diff options
Diffstat (limited to 'src')
92 files changed, 6512 insertions, 1936 deletions
diff --git a/src/blockchain_db/berkeleydb/db_bdb.cpp b/src/blockchain_db/berkeleydb/db_bdb.cpp index f827ab7c3..6c79120e8 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.cpp +++ b/src/blockchain_db/berkeleydb/db_bdb.cpp @@ -224,7 +224,7 @@ struct Dbt_safe : public Dbt namespace cryptonote { -void BlockchainBDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const crypto::hash& blk_hash) +void BlockchainBDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const crypto::hash& blk_hash) { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); @@ -255,7 +255,7 @@ void BlockchainBDB::add_block(const block& blk, const size_t& block_size, const if (res) throw0(DB_ERROR("Failed to add block blob to db transaction.")); - Dbt_copy<size_t> sz(block_size); + Dbt_copy<size_t> sz(block_weight); if (m_block_sizes->put(DB_DEFAULT_TX, &key, &sz, 0)) throw0(DB_ERROR("Failed to add block size to db transaction.")); @@ -1353,7 +1353,7 @@ uint64_t BlockchainBDB::get_top_block_timestamp() const return get_block_timestamp(m_height - 1); } -size_t BlockchainBDB::get_block_size(const uint64_t& height) const +size_t BlockchainBDB::get_block_weight(const uint64_t& height) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); @@ -1861,7 +1861,7 @@ void BlockchainBDB::block_txn_abort() // TODO } -uint64_t BlockchainBDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const std::vector<transaction>& txs) +uint64_t BlockchainBDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const std::vector<transaction>& txs) { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); @@ -1874,7 +1874,7 @@ uint64_t BlockchainBDB::add_block(const block& blk, const size_t& block_size, co uint64_t num_outputs = m_num_outputs; try { - BlockchainDB::add_block(blk, block_size, cumulative_difficulty, coins_generated, txs); + BlockchainDB::add_block(blk, block_weight, cumulative_difficulty, coins_generated, txs); m_write_txn = NULL; TIME_MEASURE_START(time1); diff --git a/src/blockchain_db/berkeleydb/db_bdb.h b/src/blockchain_db/berkeleydb/db_bdb.h index c90d030a2..76d0a0517 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.h +++ b/src/blockchain_db/berkeleydb/db_bdb.h @@ -266,7 +266,7 @@ public: virtual uint64_t get_top_block_timestamp() const; - virtual size_t get_block_size(const uint64_t& height) const; + virtual size_t get_block_weight(const uint64_t& height) const; virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const; @@ -303,7 +303,6 @@ public: virtual uint64_t get_indexing_base() const { return 1; } virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index); - virtual output_data_t get_output_key(const uint64_t& global_index) const; virtual void get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs); virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const; @@ -319,7 +318,7 @@ public: virtual bool has_key_image(const crypto::key_image& img) const; virtual uint64_t add_block( const block& blk - , const size_t& block_size + , size_t block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , const std::vector<transaction>& txs @@ -354,7 +353,7 @@ public: private: virtual void add_block( const block& blk - , const size_t& block_size + , size_t block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , const crypto::hash& block_hash @@ -419,6 +418,7 @@ private: * @return the global index of the desired output */ uint64_t get_output_global_index(const uint64_t& amount, const uint64_t& index); + output_data_t get_output_key(const uint64_t& global_index) const; void checkpoint_worker() const; void check_open() const; diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp index 8544cc3f0..be0ffeac3 100644 --- a/src/blockchain_db/blockchain_db.cpp +++ b/src/blockchain_db/blockchain_db.cpp @@ -87,8 +87,8 @@ const command_line::arg_descriptor<std::string> arg_db_type = { }; const command_line::arg_descriptor<std::string> arg_db_sync_mode = { "db-sync-mode" -, "Specify sync option, using format [safe|fast|fastest]:[sync|async]:[nblocks_per_sync]." -, "fast:async:1000" +, "Specify sync option, using format [safe|fast|fastest]:[sync|async]:[<nblocks_per_sync>[blocks]|<nbytes_per_sync>[bytes]]." +, "fast:async:250000000bytes" }; const command_line::arg_descriptor<bool> arg_db_salvage = { "db-salvage" @@ -196,7 +196,7 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti } uint64_t BlockchainDB::add_block( const block& blk - , const size_t& block_size + , size_t block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , const std::vector<transaction>& txs @@ -241,7 +241,7 @@ uint64_t BlockchainDB::add_block( const block& blk // call out to subclass implementation to add the block & metadata time1 = epee::misc_utils::get_tick_count(); - add_block(blk, block_size, cumulative_difficulty, coins_generated, num_rct_outs, blk_hash); + add_block(blk, block_weight, cumulative_difficulty, coins_generated, num_rct_outs, blk_hash); TIME_MEASURE_FINISH(time1); time_add_block1 += time1; diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 6851e2404..396ae7544 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -137,7 +137,7 @@ struct txpool_tx_meta_t { crypto::hash max_used_block_id; crypto::hash last_failed_id; - uint64_t blob_size; + uint64_t weight; uint64_t fee; uint64_t max_used_block_height; uint64_t last_failed_height; @@ -358,13 +358,13 @@ private: * subclass of DB_EXCEPTION * * @param blk the block to be added - * @param block_size the size of the block (transactions and all) + * @param block_weight the weight of the block (transactions and all) * @param cumulative_difficulty the accumulated difficulty after this block * @param coins_generated the number of coins generated total after this block * @param blk_hash the hash of the block */ virtual void add_block( const block& blk - , const size_t& block_size + , size_t block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , uint64_t num_rct_outs @@ -376,7 +376,7 @@ private: * * The subclass implementing this will remove the block data from the top * block in the chain. The data to be removed is that which was added in - * BlockchainDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const crypto::hash& blk_hash) + * BlockchainDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const crypto::hash& blk_hash) * * If any of this cannot be done, the subclass should throw the corresponding * subclass of DB_EXCEPTION @@ -789,7 +789,7 @@ public: * subclass of DB_EXCEPTION * * @param blk the block to be added - * @param block_size the size of the block (transactions and all) + * @param block_weight the size of the block (transactions and all) * @param cumulative_difficulty the accumulated difficulty after this block * @param coins_generated the number of coins generated total after this block * @param txs the transactions in the block @@ -797,7 +797,7 @@ public: * @return the height of the chain post-addition */ virtual uint64_t add_block( const block& blk - , const size_t& block_size + , size_t block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , const std::vector<transaction>& txs @@ -930,18 +930,18 @@ public: virtual uint64_t get_top_block_timestamp() const = 0; /** - * @brief fetch a block's size + * @brief fetch a block's weight * - * The subclass should return the size of the block with the + * The subclass should return the weight of the block with the * given height. * * If the block does not exist, the subclass should throw BLOCK_DNE * * @param height the height requested * - * @return the size + * @return the weight */ - virtual size_t get_block_size(const uint64_t& height) const = 0; + virtual size_t get_block_weight(const uint64_t& height) const = 0; /** * @brief fetch a block's cumulative difficulty @@ -1262,23 +1262,6 @@ public: virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index) = 0; /** - * @brief get some of an output's data - * - * The subclass should return the public key, unlock time, and block height - * for the output with the given global index, collected in a struct. - * - * If the output cannot be found, the subclass should throw OUTPUT_DNE. - * - * If any of these parts cannot be found, but some are, the subclass - * should throw DB_ERROR with a message stating as much. - * - * @param global_index the output's index (global) - * - * @return the requested output data - */ - virtual output_data_t get_output_key(const uint64_t& global_index) const = 0; - - /** * @brief gets an output's tx hash and index * * The subclass should return the hash of the transaction which created the diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 627038ca7..9e22e2e4b 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -256,7 +256,7 @@ typedef struct mdb_block_info_old uint64_t bi_height; uint64_t bi_timestamp; uint64_t bi_coins; - uint64_t bi_size; // a size_t really but we need 32-bit compat + uint64_t bi_weight; // a size_t really but we need 32-bit compat difficulty_type bi_diff; crypto::hash bi_hash; } mdb_block_info_old; @@ -266,7 +266,7 @@ typedef struct mdb_block_info uint64_t bi_height; uint64_t bi_timestamp; uint64_t bi_coins; - uint64_t bi_size; // a size_t really but we need 32-bit compat + uint64_t bi_weight; // a size_t really but we need 32-bit compat difficulty_type bi_diff; crypto::hash bi_hash; uint64_t bi_cum_rct; @@ -545,8 +545,8 @@ bool BlockchainLMDB::need_resize(uint64_t threshold_size) const LOG_PRINT_L1("Space used: " << size_used); LOG_PRINT_L1("Space remaining: " << mei.me_mapsize - size_used); LOG_PRINT_L1("Size threshold: " << threshold_size); - float resize_percent_old = RESIZE_PERCENT; - LOG_PRINT_L1(boost::format("Percent used: %.04f Percent threshold: %.04f") % ((double)size_used/mei.me_mapsize) % resize_percent_old); + float resize_percent = RESIZE_PERCENT; + LOG_PRINT_L1(boost::format("Percent used: %.04f Percent threshold: %.04f") % ((double)size_used/mei.me_mapsize) % resize_percent); if (threshold_size > 0) { @@ -559,10 +559,6 @@ bool BlockchainLMDB::need_resize(uint64_t threshold_size) const return false; } - std::mt19937 engine(std::random_device{}()); - std::uniform_real_distribution<double> fdis(0.6, 0.9); - double resize_percent = fdis(engine); - if ((double)size_used / mei.me_mapsize > resize_percent) { LOG_PRINT_L1("Threshold met (percent-based)"); @@ -656,8 +652,11 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks, uin block_rtxn_start(&rtxn, &rcurs); for (uint64_t block_num = block_start; block_num <= block_stop; ++block_num) { - uint32_t block_size = get_block_size(block_num); - total_block_size += block_size; + // we have access to block weight, which will be greater or equal to block size, + // so use this as a proxy. If it's too much off, we might have to check actual size, + // which involves reading more data, so is not really wanted + size_t block_weight = get_block_weight(block_num); + total_block_size += block_weight; // Track number of blocks being totalled here instead of assuming, in case // some blocks were to be skipped for being outliers. ++num_blocks_used; @@ -678,7 +677,7 @@ estim: return threshold_size; } -void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, +void BlockchainLMDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, uint64_t num_rct_outs, const crypto::hash& blk_hash) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -724,7 +723,7 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const bi.bi_height = m_height; bi.bi_timestamp = blk.timestamp; bi.bi_coins = coins_generated; - bi.bi_size = block_size; + bi.bi_weight = block_weight; bi.bi_diff = cumulative_difficulty; bi.bi_hash = blk_hash; bi.bi_cum_rct = num_rct_outs; @@ -747,7 +746,9 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const if (result) throw0(DB_ERROR(lmdb_error("Failed to add block height by hash to db transaction: ", result).c_str())); - m_cum_size += block_size; + // we use weight as a proxy for size, since we don't have size but weight is >= size + // and often actually equal + m_cum_size += block_weight; m_cum_count++; } @@ -1200,8 +1201,12 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) throw DB_ERROR("Database could not be opened"); } - if (tools::is_hdd(filename.c_str())) - MCLOG_RED(el::Level::Warning, "global", "The blockchain is on a rotating drive: this will be very slow, use a SSD if possible"); + boost::optional<bool> is_hdd_result = tools::is_hdd(filename.c_str()); + if (is_hdd_result) + { + if (is_hdd_result.value()) + MCLOG_RED(el::Level::Warning, "global", "The blockchain is on a rotating drive: this will be very slow, use a SSD if possible"); + } m_folder = filename; @@ -1971,14 +1976,25 @@ std::vector<uint64_t> BlockchainLMDB::get_block_cumulative_rct_outputs(const std MDB_val v; + uint64_t prev_height = heights[0]; for (uint64_t height: heights) { - MDB_val_set(v, height); - result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); + if (height == prev_height + 1) + { + MDB_val k2; + result = mdb_cursor_get(m_cur_block_info, &k2, &v, MDB_NEXT); + } + else + { + v.mv_size = sizeof(uint64_t); + v.mv_data = (void*)&height; + result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); + } if (result) throw0(DB_ERROR(lmdb_error("Error attempting to retrieve rct distribution from the db: ", result).c_str())); const mdb_block_info *bi = (const mdb_block_info *)v.mv_data; res.push_back(bi->bi_cum_rct); + prev_height = height; } TXN_POSTFIX_RDONLY(); @@ -2000,7 +2016,7 @@ uint64_t BlockchainLMDB::get_top_block_timestamp() const return get_block_timestamp(m_height - 1); } -size_t BlockchainLMDB::get_block_size(const uint64_t& height) const +size_t BlockchainLMDB::get_block_weight(const uint64_t& height) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -2018,7 +2034,7 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) const throw0(DB_ERROR("Error attempting to retrieve a block size from the db")); mdb_block_info *bi = (mdb_block_info *)result.mv_data; - size_t ret = bi->bi_size; + size_t ret = bi->bi_weight; TXN_POSTFIX_RDONLY(); return ret; } @@ -2454,55 +2470,6 @@ uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) const return num_elems; } -// This is a lot harder now that we've removed the output_keys index -output_data_t BlockchainLMDB::get_output_key(const uint64_t &global_index) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " (unused version - does nothing)"); - check_open(); - TXN_PREFIX_RDONLY(); - RCURSOR(output_txs); - RCURSOR(tx_indices); - - output_data_t od; - MDB_val_set(v, global_index); - auto get_result = mdb_cursor_get(m_cur_output_txs, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); - if (get_result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("output with given index not in db")); - else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch output tx hash")); - - outtx *ot = (outtx *)v.mv_data; - - MDB_val_set(val_h, ot->tx_hash); - get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &val_h, MDB_GET_BOTH); - if (get_result) - throw0(DB_ERROR(lmdb_error(std::string("DB error attempting to fetch transaction index from hash ") + epee::string_tools::pod_to_hex(ot->tx_hash) + ": ", get_result).c_str())); - - txindex *tip = (txindex *)val_h.mv_data; - MDB_val_set(val_tx_id, tip->data.tx_id); - MDB_val result; - get_result = mdb_cursor_get(m_cur_txs_pruned, &val_tx_id, &result, MDB_SET); - if (get_result == MDB_NOTFOUND) - throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(ot->tx_hash)).append(" not found in db").c_str())); - else if (get_result) - throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx from hash", get_result).c_str())); - - blobdata bd; - bd.assign(reinterpret_cast<char*>(result.mv_data), result.mv_size); - - transaction tx; - if (!parse_and_validate_tx_base_from_blob(bd, tx)) - throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); - - const tx_out tx_output = tx.vout[ot->local_index]; - od.unlock_time = tip->data.unlock_time; - od.height = tip->data.block_id; - od.pubkey = boost::get<txout_to_key>(tx_output.target).key; - - TXN_POSTFIX_RDONLY(); - return od; -} - output_data_t BlockchainLMDB::get_output_key(const uint64_t& amount, const uint64_t& index) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -3128,7 +3095,7 @@ void BlockchainLMDB::block_txn_abort() } } -uint64_t BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, +uint64_t BlockchainLMDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const std::vector<transaction>& txs) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -3147,7 +3114,7 @@ uint64_t BlockchainLMDB::add_block(const block& blk, const size_t& block_size, c try { - BlockchainDB::add_block(blk, block_size, cumulative_difficulty, coins_generated, txs); + BlockchainDB::add_block(blk, block_weight, cumulative_difficulty, coins_generated, txs); } catch (const DB_ERROR_TXN_START& e) { @@ -3736,9 +3703,9 @@ void BlockchainLMDB::migrate_0_1() if (result) throw0(DB_ERROR(lmdb_error("Failed to get a record from block_sizes: ", result).c_str())); if (v.mv_size == sizeof(uint32_t)) - bi.bi_size = *(uint32_t *)v.mv_data; + bi.bi_weight = *(uint32_t *)v.mv_data; else - bi.bi_size = *(uint64_t *)v.mv_data; // this is a 32/64 compat bug in version 0 + bi.bi_weight = *(uint64_t *)v.mv_data; // this is a 32/64 compat bug in version 0 result = mdb_cursor_get(c_timestamps, &k, &v, MDB_NEXT); if (result) throw0(DB_ERROR(lmdb_error("Failed to get a record from block_timestamps: ", result).c_str())); @@ -4297,7 +4264,7 @@ void BlockchainLMDB::migrate_2_3() bi.bi_height = bi_old->bi_height; bi.bi_timestamp = bi_old->bi_timestamp; bi.bi_coins = bi_old->bi_coins; - bi.bi_size = bi_old->bi_size; + bi.bi_weight = bi_old->bi_weight; bi.bi_diff = bi_old->bi_diff; bi.bi_hash = bi_old->bi_hash; if (bi_old->bi_height >= distribution.size()) diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 8b214d2df..e1f748ed8 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -205,7 +205,7 @@ public: virtual uint64_t get_top_block_timestamp() const; - virtual size_t get_block_size(const uint64_t& height) const; + virtual size_t get_block_weight(const uint64_t& height) const; virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const; @@ -243,7 +243,6 @@ public: virtual uint64_t get_num_outputs(const uint64_t& amount) const; virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index); - virtual output_data_t get_output_key(const uint64_t& global_index) const; virtual void get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs, bool allow_partial = false); virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const; @@ -274,7 +273,7 @@ public: virtual bool for_all_outputs(uint64_t amount, const std::function<bool(uint64_t height)> &f) const; virtual uint64_t add_block( const block& blk - , const size_t& block_size + , size_t block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , const std::vector<transaction>& txs @@ -318,7 +317,7 @@ private: uint64_t get_estimated_batch_size(uint64_t batch_num_blocks, uint64_t batch_bytes) const; virtual void add_block( const block& blk - , const size_t& block_size + , size_t block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , uint64_t num_rct_outs @@ -450,7 +449,7 @@ private: #endif #endif - constexpr static float RESIZE_PERCENT = 0.8f; + constexpr static float RESIZE_PERCENT = 0.9f; }; } // namespace cryptonote diff --git a/src/blockchain_utilities/CMakeLists.txt b/src/blockchain_utilities/CMakeLists.txt index 338ec3e4b..fe57895e1 100644 --- a/src/blockchain_utilities/CMakeLists.txt +++ b/src/blockchain_utilities/CMakeLists.txt @@ -91,6 +91,28 @@ monero_private_headers(blockchain_usage +set(blockchain_ancestry_sources + blockchain_ancestry.cpp + ) + +set(blockchain_ancestry_private_headers) + +monero_private_headers(blockchain_ancestry + ${blockchain_ancestry_private_headers}) + + + +set(blockchain_depth_sources + blockchain_depth.cpp + ) + +set(blockchain_depth_private_headers) + +monero_private_headers(blockchain_depth + ${blockchain_depth_private_headers}) + + + monero_add_executable(blockchain_import ${blockchain_import_sources} ${blockchain_import_private_headers} @@ -183,3 +205,45 @@ set_property(TARGET blockchain_usage OUTPUT_NAME "monero-blockchain-usage") install(TARGETS blockchain_usage DESTINATION bin) +monero_add_executable(blockchain_ancestry + ${blockchain_ancestry_sources} + ${blockchain_ancestry_private_headers}) + +target_link_libraries(blockchain_ancestry + PRIVATE + cryptonote_core + blockchain_db + version + epee + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${Boost_THREAD_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES}) + +set_property(TARGET blockchain_ancestry + PROPERTY + OUTPUT_NAME "monero-blockchain-ancestry") +install(TARGETS blockchain_ancestry DESTINATION bin) + +monero_add_executable(blockchain_depth + ${blockchain_depth_sources} + ${blockchain_depth_private_headers}) + +target_link_libraries(blockchain_depth + PRIVATE + cryptonote_core + blockchain_db + version + epee + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${Boost_THREAD_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES}) + +set_property(TARGET blockchain_depth + PROPERTY + OUTPUT_NAME "monero-blockchain-depth") +install(TARGETS blockchain_depth DESTINATION bin) + diff --git a/src/blockchain_utilities/blockchain_ancestry.cpp b/src/blockchain_utilities/blockchain_ancestry.cpp new file mode 100644 index 000000000..2f0bbffd6 --- /dev/null +++ b/src/blockchain_utilities/blockchain_ancestry.cpp @@ -0,0 +1,772 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <unordered_map> +#include <unordered_set> +#include <boost/range/adaptor/transformed.hpp> +#include <boost/algorithm/string.hpp> +#include <boost/archive/portable_binary_iarchive.hpp> +#include <boost/archive/portable_binary_oarchive.hpp> +#include "common/unordered_containers_boost_serialization.h" +#include "common/command_line.h" +#include "common/varint.h" +#include "cryptonote_basic/cryptonote_boost_serialization.h" +#include "cryptonote_core/tx_pool.h" +#include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_core/blockchain.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/db_types.h" +#include "version.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "bcutil" + +namespace po = boost::program_options; +using namespace epee; +using namespace cryptonote; + +static bool stop_requested = false; + +struct ancestor +{ + uint64_t amount; + uint64_t offset; + + bool operator==(const ancestor &other) const { return amount == other.amount && offset == other.offset; } + + template <typename t_archive> void serialize(t_archive &a, const unsigned int ver) + { + a & amount; + a & offset; + } +}; +BOOST_CLASS_VERSION(ancestor, 0) + +namespace std +{ + template<> struct hash<ancestor> + { + size_t operator()(const ancestor &a) const + { + return a.amount ^ a.offset; // not that bad, since amount almost always have a high bit set, and offset doesn't + } + }; +} + +struct tx_data_t +{ + std::vector<std::pair<uint64_t, std::vector<uint64_t>>> vin; + std::vector<crypto::public_key> vout; + bool coinbase; + + tx_data_t(): coinbase(false) {} + tx_data_t(const cryptonote::transaction &tx) + { + coinbase = tx.vin.size() == 1 && tx.vin[0].type() == typeid(cryptonote::txin_gen); + if (!coinbase) + { + vin.reserve(tx.vin.size()); + for (size_t ring = 0; ring < tx.vin.size(); ++ring) + { + if (tx.vin[ring].type() == typeid(cryptonote::txin_to_key)) + { + const cryptonote::txin_to_key &txin = boost::get<cryptonote::txin_to_key>(tx.vin[ring]); + vin.push_back(std::make_pair(txin.amount, cryptonote::relative_output_offsets_to_absolute(txin.key_offsets))); + } + else + { + LOG_PRINT_L0("Bad vin type in txid " << get_transaction_hash(tx)); + throw std::runtime_error("Bad vin type"); + } + } + } + vout.reserve(tx.vout.size()); + for (size_t out = 0; out < tx.vout.size(); ++out) + { + if (tx.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get<cryptonote::txout_to_key>(tx.vout[out].target); + vout.push_back(txout.key); + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << get_transaction_hash(tx)); + throw std::runtime_error("Bad vout type"); + } + } + } + + template <typename t_archive> void serialize(t_archive &a, const unsigned int ver) + { + a & coinbase; + a & vin; + a & vout; + } +}; + +struct ancestry_state_t +{ + uint64_t height; + std::unordered_map<crypto::hash, std::unordered_set<ancestor>> ancestry; + std::unordered_map<ancestor, crypto::hash> output_cache; + std::unordered_map<crypto::hash, ::tx_data_t> tx_cache; + std::vector<cryptonote::block> block_cache; + + template <typename t_archive> void serialize(t_archive &a, const unsigned int ver) + { + a & height; + a & ancestry; + a & output_cache; + if (ver < 1) + { + std::unordered_map<crypto::hash, cryptonote::transaction> old_tx_cache; + a & old_tx_cache; + for (const auto i: old_tx_cache) + tx_cache.insert(std::make_pair(i.first, ::tx_data_t(i.second))); + } + else + { + a & tx_cache; + } + if (ver < 2) + { + std::unordered_map<uint64_t, cryptonote::block> old_block_cache; + a & old_block_cache; + block_cache.resize(old_block_cache.size()); + for (const auto i: old_block_cache) + block_cache[i.first] = i.second; + } + else + { + a & block_cache; + } + } +}; +BOOST_CLASS_VERSION(ancestry_state_t, 2) + +static void add_ancestor(std::unordered_map<ancestor, unsigned int> &ancestry, uint64_t amount, uint64_t offset) +{ + std::pair<std::unordered_map<ancestor, unsigned int>::iterator, bool> p = ancestry.insert(std::make_pair(ancestor{amount, offset}, 1)); + if (!p.second) + { + ++p.first->second; + } +} + +static size_t get_full_ancestry(const std::unordered_map<ancestor, unsigned int> &ancestry) +{ + size_t count = 0; + for (const auto &i: ancestry) + count += i.second; + return count; +} + +static size_t get_deduplicated_ancestry(const std::unordered_map<ancestor, unsigned int> &ancestry) +{ + return ancestry.size(); +} + +static void add_ancestry(std::unordered_map<crypto::hash, std::unordered_set<ancestor>> &ancestry, const crypto::hash &txid, const std::unordered_set<ancestor> &ancestors) +{ + std::pair<std::unordered_map<crypto::hash, std::unordered_set<ancestor>>::iterator, bool> p = ancestry.insert(std::make_pair(txid, ancestors)); + if (!p.second) + { + for (const auto &e: ancestors) + p.first->second.insert(e); + } +} + +static void add_ancestry(std::unordered_map<crypto::hash, std::unordered_set<ancestor>> &ancestry, const crypto::hash &txid, const ancestor &new_ancestor) +{ + std::pair<std::unordered_map<crypto::hash, std::unordered_set<ancestor>>::iterator, bool> p = ancestry.insert(std::make_pair(txid, std::unordered_set<ancestor>())); + p.first->second.insert(new_ancestor); +} + +static std::unordered_set<ancestor> get_ancestry(const std::unordered_map<crypto::hash, std::unordered_set<ancestor>> &ancestry, const crypto::hash &txid) +{ + std::unordered_map<crypto::hash, std::unordered_set<ancestor>>::const_iterator i = ancestry.find(txid); + if (i == ancestry.end()) + { + //MERROR("txid ancestry not found: " << txid); + //throw std::runtime_error("txid ancestry not found"); + return std::unordered_set<ancestor>(); + } + return i->second; +} + +int main(int argc, char* argv[]) +{ + TRY_ENTRY(); + + epee::string_tools::set_module_name_and_folder(argv[0]); + + std::string default_db_type = "lmdb"; + + std::string available_dbs = cryptonote::blockchain_db_types(", "); + available_dbs = "available: " + available_dbs; + + uint32_t log_level = 0; + + tools::on_startup(); + + boost::filesystem::path output_file_path; + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""}; + const command_line::arg_descriptor<std::string> arg_database = { + "database", available_dbs.c_str(), default_db_type + }; + const command_line::arg_descriptor<std::string> arg_txid = {"txid", "Get ancestry for this txid", ""}; + const command_line::arg_descriptor<uint64_t> arg_height = {"height", "Get ancestry for all txes at this height", 0}; + const command_line::arg_descriptor<bool> arg_all = {"all", "Include the whole chain", false}; + const command_line::arg_descriptor<bool> arg_cache_outputs = {"cache-outputs", "Cache outputs (memory hungry)", false}; + const command_line::arg_descriptor<bool> arg_cache_txes = {"cache-txes", "Cache txes (memory hungry)", false}; + const command_line::arg_descriptor<bool> arg_cache_blocks = {"cache-blocks", "Cache blocks (memory hungry)", false}; + const command_line::arg_descriptor<bool> arg_include_coinbase = {"include-coinbase", "Including coinbase tx", false}; + const command_line::arg_descriptor<bool> arg_show_cache_stats = {"show-cache-stats", "Show cache statistics", false}; + + command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_database); + command_line::add_arg(desc_cmd_sett, arg_txid); + command_line::add_arg(desc_cmd_sett, arg_height); + command_line::add_arg(desc_cmd_sett, arg_all); + command_line::add_arg(desc_cmd_sett, arg_cache_outputs); + command_line::add_arg(desc_cmd_sett, arg_cache_txes); + command_line::add_arg(desc_cmd_sett, arg_cache_blocks); + command_line::add_arg(desc_cmd_sett, arg_include_coinbase); + command_line::add_arg(desc_cmd_sett, arg_show_cache_stats); + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + auto parser = po::command_line_parser(argc, argv).options(desc_options); + po::store(parser.run(), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + mlog_configure(mlog_get_default_log_path("monero-blockchain-ancestry.log"), true); + if (!command_line::is_arg_defaulted(vm, arg_log_level)) + mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); + else + mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str()); + + LOG_PRINT_L0("Starting..."); + + std::string opt_data_dir = command_line::get_arg(vm, cryptonote::arg_data_dir); + bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); + bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); + network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; + std::string opt_txid_string = command_line::get_arg(vm, arg_txid); + uint64_t opt_height = command_line::get_arg(vm, arg_height); + bool opt_all = command_line::get_arg(vm, arg_all); + bool opt_cache_outputs = command_line::get_arg(vm, arg_cache_outputs); + bool opt_cache_txes = command_line::get_arg(vm, arg_cache_txes); + bool opt_cache_blocks = command_line::get_arg(vm, arg_cache_blocks); + bool opt_include_coinbase = command_line::get_arg(vm, arg_include_coinbase); + bool opt_show_cache_stats = command_line::get_arg(vm, arg_show_cache_stats); + + if ((!opt_txid_string.empty()) + !!opt_height + !!opt_all > 1) + { + std::cerr << "Only one of --txid, --height and --all can be given" << std::endl; + return 1; + } + crypto::hash opt_txid = crypto::null_hash; + if (!opt_txid_string.empty()) + { + if (!epee::string_tools::hex_to_pod(opt_txid_string, opt_txid)) + { + std::cerr << "Invalid txid" << std::endl; + return 1; + } + } + + std::string db_type = command_line::get_arg(vm, arg_database); + if (!cryptonote::blockchain_valid_db_type(db_type)) + { + std::cerr << "Invalid database type: " << db_type << std::endl; + return 1; + } + + // If we wanted to use the memory pool, we would set up a fake_core. + + // Use Blockchain instead of lower-level BlockchainDB for two reasons: + // 1. Blockchain has the init() method for easy setup + // 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash() + // + // cannot match blockchain_storage setup above with just one line, + // e.g. + // Blockchain* core_storage = new Blockchain(NULL); + // because unlike blockchain_storage constructor, which takes a pointer to + // tx_memory_pool, Blockchain's constructor takes tx_memory_pool object. + LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); + std::unique_ptr<Blockchain> core_storage; + tx_memory_pool m_mempool(*core_storage); + core_storage.reset(new Blockchain(m_mempool)); + BlockchainDB *db = new_db(db_type); + if (db == NULL) + { + LOG_ERROR("Attempted to use non-existent database type: " << db_type); + throw std::runtime_error("Attempting to use non-existent database type"); + } + LOG_PRINT_L0("database: " << db_type); + + const std::string filename = (boost::filesystem::path(opt_data_dir) / db->get_db_name()).string(); + LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); + + try + { + db->open(filename, DBF_RDONLY); + } + catch (const std::exception& e) + { + LOG_PRINT_L0("Error opening database: " << e.what()); + return 1; + } + r = core_storage->init(db, net_type); + + CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage"); + LOG_PRINT_L0("Source blockchain storage initialized OK"); + + std::vector<crypto::hash> start_txids; + + // forward method + if (opt_all) + { + uint64_t cached_txes = 0, cached_blocks = 0, cached_outputs = 0, total_txes = 0, total_blocks = 0, total_outputs = 0; + ancestry_state_t state; + + const std::string state_file_path = (boost::filesystem::path(opt_data_dir) / "ancestry-state.bin").string(); + LOG_PRINT_L0("Loading state data from " << state_file_path); + std::ifstream state_data_in; + state_data_in.open(state_file_path, std::ios_base::binary | std::ios_base::in); + if (!state_data_in.fail()) + { + try + { + boost::archive::portable_binary_iarchive a(state_data_in); + a >> state; + } + catch (const std::exception &e) + { + MERROR("Failed to load state data from " << state_file_path << ", restarting from scratch"); + state = ancestry_state_t(); + } + state_data_in.close(); + } + + tools::signal_handler::install([](int type) { + stop_requested = true; + }); + + MINFO("Starting from height " << state.height); + const uint64_t db_height = db->height(); + state.block_cache.reserve(db_height); + for (uint64_t h = state.height; h < db_height; ++h) + { + size_t block_ancestry_size = 0; + const crypto::hash block_hash = db->get_block_hash_from_height(h); + const cryptonote::blobdata bd = db->get_block_blob(block_hash); + ++total_blocks; + cryptonote::block b; + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + if (opt_cache_blocks) + { + state.block_cache.resize(h + 1); + state.block_cache[h] = b; + } + std::vector<crypto::hash> txids; + txids.reserve(1 + b.tx_hashes.size()); + if (opt_include_coinbase) + txids.push_back(cryptonote::get_transaction_hash(b.miner_tx)); + for (const auto &h: b.tx_hashes) + txids.push_back(h); + for (const crypto::hash &txid: txids) + { + printf("%lu/%lu \r", (unsigned long)h, (unsigned long)db_height); + fflush(stdout); + ::tx_data_t tx_data; + std::unordered_map<crypto::hash, ::tx_data_t>::const_iterator i = state.tx_cache.find(txid); + ++total_txes; + if (i != state.tx_cache.end()) + { + ++cached_txes; + tx_data = i->second; + } + else + { + cryptonote::blobdata bd; + if (!db->get_pruned_tx_blob(txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << txid << " from db"); + return 1; + } + cryptonote::transaction tx; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) + { + LOG_PRINT_L0("Bad tx: " << txid); + return 1; + } + tx_data = ::tx_data_t(tx); + if (opt_cache_txes) + state.tx_cache.insert(std::make_pair(txid, tx_data)); + } + if (tx_data.coinbase) + { + add_ancestry(state.ancestry, txid, std::unordered_set<ancestor>()); + } + else + { + for (size_t ring = 0; ring < tx_data.vin.size(); ++ring) + { + if (1) + { + const uint64_t amount = tx_data.vin[ring].first; + const std::vector<uint64_t> &absolute_offsets = tx_data.vin[ring].second; + for (uint64_t offset: absolute_offsets) + { + const output_data_t od = db->get_output_key(amount, offset); + add_ancestry(state.ancestry, txid, ancestor{amount, offset}); + cryptonote::block b; + ++total_blocks; + if (state.block_cache.size() > od.height && !state.block_cache[od.height].miner_tx.vin.empty()) + { + ++cached_blocks; + b = state.block_cache[od.height]; + } + else + { + const crypto::hash block_hash = db->get_block_hash_from_height(od.height); + cryptonote::blobdata bd = db->get_block_blob(block_hash); + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + if (opt_cache_blocks) + { + state.block_cache.resize(od.height + 1); + state.block_cache[od.height] = b; + } + } + // find the tx which created this output + bool found = false; + std::unordered_map<ancestor, crypto::hash>::const_iterator i = state.output_cache.find({amount, offset}); + ++total_outputs; + if (i != state.output_cache.end()) + { + ++cached_outputs; + add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, i->second)); + found = true; + } + else for (size_t out = 0; out < b.miner_tx.vout.size(); ++out) + { + if (b.miner_tx.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get<cryptonote::txout_to_key>(b.miner_tx.vout[out].target); + if (txout.key == od.pubkey) + { + found = true; + add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, cryptonote::get_transaction_hash(b.miner_tx))); + if (opt_cache_outputs) + state.output_cache.insert(std::make_pair(ancestor{amount, offset}, cryptonote::get_transaction_hash(b.miner_tx))); + break; + } + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << cryptonote::get_transaction_hash(b.miner_tx)); + return 1; + } + } + for (const crypto::hash &block_txid: b.tx_hashes) + { + if (found) + break; + ::tx_data_t tx_data2; + std::unordered_map<crypto::hash, ::tx_data_t>::const_iterator i = state.tx_cache.find(block_txid); + ++total_txes; + if (i != state.tx_cache.end()) + { + ++cached_txes; + tx_data2 = i->second; + } + else + { + cryptonote::blobdata bd; + if (!db->get_pruned_tx_blob(block_txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << block_txid << " from db"); + return 1; + } + cryptonote::transaction tx; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) + { + LOG_PRINT_L0("Bad tx: " << block_txid); + return 1; + } + tx_data2 = ::tx_data_t(tx); + if (opt_cache_txes) + state.tx_cache.insert(std::make_pair(block_txid, tx_data2)); + } + for (size_t out = 0; out < tx_data2.vout.size(); ++out) + { + if (tx_data2.vout[out] == od.pubkey) + { + found = true; + add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, block_txid)); + if (opt_cache_outputs) + state.output_cache.insert(std::make_pair(ancestor{amount, offset}, block_txid)); + break; + } + } + } + if (!found) + { + LOG_PRINT_L0("Output originating transaction not found"); + return 1; + } + } + } + } + } + const size_t ancestry_size = get_ancestry(state.ancestry, txid).size(); + block_ancestry_size += ancestry_size; + MINFO(txid << ": " << ancestry_size); + } + if (!txids.empty()) + { + std::string stats_msg; + if (opt_show_cache_stats) + stats_msg = std::string(", cache: txes ") + std::to_string(cached_txes*100./total_txes) + + ", blocks " + std::to_string(cached_blocks*100./total_blocks) + ", outputs " + + std::to_string(cached_outputs*100./total_outputs); + MINFO("Height " << h << ": " << (block_ancestry_size / txids.size()) << " average over " << txids.size() << stats_msg); + } + state.height = h; + if (stop_requested) + break; + } + + LOG_PRINT_L0("Saving state data to " << state_file_path); + std::ofstream state_data_out; + state_data_out.open(state_file_path, std::ios_base::binary | std::ios_base::out | std::ios::trunc); + if (!state_data_out.fail()) + { + try + { + boost::archive::portable_binary_oarchive a(state_data_out); + a << state; + } + catch (const std::exception &e) + { + MERROR("Failed to save state data to " << state_file_path); + } + state_data_out.close(); + } + + goto done; + } + + if (!opt_txid_string.empty()) + { + start_txids.push_back(opt_txid); + } + else + { + const crypto::hash block_hash = db->get_block_hash_from_height(opt_height); + const cryptonote::blobdata bd = db->get_block_blob(block_hash); + cryptonote::block b; + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + for (const crypto::hash &txid: b.tx_hashes) + start_txids.push_back(txid); + } + + if (start_txids.empty()) + { + LOG_PRINT_L0("No transaction(s) to check"); + return 1; + } + + for (const crypto::hash &start_txid: start_txids) + { + LOG_PRINT_L0("Checking ancestry for txid " << start_txid); + + std::unordered_map<ancestor, unsigned int> ancestry; + + std::list<crypto::hash> txids; + txids.push_back(start_txid); + while (!txids.empty()) + { + const crypto::hash txid = txids.front(); + txids.pop_front(); + + cryptonote::blobdata bd; + if (!db->get_pruned_tx_blob(txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << txid << " from db"); + return 1; + } + cryptonote::transaction tx; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) + { + LOG_PRINT_L0("Bad tx: " << txid); + return 1; + } + const bool coinbase = tx.vin.size() == 1 && tx.vin[0].type() == typeid(cryptonote::txin_gen); + if (coinbase) + continue; + + for (size_t ring = 0; ring < tx.vin.size(); ++ring) + { + if (tx.vin[ring].type() == typeid(cryptonote::txin_to_key)) + { + const cryptonote::txin_to_key &txin = boost::get<cryptonote::txin_to_key>(tx.vin[ring]); + const uint64_t amount = txin.amount; + auto absolute_offsets = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets); + for (uint64_t offset: absolute_offsets) + { + add_ancestor(ancestry, amount, offset); + const output_data_t od = db->get_output_key(amount, offset); + const crypto::hash block_hash = db->get_block_hash_from_height(od.height); + bd = db->get_block_blob(block_hash); + cryptonote::block b; + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + // find the tx which created this output + bool found = false; + for (size_t out = 0; out < b.miner_tx.vout.size(); ++out) + { + if (b.miner_tx.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get<cryptonote::txout_to_key>(b.miner_tx.vout[out].target); + if (txout.key == od.pubkey) + { + found = true; + txids.push_back(cryptonote::get_transaction_hash(b.miner_tx)); + MDEBUG("adding txid: " << cryptonote::get_transaction_hash(b.miner_tx)); + break; + } + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << cryptonote::get_transaction_hash(b.miner_tx)); + return 1; + } + } + for (const crypto::hash &block_txid: b.tx_hashes) + { + if (found) + break; + if (!db->get_pruned_tx_blob(block_txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << block_txid << " from db"); + return 1; + } + cryptonote::transaction tx2; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx2)) + { + LOG_PRINT_L0("Bad tx: " << block_txid); + return 1; + } + for (size_t out = 0; out < tx2.vout.size(); ++out) + { + if (tx2.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get<cryptonote::txout_to_key>(tx2.vout[out].target); + if (txout.key == od.pubkey) + { + found = true; + txids.push_back(block_txid); + MDEBUG("adding txid: " << block_txid); + break; + } + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << block_txid); + return 1; + } + } + } + if (!found) + { + LOG_PRINT_L0("Output originating transaction not found"); + return 1; + } + } + } + else + { + LOG_PRINT_L0("Bad vin type in txid " << txid); + return 1; + } + } + } + + MINFO("Ancestry for " << start_txid << ": " << get_deduplicated_ancestry(ancestry) << " / " << get_full_ancestry(ancestry)); + for (const auto &i: ancestry) + { + MINFO(cryptonote::print_money(i.first.amount) << "/" << i.first.offset << ": " << i.second); + } + } + +done: + core_storage->deinit(); + return 0; + + CATCH_ENTRY("Depth query error", 1); +} diff --git a/src/blockchain_utilities/blockchain_depth.cpp b/src/blockchain_utilities/blockchain_depth.cpp new file mode 100644 index 000000000..dd2387e5b --- /dev/null +++ b/src/blockchain_utilities/blockchain_depth.cpp @@ -0,0 +1,351 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <boost/range/adaptor/transformed.hpp> +#include <boost/algorithm/string.hpp> +#include "common/command_line.h" +#include "common/varint.h" +#include "cryptonote_core/tx_pool.h" +#include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_core/blockchain.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/db_types.h" +#include "version.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "bcutil" + +namespace po = boost::program_options; +using namespace epee; +using namespace cryptonote; + +int main(int argc, char* argv[]) +{ + TRY_ENTRY(); + + epee::string_tools::set_module_name_and_folder(argv[0]); + + std::string default_db_type = "lmdb"; + + std::string available_dbs = cryptonote::blockchain_db_types(", "); + available_dbs = "available: " + available_dbs; + + uint32_t log_level = 0; + + tools::on_startup(); + + boost::filesystem::path output_file_path; + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""}; + const command_line::arg_descriptor<std::string> arg_database = { + "database", available_dbs.c_str(), default_db_type + }; + const command_line::arg_descriptor<std::string> arg_txid = {"txid", "Get min depth for this txid", ""}; + const command_line::arg_descriptor<uint64_t> arg_height = {"height", "Get min depth for all txes at this height", 0}; + const command_line::arg_descriptor<bool> arg_include_coinbase = {"include-coinbase", "Include coinbase in the average", false}; + + command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_database); + command_line::add_arg(desc_cmd_sett, arg_txid); + command_line::add_arg(desc_cmd_sett, arg_height); + command_line::add_arg(desc_cmd_sett, arg_include_coinbase); + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + auto parser = po::command_line_parser(argc, argv).options(desc_options); + po::store(parser.run(), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + mlog_configure(mlog_get_default_log_path("monero-blockchain-depth.log"), true); + if (!command_line::is_arg_defaulted(vm, arg_log_level)) + mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); + else + mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str()); + + LOG_PRINT_L0("Starting..."); + + std::string opt_data_dir = command_line::get_arg(vm, cryptonote::arg_data_dir); + bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); + bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); + network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; + std::string opt_txid_string = command_line::get_arg(vm, arg_txid); + uint64_t opt_height = command_line::get_arg(vm, arg_height); + bool opt_include_coinbase = command_line::get_arg(vm, arg_include_coinbase); + + if (!opt_txid_string.empty() && opt_height) + { + std::cerr << "txid and height cannot be given at the same time" << std::endl; + return 1; + } + crypto::hash opt_txid = crypto::null_hash; + if (!opt_txid_string.empty()) + { + if (!epee::string_tools::hex_to_pod(opt_txid_string, opt_txid)) + { + std::cerr << "Invalid txid" << std::endl; + return 1; + } + } + + std::string db_type = command_line::get_arg(vm, arg_database); + if (!cryptonote::blockchain_valid_db_type(db_type)) + { + std::cerr << "Invalid database type: " << db_type << std::endl; + return 1; + } + + // If we wanted to use the memory pool, we would set up a fake_core. + + // Use Blockchain instead of lower-level BlockchainDB for two reasons: + // 1. Blockchain has the init() method for easy setup + // 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash() + // + // cannot match blockchain_storage setup above with just one line, + // e.g. + // Blockchain* core_storage = new Blockchain(NULL); + // because unlike blockchain_storage constructor, which takes a pointer to + // tx_memory_pool, Blockchain's constructor takes tx_memory_pool object. + LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); + std::unique_ptr<Blockchain> core_storage; + tx_memory_pool m_mempool(*core_storage); + core_storage.reset(new Blockchain(m_mempool)); + BlockchainDB *db = new_db(db_type); + if (db == NULL) + { + LOG_ERROR("Attempted to use non-existent database type: " << db_type); + throw std::runtime_error("Attempting to use non-existent database type"); + } + LOG_PRINT_L0("database: " << db_type); + + const std::string filename = (boost::filesystem::path(opt_data_dir) / db->get_db_name()).string(); + LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); + + try + { + db->open(filename, DBF_RDONLY); + } + catch (const std::exception& e) + { + LOG_PRINT_L0("Error opening database: " << e.what()); + return 1; + } + r = core_storage->init(db, net_type); + + CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage"); + LOG_PRINT_L0("Source blockchain storage initialized OK"); + + std::vector<crypto::hash> start_txids; + if (!opt_txid_string.empty()) + { + start_txids.push_back(opt_txid); + } + else + { + const crypto::hash block_hash = db->get_block_hash_from_height(opt_height); + const cryptonote::blobdata bd = db->get_block_blob(block_hash); + cryptonote::block b; + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + for (const crypto::hash &txid: b.tx_hashes) + start_txids.push_back(txid); + if (opt_include_coinbase) + start_txids.push_back(cryptonote::get_transaction_hash(b.miner_tx)); + } + + if (start_txids.empty()) + { + LOG_PRINT_L0("No transaction(s) to check"); + return 1; + } + + std::vector<uint64_t> depths; + for (const crypto::hash &start_txid: start_txids) + { + uint64_t depth = 0; + bool coinbase = false; + + LOG_PRINT_L0("Checking depth for txid " << start_txid); + std::vector<crypto::hash> txids(1, start_txid); + while (!coinbase) + { + LOG_PRINT_L0("Considering "<< txids.size() << " transaction(s) at depth " << depth); + std::vector<crypto::hash> new_txids; + for (const crypto::hash &txid: txids) + { + cryptonote::blobdata bd; + if (!db->get_pruned_tx_blob(txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << txid << " from db"); + return 1; + } + cryptonote::transaction tx; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) + { + LOG_PRINT_L0("Bad tx: " << txid); + return 1; + } + for (size_t ring = 0; ring < tx.vin.size(); ++ring) + { + if (tx.vin[ring].type() == typeid(cryptonote::txin_gen)) + { + MDEBUG(txid << " is a coinbase transaction"); + coinbase = true; + goto done; + } + if (tx.vin[ring].type() == typeid(cryptonote::txin_to_key)) + { + const cryptonote::txin_to_key &txin = boost::get<cryptonote::txin_to_key>(tx.vin[ring]); + const uint64_t amount = txin.amount; + auto absolute_offsets = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets); + for (uint64_t offset: absolute_offsets) + { + const output_data_t od = db->get_output_key(amount, offset); + const crypto::hash block_hash = db->get_block_hash_from_height(od.height); + bd = db->get_block_blob(block_hash); + cryptonote::block b; + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + // find the tx which created this output + bool found = false; + for (size_t out = 0; out < b.miner_tx.vout.size(); ++out) + { + if (b.miner_tx.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get<cryptonote::txout_to_key>(b.miner_tx.vout[out].target); + if (txout.key == od.pubkey) + { + found = true; + new_txids.push_back(cryptonote::get_transaction_hash(b.miner_tx)); + MDEBUG("adding txid: " << cryptonote::get_transaction_hash(b.miner_tx)); + break; + } + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << cryptonote::get_transaction_hash(b.miner_tx)); + return 1; + } + } + for (const crypto::hash &block_txid: b.tx_hashes) + { + if (found) + break; + if (!db->get_pruned_tx_blob(block_txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << block_txid << " from db"); + return 1; + } + cryptonote::transaction tx2; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx2)) + { + LOG_PRINT_L0("Bad tx: " << block_txid); + return 1; + } + for (size_t out = 0; out < tx2.vout.size(); ++out) + { + if (tx2.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get<cryptonote::txout_to_key>(tx2.vout[out].target); + if (txout.key == od.pubkey) + { + found = true; + new_txids.push_back(block_txid); + MDEBUG("adding txid: " << block_txid); + break; + } + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << block_txid); + return 1; + } + } + } + if (!found) + { + LOG_PRINT_L0("Output originating transaction not found"); + return 1; + } + } + } + else + { + LOG_PRINT_L0("Bad vin type in txid " << txid); + return 1; + } + } + } + if (!coinbase) + { + std::swap(txids, new_txids); + ++depth; + } + } +done: + LOG_PRINT_L0("Min depth for txid " << start_txid << ": " << depth); + depths.push_back(depth); + } + + uint64_t cumulative_depth = 0; + for (uint64_t depth: depths) + cumulative_depth += depth; + LOG_PRINT_L0("Average min depth for " << start_txids.size() << " transaction(s): " << cumulative_depth/(float)depths.size()); + LOG_PRINT_L0("Median min depth for " << start_txids.size() << " transaction(s): " << epee::misc_utils::median(depths)); + + core_storage->deinit(); + return 0; + + CATCH_ENTRY("Depth query error", 1); +} diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp index 7291dbd68..9ec768d26 100644 --- a/src/blockchain_utilities/blockchain_import.cpp +++ b/src/blockchain_utilities/blockchain_import.cpp @@ -474,17 +474,17 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path txs.push_back(tx); } - size_t block_size; + size_t block_weight; difficulty_type cumulative_difficulty; uint64_t coins_generated; - block_size = bp.block_size; + block_weight = bp.block_weight; cumulative_difficulty = bp.cumulative_difficulty; coins_generated = bp.coins_generated; try { - core.get_blockchain_storage().get_db().add_block(b, block_size, cumulative_difficulty, coins_generated, txs); + core.get_blockchain_storage().get_db().add_block(b, block_weight, cumulative_difficulty, coins_generated, txs); } catch (const std::exception& e) { diff --git a/src/blockchain_utilities/bootstrap_file.cpp b/src/blockchain_utilities/bootstrap_file.cpp index ba2697226..beaad2abc 100644 --- a/src/blockchain_utilities/bootstrap_file.cpp +++ b/src/blockchain_utilities/bootstrap_file.cpp @@ -236,11 +236,11 @@ void BootstrapFile::write_block(block& block) bool include_extra_block_data = true; if (include_extra_block_data) { - size_t block_size = m_blockchain_storage->get_db().get_block_size(block_height); + size_t block_weight = m_blockchain_storage->get_db().get_block_weight(block_height); difficulty_type cumulative_difficulty = m_blockchain_storage->get_db().get_block_cumulative_difficulty(block_height); uint64_t coins_generated = m_blockchain_storage->get_db().get_block_already_generated_coins(block_height); - bp.block_size = block_size; + bp.block_weight = block_weight; bp.cumulative_difficulty = cumulative_difficulty; bp.coins_generated = coins_generated; } diff --git a/src/blockchain_utilities/bootstrap_serialization.h b/src/blockchain_utilities/bootstrap_serialization.h index 8755d3fe3..278a7b40f 100644 --- a/src/blockchain_utilities/bootstrap_serialization.h +++ b/src/blockchain_utilities/bootstrap_serialization.h @@ -70,14 +70,14 @@ namespace cryptonote { cryptonote::block block; std::vector<transaction> txs; - size_t block_size; + size_t block_weight; difficulty_type cumulative_difficulty; uint64_t coins_generated; BEGIN_SERIALIZE() FIELD(block) FIELD(txs) - VARINT_FIELD(block_size) + VARINT_FIELD(block_weight) VARINT_FIELD(cumulative_difficulty) VARINT_FIELD(coins_generated) END_SERIALIZE() diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index f0df05b0d..c6bac2199 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -38,7 +38,8 @@ set(common_sources password.cpp perf_timer.cpp threadpool.cpp - updates.cpp) + updates.cpp + aligned.c) if (STACK_TRACE) list(APPEND common_sources stack_trace.cpp) @@ -67,7 +68,8 @@ set(common_private_headers perf_timer.h stack_trace.h threadpool.h - updates.h) + updates.h + aligned.h) monero_private_headers(common ${common_private_headers}) diff --git a/src/common/aligned.c b/src/common/aligned.c new file mode 100644 index 000000000..763dfd0e7 --- /dev/null +++ b/src/common/aligned.c @@ -0,0 +1,139 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <stdint.h> +#include <string.h> +#include "aligned.h" + +static inline int is_power_of_2(size_t n) { return n && (n & (n-1)) == 0; } + +#define MAGIC 0xaa0817161500ff81 +#define MAGIC_FREED 0xaa0817161500ff82 + +static void local_abort(const char *msg) +{ + fprintf(stderr, "%s\n", msg); +#ifdef NDEBUG + _exit(1); +#else + abort(); +#endif +} + +typedef struct +{ + uint64_t magic; + void *raw; + size_t bytes; + size_t align; +} control; + +void *aligned_malloc(size_t bytes, size_t align) +{ + void *raw, *ptr; + control *ctrl; + + if (!is_power_of_2(align)) + return NULL; + if (bytes > (size_t)-1 - align) + return NULL; + if (bytes + align > (size_t)-1 - sizeof(control)) + return NULL; + + raw = malloc(bytes + sizeof(control) + align); + if (!raw) + return NULL; + ptr = (void*)(((uintptr_t)raw + align + sizeof(control) - 1) & ~(align-1)); + ctrl = ((control*)ptr) - 1; + ctrl->magic = MAGIC; + ctrl->raw = raw; + ctrl->bytes = bytes; + ctrl->align = align; + return ptr; +} + +void *aligned_realloc(void *ptr, size_t bytes, size_t align) +{ + void *raw, *ptr2; + control *ctrl, *ctrl2; + + if (!ptr) + return aligned_malloc(bytes, align); + if (!bytes) + { + aligned_free(ptr); + return NULL; + } + if (!is_power_of_2(align)) + return NULL; + + ctrl = ((control*)ptr) - 1; + if (ctrl->magic == MAGIC_FREED) + local_abort("Double free detected"); + if (ctrl->magic != MAGIC) + local_abort("Freeing unallocated memory"); + if (ctrl->align != align) + return NULL; + if (ctrl->bytes >= bytes) + return ptr; + + if (ctrl->bytes > (size_t)-1 - ctrl->align) + return NULL; + if (ctrl->bytes + ctrl->align > (size_t)-1 - sizeof(control)) + return NULL; + + raw = malloc(bytes + sizeof(control) + ctrl->align); + if (!raw) + return NULL; + ptr2 = (void*)(((uintptr_t)raw + ctrl->align + sizeof(control) - 1) & ~(ctrl->align-1)); + memcpy(ptr2, ptr, ctrl->bytes); + ctrl2 = ((control*)ptr2) - 1; + ctrl2->magic = MAGIC; + ctrl2->raw = raw; + ctrl2->bytes = bytes; + ctrl2->align = ctrl->align; + ctrl->magic = MAGIC_FREED; + free(ctrl->raw); + return ptr2; +} + +void aligned_free(void *ptr) +{ + if (!ptr) + return; + control *ctrl = ((control*)ptr) - 1; + if (ctrl->magic == MAGIC_FREED) + local_abort("Double free detected"); + if (ctrl->magic != MAGIC) + local_abort("Freeing unallocated memory"); + ctrl->magic = MAGIC_FREED; + free(ctrl->raw); +} diff --git a/src/common/aligned.h b/src/common/aligned.h new file mode 100644 index 000000000..fed3ccb36 --- /dev/null +++ b/src/common/aligned.h @@ -0,0 +1,41 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +void *aligned_malloc(size_t bytes, size_t align); +void *aligned_realloc(void *ptr, size_t bytes, size_t align); +void aligned_free(void *ptr); + +#ifdef __cplusplus +} +#endif diff --git a/src/common/password.cpp b/src/common/password.cpp index 3ce2ba42a..5671c4a4e 100644 --- a/src/common/password.cpp +++ b/src/common/password.cpp @@ -54,7 +54,7 @@ namespace return 0 != _isatty(_fileno(stdin)); } - bool read_from_tty(epee::wipeable_string& pass) + bool read_from_tty(epee::wipeable_string& pass, bool hide_input) { static constexpr const char BACKSPACE = 8; @@ -62,7 +62,7 @@ namespace DWORD mode_old; ::GetConsoleMode(h_cin, &mode_old); - DWORD mode_new = mode_old & ~(ENABLE_ECHO_INPUT | ENABLE_LINE_INPUT); + DWORD mode_new = mode_old & ~((hide_input ? ENABLE_ECHO_INPUT : 0) | ENABLE_LINE_INPUT); ::SetConsoleMode(h_cin, mode_new); bool r = true; @@ -107,14 +107,14 @@ namespace return 0 != isatty(fileno(stdin)); } - int getch() noexcept + int getch(bool hide_input) noexcept { struct termios tty_old; tcgetattr(STDIN_FILENO, &tty_old); struct termios tty_new; tty_new = tty_old; - tty_new.c_lflag &= ~(ICANON | ECHO); + tty_new.c_lflag &= ~(ICANON | (hide_input ? ECHO : 0)); tcsetattr(STDIN_FILENO, TCSANOW, &tty_new); int ch = getchar(); @@ -124,14 +124,14 @@ namespace return ch; } - bool read_from_tty(epee::wipeable_string& aPass) + bool read_from_tty(epee::wipeable_string& aPass, bool hide_input) { static constexpr const char BACKSPACE = 127; aPass.reserve(tools::password_container::max_password_size); while (aPass.size() < tools::password_container::max_password_size) { - int ch = getch(); + int ch = getch(hide_input); if (EOF == ch || ch == EOT) { return false; @@ -159,18 +159,18 @@ namespace #endif // end !WIN32 - bool read_from_tty(const bool verify, const char *message, epee::wipeable_string& pass1, epee::wipeable_string& pass2) + bool read_from_tty(const bool verify, const char *message, bool hide_input, epee::wipeable_string& pass1, epee::wipeable_string& pass2) { while (true) { if (message) std::cout << message <<": " << std::flush; - if (!read_from_tty(pass1)) + if (!read_from_tty(pass1, hide_input)) return false; if (verify) { std::cout << "Confirm password: "; - if (!read_from_tty(pass2)) + if (!read_from_tty(pass2, hide_input)) return false; if(pass1!=pass2) { @@ -229,12 +229,12 @@ namespace tools std::atomic<bool> password_container::is_prompting(false); - boost::optional<password_container> password_container::prompt(const bool verify, const char *message) + boost::optional<password_container> password_container::prompt(const bool verify, const char *message, bool hide_input) { is_prompting = true; password_container pass1{}; password_container pass2{}; - if (is_cin_tty() ? read_from_tty(verify, message, pass1.m_password, pass2.m_password) : read_from_file(pass1.m_password)) + if (is_cin_tty() ? read_from_tty(verify, message, hide_input, pass1.m_password, pass2.m_password) : read_from_file(pass1.m_password)) { is_prompting = false; return {std::move(pass1)}; diff --git a/src/common/password.h b/src/common/password.h index 61937b93a..529881e40 100644 --- a/src/common/password.h +++ b/src/common/password.h @@ -49,7 +49,7 @@ namespace tools password_container(std::string&& password) noexcept; //! \return A password from stdin TTY prompt or `std::cin` pipe. - static boost::optional<password_container> prompt(bool verify, const char *mesage = "Password"); + static boost::optional<password_container> prompt(bool verify, const char *mesage = "Password", bool hide_input = true); static std::atomic<bool> is_prompting; password_container(const password_container&) = delete; diff --git a/src/common/perf_timer.cpp b/src/common/perf_timer.cpp index 16abdfd99..6910ebdd4 100644 --- a/src/common/perf_timer.cpp +++ b/src/common/perf_timer.cpp @@ -33,7 +33,7 @@ #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "perf" -namespace +namespace tools { uint64_t get_tick_count() { @@ -83,7 +83,7 @@ namespace tools el::Level performance_timer_log_level = el::Level::Debug; -static __thread std::vector<PerformanceTimer*> *performance_timers = NULL; +static __thread std::vector<LoggingPerformanceTimer*> *performance_timers = NULL; void set_performance_timer_log_level(el::Level level) { @@ -96,17 +96,24 @@ void set_performance_timer_log_level(el::Level level) performance_timer_log_level = level; } -PerformanceTimer::PerformanceTimer(const std::string &s, uint64_t unit, el::Level l): name(s), unit(unit), level(l), started(false), paused(false) +PerformanceTimer::PerformanceTimer(bool paused): started(true), paused(paused) +{ + if (paused) + ticks = 0; + else + ticks = get_tick_count(); +} + +LoggingPerformanceTimer::LoggingPerformanceTimer(const std::string &s, uint64_t unit, el::Level l): PerformanceTimer(), name(s), unit(unit), level(l) { - ticks = get_tick_count(); if (!performance_timers) { MLOG(level, "PERF ----------"); - performance_timers = new std::vector<PerformanceTimer*>(); + performance_timers = new std::vector<LoggingPerformanceTimer*>(); } else { - PerformanceTimer *pt = performance_timers->back(); + LoggingPerformanceTimer *pt = performance_timers->back(); if (!pt->started && !pt->paused) { size_t size = 0; for (const auto *tmp: *performance_timers) if (!tmp->paused) ++size; @@ -119,9 +126,14 @@ PerformanceTimer::PerformanceTimer(const std::string &s, uint64_t unit, el::Leve PerformanceTimer::~PerformanceTimer() { - performance_timers->pop_back(); if (!paused) ticks = get_tick_count() - ticks; +} + +LoggingPerformanceTimer::~LoggingPerformanceTimer() +{ + pause(); + performance_timers->pop_back(); char s[12]; snprintf(s, sizeof(s), "%8llu ", (unsigned long long)(ticks_to_ns(ticks) / (1000000000 / unit))); size_t size = 0; for (const auto *tmp: *performance_timers) if (!tmp->paused || tmp==this) ++size; diff --git a/src/common/perf_timer.h b/src/common/perf_timer.h index 0e910caf9..675d6234d 100644 --- a/src/common/perf_timer.h +++ b/src/common/perf_timer.h @@ -43,30 +43,46 @@ class PerformanceTimer; extern el::Level performance_timer_log_level; +uint64_t get_tick_count(); +uint64_t get_ticks_per_ns(); +uint64_t ticks_to_ns(uint64_t ticks); + class PerformanceTimer { public: - PerformanceTimer(const std::string &s, uint64_t unit, el::Level l = el::Level::Debug); + PerformanceTimer(bool paused = false); ~PerformanceTimer(); void pause(); void resume(); + uint64_t value() const { return ticks; } +void set(uint64_t v){ticks=v;} + +protected: + uint64_t ticks; + bool started; + bool paused; +}; + +class LoggingPerformanceTimer: public PerformanceTimer +{ +public: + LoggingPerformanceTimer(const std::string &s, uint64_t unit, el::Level l = el::Level::Debug); + ~LoggingPerformanceTimer(); + private: std::string name; uint64_t unit; el::Level level; - uint64_t ticks; - bool started; - bool paused; }; void set_performance_timer_log_level(el::Level level); -#define PERF_TIMER_UNIT(name, unit) tools::PerformanceTimer pt_##name(#name, unit, tools::performance_timer_log_level) -#define PERF_TIMER_UNIT_L(name, unit, l) tools::PerformanceTimer pt_##name(#name, unit, l) +#define PERF_TIMER_UNIT(name, unit) tools::LoggingPerformanceTimer pt_##name(#name, unit, tools::performance_timer_log_level) +#define PERF_TIMER_UNIT_L(name, unit, l) tools::LoggingPerformanceTimer pt_##name(#name, unit, l) #define PERF_TIMER(name) PERF_TIMER_UNIT(name, 1000) #define PERF_TIMER_L(name, l) PERF_TIMER_UNIT_L(name, 1000, l) -#define PERF_TIMER_START_UNIT(name, unit) std::unique_ptr<tools::PerformanceTimer> pt_##name(new tools::PerformanceTimer(#name, unit, el::Level::Info)) +#define PERF_TIMER_START_UNIT(name, unit) std::unique_ptr<tools::LoggingPerformanceTimer> pt_##name(new tools::LoggingPerformanceTimer(#name, unit, el::Level::Info)) #define PERF_TIMER_START(name) PERF_TIMER_START_UNIT(name, 1000) #define PERF_TIMER_STOP(name) do { pt_##name.reset(NULL); } while(0) #define PERF_TIMER_PAUSE(name) pt_##name->pause() diff --git a/src/common/updates.cpp b/src/common/updates.cpp index 9eb402e0b..9f12f8dbc 100644 --- a/src/common/updates.cpp +++ b/src/common/updates.cpp @@ -69,12 +69,12 @@ namespace tools continue; bool alnum = true; - for (auto c: hash) + for (auto c: fields[3]) if (!isalnum(c)) alnum = false; - if (hash.size() != 64 && !alnum) + if (fields[3].size() != 64 && !alnum) { - MWARNING("Invalid hash: " << hash); + MWARNING("Invalid hash: " << fields[3]); continue; } diff --git a/src/common/util.cpp b/src/common/util.cpp index 5e0d2726e..c56c77505 100644 --- a/src/common/util.cpp +++ b/src/common/util.cpp @@ -38,7 +38,6 @@ #include <sys/types.h> #include <sys/stat.h> #include <sys/resource.h> -#include <ustat.h> #include <unistd.h> #include <dirent.h> #include <string.h> @@ -46,6 +45,13 @@ #include <string> #endif +//tools::is_hdd +#ifdef __GLIBC__ + #include <sstream> + #include <sys/sysmacros.h> + #include <fstream> +#endif + #include "unbound.h" #include "include_base_utils.h" @@ -733,62 +739,41 @@ std::string get_nix_version_display_string() #endif } - bool is_hdd(const char *path) + boost::optional<bool> is_hdd(const char *file_path) { #ifdef __GLIBC__ - std::string device = ""; - struct stat st, dst; - if (stat(path, &st) < 0) - return 0; - - DIR *dir = opendir("/dev/block"); - if (!dir) - return 0; - struct dirent *de; - while ((de = readdir(dir))) - { - if (strcmp(de->d_name, ".") && strcmp(de->d_name, "..")) + struct stat st; + std::string prefix; + if(stat(file_path, &st) == 0) + { + std::ostringstream s; + s << "/sys/dev/block/" << major(st.st_dev) << ":" << minor(st.st_dev); + prefix = s.str(); + } + else + { + return boost::none; + } + std::string attr_path = prefix + "/queue/rotational"; + std::ifstream f(attr_path, std::ios_base::in); + if(not f.is_open()) + { + attr_path = prefix + "/../queue/rotational"; + f.open(attr_path, std::ios_base::in); + if(not f.is_open()) { - std::string dev_path = std::string("/dev/block/") + de->d_name; - char resolved[PATH_MAX]; - if (realpath(dev_path.c_str(), resolved) && !strncmp(resolved, "/dev/", 5)) - { - if (stat(resolved, &dst) == 0) - { - if (dst.st_rdev == st.st_dev) - { - // take out trailing digits (eg, sda1 -> sda) - char *ptr = resolved; - while (*ptr) - ++ptr; - while (ptr > resolved && isdigit(*--ptr)) - *ptr = 0; - device = resolved + 5; - break; - } - } - } + return boost::none; } } - closedir(dir); - - if (device.empty()) - return 0; - - std::string sys_path = "/sys/block/" + device + "/queue/rotational"; - FILE *f = fopen(sys_path.c_str(), "r"); - if (!f) - return false; - char s[8]; - char *ptr = fgets(s, sizeof(s), f); - fclose(f); - if (!ptr) - return 0; - s[sizeof(s) - 1] = 0; - int n = atoi(s); // returns 0 on parse error - return n == 1; + unsigned short val = 0xdead; + f >> val; + if(not f.fail()) + { + return (val == 1); + } + return boost::none; #else - return 0; + return boost::none; #endif } diff --git a/src/common/util.h b/src/common/util.h index 8815232e2..0e0b50520 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -230,7 +230,7 @@ namespace tools bool sha256sum(const uint8_t *data, size_t len, crypto::hash &hash); bool sha256sum(const std::string &filename, crypto::hash &hash); - bool is_hdd(const char *path); + boost::optional<bool> is_hdd(const char *path); boost::optional<std::pair<uint32_t, uint32_t>> parse_subaddress_lookahead(const std::string& str); diff --git a/src/crypto/CMakeLists.txt b/src/crypto/CMakeLists.txt index 71dcedcab..0c635e7cb 100644 --- a/src/crypto/CMakeLists.txt +++ b/src/crypto/CMakeLists.txt @@ -78,6 +78,7 @@ target_link_libraries(cncrypto PUBLIC epee ${Boost_SYSTEM_LIBRARY} + ${SODIUM_LIBRARY} PRIVATE ${EXTRA_LIBRARIES}) diff --git a/src/crypto/chacha.h b/src/crypto/chacha.h index 1dc270faf..6e85ad0e9 100644 --- a/src/crypto/chacha.h +++ b/src/crypto/chacha.h @@ -40,6 +40,7 @@ #include <memory.h> #include "memwipe.h" +#include "mlocker.h" #include "hash.h" namespace crypto { @@ -50,7 +51,7 @@ namespace crypto { #if defined(__cplusplus) } - using chacha_key = tools::scrubbed_arr<uint8_t, CHACHA_KEY_SIZE>; + using chacha_key = epee::mlocked<tools::scrubbed_arr<uint8_t, CHACHA_KEY_SIZE>>; #pragma pack(push, 1) // MS VC 2012 doesn't interpret `class chacha_iv` as POD in spite of [9.0.10], so it is a struct @@ -71,20 +72,20 @@ namespace crypto { inline void generate_chacha_key(const void *data, size_t size, chacha_key& key, uint64_t kdf_rounds) { static_assert(sizeof(chacha_key) <= sizeof(hash), "Size of hash must be at least that of chacha_key"); - tools::scrubbed_arr<char, HASH_SIZE> pwd_hash; + epee::mlocked<tools::scrubbed_arr<char, HASH_SIZE>> pwd_hash; crypto::cn_slow_hash(data, size, pwd_hash.data(), 0/*variant*/, 0/*prehashed*/); for (uint64_t n = 1; n < kdf_rounds; ++n) crypto::cn_slow_hash(pwd_hash.data(), pwd_hash.size(), pwd_hash.data(), 0/*variant*/, 0/*prehashed*/); - memcpy(&unwrap(key), pwd_hash.data(), sizeof(key)); + memcpy(&unwrap(unwrap(key)), pwd_hash.data(), sizeof(key)); } inline void generate_chacha_key_prehashed(const void *data, size_t size, chacha_key& key, uint64_t kdf_rounds) { static_assert(sizeof(chacha_key) <= sizeof(hash), "Size of hash must be at least that of chacha_key"); - tools::scrubbed_arr<char, HASH_SIZE> pwd_hash; + epee::mlocked<tools::scrubbed_arr<char, HASH_SIZE>> pwd_hash; crypto::cn_slow_hash(data, size, pwd_hash.data(), 0/*variant*/, 1/*prehashed*/); for (uint64_t n = 1; n < kdf_rounds; ++n) crypto::cn_slow_hash(pwd_hash.data(), pwd_hash.size(), pwd_hash.data(), 0/*variant*/, 0/*prehashed*/); - memcpy(&unwrap(key), pwd_hash.data(), sizeof(key)); + memcpy(&unwrap(unwrap(key)), pwd_hash.data(), sizeof(key)); } inline void generate_chacha_key(std::string password, chacha_key& key, uint64_t kdf_rounds) { diff --git a/src/crypto/crypto-ops-data.c b/src/crypto/crypto-ops-data.c index 4ff4310de..1f77513ca 100644 --- a/src/crypto/crypto-ops-data.c +++ b/src/crypto/crypto-ops-data.c @@ -871,3 +871,9 @@ const fe fe_fffb2 = {8166131, -6741800, -17040804, 3154616, 21461005, 1466302, - const fe fe_fffb3 = {-13620103, 14639558, 4532995, 7679154, 16815101, -15883539, -22863840, -14813421, 13716513, -6477756}; /* sqrt(-sqrt(-1) * A * (A + 2)) */ const fe fe_fffb4 = {-21786234, -12173074, 21573800, 4524538, -4645904, 16204591, 8012863, -8444712, 3212926, 6885324}; /* sqrt(sqrt(-1) * A * (A + 2)) */ const ge_p3 ge_p3_identity = { {0}, {1, 0}, {1, 0}, {0} }; +const ge_p3 ge_p3_H = { + {7329926, -15101362, 31411471, 7614783, 27996851, -3197071, -11157635, -6878293, 466949, -7986503}, + {5858699, 5096796, 21321203, -7536921, -5553480, -11439507, -5627669, 15045946, 19977121, 5275251}, + {1, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + {23443568, -5110398, -8776029, -4345135, 6889568, -14710814, 7474843, 3279062, 14550766, -7453428} +}; diff --git a/src/crypto/crypto-ops.c b/src/crypto/crypto-ops.c index 45d412ac6..09296d6f9 100644 --- a/src/crypto/crypto-ops.c +++ b/src/crypto/crypto-ops.c @@ -3707,9 +3707,8 @@ void sc_muladd(unsigned char *s, const unsigned char *a, const unsigned char *b, s[31] = s11 >> 17; } -/* Assumes that a != INT64_MIN */ static int64_t signum(int64_t a) { - return (a >> 63) - ((-a) >> 63); + return a > 0 ? 1 : a < 0 ? -1 : 0; } int sc_check(const unsigned char *s) { @@ -3730,3 +3729,16 @@ int sc_isnonzero(const unsigned char *s) { s[18] | s[19] | s[20] | s[21] | s[22] | s[23] | s[24] | s[25] | s[26] | s[27] | s[28] | s[29] | s[30] | s[31]) - 1) >> 8) + 1; } + +int ge_p3_is_point_at_infinity(const ge_p3 *p) { + // X = 0 and Y == Z + int n; + for (n = 0; n < 10; ++n) + { + if (p->X[n] | p->T[n]) + return 0; + if (p->Y[n] != p->Z[n]) + return 0; + } + return 1; +} diff --git a/src/crypto/crypto-ops.h b/src/crypto/crypto-ops.h index dc3c60794..2910dafd4 100644 --- a/src/crypto/crypto-ops.h +++ b/src/crypto/crypto-ops.h @@ -140,6 +140,7 @@ extern const fe fe_fffb2; extern const fe fe_fffb3; extern const fe fe_fffb4; extern const ge_p3 ge_p3_identity; +extern const ge_p3 ge_p3_H; void ge_fromfe_frombytes_vartime(ge_p2 *, const unsigned char *); void sc_0(unsigned char *); void sc_reduce32(unsigned char *); @@ -158,3 +159,5 @@ void ge_sub(ge_p1p1 *r, const ge_p3 *p, const ge_cached *q); void fe_add(fe h, const fe f, const fe g); void fe_tobytes(unsigned char *, const fe); void fe_invert(fe out, const fe z); + +int ge_p3_is_point_at_infinity(const ge_p3 *p); diff --git a/src/crypto/crypto.cpp b/src/crypto/crypto.cpp index 4243c71fd..ad7721cf0 100644 --- a/src/crypto/crypto.cpp +++ b/src/crypto/crypto.cpp @@ -116,7 +116,7 @@ namespace crypto { do { generate_random_bytes_thread_safe(32, bytes); - } while (!less32(bytes, limit)); // should be good about 15/16 of the time + } while (!sc_isnonzero(bytes) && !less32(bytes, limit)); // should be good about 15/16 of the time sc_reduce32(bytes); } /* generate a random 32-byte (256-bit) integer and copy it to res */ @@ -274,11 +274,18 @@ namespace crypto { #endif buf.h = prefix_hash; buf.key = pub; + try_again: random_scalar(k); + if (((const uint32_t*)(&k))[7] == 0) // we don't want tiny numbers here + goto try_again; ge_scalarmult_base(&tmp3, &k); ge_p3_tobytes(&buf.comm, &tmp3); hash_to_scalar(&buf, sizeof(s_comm), sig.c); + if (!sc_isnonzero((const unsigned char*)sig.c.data)) + goto try_again; sc_mulsub(&sig.r, &sig.c, &unwrap(sec), &k); + if (!sc_isnonzero((const unsigned char*)sig.r.data)) + goto try_again; } bool crypto_ops::check_signature(const hash &prefix_hash, const public_key &pub, const signature &sig) { @@ -292,11 +299,14 @@ namespace crypto { if (ge_frombytes_vartime(&tmp3, &pub) != 0) { return false; } - if (sc_check(&sig.c) != 0 || sc_check(&sig.r) != 0) { + if (sc_check(&sig.c) != 0 || sc_check(&sig.r) != 0 || !sc_isnonzero(&sig.c)) { return false; } ge_double_scalarmult_base_vartime(&tmp2, &sig.c, &tmp3, &sig.r); ge_tobytes(&buf.comm, &tmp2); + static const ec_point infinity = {{ 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; + if (memcmp(&buf.comm, &infinity, 32) == 0) + return false; hash_to_scalar(&buf, sizeof(s_comm), c); sc_sub(&c, &c, &sig.c); return sc_isnonzero(&c) == 0; diff --git a/src/crypto/crypto.h b/src/crypto/crypto.h index a2d61b04e..33cc0a25a 100644 --- a/src/crypto/crypto.h +++ b/src/crypto/crypto.h @@ -41,6 +41,7 @@ #include "common/pod-class.h" #include "common/util.h" #include "memwipe.h" +#include "mlocker.h" #include "generic-ops.h" #include "hex.h" #include "span.h" @@ -65,7 +66,7 @@ namespace crypto { friend class crypto_ops; }; - using secret_key = tools::scrubbed<ec_scalar>; + using secret_key = epee::mlocked<tools::scrubbed<ec_scalar>>; POD_CLASS public_keyV { std::vector<public_key> keys; @@ -282,6 +283,6 @@ namespace crypto { } CRYPTO_MAKE_HASHABLE(public_key) -CRYPTO_MAKE_HASHABLE(secret_key) +CRYPTO_MAKE_HASHABLE_CONSTANT_TIME(secret_key) CRYPTO_MAKE_HASHABLE(key_image) CRYPTO_MAKE_COMPARABLE(signature) diff --git a/src/crypto/generic-ops.h b/src/crypto/generic-ops.h index 62bc758c9..42b98706e 100644 --- a/src/crypto/generic-ops.h +++ b/src/crypto/generic-ops.h @@ -33,19 +33,30 @@ #include <cstddef> #include <cstring> #include <functional> +#include <sodium/crypto_verify_32.h> #define CRYPTO_MAKE_COMPARABLE(type) \ namespace crypto { \ inline bool operator==(const type &_v1, const type &_v2) { \ - return std::memcmp(&_v1, &_v2, sizeof(type)) == 0; \ + return !memcmp(&_v1, &_v2, sizeof(_v1)); \ } \ inline bool operator!=(const type &_v1, const type &_v2) { \ - return std::memcmp(&_v1, &_v2, sizeof(type)) != 0; \ + return !operator==(_v1, _v2); \ } \ } -#define CRYPTO_MAKE_HASHABLE(type) \ -CRYPTO_MAKE_COMPARABLE(type) \ +#define CRYPTO_MAKE_COMPARABLE_CONSTANT_TIME(type) \ +namespace crypto { \ + inline bool operator==(const type &_v1, const type &_v2) { \ + static_assert(sizeof(_v1) == 32, "constant time comparison is only implenmted for 32 bytes"); \ + return crypto_verify_32((const unsigned char*)&_v1, (const unsigned char*)&_v2) == 0; \ + } \ + inline bool operator!=(const type &_v1, const type &_v2) { \ + return !operator==(_v1, _v2); \ + } \ +} + +#define CRYPTO_DEFINE_HASH_FUNCTIONS(type) \ namespace crypto { \ static_assert(sizeof(std::size_t) <= sizeof(type), "Size of " #type " must be at least that of size_t"); \ inline std::size_t hash_value(const type &_v) { \ @@ -60,3 +71,12 @@ namespace std { \ } \ }; \ } + +#define CRYPTO_MAKE_HASHABLE(type) \ +CRYPTO_MAKE_COMPARABLE(type) \ +CRYPTO_DEFINE_HASH_FUNCTIONS(type) + +#define CRYPTO_MAKE_HASHABLE_CONSTANT_TIME(type) \ +CRYPTO_MAKE_COMPARABLE_CONSTANT_TIME(type) \ +CRYPTO_DEFINE_HASH_FUNCTIONS(type) + diff --git a/src/crypto/keccak.c b/src/crypto/keccak.c index de8e2a5b3..8fcd2138e 100644 --- a/src/crypto/keccak.c +++ b/src/crypto/keccak.c @@ -132,3 +132,77 @@ void keccak1600(const uint8_t *in, size_t inlen, uint8_t *md) { keccak(in, inlen, md, sizeof(state_t)); } + +#define KECCAK_FINALIZED 0x80000000 +#define KECCAK_BLOCKLEN 136 +#define KECCAK_WORDS 17 +#define KECCAK_DIGESTSIZE 32 +#define IS_ALIGNED_64(p) (0 == (7 & ((const char*)(p) - (const char*)0))) +#define KECCAK_PROCESS_BLOCK(st, block) { \ + for (int i_ = 0; i_ < KECCAK_WORDS; i_++){ \ + ((st))[i_] ^= ((block))[i_]; \ + }; \ + keccakf(st, KECCAK_ROUNDS); } + + +void keccak_init(KECCAK_CTX * ctx){ + memset(ctx, 0, sizeof(KECCAK_CTX)); +} + +void keccak_update(KECCAK_CTX * ctx, const uint8_t *in, size_t inlen){ + if (ctx->rest & KECCAK_FINALIZED) { + local_abort("Bad keccak use"); + } + + const size_t idx = ctx->rest; + ctx->rest = (ctx->rest + inlen) % KECCAK_BLOCKLEN; + + // fill partial block + if (idx) { + size_t left = KECCAK_BLOCKLEN - idx; + memcpy((char*)ctx->message + idx, in, (inlen < left ? inlen : left)); + if (inlen < left) return; + + KECCAK_PROCESS_BLOCK(ctx->hash, ctx->message); + + in += left; + inlen -= left; + } + + const bool is_aligned = IS_ALIGNED_64(in); + while (inlen >= KECCAK_BLOCKLEN) { + const uint64_t* aligned_message_block; + if (is_aligned) { + aligned_message_block = (uint64_t*)in; + } else { + memcpy(ctx->message, in, KECCAK_BLOCKLEN); + aligned_message_block = ctx->message; + } + + KECCAK_PROCESS_BLOCK(ctx->hash, aligned_message_block); + in += KECCAK_BLOCKLEN; + inlen -= KECCAK_BLOCKLEN; + } + if (inlen) { + memcpy(ctx->message, in, inlen); + } +} + +void keccak_finish(KECCAK_CTX * ctx, uint8_t *md){ + if (!(ctx->rest & KECCAK_FINALIZED)) + { + // clear the rest of the data queue + memset((char*)ctx->message + ctx->rest, 0, KECCAK_BLOCKLEN - ctx->rest); + ((char*)ctx->message)[ctx->rest] |= 0x01; + ((char*)ctx->message)[KECCAK_BLOCKLEN - 1] |= 0x80; + + // process final block + KECCAK_PROCESS_BLOCK(ctx->hash, ctx->message); + ctx->rest = KECCAK_FINALIZED; // mark context as finalized + } + + static_assert(KECCAK_BLOCKLEN > KECCAK_DIGESTSIZE, ""); + if (md) { + memcpy(md, ctx->hash, KECCAK_DIGESTSIZE); + } +} diff --git a/src/crypto/keccak.h b/src/crypto/keccak.h index fb9d8bd04..9123c7a3b 100644 --- a/src/crypto/keccak.h +++ b/src/crypto/keccak.h @@ -15,6 +15,17 @@ #define ROTL64(x, y) (((x) << (y)) | ((x) >> (64 - (y)))) #endif +// SHA3 Algorithm context. +typedef struct KECCAK_CTX +{ + // 1600 bits algorithm hashing state + uint64_t hash[25]; + // 1088-bit buffer for leftovers, block size = 136 B for 256-bit keccak + uint64_t message[17]; + // count of bytes in the message[] buffer + size_t rest; +} KECCAK_CTX; + // compute a keccak hash (md) of given byte length from "in" void keccak(const uint8_t *in, size_t inlen, uint8_t *md, int mdlen); @@ -23,4 +34,7 @@ void keccakf(uint64_t st[25], int norounds); void keccak1600(const uint8_t *in, size_t inlen, uint8_t *md); +void keccak_init(KECCAK_CTX * ctx); +void keccak_update(KECCAK_CTX * ctx, const uint8_t *in, size_t inlen); +void keccak_finish(KECCAK_CTX * ctx, uint8_t *md); #endif diff --git a/src/crypto/slow-hash.c b/src/crypto/slow-hash.c index 9d4fc0dfa..a4d2b58de 100644 --- a/src/crypto/slow-hash.c +++ b/src/crypto/slow-hash.c @@ -38,6 +38,7 @@ #include "common/int-util.h" #include "hash-ops.h" #include "oaes_lib.h" +#include "variant2_int_sqrt.h" #define MEMORY (1 << 21) // 2MB scratchpad #define ITER (1 << 20) @@ -50,7 +51,7 @@ extern int aesb_single_round(const uint8_t *in, uint8_t*out, const uint8_t *expa extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *expandedKey); #define VARIANT1_1(p) \ - do if (variant > 0) \ + do if (variant == 1) \ { \ const uint8_t tmp = ((const uint8_t*)(p))[11]; \ static const uint32_t table = 0x75310; \ @@ -59,7 +60,7 @@ extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *exp } while(0) #define VARIANT1_2(p) \ - do if (variant > 0) \ + do if (variant == 1) \ { \ xor64(p, tweak1_2); \ } while(0) @@ -67,7 +68,7 @@ extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *exp #define VARIANT1_CHECK() \ do if (length < 43) \ { \ - fprintf(stderr, "Cryptonight variants need at least 43 bytes of data"); \ + fprintf(stderr, "Cryptonight variant 1 needs at least 43 bytes of data"); \ _exit(1); \ } while(0) @@ -75,7 +76,7 @@ extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *exp #define VARIANT1_PORTABLE_INIT() \ uint8_t tweak1_2[8]; \ - do if (variant > 0) \ + do if (variant == 1) \ { \ VARIANT1_CHECK(); \ memcpy(&tweak1_2, &state.hs.b[192], sizeof(tweak1_2)); \ @@ -83,11 +84,119 @@ extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *exp } while(0) #define VARIANT1_INIT64() \ - if (variant > 0) \ + if (variant == 1) \ { \ VARIANT1_CHECK(); \ } \ - const uint64_t tweak1_2 = variant > 0 ? (state.hs.w[24] ^ (*((const uint64_t*)NONCE_POINTER))) : 0 + const uint64_t tweak1_2 = (variant == 1) ? (state.hs.w[24] ^ (*((const uint64_t*)NONCE_POINTER))) : 0 + +#define VARIANT2_INIT64() \ + uint64_t division_result = 0; \ + uint64_t sqrt_result = 0; \ + do if (variant >= 2) \ + { \ + U64(b)[2] = state.hs.w[8] ^ state.hs.w[10]; \ + U64(b)[3] = state.hs.w[9] ^ state.hs.w[11]; \ + division_result = state.hs.w[12]; \ + sqrt_result = state.hs.w[13]; \ + } while (0) + +#define VARIANT2_PORTABLE_INIT() \ + uint64_t division_result = 0; \ + uint64_t sqrt_result = 0; \ + do if (variant >= 2) \ + { \ + memcpy(b + AES_BLOCK_SIZE, state.hs.b + 64, AES_BLOCK_SIZE); \ + xor64(b + AES_BLOCK_SIZE, state.hs.b + 80); \ + xor64(b + AES_BLOCK_SIZE + 8, state.hs.b + 88); \ + division_result = state.hs.w[12]; \ + sqrt_result = state.hs.w[13]; \ + } while (0) + +#define VARIANT2_SHUFFLE_ADD_SSE2(base_ptr, offset) \ + do if (variant >= 2) \ + { \ + const __m128i chunk1 = _mm_load_si128((__m128i *)((base_ptr) + ((offset) ^ 0x10))); \ + const __m128i chunk2 = _mm_load_si128((__m128i *)((base_ptr) + ((offset) ^ 0x20))); \ + const __m128i chunk3 = _mm_load_si128((__m128i *)((base_ptr) + ((offset) ^ 0x30))); \ + _mm_store_si128((__m128i *)((base_ptr) + ((offset) ^ 0x10)), _mm_add_epi64(chunk3, _b1)); \ + _mm_store_si128((__m128i *)((base_ptr) + ((offset) ^ 0x20)), _mm_add_epi64(chunk1, _b)); \ + _mm_store_si128((__m128i *)((base_ptr) + ((offset) ^ 0x30)), _mm_add_epi64(chunk2, _a)); \ + } while (0) + +#define VARIANT2_SHUFFLE_ADD_NEON(base_ptr, offset) \ + do if (variant >= 2) \ + { \ + const uint64x2_t chunk1 = vld1q_u64(U64((base_ptr) + ((offset) ^ 0x10))); \ + const uint64x2_t chunk2 = vld1q_u64(U64((base_ptr) + ((offset) ^ 0x20))); \ + const uint64x2_t chunk3 = vld1q_u64(U64((base_ptr) + ((offset) ^ 0x30))); \ + vst1q_u64(U64((base_ptr) + ((offset) ^ 0x10)), vaddq_u64(chunk3, vreinterpretq_u64_u8(_b1))); \ + vst1q_u64(U64((base_ptr) + ((offset) ^ 0x20)), vaddq_u64(chunk1, vreinterpretq_u64_u8(_b))); \ + vst1q_u64(U64((base_ptr) + ((offset) ^ 0x30)), vaddq_u64(chunk2, vreinterpretq_u64_u8(_a))); \ + } while (0) + +#define VARIANT2_PORTABLE_SHUFFLE_ADD(base_ptr, offset) \ + do if (variant >= 2) \ + { \ + uint64_t* chunk1 = U64((base_ptr) + ((offset) ^ 0x10)); \ + uint64_t* chunk2 = U64((base_ptr) + ((offset) ^ 0x20)); \ + uint64_t* chunk3 = U64((base_ptr) + ((offset) ^ 0x30)); \ + \ + const uint64_t chunk1_old[2] = { chunk1[0], chunk1[1] }; \ + \ + uint64_t b1[2]; \ + memcpy(b1, b + 16, 16); \ + chunk1[0] = chunk3[0] + b1[0]; \ + chunk1[1] = chunk3[1] + b1[1]; \ + \ + uint64_t a0[2]; \ + memcpy(a0, a, 16); \ + chunk3[0] = chunk2[0] + a0[0]; \ + chunk3[1] = chunk2[1] + a0[1]; \ + \ + uint64_t b0[2]; \ + memcpy(b0, b, 16); \ + chunk2[0] = chunk1_old[0] + b0[0]; \ + chunk2[1] = chunk1_old[1] + b0[1]; \ + } while (0) + +#define VARIANT2_INTEGER_MATH_DIVISION_STEP(b, ptr) \ + ((uint64_t*)(b))[0] ^= division_result ^ (sqrt_result << 32); \ + { \ + const uint64_t dividend = ((uint64_t*)(ptr))[1]; \ + const uint32_t divisor = (((uint64_t*)(ptr))[0] + (uint32_t)(sqrt_result << 1)) | 0x80000001UL; \ + division_result = ((uint32_t)(dividend / divisor)) + \ + (((uint64_t)(dividend % divisor)) << 32); \ + } \ + const uint64_t sqrt_input = ((uint64_t*)(ptr))[0] + division_result + +#define VARIANT2_INTEGER_MATH_SSE2(b, ptr) \ + do if (variant >= 2) \ + { \ + VARIANT2_INTEGER_MATH_DIVISION_STEP(b, ptr); \ + VARIANT2_INTEGER_MATH_SQRT_STEP_SSE2(); \ + VARIANT2_INTEGER_MATH_SQRT_FIXUP(sqrt_result); \ + } while(0) + +#if defined DBL_MANT_DIG && (DBL_MANT_DIG >= 50) + // double precision floating point type has enough bits of precision on current platform + #define VARIANT2_PORTABLE_INTEGER_MATH(b, ptr) \ + do if (variant >= 2) \ + { \ + VARIANT2_INTEGER_MATH_DIVISION_STEP(b, ptr); \ + VARIANT2_INTEGER_MATH_SQRT_STEP_FP64(); \ + VARIANT2_INTEGER_MATH_SQRT_FIXUP(sqrt_result); \ + } while (0) +#else + // double precision floating point type is not good enough on current platform + // fall back to the reference code (integer only) + #define VARIANT2_PORTABLE_INTEGER_MATH(b, ptr) \ + do if (variant >= 2) \ + { \ + VARIANT2_INTEGER_MATH_DIVISION_STEP(b, ptr); \ + VARIANT2_INTEGER_MATH_SQRT_STEP_REF(); \ + } while (0) +#endif #if !defined NO_AES && (defined(__x86_64__) || (defined(_MSC_VER) && defined(_WIN64))) // Optimised code below, uses x86-specific intrinsics, SSE2, AES-NI @@ -164,19 +273,22 @@ extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *exp * This code is based upon an optimized implementation by dga. */ #define post_aes() \ + VARIANT2_SHUFFLE_ADD_SSE2(hp_state, j); \ _mm_store_si128(R128(c), _c); \ - _b = _mm_xor_si128(_b, _c); \ - _mm_store_si128(R128(&hp_state[j]), _b); \ + _mm_store_si128(R128(&hp_state[j]), _mm_xor_si128(_b, _c)); \ VARIANT1_1(&hp_state[j]); \ j = state_index(c); \ p = U64(&hp_state[j]); \ b[0] = p[0]; b[1] = p[1]; \ + VARIANT2_INTEGER_MATH_SSE2(b, c); \ __mul(); \ + VARIANT2_SHUFFLE_ADD_SSE2(hp_state, j); \ a[0] += hi; a[1] += lo; \ p = U64(&hp_state[j]); \ p[0] = a[0]; p[1] = a[1]; \ a[0] ^= b[0]; a[1] ^= b[1]; \ VARIANT1_2(p + 1); \ + _b1 = _b; \ _b = _c; \ #if defined(_MSC_VER) @@ -570,10 +682,10 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int uint8_t text[INIT_SIZE_BYTE]; RDATA_ALIGN16 uint64_t a[2]; - RDATA_ALIGN16 uint64_t b[2]; + RDATA_ALIGN16 uint64_t b[4]; RDATA_ALIGN16 uint64_t c[2]; union cn_slow_hash_state state; - __m128i _a, _b, _c; + __m128i _a, _b, _b1, _c; uint64_t hi, lo; size_t i, j; @@ -599,6 +711,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int memcpy(text, state.init, INIT_SIZE_BYTE); VARIANT1_INIT64(); + VARIANT2_INIT64(); /* CryptoNight Step 2: Iteratively encrypt the results from Keccak to fill * the 2MB large random access buffer. @@ -637,6 +750,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int */ _b = _mm_load_si128(R128(b)); + _b1 = _mm_load_si128(R128(b) + 1); // Two independent versions, one with AES, one without, to ensure that // the useAes test is only performed once, not every iteration. if(useAes) @@ -761,19 +875,22 @@ union cn_slow_hash_state _a = vld1q_u8((const uint8_t *)a); \ #define post_aes() \ + VARIANT2_SHUFFLE_ADD_NEON(hp_state, j); \ vst1q_u8((uint8_t *)c, _c); \ - _b = veorq_u8(_b, _c); \ - vst1q_u8(&hp_state[j], _b); \ + vst1q_u8(&hp_state[j], veorq_u8(_b, _c)); \ VARIANT1_1(&hp_state[j]); \ j = state_index(c); \ p = U64(&hp_state[j]); \ b[0] = p[0]; b[1] = p[1]; \ + VARIANT2_PORTABLE_INTEGER_MATH(b, c); \ __mul(); \ + VARIANT2_SHUFFLE_ADD_NEON(hp_state, j); \ a[0] += hi; a[1] += lo; \ p = U64(&hp_state[j]); \ p[0] = a[0]; p[1] = a[1]; \ a[0] ^= b[0]; a[1] ^= b[1]; \ VARIANT1_2(p + 1); \ + _b1 = _b; \ _b = _c; \ @@ -912,10 +1029,10 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int uint8_t text[INIT_SIZE_BYTE]; RDATA_ALIGN16 uint64_t a[2]; - RDATA_ALIGN16 uint64_t b[2]; + RDATA_ALIGN16 uint64_t b[4]; RDATA_ALIGN16 uint64_t c[2]; union cn_slow_hash_state state; - uint8x16_t _a, _b, _c, zero = {0}; + uint8x16_t _a, _b, _b1, _c, zero = {0}; uint64_t hi, lo; size_t i, j; @@ -936,6 +1053,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int memcpy(text, state.init, INIT_SIZE_BYTE); VARIANT1_INIT64(); + VARIANT2_INIT64(); /* CryptoNight Step 2: Iteratively encrypt the results from Keccak to fill * the 2MB large random access buffer. @@ -959,7 +1077,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int */ _b = vld1q_u8((const uint8_t *)b); - + _b1 = vld1q_u8(((const uint8_t *)b) + AES_BLOCK_SIZE); for(i = 0; i < ITER / 2; i++) { @@ -1075,6 +1193,11 @@ __asm__ __volatile__( #endif /* !aarch64 */ #endif // NO_OPTIMIZED_MULTIPLY_ON_ARM +STATIC INLINE void copy_block(uint8_t* dst, const uint8_t* src) +{ + memcpy(dst, src, AES_BLOCK_SIZE); +} + STATIC INLINE void sum_half_blocks(uint8_t* a, const uint8_t* b) { uint64_t a0, a1, b0, b1; @@ -1109,7 +1232,9 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int { uint8_t text[INIT_SIZE_BYTE]; uint8_t a[AES_BLOCK_SIZE]; - uint8_t b[AES_BLOCK_SIZE]; + uint8_t b[AES_BLOCK_SIZE * 2]; + uint8_t c[AES_BLOCK_SIZE]; + uint8_t c1[AES_BLOCK_SIZE]; uint8_t d[AES_BLOCK_SIZE]; uint8_t aes_key[AES_KEY_SIZE]; RDATA_ALIGN16 uint8_t expandedKey[256]; @@ -1138,11 +1263,12 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int } memcpy(text, state.init, INIT_SIZE_BYTE); - VARIANT1_INIT64(); - aes_ctx = (oaes_ctx *) oaes_alloc(); oaes_key_import_data(aes_ctx, state.hs.b, AES_KEY_SIZE); + VARIANT1_INIT64(); + VARIANT2_INIT64(); + // use aligned data memcpy(expandedKey, aes_ctx->key->exp_data, aes_ctx->key->exp_data_len); for(i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) @@ -1163,23 +1289,33 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int #define state_index(x) ((*(uint32_t *) x) & MASK) // Iteration 1 - p = &long_state[state_index(a)]; + j = state_index(a); + p = &long_state[j]; aesb_single_round(p, p, a); + copy_block(c1, p); - xor_blocks(b, p); - swap_blocks(b, p); - swap_blocks(a, b); + VARIANT2_PORTABLE_SHUFFLE_ADD(long_state, j); + xor_blocks(p, b); VARIANT1_1(p); // Iteration 2 - p = &long_state[state_index(a)]; - - mul(a, p, d); - sum_half_blocks(b, d); - swap_blocks(b, p); - xor_blocks(b, p); - swap_blocks(a, b); - VARIANT1_2(U64(p) + 1); + j = state_index(c1); + p = &long_state[j]; + copy_block(c, p); + + VARIANT2_PORTABLE_INTEGER_MATH(c, c1); + mul(c1, c, d); + VARIANT2_PORTABLE_SHUFFLE_ADD(long_state, j); + sum_half_blocks(a, d); + swap_blocks(a, c); + xor_blocks(a, c); + VARIANT1_2(U64(c) + 1); + copy_block(p, c); + + if (variant >= 2) { + copy_block(b + AES_BLOCK_SIZE, b); + } + copy_block(b, c1); } memcpy(text, state.init, INIT_SIZE_BYTE); @@ -1298,8 +1434,9 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int union cn_slow_hash_state state; uint8_t text[INIT_SIZE_BYTE]; uint8_t a[AES_BLOCK_SIZE]; - uint8_t b[AES_BLOCK_SIZE]; - uint8_t c[AES_BLOCK_SIZE]; + uint8_t b[AES_BLOCK_SIZE * 2]; + uint8_t c1[AES_BLOCK_SIZE]; + uint8_t c2[AES_BLOCK_SIZE]; uint8_t d[AES_BLOCK_SIZE]; size_t i, j; uint8_t aes_key[AES_KEY_SIZE]; @@ -1315,6 +1452,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int aes_ctx = (oaes_ctx *) oaes_alloc(); VARIANT1_PORTABLE_INIT(); + VARIANT2_PORTABLE_INIT(); oaes_key_import_data(aes_ctx, aes_key, AES_KEY_SIZE); for (i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) { @@ -1324,9 +1462,9 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int memcpy(&long_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE); } - for (i = 0; i < 16; i++) { - a[i] = state.k[ i] ^ state.k[32 + i]; - b[i] = state.k[16 + i] ^ state.k[48 + i]; + for (i = 0; i < AES_BLOCK_SIZE; i++) { + a[i] = state.k[ i] ^ state.k[AES_BLOCK_SIZE * 2 + i]; + b[i] = state.k[AES_BLOCK_SIZE + i] ^ state.k[AES_BLOCK_SIZE * 3 + i]; } for (i = 0; i < ITER / 2; i++) { @@ -1335,26 +1473,32 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int * next address <-+ */ /* Iteration 1 */ - j = e2i(a, MEMORY / AES_BLOCK_SIZE); - copy_block(c, &long_state[j * AES_BLOCK_SIZE]); - aesb_single_round(c, c, a); - xor_blocks(b, c); - swap_blocks(b, c); - copy_block(&long_state[j * AES_BLOCK_SIZE], c); - assert(j == e2i(a, MEMORY / AES_BLOCK_SIZE)); - swap_blocks(a, b); - VARIANT1_1(&long_state[j * AES_BLOCK_SIZE]); + j = e2i(a, MEMORY / AES_BLOCK_SIZE) * AES_BLOCK_SIZE; + copy_block(c1, &long_state[j]); + aesb_single_round(c1, c1, a); + VARIANT2_PORTABLE_SHUFFLE_ADD(long_state, j); + copy_block(&long_state[j], c1); + xor_blocks(&long_state[j], b); + assert(j == e2i(a, MEMORY / AES_BLOCK_SIZE) * AES_BLOCK_SIZE); + VARIANT1_1(&long_state[j]); /* Iteration 2 */ - j = e2i(a, MEMORY / AES_BLOCK_SIZE); - copy_block(c, &long_state[j * AES_BLOCK_SIZE]); - mul(a, c, d); - sum_half_blocks(b, d); - swap_blocks(b, c); - xor_blocks(b, c); - VARIANT1_2(c + 8); - copy_block(&long_state[j * AES_BLOCK_SIZE], c); - assert(j == e2i(a, MEMORY / AES_BLOCK_SIZE)); - swap_blocks(a, b); + j = e2i(c1, MEMORY / AES_BLOCK_SIZE) * AES_BLOCK_SIZE; + copy_block(c2, &long_state[j]); + VARIANT2_PORTABLE_INTEGER_MATH(c2, c1); + mul(c1, c2, d); + VARIANT2_PORTABLE_SHUFFLE_ADD(long_state, j); + swap_blocks(a, c1); + sum_half_blocks(c1, d); + swap_blocks(c1, c2); + xor_blocks(c1, c2); + VARIANT1_2(c2 + 8); + copy_block(&long_state[j], c2); + assert(j == e2i(a, MEMORY / AES_BLOCK_SIZE) * AES_BLOCK_SIZE); + if (variant >= 2) { + copy_block(b + AES_BLOCK_SIZE, b); + } + copy_block(b, a); + copy_block(a, c1); } memcpy(text, state.init, INIT_SIZE_BYTE); diff --git a/src/crypto/variant2_int_sqrt.h b/src/crypto/variant2_int_sqrt.h new file mode 100644 index 000000000..b405bb798 --- /dev/null +++ b/src/crypto/variant2_int_sqrt.h @@ -0,0 +1,163 @@ +#ifndef VARIANT2_INT_SQRT_H +#define VARIANT2_INT_SQRT_H + +#include <math.h> +#include <float.h> + +#define VARIANT2_INTEGER_MATH_SQRT_STEP_SSE2() \ + do { \ + const __m128i exp_double_bias = _mm_set_epi64x(0, 1023ULL << 52); \ + __m128d x = _mm_castsi128_pd(_mm_add_epi64(_mm_cvtsi64_si128(sqrt_input >> 12), exp_double_bias)); \ + x = _mm_sqrt_sd(_mm_setzero_pd(), x); \ + sqrt_result = (uint64_t)(_mm_cvtsi128_si64(_mm_sub_epi64(_mm_castpd_si128(x), exp_double_bias))) >> 19; \ + } while(0) + +#define VARIANT2_INTEGER_MATH_SQRT_STEP_FP64() \ + do { \ + sqrt_result = sqrt(sqrt_input + 18446744073709551616.0) * 2.0 - 8589934592.0; \ + } while(0) + +#define VARIANT2_INTEGER_MATH_SQRT_STEP_REF() \ + sqrt_result = integer_square_root_v2(sqrt_input) + +// Reference implementation of the integer square root for Cryptonight variant 2 +// Computes integer part of "sqrt(2^64 + n) * 2 - 2^33" +// +// In other words, given 64-bit unsigned integer n: +// 1) Write it as x = 1.NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN000... in binary (1 <= x < 2, all 64 bits of n are used) +// 2) Calculate sqrt(x) = 1.0RRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRR... (1 <= sqrt(x) < sqrt(2), so it will always start with "1.0" in binary) +// 3) Take 32 bits that come after "1.0" and return them as a 32-bit unsigned integer, discard all remaining bits +// +// Some sample inputs and outputs: +// +// Input | Output | Exact value of "sqrt(2^64 + n) * 2 - 2^33" +// -----------------|------------|------------------------------------------- +// 0 | 0 | 0 +// 2^32 | 0 | 0.99999999994179233909330885695244... +// 2^32 + 1 | 1 | 1.0000000001746229827200734316305... +// 2^50 | 262140 | 262140.00012206565608606978175873... +// 2^55 + 20963331 | 8384515 | 8384515.9999999997673963974959744... +// 2^55 + 20963332 | 8384516 | 8384516 +// 2^62 + 26599786 | 1013904242 | 1013904242.9999999999479374853545... +// 2^62 + 26599787 | 1013904243 | 1013904243.0000000001561875439364... +// 2^64 - 1 | 3558067407 | 3558067407.9041987696409179931096... + +// The reference implementation as it is now uses only unsigned int64 arithmetic, so it can't have undefined behavior +// It was tested once for all edge cases and confirmed correct +static inline uint32_t integer_square_root_v2(uint64_t n) +{ + uint64_t r = 1ULL << 63; + + for (uint64_t bit = 1ULL << 60; bit; bit >>= 2) + { + const bool b = (n < r + bit); + const uint64_t n_next = n - (r + bit); + const uint64_t r_next = r + bit * 2; + n = b ? n : n_next; + r = b ? r : r_next; + r >>= 1; + } + + return r * 2 + ((n > r) ? 1 : 0); +} + +/* +VARIANT2_INTEGER_MATH_SQRT_FIXUP checks that "r" is an integer part of "sqrt(2^64 + sqrt_input) * 2 - 2^33" and adds or subtracts 1 if needed +It's hard to understand how it works, so here is a full calculation of formulas used in VARIANT2_INTEGER_MATH_SQRT_FIXUP + +The following inequalities must hold for r if it's an integer part of "sqrt(2^64 + sqrt_input) * 2 - 2^33": +1) r <= sqrt(2^64 + sqrt_input) * 2 - 2^33 +2) r + 1 > sqrt(2^64 + sqrt_input) * 2 - 2^33 + +We need to check them using only unsigned integer arithmetic to avoid rounding errors and undefined behavior + +First inequality: r <= sqrt(2^64 + sqrt_input) * 2 - 2^33 +----------------------------------------------------------------------------------- +r <= sqrt(2^64 + sqrt_input) * 2 - 2^33 +r + 2^33 <= sqrt(2^64 + sqrt_input) * 2 +r/2 + 2^32 <= sqrt(2^64 + sqrt_input) +(r/2 + 2^32)^2 <= 2^64 + sqrt_input + +Rewrite r as r = s * 2 + b (s = trunc(r/2), b is 0 or 1) + +((s*2+b)/2 + 2^32)^2 <= 2^64 + sqrt_input +(s*2+b)^2/4 + 2*2^32*(s*2+b)/2 + 2^64 <= 2^64 + sqrt_input +(s*2+b)^2/4 + 2*2^32*(s*2+b)/2 <= sqrt_input +(s*2+b)^2/4 + 2^32*r <= sqrt_input +(s^2*4+2*s*2*b+b^2)/4 + 2^32*r <= sqrt_input +s^2+s*b+b^2/4 + 2^32*r <= sqrt_input +s*(s+b) + b^2/4 + 2^32*r <= sqrt_input + +Let r2 = s*(s+b) + r*2^32 +r2 + b^2/4 <= sqrt_input + +If this inequality doesn't hold, then we must decrement r: IF "r2 + b^2/4 > sqrt_input" THEN r = r - 1 + +b can be 0 or 1 +If b is 0 then we need to compare "r2 > sqrt_input" +If b is 1 then b^2/4 = 0.25, so we need to compare "r2 + 0.25 > sqrt_input" +Since both r2 and sqrt_input are integers, we can safely replace it with "r2 + 1 > sqrt_input" +----------------------------------------------------------------------------------- +Both cases can be merged to a single expression "r2 + b > sqrt_input" +----------------------------------------------------------------------------------- +There will be no overflow when calculating "r2 + b", so it's safe to compare with sqrt_input: +r2 + b = s*(s+b) + r*2^32 + b +The largest value s, b and r can have is s = 1779033703, b = 1, r = 3558067407 when sqrt_input = 2^64 - 1 +r2 + b <= 1779033703*1779033704 + 3558067407*2^32 + 1 = 18446744068217447385 < 2^64 + +Second inequality: r + 1 > sqrt(2^64 + sqrt_input) * 2 - 2^33 +----------------------------------------------------------------------------------- +r + 1 > sqrt(2^64 + sqrt_input) * 2 - 2^33 +r + 1 + 2^33 > sqrt(2^64 + sqrt_input) * 2 +((r+1)/2 + 2^32)^2 > 2^64 + sqrt_input + +Rewrite r as r = s * 2 + b (s = trunc(r/2), b is 0 or 1) + +((s*2+b+1)/2 + 2^32)^2 > 2^64 + sqrt_input +(s*2+b+1)^2/4 + 2*(s*2+b+1)/2*2^32 + 2^64 > 2^64 + sqrt_input +(s*2+b+1)^2/4 + (s*2+b+1)*2^32 > sqrt_input +(s*2+b+1)^2/4 + (r+1)*2^32 > sqrt_input +(s*2+(b+1))^2/4 + r*2^32 + 2^32 > sqrt_input +(s^2*4+2*s*2*(b+1)+(b+1)^2)/4 + r*2^32 + 2^32 > sqrt_input +s^2+s*(b+1)+(b+1)^2/4 + r*2^32 + 2^32 > sqrt_input +s*(s+b) + s + (b+1)^2/4 + r*2^32 + 2^32 > sqrt_input + +Let r2 = s*(s+b) + r*2^32 + +r2 + s + (b+1)^2/4 + 2^32 > sqrt_input +r2 + 2^32 + (b+1)^2/4 > sqrt_input - s + +If this inequality doesn't hold, then we must decrement r: IF "r2 + 2^32 + (b+1)^2/4 <= sqrt_input - s" THEN r = r - 1 +b can be 0 or 1 +If b is 0 then we need to compare "r2 + 2^32 + 1/4 <= sqrt_input - s" which is equal to "r2 + 2^32 < sqrt_input - s" because all numbers here are integers +If b is 1 then (b+1)^2/4 = 1, so we need to compare "r2 + 2^32 + 1 <= sqrt_input - s" which is also equal to "r2 + 2^32 < sqrt_input - s" +----------------------------------------------------------------------------------- +Both cases can be merged to a single expression "r2 + 2^32 < sqrt_input - s" +----------------------------------------------------------------------------------- +There will be no overflow when calculating "r2 + 2^32": +r2 + 2^32 = s*(s+b) + r*2^32 + 2^32 = s*(s+b) + (r+1)*2^32 +The largest value s, b and r can have is s = 1779033703, b = 1, r = 3558067407 when sqrt_input = 2^64 - 1 +r2 + b <= 1779033703*1779033704 + 3558067408*2^32 = 18446744072512414680 < 2^64 + +There will be no integer overflow when calculating "sqrt_input - s", i.e. "sqrt_input >= s" at all times: +s = trunc(r/2) = trunc(sqrt(2^64 + sqrt_input) - 2^32) < sqrt(2^64 + sqrt_input) - 2^32 + 1 +sqrt_input > sqrt(2^64 + sqrt_input) - 2^32 + 1 +sqrt_input + 2^32 - 1 > sqrt(2^64 + sqrt_input) +(sqrt_input + 2^32 - 1)^2 > sqrt_input + 2^64 +sqrt_input^2 + 2*sqrt_input*(2^32 - 1) + (2^32-1)^2 > sqrt_input + 2^64 +sqrt_input^2 + sqrt_input*(2^33 - 2) + (2^32-1)^2 > sqrt_input + 2^64 +sqrt_input^2 + sqrt_input*(2^33 - 3) + (2^32-1)^2 > 2^64 +sqrt_input^2 + sqrt_input*(2^33 - 3) + 2^64-2^33+1 > 2^64 +sqrt_input^2 + sqrt_input*(2^33 - 3) - 2^33 + 1 > 0 +This inequality is true if sqrt_input > 1 and it's easy to check that s = 0 if sqrt_input is 0 or 1, so there will be no integer overflow +*/ + +#define VARIANT2_INTEGER_MATH_SQRT_FIXUP(r) \ + do { \ + const uint64_t s = r >> 1; \ + const uint64_t b = r & 1; \ + const uint64_t r2 = (uint64_t)(s) * (s + b) + (r << 32); \ + r += ((r2 + b > sqrt_input) ? -1 : 0) + ((r2 + (1ULL << 32) < sqrt_input - s) ? 1 : 0); \ + } while(0) + +#endif diff --git a/src/cryptonote_basic/account.cpp b/src/cryptonote_basic/account.cpp index aac6ec22b..4cbfa8142 100644 --- a/src/cryptonote_basic/account.cpp +++ b/src/cryptonote_basic/account.cpp @@ -44,6 +44,9 @@ extern "C" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "account" +#define KEYS_ENCRYPTION_SALT 'k' + + using namespace std; DISABLE_VS_WARNINGS(4244 4345) @@ -60,7 +63,70 @@ DISABLE_VS_WARNINGS(4244 4345) m_device = &hwdev; MCDEBUG("device", "account_keys::set_device device type: "<<typeid(hwdev).name()); } + //----------------------------------------------------------------- + static void derive_key(const crypto::chacha_key &base_key, crypto::chacha_key &key) + { + static_assert(sizeof(base_key) == sizeof(crypto::hash), "chacha key and hash should be the same size"); + epee::mlocked<tools::scrubbed_arr<char, sizeof(base_key)+1>> data; + memcpy(data.data(), &base_key, sizeof(base_key)); + data[sizeof(base_key)] = KEYS_ENCRYPTION_SALT; + crypto::generate_chacha_key(data.data(), sizeof(data), key, 1); + } + //----------------------------------------------------------------- + static epee::wipeable_string get_key_stream(const crypto::chacha_key &base_key, const crypto::chacha_iv &iv, size_t bytes) + { + // derive a new key + crypto::chacha_key key; + derive_key(base_key, key); + // chacha + epee::wipeable_string buffer0(std::string(bytes, '\0')); + epee::wipeable_string buffer1 = buffer0; + crypto::chacha20(buffer0.data(), buffer0.size(), key, iv, buffer1.data()); + return buffer1; + } + //----------------------------------------------------------------- + void account_keys::xor_with_key_stream(const crypto::chacha_key &key) + { + // encrypt a large enough byte stream with chacha20 + epee::wipeable_string key_stream = get_key_stream(key, m_encryption_iv, sizeof(crypto::secret_key) * (2 + m_multisig_keys.size())); + const char *ptr = key_stream.data(); + for (size_t i = 0; i < sizeof(crypto::secret_key); ++i) + m_spend_secret_key.data[i] ^= *ptr++; + for (size_t i = 0; i < sizeof(crypto::secret_key); ++i) + m_view_secret_key.data[i] ^= *ptr++; + for (crypto::secret_key &k: m_multisig_keys) + { + for (size_t i = 0; i < sizeof(crypto::secret_key); ++i) + k.data[i] ^= *ptr++; + } + } + //----------------------------------------------------------------- + void account_keys::encrypt(const crypto::chacha_key &key) + { + m_encryption_iv = crypto::rand<crypto::chacha_iv>(); + xor_with_key_stream(key); + } + //----------------------------------------------------------------- + void account_keys::decrypt(const crypto::chacha_key &key) + { + xor_with_key_stream(key); + } + //----------------------------------------------------------------- + void account_keys::encrypt_viewkey(const crypto::chacha_key &key) + { + // encrypt a large enough byte stream with chacha20 + epee::wipeable_string key_stream = get_key_stream(key, m_encryption_iv, sizeof(crypto::secret_key) * 2); + const char *ptr = key_stream.data(); + ptr += sizeof(crypto::secret_key); + for (size_t i = 0; i < sizeof(crypto::secret_key); ++i) + m_view_secret_key.data[i] ^= *ptr++; + } + //----------------------------------------------------------------- + void account_keys::decrypt_viewkey(const crypto::chacha_key &key) + { + encrypt_viewkey(key); + } //----------------------------------------------------------------- account_base::account_base() { @@ -157,7 +223,7 @@ DISABLE_VS_WARNINGS(4244 4345) void account_base::create_from_viewkey(const cryptonote::account_public_address& address, const crypto::secret_key& viewkey) { crypto::secret_key fake; - memset(&unwrap(fake), 0, sizeof(fake)); + memset(&unwrap(unwrap(fake)), 0, sizeof(fake)); create_from_keys(address, fake, viewkey); } //----------------------------------------------------------------- diff --git a/src/cryptonote_basic/account.h b/src/cryptonote_basic/account.h index b5d119c46..dac66ff1a 100644 --- a/src/cryptonote_basic/account.h +++ b/src/cryptonote_basic/account.h @@ -44,18 +44,29 @@ namespace cryptonote crypto::secret_key m_view_secret_key; std::vector<crypto::secret_key> m_multisig_keys; hw::device *m_device = &hw::get_device("default"); + crypto::chacha_iv m_encryption_iv; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(m_account_address) KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE(m_spend_secret_key) KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE(m_view_secret_key) KV_SERIALIZE_CONTAINER_POD_AS_BLOB(m_multisig_keys) + const crypto::chacha_iv default_iv{{0, 0, 0, 0, 0, 0, 0, 0}}; + KV_SERIALIZE_VAL_POD_AS_BLOB_OPT(m_encryption_iv, default_iv) END_KV_SERIALIZE_MAP() account_keys& operator=(account_keys const&) = default; + void encrypt(const crypto::chacha_key &key); + void decrypt(const crypto::chacha_key &key); + void encrypt_viewkey(const crypto::chacha_key &key); + void decrypt_viewkey(const crypto::chacha_key &key); + hw::device& get_device() const ; void set_device( hw::device &hwdev) ; + + private: + void xor_with_key_stream(const crypto::chacha_key &key); }; /************************************************************************/ @@ -87,6 +98,11 @@ namespace cryptonote void forget_spend_key(); const std::vector<crypto::secret_key> &get_multisig_keys() const { return m_keys.m_multisig_keys; } + void encrypt_keys(const crypto::chacha_key &key) { m_keys.encrypt(key); } + void decrypt_keys(const crypto::chacha_key &key) { m_keys.decrypt(key); } + void encrypt_viewkey(const crypto::chacha_key &key) { m_keys.encrypt_viewkey(key); } + void decrypt_viewkey(const crypto::chacha_key &key) { m_keys.decrypt_viewkey(key); } + template <class t_archive> inline void serialize(t_archive &a, const unsigned int /*ver*/) { diff --git a/src/cryptonote_basic/cryptonote_basic_impl.cpp b/src/cryptonote_basic/cryptonote_basic_impl.cpp index cff23695f..b18ef1c5c 100644 --- a/src/cryptonote_basic/cryptonote_basic_impl.cpp +++ b/src/cryptonote_basic/cryptonote_basic_impl.cpp @@ -67,7 +67,7 @@ namespace cryptonote { /* Cryptonote helper functions */ /************************************************************************/ //----------------------------------------------------------------------------------------------- - size_t get_min_block_size(uint8_t version) + size_t get_min_block_weight(uint8_t version) { if (version < 2) return CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V1; @@ -86,7 +86,7 @@ namespace cryptonote { return CRYPTONOTE_MAX_TX_SIZE; } //----------------------------------------------------------------------------------------------- - bool get_block_reward(size_t median_size, size_t current_block_size, uint64_t already_generated_coins, uint64_t &reward, uint8_t version) { + bool get_block_reward(size_t median_weight, size_t current_block_weight, uint64_t already_generated_coins, uint64_t &reward, uint8_t version) { static_assert(DIFFICULTY_TARGET_V2%60==0&&DIFFICULTY_TARGET_V1%60==0,"difficulty targets must be a multiple of 60"); const int target = version < 2 ? DIFFICULTY_TARGET_V1 : DIFFICULTY_TARGET_V2; const int target_minutes = target / 60; @@ -98,37 +98,37 @@ namespace cryptonote { base_reward = FINAL_SUBSIDY_PER_MINUTE*target_minutes; } - uint64_t full_reward_zone = get_min_block_size(version); + uint64_t full_reward_zone = get_min_block_weight(version); //make it soft - if (median_size < full_reward_zone) { - median_size = full_reward_zone; + if (median_weight < full_reward_zone) { + median_weight = full_reward_zone; } - if (current_block_size <= median_size) { + if (current_block_weight <= median_weight) { reward = base_reward; return true; } - if(current_block_size > 2 * median_size) { - MERROR("Block cumulative size is too big: " << current_block_size << ", expected less than " << 2 * median_size); + if(current_block_weight > 2 * median_weight) { + MERROR("Block cumulative weight is too big: " << current_block_weight << ", expected less than " << 2 * median_weight); return false; } - assert(median_size < std::numeric_limits<uint32_t>::max()); - assert(current_block_size < std::numeric_limits<uint32_t>::max()); + assert(median_weight < std::numeric_limits<uint32_t>::max()); + assert(current_block_weight < std::numeric_limits<uint32_t>::max()); uint64_t product_hi; // BUGFIX: 32-bit saturation bug (e.g. ARM7), the result was being // treated as 32-bit by default. - uint64_t multiplicand = 2 * median_size - current_block_size; - multiplicand *= current_block_size; + uint64_t multiplicand = 2 * median_weight - current_block_weight; + multiplicand *= current_block_weight; uint64_t product_lo = mul128(base_reward, multiplicand, &product_hi); uint64_t reward_hi; uint64_t reward_lo; - div128_32(product_hi, product_lo, static_cast<uint32_t>(median_size), &reward_hi, &reward_lo); - div128_32(reward_hi, reward_lo, static_cast<uint32_t>(median_size), &reward_hi, &reward_lo); + div128_32(product_hi, product_lo, static_cast<uint32_t>(median_weight), &reward_hi, &reward_lo); + div128_32(reward_hi, reward_lo, static_cast<uint32_t>(median_weight), &reward_hi, &reward_lo); assert(0 == reward_hi); assert(reward_lo < base_reward); diff --git a/src/cryptonote_basic/cryptonote_basic_impl.h b/src/cryptonote_basic/cryptonote_basic_impl.h index f59785021..c804a88fa 100644 --- a/src/cryptonote_basic/cryptonote_basic_impl.h +++ b/src/cryptonote_basic/cryptonote_basic_impl.h @@ -86,10 +86,10 @@ namespace cryptonote { /************************************************************************/ /* Cryptonote helper functions */ /************************************************************************/ - size_t get_min_block_size(uint8_t version); + size_t get_min_block_weight(uint8_t version); size_t get_max_block_size(); size_t get_max_tx_size(); - bool get_block_reward(size_t median_size, size_t current_block_size, uint64_t already_generated_coins, uint64_t &reward, uint8_t version); + bool get_block_reward(size_t median_weight, size_t current_block_weight, uint64_t already_generated_coins, uint64_t &reward, uint8_t version); uint8_t get_account_address_checksum(const public_address_outer_blob& bl); uint8_t get_account_integrated_address_checksum(const public_integrated_address_outer_blob& bl); diff --git a/src/cryptonote_basic/cryptonote_boost_serialization.h b/src/cryptonote_basic/cryptonote_boost_serialization.h index 143133163..0725a2bb8 100644 --- a/src/cryptonote_basic/cryptonote_boost_serialization.h +++ b/src/cryptonote_basic/cryptonote_boost_serialization.h @@ -295,7 +295,7 @@ namespace boost a & x.type; if (x.type == rct::RCTTypeNull) return; - if (x.type != rct::RCTTypeFull && x.type != rct::RCTTypeFullBulletproof && x.type != rct::RCTTypeSimple && x.type != rct::RCTTypeSimpleBulletproof) + if (x.type != rct::RCTTypeFull && x.type != rct::RCTTypeSimple && x.type != rct::RCTTypeBulletproof) throw boost::archive::archive_exception(boost::archive::archive_exception::other_exception, "Unsupported rct type"); // a & x.message; message is not serialized, as it can be reconstructed from the tx data // a & x.mixRing; mixRing is not serialized, as it can be reconstructed from the offsets @@ -323,7 +323,7 @@ namespace boost a & x.type; if (x.type == rct::RCTTypeNull) return; - if (x.type != rct::RCTTypeFull && x.type != rct::RCTTypeFullBulletproof && x.type != rct::RCTTypeSimple && x.type != rct::RCTTypeSimpleBulletproof) + if (x.type != rct::RCTTypeFull && x.type != rct::RCTTypeSimple && x.type != rct::RCTTypeBulletproof) throw boost::archive::archive_exception(boost::archive::archive_exception::other_exception, "Unsupported rct type"); // a & x.message; message is not serialized, as it can be reconstructed from the tx data // a & x.mixRing; mixRing is not serialized, as it can be reconstructed from the offsets @@ -337,7 +337,7 @@ namespace boost if (x.p.rangeSigs.empty()) a & x.p.bulletproofs; a & x.p.MGs; - if (x.type == rct::RCTTypeSimpleBulletproof) + if (x.type == rct::RCTTypeBulletproof) a & x.p.pseudoOuts; } } diff --git a/src/cryptonote_basic/cryptonote_format_utils.cpp b/src/cryptonote_basic/cryptonote_format_utils.cpp index 428be1c9c..7ea4718d2 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.cpp +++ b/src/cryptonote_basic/cryptonote_format_utils.cpp @@ -135,23 +135,41 @@ namespace cryptonote return false; } for (size_t n = 0; n < tx.rct_signatures.outPk.size(); ++n) + { + if (tx.vout[n].target.type() != typeid(txout_to_key)) + { + LOG_PRINT_L1("Unsupported output type in tx " << get_transaction_hash(tx)); + return false; + } rv.outPk[n].dest = rct::pk2rct(boost::get<txout_to_key>(tx.vout[n].target).key); + } if (!base_only) { - const bool bulletproof = rv.type == rct::RCTTypeFullBulletproof || rv.type == rct::RCTTypeSimpleBulletproof; + const bool bulletproof = rct::is_rct_bulletproof(rv.type); if (bulletproof) { - if (rv.p.bulletproofs.size() != tx.vout.size()) + if (rv.p.bulletproofs.size() != 1) { LOG_PRINT_L1("Failed to parse transaction from blob, bad bulletproofs size in tx " << get_transaction_hash(tx)); return false; } - for (size_t n = 0; n < rv.outPk.size(); ++n) + if (rv.p.bulletproofs[0].L.size() < 6) { - rv.p.bulletproofs[n].V.resize(1); - rv.p.bulletproofs[n].V[0] = rv.outPk[n].mask; + LOG_PRINT_L1("Failed to parse transaction from blob, bad bulletproofs L size in tx " << get_transaction_hash(tx)); + return false; + } + const size_t max_outputs = 1 << (rv.p.bulletproofs[0].L.size() - 6); + if (max_outputs < tx.vout.size()) + { + LOG_PRINT_L1("Failed to parse transaction from blob, bad bulletproofs max outputs in tx " << get_transaction_hash(tx)); + return false; } + const size_t n_amounts = tx.vout.size(); + CHECK_AND_ASSERT_MES(n_amounts == rv.outPk.size(), false, "Internal error filling out V"); + rv.p.bulletproofs[0].V.resize(n_amounts); + for (size_t i = 0; i < n_amounts; ++i) + rv.p.bulletproofs[0].V[i] = rct::scalarmultKey(rv.outPk[i].mask, rct::INV_EIGHT); } } } @@ -318,6 +336,37 @@ namespace cryptonote return string_tools::get_xtype_from_string(amount, str_amount); } //--------------------------------------------------------------- + uint64_t get_transaction_weight(const transaction &tx, size_t blob_size) + { + if (tx.version < 2) + return blob_size; + const rct::rctSig &rv = tx.rct_signatures; + if (!rct::is_rct_bulletproof(rv.type)) + return blob_size; + const size_t n_outputs = tx.vout.size(); + if (n_outputs <= 2) + return blob_size; + const uint64_t bp_base = 368; + const size_t n_padded_outputs = rct::n_bulletproof_max_amounts(rv.p.bulletproofs); + size_t nlr = 0; + for (const auto &bp: rv.p.bulletproofs) + nlr += bp.L.size() * 2; + const size_t bp_size = 32 * (9 + nlr); + CHECK_AND_ASSERT_THROW_MES_L1(bp_base * n_padded_outputs >= bp_size, "Invalid bulletproof clawback"); + const uint64_t bp_clawback = (bp_base * n_padded_outputs - bp_size) * 4 / 5; + CHECK_AND_ASSERT_THROW_MES_L1(bp_clawback <= std::numeric_limits<uint64_t>::max() - blob_size, "Weight overflow"); + return blob_size + bp_clawback; + } + //--------------------------------------------------------------- + uint64_t get_transaction_weight(const transaction &tx) + { + std::ostringstream s; + binary_archive<true> a(s); + ::serialization::serialize(a, const_cast<transaction&>(tx)); + const cryptonote::blobdata blob = s.str(); + return get_transaction_weight(tx, blob.size()); + } + //--------------------------------------------------------------- bool get_tx_fee(const transaction& tx, uint64_t & fee) { if (tx.version > 1) diff --git a/src/cryptonote_basic/cryptonote_format_utils.h b/src/cryptonote_basic/cryptonote_format_utils.h index 8a5296d5b..bf71eb591 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.h +++ b/src/cryptonote_basic/cryptonote_format_utils.h @@ -117,6 +117,8 @@ namespace cryptonote bool check_inputs_types_supported(const transaction& tx); bool check_outs_valid(const transaction& tx); bool parse_amount(uint64_t& amount, const std::string& str_amount); + uint64_t get_transaction_weight(const transaction &tx); + uint64_t get_transaction_weight(const transaction &tx, size_t blob_size); bool check_money_overflow(const transaction& tx); bool check_outs_overflow(const transaction& tx); diff --git a/src/cryptonote_config.h b/src/cryptonote_config.h index a0dcf2df1..a6858ce7c 100644 --- a/src/cryptonote_config.h +++ b/src/cryptonote_config.h @@ -65,9 +65,11 @@ #define FEE_PER_KB_OLD ((uint64_t)10000000000) // pow(10, 10) #define FEE_PER_KB ((uint64_t)2000000000) // 2 * pow(10, 9) +#define FEE_PER_BYTE ((uint64_t)300000) #define DYNAMIC_FEE_PER_KB_BASE_FEE ((uint64_t)2000000000) // 2 * pow(10,9) #define DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD ((uint64_t)10000000000000) // 10 * pow(10,12) #define DYNAMIC_FEE_PER_KB_BASE_FEE_V5 ((uint64_t)2000000000 * (uint64_t)CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V2 / CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5) +#define DYNAMIC_FEE_REFERENCE_TRANSACTION_WEIGHT ((uint64_t)3000) #define ORPHANED_BLOCKS_MAX_COUNT 100 @@ -133,13 +135,17 @@ #define HF_VERSION_DYNAMIC_FEE 4 #define HF_VERSION_MIN_MIXIN_4 6 #define HF_VERSION_MIN_MIXIN_6 7 +#define HF_VERSION_MIN_MIXIN_10 8 #define HF_VERSION_ENFORCE_RCT 6 +#define HF_VERSION_PER_BYTE_FEE 8 #define PER_KB_FEE_QUANTIZATION_DECIMALS 8 #define HASH_OF_HASHES_STEP 256 -#define DEFAULT_TXPOOL_MAX_SIZE 648000000ull // 3 days at 300000, in bytes +#define DEFAULT_TXPOOL_MAX_WEIGHT 648000000ull // 3 days at 300000, in bytes + +#define BULLETPROOF_MAX_OUTPUTS 16 // New constants are intended to go here namespace config diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 87ef47c11..b20fe9869 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -132,6 +132,7 @@ static const struct { { 6, 971400, 0, 1501709789 }, { 7, 1057027, 0, 1512211236 }, { 8, 1057058, 0, 1515967497 }, + { 9, 1057778, 0, 1515967498 }, }; static const uint64_t testnet_hard_fork_version_1_till = 624633; @@ -155,10 +156,11 @@ static const struct { //------------------------------------------------------------------ Blockchain::Blockchain(tx_memory_pool& tx_pool) : - m_db(), m_tx_pool(tx_pool), m_hardfork(NULL), m_timestamps_and_difficulties_height(0), m_current_block_cumul_sz_limit(0), m_current_block_cumul_sz_median(0), - m_enforce_dns_checkpoints(false), m_max_prepare_blocks_threads(4), m_db_blocks_per_sync(1), m_db_sync_mode(db_async), m_db_default_sync(false), m_fast_sync(true), m_show_time_stats(false), m_sync_counter(0), m_cancel(false), + m_db(), m_tx_pool(tx_pool), m_hardfork(NULL), m_timestamps_and_difficulties_height(0), m_current_block_cumul_weight_limit(0), m_current_block_cumul_weight_median(0), + m_enforce_dns_checkpoints(false), m_max_prepare_blocks_threads(4), m_db_sync_on_blocks(true), m_db_sync_threshold(1), m_db_sync_mode(db_async), m_db_default_sync(false), m_fast_sync(true), m_show_time_stats(false), m_sync_counter(0), m_bytes_to_sync(0), m_cancel(false), m_difficulty_for_next_block_top_hash(crypto::null_hash), - m_difficulty_for_next_block(1) + m_difficulty_for_next_block(1), + m_btc_valid(false) { LOG_PRINT_L3("Blockchain::" << __func__); } @@ -481,7 +483,7 @@ bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline m_tx_pool.on_blockchain_dec(m_db->height()-1, get_tail_id()); } - update_next_cumulative_size_limit(); + update_next_cumulative_weight_limit(); return true; } //------------------------------------------------------------------ @@ -630,8 +632,9 @@ block Blockchain::pop_block_from_blockchain() m_blocks_txs_check.clear(); m_check_txin_table.clear(); - update_next_cumulative_size_limit(); + update_next_cumulative_weight_limit(); m_tx_pool.on_blockchain_dec(m_db->height()-1, get_tail_id()); + invalidate_block_template_cache(); return popped_block; } @@ -642,12 +645,13 @@ bool Blockchain::reset_and_set_genesis_block(const block& b) CRITICAL_REGION_LOCAL(m_blockchain_lock); m_timestamps_and_difficulties_height = 0; m_alternative_chains.clear(); + invalidate_block_template_cache(); m_db->reset(); m_hardfork->init(); block_verification_context bvc = boost::value_initialized<block_verification_context>(); add_new_block(b, bvc); - update_next_cumulative_size_limit(); + update_next_cumulative_weight_limit(); return bvc.m_added_to_main_chain && !bvc.m_verifivation_failed; } //------------------------------------------------------------------ @@ -1110,7 +1114,7 @@ bool Blockchain::prevalidate_miner_transaction(const block& b, uint64_t height) } //------------------------------------------------------------------ // This function validates the miner transaction reward -bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins, bool &partial_block_reward, uint8_t version) +bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_block_weight, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins, bool &partial_block_reward, uint8_t version) { LOG_PRINT_L3("Blockchain::" << __func__); //validate reward @@ -1128,11 +1132,11 @@ bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_bl } } - std::vector<size_t> last_blocks_sizes; - get_last_n_blocks_sizes(last_blocks_sizes, CRYPTONOTE_REWARD_BLOCKS_WINDOW); - if (!get_block_reward(epee::misc_utils::median(last_blocks_sizes), cumulative_block_size, already_generated_coins, base_reward, version)) + std::vector<size_t> last_blocks_weights; + get_last_n_blocks_weights(last_blocks_weights, CRYPTONOTE_REWARD_BLOCKS_WINDOW); + if (!get_block_reward(epee::misc_utils::median(last_blocks_weights), cumulative_block_weight, already_generated_coins, base_reward, version)) { - MERROR_VER("block size " << cumulative_block_size << " is bigger than allowed for this blockchain"); + MERROR_VER("block weight " << cumulative_block_weight << " is bigger than allowed for this blockchain"); return false; } if(base_reward + fee < money_in_use) @@ -1162,8 +1166,8 @@ bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_bl return true; } //------------------------------------------------------------------ -// get the block sizes of the last <count> blocks, and return by reference <sz>. -void Blockchain::get_last_n_blocks_sizes(std::vector<size_t>& sz, size_t count) const +// get the block weights of the last <count> blocks, and return by reference <sz>. +void Blockchain::get_last_n_blocks_weights(std::vector<size_t>& weights, size_t count) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -1174,26 +1178,26 @@ void Blockchain::get_last_n_blocks_sizes(std::vector<size_t>& sz, size_t count) return; m_db->block_txn_start(true); - // add size of last <count> blocks to vector <sz> (or less, if blockchain size < count) + // add weight of last <count> blocks to vector <weights> (or less, if blockchain size < count) size_t start_offset = h - std::min<size_t>(h, count); - sz.reserve(sz.size() + h - start_offset); + weights.reserve(weights.size() + h - start_offset); for(size_t i = start_offset; i < h; i++) { - sz.push_back(m_db->get_block_size(i)); + weights.push_back(m_db->get_block_weight(i)); } m_db->block_txn_stop(); } //------------------------------------------------------------------ -uint64_t Blockchain::get_current_cumulative_blocksize_limit() const +uint64_t Blockchain::get_current_cumulative_block_weight_limit() const { LOG_PRINT_L3("Blockchain::" << __func__); - return m_current_block_cumul_sz_limit; + return m_current_block_cumul_weight_limit; } //------------------------------------------------------------------ -uint64_t Blockchain::get_current_cumulative_blocksize_median() const +uint64_t Blockchain::get_current_cumulative_block_weight_median() const { LOG_PRINT_L3("Blockchain::" << __func__); - return m_current_block_cumul_sz_median; + return m_current_block_cumul_weight_median; } //------------------------------------------------------------------ //TODO: This function only needed minor modification to work with BlockchainDB, @@ -1210,11 +1214,28 @@ uint64_t Blockchain::get_current_cumulative_blocksize_median() const bool Blockchain::create_block_template(block& b, const account_public_address& miner_address, difficulty_type& diffic, uint64_t& height, uint64_t& expected_reward, const blobdata& ex_nonce) { LOG_PRINT_L3("Blockchain::" << __func__); - size_t median_size; + size_t median_weight; uint64_t already_generated_coins; + uint64_t pool_cookie; CRITICAL_REGION_BEGIN(m_blockchain_lock); height = m_db->height(); + if (m_btc_valid) { + // The pool cookie is atomic. The lack of locking is OK, as if it changes + // just as we compare it, we'll just use a slightly old template, but + // this would be the case anyway if we'd lock, and the change happened + // just after the block template was created + if (!memcmp(&miner_address, &m_btc_address, sizeof(cryptonote::account_public_address)) && m_btc_nonce == ex_nonce && m_btc_pool_cookie == m_tx_pool.cookie()) { + MDEBUG("Using cached template"); + m_btc.timestamp = time(NULL); // update timestamp unconditionally + b = m_btc; + diffic = m_btc_difficulty; + expected_reward = m_btc_expected_reward; + return true; + } + MDEBUG("Not using cached template: address " << (!memcmp(&miner_address, &m_btc_address, sizeof(cryptonote::account_public_address))) << ", nonce " << (m_btc_nonce == ex_nonce) << ", cookie " << (m_btc_pool_cookie == m_tx_pool.cookie())); + invalidate_block_template_cache(); + } b.major_version = m_hardfork->get_current_version(); b.minor_version = m_hardfork->get_ideal_version(); @@ -1230,19 +1251,20 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m diffic = get_difficulty_for_next_block(); CHECK_AND_ASSERT_MES(diffic, false, "difficulty overhead."); - median_size = m_current_block_cumul_sz_limit / 2; + median_weight = m_current_block_cumul_weight_limit / 2; already_generated_coins = m_db->get_block_already_generated_coins(height - 1); CRITICAL_REGION_END(); - size_t txs_size; + size_t txs_weight; uint64_t fee; - if (!m_tx_pool.fill_block_template(b, median_size, already_generated_coins, txs_size, fee, expected_reward, m_hardfork->get_current_version())) + if (!m_tx_pool.fill_block_template(b, median_weight, already_generated_coins, txs_weight, fee, expected_reward, m_hardfork->get_current_version())) { return false; } + pool_cookie = m_tx_pool.cookie(); #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) - size_t real_txs_size = 0; + size_t real_txs_weight = 0; uint64_t real_fee = 0; CRITICAL_REGION_BEGIN(m_tx_pool.m_transactions_lock); for(crypto::hash &cur_hash: b.tx_hashes) @@ -1254,11 +1276,11 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m continue; } tx_memory_pool::tx_details &cur_tx = cur_res->second; - real_txs_size += cur_tx.blob_size; + real_txs_weight += cur_tx.weight; real_fee += cur_tx.fee; - if (cur_tx.blob_size != get_object_blobsize(cur_tx.tx)) + if (cur_tx.weight != get_transaction_weight(cur_tx.tx)) { - LOG_ERROR("Creating block template: error: invalid transaction size"); + LOG_ERROR("Creating block template: error: invalid transaction weight"); } if (cur_tx.tx.version == 1) { @@ -1280,9 +1302,9 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m } } } - if (txs_size != real_txs_size) + if (txs_weight != real_txs_weight) { - LOG_ERROR("Creating block template: error: wrongly calculated transaction size"); + LOG_ERROR("Creating block template: error: wrongly calculated transaction weight"); } if (fee != real_fee) { @@ -1290,71 +1312,73 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m } CRITICAL_REGION_END(); MDEBUG("Creating block template: height " << height << - ", median size " << median_size << + ", median weight " << median_weight << ", already generated coins " << already_generated_coins << - ", transaction size " << txs_size << + ", transaction weight " << txs_weight << ", fee " << fee); #endif /* - two-phase miner transaction generation: we don't know exact block size until we prepare block, but we don't know reward until we know - block size, so first miner transaction generated with fake amount of money, and with phase we know think we know expected block size + two-phase miner transaction generation: we don't know exact block weight until we prepare block, but we don't know reward until we know + block weight, so first miner transaction generated with fake amount of money, and with phase we know think we know expected block weight */ - //make blocks coin-base tx looks close to real coinbase tx to get truthful blob size + //make blocks coin-base tx looks close to real coinbase tx to get truthful blob weight uint8_t hf_version = m_hardfork->get_current_version(); size_t max_outs = hf_version >= 4 ? 1 : 11; - bool r = construct_miner_tx(height, median_size, already_generated_coins, txs_size, fee, miner_address, b.miner_tx, ex_nonce, max_outs, hf_version); + bool r = construct_miner_tx(height, median_weight, already_generated_coins, txs_weight, fee, miner_address, b.miner_tx, ex_nonce, max_outs, hf_version); CHECK_AND_ASSERT_MES(r, false, "Failed to construct miner tx, first chance"); - size_t cumulative_size = txs_size + get_object_blobsize(b.miner_tx); + size_t cumulative_weight = txs_weight + get_transaction_weight(b.miner_tx); #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) - MDEBUG("Creating block template: miner tx size " << get_object_blobsize(b.miner_tx) << - ", cumulative size " << cumulative_size); + MDEBUG("Creating block template: miner tx weight " << get_transaction_weight(b.miner_tx) << + ", cumulative weight " << cumulative_weight); #endif for (size_t try_count = 0; try_count != 10; ++try_count) { - r = construct_miner_tx(height, median_size, already_generated_coins, cumulative_size, fee, miner_address, b.miner_tx, ex_nonce, max_outs, hf_version); + r = construct_miner_tx(height, median_weight, already_generated_coins, cumulative_weight, fee, miner_address, b.miner_tx, ex_nonce, max_outs, hf_version); CHECK_AND_ASSERT_MES(r, false, "Failed to construct miner tx, second chance"); - size_t coinbase_blob_size = get_object_blobsize(b.miner_tx); - if (coinbase_blob_size > cumulative_size - txs_size) + size_t coinbase_weight = get_transaction_weight(b.miner_tx); + if (coinbase_weight > cumulative_weight - txs_weight) { - cumulative_size = txs_size + coinbase_blob_size; + cumulative_weight = txs_weight + coinbase_weight; #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) - MDEBUG("Creating block template: miner tx size " << coinbase_blob_size << - ", cumulative size " << cumulative_size << " is greater than before"); + MDEBUG("Creating block template: miner tx weight " << coinbase_weight << + ", cumulative weight " << cumulative_weight << " is greater than before"); #endif continue; } - if (coinbase_blob_size < cumulative_size - txs_size) + if (coinbase_weight < cumulative_weight - txs_weight) { - size_t delta = cumulative_size - txs_size - coinbase_blob_size; + size_t delta = cumulative_weight - txs_weight - coinbase_weight; #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) - MDEBUG("Creating block template: miner tx size " << coinbase_blob_size << - ", cumulative size " << txs_size + coinbase_blob_size << + MDEBUG("Creating block template: miner tx weight " << coinbase_weight << + ", cumulative weight " << txs_weight + coinbase_weight << " is less than before, adding " << delta << " zero bytes"); #endif b.miner_tx.extra.insert(b.miner_tx.extra.end(), delta, 0); //here could be 1 byte difference, because of extra field counter is varint, and it can become from 1-byte len to 2-bytes len. - if (cumulative_size != txs_size + get_object_blobsize(b.miner_tx)) + if (cumulative_weight != txs_weight + get_transaction_weight(b.miner_tx)) { - CHECK_AND_ASSERT_MES(cumulative_size + 1 == txs_size + get_object_blobsize(b.miner_tx), false, "unexpected case: cumulative_size=" << cumulative_size << " + 1 is not equal txs_cumulative_size=" << txs_size << " + get_object_blobsize(b.miner_tx)=" << get_object_blobsize(b.miner_tx)); + CHECK_AND_ASSERT_MES(cumulative_weight + 1 == txs_weight + get_transaction_weight(b.miner_tx), false, "unexpected case: cumulative_weight=" << cumulative_weight << " + 1 is not equal txs_cumulative_weight=" << txs_weight << " + get_transaction_weight(b.miner_tx)=" << get_transaction_weight(b.miner_tx)); b.miner_tx.extra.resize(b.miner_tx.extra.size() - 1); - if (cumulative_size != txs_size + get_object_blobsize(b.miner_tx)) + if (cumulative_weight != txs_weight + get_transaction_weight(b.miner_tx)) { - //fuck, not lucky, -1 makes varint-counter size smaller, in that case we continue to grow with cumulative_size + //fuck, not lucky, -1 makes varint-counter size smaller, in that case we continue to grow with cumulative_weight MDEBUG("Miner tx creation has no luck with delta_extra size = " << delta << " and " << delta - 1); - cumulative_size += delta - 1; + cumulative_weight += delta - 1; continue; } MDEBUG("Setting extra for block: " << b.miner_tx.extra.size() << ", try_count=" << try_count); } } - CHECK_AND_ASSERT_MES(cumulative_size == txs_size + get_object_blobsize(b.miner_tx), false, "unexpected case: cumulative_size=" << cumulative_size << " is not equal txs_cumulative_size=" << txs_size << " + get_object_blobsize(b.miner_tx)=" << get_object_blobsize(b.miner_tx)); + CHECK_AND_ASSERT_MES(cumulative_weight == txs_weight + get_transaction_weight(b.miner_tx), false, "unexpected case: cumulative_weight=" << cumulative_weight << " is not equal txs_cumulative_weight=" << txs_weight << " + get_transaction_weight(b.miner_tx)=" << get_transaction_weight(b.miner_tx)); #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) - MDEBUG("Creating block template: miner tx size " << coinbase_blob_size << - ", cumulative size " << cumulative_size << " is now good"); + MDEBUG("Creating block template: miner tx weight " << coinbase_weight << + ", cumulative weight " << cumulative_weight << " is now good"); #endif + + cache_block_template(b, miner_address, ex_nonce, diffic, expected_reward, pool_cookie); return true; } LOG_ERROR("Failed to create_block_template with " << 10 << " tries"); @@ -2294,7 +2318,7 @@ bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, cons total_height = get_current_blockchain_height(); size_t count = 0, size = 0; blocks.reserve(std::min(std::min(max_count, (size_t)10000), (size_t)(total_height - start_height))); - for(size_t i = start_height; i < total_height && count < max_count && (size < FIND_BLOCKCHAIN_SUPPLEMENT_MAX_SIZE || count < 3); i++, count++) + for(uint64_t i = start_height; i < total_height && count < max_count && (size < FIND_BLOCKCHAIN_SUPPLEMENT_MAX_SIZE || count < 3); i++, count++) { blocks.resize(blocks.size()+1); blocks.back().first.first = m_db->get_block_blob_from_height(i); @@ -2517,7 +2541,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, uint64_t& max_used_block_heigh if(m_show_time_stats) { size_t ring_size = !tx.vin.empty() && tx.vin[0].type() == typeid(txin_to_key) ? boost::get<txin_to_key>(tx.vin[0]).key_offsets.size() : 0; - MINFO("HASH: " << get_transaction_hash(tx) << " I/M/O: " << tx.vin.size() << "/" << ring_size << "/" << tx.vout.size() << " H: " << max_used_block_height << " ms: " << a + m_fake_scan_time << " B: " << get_object_blobsize(tx)); + MINFO("HASH: " << get_transaction_hash(tx) << " I/M/O: " << tx.vin.size() << "/" << ring_size << "/" << tx.vout.size() << " H: " << max_used_block_height << " ms: " << a + m_fake_scan_time << " B: " << get_object_blobsize(tx) << " W: " << get_transaction_weight(tx)); } if (!res) return false; @@ -2574,12 +2598,27 @@ bool Blockchain::check_tx_outputs(const transaction& tx, tx_verification_context // from v8, allow bulletproofs if (hf_version < 8) { - const bool bulletproof = tx.rct_signatures.type == rct::RCTTypeFullBulletproof || tx.rct_signatures.type == rct::RCTTypeSimpleBulletproof; - if (bulletproof || !tx.rct_signatures.p.bulletproofs.empty()) - { - MERROR("Bulletproofs are not allowed before v8"); - tvc.m_invalid_output = true; - return false; + if (tx.version >= 2) { + const bool bulletproof = rct::is_rct_bulletproof(tx.rct_signatures.type); + if (bulletproof || !tx.rct_signatures.p.bulletproofs.empty()) + { + MERROR("Bulletproofs are not allowed before v8"); + tvc.m_invalid_output = true; + return false; + } + } + } + + // from v9, forbid borromean range proofs + if (hf_version > 8) { + if (tx.version >= 2) { + const bool borromean = rct::is_rct_borromean(tx.rct_signatures.type); + if (borromean) + { + MERROR("Borromean range proofs are not allowed after v8"); + tvc.m_invalid_output = true; + return false; + } } } @@ -2608,7 +2647,7 @@ bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_pr rv.message = rct::hash2rct(tx_prefix_hash); // mixRing - full and simple store it in opposite ways - if (rv.type == rct::RCTTypeFull || rv.type == rct::RCTTypeFullBulletproof) + if (rv.type == rct::RCTTypeFull) { CHECK_AND_ASSERT_MES(!pubkeys.empty() && !pubkeys[0].empty(), false, "empty pubkeys"); rv.mixRing.resize(pubkeys[0].size()); @@ -2623,7 +2662,7 @@ bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_pr } } } - else if (rv.type == rct::RCTTypeSimple || rv.type == rct::RCTTypeSimpleBulletproof) + else if (rv.type == rct::RCTTypeSimple || rv.type == rct::RCTTypeBulletproof) { CHECK_AND_ASSERT_MES(!pubkeys.empty() && !pubkeys[0].empty(), false, "empty pubkeys"); rv.mixRing.resize(pubkeys.size()); @@ -2642,14 +2681,14 @@ bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_pr } // II - if (rv.type == rct::RCTTypeFull || rv.type == rct::RCTTypeFullBulletproof) + if (rv.type == rct::RCTTypeFull) { rv.p.MGs.resize(1); rv.p.MGs[0].II.resize(tx.vin.size()); for (size_t n = 0; n < tx.vin.size(); ++n) rv.p.MGs[0].II[n] = rct::ki2rct(boost::get<txin_to_key>(tx.vin[n]).k_image); } - else if (rv.type == rct::RCTTypeSimple || rv.type == rct::RCTTypeSimpleBulletproof) + else if (rv.type == rct::RCTTypeSimple || rv.type == rct::RCTTypeBulletproof) { CHECK_AND_ASSERT_MES(rv.p.MGs.size() == tx.vin.size(), false, "Bad MGs size"); for (size_t n = 0; n < tx.vin.size(); ++n) @@ -2691,7 +2730,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, { size_t n_unmixable = 0, n_mixable = 0; size_t mixin = std::numeric_limits<size_t>::max(); - const size_t min_mixin = hf_version >= HF_VERSION_MIN_MIXIN_6 ? 6 : hf_version >= HF_VERSION_MIN_MIXIN_4 ? 4 : 2; + const size_t min_mixin = hf_version >= HF_VERSION_MIN_MIXIN_10 ? 10 : hf_version >= HF_VERSION_MIN_MIXIN_6 ? 6 : hf_version >= HF_VERSION_MIN_MIXIN_4 ? 4 : 2; for (const auto& txin : tx.vin) { // non txin_to_key inputs will be rejected below @@ -2720,6 +2759,13 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, } } + if (hf_version >= HF_VERSION_MIN_MIXIN_10 && mixin != 10) + { + MERROR_VER("Tx " << get_transaction_hash(tx) << " has invalid ring size (" << (mixin + 1) << "), it should be 11"); + tvc.m_low_mixin = true; + return false; + } + if (mixin < min_mixin) { if (n_unmixable == 0) @@ -2915,7 +2961,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, return false; } case rct::RCTTypeSimple: - case rct::RCTTypeSimpleBulletproof: + case rct::RCTTypeBulletproof: { // check all this, either reconstructed (so should really pass), or not { @@ -2965,7 +3011,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, } } - if (!rct::verRctSimple(rv, false)) + if (!rct::verRctNonSemanticsSimple(rv)) { MERROR_VER("Failed to check ringct signatures!"); return false; @@ -2973,7 +3019,6 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, break; } case rct::RCTTypeFull: - case rct::RCTTypeFullBulletproof: { // check all this, either reconstructed (so should really pass), or not { @@ -3036,6 +3081,22 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, MERROR_VER("Unsupported rct type: " << rv.type); return false; } + + // for bulletproofs, check they're only multi-output after v8 + if (rct::is_rct_bulletproof(rv.type)) + { + if (hf_version < 8) + { + for (const rct::Bulletproof &proof: rv.p.bulletproofs) + { + if (proof.V.size() > 1) + { + MERROR_VER("Multi output bulletproofs are invalid before v8"); + return false; + } + } + } + } } return true; } @@ -3055,7 +3116,7 @@ void Blockchain::check_ring_signature(const crypto::hash &tx_prefix_hash, const } //------------------------------------------------------------------ -static uint64_t get_fee_quantization_mask() +uint64_t Blockchain::get_fee_quantization_mask() { static uint64_t mask = 0; if (mask == 0) @@ -3068,16 +3129,27 @@ static uint64_t get_fee_quantization_mask() } //------------------------------------------------------------------ -uint64_t Blockchain::get_dynamic_per_kb_fee(uint64_t block_reward, size_t median_block_size, uint8_t version) +uint64_t Blockchain::get_dynamic_base_fee(uint64_t block_reward, size_t median_block_weight, uint8_t version) { - const uint64_t min_block_size = get_min_block_size(version); - const uint64_t fee_per_kb_base = version >= 5 ? DYNAMIC_FEE_PER_KB_BASE_FEE_V5 : DYNAMIC_FEE_PER_KB_BASE_FEE; + const uint64_t min_block_weight = get_min_block_weight(version); + if (median_block_weight < min_block_weight) + median_block_weight = min_block_weight; + uint64_t hi, lo; - if (median_block_size < min_block_size) - median_block_size = min_block_size; + if (version >= HF_VERSION_PER_BYTE_FEE) + { + lo = mul128(block_reward, DYNAMIC_FEE_REFERENCE_TRANSACTION_WEIGHT, &hi); + div128_32(hi, lo, min_block_weight, &hi, &lo); + div128_32(hi, lo, median_block_weight, &hi, &lo); + assert(hi == 0); + lo /= 5; + return lo; + } + + const uint64_t fee_base = version >= 5 ? DYNAMIC_FEE_PER_KB_BASE_FEE_V5 : DYNAMIC_FEE_PER_KB_BASE_FEE; - uint64_t unscaled_fee_per_kb = (fee_per_kb_base * min_block_size / median_block_size); - uint64_t hi, lo = mul128(unscaled_fee_per_kb, block_reward, &hi); + uint64_t unscaled_fee_base = (fee_base * min_block_weight / median_block_weight); + lo = mul128(unscaled_fee_base, block_reward, &hi); static_assert(DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD % 1000000 == 0, "DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD must be divisible by 1000000"); static_assert(DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD / 1000000 <= std::numeric_limits<uint32_t>::max(), "DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD is too large"); @@ -3095,29 +3167,48 @@ uint64_t Blockchain::get_dynamic_per_kb_fee(uint64_t block_reward, size_t median } //------------------------------------------------------------------ -bool Blockchain::check_fee(size_t blob_size, uint64_t fee) const +bool Blockchain::check_fee(size_t tx_weight, uint64_t fee) const { const uint8_t version = get_current_hard_fork_version(); - uint64_t fee_per_kb; - if (version < HF_VERSION_DYNAMIC_FEE) - { - fee_per_kb = FEE_PER_KB; - } - else + uint64_t median = 0; + uint64_t already_generated_coins = 0; + uint64_t base_reward = 0; + if (version >= HF_VERSION_DYNAMIC_FEE) { - uint64_t median = m_current_block_cumul_sz_limit / 2; - uint64_t already_generated_coins = m_db->height() ? m_db->get_block_already_generated_coins(m_db->height() - 1) : 0; - uint64_t base_reward; + median = m_current_block_cumul_weight_limit / 2; + already_generated_coins = m_db->height() ? m_db->get_block_already_generated_coins(m_db->height() - 1) : 0; if (!get_block_reward(median, 1, already_generated_coins, base_reward, version)) return false; - fee_per_kb = get_dynamic_per_kb_fee(base_reward, median, version); } - MDEBUG("Using " << print_money(fee_per_kb) << "/kB fee"); - uint64_t needed_fee = blob_size / 1024; - needed_fee += (blob_size % 1024) ? 1 : 0; - needed_fee *= fee_per_kb; + uint64_t needed_fee; + if (version >= HF_VERSION_PER_BYTE_FEE) + { + uint64_t fee_per_byte = get_dynamic_base_fee(base_reward, median, version); + MDEBUG("Using " << print_money(fee_per_byte) << "/byte fee"); + needed_fee = tx_weight * fee_per_byte; + // quantize fee up to 8 decimals + const uint64_t mask = get_fee_quantization_mask(); + needed_fee = (needed_fee + mask - 1) / mask * mask; + } + else + { + uint64_t fee_per_kb; + if (version < HF_VERSION_DYNAMIC_FEE) + { + fee_per_kb = FEE_PER_KB; + } + else + { + fee_per_kb = get_dynamic_base_fee(base_reward, median, version); + } + MDEBUG("Using " << print_money(fee_per_kb) << "/kB fee"); + + needed_fee = tx_weight / 1024; + needed_fee += (tx_weight % 1024) ? 1 : 0; + needed_fee *= fee_per_kb; + } if (fee < needed_fee - needed_fee / 50) // keep a little 2% buffer on acceptance - no integer overflow { @@ -3128,7 +3219,7 @@ bool Blockchain::check_fee(size_t blob_size, uint64_t fee) const } //------------------------------------------------------------------ -uint64_t Blockchain::get_dynamic_per_kb_fee_estimate(uint64_t grace_blocks) const +uint64_t Blockchain::get_dynamic_base_fee_estimate(uint64_t grace_blocks) const { const uint8_t version = get_current_hard_fork_version(); @@ -3138,16 +3229,16 @@ uint64_t Blockchain::get_dynamic_per_kb_fee_estimate(uint64_t grace_blocks) cons if (grace_blocks >= CRYPTONOTE_REWARD_BLOCKS_WINDOW) grace_blocks = CRYPTONOTE_REWARD_BLOCKS_WINDOW - 1; - const uint64_t min_block_size = get_min_block_size(version); - std::vector<size_t> sz; - get_last_n_blocks_sizes(sz, CRYPTONOTE_REWARD_BLOCKS_WINDOW - grace_blocks); - sz.reserve(grace_blocks); + const uint64_t min_block_weight = get_min_block_weight(version); + std::vector<size_t> weights; + get_last_n_blocks_weights(weights, CRYPTONOTE_REWARD_BLOCKS_WINDOW - grace_blocks); + weights.reserve(grace_blocks); for (size_t i = 0; i < grace_blocks; ++i) - sz.push_back(min_block_size); + weights.push_back(min_block_weight); - uint64_t median = epee::misc_utils::median(sz); - if(median <= min_block_size) - median = min_block_size; + uint64_t median = epee::misc_utils::median(weights); + if(median <= min_block_weight) + median = min_block_weight; uint64_t already_generated_coins = m_db->height() ? m_db->get_block_already_generated_coins(m_db->height() - 1) : 0; uint64_t base_reward; @@ -3157,8 +3248,9 @@ uint64_t Blockchain::get_dynamic_per_kb_fee_estimate(uint64_t grace_blocks) cons base_reward = BLOCK_REWARD_OVERESTIMATE; } - uint64_t fee = get_dynamic_per_kb_fee(base_reward, median, version); - MDEBUG("Estimating " << grace_blocks << "-block fee at " << print_money(fee) << "/kB"); + uint64_t fee = get_dynamic_base_fee(base_reward, median, version); + const bool per_byte = version < HF_VERSION_PER_BYTE_FEE; + MDEBUG("Estimating " << grace_blocks << "-block fee at " << print_money(fee) << "/" << (per_byte ? "byte" : "kB")); return fee; } @@ -3334,11 +3426,11 @@ bool Blockchain::flush_txes_from_pool(const std::vector<crypto::hash> &txids) for (const auto &txid: txids) { cryptonote::transaction tx; - size_t blob_size; + size_t tx_weight; uint64_t fee; bool relayed, do_not_relay, double_spend_seen; MINFO("Removing txid " << txid << " from the pool"); - if(m_tx_pool.have_tx(txid) && !m_tx_pool.take_tx(txid, tx, blob_size, fee, relayed, do_not_relay, double_spend_seen)) + if(m_tx_pool.have_tx(txid) && !m_tx_pool.take_tx(txid, tx, tx_weight, fee, relayed, do_not_relay, double_spend_seen)) { MERROR("Failed to remove txid " << txid << " from the pool"); res = false; @@ -3498,8 +3590,8 @@ leave: goto leave; } - size_t coinbase_blob_size = get_object_blobsize(bl.miner_tx); - size_t cumulative_block_size = coinbase_blob_size; + size_t coinbase_weight = get_transaction_weight(bl.miner_tx); + size_t cumulative_block_weight = coinbase_weight; std::vector<transaction> txs; key_images_container keys; @@ -3521,7 +3613,7 @@ leave: for (const crypto::hash& tx_id : bl.tx_hashes) { transaction tx; - size_t blob_size = 0; + size_t tx_weight = 0; uint64_t fee = 0; bool relayed = false, do_not_relay = false, double_spend_seen = false; TIME_MEASURE_START(aa); @@ -3540,7 +3632,7 @@ leave: TIME_MEASURE_START(bb); // get transaction with hash <tx_id> from tx_pool - if(!m_tx_pool.take_tx(tx_id, tx, blob_size, fee, relayed, do_not_relay, double_spend_seen)) + if(!m_tx_pool.take_tx(tx_id, tx, tx_weight, fee, relayed, do_not_relay, double_spend_seen)) { MERROR_VER("Block with id: " << id << " has at least one unknown transaction with id: " << tx_id); bvc.m_verifivation_failed = true; @@ -3611,7 +3703,7 @@ leave: TIME_MEASURE_FINISH(cc); t_checktx += cc; fee_summary += fee; - cumulative_block_size += blob_size; + cumulative_block_weight += tx_weight; } m_blocks_txs_check.clear(); @@ -3619,7 +3711,7 @@ leave: TIME_MEASURE_START(vmt); uint64_t base_reward = 0; uint64_t already_generated_coins = m_db->height() ? m_db->get_block_already_generated_coins(m_db->height() - 1) : 0; - if(!validate_miner_transaction(bl, cumulative_block_size, fee_summary, base_reward, already_generated_coins, bvc.m_partial_block_reward, m_hardfork->get_current_version())) + if(!validate_miner_transaction(bl, cumulative_block_weight, fee_summary, base_reward, already_generated_coins, bvc.m_partial_block_reward, m_hardfork->get_current_version())) { MERROR_VER("Block with id: " << id << " has incorrect miner transaction"); bvc.m_verifivation_failed = true; @@ -3628,11 +3720,11 @@ leave: } TIME_MEASURE_FINISH(vmt); - size_t block_size; + size_t block_weight; difficulty_type cumulative_difficulty; // populate various metadata about the block to be stored alongside it. - block_size = cumulative_block_size; + block_weight = cumulative_block_weight; cumulative_difficulty = current_diffic; // In the "tail" state when the minimum subsidy (implemented in get_block_reward) is in effect, the number of // coins will eventually exceed MONEY_SUPPLY and overflow a uint64. To prevent overflow, cap already_generated_coins @@ -3653,7 +3745,7 @@ leave: { try { - new_height = m_db->add_block(bl, block_size, cumulative_difficulty, already_generated_coins, txs); + new_height = m_db->add_block(bl, block_weight, cumulative_difficulty, already_generated_coins, txs); } catch (const KEY_IMAGE_EXISTS& e) { @@ -3678,14 +3770,14 @@ leave: TIME_MEASURE_FINISH(addblock); - // do this after updating the hard fork state since the size limit may change due to fork - update_next_cumulative_size_limit(); + // do this after updating the hard fork state since the weight limit may change due to fork + update_next_cumulative_weight_limit(); - MINFO("+++++ BLOCK SUCCESSFULLY ADDED" << std::endl << "id:\t" << id << std::endl << "PoW:\t" << proof_of_work << std::endl << "HEIGHT " << new_height-1 << ", difficulty:\t" << current_diffic << std::endl << "block reward: " << print_money(fee_summary + base_reward) << "(" << print_money(base_reward) << " + " << print_money(fee_summary) << "), coinbase_blob_size: " << coinbase_blob_size << ", cumulative size: " << cumulative_block_size << ", " << block_processing_time << "(" << target_calculating_time << "/" << longhash_calculating_time << ")ms"); + MINFO("+++++ BLOCK SUCCESSFULLY ADDED" << std::endl << "id:\t" << id << std::endl << "PoW:\t" << proof_of_work << std::endl << "HEIGHT " << new_height-1 << ", difficulty:\t" << current_diffic << std::endl << "block reward: " << print_money(fee_summary + base_reward) << "(" << print_money(base_reward) << " + " << print_money(fee_summary) << "), coinbase_weight: " << coinbase_weight << ", cumulative weight: " << cumulative_block_weight << ", " << block_processing_time << "(" << target_calculating_time << "/" << longhash_calculating_time << ")ms"); if(m_show_time_stats) { - MINFO("Height: " << new_height << " blob: " << coinbase_blob_size << " cumm: " - << cumulative_block_size << " p/t: " << block_processing_time << " (" + MINFO("Height: " << new_height << " coinbase weight: " << coinbase_weight << " cumm: " + << cumulative_block_weight << " p/t: " << block_processing_time << " (" << target_calculating_time << "/" << longhash_calculating_time << "/" << t1 << "/" << t2 << "/" << t3 << "/" << t_exists << "/" << t_pool << "/" << t_checktx << "/" << t_dblspnd << "/" << vmt << "/" << addblock << ")ms"); @@ -3697,24 +3789,25 @@ leave: // appears to be a NOP *and* is called elsewhere. wat? m_tx_pool.on_blockchain_inc(new_height, id); get_difficulty_for_next_block(); // just to cache it + invalidate_block_template_cache(); return true; } //------------------------------------------------------------------ -bool Blockchain::update_next_cumulative_size_limit() +bool Blockchain::update_next_cumulative_weight_limit() { - uint64_t full_reward_zone = get_min_block_size(get_current_hard_fork_version()); + uint64_t full_reward_zone = get_min_block_weight(get_current_hard_fork_version()); LOG_PRINT_L3("Blockchain::" << __func__); - std::vector<size_t> sz; - get_last_n_blocks_sizes(sz, CRYPTONOTE_REWARD_BLOCKS_WINDOW); + std::vector<size_t> weights; + get_last_n_blocks_weights(weights, CRYPTONOTE_REWARD_BLOCKS_WINDOW); - uint64_t median = epee::misc_utils::median(sz); - m_current_block_cumul_sz_median = median; + uint64_t median = epee::misc_utils::median(weights); + m_current_block_cumul_weight_median = median; if(median <= full_reward_zone) median = full_reward_zone; - m_current_block_cumul_sz_limit = median*2; + m_current_block_cumul_weight_limit = median*2; return true; } //------------------------------------------------------------------ @@ -3877,11 +3970,13 @@ bool Blockchain::cleanup_handle_incoming_blocks(bool force_sync) store_blockchain(); m_sync_counter = 0; } - else if (m_db_blocks_per_sync && m_sync_counter >= m_db_blocks_per_sync) + else if (m_db_sync_threshold && ((m_db_sync_on_blocks && m_sync_counter >= m_db_sync_threshold) || (!m_db_sync_on_blocks && m_bytes_to_sync >= m_db_sync_threshold))) { + MDEBUG("Sync threshold met, syncing"); if(m_db_sync_mode == db_async) { m_sync_counter = 0; + m_bytes_to_sync = 0; m_async_service.dispatch(boost::bind(&Blockchain::store_blockchain, this)); } else if(m_db_sync_mode == db_sync) @@ -4073,6 +4168,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete } total_txs += entry.txs.size(); } + m_bytes_to_sync += bytes; while (!(stop_batch = m_db->batch_start(blocks_entry.size(), bytes))) { m_blockchain_lock.unlock(); m_tx_pool.unlock(); @@ -4423,7 +4519,7 @@ bool Blockchain::for_all_txpool_txes(std::function<bool(const crypto::hash&, con return m_db->for_all_txpool_txes(f, include_blob, include_unrelayed_txes); } -void Blockchain::set_user_options(uint64_t maxthreads, uint64_t blocks_per_sync, blockchain_db_sync_mode sync_mode, bool fast_sync) +void Blockchain::set_user_options(uint64_t maxthreads, bool sync_on_blocks, uint64_t sync_threshold, blockchain_db_sync_mode sync_mode, bool fast_sync) { if (sync_mode == db_defaultsync) { @@ -4432,7 +4528,8 @@ void Blockchain::set_user_options(uint64_t maxthreads, uint64_t blocks_per_sync, } m_db_sync_mode = sync_mode; m_fast_sync = fast_sync; - m_db_blocks_per_sync = blocks_per_sync; + m_db_sync_on_blocks = sync_on_blocks; + m_db_sync_threshold = sync_threshold; m_max_prepare_blocks_threads = maxthreads; } @@ -4607,14 +4704,14 @@ void Blockchain::load_compiled_in_block_hashes() std::vector<transaction> txs; m_tx_pool.get_transactions(txs); - size_t blob_size; + size_t tx_weight; uint64_t fee; bool relayed, do_not_relay, double_spend_seen; transaction pool_tx; for(const transaction &tx : txs) { crypto::hash tx_hash = get_transaction_hash(tx); - m_tx_pool.take_tx(tx_hash, pool_tx, blob_size, fee, relayed, do_not_relay, double_spend_seen); + m_tx_pool.take_tx(tx_hash, pool_tx, tx_weight, fee, relayed, do_not_relay, double_spend_seen); } } } @@ -4666,6 +4763,24 @@ bool Blockchain::for_all_outputs(uint64_t amount, std::function<bool(uint64_t he return m_db->for_all_outputs(amount, f);; } +void Blockchain::invalidate_block_template_cache() +{ + MDEBUG("Invalidating block template cache"); + m_btc_valid = false; +} + +void Blockchain::cache_block_template(const block &b, const cryptonote::account_public_address &address, const blobdata &nonce, const difficulty_type &diff, uint64_t expected_reward, uint64_t pool_cookie) +{ + MDEBUG("Setting block template cache"); + m_btc = b; + m_btc_address = address; + m_btc_nonce = nonce; + m_btc_difficulty = diff; + m_btc_expected_reward = expected_reward; + m_btc_pool_cookie = pool_cookie; + m_btc_valid = true; +} + namespace cryptonote { template bool Blockchain::get_transactions(const std::vector<crypto::hash>&, std::vector<transaction>&, std::vector<crypto::hash>&) const; template bool Blockchain::get_transactions_blobs(const std::vector<crypto::hash>&, std::vector<cryptonote::blobdata>&, std::vector<crypto::hash>&, bool) const; diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index d95c8ed15..7e2ba7a39 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -95,7 +95,7 @@ namespace cryptonote { block bl; //!< the block uint64_t height; //!< the height of the block in the blockchain - size_t block_cumulative_size; //!< the size (in bytes) of the block + size_t block_cumulative_weight; //!< the weight of the block difficulty_type cumulative_difficulty; //!< the accumulated difficulty after that block uint64_t already_generated_coins; //!< the total coins minted after that block }; @@ -579,46 +579,57 @@ namespace cryptonote bool check_tx_inputs(transaction& tx, uint64_t& pmax_used_block_height, crypto::hash& max_used_block_id, tx_verification_context &tvc, bool kept_by_block = false); /** - * @brief get dynamic per kB fee for a given block size + * @brief get fee quantization mask * - * The dynamic fee is based on the block size in a past window, and - * the current block reward. It is expressed by kB. + * The dynamic fee may be quantized, to mask out the last decimal places + * + * @return the fee quantized mask + */ + static uint64_t get_fee_quantization_mask(); + + /** + * @brief get dynamic per kB or byte fee for a given block weight + * + * The dynamic fee is based on the block weight in a past window, and + * the current block reward. It is expressed by kB before v8, and + * per byte from v8. * * @param block_reward the current block reward - * @param median_block_size the median blob's size in the past window + * @param median_block_weight the median block weight in the past window * @param version hard fork version for rules and constants to use * - * @return the per kB fee + * @return the fee */ - static uint64_t get_dynamic_per_kb_fee(uint64_t block_reward, size_t median_block_size, uint8_t version); + static uint64_t get_dynamic_base_fee(uint64_t block_reward, size_t median_block_weight, uint8_t version); /** - * @brief get dynamic per kB fee estimate for the next few blocks + * @brief get dynamic per kB or byte fee estimate for the next few blocks * - * The dynamic fee is based on the block size in a past window, and - * the current block reward. It is expressed by kB. This function - * calculates an estimate for a dynamic fee which will be valid for - * the next grace_blocks + * The dynamic fee is based on the block weight in a past window, and + * the current block reward. It is expressed by kB before v8, and + * per byte from v8. + * This function calculates an estimate for a dynamic fee which will be + * valid for the next grace_blocks * * @param grace_blocks number of blocks we want the fee to be valid for * - * @return the per kB fee estimate + * @return the fee estimate */ - uint64_t get_dynamic_per_kb_fee_estimate(uint64_t grace_blocks) const; + uint64_t get_dynamic_base_fee_estimate(uint64_t grace_blocks) const; /** * @brief validate a transaction's fee * * This function validates the fee is enough for the transaction. - * This is based on the size of the transaction blob, and, after a - * height threshold, on the average size of transaction in a past window + * This is based on the weight of the transaction, and, after a + * height threshold, on the average weight of transaction in a past window * - * @param blob_size the transaction blob's size + * @param tx_weight the transaction weight * @param fee the fee * * @return true if the fee is enough, false otherwise */ - bool check_fee(size_t blob_size, uint64_t fee) const; + bool check_fee(size_t tx_weight, uint64_t fee) const; /** * @brief check that a transaction's outputs conform to current standards @@ -635,18 +646,18 @@ namespace cryptonote bool check_tx_outputs(const transaction& tx, tx_verification_context &tvc); /** - * @brief gets the blocksize limit based on recent blocks + * @brief gets the block weight limit based on recent blocks * * @return the limit */ - uint64_t get_current_cumulative_blocksize_limit() const; + uint64_t get_current_cumulative_block_weight_limit() const; /** - * @brief gets the blocksize median based on recent blocks (same window as for the limit) + * @brief gets the block weight median based on recent blocks (same window as for the limit) * * @return the median */ - uint64_t get_current_cumulative_blocksize_median() const; + uint64_t get_current_cumulative_block_weight_median() const; /** * @brief gets the difficulty of the block with a given height @@ -729,11 +740,12 @@ namespace cryptonote * @brief sets various performance options * * @param maxthreads max number of threads when preparing blocks for addition - * @param blocks_per_sync number of blocks to cache before syncing to database + * @param sync_on_blocks whether to sync based on blocks or bytes + * @param sync_threshold number of blocks/bytes to cache before syncing to database * @param sync_mode the ::blockchain_db_sync_mode to use * @param fast_sync sync using built-in block hashes as trusted */ - void set_user_options(uint64_t maxthreads, uint64_t blocks_per_sync, + void set_user_options(uint64_t maxthreads, bool sync_on_blocks, uint64_t sync_threshold, blockchain_db_sync_mode sync_mode, bool fast_sync); /** @@ -1000,8 +1012,8 @@ namespace cryptonote // main chain transactions_container m_transactions; - size_t m_current_block_cumul_sz_limit; - size_t m_current_block_cumul_sz_median; + size_t m_current_block_cumul_weight_limit; + size_t m_current_block_cumul_weight_median; // metadata containers std::unordered_map<crypto::hash, std::unordered_map<crypto::key_image, std::vector<output_data_t>>> m_scan_table; @@ -1017,11 +1029,13 @@ namespace cryptonote bool m_fast_sync; bool m_show_time_stats; bool m_db_default_sync; - uint64_t m_db_blocks_per_sync; + bool m_db_sync_on_blocks; + uint64_t m_db_sync_threshold; uint64_t m_max_prepare_blocks_threads; uint64_t m_fake_pow_calc_time; uint64_t m_fake_scan_time; uint64_t m_sync_counter; + uint64_t m_bytes_to_sync; std::vector<uint64_t> m_timestamps; std::vector<difficulty_type> m_difficulties; uint64_t m_timestamps_and_difficulties_height; @@ -1052,6 +1066,15 @@ namespace cryptonote std::atomic<bool> m_cancel; + // block template cache + block m_btc; + account_public_address m_btc_address; + blobdata m_btc_nonce; + difficulty_type m_btc_difficulty; + uint64_t m_btc_pool_cookie; + uint64_t m_btc_expected_reward; + bool m_btc_valid; + /** * @brief collects the keys for all outputs being "spent" as an input * @@ -1213,7 +1236,7 @@ namespace cryptonote * and that his miner transaction totals reward + fee. * * @param b the block containing the miner transaction to be validated - * @param cumulative_block_size the block's size + * @param cumulative_block_weight the block's weight * @param fee the total fees collected in the block * @param base_reward return-by-reference the new block's generated coins * @param already_generated_coins the amount of currency generated prior to this block @@ -1222,7 +1245,7 @@ namespace cryptonote * * @return false if anything is found wrong with the miner transaction, otherwise true */ - bool validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins, bool &partial_block_reward, uint8_t version); + bool validate_miner_transaction(const block& b, size_t cumulative_block_weight, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins, bool &partial_block_reward, uint8_t version); /** * @brief reverts the blockchain to its previous state following a failed switch @@ -1239,14 +1262,14 @@ namespace cryptonote bool rollback_blockchain_switching(std::list<block>& original_chain, uint64_t rollback_height); /** - * @brief gets recent block sizes for median calculation + * @brief gets recent block weights for median calculation * - * get the block sizes of the last <count> blocks, and return by reference <sz>. + * get the block weights of the last <count> blocks, and return by reference <sz>. * - * @param sz return-by-reference the list of sizes - * @param count the number of blocks to get sizes for + * @param sz return-by-reference the list of weights + * @param count the number of blocks to get weights for */ - void get_last_n_blocks_sizes(std::vector<size_t>& sz, size_t count) const; + void get_last_n_blocks_weights(std::vector<size_t>& weights, size_t count) const; /** * @brief adds the given output to the requested set of random outputs @@ -1361,11 +1384,11 @@ namespace cryptonote bool complete_timestamps_vector(uint64_t start_height, std::vector<uint64_t>& timestamps); /** - * @brief calculate the block size limit for the next block to be added + * @brief calculate the block weight limit for the next block to be added * * @return true */ - bool update_next_cumulative_size_limit(); + bool update_next_cumulative_weight_limit(); void return_tx_to_pool(std::vector<transaction> &txs); /** @@ -1407,5 +1430,17 @@ namespace cryptonote * that implicit data. */ bool expand_transaction_2(transaction &tx, const crypto::hash &tx_prefix_hash, const std::vector<std::vector<rct::ctkey>> &pubkeys); + + /** + * @brief invalidates any cached block template + */ + void invalidate_block_template_cache(); + + /** + * @brief stores a new cached block template + * + * At some point, may be used to push an update to miners + */ + void cache_block_template(const block &b, const cryptonote::account_public_address &address, const blobdata &nonce, const difficulty_type &diff, uint64_t expected_reward, uint64_t pool_cookie); }; } // namespace cryptonote diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 18490c65e..c4eaa0cc4 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -162,10 +162,10 @@ namespace cryptonote , "Relay blocks as normal blocks" , false }; - static const command_line::arg_descriptor<size_t> arg_max_txpool_size = { - "max-txpool-size" - , "Set maximum txpool size in bytes." - , DEFAULT_TXPOOL_MAX_SIZE + static const command_line::arg_descriptor<size_t> arg_max_txpool_weight = { + "max-txpool-weight" + , "Set maximum txpool weight in bytes." + , DEFAULT_TXPOOL_MAX_WEIGHT }; //----------------------------------------------------------------------------------------------- @@ -274,7 +274,7 @@ namespace cryptonote command_line::add_arg(desc, arg_test_dbg_lock_sleep); command_line::add_arg(desc, arg_offline); command_line::add_arg(desc, arg_disable_dns_checkpoints); - command_line::add_arg(desc, arg_max_txpool_size); + command_line::add_arg(desc, arg_max_txpool_weight); miner::init_options(desc); BlockchainDB::init_options(desc); @@ -402,7 +402,7 @@ namespace cryptonote bool fast_sync = command_line::get_arg(vm, arg_fast_block_sync) != 0; uint64_t blocks_threads = command_line::get_arg(vm, arg_prep_blocks_threads); std::string check_updates_string = command_line::get_arg(vm, arg_check_updates); - size_t max_txpool_size = command_line::get_arg(vm, arg_max_txpool_size); + size_t max_txpool_weight = command_line::get_arg(vm, arg_max_txpool_weight); boost::filesystem::path folder(m_config_folder); if (m_nettype == FAKECHAIN) @@ -439,9 +439,10 @@ namespace cryptonote MGINFO("Loading blockchain from folder " << folder.string() << " ..."); const std::string filename = folder.string(); - // default to fast:async:1 + // default to fast:async:1 if overridden blockchain_db_sync_mode sync_mode = db_defaultsync; - uint64_t blocks_per_sync = 1; + bool sync_on_blocks = true; + uint64_t sync_threshold = 1; if (m_nettype == FAKECHAIN) { @@ -491,7 +492,7 @@ namespace cryptonote else if(options[0] == "fastest") { db_flags = DBF_FASTEST; - blocks_per_sync = 1000; // default to fastest:async:1000 + sync_threshold = 1000; // default to fastest:async:1000 sync_mode = db_sync_mode_is_default ? db_defaultsync : db_async; } else @@ -509,9 +510,22 @@ namespace cryptonote if(options.size() >= 3 && !safemode) { char *endptr; - uint64_t bps = strtoull(options[2].c_str(), &endptr, 0); - if (*endptr == '\0') - blocks_per_sync = bps; + uint64_t threshold = strtoull(options[2].c_str(), &endptr, 0); + if (*endptr == '\0' || !strcmp(endptr, "blocks")) + { + sync_on_blocks = true; + sync_threshold = threshold; + } + else if (!strcmp(endptr, "bytes")) + { + sync_on_blocks = false; + sync_threshold = threshold; + } + else + { + LOG_ERROR("Invalid db sync mode: " << options[2]); + return false; + } } if (db_salvage) @@ -528,7 +542,7 @@ namespace cryptonote } m_blockchain_storage.set_user_options(blocks_threads, - blocks_per_sync, sync_mode, fast_sync); + sync_on_blocks, sync_threshold, sync_mode, fast_sync); const std::pair<uint8_t, uint64_t> regtest_hard_forks[3] = {std::make_pair(1, 0), std::make_pair(Blockchain::get_hard_fork_heights(MAINNET).back().version, 1), std::make_pair(0, 0)}; const cryptonote::test_options regtest_test_options = { @@ -537,7 +551,7 @@ namespace cryptonote const difficulty_type fixed_difficulty = command_line::get_arg(vm, arg_fixed_difficulty); r = m_blockchain_storage.init(db.release(), m_nettype, m_offline, regtest ? ®test_test_options : test_options, fixed_difficulty); - r = m_mempool.init(max_txpool_size); + r = m_mempool.init(max_txpool_weight); CHECK_AND_ASSERT_MES(r, false, "Failed to initialize memory pool"); // now that we have a valid m_blockchain_storage, we can clean out any @@ -678,26 +692,123 @@ namespace cryptonote return false; } + return true; + } + //----------------------------------------------------------------------------------------------- + void core::set_semantics_failed(const crypto::hash &tx_hash) + { + LOG_PRINT_L1("WRONG TRANSACTION BLOB, Failed to check tx " << tx_hash << " semantic, rejected"); + bad_semantics_txes_lock.lock(); + bad_semantics_txes[0].insert(tx_hash); + if (bad_semantics_txes[0].size() >= BAD_SEMANTICS_TXES_MAX_SIZE) + { + std::swap(bad_semantics_txes[0], bad_semantics_txes[1]); + bad_semantics_txes[0].clear(); + } + bad_semantics_txes_lock.unlock(); + } + //----------------------------------------------------------------------------------------------- + static bool is_canonical_bulletproof_layout(const std::vector<rct::Bulletproof> &proofs) + { + if (proofs.size() != 1) + return false; + const size_t sz = proofs[0].V.size(); + if (sz == 0 || sz > BULLETPROOF_MAX_OUTPUTS) + return false; + return true; + } + //----------------------------------------------------------------------------------------------- + bool core::handle_incoming_tx_accumulated_batch(std::vector<tx_verification_batch_info> &tx_info, bool keeped_by_block) + { + bool ret = true; if (keeped_by_block && get_blockchain_storage().is_within_compiled_block_hash_area()) { MTRACE("Skipping semantics check for tx kept by block in embedded hash area"); + return true; } - else if(!check_tx_semantic(tx, keeped_by_block)) + + std::vector<const rct::rctSig*> rvv; + for (size_t n = 0; n < tx_info.size(); ++n) { - LOG_PRINT_L1("WRONG TRANSACTION BLOB, Failed to check tx " << tx_hash << " semantic, rejected"); - tvc.m_verifivation_failed = true; - bad_semantics_txes_lock.lock(); - bad_semantics_txes[0].insert(tx_hash); - if (bad_semantics_txes[0].size() >= BAD_SEMANTICS_TXES_MAX_SIZE) + if (!check_tx_semantic(*tx_info[n].tx, keeped_by_block)) { - std::swap(bad_semantics_txes[0], bad_semantics_txes[1]); - bad_semantics_txes[0].clear(); + set_semantics_failed(tx_info[n].tx_hash); + tx_info[n].tvc.m_verifivation_failed = true; + tx_info[n].result = false; + continue; + } + + if (tx_info[n].tx->version < 2) + continue; + const rct::rctSig &rv = tx_info[n].tx->rct_signatures; + switch (rv.type) { + case rct::RCTTypeNull: + // coinbase should not come here, so we reject for all other types + MERROR_VER("Unexpected Null rctSig type"); + set_semantics_failed(tx_info[n].tx_hash); + tx_info[n].tvc.m_verifivation_failed = true; + tx_info[n].result = false; + break; + case rct::RCTTypeSimple: + if (!rct::verRctSemanticsSimple(rv)) + { + MERROR_VER("rct signature semantics check failed"); + set_semantics_failed(tx_info[n].tx_hash); + tx_info[n].tvc.m_verifivation_failed = true; + tx_info[n].result = false; + break; + } + break; + case rct::RCTTypeFull: + if (!rct::verRct(rv, true)) + { + MERROR_VER("rct signature semantics check failed"); + set_semantics_failed(tx_info[n].tx_hash); + tx_info[n].tvc.m_verifivation_failed = true; + tx_info[n].result = false; + break; + } + break; + case rct::RCTTypeBulletproof: + if (!is_canonical_bulletproof_layout(rv.p.bulletproofs)) + { + MERROR_VER("Bulletproof does not have canonical form"); + set_semantics_failed(tx_info[n].tx_hash); + tx_info[n].tvc.m_verifivation_failed = true; + tx_info[n].result = false; + break; + } + rvv.push_back(&rv); // delayed batch verification + break; + default: + MERROR_VER("Unknown rct type: " << rv.type); + set_semantics_failed(tx_info[n].tx_hash); + tx_info[n].tvc.m_verifivation_failed = true; + tx_info[n].result = false; + break; + } + } + if (!rvv.empty() && !rct::verRctSemanticsSimple(rvv)) + { + LOG_PRINT_L1("One transaction among this group has bad semantics, verifying one at a time"); + ret = false; + const bool assumed_bad = rvv.size() == 1; // if there's only one tx, it must be the bad one + for (size_t n = 0; n < tx_info.size(); ++n) + { + if (!tx_info[n].result) + continue; + if (tx_info[n].tx->rct_signatures.type != rct::RCTTypeBulletproof) + continue; + if (assumed_bad || !rct::verRctSemanticsSimple(tx_info[n].tx->rct_signatures)) + { + set_semantics_failed(tx_info[n].tx_hash); + tx_info[n].tvc.m_verifivation_failed = true; + tx_info[n].result = false; + } } - bad_semantics_txes_lock.unlock(); - return false; } - return true; + return ret; } //----------------------------------------------------------------------------------------------- bool core::handle_incoming_txs(const std::vector<blobdata>& tx_blobs, std::vector<tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) @@ -755,6 +866,16 @@ namespace cryptonote } waiter.wait(&tpool); + std::vector<tx_verification_batch_info> tx_info; + tx_info.reserve(tx_blobs.size()); + for (size_t i = 0; i < tx_blobs.size(); i++) { + if (!results[i].res) + continue; + tx_info.push_back({&results[i].tx, results[i].hash, tvc[i], results[i].res}); + } + if (!tx_info.empty()) + handle_incoming_tx_accumulated_batch(tx_info, keeped_by_block); + bool ok = true; it = tx_blobs.begin(); for (size_t i = 0; i < tx_blobs.size(); i++, ++it) { @@ -764,7 +885,8 @@ namespace cryptonote continue; } - ok &= add_new_tx(results[i].tx, results[i].hash, results[i].prefix_hash, it->size(), tvc[i], keeped_by_block, relayed, do_not_relay); + const size_t weight = get_transaction_weight(results[i].tx, it->size()); + ok &= add_new_tx(results[i].tx, results[i].hash, results[i].prefix_hash, weight, tvc[i], keeped_by_block, relayed, do_not_relay); if(tvc[i].m_verifivation_failed) {MERROR_VER("Transaction verification failed: " << results[i].hash);} else if(tvc[i].m_verifivation_impossible) @@ -847,9 +969,9 @@ namespace cryptonote } // for version > 1, ringct signatures check verifies amounts match - if(!keeped_by_block && get_object_blobsize(tx) >= m_blockchain_storage.get_current_cumulative_blocksize_limit() - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE) + if(!keeped_by_block && get_transaction_weight(tx) >= m_blockchain_storage.get_current_cumulative_block_weight_limit() - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE) { - MERROR_VER("tx is too large " << get_object_blobsize(tx) << ", expected not bigger than " << m_blockchain_storage.get_current_cumulative_blocksize_limit() - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE); + MERROR_VER("tx is too large " << get_transaction_weight(tx) << ", expected not bigger than " << m_blockchain_storage.get_current_cumulative_block_weight_limit() - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE); return false; } @@ -872,36 +994,6 @@ namespace cryptonote return false; } - if (tx.version >= 2) - { - const rct::rctSig &rv = tx.rct_signatures; - switch (rv.type) { - case rct::RCTTypeNull: - // coinbase should not come here, so we reject for all other types - MERROR_VER("Unexpected Null rctSig type"); - return false; - case rct::RCTTypeSimple: - case rct::RCTTypeSimpleBulletproof: - if (!rct::verRctSimple(rv, true)) - { - MERROR_VER("rct signature semantics check failed"); - return false; - } - break; - case rct::RCTTypeFull: - case rct::RCTTypeFullBulletproof: - if (!rct::verRct(rv, true)) - { - MERROR_VER("rct signature semantics check failed"); - return false; - } - break; - default: - MERROR_VER("Unknown rct type: " << rv.type); - return false; - } - } - return true; } //----------------------------------------------------------------------------------------------- @@ -1011,7 +1103,8 @@ namespace cryptonote crypto::hash tx_prefix_hash = get_transaction_prefix_hash(tx); blobdata bl; t_serializable_object_to_blob(tx, bl); - return add_new_tx(tx, tx_hash, tx_prefix_hash, bl.size(), tvc, keeped_by_block, relayed, do_not_relay); + size_t tx_weight = get_transaction_weight(tx, bl.size()); + return add_new_tx(tx, tx_hash, tx_prefix_hash, tx_weight, tvc, keeped_by_block, relayed, do_not_relay); } //----------------------------------------------------------------------------------------------- size_t core::get_blockchain_total_transactions() const @@ -1019,7 +1112,7 @@ namespace cryptonote return m_blockchain_storage.get_total_transactions(); } //----------------------------------------------------------------------------------------------- - bool core::add_new_tx(transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t blob_size, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) + bool core::add_new_tx(transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t tx_weight, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { if (keeped_by_block) get_blockchain_storage().on_new_tx_from_block(tx); @@ -1037,7 +1130,7 @@ namespace cryptonote } uint8_t version = m_blockchain_storage.get_current_hard_fork_version(); - return m_mempool.add_tx(tx, tx_hash, blob_size, tvc, keeped_by_block, relayed, do_not_relay, version); + return m_mempool.add_tx(tx, tx_hash, tx_weight, tvc, keeped_by_block, relayed, do_not_relay, version); } //----------------------------------------------------------------------------------------------- bool core::relay_txpool_transactions() diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 84e1bb918..8b68f5e2b 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -788,12 +788,12 @@ namespace cryptonote * * @param tx_hash the transaction's hash * @param tx_prefix_hash the transaction prefix' hash - * @param blob_size the size of the transaction + * @param tx_weight the weight of the transaction * @param relayed whether or not the transaction was relayed to us * @param do_not_relay whether to prevent the transaction from being relayed * */ - bool add_new_tx(transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t blob_size, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); + bool add_new_tx(transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t tx_weight, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); /** * @brief add a new transaction to the transaction pool @@ -864,9 +864,12 @@ namespace cryptonote * @return true if all the checks pass, otherwise false */ bool check_tx_semantic(const transaction& tx, bool keeped_by_block) const; + void set_semantics_failed(const crypto::hash &tx_hash); bool handle_incoming_tx_pre(const blobdata& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, crypto::hash &tx_prefixt_hash, bool keeped_by_block, bool relayed, bool do_not_relay); bool handle_incoming_tx_post(const blobdata& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, crypto::hash &tx_prefixt_hash, bool keeped_by_block, bool relayed, bool do_not_relay); + struct tx_verification_batch_info { const cryptonote::transaction *tx; crypto::hash tx_hash; tx_verification_context &tvc; bool &result; }; + bool handle_incoming_tx_accumulated_batch(std::vector<tx_verification_batch_info> &tx_info, bool keeped_by_block); /** * @copydoc miner::on_block_chain_update diff --git a/src/cryptonote_core/cryptonote_tx_utils.cpp b/src/cryptonote_core/cryptonote_tx_utils.cpp index 071ce591e..fb2af9ceb 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.cpp +++ b/src/cryptonote_core/cryptonote_tx_utils.cpp @@ -74,7 +74,7 @@ namespace cryptonote LOG_PRINT_L2("destinations include " << num_stdaddresses << " standard addresses and " << num_subaddresses << " subaddresses"); } //--------------------------------------------------------------- - bool construct_miner_tx(size_t height, size_t median_size, uint64_t already_generated_coins, size_t current_block_size, uint64_t fee, const account_public_address &miner_address, transaction& tx, const blobdata& extra_nonce, size_t max_outs, uint8_t hard_fork_version) { + bool construct_miner_tx(size_t height, size_t median_weight, uint64_t already_generated_coins, size_t current_block_weight, uint64_t fee, const account_public_address &miner_address, transaction& tx, const blobdata& extra_nonce, size_t max_outs, uint8_t hard_fork_version) { tx.vin.clear(); tx.vout.clear(); tx.extra.clear(); @@ -89,7 +89,7 @@ namespace cryptonote in.height = height; uint64_t block_reward; - if(!get_block_reward(median_size, current_block_size, already_generated_coins, block_reward, hard_fork_version)) + if(!get_block_reward(median_weight, current_block_weight, already_generated_coins, block_reward, hard_fork_version)) { LOG_PRINT_L0("Block is too big"); return false; @@ -195,7 +195,7 @@ namespace cryptonote return addr.m_view_public_key; } //--------------------------------------------------------------- - bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct, bool bulletproof, rct::multisig_out *msout, bool shuffle_outs) + bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct, rct::RangeProofType range_proof_type, rct::multisig_out *msout, bool shuffle_outs) { hw::device &hwdev = sender_account_keys.get_device(); @@ -491,7 +491,7 @@ namespace cryptonote // the non-simple version is slightly smaller, but assumes all real inputs // are on the same index, so can only be used if there just one ring. - bool use_simple_rct = sources.size() > 1; + bool use_simple_rct = sources.size() > 1 || range_proof_type != rct::RangeProofBorromean; if (!use_simple_rct) { @@ -516,6 +516,7 @@ namespace cryptonote uint64_t amount_in = 0, amount_out = 0; rct::ctkeyV inSk; + inSk.reserve(sources.size()); // mixRing indexing is done the other way round for simple rct::ctkeyM mixRing(use_simple_rct ? sources.size() : n_total_outs); rct::keyV destinations; @@ -532,6 +533,7 @@ namespace cryptonote ctkey.dest = rct::sk2rct(in_contexts[i].in_ephemeral.sec); ctkey.mask = sources[i].mask; inSk.push_back(ctkey); + memwipe(&ctkey, sizeof(rct::ctkey)); // inPk: (public key, commitment) // will be done when filling in mixRing if (msout) @@ -587,9 +589,10 @@ namespace cryptonote get_transaction_prefix_hash(tx, tx_prefix_hash); rct::ctkeyV outSk; if (use_simple_rct) - tx.rct_signatures = rct::genRctSimple(rct::hash2rct(tx_prefix_hash), inSk, destinations, inamounts, outamounts, amount_in - amount_out, mixRing, amount_keys, msout ? &kLRki : NULL, msout, index, outSk, bulletproof, hwdev); + tx.rct_signatures = rct::genRctSimple(rct::hash2rct(tx_prefix_hash), inSk, destinations, inamounts, outamounts, amount_in - amount_out, mixRing, amount_keys, msout ? &kLRki : NULL, msout, index, outSk, range_proof_type, hwdev); else - tx.rct_signatures = rct::genRct(rct::hash2rct(tx_prefix_hash), inSk, destinations, outamounts, mixRing, amount_keys, msout ? &kLRki[0] : NULL, msout, sources[0].real_output, outSk, bulletproof, hwdev); // same index assumption + tx.rct_signatures = rct::genRct(rct::hash2rct(tx_prefix_hash), inSk, destinations, outamounts, mixRing, amount_keys, msout ? &kLRki[0] : NULL, msout, sources[0].real_output, outSk, hwdev); // same index assumption + memwipe(inSk.data(), inSk.size() * sizeof(rct::ctkey)); CHECK_AND_ASSERT_MES(tx.vout.size() == outSk.size(), false, "outSk size does not match vout"); @@ -601,7 +604,7 @@ namespace cryptonote return true; } //--------------------------------------------------------------- - bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys, bool rct, bool bulletproof, rct::multisig_out *msout) + bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys, bool rct, rct::RangeProofType range_proof_type, rct::multisig_out *msout) { hw::device &hwdev = sender_account_keys.get_device(); hwdev.open_tx(tx_key); @@ -619,7 +622,7 @@ namespace cryptonote additional_tx_keys.push_back(keypair::generate(sender_account_keys.get_device()).sec); } - bool r = construct_tx_with_tx_key(sender_account_keys, subaddresses, sources, destinations, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, rct, bulletproof, msout); + bool r = construct_tx_with_tx_key(sender_account_keys, subaddresses, sources, destinations, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, rct, range_proof_type, msout); hwdev.close_tx(); return r; } @@ -631,7 +634,7 @@ namespace cryptonote crypto::secret_key tx_key; std::vector<crypto::secret_key> additional_tx_keys; std::vector<tx_destination_entry> destinations_copy = destinations; - return construct_tx_and_get_tx_key(sender_account_keys, subaddresses, sources, destinations_copy, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, false, NULL); + return construct_tx_and_get_tx_key(sender_account_keys, subaddresses, sources, destinations_copy, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, rct::RangeProofBorromean, NULL); } //--------------------------------------------------------------- bool generate_genesis_block( diff --git a/src/cryptonote_core/cryptonote_tx_utils.h b/src/cryptonote_core/cryptonote_tx_utils.h index a5d149fca..f2cf7b6ca 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.h +++ b/src/cryptonote_core/cryptonote_tx_utils.h @@ -37,7 +37,7 @@ namespace cryptonote { //--------------------------------------------------------------- - bool construct_miner_tx(size_t height, size_t median_size, uint64_t already_generated_coins, size_t current_block_size, uint64_t fee, const account_public_address &miner_address, transaction& tx, const blobdata& extra_nonce = blobdata(), size_t max_outs = 999, uint8_t hard_fork_version = 1); + bool construct_miner_tx(size_t height, size_t median_weight, uint64_t already_generated_coins, size_t current_block_weight, uint64_t fee, const account_public_address &miner_address, transaction& tx, const blobdata& extra_nonce = blobdata(), size_t max_outs = 999, uint8_t hard_fork_version = 1); struct tx_source_entry { @@ -90,8 +90,8 @@ namespace cryptonote //--------------------------------------------------------------- crypto::public_key get_destination_view_key_pub(const std::vector<tx_destination_entry> &destinations, const boost::optional<cryptonote::account_public_address>& change_addr); bool construct_tx(const account_keys& sender_account_keys, std::vector<tx_source_entry> &sources, const std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time); - bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, bool bulletproof = false, rct::multisig_out *msout = NULL, bool shuffle_outs = true); - bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, bool bulletproof = false, rct::multisig_out *msout = NULL); + bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, rct::RangeProofType range_proof_type = rct::RangeProofBorromean, rct::multisig_out *msout = NULL, bool shuffle_outs = true); + bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, rct::RangeProofType range_proof_type = rct::RangeProofBorromean, rct::multisig_out *msout = NULL); bool generate_genesis_block( block& bl diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index eac0f1f57..b12a329bb 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -80,9 +80,13 @@ namespace cryptonote return amount * ACCEPT_THRESHOLD; } - uint64_t get_transaction_size_limit(uint8_t version) + uint64_t get_transaction_weight_limit(uint8_t version) { - return get_min_block_size(version) - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; + // from v8, limit a tx to 50% of the minimum block weight + if (version >= 8) + return get_min_block_weight(version) / 2 - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; + else + return get_min_block_weight(version) - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; } // This class is meant to create a batch when none currently exists. @@ -102,12 +106,12 @@ namespace cryptonote } //--------------------------------------------------------------------------------- //--------------------------------------------------------------------------------- - tx_memory_pool::tx_memory_pool(Blockchain& bchs): m_blockchain(bchs), m_txpool_max_size(DEFAULT_TXPOOL_MAX_SIZE), m_txpool_size(0) + tx_memory_pool::tx_memory_pool(Blockchain& bchs): m_blockchain(bchs), m_txpool_max_weight(DEFAULT_TXPOOL_MAX_WEIGHT), m_txpool_weight(0), m_cookie(0) { } //--------------------------------------------------------------------------------- - bool tx_memory_pool::add_tx(transaction &tx, /*const crypto::hash& tx_prefix_hash,*/ const crypto::hash &id, size_t blob_size, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version) + bool tx_memory_pool::add_tx(transaction &tx, /*const crypto::hash& tx_prefix_hash,*/ const crypto::hash &id, size_t tx_weight, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version) { // this should already be called with that lock, but let's make it explicit for clarity CRITICAL_REGION_LOCAL(m_transactions_lock); @@ -173,17 +177,17 @@ namespace cryptonote fee = tx.rct_signatures.txnFee; } - if (!kept_by_block && !m_blockchain.check_fee(blob_size, fee)) + if (!kept_by_block && !m_blockchain.check_fee(tx_weight, fee)) { tvc.m_verifivation_failed = true; tvc.m_fee_too_low = true; return false; } - size_t tx_size_limit = get_transaction_size_limit(version); - if (!kept_by_block && blob_size > tx_size_limit) + size_t tx_weight_limit = get_transaction_weight_limit(version); + if (!kept_by_block && tx_weight > tx_weight_limit) { - LOG_PRINT_L1("transaction is too big: " << blob_size << " bytes, maximum size: " << tx_size_limit); + LOG_PRINT_L1("transaction is too heavy: " << tx_weight << " bytes, maximum weight: " << tx_weight_limit); tvc.m_verifivation_failed = true; tvc.m_too_big = true; return false; @@ -227,7 +231,7 @@ namespace cryptonote // may become valid again, so ignore the failed inputs check. if(kept_by_block) { - meta.blob_size = blob_size; + meta.weight = tx_weight; meta.fee = fee; meta.max_used_block_id = null_hash; meta.max_used_block_height = 0; @@ -248,7 +252,7 @@ namespace cryptonote m_blockchain.add_txpool_tx(tx, meta); if (!insert_key_images(tx, kept_by_block)) return false; - m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)blob_size, receive_time), id); + m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)tx_weight, receive_time), id); } catch (const std::exception &e) { @@ -267,7 +271,7 @@ namespace cryptonote }else { //update transactions container - meta.blob_size = blob_size; + meta.weight = tx_weight; meta.kept_by_block = kept_by_block; meta.fee = fee; meta.max_used_block_id = max_used_block_id; @@ -290,7 +294,7 @@ namespace cryptonote m_blockchain.add_txpool_tx(tx, meta); if (!insert_key_images(tx, kept_by_block)) return false; - m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)blob_size, receive_time), id); + m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)tx_weight, receive_time), id); } catch (const std::exception &e) { @@ -304,11 +308,13 @@ namespace cryptonote } tvc.m_verifivation_failed = false; - m_txpool_size += blob_size; + m_txpool_weight += tx_weight; + + ++m_cookie; - MINFO("Transaction added to pool: txid " << id << " bytes: " << blob_size << " fee/byte: " << (fee / (double)blob_size)); + MINFO("Transaction added to pool: txid " << id << " weight: " << tx_weight << " fee/byte: " << (fee / (double)tx_weight)); - prune(m_txpool_max_size); + prune(m_txpool_max_weight); return true; } @@ -319,34 +325,35 @@ namespace cryptonote size_t blob_size = 0; if (!get_transaction_hash(tx, h, blob_size) || blob_size == 0) return false; - return add_tx(tx, h, blob_size, tvc, keeped_by_block, relayed, do_not_relay, version); + return add_tx(tx, h, get_transaction_weight(tx, blob_size), tvc, keeped_by_block, relayed, do_not_relay, version); } //--------------------------------------------------------------------------------- - size_t tx_memory_pool::get_txpool_size() const + size_t tx_memory_pool::get_txpool_weight() const { CRITICAL_REGION_LOCAL(m_transactions_lock); - return m_txpool_size; + return m_txpool_weight; } //--------------------------------------------------------------------------------- - void tx_memory_pool::set_txpool_max_size(size_t bytes) + void tx_memory_pool::set_txpool_max_weight(size_t bytes) { CRITICAL_REGION_LOCAL(m_transactions_lock); - m_txpool_max_size = bytes; + m_txpool_max_weight = bytes; } //--------------------------------------------------------------------------------- void tx_memory_pool::prune(size_t bytes) { CRITICAL_REGION_LOCAL(m_transactions_lock); if (bytes == 0) - bytes = m_txpool_max_size; + bytes = m_txpool_max_weight; CRITICAL_REGION_LOCAL1(m_blockchain); LockedTXN lock(m_blockchain); + bool changed = false; // this will never remove the first one, but we don't care auto it = --m_txs_by_fee_and_receive_time.end(); while (it != m_txs_by_fee_and_receive_time.begin()) { - if (m_txpool_size <= bytes) + if (m_txpool_weight <= bytes) break; try { @@ -371,12 +378,13 @@ namespace cryptonote return; } // remove first, in case this throws, so key images aren't removed - MINFO("Pruning tx " << txid << " from txpool: size: " << it->first.second << ", fee/byte: " << it->first.first); + MINFO("Pruning tx " << txid << " from txpool: weight: " << it->first.second << ", fee/byte: " << it->first.first); m_blockchain.remove_txpool_tx(txid); - m_txpool_size -= txblob.size(); + m_txpool_weight -= it->first.second; remove_transaction_keyimages(tx); - MINFO("Pruned tx " << txid << " from txpool: size: " << it->first.second << ", fee/byte: " << it->first.first); + MINFO("Pruned tx " << txid << " from txpool: weight: " << it->first.second << ", fee/byte: " << it->first.first); m_txs_by_fee_and_receive_time.erase(it--); + changed = true; } catch (const std::exception &e) { @@ -384,8 +392,10 @@ namespace cryptonote return; } } - if (m_txpool_size > bytes) - MINFO("Pool size after pruning is larger than limit: " << m_txpool_size << "/" << bytes); + if (changed) + ++m_cookie; + if (m_txpool_weight > bytes) + MINFO("Pool weight after pruning is larger than limit: " << m_txpool_weight << "/" << bytes); } //--------------------------------------------------------------------------------- bool tx_memory_pool::insert_key_images(const transaction &tx, bool kept_by_block) @@ -401,6 +411,7 @@ namespace cryptonote auto ins_res = kei_image_set.insert(id); CHECK_AND_ASSERT_MES(ins_res.second, false, "internal error: try to insert duplicate iterator in key_image set"); } + ++m_cookie; return true; } //--------------------------------------------------------------------------------- @@ -435,10 +446,11 @@ namespace cryptonote } } + ++m_cookie; return true; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::take_tx(const crypto::hash &id, transaction &tx, size_t& blob_size, uint64_t& fee, bool &relayed, bool &do_not_relay, bool &double_spend_seen) + bool tx_memory_pool::take_tx(const crypto::hash &id, transaction &tx, size_t& tx_weight, uint64_t& fee, bool &relayed, bool &do_not_relay, bool &double_spend_seen) { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); @@ -462,7 +474,7 @@ namespace cryptonote MERROR("Failed to parse tx from txpool"); return false; } - blob_size = meta.blob_size; + tx_weight = meta.weight; fee = meta.fee; relayed = meta.relayed; do_not_relay = meta.do_not_relay; @@ -470,7 +482,7 @@ namespace cryptonote // remove first, in case this throws, so key images aren't removed m_blockchain.remove_txpool_tx(id); - m_txpool_size -= blob_size; + m_txpool_weight -= tx_weight; remove_transaction_keyimages(tx); } catch (const std::exception &e) @@ -480,6 +492,7 @@ namespace cryptonote } m_txs_by_fee_and_receive_time.erase(sorted_it); + ++m_cookie; return true; } //--------------------------------------------------------------------------------- @@ -543,7 +556,7 @@ namespace cryptonote { // remove first, so we only remove key images if the tx removal succeeds m_blockchain.remove_txpool_tx(txid); - m_txpool_size -= bd.size(); + m_txpool_weight -= get_transaction_weight(tx, bd.size()); remove_transaction_keyimages(tx); } } @@ -553,6 +566,7 @@ namespace cryptonote // ignore error } } + ++m_cookie; } return true; } @@ -660,7 +674,7 @@ namespace cryptonote const uint64_t now = time(NULL); backlog.reserve(m_blockchain.get_txpool_tx_count(include_unrelayed_txes)); m_blockchain.for_all_txpool_txes([&backlog, now](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ - backlog.push_back({meta.blob_size, meta.fee, meta.receive_time - now}); + backlog.push_back({meta.weight, meta.fee, meta.receive_time - now}); return true; }, false, include_unrelayed_txes); } @@ -672,15 +686,15 @@ namespace cryptonote const uint64_t now = time(NULL); std::map<uint64_t, txpool_histo> agebytes; stats.txs_total = m_blockchain.get_txpool_tx_count(include_unrelayed_txes); - std::vector<uint32_t> sizes; - sizes.reserve(stats.txs_total); - m_blockchain.for_all_txpool_txes([&stats, &sizes, now, &agebytes](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ - sizes.push_back(meta.blob_size); - stats.bytes_total += meta.blob_size; - if (!stats.bytes_min || meta.blob_size < stats.bytes_min) - stats.bytes_min = meta.blob_size; - if (meta.blob_size > stats.bytes_max) - stats.bytes_max = meta.blob_size; + std::vector<uint32_t> weights; + weights.reserve(stats.txs_total); + m_blockchain.for_all_txpool_txes([&stats, &weights, now, &agebytes](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ + weights.push_back(meta.weight); + stats.bytes_total += meta.weight; + if (!stats.bytes_min || meta.weight < stats.bytes_min) + stats.bytes_min = meta.weight; + if (meta.weight > stats.bytes_max) + stats.bytes_max = meta.weight; if (!meta.relayed) stats.num_not_relayed++; stats.fee_total += meta.fee; @@ -692,12 +706,12 @@ namespace cryptonote stats.num_failing++; uint64_t age = now - meta.receive_time + (now == meta.receive_time); agebytes[age].txs++; - agebytes[age].bytes += meta.blob_size; + agebytes[age].bytes += meta.weight; if (meta.double_spend_seen) ++stats.num_double_spends; return true; }, false, include_unrelayed_txes); - stats.bytes_med = epee::misc_utils::median(sizes); + stats.bytes_med = epee::misc_utils::median(weights); if (stats.txs_total > 1) { /* looking for 98th percentile */ @@ -761,7 +775,8 @@ namespace cryptonote return true; } txi.tx_json = obj_to_json_str(tx); - txi.blob_size = meta.blob_size; + txi.blob_size = bd->size(); + txi.weight = meta.weight; txi.fee = meta.fee; txi.kept_by_block = meta.kept_by_block; txi.max_used_block_height = meta.max_used_block_height; @@ -832,7 +847,8 @@ namespace cryptonote return true; } txi.tx = tx; - txi.blob_size = meta.blob_size; + txi.blob_size = bd->size(); + txi.weight = meta.weight; txi.fee = meta.fee; txi.kept_by_block = meta.kept_by_block; txi.max_used_block_height = meta.max_used_block_height; @@ -1051,6 +1067,7 @@ namespace cryptonote { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); + bool changed = false; LockedTXN lock(m_blockchain); for(size_t i = 0; i!= tx.vin.size(); i++) { @@ -1071,6 +1088,7 @@ namespace cryptonote { MDEBUG("Marking " << txid << " as double spending " << itk.k_image); meta.double_spend_seen = true; + changed = true; try { m_blockchain.update_txpool_tx(txid, meta); @@ -1084,6 +1102,8 @@ namespace cryptonote } } } + if (changed) + ++m_cookie; } //--------------------------------------------------------------------------------- std::string tx_memory_pool::print_pool(bool short_format) const @@ -1102,7 +1122,8 @@ namespace cryptonote } ss << obj_to_json_str(tx) << std::endl; } - ss << "blob_size: " << meta.blob_size << std::endl + ss << "blob_size: " << (short_format ? "-" : std::to_string(txblob->size())) << std::endl + << "weight: " << meta.weight << std::endl << "fee: " << print_money(meta.fee) << std::endl << "kept_by_block: " << (meta.kept_by_block ? 'T' : 'F') << std::endl << "double_spend_seen: " << (meta.double_spend_seen ? 'T' : 'F') << std::endl @@ -1117,30 +1138,30 @@ namespace cryptonote } //--------------------------------------------------------------------------------- //TODO: investigate whether boolean return is appropriate - bool tx_memory_pool::fill_block_template(block &bl, size_t median_size, uint64_t already_generated_coins, size_t &total_size, uint64_t &fee, uint64_t &expected_reward, uint8_t version) + bool tx_memory_pool::fill_block_template(block &bl, size_t median_weight, uint64_t already_generated_coins, size_t &total_weight, uint64_t &fee, uint64_t &expected_reward, uint8_t version) { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); uint64_t best_coinbase = 0, coinbase = 0; - total_size = 0; + total_weight = 0; fee = 0; //baseline empty block - get_block_reward(median_size, total_size, already_generated_coins, best_coinbase, version); + get_block_reward(median_weight, total_weight, already_generated_coins, best_coinbase, version); - size_t max_total_size_pre_v5 = (130 * median_size) / 100 - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; - size_t max_total_size_v5 = 2 * median_size - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; - size_t max_total_size = version >= 5 ? max_total_size_v5 : max_total_size_pre_v5; + size_t max_total_weight_pre_v5 = (130 * median_weight) / 100 - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; + size_t max_total_weight_v5 = 2 * median_weight - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; + size_t max_total_weight = version >= 5 ? max_total_weight_v5 : max_total_weight_pre_v5; std::unordered_set<crypto::key_image> k_images; - LOG_PRINT_L2("Filling block template, median size " << median_size << ", " << m_txs_by_fee_and_receive_time.size() << " txes in the pool"); + LOG_PRINT_L2("Filling block template, median weight " << median_weight << ", " << m_txs_by_fee_and_receive_time.size() << " txes in the pool"); LockedTXN lock(m_blockchain); auto sorted_it = m_txs_by_fee_and_receive_time.begin(); - while (sorted_it != m_txs_by_fee_and_receive_time.end()) + for (; sorted_it != m_txs_by_fee_and_receive_time.end(); ++sorted_it) { txpool_tx_meta_t meta; if (!m_blockchain.get_txpool_tx_meta(sorted_it->second, meta)) @@ -1148,13 +1169,12 @@ namespace cryptonote MERROR(" failed to find tx meta"); continue; } - LOG_PRINT_L2("Considering " << sorted_it->second << ", size " << meta.blob_size << ", current block size " << total_size << "/" << max_total_size << ", current coinbase " << print_money(best_coinbase)); + LOG_PRINT_L2("Considering " << sorted_it->second << ", weight " << meta.weight << ", current block weight " << total_weight << "/" << max_total_weight << ", current coinbase " << print_money(best_coinbase)); - // Can not exceed maximum block size - if (max_total_size < total_size + meta.blob_size) + // Can not exceed maximum block weight + if (max_total_weight < total_weight + meta.weight) { - LOG_PRINT_L2(" would exceed maximum block size"); - sorted_it++; + LOG_PRINT_L2(" would exceed maximum block weight"); continue; } @@ -1164,27 +1184,25 @@ namespace cryptonote // If we're getting lower coinbase tx, // stop including more tx uint64_t block_reward; - if(!get_block_reward(median_size, total_size + meta.blob_size, already_generated_coins, block_reward, version)) + if(!get_block_reward(median_weight, total_weight + meta.weight, already_generated_coins, block_reward, version)) { - LOG_PRINT_L2(" would exceed maximum block size"); - sorted_it++; + LOG_PRINT_L2(" would exceed maximum block weight"); continue; } coinbase = block_reward + fee + meta.fee; if (coinbase < template_accept_threshold(best_coinbase)) { LOG_PRINT_L2(" would decrease coinbase to " << print_money(coinbase)); - sorted_it++; continue; } } else { - // If we've exceeded the penalty free size, + // If we've exceeded the penalty free weight, // stop including more tx - if (total_size > median_size) + if (total_weight > median_weight) { - LOG_PRINT_L2(" would exceed median block size"); + LOG_PRINT_L2(" would exceed median block weight"); break; } } @@ -1221,28 +1239,25 @@ namespace cryptonote if (!ready) { LOG_PRINT_L2(" not ready to go"); - sorted_it++; continue; } if (have_key_images(k_images, tx)) { LOG_PRINT_L2(" key images already seen"); - sorted_it++; continue; } bl.tx_hashes.push_back(sorted_it->second); - total_size += meta.blob_size; + total_weight += meta.weight; fee += meta.fee; best_coinbase = coinbase; append_key_images(k_images, tx); - sorted_it++; - LOG_PRINT_L2(" added, new block size " << total_size << "/" << max_total_size << ", coinbase " << print_money(best_coinbase)); + LOG_PRINT_L2(" added, new block weight " << total_weight << "/" << max_total_weight << ", coinbase " << print_money(best_coinbase)); } expected_reward = best_coinbase; - LOG_PRINT_L2("Block template filled with " << bl.tx_hashes.size() << " txes, size " - << total_size << "/" << max_total_size << ", coinbase " << print_money(best_coinbase) + LOG_PRINT_L2("Block template filled with " << bl.tx_hashes.size() << " txes, weight " + << total_weight << "/" << max_total_weight << ", coinbase " << print_money(best_coinbase) << " (including " << print_money(fee) << " in fees)"); return true; } @@ -1251,14 +1266,14 @@ namespace cryptonote { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); - size_t tx_size_limit = get_transaction_size_limit(version); + size_t tx_weight_limit = get_transaction_weight_limit(version); std::unordered_set<crypto::hash> remove; - m_txpool_size = 0; - m_blockchain.for_all_txpool_txes([this, &remove, tx_size_limit](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata*) { - m_txpool_size += meta.blob_size; - if (meta.blob_size > tx_size_limit) { - LOG_PRINT_L1("Transaction " << txid << " is too big (" << meta.blob_size << " bytes), removing it from pool"); + m_txpool_weight = 0; + m_blockchain.for_all_txpool_txes([this, &remove, tx_weight_limit](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata*) { + m_txpool_weight += meta.weight; + if (meta.weight > tx_weight_limit) { + LOG_PRINT_L1("Transaction " << txid << " is too big (" << meta.weight << " bytes), removing it from pool"); remove.insert(txid); } else if (m_blockchain.have_tx(txid)) { @@ -1285,7 +1300,7 @@ namespace cryptonote } // remove tx from db first m_blockchain.remove_txpool_tx(txid); - m_txpool_size -= txblob.size(); + m_txpool_weight -= get_transaction_weight(tx, txblob.size()); remove_transaction_keyimages(tx); auto sorted_it = find_tx_in_sorted_container(txid); if (sorted_it == m_txs_by_fee_and_receive_time.end()) @@ -1305,18 +1320,20 @@ namespace cryptonote } } } + if (n_removed > 0) + ++m_cookie; return n_removed; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::init(size_t max_txpool_size) + bool tx_memory_pool::init(size_t max_txpool_weight) { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); - m_txpool_max_size = max_txpool_size ? max_txpool_size : DEFAULT_TXPOOL_MAX_SIZE; + m_txpool_max_weight = max_txpool_weight ? max_txpool_weight : DEFAULT_TXPOOL_MAX_WEIGHT; m_txs_by_fee_and_receive_time.clear(); m_spent_key_images.clear(); - m_txpool_size = 0; + m_txpool_weight = 0; std::vector<crypto::hash> remove; // first add the not kept by block, then the kept by block, @@ -1338,8 +1355,8 @@ namespace cryptonote MFATAL("Failed to insert key images from txpool tx"); return false; } - m_txs_by_fee_and_receive_time.emplace(std::pair<double, time_t>(meta.fee / (double)meta.blob_size, meta.receive_time), txid); - m_txpool_size += meta.blob_size; + m_txs_by_fee_and_receive_time.emplace(std::pair<double, time_t>(meta.fee / (double)meta.weight, meta.receive_time), txid); + m_txpool_weight += meta.weight; return true; }, true); if (!r) @@ -1361,6 +1378,10 @@ namespace cryptonote } } } + + m_cookie = 0; + + // Ignore deserialization error return true; } diff --git a/src/cryptonote_core/tx_pool.h b/src/cryptonote_core/tx_pool.h index 4ade7ddbe..892cadc69 100644 --- a/src/cryptonote_core/tx_pool.h +++ b/src/cryptonote_core/tx_pool.h @@ -84,7 +84,7 @@ namespace cryptonote * * This handling includes: * storing the transactions - * organizing the transactions by fee per size + * organizing the transactions by fee per weight unit * taking/giving transactions to and from various other components * saving the transactions to disk on shutdown * helping create a new block template by choosing transactions for it @@ -105,9 +105,9 @@ namespace cryptonote * @copydoc add_tx(transaction&, tx_verification_context&, bool, bool, uint8_t) * * @param id the transaction's hash - * @param blob_size the transaction's size + * @param tx_weight the transaction's weight */ - bool add_tx(transaction &tx, const crypto::hash &id, size_t blob_size, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version); + bool add_tx(transaction &tx, const crypto::hash &id, size_t tx_weight, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version); /** * @brief add a transaction to the transaction pool @@ -133,7 +133,7 @@ namespace cryptonote * * @param id the hash of the transaction * @param tx return-by-reference the transaction taken - * @param blob_size return-by-reference the transaction's size + * @param tx_weight return-by-reference the transaction's weight * @param fee the transaction fee * @param relayed return-by-reference was transaction relayed to us by the network? * @param do_not_relay return-by-reference is transaction not to be relayed to the network? @@ -141,7 +141,7 @@ namespace cryptonote * * @return true unless the transaction cannot be found in the pool */ - bool take_tx(const crypto::hash &id, transaction &tx, size_t& blob_size, uint64_t& fee, bool &relayed, bool &do_not_relay, bool &double_spend_seen); + bool take_tx(const crypto::hash &id, transaction &tx, size_t& tx_weight, uint64_t& fee, bool &relayed, bool &do_not_relay, bool &double_spend_seen); /** * @brief checks if the pool has a transaction with the given hash @@ -198,11 +198,11 @@ namespace cryptonote /** * @brief loads pool state (if any) from disk, and initializes pool * - * @param max_txpool_size the max size in bytes + * @param max_txpool_weight the max weight in bytes * * @return true */ - bool init(size_t max_txpool_size = 0); + bool init(size_t max_txpool_weight = 0); /** * @brief attempts to save the transaction pool state to disk @@ -219,16 +219,16 @@ namespace cryptonote * @brief Chooses transactions for a block to include * * @param bl return-by-reference the block to fill in with transactions - * @param median_size the current median block size + * @param median_weight the current median block weight * @param already_generated_coins the current total number of coins "minted" - * @param total_size return-by-reference the total size of the new block + * @param total_weight return-by-reference the total weight of the new block * @param fee return-by-reference the total of fees from the included transactions * @param expected_reward return-by-reference the total reward awarded to the miner finding this block, including transaction fees * @param version hard fork version to use for consensus rules * * @return true */ - bool fill_block_template(block &bl, size_t median_size, uint64_t already_generated_coins, size_t &total_size, uint64_t &fee, uint64_t &expected_reward, uint8_t version); + bool fill_block_template(block &bl, size_t median_weight, uint64_t already_generated_coins, size_t &total_weight, uint64_t &fee, uint64_t &expected_reward, uint8_t version); /** * @brief get a list of all transactions in the pool @@ -249,7 +249,7 @@ namespace cryptonote void get_transaction_hashes(std::vector<crypto::hash>& txs, bool include_unrelayed_txes = true) const; /** - * @brief get (size, fee, receive time) for all transaction in the pool + * @brief get (weight, fee, receive time) for all transaction in the pool * * @param txs return-by-reference that data * @param include_unrelayed_txes include unrelayed txes in the result @@ -362,22 +362,29 @@ namespace cryptonote */ size_t validate(uint8_t version); + /** + * @brief return the cookie + * + * @return the cookie + */ + uint64_t cookie() const { return m_cookie; } + /** - * @brief get the cumulative txpool size in bytes + * @brief get the cumulative txpool weight in bytes * - * @return the cumulative txpool size in bytes + * @return the cumulative txpool weight in bytes */ - size_t get_txpool_size() const; + size_t get_txpool_weight() const; /** - * @brief set the max cumulative txpool size in bytes + * @brief set the max cumulative txpool weight in bytes * - * @param bytes the max cumulative txpool size in bytes + * @param bytes the max cumulative txpool weight in bytes */ - void set_txpool_max_size(size_t bytes); + void set_txpool_max_weight(size_t bytes); #define CURRENT_MEMPOOL_ARCHIVE_VER 11 -#define CURRENT_MEMPOOL_TX_DETAILS_ARCHIVE_VER 12 +#define CURRENT_MEMPOOL_TX_DETAILS_ARCHIVE_VER 13 /** * @brief information about a single transaction @@ -386,6 +393,7 @@ namespace cryptonote { transaction tx; //!< the transaction size_t blob_size; //!< the transaction's size + size_t weight; //!< the transaction's weight uint64_t fee; //!< the transaction's fee amount crypto::hash max_used_block_id; //!< the hash of the highest block referenced by an input uint64_t max_used_block_height; //!< the height of the highest block referenced by an input @@ -515,7 +523,7 @@ namespace cryptonote /** * @brief prune lowest fee/byte txes till we're not above bytes * - * if bytes is 0, use m_txpool_max_size + * if bytes is 0, use m_txpool_max_weight */ void prune(size_t bytes = 0); @@ -549,6 +557,8 @@ private: //!< container for transactions organized by fee per size and receive time sorted_tx_container m_txs_by_fee_and_receive_time; + std::atomic<uint64_t> m_cookie; //!< incremented at each change + /** * @brief get an iterator to a transaction in the sorted container * @@ -569,8 +579,8 @@ private: Blockchain& m_blockchain; //!< reference to the Blockchain object - size_t m_txpool_max_size; - size_t m_txpool_size; + size_t m_txpool_max_weight; + size_t m_txpool_weight; mutable std::unordered_map<crypto::hash, std::tuple<bool, tx_verification_context, uint64_t, crypto::hash>> m_input_cache; }; @@ -599,6 +609,9 @@ namespace boost if (version < 12) return; ar & td.do_not_relay; + if (version < 13) + return; + ar & td.weight; } } } diff --git a/src/cryptonote_protocol/block_queue.cpp b/src/cryptonote_protocol/block_queue.cpp index c39d67ceb..05f4189fb 100644 --- a/src/cryptonote_protocol/block_queue.cpp +++ b/src/cryptonote_protocol/block_queue.cpp @@ -57,7 +57,11 @@ void block_queue::add_blocks(uint64_t height, std::vector<cryptonote::block_comp bool has_hashes = remove_span(height, &hashes); blocks.insert(span(height, std::move(bcel), connection_id, rate, size)); if (has_hashes) + { + for (const crypto::hash &h: hashes) + requested_hashes.insert(h); set_span_hashes(height, connection_id, hashes); + } } void block_queue::add_blocks(uint64_t height, uint64_t nblocks, const boost::uuids::uuid &connection_id, boost::posix_time::ptime time) @@ -76,11 +80,19 @@ void block_queue::flush_spans(const boost::uuids::uuid &connection_id, bool all) block_map::iterator j = i++; if (j->connection_id == connection_id && (all || j->blocks.size() == 0)) { - blocks.erase(j); + erase_block(j); } } } +void block_queue::erase_block(block_map::iterator j) +{ + CHECK_AND_ASSERT_THROW_MES(j != blocks.end(), "Invalid iterator"); + for (const crypto::hash &h: j->hashes) + requested_hashes.erase(h); + blocks.erase(j); +} + void block_queue::flush_stale_spans(const std::set<boost::uuids::uuid> &live_connections) { boost::unique_lock<boost::recursive_mutex> lock(mutex); @@ -92,7 +104,7 @@ void block_queue::flush_stale_spans(const std::set<boost::uuids::uuid> &live_con block_map::iterator j = i++; if (live_connections.find(j->connection_id) == live_connections.end() && j->blocks.size() == 0) { - blocks.erase(j); + erase_block(j); } } } @@ -106,7 +118,7 @@ bool block_queue::remove_span(uint64_t start_block_height, std::vector<crypto::h { if (hashes) *hashes = std::move(i->hashes); - blocks.erase(i); + erase_block(i); return true; } } @@ -121,7 +133,7 @@ void block_queue::remove_spans(const boost::uuids::uuid &connection_id, uint64_t block_map::iterator j = i++; if (j->connection_id == connection_id && j->start_block_height <= start_block_height) { - blocks.erase(j); + erase_block(j); } } } @@ -160,16 +172,15 @@ std::string block_queue::get_overview() const return s; } +inline bool block_queue::requested_internal(const crypto::hash &hash) const +{ + return requested_hashes.find(hash) != requested_hashes.end(); +} + bool block_queue::requested(const crypto::hash &hash) const { boost::unique_lock<boost::recursive_mutex> lock(mutex); - for (const auto &span: blocks) - { - for (const auto &h: span.hashes) - if (h == hash) - return true; - } - return false; + return requested_internal(hash); } std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, const std::vector<crypto::hash> &block_hashes, boost::posix_time::ptime time) @@ -184,7 +195,7 @@ std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_hei uint64_t span_start_height = last_block_height - block_hashes.size() + 1; std::vector<crypto::hash>::const_iterator i = block_hashes.begin(); - while (i != block_hashes.end() && requested(*i)) + while (i != block_hashes.end() && requested_internal(*i)) { ++i; ++span_start_height; @@ -256,8 +267,10 @@ void block_queue::set_span_hashes(uint64_t start_height, const boost::uuids::uui if (i->start_block_height == start_height && i->connection_id == connection_id) { span s = *i; - blocks.erase(i); + erase_block(i); s.hashes = std::move(hashes); + for (const crypto::hash &h: s.hashes) + requested_hashes.insert(h); blocks.insert(s); return; } diff --git a/src/cryptonote_protocol/block_queue.h b/src/cryptonote_protocol/block_queue.h index 9059e89ac..9cce95075 100644 --- a/src/cryptonote_protocol/block_queue.h +++ b/src/cryptonote_protocol/block_queue.h @@ -33,6 +33,7 @@ #include <string> #include <vector> #include <set> +#include <unordered_set> #include <boost/thread/recursive_mutex.hpp> #include <boost/uuid/uuid.hpp> @@ -93,7 +94,12 @@ namespace cryptonote bool requested(const crypto::hash &hash) const; private: + void erase_block(block_map::iterator j); + inline bool requested_internal(const crypto::hash &hash) const; + + private: block_map blocks; mutable boost::recursive_mutex mutex; + std::unordered_set<crypto::hash> requested_hashes; }; } diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index a931d3b57..c2c660e8c 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -1683,11 +1683,6 @@ skip: fluffy_arg.b = arg.b; fluffy_arg.b.txs = fluffy_txs; - // pre-serialize them - std::string fullBlob, fluffyBlob; - epee::serialization::store_t_to_binary(arg, fullBlob); - epee::serialization::store_t_to_binary(fluffy_arg, fluffyBlob); - // sort peers between fluffy ones and others std::list<boost::uuids::uuid> fullConnections, fluffyConnections; m_p2p->for_each_connection([this, &exclude_context, &fullConnections, &fluffyConnections](connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags) @@ -1709,8 +1704,18 @@ skip: }); // send fluffy ones first, we want to encourage people to run that - m_p2p->relay_notify_to_list(NOTIFY_NEW_FLUFFY_BLOCK::ID, fluffyBlob, fluffyConnections); - m_p2p->relay_notify_to_list(NOTIFY_NEW_BLOCK::ID, fullBlob, fullConnections); + if (!fluffyConnections.empty()) + { + std::string fluffyBlob; + epee::serialization::store_t_to_binary(fluffy_arg, fluffyBlob); + m_p2p->relay_notify_to_list(NOTIFY_NEW_FLUFFY_BLOCK::ID, fluffyBlob, fluffyConnections); + } + if (!fullConnections.empty()) + { + std::string fullBlob; + epee::serialization::store_t_to_binary(arg, fullBlob); + m_p2p->relay_notify_to_list(NOTIFY_NEW_BLOCK::ID, fullBlob, fullConnections); + } return true; } diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp index 45ba81e16..9ab1be246 100644 --- a/src/daemon/rpc_command_executor.cpp +++ b/src/daemon/rpc_command_executor.cpp @@ -76,6 +76,7 @@ namespace { << "difficulty: " << boost::lexical_cast<std::string>(header.difficulty) << std::endl << "POW hash: " << header.pow_hash << std::endl << "block size: " << header.block_size << std::endl + << "block weight: " << header.block_weight << std::endl << "num txes: " << header.num_txes << std::endl << "reward: " << cryptonote::print_money(header.reward); } @@ -558,7 +559,7 @@ bool t_rpc_command_executor::print_blockchain_info(uint64_t start_block_index, u std::cout << std::endl; std::cout << "height: " << header.height << ", timestamp: " << header.timestamp - << ", size: " << header.block_size << ", transactions: " << header.num_txes << std::endl + << ", size: " << header.block_size << ", weight: " << header.block_weight << ", transactions: " << header.num_txes << std::endl << "major version: " << (unsigned)header.major_version << ", minor version: " << (unsigned)header.minor_version << std::endl << "block id: " << header.hash << ", previous block id: " << header.prev_hash << std::endl << "difficulty: " << header.difficulty << ", nonce " << header.nonce << ", reward " << cryptonote::print_money(header.reward) << std::endl; @@ -857,8 +858,9 @@ bool t_rpc_command_executor::print_transaction_pool_long() { tools::msg_writer() << "id: " << tx_info.id_hash << std::endl << tx_info.tx_json << std::endl << "blob_size: " << tx_info.blob_size << std::endl + << "weight: " << tx_info.weight << std::endl << "fee: " << cryptonote::print_money(tx_info.fee) << std::endl - << "fee/byte: " << cryptonote::print_money(tx_info.fee / (double)tx_info.blob_size) << std::endl + << "fee/byte: " << cryptonote::print_money(tx_info.fee / (double)tx_info.weight) << std::endl << "receive_time: " << tx_info.receive_time << " (" << get_human_time_ago(tx_info.receive_time, now) << ")" << std::endl << "relayed: " << [&](const cryptonote::tx_info &tx_info)->std::string { if (!tx_info.relayed) return "no"; return boost::lexical_cast<std::string>(tx_info.last_relayed_time) + " (" + get_human_time_ago(tx_info.last_relayed_time, now) + ")"; } (tx_info) << std::endl << "do_not_relay: " << (tx_info.do_not_relay ? 'T' : 'F') << std::endl @@ -940,8 +942,9 @@ bool t_rpc_command_executor::print_transaction_pool_short() { { tools::msg_writer() << "id: " << tx_info.id_hash << std::endl << "blob_size: " << tx_info.blob_size << std::endl + << "weight: " << tx_info.weight << std::endl << "fee: " << cryptonote::print_money(tx_info.fee) << std::endl - << "fee/byte: " << cryptonote::print_money(tx_info.fee / (double)tx_info.blob_size) << std::endl + << "fee/byte: " << cryptonote::print_money(tx_info.fee / (double)tx_info.weight) << std::endl << "receive_time: " << tx_info.receive_time << " (" << get_human_time_ago(tx_info.receive_time, now) << ")" << std::endl << "relayed: " << [&](const cryptonote::tx_info &tx_info)->std::string { if (!tx_info.relayed) return "no"; return boost::lexical_cast<std::string>(tx_info.last_relayed_time) + " (" + get_human_time_ago(tx_info.last_relayed_time, now) + ")"; } (tx_info) << std::endl << "do_not_relay: " << (tx_info.do_not_relay ? 'T' : 'F') << std::endl @@ -996,7 +999,7 @@ bool t_rpc_command_executor::print_transaction_pool_stats() { size_t avg_bytes = n_transactions ? res.pool_stats.bytes_total / n_transactions : 0; std::string backlog_message; - const uint64_t full_reward_zone = ires.block_size_limit / 2; + const uint64_t full_reward_zone = ires.block_weight_limit / 2; if (res.pool_stats.bytes_total <= full_reward_zone) { backlog_message = "no backlog"; @@ -1701,8 +1704,8 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks) cryptonote::COMMAND_RPC_GET_INFO::response ires; cryptonote::COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::request bhreq; cryptonote::COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::response bhres; - cryptonote::COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::request fereq; - cryptonote::COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::response feres; + cryptonote::COMMAND_RPC_GET_BASE_FEE_ESTIMATE::request fereq; + cryptonote::COMMAND_RPC_GET_BASE_FEE_ESTIMATE::response feres; epee::json_rpc::error error_resp; std::string fail_message = "Problem fetching info"; @@ -1726,7 +1729,7 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks) tools::fail_msg_writer() << make_error(fail_message, ires.status); return true; } - if (!m_rpc_server->on_get_per_kb_fee_estimate(fereq, feres, error_resp) || feres.status != CORE_RPC_STATUS_OK) + if (!m_rpc_server->on_get_base_fee_estimate(fereq, feres, error_resp) || feres.status != CORE_RPC_STATUS_OK) { tools::fail_msg_writer() << make_error(fail_message, feres.status); return true; @@ -1762,8 +1765,8 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks) double avgdiff = 0; double avgnumtxes = 0; double avgreward = 0; - std::vector<uint64_t> sizes; - sizes.reserve(nblocks); + std::vector<uint64_t> weights; + weights.reserve(nblocks); uint64_t earliest = std::numeric_limits<uint64_t>::max(), latest = 0; std::vector<unsigned> major_versions(256, 0), minor_versions(256, 0); for (const auto &bhr: bhres.headers) @@ -1771,7 +1774,7 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks) avgdiff += bhr.difficulty; avgnumtxes += bhr.num_txes; avgreward += bhr.reward; - sizes.push_back(bhr.block_size); + weights.push_back(bhr.block_weight); static_assert(sizeof(bhr.major_version) == 1, "major_version expected to be uint8_t"); static_assert(sizeof(bhr.minor_version) == 1, "major_version expected to be uint8_t"); major_versions[(unsigned)bhr.major_version]++; @@ -1782,9 +1785,9 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks) avgdiff /= nblocks; avgnumtxes /= nblocks; avgreward /= nblocks; - uint64_t median_block_size = epee::misc_utils::median(sizes); + uint64_t median_block_weight = epee::misc_utils::median(weights); tools::msg_writer() << "Last " << nblocks << ": avg. diff " << (uint64_t)avgdiff << ", " << (latest - earliest) / nblocks << " avg sec/block, avg num txes " << avgnumtxes - << ", avg. reward " << cryptonote::print_money(avgreward) << ", median block size " << median_block_size; + << ", avg. reward " << cryptonote::print_money(avgreward) << ", median block weight " << median_block_weight; unsigned int max_major = 256, max_minor = 256; while (max_major > 0 && !major_versions[--max_major]); diff --git a/src/debug_utilities/cn_deserialize.cpp b/src/debug_utilities/cn_deserialize.cpp index 6c09b0f18..bf228fe14 100644 --- a/src/debug_utilities/cn_deserialize.cpp +++ b/src/debug_utilities/cn_deserialize.cpp @@ -43,6 +43,15 @@ using namespace epee; using namespace cryptonote; +static std::string extra_nonce_to_string(const cryptonote::tx_extra_nonce &extra_nonce) +{ + if (extra_nonce.nonce.size() == 9 && extra_nonce.nonce[0] == TX_EXTRA_NONCE_ENCRYPTED_PAYMENT_ID) + return "encrypted payment ID: " + epee::string_tools::buff_to_hex_nodelimer(extra_nonce.nonce.substr(1)); + if (extra_nonce.nonce.size() == 33 && extra_nonce.nonce[0] == TX_EXTRA_NONCE_PAYMENT_ID) + return "plaintext payment ID: " + epee::string_tools::buff_to_hex_nodelimer(extra_nonce.nonce.substr(1)); + return epee::string_tools::buff_to_hex_nodelimer(extra_nonce.nonce); +} + static void print_extra_fields(const std::vector<cryptonote::tx_extra_field> &fields) { std::cout << "tx_extra has " << fields.size() << " field(s)" << std::endl; @@ -51,7 +60,7 @@ static void print_extra_fields(const std::vector<cryptonote::tx_extra_field> &fi std::cout << "field " << n << ": "; if (typeid(cryptonote::tx_extra_padding) == fields[n].type()) std::cout << "extra padding: " << boost::get<cryptonote::tx_extra_padding>(fields[n]).size << " bytes"; else if (typeid(cryptonote::tx_extra_pub_key) == fields[n].type()) std::cout << "extra pub key: " << boost::get<cryptonote::tx_extra_pub_key>(fields[n]).pub_key; - else if (typeid(cryptonote::tx_extra_nonce) == fields[n].type()) std::cout << "extra nonce: " << epee::string_tools::buff_to_hex_nodelimer(boost::get<cryptonote::tx_extra_nonce>(fields[n]).nonce); + else if (typeid(cryptonote::tx_extra_nonce) == fields[n].type()) std::cout << "extra nonce: " << extra_nonce_to_string(boost::get<cryptonote::tx_extra_nonce>(fields[n])); else if (typeid(cryptonote::tx_extra_merge_mining_tag) == fields[n].type()) std::cout << "extra merge mining tag: depth " << boost::get<cryptonote::tx_extra_merge_mining_tag>(fields[n]).depth << ", merkle root " << boost::get<cryptonote::tx_extra_merge_mining_tag>(fields[n]).merkle_root; else if (typeid(cryptonote::tx_extra_additional_pub_keys) == fields[n].type()) std::cout << "additional tx pubkeys: " << boost::join(boost::get<cryptonote::tx_extra_additional_pub_keys>(fields[n]).data | boost::adaptors::transformed([](const crypto::public_key &key){ return epee::string_tools::pod_to_hex(key); }), ", " ); else if (typeid(cryptonote::tx_extra_mysterious_minergate) == fields[n].type()) std::cout << "extra minergate custom: " << epee::string_tools::buff_to_hex_nodelimer(boost::get<cryptonote::tx_extra_mysterious_minergate>(fields[n]).data); @@ -168,9 +177,12 @@ int main(int argc, char* argv[]) std::cout << "Parsed block:" << std::endl; std::cout << cryptonote::obj_to_json_str(block) << std::endl; } - else if (cryptonote::parse_and_validate_tx_from_blob(blob, tx)) + else if (cryptonote::parse_and_validate_tx_from_blob(blob, tx) || cryptonote::parse_and_validate_tx_base_from_blob(blob, tx)) { - std::cout << "Parsed transaction:" << std::endl; + if (tx.pruned) + std::cout << "Parsed pruned transaction:" << std::endl; + else + std::cout << "Parsed transaction:" << std::endl; std::cout << cryptonote::obj_to_json_str(tx) << std::endl; bool parsed = cryptonote::parse_tx_extra(tx.extra, fields); diff --git a/src/device/device_default.cpp b/src/device/device_default.cpp index bf14813ea..a4f40e041 100644 --- a/src/device/device_default.cpp +++ b/src/device/device_default.cpp @@ -103,7 +103,7 @@ namespace hw { bool device_default::generate_chacha_key(const cryptonote::account_keys &keys, crypto::chacha_key &key, uint64_t kdf_rounds) { const crypto::secret_key &view_key = keys.m_view_secret_key; const crypto::secret_key &spend_key = keys.m_spend_secret_key; - tools::scrubbed_arr<char, sizeof(view_key) + sizeof(spend_key) + 1> data; + epee::mlocked<tools::scrubbed_arr<char, sizeof(view_key) + sizeof(spend_key) + 1>> data; memcpy(data.data(), &view_key, sizeof(view_key)); memcpy(data.data() + sizeof(view_key), &spend_key, sizeof(spend_key)); data[sizeof(data) - 1] = CHACHA8_KEY_TAIL; diff --git a/src/device/device_ledger.cpp b/src/device/device_ledger.cpp index 7a34dad5e..658b379e4 100644 --- a/src/device/device_ledger.cpp +++ b/src/device/device_ledger.cpp @@ -136,7 +136,8 @@ namespace hw { } bool operator==(const crypto::key_derivation &d0, const crypto::key_derivation &d1) { - return !memcmp(&d0, &d1, sizeof(d0)); + static_assert(sizeof(crypto::key_derivation) == 32, "key_derivation must be 32 bytes"); + return !crypto_verify_32((const unsigned char*)&d0, (const unsigned char*)&d1); } /* ===================================================================== */ @@ -1353,7 +1354,7 @@ namespace hw { this->exchange(); //pseudoOuts - if ((type == rct::RCTTypeSimple) || (type == rct::RCTTypeSimpleBulletproof)) { + if ((type == rct::RCTTypeSimple) || (type == rct::RCTTypeBulletproof)) { for ( i = 0; i < inputs_size; i++) { offset = set_command_header(INS_VALIDATE, 0x01, i+2); //options diff --git a/src/gen_multisig/gen_multisig.cpp b/src/gen_multisig/gen_multisig.cpp index 03e0a7946..e680a8157 100644 --- a/src/gen_multisig/gen_multisig.cpp +++ b/src/gen_multisig/gen_multisig.cpp @@ -92,7 +92,7 @@ static bool generate_multisig(uint32_t threshold, uint32_t total, const std::str { std::string name = basename + "-" + std::to_string(n + 1); wallets[n].reset(new tools::wallet2(nettype)); - wallets[n]->init(""); + wallets[n]->init(false, ""); wallets[n]->generate(name, pwd_container->password(), rct::rct2sk(rct::skGen()), false, false, create_address_file); } @@ -101,11 +101,13 @@ static bool generate_multisig(uint32_t threshold, uint32_t total, const std::str std::vector<crypto::public_key> pk(total); for (size_t n = 0; n < total; ++n) { + wallets[n]->decrypt_keys(pwd_container->password()); if (!tools::wallet2::verify_multisig_info(wallets[n]->get_multisig_info(), sk[n], pk[n])) { tools::fail_msg_writer() << tr("Failed to verify multisig info"); return false; } + wallets[n]->encrypt_keys(pwd_container->password()); } // make the wallets multisig diff --git a/src/mnemonics/electrum-words.cpp b/src/mnemonics/electrum-words.cpp index 19a9c26bb..3d6338856 100644 --- a/src/mnemonics/electrum-words.cpp +++ b/src/mnemonics/electrum-words.cpp @@ -43,8 +43,11 @@ #include <vector> #include <unordered_map> #include <boost/algorithm/string.hpp> +#include "wipeable_string.h" +#include "misc_language.h" #include "crypto/crypto.h" // for declaration of crypto::secret_key #include <fstream> +#include "common/int-util.h" #include "mnemonics/electrum-words.h" #include <stdexcept> #include <boost/filesystem.hpp> @@ -80,9 +83,9 @@ namespace crypto namespace { - uint32_t create_checksum_index(const std::vector<std::string> &word_list, + uint32_t create_checksum_index(const std::vector<epee::wipeable_string> &word_list, uint32_t unique_prefix_length); - bool checksum_test(std::vector<std::string> seed, uint32_t unique_prefix_length); + bool checksum_test(std::vector<epee::wipeable_string> seed, uint32_t unique_prefix_length); /*! * \brief Finds the word list that contains the seed words and puts the indices @@ -93,7 +96,7 @@ namespace * \param language Language instance pointer to write to after it is found. * \return true if all the words were present in some language false if not. */ - bool find_seed_language(const std::vector<std::string> &seed, + bool find_seed_language(const std::vector<epee::wipeable_string> &seed, bool has_checksum, std::vector<uint32_t> &matched_indices, Language::Base **language) { // If there's a new language added, add an instance of it here. @@ -114,17 +117,19 @@ namespace }); Language::Base *fallback = NULL; + std::vector<epee::wipeable_string>::const_iterator it2; + matched_indices.reserve(seed.size()); + // Iterate through all the languages and find a match for (std::vector<Language::Base*>::iterator it1 = language_instances.begin(); it1 != language_instances.end(); it1++) { - const std::unordered_map<std::string, uint32_t> &word_map = (*it1)->get_word_map(); - const std::unordered_map<std::string, uint32_t> &trimmed_word_map = (*it1)->get_trimmed_word_map(); + const std::unordered_map<epee::wipeable_string, uint32_t> &word_map = (*it1)->get_word_map(); + const std::unordered_map<epee::wipeable_string, uint32_t> &trimmed_word_map = (*it1)->get_trimmed_word_map(); // To iterate through seed words - std::vector<std::string>::const_iterator it2; bool full_match = true; - std::string trimmed_word; + epee::wipeable_string trimmed_word; // Iterate through all the words and see if they're all present for (it2 = seed.begin(); it2 != seed.end(); it2++) { @@ -167,6 +172,7 @@ namespace return true; } // Some didn't match. Clear the index array. + memwipe(matched_indices.data(), matched_indices.size() * sizeof(matched_indices[0])); matched_indices.clear(); } @@ -181,6 +187,7 @@ namespace } MINFO("No match found"); + memwipe(matched_indices.data(), matched_indices.size() * sizeof(matched_indices[0])); return false; } @@ -190,12 +197,12 @@ namespace * \param unique_prefix_length the prefix length of each word to use for checksum * \return Checksum index */ - uint32_t create_checksum_index(const std::vector<std::string> &word_list, + uint32_t create_checksum_index(const std::vector<epee::wipeable_string> &word_list, uint32_t unique_prefix_length) { - std::string trimmed_words = ""; + epee::wipeable_string trimmed_words = ""; - for (std::vector<std::string>::const_iterator it = word_list.begin(); it != word_list.end(); it++) + for (std::vector<epee::wipeable_string>::const_iterator it = word_list.begin(); it != word_list.end(); it++) { if (it->length() > unique_prefix_length) { @@ -217,22 +224,22 @@ namespace * \param unique_prefix_length the prefix length of each word to use for checksum * \return True if the test passed false if not. */ - bool checksum_test(std::vector<std::string> seed, uint32_t unique_prefix_length) + bool checksum_test(std::vector<epee::wipeable_string> seed, uint32_t unique_prefix_length) { if (seed.empty()) return false; // The last word is the checksum. - std::string last_word = seed.back(); + epee::wipeable_string last_word = seed.back(); seed.pop_back(); - std::string checksum = seed[create_checksum_index(seed, unique_prefix_length)]; + epee::wipeable_string checksum = seed[create_checksum_index(seed, unique_prefix_length)]; - std::string trimmed_checksum = checksum.length() > unique_prefix_length ? Language::utf8prefix(checksum, unique_prefix_length) : + epee::wipeable_string trimmed_checksum = checksum.length() > unique_prefix_length ? Language::utf8prefix(checksum, unique_prefix_length) : checksum; - std::string trimmed_last_word = last_word.length() > unique_prefix_length ? Language::utf8prefix(last_word, unique_prefix_length) : + epee::wipeable_string trimmed_last_word = last_word.length() > unique_prefix_length ? Language::utf8prefix(last_word, unique_prefix_length) : last_word; bool ret = trimmed_checksum == trimmed_last_word; - MINFO("Checksum is %s" << (ret ? "valid" : "invalid")); + MINFO("Checksum is " << (ret ? "valid" : "invalid")); return ret; } } @@ -260,13 +267,12 @@ namespace crypto * \param language_name Language of the seed as found gets written here. * \return false if not a multiple of 3 words, or if word is not in the words list */ - bool words_to_bytes(std::string words, std::string& dst, size_t len, bool duplicate, + bool words_to_bytes(const epee::wipeable_string &words, epee::wipeable_string& dst, size_t len, bool duplicate, std::string &language_name) { - std::vector<std::string> seed; + std::vector<epee::wipeable_string> seed; - boost::algorithm::trim(words); - boost::split(seed, words, boost::is_any_of(" "), boost::token_compress_on); + words.split(seed); if (len % 4) { @@ -291,6 +297,7 @@ namespace crypto } std::vector<uint32_t> matched_indices; + auto wiper = epee::misc_utils::create_scope_leave_handler([&](){memwipe(matched_indices.data(), matched_indices.size() * sizeof(matched_indices[0]));}); Language::Base *language; if (!find_seed_language(seed, has_checksum, matched_indices, &language)) { @@ -313,33 +320,33 @@ namespace crypto for (unsigned int i=0; i < seed.size() / 3; i++) { - uint32_t val; - uint32_t w1, w2, w3; - w1 = matched_indices[i*3]; - w2 = matched_indices[i*3 + 1]; - w3 = matched_indices[i*3 + 2]; + uint32_t w[4]; + w[1] = matched_indices[i*3]; + w[2] = matched_indices[i*3 + 1]; + w[3] = matched_indices[i*3 + 2]; - val = w1 + word_list_length * (((word_list_length - w1) + w2) % word_list_length) + - word_list_length * word_list_length * (((word_list_length - w2) + w3) % word_list_length); + w[0]= w[1] + word_list_length * (((word_list_length - w[1]) + w[2]) % word_list_length) + + word_list_length * word_list_length * (((word_list_length - w[2]) + w[3]) % word_list_length); - if (!(val % word_list_length == w1)) + if (!(w[0]% word_list_length == w[1])) { + memwipe(w, sizeof(w)); MERROR("Invalid seed: mumble mumble"); return false; } - dst.append((const char*)&val, 4); // copy 4 bytes to position + dst.append((const char*)&w[0], 4); // copy 4 bytes to position + memwipe(w, sizeof(w)); } if (len > 0 && duplicate) { const size_t expected = len * 3 / 32; - std::string wlist_copy = words; if (seed.size() == expected/2) { - dst.append(dst); // if electrum 12-word seed, duplicate - wlist_copy += ' '; - wlist_copy += words; + dst += ' '; // if electrum 12-word seed, duplicate + dst += dst; // if electrum 12-word seed, duplicate + dst.pop_back(); // trailing space } } @@ -353,10 +360,10 @@ namespace crypto * \param language_name Language of the seed as found gets written here. * \return false if not a multiple of 3 words, or if word is not in the words list */ - bool words_to_bytes(std::string words, crypto::secret_key& dst, + bool words_to_bytes(const epee::wipeable_string &words, crypto::secret_key& dst, std::string &language_name) { - std::string s; + epee::wipeable_string s; if (!words_to_bytes(words, s, sizeof(dst), true, language_name)) { MERROR("Invalid seed: failed to convert words to bytes"); @@ -378,7 +385,7 @@ namespace crypto * \param language_name Seed language name * \return true if successful false if not. Unsuccessful if wrong key size. */ - bool bytes_to_words(const char *src, size_t len, std::string& words, + bool bytes_to_words(const char *src, size_t len, epee::wipeable_string& words, const std::string &language_name) { @@ -397,39 +404,38 @@ namespace crypto } const std::vector<std::string> &word_list = language->get_word_list(); // To store the words for random access to add the checksum word later. - std::vector<std::string> words_store; + std::vector<epee::wipeable_string> words_store; uint32_t word_list_length = word_list.size(); // 4 bytes -> 3 words. 8 digits base 16 -> 3 digits base 1626 - for (unsigned int i=0; i < len/4; i++, words += ' ') + for (unsigned int i=0; i < len/4; i++, words.push_back(' ')) { - uint32_t w1, w2, w3; - - uint32_t val; + uint32_t w[4]; - memcpy(&val, src + (i * 4), 4); + w[0] = SWAP32LE(*(const uint32_t*)(src + (i * 4))); - w1 = val % word_list_length; - w2 = ((val / word_list_length) + w1) % word_list_length; - w3 = (((val / word_list_length) / word_list_length) + w2) % word_list_length; + w[1] = w[0] % word_list_length; + w[2] = ((w[0] / word_list_length) + w[1]) % word_list_length; + w[3] = (((w[0] / word_list_length) / word_list_length) + w[2]) % word_list_length; - words += word_list[w1]; + words += word_list[w[1]]; words += ' '; - words += word_list[w2]; + words += word_list[w[2]]; words += ' '; - words += word_list[w3]; + words += word_list[w[3]]; + + words_store.push_back(word_list[w[1]]); + words_store.push_back(word_list[w[2]]); + words_store.push_back(word_list[w[3]]); - words_store.push_back(word_list[w1]); - words_store.push_back(word_list[w2]); - words_store.push_back(word_list[w3]); + memwipe(w, sizeof(w)); } - words.pop_back(); - words += (' ' + words_store[create_checksum_index(words_store, language->get_unique_prefix_length())]); + words += words_store[create_checksum_index(words_store, language->get_unique_prefix_length())]; return true; } - bool bytes_to_words(const crypto::secret_key& src, std::string& words, + bool bytes_to_words(const crypto::secret_key& src, epee::wipeable_string& words, const std::string &language_name) { return bytes_to_words(src.data, sizeof(src), words, language_name); @@ -473,11 +479,10 @@ namespace crypto * \param seed The seed to check (a space delimited concatenated word list) * \return true if the seed passed is a old style seed false if not. */ - bool get_is_old_style_seed(std::string seed) + bool get_is_old_style_seed(const epee::wipeable_string &seed) { - std::vector<std::string> word_list; - boost::algorithm::trim(seed); - boost::split(word_list, seed, boost::is_any_of(" "), boost::token_compress_on); + std::vector<epee::wipeable_string> word_list; + seed.split(word_list); return word_list.size() != (seed_length + 1); } diff --git a/src/mnemonics/electrum-words.h b/src/mnemonics/electrum-words.h index 856edb92a..5401b9779 100644 --- a/src/mnemonics/electrum-words.h +++ b/src/mnemonics/electrum-words.h @@ -44,6 +44,8 @@ #include <map> #include "crypto/crypto.h" // for declaration of crypto::secret_key +namespace epee { class wipeable_string; } + /*! * \namespace crypto * @@ -70,7 +72,7 @@ namespace crypto * \param language_name Language of the seed as found gets written here. * \return false if not a multiple of 3 words, or if word is not in the words list */ - bool words_to_bytes(std::string words, std::string& dst, size_t len, bool duplicate, + bool words_to_bytes(const epee::wipeable_string &words, epee::wipeable_string& dst, size_t len, bool duplicate, std::string &language_name); /*! * \brief Converts seed words to bytes (secret key). @@ -79,7 +81,7 @@ namespace crypto * \param language_name Language of the seed as found gets written here. * \return false if not a multiple of 3 words, or if word is not in the words list */ - bool words_to_bytes(std::string words, crypto::secret_key& dst, + bool words_to_bytes(const epee::wipeable_string &words, crypto::secret_key& dst, std::string &language_name); /*! @@ -90,7 +92,7 @@ namespace crypto * \param language_name Seed language name * \return true if successful false if not. Unsuccessful if wrong key size. */ - bool bytes_to_words(const char *src, size_t len, std::string& words, + bool bytes_to_words(const char *src, size_t len, epee::wipeable_string& words, const std::string &language_name); /*! @@ -100,7 +102,7 @@ namespace crypto * \param language_name Seed language name * \return true if successful false if not. Unsuccessful if wrong key size. */ - bool bytes_to_words(const crypto::secret_key& src, std::string& words, + bool bytes_to_words(const crypto::secret_key& src, epee::wipeable_string& words, const std::string &language_name); /*! @@ -115,7 +117,7 @@ namespace crypto * \param seed The seed to check (a space delimited concatenated word list) * \return true if the seed passed is a old style seed false if not. */ - bool get_is_old_style_seed(std::string seed); + bool get_is_old_style_seed(const epee::wipeable_string &seed); /*! * \brief Returns the name of a language in English diff --git a/src/mnemonics/language_base.h b/src/mnemonics/language_base.h index 2b0c37c6b..cf518ab2a 100644 --- a/src/mnemonics/language_base.h +++ b/src/mnemonics/language_base.h @@ -53,15 +53,20 @@ namespace Language * \param count How many characters to return.
* \return A string consisting of the first count characters in s.
*/
- inline std::string utf8prefix(const std::string &s, size_t count)
+ template<typename T>
+ inline T utf8prefix(const T &s, size_t count)
{
- std::string prefix = "";
- const char *ptr = s.c_str();
- while (count-- && *ptr)
+ T prefix = "";
+ size_t avail = s.size();
+ const char *ptr = s.data();
+ while (count-- && avail--)
{
prefix += *ptr++;
- while (((*ptr) & 0xc0) == 0x80)
+ while (avail && ((*ptr) & 0xc0) == 0x80)
+ {
prefix += *ptr++;
+ --avail;
+ }
}
return prefix;
}
@@ -79,8 +84,8 @@ namespace Language ALLOW_DUPLICATE_PREFIXES = 1<<1,
};
const std::vector<std::string> word_list; /*!< A pointer to the array of words */
- std::unordered_map<std::string, uint32_t> word_map; /*!< hash table to find word's index */
- std::unordered_map<std::string, uint32_t> trimmed_word_map; /*!< hash table to find word's trimmed index */
+ std::unordered_map<epee::wipeable_string, uint32_t> word_map; /*!< hash table to find word's index */
+ std::unordered_map<epee::wipeable_string, uint32_t> trimmed_word_map; /*!< hash table to find word's trimmed index */
std::string language_name; /*!< Name of language */
std::string english_language_name; /*!< Name of language */
uint32_t unique_prefix_length; /*!< Number of unique starting characters to trim the wordlist to when matching */
@@ -103,7 +108,7 @@ namespace Language else
throw std::runtime_error("Too short word in " + language_name + " word list: " + *it);
}
- std::string trimmed;
+ epee::wipeable_string trimmed;
if (it->length() > unique_prefix_length)
{
trimmed = utf8prefix(*it, unique_prefix_length);
@@ -115,9 +120,9 @@ namespace Language if (trimmed_word_map.find(trimmed) != trimmed_word_map.end())
{
if (flags & ALLOW_DUPLICATE_PREFIXES)
- MWARNING("Duplicate prefix in " << language_name << " word list: " << trimmed);
+ MWARNING("Duplicate prefix in " << language_name << " word list: " << std::string(trimmed.data(), trimmed.size()));
else
- throw std::runtime_error("Duplicate prefix in " + language_name + " word list: " + trimmed);
+ throw std::runtime_error("Duplicate prefix in " + language_name + " word list: " + std::string(trimmed.data(), trimmed.size()));
}
trimmed_word_map[trimmed] = ii;
}
@@ -145,7 +150,7 @@ namespace Language * \brief Returns a pointer to the word map.
* \return A pointer to the word map.
*/
- const std::unordered_map<std::string, uint32_t>& get_word_map() const
+ const std::unordered_map<epee::wipeable_string, uint32_t>& get_word_map() const
{
return word_map;
}
@@ -153,7 +158,7 @@ namespace Language * \brief Returns a pointer to the trimmed word map.
* \return A pointer to the trimmed word map.
*/
- const std::unordered_map<std::string, uint32_t>& get_trimmed_word_map() const
+ const std::unordered_map<epee::wipeable_string, uint32_t>& get_trimmed_word_map() const
{
return trimmed_word_map;
}
diff --git a/src/multisig/multisig.cpp b/src/multisig/multisig.cpp index d85c47772..a0a788b7d 100644 --- a/src/multisig/multisig.cpp +++ b/src/multisig/multisig.cpp @@ -47,9 +47,12 @@ namespace cryptonote crypto::secret_key get_multisig_blinded_secret_key(const crypto::secret_key &key) { rct::keyV data; + data.reserve(2); data.push_back(rct::sk2rct(key)); data.push_back(multisig_salt); - return rct::rct2sk(rct::hash_to_scalar(data)); + crypto::secret_key result = rct::rct2sk(rct::hash_to_scalar(data)); + memwipe(&data[0], sizeof(rct::key)); + return result; } //----------------------------------------------------------------- void generate_multisig_N_N(const account_keys &keys, const std::vector<crypto::public_key> &spend_keys, std::vector<crypto::secret_key> &multisig_keys, rct::key &spend_skey, rct::key &spend_pkey) diff --git a/src/ringct/CMakeLists.txt b/src/ringct/CMakeLists.txt index c8dcdca26..29f166a3b 100644 --- a/src/ringct/CMakeLists.txt +++ b/src/ringct/CMakeLists.txt @@ -30,11 +30,13 @@ set(ringct_basic_sources rctOps.cpp rctTypes.cpp rctCryptoOps.c + multiexp.cc bulletproofs.cc) set(ringct_basic_private_headers rctOps.h rctTypes.h + multiexp.h bulletproofs.h) monero_private_headers(ringct_basic diff --git a/src/ringct/bulletproofs.cc b/src/ringct/bulletproofs.cc index fd15ffbc4..abe4ef18d 100644 --- a/src/ringct/bulletproofs.cc +++ b/src/ringct/bulletproofs.cc @@ -30,14 +30,17 @@ #include <stdlib.h> #include <openssl/ssl.h> +#include <openssl/bn.h> #include <boost/thread/mutex.hpp> #include "misc_log_ex.h" #include "common/perf_timer.h" +#include "cryptonote_config.h" extern "C" { #include "crypto/crypto-ops.h" } #include "rctOps.h" +#include "multiexp.h" #include "bulletproofs.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -47,27 +50,99 @@ extern "C" #define PERF_TIMER_START_BP(x) PERF_TIMER_START_UNIT(x, 1000000) +#define STRAUS_SIZE_LIMIT 128 +#define PIPPENGER_SIZE_LIMIT 0 + namespace rct { static rct::key vector_exponent(const rct::keyV &a, const rct::keyV &b); -static rct::keyV vector_powers(rct::key x, size_t n); +static rct::keyV vector_powers(const rct::key &x, size_t n); +static rct::keyV vector_dup(const rct::key &x, size_t n); static rct::key inner_product(const rct::keyV &a, const rct::keyV &b); static constexpr size_t maxN = 64; -static rct::key Hi[maxN], Gi[maxN]; -static ge_dsmp Gprecomp[64], Hprecomp[64]; +static constexpr size_t maxM = BULLETPROOF_MAX_OUTPUTS; +static rct::key Hi[maxN*maxM], Gi[maxN*maxM]; +static ge_p3 Hi_p3[maxN*maxM], Gi_p3[maxN*maxM]; +static std::shared_ptr<straus_cached_data> straus_HiGi_cache; +static std::shared_ptr<pippenger_cached_data> pippenger_HiGi_cache; static const rct::key TWO = { {0x02, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 } }; -static const rct::keyV oneN = vector_powers(rct::identity(), maxN); +static const rct::key MINUS_ONE = { { 0xec, 0xd3, 0xf5, 0x5c, 0x1a, 0x63, 0x12, 0x58, 0xd6, 0x9c, 0xf7, 0xa2, 0xde, 0xf9, 0xde, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10 } }; +static const rct::key MINUS_INV_EIGHT = { { 0x74, 0xa4, 0x19, 0x7a, 0xf0, 0x7d, 0x0b, 0xf7, 0x05, 0xc2, 0xda, 0x25, 0x2b, 0x5c, 0x0b, 0x0d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a } }; +static const rct::keyV oneN = vector_dup(rct::identity(), maxN); static const rct::keyV twoN = vector_powers(TWO, maxN); static const rct::key ip12 = inner_product(oneN, twoN); static boost::mutex init_mutex; +static inline rct::key multiexp(const std::vector<MultiexpData> &data, bool HiGi) +{ + if (HiGi) + { + static_assert(128 <= STRAUS_SIZE_LIMIT, "Straus in precalc mode can only be calculated till STRAUS_SIZE_LIMIT"); + return data.size() <= 128 ? straus(data, straus_HiGi_cache, 0) : pippenger(data, pippenger_HiGi_cache, get_pippenger_c(data.size())); + } + else + return data.size() <= 64 ? straus(data, NULL, 0) : pippenger(data, NULL, get_pippenger_c(data.size())); +} + +static bool is_reduced(const rct::key &scalar) +{ + rct::key reduced = scalar; + sc_reduce32(reduced.bytes); + return scalar == reduced; +} + +static void addKeys_acc_p3(ge_p3 *acc_p3, const rct::key &a, const rct::key &point) +{ + ge_p3 p3; + CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&p3, point.bytes) == 0, "ge_frombytes_vartime failed"); + ge_scalarmult_p3(&p3, a.bytes, &p3); + ge_cached cached; + ge_p3_to_cached(&cached, acc_p3); + ge_p1p1 p1; + ge_add(&p1, &p3, &cached); + ge_p1p1_to_p3(acc_p3, &p1); +} + +static void add_acc_p3(ge_p3 *acc_p3, const rct::key &point) +{ + ge_p3 p3; + CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&p3, point.bytes) == 0, "ge_frombytes_vartime failed"); + ge_cached cached; + ge_p3_to_cached(&cached, &p3); + ge_p1p1 p1; + ge_add(&p1, acc_p3, &cached); + ge_p1p1_to_p3(acc_p3, &p1); +} + +static void sub_acc_p3(ge_p3 *acc_p3, const rct::key &point) +{ + ge_p3 p3; + CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&p3, point.bytes) == 0, "ge_frombytes_vartime failed"); + ge_cached cached; + ge_p3_to_cached(&cached, &p3); + ge_p1p1 p1; + ge_sub(&p1, acc_p3, &cached); + ge_p1p1_to_p3(acc_p3, &p1); +} + +static rct::key scalarmultKey(const ge_p3 &P, const rct::key &a) +{ + ge_p2 R; + ge_scalarmult(&R, a.bytes, &P); + rct::key aP; + ge_tobytes(aP.bytes, &R); + return aP; +} + static rct::key get_exponent(const rct::key &base, size_t idx) { static const std::string salt("bulletproof"); std::string hashed = std::string((const char*)base.bytes, sizeof(base)) + salt + tools::get_varint_data(idx); - return rct::hashToPoint(rct::hash2rct(crypto::cn_fast_hash(hashed.data(), hashed.size()))); + const rct::key e = rct::hashToPoint(rct::hash2rct(crypto::cn_fast_hash(hashed.data(), hashed.size()))); + CHECK_AND_ASSERT_THROW_MES(!(e == rct::identity()), "Exponent is point at infinity"); + return e; } static void init_exponents() @@ -77,13 +152,27 @@ static void init_exponents() static bool init_done = false; if (init_done) return; - for (size_t i = 0; i < maxN; ++i) + std::vector<MultiexpData> data; + for (size_t i = 0; i < maxN*maxM; ++i) { Hi[i] = get_exponent(rct::H, i * 2); - rct::precomp(Hprecomp[i], Hi[i]); + CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&Hi_p3[i], Hi[i].bytes) == 0, "ge_frombytes_vartime failed"); Gi[i] = get_exponent(rct::H, i * 2 + 1); - rct::precomp(Gprecomp[i], Gi[i]); + CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&Gi_p3[i], Gi[i].bytes) == 0, "ge_frombytes_vartime failed"); + + data.push_back({rct::zero(), Gi[i]}); + data.push_back({rct::zero(), Hi[i]}); } + + straus_HiGi_cache = straus_init_cache(data, STRAUS_SIZE_LIMIT); + pippenger_HiGi_cache = pippenger_init_cache(data, PIPPENGER_SIZE_LIMIT); + + MINFO("Hi/Gi cache size: " << (sizeof(Hi)+sizeof(Gi))/1024 << " kB"); + MINFO("Hi_p3/Gi_p3 cache size: " << (sizeof(Hi_p3)+sizeof(Gi_p3))/1024 << " kB"); + MINFO("Straus cache size: " << straus_get_cache_size(straus_HiGi_cache)/1024 << " kB"); + MINFO("Pippenger cache size: " << pippenger_get_cache_size(pippenger_HiGi_cache)/1024 << " kB"); + size_t cache_size = (sizeof(Hi)+sizeof(Hi_p3))*2 + straus_get_cache_size(straus_HiGi_cache) + pippenger_get_cache_size(pippenger_HiGi_cache); + MINFO("Total cache size: " << cache_size/1024 << "kB"); init_done = true; } @@ -91,15 +180,16 @@ static void init_exponents() static rct::key vector_exponent(const rct::keyV &a, const rct::keyV &b) { CHECK_AND_ASSERT_THROW_MES(a.size() == b.size(), "Incompatible sizes of a and b"); - CHECK_AND_ASSERT_THROW_MES(a.size() <= maxN, "Incompatible sizes of a and maxN"); - rct::key res = rct::identity(); + CHECK_AND_ASSERT_THROW_MES(a.size() <= maxN*maxM, "Incompatible sizes of a and maxN"); + + std::vector<MultiexpData> multiexp_data; + multiexp_data.reserve(a.size()*2); for (size_t i = 0; i < a.size(); ++i) { - rct::key term; - rct::addKeys3(term, a[i], Gprecomp[i], b[i], Hprecomp[i]); - rct::addKeys(res, res, term); + multiexp_data.emplace_back(a[i], Gi_p3[i]); + multiexp_data.emplace_back(b[i], Hi_p3[i]); } - return res; + return multiexp(multiexp_data, true); } /* Compute a custom vector-scalar commitment */ @@ -108,44 +198,24 @@ static rct::key vector_exponent_custom(const rct::keyV &A, const rct::keyV &B, c CHECK_AND_ASSERT_THROW_MES(A.size() == B.size(), "Incompatible sizes of A and B"); CHECK_AND_ASSERT_THROW_MES(a.size() == b.size(), "Incompatible sizes of a and b"); CHECK_AND_ASSERT_THROW_MES(a.size() == A.size(), "Incompatible sizes of a and A"); - CHECK_AND_ASSERT_THROW_MES(a.size() <= maxN, "Incompatible sizes of a and maxN"); - rct::key res = rct::identity(); + CHECK_AND_ASSERT_THROW_MES(a.size() <= maxN*maxM, "Incompatible sizes of a and maxN"); + + std::vector<MultiexpData> multiexp_data; + multiexp_data.reserve(a.size()*2); for (size_t i = 0; i < a.size(); ++i) { - rct::key term; -#if 0 - // we happen to know where A and B might fall, so don't bother checking the rest - ge_dsmp *Acache = NULL, *Bcache = NULL; - ge_dsmp Acache_custom[1], Bcache_custom[1]; - if (Gi[i] == A[i]) - Acache = Gprecomp + i; - else if (i<32 && Gi[i+32] == A[i]) - Acache = Gprecomp + i + 32; - else - { - rct::precomp(Acache_custom[0], A[i]); - Acache = Acache_custom; - } - if (i == 0 && B[i] == Hi[0]) - Bcache = Hprecomp; - else - { - rct::precomp(Bcache_custom[0], B[i]); - Bcache = Bcache_custom; - } - rct::addKeys3(term, a[i], *Acache, b[i], *Bcache); -#else - ge_dsmp Acache, Bcache; - rct::precomp(Bcache, B[i]); - rct::addKeys3(term, a[i], A[i], b[i], Bcache); -#endif - rct::addKeys(res, res, term); + multiexp_data.resize(multiexp_data.size() + 1); + multiexp_data.back().scalar = a[i]; + CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&multiexp_data.back().point, A[i].bytes) == 0, "ge_frombytes_vartime failed"); + multiexp_data.resize(multiexp_data.size() + 1); + multiexp_data.back().scalar = b[i]; + CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&multiexp_data.back().point, B[i].bytes) == 0, "ge_frombytes_vartime failed"); } - return res; + return multiexp(multiexp_data, false); } /* Given a scalar, construct a vector of powers */ -static rct::keyV vector_powers(rct::key x, size_t n) +static rct::keyV vector_powers(const rct::key &x, size_t n) { rct::keyV res(n); if (n == 0) @@ -161,6 +231,24 @@ static rct::keyV vector_powers(rct::key x, size_t n) return res; } +/* Given a scalar, return the sum of its powers from 0 to n-1 */ +static rct::key vector_power_sum(const rct::key &x, size_t n) +{ + if (n == 0) + return rct::zero(); + rct::key res = rct::identity(); + if (n == 1) + return res; + rct::key prev = x; + for (size_t i = 1; i < n; ++i) + { + if (i > 1) + sc_mul(prev.bytes, prev.bytes, x.bytes); + sc_add(res.bytes, res.bytes, prev.bytes); + } + return res; +} + /* Given two scalar arrays, construct the inner product */ static rct::key inner_product(const rct::keyV &a, const rct::keyV &b) { @@ -232,6 +320,12 @@ static rct::keyV vector_scalar(const rct::keyV &a, const rct::key &x) return res; } +/* Create a vector from copies of a single value */ +static rct::keyV vector_dup(const rct::key &x, size_t N) +{ + return rct::keyV(N, x); +} + /* Exponentiate a curve vector by a scalar */ static rct::keyV vector_scalar2(const rct::keyV &a, const rct::key &x) { @@ -243,6 +337,17 @@ static rct::keyV vector_scalar2(const rct::keyV &a, const rct::key &x) return res; } +/* Get the sum of a vector's elements */ +static rct::key vector_sum(const rct::keyV &a) +{ + rct::key res = rct::zero(); + for (size_t i = 0; i < a.size(); ++i) + { + sc_add(res.bytes, res.bytes, a[i].bytes); + } + return res; +} + static rct::key switch_endianness(rct::key k) { std::reverse(k.bytes, k.bytes + sizeof(k)); @@ -345,6 +450,7 @@ Bulletproof bulletproof_PROVE(const rct::key &sv, const rct::key &gamma) PERF_TIMER_START_BP(PROVE_v); rct::addKeys2(V, gamma, sv, rct::H); + V = rct::scalarmultKey(V, INV_EIGHT); PERF_TIMER_STOP(PROVE_v); PERF_TIMER_START_BP(PROVE_aLaR); @@ -380,12 +486,14 @@ Bulletproof bulletproof_PROVE(const rct::key &sv, const rct::key &gamma) CHECK_AND_ASSERT_THROW_MES(test_aR == v_test, "test_aR failed"); #endif +try_again: PERF_TIMER_START_BP(PROVE_step1); // PAPER LINES 38-39 rct::key alpha = rct::skGen(); rct::key ve = vector_exponent(aL, aR); rct::key A; rct::addKeys(A, ve, rct::scalarmultBase(alpha)); + A = rct::scalarmultKey(A, INV_EIGHT); // PAPER LINES 40-42 rct::keyV sL = rct::skvGen(N), sR = rct::skvGen(N); @@ -393,10 +501,23 @@ Bulletproof bulletproof_PROVE(const rct::key &sv, const rct::key &gamma) ve = vector_exponent(sL, sR); rct::key S; rct::addKeys(S, ve, rct::scalarmultBase(rho)); + S = rct::scalarmultKey(S, INV_EIGHT); // PAPER LINES 43-45 rct::key y = hash_cache_mash(hash_cache, A, S); + if (y == rct::zero()) + { + PERF_TIMER_STOP(PROVE_step1); + MINFO("y is 0, trying again"); + goto try_again; + } rct::key z = hash_cache = rct::hash_to_scalar(y); + if (z == rct::zero()) + { + PERF_TIMER_STOP(PROVE_step1); + MINFO("z is 0, trying again"); + goto try_again; + } // Polynomial construction before PAPER LINE 46 rct::key t0 = rct::zero(); @@ -405,7 +526,7 @@ Bulletproof bulletproof_PROVE(const rct::key &sv, const rct::key &gamma) const auto yN = vector_powers(y, N); - rct::key ip1y = inner_product(oneN, yN); + rct::key ip1y = vector_sum(yN); rct::key tmp; sc_muladd(t0.bytes, z.bytes, ip1y.bytes, t0.bytes); @@ -437,7 +558,7 @@ Bulletproof bulletproof_PROVE(const rct::key &sv, const rct::key &gamma) PERF_TIMER_START_BP(PROVE_step2); const auto HyNsR = hadamard(yN, sR); - const auto vpIz = vector_scalar(oneN, z); + const auto vpIz = vector_dup(z, N); const auto vp2zsq = vector_scalar(twoN, zsq); const auto aL_vpIz = vector_subtract(aL, vpIz); const auto aR_vpIz = vector_add(aR, vpIz); @@ -454,11 +575,19 @@ Bulletproof bulletproof_PROVE(const rct::key &sv, const rct::key &gamma) // PAPER LINES 47-48 rct::key tau1 = rct::skGen(), tau2 = rct::skGen(); - rct::key T1 = rct::addKeys(rct::scalarmultKey(rct::H, t1), rct::scalarmultBase(tau1)); - rct::key T2 = rct::addKeys(rct::scalarmultKey(rct::H, t2), rct::scalarmultBase(tau2)); + rct::key T1 = rct::addKeys(rct::scalarmultH(t1), rct::scalarmultBase(tau1)); + T1 = rct::scalarmultKey(T1, INV_EIGHT); + rct::key T2 = rct::addKeys(rct::scalarmultH(t2), rct::scalarmultBase(tau2)); + T2 = rct::scalarmultKey(T2, INV_EIGHT); // PAPER LINES 49-51 rct::key x = hash_cache_mash(hash_cache, z, T1, T2); + if (x == rct::zero()) + { + PERF_TIMER_STOP(PROVE_step2); + MINFO("x is 0, trying again"); + goto try_again; + } // PAPER LINES 52-53 rct::key taux = rct::zero(); @@ -500,7 +629,7 @@ Bulletproof bulletproof_PROVE(const rct::key &sv, const rct::key &gamma) for (size_t i = 0; i < N; ++i) { Gprime[i] = Gi[i]; - Hprime[i] = scalarmultKey(Hi[i], yinvpow); + Hprime[i] = scalarmultKey(Hi_p3[i], yinvpow); sc_mul(yinvpow.bytes, yinvpow.bytes, yinv.bytes); aprime[i] = l[i]; bprime[i] = r[i]; @@ -525,13 +654,21 @@ Bulletproof bulletproof_PROVE(const rct::key &sv, const rct::key &gamma) // PAPER LINES 18-19 L[round] = vector_exponent_custom(slice(Gprime, nprime, Gprime.size()), slice(Hprime, 0, nprime), slice(aprime, 0, nprime), slice(bprime, nprime, bprime.size())); sc_mul(tmp.bytes, cL.bytes, x_ip.bytes); - rct::addKeys(L[round], L[round], rct::scalarmultKey(rct::H, tmp)); + rct::addKeys(L[round], L[round], rct::scalarmultH(tmp)); + L[round] = rct::scalarmultKey(L[round], INV_EIGHT); R[round] = vector_exponent_custom(slice(Gprime, 0, nprime), slice(Hprime, nprime, Hprime.size()), slice(aprime, nprime, aprime.size()), slice(bprime, 0, nprime)); sc_mul(tmp.bytes, cR.bytes, x_ip.bytes); - rct::addKeys(R[round], R[round], rct::scalarmultKey(rct::H, tmp)); + rct::addKeys(R[round], R[round], rct::scalarmultH(tmp)); + R[round] = rct::scalarmultKey(R[round], INV_EIGHT); // PAPER LINES 21-22 w[round] = hash_cache_mash(hash_cache, L[round], R[round]); + if (w[round] == rct::zero()) + { + PERF_TIMER_STOP(PROVE_step4); + MINFO("w[round] is 0, trying again"); + goto try_again; + } // PAPER LINES 24-25 const rct::key winv = invert(w[round]); @@ -567,179 +704,540 @@ Bulletproof bulletproof_PROVE(uint64_t v, const rct::key &gamma) return bulletproof_PROVE(sv, gamma); } -/* Given a range proof, determine if it is valid */ -bool bulletproof_VERIFY(const Bulletproof &proof) +/* Given a set of values v (0..2^N-1) and masks gamma, construct a range proof */ +Bulletproof bulletproof_PROVE(const rct::keyV &sv, const rct::keyV &gamma) { + CHECK_AND_ASSERT_THROW_MES(sv.size() == gamma.size(), "Incompatible sizes of sv and gamma"); + CHECK_AND_ASSERT_THROW_MES(!sv.empty(), "sv is empty"); + for (const rct::key &sve: sv) + CHECK_AND_ASSERT_THROW_MES(is_reduced(sve), "Invalid sv input"); + for (const rct::key &g: gamma) + CHECK_AND_ASSERT_THROW_MES(is_reduced(g), "Invalid gamma input"); + init_exponents(); - CHECK_AND_ASSERT_MES(proof.V.size() == 1, false, "V does not have exactly one element"); - CHECK_AND_ASSERT_MES(proof.L.size() == proof.R.size(), false, "Mismatched L and R sizes"); - CHECK_AND_ASSERT_MES(proof.L.size() > 0, false, "Empty proof"); - CHECK_AND_ASSERT_MES(proof.L.size() == 6, false, "Proof is not for 64 bits"); + PERF_TIMER_UNIT(PROVE, 1000000); - const size_t logN = proof.L.size(); - const size_t N = 1 << logN; + constexpr size_t logN = 6; // log2(64) + constexpr size_t N = 1<<logN; + size_t M, logM; + for (logM = 0; (M = 1<<logM) <= maxM && M < sv.size(); ++logM); + CHECK_AND_ASSERT_THROW_MES(M <= maxM, "sv/gamma are too large"); + const size_t logMN = logM + logN; + const size_t MN = M * N; + + rct::keyV V(sv.size()); + rct::keyV aL(MN), aR(MN); + rct::key tmp; - // Reconstruct the challenges - PERF_TIMER_START_BP(VERIFY); - PERF_TIMER_START_BP(VERIFY_start); - rct::key hash_cache = rct::hash_to_scalar(proof.V[0]); - rct::key y = hash_cache_mash(hash_cache, proof.A, proof.S); + PERF_TIMER_START_BP(PROVE_v); + for (size_t i = 0; i < sv.size(); ++i) + { + rct::addKeys2(V[i], gamma[i], sv[i], rct::H); + V[i] = rct::scalarmultKey(V[i], INV_EIGHT); + } + PERF_TIMER_STOP(PROVE_v); + + PERF_TIMER_START_BP(PROVE_aLaR); + for (size_t j = 0; j < M; ++j) + { + for (size_t i = N; i-- > 0; ) + { + if (j >= sv.size()) + { + aL[j*N+i] = rct::zero(); + } + else if (sv[j][i/8] & (((uint64_t)1)<<(i%8))) + { + aL[j*N+i] = rct::identity(); + } + else + { + aL[j*N+i] = rct::zero(); + } + sc_sub(aR[j*N+i].bytes, aL[j*N+i].bytes, rct::identity().bytes); + } + } + PERF_TIMER_STOP(PROVE_aLaR); + + // DEBUG: Test to ensure this recovers the value +#ifdef DEBUG_BP + for (size_t j = 0; j < M; ++j) + { + uint64_t test_aL = 0, test_aR = 0; + for (size_t i = 0; i < N; ++i) + { + if (aL[j*N+i] == rct::identity()) + test_aL += ((uint64_t)1)<<i; + if (aR[j*N+i] == rct::zero()) + test_aR += ((uint64_t)1)<<i; + } + uint64_t v_test = 0; + if (j < sv.size()) + for (int n = 0; n < 8; ++n) v_test |= (((uint64_t)sv[j][n]) << (8*n)); + CHECK_AND_ASSERT_THROW_MES(test_aL == v_test, "test_aL failed"); + CHECK_AND_ASSERT_THROW_MES(test_aR == v_test, "test_aR failed"); + } +#endif + +try_again: + rct::key hash_cache = rct::hash_to_scalar(V); + + PERF_TIMER_START_BP(PROVE_step1); + // PAPER LINES 38-39 + rct::key alpha = rct::skGen(); + rct::key ve = vector_exponent(aL, aR); + rct::key A; + rct::addKeys(A, ve, rct::scalarmultBase(alpha)); + A = rct::scalarmultKey(A, INV_EIGHT); + + // PAPER LINES 40-42 + rct::keyV sL = rct::skvGen(MN), sR = rct::skvGen(MN); + rct::key rho = rct::skGen(); + ve = vector_exponent(sL, sR); + rct::key S; + rct::addKeys(S, ve, rct::scalarmultBase(rho)); + S = rct::scalarmultKey(S, INV_EIGHT); + + // PAPER LINES 43-45 + rct::key y = hash_cache_mash(hash_cache, A, S); + if (y == rct::zero()) + { + PERF_TIMER_STOP(PROVE_step1); + MINFO("y is 0, trying again"); + goto try_again; + } rct::key z = hash_cache = rct::hash_to_scalar(y); - rct::key x = hash_cache_mash(hash_cache, z, proof.T1, proof.T2); - PERF_TIMER_STOP(VERIFY_start); + if (z == rct::zero()) + { + PERF_TIMER_STOP(PROVE_step1); + MINFO("z is 0, trying again"); + goto try_again; + } - PERF_TIMER_START_BP(VERIFY_line_60); - // Reconstruct the challenges - rct::key x_ip = hash_cache_mash(hash_cache, x, proof.taux, proof.mu, proof.t); - PERF_TIMER_STOP(VERIFY_line_60); + // Polynomial construction by coefficients + const auto zMN = vector_dup(z, MN); + rct::keyV l0 = vector_subtract(aL, zMN); + const rct::keyV &l1 = sL; - PERF_TIMER_START_BP(VERIFY_line_61); - // PAPER LINE 61 - rct::key L61Left = rct::addKeys(rct::scalarmultBase(proof.taux), rct::scalarmultKey(rct::H, proof.t)); + // This computes the ugly sum/concatenation from PAPER LINE 65 + rct::keyV zero_twos(MN); + const rct::keyV zpow = vector_powers(z, M+2); + for (size_t i = 0; i < MN; ++i) + { + zero_twos[i] = rct::zero(); + for (size_t j = 1; j <= M; ++j) + { + if (i >= (j-1)*N && i < j*N) + { + CHECK_AND_ASSERT_THROW_MES(1+j < zpow.size(), "invalid zpow index"); + CHECK_AND_ASSERT_THROW_MES(i-(j-1)*N < twoN.size(), "invalid twoN index"); + sc_muladd(zero_twos[i].bytes, zpow[1+j].bytes, twoN[i-(j-1)*N].bytes, zero_twos[i].bytes); + } + } + } - rct::key k = rct::zero(); - const auto yN = vector_powers(y, N); - rct::key ip1y = inner_product(oneN, yN); - rct::key zsq; - sc_mul(zsq.bytes, z.bytes, z.bytes); - rct::key tmp, tmp2; - sc_mulsub(k.bytes, zsq.bytes, ip1y.bytes, k.bytes); - rct::key zcu; - sc_mul(zcu.bytes, zsq.bytes, z.bytes); - sc_mulsub(k.bytes, zcu.bytes, ip12.bytes, k.bytes); - PERF_TIMER_STOP(VERIFY_line_61); + rct::keyV r0 = vector_add(aR, zMN); + const auto yMN = vector_powers(y, MN); + r0 = hadamard(r0, yMN); + r0 = vector_add(r0, zero_twos); + rct::keyV r1 = hadamard(yMN, sR); - PERF_TIMER_START_BP(VERIFY_line_61rl); - sc_muladd(tmp.bytes, z.bytes, ip1y.bytes, k.bytes); - rct::key L61Right = rct::scalarmultKey(rct::H, tmp); + // Polynomial construction before PAPER LINE 46 + rct::key t1_1 = inner_product(l0, r1); + rct::key t1_2 = inner_product(l1, r0); + rct::key t1; + sc_add(t1.bytes, t1_1.bytes, t1_2.bytes); + rct::key t2 = inner_product(l1, r1); - CHECK_AND_ASSERT_MES(proof.V.size() == 1, false, "proof.V does not have exactly one element"); - tmp = rct::scalarmultKey(proof.V[0], zsq); - rct::addKeys(L61Right, L61Right, tmp); + PERF_TIMER_STOP(PROVE_step1); - tmp = rct::scalarmultKey(proof.T1, x); - rct::addKeys(L61Right, L61Right, tmp); + PERF_TIMER_START_BP(PROVE_step2); + // PAPER LINES 47-48 + rct::key tau1 = rct::skGen(), tau2 = rct::skGen(); + + rct::key T1 = rct::addKeys(rct::scalarmultH(t1), rct::scalarmultBase(tau1)); + T1 = rct::scalarmultKey(T1, INV_EIGHT); + rct::key T2 = rct::addKeys(rct::scalarmultH(t2), rct::scalarmultBase(tau2)); + T2 = rct::scalarmultKey(T2, INV_EIGHT); + // PAPER LINES 49-51 + rct::key x = hash_cache_mash(hash_cache, z, T1, T2); + if (x == rct::zero()) + { + PERF_TIMER_STOP(PROVE_step2); + MINFO("x is 0, trying again"); + goto try_again; + } + + // PAPER LINES 52-53 + rct::key taux; + sc_mul(taux.bytes, tau1.bytes, x.bytes); rct::key xsq; sc_mul(xsq.bytes, x.bytes, x.bytes); - tmp = rct::scalarmultKey(proof.T2, xsq); - rct::addKeys(L61Right, L61Right, tmp); - PERF_TIMER_STOP(VERIFY_line_61rl); - - if (!(L61Right == L61Left)) + sc_muladd(taux.bytes, tau2.bytes, xsq.bytes, taux.bytes); + for (size_t j = 1; j <= sv.size(); ++j) { - MERROR("Verification failure at step 1"); - return false; + CHECK_AND_ASSERT_THROW_MES(j+1 < zpow.size(), "invalid zpow index"); + sc_muladd(taux.bytes, zpow[j+1].bytes, gamma[j-1].bytes, taux.bytes); } + rct::key mu; + sc_muladd(mu.bytes, x.bytes, rho.bytes, alpha.bytes); - PERF_TIMER_START_BP(VERIFY_line_62); - // PAPER LINE 62 - rct::key P = rct::addKeys(proof.A, rct::scalarmultKey(proof.S, x)); - PERF_TIMER_STOP(VERIFY_line_62); + // PAPER LINES 54-57 + rct::keyV l = l0; + l = vector_add(l, vector_scalar(l1, x)); + rct::keyV r = r0; + r = vector_add(r, vector_scalar(r1, x)); + PERF_TIMER_STOP(PROVE_step2); - // Compute the number of rounds for the inner product - const size_t rounds = proof.L.size(); - CHECK_AND_ASSERT_MES(rounds > 0, false, "Zero rounds"); + PERF_TIMER_START_BP(PROVE_step3); + rct::key t = inner_product(l, r); - PERF_TIMER_START_BP(VERIFY_line_21_22); - // PAPER LINES 21-22 - // The inner product challenges are computed per round - rct::keyV w(rounds); - for (size_t i = 0; i < rounds; ++i) + // DEBUG: Test if the l and r vectors match the polynomial forms +#ifdef DEBUG_BP + rct::key test_t; + const rct::key t0 = inner_product(l0, r0); + sc_muladd(test_t.bytes, t1.bytes, x.bytes, t0.bytes); + sc_muladd(test_t.bytes, t2.bytes, xsq.bytes, test_t.bytes); + CHECK_AND_ASSERT_THROW_MES(test_t == t, "test_t check failed"); +#endif + + // PAPER LINES 32-33 + rct::key x_ip = hash_cache_mash(hash_cache, x, taux, mu, t); + if (x_ip == rct::zero()) { - w[i] = hash_cache_mash(hash_cache, proof.L[i], proof.R[i]); + PERF_TIMER_STOP(PROVE_step3); + MINFO("x_ip is 0, trying again"); + goto try_again; } - PERF_TIMER_STOP(VERIFY_line_21_22); - PERF_TIMER_START_BP(VERIFY_line_24_25); - // Basically PAPER LINES 24-25 - // Compute the curvepoints from G[i] and H[i] - rct::key inner_prod = rct::identity(); + // These are used in the inner product rounds + size_t nprime = MN; + rct::keyV Gprime(MN); + rct::keyV Hprime(MN); + rct::keyV aprime(MN); + rct::keyV bprime(MN); + const rct::key yinv = invert(y); rct::key yinvpow = rct::identity(); - rct::key ypow = rct::identity(); + for (size_t i = 0; i < MN; ++i) + { + Gprime[i] = Gi[i]; + Hprime[i] = scalarmultKey(Hi_p3[i], yinvpow); + sc_mul(yinvpow.bytes, yinvpow.bytes, yinv.bytes); + aprime[i] = l[i]; + bprime[i] = r[i]; + } + rct::keyV L(logMN); + rct::keyV R(logMN); + int round = 0; + rct::keyV w(logMN); // this is the challenge x in the inner product protocol + PERF_TIMER_STOP(PROVE_step3); - PERF_TIMER_START_BP(VERIFY_line_24_25_invert); - const rct::key yinv = invert(y); - rct::keyV winv(rounds); - for (size_t i = 0; i < rounds; ++i) - winv[i] = invert(w[i]); - PERF_TIMER_STOP(VERIFY_line_24_25_invert); + PERF_TIMER_START_BP(PROVE_step4); + // PAPER LINE 13 + while (nprime > 1) + { + // PAPER LINE 15 + nprime /= 2; - for (size_t i = 0; i < N; ++i) + // PAPER LINES 16-17 + rct::key cL = inner_product(slice(aprime, 0, nprime), slice(bprime, nprime, bprime.size())); + rct::key cR = inner_product(slice(aprime, nprime, aprime.size()), slice(bprime, 0, nprime)); + + // PAPER LINES 18-19 + L[round] = vector_exponent_custom(slice(Gprime, nprime, Gprime.size()), slice(Hprime, 0, nprime), slice(aprime, 0, nprime), slice(bprime, nprime, bprime.size())); + sc_mul(tmp.bytes, cL.bytes, x_ip.bytes); + rct::addKeys(L[round], L[round], rct::scalarmultH(tmp)); + L[round] = rct::scalarmultKey(L[round], INV_EIGHT); + R[round] = vector_exponent_custom(slice(Gprime, 0, nprime), slice(Hprime, nprime, Hprime.size()), slice(aprime, nprime, aprime.size()), slice(bprime, 0, nprime)); + sc_mul(tmp.bytes, cR.bytes, x_ip.bytes); + rct::addKeys(R[round], R[round], rct::scalarmultH(tmp)); + R[round] = rct::scalarmultKey(R[round], INV_EIGHT); + + // PAPER LINES 21-22 + w[round] = hash_cache_mash(hash_cache, L[round], R[round]); + if (w[round] == rct::zero()) + { + PERF_TIMER_STOP(PROVE_step4); + MINFO("w[round] is 0, trying again"); + goto try_again; + } + + // PAPER LINES 24-25 + const rct::key winv = invert(w[round]); + Gprime = hadamard2(vector_scalar2(slice(Gprime, 0, nprime), winv), vector_scalar2(slice(Gprime, nprime, Gprime.size()), w[round])); + Hprime = hadamard2(vector_scalar2(slice(Hprime, 0, nprime), w[round]), vector_scalar2(slice(Hprime, nprime, Hprime.size()), winv)); + + // PAPER LINES 28-29 + aprime = vector_add(vector_scalar(slice(aprime, 0, nprime), w[round]), vector_scalar(slice(aprime, nprime, aprime.size()), winv)); + bprime = vector_add(vector_scalar(slice(bprime, 0, nprime), winv), vector_scalar(slice(bprime, nprime, bprime.size()), w[round])); + + ++round; + } + PERF_TIMER_STOP(PROVE_step4); + + // PAPER LINE 58 (with inclusions from PAPER LINE 8 and PAPER LINE 20) + return Bulletproof(V, A, S, T1, T2, taux, mu, L, R, aprime[0], bprime[0], t); +} + +Bulletproof bulletproof_PROVE(const std::vector<uint64_t> &v, const rct::keyV &gamma) +{ + CHECK_AND_ASSERT_THROW_MES(v.size() == gamma.size(), "Incompatible sizes of v and gamma"); + + // vG + gammaH + PERF_TIMER_START_BP(PROVE_v); + rct::keyV sv(v.size()); + for (size_t i = 0; i < v.size(); ++i) { - // Convert the index to binary IN REVERSE and construct the scalar exponent - rct::key g_scalar = proof.a; - rct::key h_scalar; - sc_mul(h_scalar.bytes, proof.b.bytes, yinvpow.bytes); + sv[i] = rct::zero(); + sv[i].bytes[0] = v[i] & 255; + sv[i].bytes[1] = (v[i] >> 8) & 255; + sv[i].bytes[2] = (v[i] >> 16) & 255; + sv[i].bytes[3] = (v[i] >> 24) & 255; + sv[i].bytes[4] = (v[i] >> 32) & 255; + sv[i].bytes[5] = (v[i] >> 40) & 255; + sv[i].bytes[6] = (v[i] >> 48) & 255; + sv[i].bytes[7] = (v[i] >> 56) & 255; + } + PERF_TIMER_STOP(PROVE_v); + return bulletproof_PROVE(sv, gamma); +} + +/* Given a range proof, determine if it is valid */ +bool bulletproof_VERIFY(const std::vector<const Bulletproof*> &proofs) +{ + init_exponents(); + + PERF_TIMER_START_BP(VERIFY); + + // sanity and figure out which proof is longest + size_t max_length = 0; + for (const Bulletproof *p: proofs) + { + const Bulletproof &proof = *p; + + // check scalar range + CHECK_AND_ASSERT_MES(is_reduced(proof.taux), false, "Input scalar not in range"); + CHECK_AND_ASSERT_MES(is_reduced(proof.mu), false, "Input scalar not in range"); + CHECK_AND_ASSERT_MES(is_reduced(proof.a), false, "Input scalar not in range"); + CHECK_AND_ASSERT_MES(is_reduced(proof.b), false, "Input scalar not in range"); + CHECK_AND_ASSERT_MES(is_reduced(proof.t), false, "Input scalar not in range"); + + CHECK_AND_ASSERT_MES(proof.V.size() >= 1, false, "V does not have at least one element"); + CHECK_AND_ASSERT_MES(proof.L.size() == proof.R.size(), false, "Mismatched L and R sizes"); + CHECK_AND_ASSERT_MES(proof.L.size() > 0, false, "Empty proof"); - for (size_t j = rounds; j-- > 0; ) + max_length = std::max(max_length, proof.L.size()); + } + CHECK_AND_ASSERT_MES(max_length < 32, false, "At least one proof is too large"); + size_t maxMN = 1u << max_length; + + const size_t logN = 6; + const size_t N = 1 << logN; + rct::key tmp; + + // setup weighted aggregates + rct::key Z0 = rct::identity(); + rct::key z1 = rct::zero(); + rct::key Z2 = rct::identity(); + rct::key z3 = rct::zero(); + rct::keyV z4(maxMN, rct::zero()), z5(maxMN, rct::zero()); + rct::key Y2 = rct::identity(), Y3 = rct::identity(), Y4 = rct::identity(); + rct::key y0 = rct::zero(), y1 = rct::zero(); + for (const Bulletproof *p: proofs) + { + const Bulletproof &proof = *p; + + size_t M, logM; + for (logM = 0; (M = 1<<logM) <= maxM && M < proof.V.size(); ++logM); + CHECK_AND_ASSERT_MES(proof.L.size() == 6+logM, false, "Proof is not the expected size"); + const size_t MN = M*N; + rct::key weight = rct::skGen(); + + // Reconstruct the challenges + PERF_TIMER_START_BP(VERIFY_start); + rct::key hash_cache = rct::hash_to_scalar(proof.V); + rct::key y = hash_cache_mash(hash_cache, proof.A, proof.S); + CHECK_AND_ASSERT_MES(!(y == rct::zero()), false, "y == 0"); + rct::key z = hash_cache = rct::hash_to_scalar(y); + CHECK_AND_ASSERT_MES(!(z == rct::zero()), false, "z == 0"); + rct::key x = hash_cache_mash(hash_cache, z, proof.T1, proof.T2); + CHECK_AND_ASSERT_MES(!(x == rct::zero()), false, "x == 0"); + rct::key x_ip = hash_cache_mash(hash_cache, x, proof.taux, proof.mu, proof.t); + CHECK_AND_ASSERT_MES(!(x_ip == rct::zero()), false, "x_ip == 0"); + PERF_TIMER_STOP(VERIFY_start); + + PERF_TIMER_START_BP(VERIFY_line_61); + // PAPER LINE 61 + sc_muladd(y0.bytes, proof.taux.bytes, weight.bytes, y0.bytes); + + const rct::keyV zpow = vector_powers(z, M+3); + + rct::key k; + const rct::key ip1y = vector_power_sum(y, MN); + sc_mulsub(k.bytes, zpow[2].bytes, ip1y.bytes, rct::zero().bytes); + for (size_t j = 1; j <= M; ++j) + { + CHECK_AND_ASSERT_MES(j+2 < zpow.size(), false, "invalid zpow index"); + sc_mulsub(k.bytes, zpow[j+2].bytes, ip12.bytes, k.bytes); + } + PERF_TIMER_STOP(VERIFY_line_61); + + PERF_TIMER_START_BP(VERIFY_line_61rl_new); + sc_muladd(tmp.bytes, z.bytes, ip1y.bytes, k.bytes); + std::vector<MultiexpData> multiexp_data; + multiexp_data.reserve(proof.V.size()); + sc_sub(tmp.bytes, proof.t.bytes, tmp.bytes); + sc_muladd(y1.bytes, tmp.bytes, weight.bytes, y1.bytes); + for (size_t j = 0; j < proof.V.size(); j++) { - size_t J = w.size() - j - 1; + sc_mul(tmp.bytes, zpow[j+2].bytes, EIGHT.bytes); + multiexp_data.emplace_back(tmp, proof.V[j]); + } + rct::addKeys(Y2, Y2, rct::scalarmultKey(multiexp(multiexp_data, false), weight)); + rct::key weight8; + sc_mul(weight8.bytes, weight.bytes, EIGHT.bytes); + sc_mul(tmp.bytes, x.bytes, weight8.bytes); + rct::addKeys(Y3, Y3, rct::scalarmultKey(proof.T1, tmp)); + rct::key xsq; + sc_mul(xsq.bytes, x.bytes, x.bytes); + sc_mul(tmp.bytes, xsq.bytes, weight8.bytes); + rct::addKeys(Y4, Y4, rct::scalarmultKey(proof.T2, tmp)); + PERF_TIMER_STOP(VERIFY_line_61rl_new); + + PERF_TIMER_START_BP(VERIFY_line_62); + // PAPER LINE 62 + sc_mul(tmp.bytes, x.bytes, EIGHT.bytes); + rct::addKeys(Z0, Z0, rct::scalarmultKey(rct::addKeys(rct::scalarmult8(proof.A), rct::scalarmultKey(proof.S, tmp)), weight)); + PERF_TIMER_STOP(VERIFY_line_62); + + // Compute the number of rounds for the inner product + const size_t rounds = logM+logN; + CHECK_AND_ASSERT_MES(rounds > 0, false, "Zero rounds"); + + PERF_TIMER_START_BP(VERIFY_line_21_22); + // PAPER LINES 21-22 + // The inner product challenges are computed per round + rct::keyV w(rounds); + for (size_t i = 0; i < rounds; ++i) + { + w[i] = hash_cache_mash(hash_cache, proof.L[i], proof.R[i]); + CHECK_AND_ASSERT_MES(!(w[i] == rct::zero()), false, "w[i] == 0"); + } + PERF_TIMER_STOP(VERIFY_line_21_22); + + PERF_TIMER_START_BP(VERIFY_line_24_25); + // Basically PAPER LINES 24-25 + // Compute the curvepoints from G[i] and H[i] + rct::key yinvpow = rct::identity(); + rct::key ypow = rct::identity(); + + PERF_TIMER_START_BP(VERIFY_line_24_25_invert); + const rct::key yinv = invert(y); + rct::keyV winv(rounds); + for (size_t i = 0; i < rounds; ++i) + winv[i] = invert(w[i]); + PERF_TIMER_STOP(VERIFY_line_24_25_invert); + + for (size_t i = 0; i < MN; ++i) + { + // Convert the index to binary IN REVERSE and construct the scalar exponent + rct::key g_scalar = proof.a; + rct::key h_scalar; + sc_mul(h_scalar.bytes, proof.b.bytes, yinvpow.bytes); - if ((i & (((size_t)1)<<j)) == 0) + for (size_t j = rounds; j-- > 0; ) { - sc_mul(g_scalar.bytes, g_scalar.bytes, winv[J].bytes); - sc_mul(h_scalar.bytes, h_scalar.bytes, w[J].bytes); + size_t J = w.size() - j - 1; + + if ((i & (((size_t)1)<<j)) == 0) + { + sc_mul(g_scalar.bytes, g_scalar.bytes, winv[J].bytes); + sc_mul(h_scalar.bytes, h_scalar.bytes, w[J].bytes); + } + else + { + sc_mul(g_scalar.bytes, g_scalar.bytes, w[J].bytes); + sc_mul(h_scalar.bytes, h_scalar.bytes, winv[J].bytes); + } } - else + + // Adjust the scalars using the exponents from PAPER LINE 62 + sc_add(g_scalar.bytes, g_scalar.bytes, z.bytes); + CHECK_AND_ASSERT_MES(2+i/N < zpow.size(), false, "invalid zpow index"); + CHECK_AND_ASSERT_MES(i%N < twoN.size(), false, "invalid twoN index"); + sc_mul(tmp.bytes, zpow[2+i/N].bytes, twoN[i%N].bytes); + sc_muladd(tmp.bytes, z.bytes, ypow.bytes, tmp.bytes); + sc_mulsub(h_scalar.bytes, tmp.bytes, yinvpow.bytes, h_scalar.bytes); + + sc_muladd(z4[i].bytes, g_scalar.bytes, weight.bytes, z4[i].bytes); + sc_muladd(z5[i].bytes, h_scalar.bytes, weight.bytes, z5[i].bytes); + + if (i != MN-1) { - sc_mul(g_scalar.bytes, g_scalar.bytes, w[J].bytes); - sc_mul(h_scalar.bytes, h_scalar.bytes, winv[J].bytes); + sc_mul(yinvpow.bytes, yinvpow.bytes, yinv.bytes); + sc_mul(ypow.bytes, ypow.bytes, y.bytes); } } - // Adjust the scalars using the exponents from PAPER LINE 62 - sc_add(g_scalar.bytes, g_scalar.bytes, z.bytes); - sc_mul(tmp.bytes, zsq.bytes, twoN[i].bytes); - sc_muladd(tmp.bytes, z.bytes, ypow.bytes, tmp.bytes); - sc_mulsub(h_scalar.bytes, tmp.bytes, yinvpow.bytes, h_scalar.bytes); + PERF_TIMER_STOP(VERIFY_line_24_25); - // Now compute the basepoint's scalar multiplication - // Each of these could be written as a multiexp operation instead - rct::addKeys3(tmp, g_scalar, Gprecomp[i], h_scalar, Hprecomp[i]); - rct::addKeys(inner_prod, inner_prod, tmp); + // PAPER LINE 26 + PERF_TIMER_START_BP(VERIFY_line_26_new); + multiexp_data.clear(); + multiexp_data.reserve(2*rounds); - if (i != N-1) + sc_muladd(z1.bytes, proof.mu.bytes, weight.bytes, z1.bytes); + for (size_t i = 0; i < rounds; ++i) { - sc_mul(yinvpow.bytes, yinvpow.bytes, yinv.bytes); - sc_mul(ypow.bytes, ypow.bytes, y.bytes); + sc_mul(tmp.bytes, w[i].bytes, w[i].bytes); + sc_mul(tmp.bytes, tmp.bytes, EIGHT.bytes); + multiexp_data.emplace_back(tmp, proof.L[i]); + sc_mul(tmp.bytes, winv[i].bytes, winv[i].bytes); + sc_mul(tmp.bytes, tmp.bytes, EIGHT.bytes); + multiexp_data.emplace_back(tmp, proof.R[i]); } + rct::key acc = multiexp(multiexp_data, false); + rct::addKeys(Z2, Z2, rct::scalarmultKey(acc, weight)); + sc_mulsub(tmp.bytes, proof.a.bytes, proof.b.bytes, proof.t.bytes); + sc_mul(tmp.bytes, tmp.bytes, x_ip.bytes); + sc_muladd(z3.bytes, tmp.bytes, weight.bytes, z3.bytes); + PERF_TIMER_STOP(VERIFY_line_26_new); } - PERF_TIMER_STOP(VERIFY_line_24_25); - - PERF_TIMER_START_BP(VERIFY_line_26); - // PAPER LINE 26 - rct::key pprime; - sc_sub(tmp.bytes, rct::zero().bytes, proof.mu.bytes); - rct::addKeys(pprime, P, rct::scalarmultBase(tmp)); - - for (size_t i = 0; i < rounds; ++i) - { - sc_mul(tmp.bytes, w[i].bytes, w[i].bytes); - sc_mul(tmp2.bytes, winv[i].bytes, winv[i].bytes); -#if 1 - ge_dsmp cacheL, cacheR; - rct::precomp(cacheL, proof.L[i]); - rct::precomp(cacheR, proof.R[i]); - rct::addKeys3(tmp, tmp, cacheL, tmp2, cacheR); - rct::addKeys(pprime, pprime, tmp); -#else - rct::addKeys(pprime, pprime, rct::scalarmultKey(proof.L[i], tmp)); - rct::addKeys(pprime, pprime, rct::scalarmultKey(proof.R[i], tmp2)); -#endif - } - sc_mul(tmp.bytes, proof.t.bytes, x_ip.bytes); - rct::addKeys(pprime, pprime, rct::scalarmultKey(rct::H, tmp)); - PERF_TIMER_STOP(VERIFY_line_26); + // now check all proofs at once PERF_TIMER_START_BP(VERIFY_step2_check); - sc_mul(tmp.bytes, proof.a.bytes, proof.b.bytes); - sc_mul(tmp.bytes, tmp.bytes, x_ip.bytes); - tmp = rct::scalarmultKey(rct::H, tmp); - rct::addKeys(tmp, tmp, inner_prod); + ge_p3 check1; + ge_scalarmult_base(&check1, y0.bytes); + addKeys_acc_p3(&check1, y1, rct::H); + sub_acc_p3(&check1, Y2); + sub_acc_p3(&check1, Y3); + sub_acc_p3(&check1, Y4); + if (!ge_p3_is_point_at_infinity(&check1)) + { + MERROR("Verification failure at step 1"); + return false; + } + ge_p3 check2; + sc_sub(tmp.bytes, rct::zero().bytes, z1.bytes); + ge_double_scalarmult_base_vartime_p3(&check2, z3.bytes, &ge_p3_H, tmp.bytes); + add_acc_p3(&check2, Z0); + add_acc_p3(&check2, Z2); + + std::vector<MultiexpData> multiexp_data; + multiexp_data.reserve(2 * maxMN); + for (size_t i = 0; i < maxMN; ++i) + { + sc_sub(tmp.bytes, rct::zero().bytes, z4[i].bytes); + multiexp_data.emplace_back(tmp, Gi_p3[i]); + sc_sub(tmp.bytes, rct::zero().bytes, z5[i].bytes); + multiexp_data.emplace_back(tmp, Hi_p3[i]); + } + add_acc_p3(&check2, multiexp(multiexp_data, true)); PERF_TIMER_STOP(VERIFY_step2_check); - if (!(pprime == tmp)) + + if (!ge_p3_is_point_at_infinity(&check2)) { MERROR("Verification failure at step 2"); return false; @@ -749,4 +1247,19 @@ bool bulletproof_VERIFY(const Bulletproof &proof) return true; } +bool bulletproof_VERIFY(const std::vector<Bulletproof> &proofs) +{ + std::vector<const Bulletproof*> proof_pointers; + for (const Bulletproof &proof: proofs) + proof_pointers.push_back(&proof); + return bulletproof_VERIFY(proof_pointers); +} + +bool bulletproof_VERIFY(const Bulletproof &proof) +{ + std::vector<const Bulletproof*> proofs; + proofs.push_back(&proof); + return bulletproof_VERIFY(proofs); +} + } diff --git a/src/ringct/bulletproofs.h b/src/ringct/bulletproofs.h index 3061d272e..b86202ccc 100644 --- a/src/ringct/bulletproofs.h +++ b/src/ringct/bulletproofs.h @@ -40,7 +40,11 @@ namespace rct Bulletproof bulletproof_PROVE(const rct::key &v, const rct::key &gamma); Bulletproof bulletproof_PROVE(uint64_t v, const rct::key &gamma); +Bulletproof bulletproof_PROVE(const rct::keyV &v, const rct::keyV &gamma); +Bulletproof bulletproof_PROVE(const std::vector<uint64_t> &v, const rct::keyV &gamma); bool bulletproof_VERIFY(const Bulletproof &proof); +bool bulletproof_VERIFY(const std::vector<const Bulletproof*> &proofs); +bool bulletproof_VERIFY(const std::vector<Bulletproof> &proofs); } diff --git a/src/ringct/multiexp.cc b/src/ringct/multiexp.cc new file mode 100644 index 000000000..21957b94c --- /dev/null +++ b/src/ringct/multiexp.cc @@ -0,0 +1,665 @@ +// Copyright (c) 2017, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Adapted from Python code by Sarang Noether + +#include "misc_log_ex.h" +#include "common/perf_timer.h" +extern "C" +{ +#include "crypto/crypto-ops.h" +} +#include "common/aligned.h" +#include "rctOps.h" +#include "multiexp.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "multiexp" + +//#define MULTIEXP_PERF(x) x +#define MULTIEXP_PERF(x) + +#define RAW_MEMORY_BLOCK +//#define ALTERNATE_LAYOUT +//#define TRACK_STRAUS_ZERO_IDENTITY + +// per points us for N/B points (B point bands) +// raw alt 128/192 4096/192 4096/4096 +// 0 0 52.6 71 71.2 +// 0 1 53.2 72.2 72.4 +// 1 0 52.7 67 67.1 +// 1 1 52.8 70.4 70.2 + +// Pippenger: +// 1 2 3 4 5 6 7 8 9 bestN +// 2 555 598 621 804 1038 1733 2486 5020 8304 1 +// 4 783 747 800 1006 1428 2132 3285 5185 9806 2 +// 8 1174 1071 1095 1286 1640 2398 3869 6378 12080 2 +// 16 2279 1874 1745 1739 2144 2831 4209 6964 12007 4 +// 32 3910 3706 2588 2477 2782 3467 4856 7489 12618 4 +// 64 7184 5429 4710 4368 4010 4672 6027 8559 13684 5 +// 128 14097 10574 8452 7297 6841 6718 8615 10580 15641 6 +// 256 27715 20800 16000 13550 11875 11400 11505 14090 18460 6 +// 512 55100 41250 31740 26570 22030 19830 20760 21380 25215 6 +// 1024 111520 79000 61080 49720 43080 38320 37600 35040 36750 8 +// 2048 219480 162680 122120 102080 83760 70360 66600 63920 66160 8 +// 4096 453320 323080 247240 210200 180040 150240 132440 114920 110560 9 + +// 2 4 8 16 32 64 128 256 512 1024 2048 4096 +// Bos Coster 858 994 1316 1949 3183 5512 9865 17830 33485 63160 124280 246320 +// Straus 226 341 548 980 1870 3538 7039 14490 29020 57200 118640 233640 +// Straus/cached 226 315 485 785 1514 2858 5753 11065 22970 45120 98880 194840 +// Pippenger 555 747 1071 1739 2477 4010 6718 11400 19830 35040 63920 110560 + +// Best/cached Straus Straus Straus Straus Straus Straus Straus Straus Pip Pip Pip Pip +// Best/uncached Straus Straus Straus Straus Straus Straus Pip Pip Pip Pip Pip Pip + +namespace rct +{ + +static inline bool operator<(const rct::key &k0, const rct::key&k1) +{ + for (int n = 31; n >= 0; --n) + { + if (k0.bytes[n] < k1.bytes[n]) + return true; + if (k0.bytes[n] > k1.bytes[n]) + return false; + } + return false; +} + +static inline rct::key div2(const rct::key &k) +{ + rct::key res; + int carry = 0; + for (int n = 31; n >= 0; --n) + { + int new_carry = (k.bytes[n] & 1) << 7; + res.bytes[n] = k.bytes[n] / 2 + carry; + carry = new_carry; + } + return res; +} + +static inline rct::key pow2(size_t n) +{ + CHECK_AND_ASSERT_THROW_MES(n < 256, "Invalid pow2 argument"); + rct::key res = rct::zero(); + res[n >> 3] |= 1<<(n&7); + return res; +} + +static inline int test(const rct::key &k, size_t n) +{ + if (n >= 256) return 0; + return k[n >> 3] & (1 << (n & 7)); +} + +static inline void add(ge_p3 &p3, const ge_cached &other) +{ + ge_p1p1 p1; + ge_add(&p1, &p3, &other); + ge_p1p1_to_p3(&p3, &p1); +} + +static inline void add(ge_p3 &p3, const ge_p3 &other) +{ + ge_cached cached; + ge_p3_to_cached(&cached, &other); + add(p3, cached); +} + +rct::key bos_coster_heap_conv(std::vector<MultiexpData> data) +{ + MULTIEXP_PERF(PERF_TIMER_START_UNIT(bos_coster, 1000000)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(setup, 1000000)); + size_t points = data.size(); + CHECK_AND_ASSERT_THROW_MES(points > 1, "Not enough points"); + std::vector<size_t> heap(points); + for (size_t n = 0; n < points; ++n) + heap[n] = n; + + auto Comp = [&](size_t e0, size_t e1) { return data[e0].scalar < data[e1].scalar; }; + std::make_heap(heap.begin(), heap.end(), Comp); + MULTIEXP_PERF(PERF_TIMER_STOP(setup)); + + MULTIEXP_PERF(PERF_TIMER_START_UNIT(loop, 1000000)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(pop, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(pop)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(add, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(add)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(sub, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(sub)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(push, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(push)); + while (heap.size() > 1) + { + MULTIEXP_PERF(PERF_TIMER_RESUME(pop)); + std::pop_heap(heap.begin(), heap.end(), Comp); + size_t index1 = heap.back(); + heap.pop_back(); + std::pop_heap(heap.begin(), heap.end(), Comp); + size_t index2 = heap.back(); + heap.pop_back(); + MULTIEXP_PERF(PERF_TIMER_PAUSE(pop)); + + MULTIEXP_PERF(PERF_TIMER_RESUME(add)); + ge_cached cached; + ge_p3_to_cached(&cached, &data[index1].point); + ge_p1p1 p1; + ge_add(&p1, &data[index2].point, &cached); + ge_p1p1_to_p3(&data[index2].point, &p1); + MULTIEXP_PERF(PERF_TIMER_PAUSE(add)); + + MULTIEXP_PERF(PERF_TIMER_RESUME(sub)); + sc_sub(data[index1].scalar.bytes, data[index1].scalar.bytes, data[index2].scalar.bytes); + MULTIEXP_PERF(PERF_TIMER_PAUSE(sub)); + + MULTIEXP_PERF(PERF_TIMER_RESUME(push)); + if (!(data[index1].scalar == rct::zero())) + { + heap.push_back(index1); + std::push_heap(heap.begin(), heap.end(), Comp); + } + + heap.push_back(index2); + std::push_heap(heap.begin(), heap.end(), Comp); + MULTIEXP_PERF(PERF_TIMER_PAUSE(push)); + } + MULTIEXP_PERF(PERF_TIMER_STOP(push)); + MULTIEXP_PERF(PERF_TIMER_STOP(sub)); + MULTIEXP_PERF(PERF_TIMER_STOP(add)); + MULTIEXP_PERF(PERF_TIMER_STOP(pop)); + MULTIEXP_PERF(PERF_TIMER_STOP(loop)); + + MULTIEXP_PERF(PERF_TIMER_START_UNIT(end, 1000000)); + //return rct::scalarmultKey(data[index1].point, data[index1].scalar); + std::pop_heap(heap.begin(), heap.end(), Comp); + size_t index1 = heap.back(); + heap.pop_back(); + ge_p2 p2; + ge_scalarmult(&p2, data[index1].scalar.bytes, &data[index1].point); + rct::key res; + ge_tobytes(res.bytes, &p2); + return res; +} + +rct::key bos_coster_heap_conv_robust(std::vector<MultiexpData> data) +{ + MULTIEXP_PERF(PERF_TIMER_START_UNIT(bos_coster, 1000000)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(setup, 1000000)); + size_t points = data.size(); + CHECK_AND_ASSERT_THROW_MES(points > 0, "Not enough points"); + std::vector<size_t> heap; + heap.reserve(points); + for (size_t n = 0; n < points; ++n) + { + if (!(data[n].scalar == rct::zero()) && !ge_p3_is_point_at_infinity(&data[n].point)) + heap.push_back(n); + } + points = heap.size(); + if (points == 0) + return rct::identity(); + + auto Comp = [&](size_t e0, size_t e1) { return data[e0].scalar < data[e1].scalar; }; + std::make_heap(heap.begin(), heap.end(), Comp); + + if (points < 2) + { + std::pop_heap(heap.begin(), heap.end(), Comp); + size_t index1 = heap.back(); + ge_p2 p2; + ge_scalarmult(&p2, data[index1].scalar.bytes, &data[index1].point); + rct::key res; + ge_tobytes(res.bytes, &p2); + return res; + } + + MULTIEXP_PERF(PERF_TIMER_STOP(setup)); + + MULTIEXP_PERF(PERF_TIMER_START_UNIT(loop, 1000000)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(pop, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(pop)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(div, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(div)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(add, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(add)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(sub, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(sub)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(push, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(push)); + while (heap.size() > 1) + { + MULTIEXP_PERF(PERF_TIMER_RESUME(pop)); + std::pop_heap(heap.begin(), heap.end(), Comp); + size_t index1 = heap.back(); + heap.pop_back(); + std::pop_heap(heap.begin(), heap.end(), Comp); + size_t index2 = heap.back(); + heap.pop_back(); + MULTIEXP_PERF(PERF_TIMER_PAUSE(pop)); + + ge_cached cached; + ge_p1p1 p1; + ge_p2 p2; + + MULTIEXP_PERF(PERF_TIMER_RESUME(div)); + while (1) + { + rct::key s1_2 = div2(data[index1].scalar); + if (!(data[index2].scalar < s1_2)) + break; + if (data[index1].scalar.bytes[0] & 1) + { + data.resize(data.size()+1); + data.back().scalar = rct::identity(); + data.back().point = data[index1].point; + heap.push_back(data.size() - 1); + std::push_heap(heap.begin(), heap.end(), Comp); + } + data[index1].scalar = div2(data[index1].scalar); + ge_p3_to_p2(&p2, &data[index1].point); + ge_p2_dbl(&p1, &p2); + ge_p1p1_to_p3(&data[index1].point, &p1); + } + MULTIEXP_PERF(PERF_TIMER_PAUSE(div)); + + MULTIEXP_PERF(PERF_TIMER_RESUME(add)); + ge_p3_to_cached(&cached, &data[index1].point); + ge_add(&p1, &data[index2].point, &cached); + ge_p1p1_to_p3(&data[index2].point, &p1); + MULTIEXP_PERF(PERF_TIMER_PAUSE(add)); + + MULTIEXP_PERF(PERF_TIMER_RESUME(sub)); + sc_sub(data[index1].scalar.bytes, data[index1].scalar.bytes, data[index2].scalar.bytes); + MULTIEXP_PERF(PERF_TIMER_PAUSE(sub)); + + MULTIEXP_PERF(PERF_TIMER_RESUME(push)); + if (!(data[index1].scalar == rct::zero())) + { + heap.push_back(index1); + std::push_heap(heap.begin(), heap.end(), Comp); + } + + heap.push_back(index2); + std::push_heap(heap.begin(), heap.end(), Comp); + MULTIEXP_PERF(PERF_TIMER_PAUSE(push)); + } + MULTIEXP_PERF(PERF_TIMER_STOP(push)); + MULTIEXP_PERF(PERF_TIMER_STOP(sub)); + MULTIEXP_PERF(PERF_TIMER_STOP(add)); + MULTIEXP_PERF(PERF_TIMER_STOP(pop)); + MULTIEXP_PERF(PERF_TIMER_STOP(loop)); + + MULTIEXP_PERF(PERF_TIMER_START_UNIT(end, 1000000)); + //return rct::scalarmultKey(data[index1].point, data[index1].scalar); + std::pop_heap(heap.begin(), heap.end(), Comp); + size_t index1 = heap.back(); + heap.pop_back(); + ge_p2 p2; + ge_scalarmult(&p2, data[index1].scalar.bytes, &data[index1].point); + rct::key res; + ge_tobytes(res.bytes, &p2); + return res; +} + +static constexpr unsigned int STRAUS_C = 4; + +struct straus_cached_data +{ +#ifdef RAW_MEMORY_BLOCK + size_t size; + ge_cached *multiples; + straus_cached_data(): size(0), multiples(NULL) {} + ~straus_cached_data() { aligned_free(multiples); } +#else + std::vector<std::vector<ge_cached>> multiples; +#endif +}; +#ifdef RAW_MEMORY_BLOCK +#ifdef ALTERNATE_LAYOUT +#define CACHE_OFFSET(cache,point,digit) cache->multiples[(point)*((1<<STRAUS_C)-1)+((digit)-1)] +#else +#define CACHE_OFFSET(cache,point,digit) cache->multiples[(point)+cache->size*((digit)-1)] +#endif +#else +#ifdef ALTERNATE_LAYOUT +#define CACHE_OFFSET(cache,point,digit) local_cache->multiples[j][digit-1] +#else +#define CACHE_OFFSET(cache,point,digit) local_cache->multiples[digit][j] +#endif +#endif + +std::shared_ptr<straus_cached_data> straus_init_cache(const std::vector<MultiexpData> &data, size_t N) +{ + MULTIEXP_PERF(PERF_TIMER_START_UNIT(multiples, 1000000)); + if (N == 0) + N = data.size(); + CHECK_AND_ASSERT_THROW_MES(N <= data.size(), "Bad cache base data"); + ge_cached cached; + ge_p1p1 p1; + ge_p3 p3; + std::shared_ptr<straus_cached_data> cache(new straus_cached_data()); + +#ifdef RAW_MEMORY_BLOCK + const size_t offset = cache->size; + cache->multiples = (ge_cached*)aligned_realloc(cache->multiples, sizeof(ge_cached) * ((1<<STRAUS_C)-1) * std::max(offset, N), 4096); + CHECK_AND_ASSERT_THROW_MES(cache->multiples, "Out of memory"); + cache->size = N; + for (size_t j=offset;j<N;++j) + { + ge_p3_to_cached(&CACHE_OFFSET(cache, j, 1), &data[j].point); + for (size_t i=2;i<1<<STRAUS_C;++i) + { + ge_add(&p1, &data[j].point, &CACHE_OFFSET(cache, j, i-1)); + ge_p1p1_to_p3(&p3, &p1); + ge_p3_to_cached(&CACHE_OFFSET(cache, j, i), &p3); + } + } +#else +#ifdef ALTERNATE_LAYOUT + const size_t offset = cache->multiples.size(); + cache->multiples.resize(std::max(offset, N)); + for (size_t i = offset; i < N; ++i) + { + cache->multiples[i].resize((1<<STRAUS_C)-1); + ge_p3_to_cached(&cache->multiples[i][0], &data[i].point); + for (size_t j=2;j<1<<STRAUS_C;++j) + { + ge_add(&p1, &data[i].point, &cache->multiples[i][j-2]); + ge_p1p1_to_p3(&p3, &p1); + ge_p3_to_cached(&cache->multiples[i][j-1], &p3); + } + } +#else + cache->multiples.resize(1<<STRAUS_C); + size_t offset = cache->multiples[1].size(); + cache->multiples[1].resize(std::max(offset, N)); + for (size_t i = offset; i < N; ++i) + ge_p3_to_cached(&cache->multiples[1][i], &data[i].point); + for (size_t i=2;i<1<<STRAUS_C;++i) + cache->multiples[i].resize(std::max(offset, N)); + for (size_t j=offset;j<N;++j) + { + for (size_t i=2;i<1<<STRAUS_C;++i) + { + ge_add(&p1, &data[j].point, &cache->multiples[i-1][j]); + ge_p1p1_to_p3(&p3, &p1); + ge_p3_to_cached(&cache->multiples[i][j], &p3); + } + } +#endif +#endif + MULTIEXP_PERF(PERF_TIMER_STOP(multiples)); + + return cache; +} + +size_t straus_get_cache_size(const std::shared_ptr<straus_cached_data> &cache) +{ + size_t sz = 0; +#ifdef RAW_MEMORY_BLOCK + sz += cache->size * sizeof(ge_cached) * ((1<<STRAUS_C)-1); +#else + for (const auto &e0: cache->multiples) + sz += e0.size() * sizeof(ge_cached); +#endif + return sz; +} + +rct::key straus(const std::vector<MultiexpData> &data, const std::shared_ptr<straus_cached_data> &cache, size_t STEP) +{ + CHECK_AND_ASSERT_THROW_MES(cache == NULL || cache->size >= data.size(), "Cache is too small"); + MULTIEXP_PERF(PERF_TIMER_UNIT(straus, 1000000)); + bool HiGi = cache != NULL; + STEP = STEP ? STEP : 192; + + MULTIEXP_PERF(PERF_TIMER_START_UNIT(setup, 1000000)); + static constexpr unsigned int mask = (1<<STRAUS_C)-1; + std::shared_ptr<straus_cached_data> local_cache = cache == NULL ? straus_init_cache(data) : cache; + ge_cached cached; + ge_p1p1 p1; + ge_p3 p3; + +#ifdef TRACK_STRAUS_ZERO_IDENTITY + MULTIEXP_PERF(PERF_TIMER_START_UNIT(skip, 1000000)); + std::vector<uint8_t> skip(data.size()); + for (size_t i = 0; i < data.size(); ++i) + skip[i] = data[i].scalar == rct::zero() || ge_p3_is_point_at_infinity(&data[i].point); + MULTIEXP_PERF(PERF_TIMER_STOP(skip)); +#endif + + MULTIEXP_PERF(PERF_TIMER_START_UNIT(digits, 1000000)); + std::unique_ptr<uint8_t[]> digits{new uint8_t[256 * data.size()]}; + for (size_t j = 0; j < data.size(); ++j) + { + unsigned char bytes33[33]; + memcpy(bytes33, data[j].scalar.bytes, 32); + bytes33[32] = 0; + const unsigned char *bytes = bytes33; +#if 1 + static_assert(STRAUS_C == 4, "optimized version needs STRAUS_C == 4"); + unsigned int i; + for (i = 0; i < 256; i += 8, bytes++) + { + digits[j*256+i] = bytes[0] & 0xf; + digits[j*256+i+1] = (bytes[0] >> 1) & 0xf; + digits[j*256+i+2] = (bytes[0] >> 2) & 0xf; + digits[j*256+i+3] = (bytes[0] >> 3) & 0xf; + digits[j*256+i+4] = ((bytes[0] >> 4) | (bytes[1]<<4)) & 0xf; + digits[j*256+i+5] = ((bytes[0] >> 5) | (bytes[1]<<3)) & 0xf; + digits[j*256+i+6] = ((bytes[0] >> 6) | (bytes[1]<<2)) & 0xf; + digits[j*256+i+7] = ((bytes[0] >> 7) | (bytes[1]<<1)) & 0xf; + } +#elif 1 + for (size_t i = 0; i < 256; ++i) + digits[j*256+i] = ((bytes[i>>3] | (bytes[(i>>3)+1]<<8)) >> (i&7)) & mask; +#else + rct::key shifted = data[j].scalar; + for (size_t i = 0; i < 256; ++i) + { + digits[j*256+i] = shifted.bytes[0] & 0xf; + shifted = div2(shifted, (256-i)>>3); + } +#endif + } + MULTIEXP_PERF(PERF_TIMER_STOP(digits)); + + rct::key maxscalar = rct::zero(); + for (size_t i = 0; i < data.size(); ++i) + if (maxscalar < data[i].scalar) + maxscalar = data[i].scalar; + size_t start_i = 0; + while (start_i < 256 && !(maxscalar < pow2(start_i))) + start_i += STRAUS_C; + MULTIEXP_PERF(PERF_TIMER_STOP(setup)); + + ge_p3 res_p3 = ge_p3_identity; + + for (size_t start_offset = 0; start_offset < data.size(); start_offset += STEP) + { + const size_t num_points = std::min(data.size() - start_offset, STEP); + + ge_p3 band_p3 = ge_p3_identity; + size_t i = start_i; + if (!(i < STRAUS_C)) + goto skipfirst; + while (!(i < STRAUS_C)) + { + ge_p2 p2; + ge_p3_to_p2(&p2, &band_p3); + for (size_t j = 0; j < STRAUS_C; ++j) + { + ge_p2_dbl(&p1, &p2); + if (j == STRAUS_C - 1) + ge_p1p1_to_p3(&band_p3, &p1); + else + ge_p1p1_to_p2(&p2, &p1); + } +skipfirst: + i -= STRAUS_C; + for (size_t j = start_offset; j < start_offset + num_points; ++j) + { +#ifdef TRACK_STRAUS_ZERO_IDENTITY + if (skip[j]) + continue; +#endif + const uint8_t digit = digits[j*256+i]; + if (digit) + { + ge_add(&p1, &band_p3, &CACHE_OFFSET(local_cache, j, digit)); + ge_p1p1_to_p3(&band_p3, &p1); + } + } + } + + ge_p3_to_cached(&cached, &band_p3); + ge_add(&p1, &res_p3, &cached); + ge_p1p1_to_p3(&res_p3, &p1); + } + + rct::key res; + ge_p3_tobytes(res.bytes, &res_p3); + return res; +} + +size_t get_pippenger_c(size_t N) +{ +// uncached: 2:1, 4:2, 8:2, 16:3, 32:4, 64:4, 128:5, 256:6, 512:7, 1024:7, 2048:8, 4096:9 +// cached: 2:1, 4:2, 8:2, 16:3, 32:4, 64:4, 128:5, 256:6, 512:7, 1024:7, 2048:8, 4096:9 + if (N <= 2) return 1; + if (N <= 8) return 2; + if (N <= 16) return 3; + if (N <= 64) return 4; + if (N <= 128) return 5; + if (N <= 256) return 6; + if (N <= 1024) return 7; + if (N <= 2048) return 8; + return 9; +} + +struct pippenger_cached_data +{ + size_t size; + ge_cached *cached; + pippenger_cached_data(): size(0), cached(NULL) {} + ~pippenger_cached_data() { aligned_free(cached); } +}; + +std::shared_ptr<pippenger_cached_data> pippenger_init_cache(const std::vector<MultiexpData> &data, size_t N) +{ + MULTIEXP_PERF(PERF_TIMER_START_UNIT(pippenger_init_cache, 1000000)); + if (N == 0) + N = data.size(); + CHECK_AND_ASSERT_THROW_MES(N <= data.size(), "Bad cache base data"); + ge_cached cached; + std::shared_ptr<pippenger_cached_data> cache(new pippenger_cached_data()); + + cache->size = N; + cache->cached = (ge_cached*)aligned_realloc(cache->cached, N * sizeof(ge_cached), 4096); + CHECK_AND_ASSERT_THROW_MES(cache->cached, "Out of memory"); + for (size_t i = 0; i < N; ++i) + ge_p3_to_cached(&cache->cached[i], &data[i].point); + + MULTIEXP_PERF(PERF_TIMER_STOP(pippenger_init_cache)); + return cache; +} + +size_t pippenger_get_cache_size(const std::shared_ptr<pippenger_cached_data> &cache) +{ + return cache->size * sizeof(*cache->cached); +} + +rct::key pippenger(const std::vector<MultiexpData> &data, const std::shared_ptr<pippenger_cached_data> &cache, size_t c) +{ + CHECK_AND_ASSERT_THROW_MES(cache == NULL || cache->size >= data.size(), "Cache is too small"); + if (c == 0) + c = get_pippenger_c(data.size()); + CHECK_AND_ASSERT_THROW_MES(c <= 9, "c is too large"); + + ge_p3 result = ge_p3_identity; + std::unique_ptr<ge_p3[]> buckets{new ge_p3[1<<c]}; + std::shared_ptr<pippenger_cached_data> local_cache = cache == NULL ? pippenger_init_cache(data) : cache; + + rct::key maxscalar = rct::zero(); + for (size_t i = 0; i < data.size(); ++i) + { + if (maxscalar < data[i].scalar) + maxscalar = data[i].scalar; + } + size_t groups = 0; + while (groups < 256 && !(maxscalar < pow2(groups))) + ++groups; + groups = (groups + c - 1) / c; + + for (size_t k = groups; k-- > 0; ) + { + if (!ge_p3_is_point_at_infinity(&result)) + { + ge_p2 p2; + ge_p3_to_p2(&p2, &result); + for (size_t i = 0; i < c; ++i) + { + ge_p1p1 p1; + ge_p2_dbl(&p1, &p2); + if (i == c - 1) + ge_p1p1_to_p3(&result, &p1); + else + ge_p1p1_to_p2(&p2, &p1); + } + } + for (size_t i = 0; i < (1u<<c); ++i) + buckets[i] = ge_p3_identity; + + // partition scalars into buckets + for (size_t i = 0; i < data.size(); ++i) + { + unsigned int bucket = 0; + for (size_t j = 0; j < c; ++j) + if (test(data[i].scalar, k*c+j)) + bucket |= 1<<j; + if (bucket == 0) + continue; + CHECK_AND_ASSERT_THROW_MES(bucket < (1u<<c), "bucket overflow"); + if (!ge_p3_is_point_at_infinity(&buckets[bucket])) + { + add(buckets[bucket], local_cache->cached[i]); + } + else + buckets[bucket] = data[i].point; + } + + // sum the buckets + ge_p3 pail = ge_p3_identity; + for (size_t i = (1<<c)-1; i > 0; --i) + { + if (!ge_p3_is_point_at_infinity(&buckets[i])) + add(pail, buckets[i]); + if (!ge_p3_is_point_at_infinity(&pail)) + add(result, pail); + } + } + + rct::key res; + ge_p3_tobytes(res.bytes, &result); + return res; +} + +} diff --git a/src/ringct/multiexp.h b/src/ringct/multiexp.h new file mode 100644 index 000000000..559ab664a --- /dev/null +++ b/src/ringct/multiexp.h @@ -0,0 +1,71 @@ +// Copyright (c) 2017, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Adapted from Python code by Sarang Noether + +#pragma once + +#ifndef MULTIEXP_H +#define MULTIEXP_H + +#include <vector> +#include "crypto/crypto.h" +#include "rctTypes.h" +#include "misc_log_ex.h" + +namespace rct +{ + +struct MultiexpData { + rct::key scalar; + ge_p3 point; + + MultiexpData() {} + MultiexpData(const rct::key &s, const ge_p3 &p): scalar(s), point(p) {} + MultiexpData(const rct::key &s, const rct::key &p): scalar(s) + { + CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&point, p.bytes) == 0, "ge_frombytes_vartime failed"); + } +}; + +struct straus_cached_data; +struct pippenger_cached_data; + +rct::key bos_coster_heap_conv(std::vector<MultiexpData> data); +rct::key bos_coster_heap_conv_robust(std::vector<MultiexpData> data); +std::shared_ptr<straus_cached_data> straus_init_cache(const std::vector<MultiexpData> &data, size_t N =0); +size_t straus_get_cache_size(const std::shared_ptr<straus_cached_data> &cache); +rct::key straus(const std::vector<MultiexpData> &data, const std::shared_ptr<straus_cached_data> &cache = NULL, size_t STEP = 0); +std::shared_ptr<pippenger_cached_data> pippenger_init_cache(const std::vector<MultiexpData> &data, size_t N =0); +size_t pippenger_get_cache_size(const std::shared_ptr<pippenger_cached_data> &cache); +size_t get_pippenger_c(size_t N); +rct::key pippenger(const std::vector<MultiexpData> &data, const std::shared_ptr<pippenger_cached_data> &cache = NULL, size_t c = 0); + +} + +#endif diff --git a/src/ringct/rctOps.cpp b/src/ringct/rctOps.cpp index 50693bad7..6c3c4500e 100644 --- a/src/ringct/rctOps.cpp +++ b/src/ringct/rctOps.cpp @@ -60,6 +60,17 @@ namespace rct { //Various key generation functions + bool toPointCheckOrder(ge_p3 *P, const unsigned char *data) + { + if (ge_frombytes_vartime(P, data)) + return false; + ge_p2 R; + ge_scalarmult(&R, curveOrder().bytes, P); + key tmp; + ge_tobytes(tmp.bytes, &R); + return tmp == identity(); + } + //generates a random scalar which can be used as a secret key or mask void skGen(key &sk) { random32_unbiased(sk.bytes); @@ -193,15 +204,33 @@ namespace rct { //Computes aH where H= toPoint(cn_fast_hash(G)), G the basepoint key scalarmultH(const key & a) { - ge_p3 A; ge_p2 R; - CHECK_AND_ASSERT_THROW_MES_L1(ge_frombytes_vartime(&A, H.bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast<std::string>(__LINE__)); - ge_scalarmult(&R, a.bytes, &A); + ge_scalarmult(&R, a.bytes, &ge_p3_H); key aP; ge_tobytes(aP.bytes, &R); return aP; } + //Computes 8P + key scalarmult8(const key & P) { + ge_p3 p3; + CHECK_AND_ASSERT_THROW_MES_L1(ge_frombytes_vartime(&p3, P.bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast<std::string>(__LINE__)); + ge_p2 p2; + ge_p3_to_p2(&p2, &p3); + ge_p1p1 p1; + ge_mul8(&p1, &p2); + ge_p1p1_to_p2(&p2, &p1); + rct::key res; + ge_tobytes(res.bytes, &p2); + return res; + } + + //Computes aL where L is the curve order + bool isInMainSubgroup(const key & a) { + ge_p3 p3; + return toPointCheckOrder(&p3, a.bytes); + } + //Curve addition / subtractions //for curve points: AB = A + B diff --git a/src/ringct/rctOps.h b/src/ringct/rctOps.h index f8889af5c..50645821c 100644 --- a/src/ringct/rctOps.h +++ b/src/ringct/rctOps.h @@ -63,6 +63,8 @@ namespace rct { static const key I = { {0x01, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 } }; static const key L = { {0xed, 0xd3, 0xf5, 0x5c, 0x1a, 0x63, 0x12, 0x58, 0xd6, 0x9c, 0xf7, 0xa2, 0xde, 0xf9, 0xde, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10 } }; static const key G = { {0x58, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66 } }; + static const key EIGHT = { {0x08, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 } }; + static const key INV_EIGHT = { { 0x79, 0x2f, 0xdc, 0xe2, 0x29, 0xe5, 0x06, 0x61, 0xd0, 0xda, 0x1c, 0x7d, 0xb3, 0x9d, 0xd3, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06 } }; //Creates a zero scalar inline key zero() { return Z; } @@ -83,6 +85,7 @@ namespace rct { keyM keyMInit(size_t rows, size_t cols); //Various key generation functions + bool toPointCheckOrder(ge_p3 *P, const unsigned char *data); //generates a random scalar which can be used as a secret key or mask key skGen(); @@ -119,6 +122,10 @@ namespace rct { key scalarmultKey(const key &P, const key &a); //Computes aH where H= toPoint(cn_fast_hash(G)), G the basepoint key scalarmultH(const key & a); + // multiplies a point by 8 + key scalarmult8(const key & P); + // checks a is in the main subgroup (ie, not a small one) + bool isInMainSubgroup(const key & a); //Curve addition / subtractions diff --git a/src/ringct/rctSigs.cpp b/src/ringct/rctSigs.cpp index f74216ed4..fe0cd9c57 100644 --- a/src/ringct/rctSigs.cpp +++ b/src/ringct/rctSigs.cpp @@ -45,30 +45,6 @@ using namespace std; #define CHECK_AND_ASSERT_MES_L1(expr, ret, message) {if(!(expr)) {MCERROR("verify", message); return ret;}} namespace rct { - bool is_simple(int type) - { - switch (type) - { - case RCTTypeSimple: - case RCTTypeSimpleBulletproof: - return true; - default: - return false; - } - } - - bool is_bulletproof(int type) - { - switch (type) - { - case RCTTypeSimpleBulletproof: - case RCTTypeFullBulletproof: - return true; - default: - return false; - } - } - Bulletproof proveRangeBulletproof(key &C, key &mask, uint64_t amount) { mask = rct::skGen(); @@ -78,6 +54,15 @@ namespace rct { return proof; } + Bulletproof proveRangeBulletproof(keyV &C, keyV &masks, const std::vector<uint64_t> &amounts) + { + masks = rct::skvGen(amounts.size()); + Bulletproof proof = bulletproof_PROVE(amounts, masks); + CHECK_AND_ASSERT_THROW_MES(proof.V.size() == amounts.size(), "V does not have the expected size"); + C = proof.V; + return proof; + } + bool verBulletproof(const Bulletproof &proof) { try { return bulletproof_VERIFY(proof); } @@ -85,6 +70,13 @@ namespace rct { catch (...) { return false; } } + bool verBulletproof(const std::vector<const Bulletproof*> &proofs) + { + try { return bulletproof_VERIFY(proofs); } + // we can get deep throws from ge_frombytes_vartime if input isn't valid + catch (...) { return false; } + } + //Borromean (c.f. gmax/andytoshi's paper) boroSig genBorromean(const key64 x, const key64 P1, const key64 P2, const bits indices) { key64 L[2], alpha; @@ -285,6 +277,7 @@ namespace rct { for (j = 0; j < dsRows; j++) { addKeys2(L, rv.ss[i][j], c_old, pk[i][j]); hashToPoint(Hi, pk[i][j]); + CHECK_AND_ASSERT_MES(!(Hi == rct::identity()), false, "Data hashed to point at infinity"); addKeys3(R, rv.ss[i][j], Hi, c_old, Ip[j].k); toHash[3 * j + 1] = pk[i][j]; toHash[3 * j + 2] = L; @@ -389,7 +382,7 @@ namespace rct { std::stringstream ss; binary_archive<true> ba(ss); CHECK_AND_ASSERT_THROW_MES(!rv.mixRing.empty(), "Empty mixRing"); - const size_t inputs = is_simple(rv.type) ? rv.mixRing.size() : rv.mixRing[0].size(); + const size_t inputs = is_rct_simple(rv.type) ? rv.mixRing.size() : rv.mixRing[0].size(); const size_t outputs = rv.ecdhInfo.size(); key prehash; CHECK_AND_ASSERT_THROW_MES(const_cast<rctSig&>(rv).serialize_rctsig_base(ba, inputs, outputs), @@ -398,7 +391,7 @@ namespace rct { hashes.push_back(hash2rct(h)); keyV kv; - if (rv.type == RCTTypeSimpleBulletproof || rv.type == RCTTypeFullBulletproof) + if (rv.type == RCTTypeBulletproof) { kv.reserve((6*2+9) * rv.p.bulletproofs.size()); for (const auto &p: rv.p.bulletproofs) @@ -492,7 +485,9 @@ namespace rct { for (size_t j = 0; j < outPk.size(); j++) { sc_sub(sk[rows].bytes, sk[rows].bytes, outSk[j].mask.bytes); //subtract output masks in last row.. } - return MLSAG_Gen(message, M, sk, kLRki, mscout, index, rows, hwdev); + mgSig result = MLSAG_Gen(message, M, sk, kLRki, mscout, index, rows, hwdev); + memwipe(sk.data(), sk.size() * sizeof(key)); + return result; } @@ -521,7 +516,9 @@ namespace rct { M[i][0] = pubs[i].dest; subKeys(M[i][1], pubs[i].mask, Cout); } - return MLSAG_Gen(message, M, sk, kLRki, mscout, index, rows, hwdev); + mgSig result = MLSAG_Gen(message, M, sk, kLRki, mscout, index, rows, hwdev); + memwipe(&sk[0], sizeof(key)); + return result; } @@ -655,7 +652,7 @@ namespace rct { // must know the destination private key to find the correct amount, else will return a random number // Note: For txn fees, the last index in the amounts vector should contain that // Thus the amounts vector will be "one" longer than the destinations vectort - rctSig genRct(const key &message, const ctkeyV & inSk, const keyV & destinations, const vector<xmr_amount> & amounts, const ctkeyM &mixRing, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, unsigned int index, ctkeyV &outSk, bool bulletproof, hw::device &hwdev) { + rctSig genRct(const key &message, const ctkeyV & inSk, const keyV & destinations, const vector<xmr_amount> & amounts, const ctkeyM &mixRing, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, unsigned int index, ctkeyV &outSk, hw::device &hwdev) { CHECK_AND_ASSERT_THROW_MES(amounts.size() == destinations.size() || amounts.size() == destinations.size() + 1, "Different number of amounts/destinations"); CHECK_AND_ASSERT_THROW_MES(amount_keys.size() == destinations.size(), "Different number of amount_keys/destinations"); CHECK_AND_ASSERT_THROW_MES(index < mixRing.size(), "Bad index into mixRing"); @@ -665,13 +662,10 @@ namespace rct { CHECK_AND_ASSERT_THROW_MES((kLRki && msout) || (!kLRki && !msout), "Only one of kLRki/msout is present"); rctSig rv; - rv.type = bulletproof ? RCTTypeFullBulletproof : RCTTypeFull; + rv.type = RCTTypeFull; rv.message = message; rv.outPk.resize(destinations.size()); - if (bulletproof) - rv.p.bulletproofs.resize(destinations.size()); - else - rv.p.rangeSigs.resize(destinations.size()); + rv.p.rangeSigs.resize(destinations.size()); rv.ecdhInfo.resize(destinations.size()); size_t i = 0; @@ -681,17 +675,10 @@ namespace rct { //add destination to sig rv.outPk[i].dest = copy(destinations[i]); //compute range proof - if (bulletproof) - rv.p.bulletproofs[i] = proveRangeBulletproof(rv.outPk[i].mask, outSk[i].mask, amounts[i]); - else - rv.p.rangeSigs[i] = proveRange(rv.outPk[i].mask, outSk[i].mask, amounts[i]); + rv.p.rangeSigs[i] = proveRange(rv.outPk[i].mask, outSk[i].mask, amounts[i]); #ifdef DBG - if (bulletproof) - CHECK_AND_ASSERT_THROW_MES(verBulletproof(rv.p.bulletproofs[i]), "verBulletproof failed on newly created proof"); - else - CHECK_AND_ASSERT_THROW_MES(verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]), "verRange failed on newly created proof"); + CHECK_AND_ASSERT_THROW_MES(verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]), "verRange failed on newly created proof"); #endif - //mask amount and mask rv.ecdhInfo[i].mask = copy(outSk[i].mask); rv.ecdhInfo[i].amount = d2h(amounts[i]); @@ -721,12 +708,13 @@ namespace rct { ctkeyM mixRing; ctkeyV outSk; tie(mixRing, index) = populateFromBlockchain(inPk, mixin); - return genRct(message, inSk, destinations, amounts, mixRing, amount_keys, kLRki, msout, index, outSk, false, hwdev); + return genRct(message, inSk, destinations, amounts, mixRing, amount_keys, kLRki, msout, index, outSk, hwdev); } //RCT simple //for post-rct only - rctSig genRctSimple(const key &message, const ctkeyV & inSk, const keyV & destinations, const vector<xmr_amount> &inamounts, const vector<xmr_amount> &outamounts, xmr_amount txnFee, const ctkeyM & mixRing, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, const std::vector<unsigned int> & index, ctkeyV &outSk, bool bulletproof, hw::device &hwdev) { + rctSig genRctSimple(const key &message, const ctkeyV & inSk, const keyV & destinations, const vector<xmr_amount> &inamounts, const vector<xmr_amount> &outamounts, xmr_amount txnFee, const ctkeyM & mixRing, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, const std::vector<unsigned int> & index, ctkeyV &outSk, RangeProofType range_proof_type, hw::device &hwdev) { + const bool bulletproof = range_proof_type != RangeProofBorromean; CHECK_AND_ASSERT_THROW_MES(inamounts.size() > 0, "Empty inamounts"); CHECK_AND_ASSERT_THROW_MES(inamounts.size() == inSk.size(), "Different number of inamounts/inSk"); CHECK_AND_ASSERT_THROW_MES(outamounts.size() == destinations.size(), "Different number of amounts/destinations"); @@ -742,35 +730,74 @@ namespace rct { } rctSig rv; - rv.type = bulletproof ? RCTTypeSimpleBulletproof : RCTTypeSimple; + rv.type = bulletproof ? RCTTypeBulletproof : RCTTypeSimple; rv.message = message; rv.outPk.resize(destinations.size()); - if (bulletproof) - rv.p.bulletproofs.resize(destinations.size()); - else + if (!bulletproof) rv.p.rangeSigs.resize(destinations.size()); rv.ecdhInfo.resize(destinations.size()); size_t i; keyV masks(destinations.size()); //sk mask.. outSk.resize(destinations.size()); - key sumout = zero(); for (i = 0; i < destinations.size(); i++) { //add destination to sig rv.outPk[i].dest = copy(destinations[i]); //compute range proof - if (bulletproof) - rv.p.bulletproofs[i] = proveRangeBulletproof(rv.outPk[i].mask, outSk[i].mask, outamounts[i]); - else + if (!bulletproof) rv.p.rangeSigs[i] = proveRange(rv.outPk[i].mask, outSk[i].mask, outamounts[i]); #ifdef DBG - if (bulletproof) - CHECK_AND_ASSERT_THROW_MES(verBulletproof(rv.p.bulletproofs[i]), "verBulletproof failed on newly created proof"); - else + if (!bulletproof) CHECK_AND_ASSERT_THROW_MES(verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]), "verRange failed on newly created proof"); #endif - + } + + rv.p.bulletproofs.clear(); + if (bulletproof) + { + std::vector<uint64_t> proof_amounts; + size_t n_amounts = outamounts.size(); + size_t amounts_proved = 0; + if (range_proof_type == RangeProofPaddedBulletproof) + { + rct::keyV C, masks; + rv.p.bulletproofs.push_back(proveRangeBulletproof(C, masks, outamounts)); + #ifdef DBG + CHECK_AND_ASSERT_THROW_MES(verBulletproof(rv.p.bulletproofs.back()), "verBulletproof failed on newly created proof"); + #endif + for (i = 0; i < outamounts.size(); ++i) + { + rv.outPk[i].mask = rct::scalarmult8(C[i]); + outSk[i].mask = masks[i]; + } + } + else while (amounts_proved < n_amounts) + { + size_t batch_size = 1; + if (range_proof_type == RangeProofMultiOutputBulletproof) + while (batch_size * 2 + amounts_proved <= n_amounts && batch_size * 2 <= BULLETPROOF_MAX_OUTPUTS) + batch_size *= 2; + rct::keyV C, masks; + std::vector<uint64_t> batch_amounts(batch_size); + for (i = 0; i < batch_size; ++i) + batch_amounts[i] = outamounts[i + amounts_proved]; + rv.p.bulletproofs.push_back(proveRangeBulletproof(C, masks, batch_amounts)); + #ifdef DBG + CHECK_AND_ASSERT_THROW_MES(verBulletproof(rv.p.bulletproofs.back()), "verBulletproof failed on newly created proof"); + #endif + for (i = 0; i < batch_size; ++i) + { + rv.outPk[i + amounts_proved].mask = rct::scalarmult8(C[i]); + outSk[i + amounts_proved].mask = masks[i]; + } + amounts_proved += batch_size; + } + } + + key sumout = zero(); + for (i = 0; i < outSk.size(); ++i) + { sc_add(sumout.bytes, outSk[i].mask.bytes, sumout.bytes); //mask amount and mask @@ -818,7 +845,7 @@ namespace rct { mixRing[i].resize(mixin+1); index[i] = populateFromBlockchainSimple(mixRing[i], inPk[i], mixin); } - return genRctSimple(message, inSk, destinations, inamounts, outamounts, txnFee, mixRing, amount_keys, kLRki, msout, index, outSk, false, hwdev); + return genRctSimple(message, inSk, destinations, inamounts, outamounts, txnFee, mixRing, amount_keys, kLRki, msout, index, outSk, RangeProofBorromean, hwdev); } //RingCT protocol @@ -833,13 +860,10 @@ namespace rct { // must know the destination private key to find the correct amount, else will return a random number bool verRct(const rctSig & rv, bool semantics) { PERF_TIMER(verRct); - CHECK_AND_ASSERT_MES(rv.type == RCTTypeFull || rv.type == RCTTypeFullBulletproof, false, "verRct called on non-full rctSig"); + CHECK_AND_ASSERT_MES(rv.type == RCTTypeFull, false, "verRct called on non-full rctSig"); if (semantics) { - if (rv.type == RCTTypeFullBulletproof) - CHECK_AND_ASSERT_MES(rv.outPk.size() == rv.p.bulletproofs.size(), false, "Mismatched sizes of outPk and rv.p.bulletproofs"); - else - CHECK_AND_ASSERT_MES(rv.outPk.size() == rv.p.rangeSigs.size(), false, "Mismatched sizes of outPk and rv.p.rangeSigs"); + CHECK_AND_ASSERT_MES(rv.outPk.size() == rv.p.rangeSigs.size(), false, "Mismatched sizes of outPk and rv.p.rangeSigs"); CHECK_AND_ASSERT_MES(rv.outPk.size() == rv.ecdhInfo.size(), false, "Mismatched sizes of outPk and rv.ecdhInfo"); CHECK_AND_ASSERT_MES(rv.p.MGs.size() == 1, false, "full rctSig has not one MG"); } @@ -856,19 +880,13 @@ namespace rct { tools::threadpool::waiter waiter; std::deque<bool> results(rv.outPk.size(), false); DP("range proofs verified?"); - for (size_t i = 0; i < rv.outPk.size(); i++) { - tpool.submit(&waiter, [&, i] { - if (rv.p.rangeSigs.empty()) - results[i] = verBulletproof(rv.p.bulletproofs[i]); - else - results[i] = verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]); - }, true); - } + for (size_t i = 0; i < rv.outPk.size(); i++) + tpool.submit(&waiter, [&, i] { results[i] = verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]); }); waiter.wait(&tpool); - for (size_t i = 0; i < rv.outPk.size(); ++i) { + for (size_t i = 0; i < results.size(); ++i) { if (!results[i]) { - LOG_PRINT_L1("Range proof verified failed for output " << i); + LOG_PRINT_L1("Range proof verified failed for proof " << i); return false; } } @@ -902,17 +920,26 @@ namespace rct { //ver RingCT simple //assumes only post-rct style inputs (at least for max anonymity) - bool verRctSimple(const rctSig & rv, bool semantics) { + bool verRctSemanticsSimple(const std::vector<const rctSig*> & rvv) { try { - PERF_TIMER(verRctSimple); + PERF_TIMER(verRctSemanticsSimple); - CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeSimpleBulletproof, false, "verRctSimple called on non simple rctSig"); - if (semantics) + tools::threadpool& tpool = tools::threadpool::getInstance(); + tools::threadpool::waiter waiter; + std::deque<bool> results; + std::vector<const Bulletproof*> proofs; + size_t max_non_bp_proofs = 0, offset = 0; + + for (const rctSig *rvp: rvv) { - if (rv.type == RCTTypeSimpleBulletproof) + CHECK_AND_ASSERT_MES(rvp, false, "rctSig pointer is NULL"); + const rctSig &rv = *rvp; + CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof, false, "verRctSemanticsSimple called on non simple rctSig"); + const bool bulletproof = is_rct_bulletproof(rv.type); + if (bulletproof) { - CHECK_AND_ASSERT_MES(rv.outPk.size() == rv.p.bulletproofs.size(), false, "Mismatched sizes of outPk and rv.p.bulletproofs"); + CHECK_AND_ASSERT_MES(rv.outPk.size() == n_bulletproof_amounts(rv.p.bulletproofs), false, "Mismatched sizes of outPk and bulletproofs"); CHECK_AND_ASSERT_MES(rv.p.pseudoOuts.size() == rv.p.MGs.size(), false, "Mismatched sizes of rv.p.pseudoOuts and rv.p.MGs"); CHECK_AND_ASSERT_MES(rv.pseudoOuts.empty(), false, "rv.pseudoOuts is not empty"); } @@ -923,28 +950,22 @@ namespace rct { CHECK_AND_ASSERT_MES(rv.p.pseudoOuts.empty(), false, "rv.p.pseudoOuts is not empty"); } CHECK_AND_ASSERT_MES(rv.outPk.size() == rv.ecdhInfo.size(), false, "Mismatched sizes of outPk and rv.ecdhInfo"); - } - else - { - // semantics check is early, and mixRing/MGs aren't resolved yet - if (rv.type == RCTTypeSimpleBulletproof) - CHECK_AND_ASSERT_MES(rv.p.pseudoOuts.size() == rv.mixRing.size(), false, "Mismatched sizes of rv.p.pseudoOuts and mixRing"); - else - CHECK_AND_ASSERT_MES(rv.pseudoOuts.size() == rv.mixRing.size(), false, "Mismatched sizes of rv.pseudoOuts and mixRing"); - } - const size_t threads = std::max(rv.outPk.size(), rv.mixRing.size()); + if (!bulletproof) + max_non_bp_proofs += rv.p.rangeSigs.size(); + } - std::deque<bool> results(threads); - tools::threadpool& tpool = tools::threadpool::getInstance(); - tools::threadpool::waiter waiter; + results.resize(max_non_bp_proofs); + for (const rctSig *rvp: rvv) + { + const rctSig &rv = *rvp; - const keyV &pseudoOuts = is_bulletproof(rv.type) ? rv.p.pseudoOuts : rv.pseudoOuts; + const bool bulletproof = is_rct_bulletproof(rv.type); + const keyV &pseudoOuts = bulletproof ? rv.p.pseudoOuts : rv.pseudoOuts; - if (semantics) { key sumOutpks = identity(); for (size_t i = 0; i < rv.outPk.size(); i++) { - addKeys(sumOutpks, sumOutpks, rv.outPk[i].mask); + addKeys(sumOutpks, sumOutpks, rv.outPk[i].mask); } DP(sumOutpks); key txnFeeKey = scalarmultH(d2h(rv.txnFee)); @@ -952,52 +973,100 @@ namespace rct { key sumPseudoOuts = identity(); for (size_t i = 0 ; i < pseudoOuts.size() ; i++) { - addKeys(sumPseudoOuts, sumPseudoOuts, pseudoOuts[i]); + addKeys(sumPseudoOuts, sumPseudoOuts, pseudoOuts[i]); } DP(sumPseudoOuts); //check pseudoOuts vs Outs.. if (!equalKeys(sumPseudoOuts, sumOutpks)) { - LOG_PRINT_L1("Sum check failed"); - return false; + LOG_PRINT_L1("Sum check failed"); + return false; } - results.clear(); - results.resize(rv.outPk.size()); - for (size_t i = 0; i < rv.outPk.size(); i++) { - tpool.submit(&waiter, [&, i] { - if (rv.p.rangeSigs.empty()) - results[i] = verBulletproof(rv.p.bulletproofs[i]); - else - results[i] = verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]); - }, true); + if (bulletproof) + { + for (size_t i = 0; i < rv.p.bulletproofs.size(); i++) + proofs.push_back(&rv.p.bulletproofs[i]); } - waiter.wait(&tpool); - - for (size_t i = 0; i < results.size(); ++i) { - if (!results[i]) { - LOG_PRINT_L1("Range proof verified failed for output " << i); - return false; - } + else + { + for (size_t i = 0; i < rv.p.rangeSigs.size(); i++) + tpool.submit(&waiter, [&, i, offset] { results[i+offset] = verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]); }); + offset += rv.p.rangeSigs.size(); } } - else { - const key message = get_pre_mlsag_hash(rv, hw::get_device("default")); - - results.clear(); - results.resize(rv.mixRing.size()); - for (size_t i = 0 ; i < rv.mixRing.size() ; i++) { - tpool.submit(&waiter, [&, i] { - results[i] = verRctMGSimple(message, rv.p.MGs[i], rv.mixRing[i], pseudoOuts[i]); - }, true); + if (!proofs.empty() && !verBulletproof(proofs)) + { + LOG_PRINT_L1("Aggregate range proof verified failed"); + return false; + } + + waiter.wait(&tpool); + for (size_t i = 0; i < results.size(); ++i) { + if (!results[i]) { + LOG_PRINT_L1("Range proof verified failed for proof " << i); + return false; } - waiter.wait(&tpool); + } - for (size_t i = 0; i < results.size(); ++i) { - if (!results[i]) { - LOG_PRINT_L1("verRctMGSimple failed for input " << i); - return false; - } + return true; + } + // we can get deep throws from ge_frombytes_vartime if input isn't valid + catch (const std::exception &e) + { + LOG_PRINT_L1("Error in verRctSemanticsSimple: " << e.what()); + return false; + } + catch (...) + { + LOG_PRINT_L1("Error in verRctSemanticsSimple, but not an actual exception"); + return false; + } + } + + bool verRctSemanticsSimple(const rctSig & rv) + { + return verRctSemanticsSimple(std::vector<const rctSig*>(1, &rv)); + } + + //ver RingCT simple + //assumes only post-rct style inputs (at least for max anonymity) + bool verRctNonSemanticsSimple(const rctSig & rv) { + try + { + PERF_TIMER(verRctNonSemanticsSimple); + + CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof, false, "verRctNonSemanticsSimple called on non simple rctSig"); + const bool bulletproof = is_rct_bulletproof(rv.type); + // semantics check is early, and mixRing/MGs aren't resolved yet + if (bulletproof) + CHECK_AND_ASSERT_MES(rv.p.pseudoOuts.size() == rv.mixRing.size(), false, "Mismatched sizes of rv.p.pseudoOuts and mixRing"); + else + CHECK_AND_ASSERT_MES(rv.pseudoOuts.size() == rv.mixRing.size(), false, "Mismatched sizes of rv.pseudoOuts and mixRing"); + + const size_t threads = std::max(rv.outPk.size(), rv.mixRing.size()); + + std::deque<bool> results(threads); + tools::threadpool& tpool = tools::threadpool::getInstance(); + tools::threadpool::waiter waiter; + + const keyV &pseudoOuts = bulletproof ? rv.p.pseudoOuts : rv.pseudoOuts; + + const key message = get_pre_mlsag_hash(rv, hw::get_device("default")); + + results.clear(); + results.resize(rv.mixRing.size()); + for (size_t i = 0 ; i < rv.mixRing.size() ; i++) { + tpool.submit(&waiter, [&, i] { + results[i] = verRctMGSimple(message, rv.p.MGs[i], rv.mixRing[i], pseudoOuts[i]); + }); + } + waiter.wait(&tpool); + + for (size_t i = 0; i < results.size(); ++i) { + if (!results[i]) { + LOG_PRINT_L1("verRctMGSimple failed for input " << i); + return false; } } @@ -1006,12 +1075,12 @@ namespace rct { // we can get deep throws from ge_frombytes_vartime if input isn't valid catch (const std::exception &e) { - LOG_PRINT_L1("Error in verRct: " << e.what()); + LOG_PRINT_L1("Error in verRctNonSemanticsSimple: " << e.what()); return false; } catch (...) { - LOG_PRINT_L1("Error in verRct, but not an actual exception"); + LOG_PRINT_L1("Error in verRctNonSemanticsSimple, but not an actual exception"); return false; } } @@ -1027,7 +1096,7 @@ namespace rct { // uses the attached ecdh info to find the amounts represented by each output commitment // must know the destination private key to find the correct amount, else will return a random number xmr_amount decodeRct(const rctSig & rv, const key & sk, unsigned int i, key & mask, hw::device &hwdev) { - CHECK_AND_ASSERT_MES(rv.type == RCTTypeFull || rv.type == RCTTypeFullBulletproof, false, "decodeRct called on non-full rctSig"); + CHECK_AND_ASSERT_MES(rv.type == RCTTypeFull, false, "decodeRct called on non-full rctSig"); CHECK_AND_ASSERT_THROW_MES(i < rv.ecdhInfo.size(), "Bad index"); CHECK_AND_ASSERT_THROW_MES(rv.outPk.size() == rv.ecdhInfo.size(), "Mismatched sizes of rv.outPk and rv.ecdhInfo"); @@ -1040,6 +1109,8 @@ namespace rct { DP("C"); DP(C); key Ctmp; + CHECK_AND_ASSERT_THROW_MES(sc_check(mask.bytes) == 0, "warning, bad ECDH mask"); + CHECK_AND_ASSERT_THROW_MES(sc_check(amount.bytes) == 0, "warning, bad ECDH amount"); addKeys2(Ctmp, mask, amount, H); DP("Ctmp"); DP(Ctmp); @@ -1055,7 +1126,7 @@ namespace rct { } xmr_amount decodeRctSimple(const rctSig & rv, const key & sk, unsigned int i, key &mask, hw::device &hwdev) { - CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeSimpleBulletproof, false, "decodeRct called on non simple rctSig"); + CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof, false, "decodeRct called on non simple rctSig"); CHECK_AND_ASSERT_THROW_MES(i < rv.ecdhInfo.size(), "Bad index"); CHECK_AND_ASSERT_THROW_MES(rv.outPk.size() == rv.ecdhInfo.size(), "Mismatched sizes of rv.outPk and rv.ecdhInfo"); @@ -1068,6 +1139,8 @@ namespace rct { DP("C"); DP(C); key Ctmp; + CHECK_AND_ASSERT_THROW_MES(sc_check(mask.bytes) == 0, "warning, bad ECDH mask"); + CHECK_AND_ASSERT_THROW_MES(sc_check(amount.bytes) == 0, "warning, bad ECDH amount"); addKeys2(Ctmp, mask, amount, H); DP("Ctmp"); DP(Ctmp); @@ -1083,12 +1156,12 @@ namespace rct { } bool signMultisig(rctSig &rv, const std::vector<unsigned int> &indices, const keyV &k, const multisig_out &msout, const key &secret_key) { - CHECK_AND_ASSERT_MES(rv.type == RCTTypeFull || rv.type == RCTTypeSimple || rv.type == RCTTypeFullBulletproof || rv.type == RCTTypeSimpleBulletproof, + CHECK_AND_ASSERT_MES(rv.type == RCTTypeFull || rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof, false, "unsupported rct type"); CHECK_AND_ASSERT_MES(indices.size() == k.size(), false, "Mismatched k/indices sizes"); CHECK_AND_ASSERT_MES(k.size() == rv.p.MGs.size(), false, "Mismatched k/MGs size"); CHECK_AND_ASSERT_MES(k.size() == msout.c.size(), false, "Mismatched k/msout.c size"); - if (rv.type == RCTTypeFull || rv.type == RCTTypeFullBulletproof) + if (rv.type == RCTTypeFull) { CHECK_AND_ASSERT_MES(rv.p.MGs.size() == 1, false, "MGs not a single element"); } diff --git a/src/ringct/rctSigs.h b/src/ringct/rctSigs.h index 5a9b2dd44..ae8bb91d7 100644 --- a/src/ringct/rctSigs.h +++ b/src/ringct/rctSigs.h @@ -119,14 +119,16 @@ namespace rct { //decodeRct: (c.f. https://eprint.iacr.org/2015/1098 section 5.1.1) // uses the attached ecdh info to find the amounts represented by each output commitment // must know the destination private key to find the correct amount, else will return a random number - rctSig genRct(const key &message, const ctkeyV & inSk, const keyV & destinations, const std::vector<xmr_amount> & amounts, const ctkeyM &mixRing, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, unsigned int index, ctkeyV &outSk, bool bulletproof, hw::device &hwdev); + rctSig genRct(const key &message, const ctkeyV & inSk, const keyV & destinations, const std::vector<xmr_amount> & amounts, const ctkeyM &mixRing, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, unsigned int index, ctkeyV &outSk, hw::device &hwdev); rctSig genRct(const key &message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const std::vector<xmr_amount> & amounts, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, const int mixin, hw::device &hwdev); rctSig genRctSimple(const key & message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const std::vector<xmr_amount> & inamounts, const std::vector<xmr_amount> & outamounts, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, xmr_amount txnFee, unsigned int mixin, hw::device &hwdev); - rctSig genRctSimple(const key & message, const ctkeyV & inSk, const keyV & destinations, const std::vector<xmr_amount> & inamounts, const std::vector<xmr_amount> & outamounts, xmr_amount txnFee, const ctkeyM & mixRing, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, const std::vector<unsigned int> & index, ctkeyV &outSk, bool bulletproof, hw::device &hwdev); + rctSig genRctSimple(const key & message, const ctkeyV & inSk, const keyV & destinations, const std::vector<xmr_amount> & inamounts, const std::vector<xmr_amount> & outamounts, xmr_amount txnFee, const ctkeyM & mixRing, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, const std::vector<unsigned int> & index, ctkeyV &outSk, RangeProofType range_proof_type, hw::device &hwdev); bool verRct(const rctSig & rv, bool semantics); static inline bool verRct(const rctSig & rv) { return verRct(rv, true) && verRct(rv, false); } - bool verRctSimple(const rctSig & rv, bool semantics); - static inline bool verRctSimple(const rctSig & rv) { return verRctSimple(rv, true) && verRctSimple(rv, false); } + bool verRctSemanticsSimple(const rctSig & rv); + bool verRctSemanticsSimple(const std::vector<const rctSig*> & rv); + bool verRctNonSemanticsSimple(const rctSig & rv); + static inline bool verRctSimple(const rctSig & rv) { return verRctSemanticsSimple(rv) && verRctNonSemanticsSimple(rv); } xmr_amount decodeRct(const rctSig & rv, const key & sk, unsigned int i, key & mask, hw::device &hwdev); xmr_amount decodeRct(const rctSig & rv, const key & sk, unsigned int i, hw::device &hwdev); xmr_amount decodeRctSimple(const rctSig & rv, const key & sk, unsigned int i, key & mask, hw::device &hwdev); diff --git a/src/ringct/rctTypes.cpp b/src/ringct/rctTypes.cpp index 5650b3ba1..90ed65df0 100644 --- a/src/ringct/rctTypes.cpp +++ b/src/ringct/rctTypes.cpp @@ -28,6 +28,8 @@ // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include "misc_log_ex.h" +#include "cryptonote_config.h" #include "rctTypes.h" using namespace crypto; using namespace std; @@ -209,4 +211,90 @@ namespace rct { return vali; } + bool is_rct_simple(int type) + { + switch (type) + { + case RCTTypeSimple: + case RCTTypeBulletproof: + return true; + default: + return false; + } + } + + bool is_rct_bulletproof(int type) + { + switch (type) + { + case RCTTypeBulletproof: + return true; + default: + return false; + } + } + + bool is_rct_borromean(int type) + { + switch (type) + { + case RCTTypeSimple: + case RCTTypeFull: + return true; + default: + return false; + } + } + + size_t n_bulletproof_amounts(const Bulletproof &proof) + { + CHECK_AND_ASSERT_MES(proof.L.size() >= 6, 0, "Invalid bulletproof L size"); + CHECK_AND_ASSERT_MES(proof.L.size() == proof.R.size(), 0, "Mismatched bulletproof L/R size"); + static const size_t extra_bits = 4; + static_assert((1 << extra_bits) == BULLETPROOF_MAX_OUTPUTS, "log2(BULLETPROOF_MAX_OUTPUTS) is out of date"); + CHECK_AND_ASSERT_MES(proof.L.size() <= 6 + extra_bits, 0, "Invalid bulletproof L size"); + CHECK_AND_ASSERT_MES(proof.V.size() <= (1u<<(proof.L.size()-6)), 0, "Invalid bulletproof V/L"); + CHECK_AND_ASSERT_MES(proof.V.size() * 2 > (1u<<(proof.L.size()-6)), 0, "Invalid bulletproof V/L"); + CHECK_AND_ASSERT_MES(proof.V.size() > 0, 0, "Empty bulletproof"); + return proof.V.size(); + } + + size_t n_bulletproof_amounts(const std::vector<Bulletproof> &proofs) + { + size_t n = 0; + for (const Bulletproof &proof: proofs) + { + size_t n2 = n_bulletproof_amounts(proof); + CHECK_AND_ASSERT_MES(n2 < std::numeric_limits<uint32_t>::max() - n, 0, "Invalid number of bulletproofs"); + if (n2 == 0) + return 0; + n += n2; + } + return n; + } + + size_t n_bulletproof_max_amounts(const Bulletproof &proof) + { + CHECK_AND_ASSERT_MES(proof.L.size() >= 6, 0, "Invalid bulletproof L size"); + CHECK_AND_ASSERT_MES(proof.L.size() == proof.R.size(), 0, "Mismatched bulletproof L/R size"); + static const size_t extra_bits = 4; + static_assert((1 << extra_bits) == BULLETPROOF_MAX_OUTPUTS, "log2(BULLETPROOF_MAX_OUTPUTS) is out of date"); + CHECK_AND_ASSERT_MES(proof.L.size() <= 6 + extra_bits, 0, "Invalid bulletproof L size"); + return 1 << (proof.L.size() - 6); + } + + size_t n_bulletproof_max_amounts(const std::vector<Bulletproof> &proofs) + { + size_t n = 0; + for (const Bulletproof &proof: proofs) + { + size_t n2 = n_bulletproof_max_amounts(proof); + CHECK_AND_ASSERT_MES(n2 < std::numeric_limits<uint32_t>::max() - n, 0, "Invalid number of bulletproofs"); + if (n2 == 0) + return 0; + n += n2; + } + return n; + } + } diff --git a/src/ringct/rctTypes.h b/src/ringct/rctTypes.h index 844291d0c..ffc4df3ed 100644 --- a/src/ringct/rctTypes.h +++ b/src/ringct/rctTypes.h @@ -36,6 +36,7 @@ #include <vector> #include <iostream> #include <cinttypes> +#include <sodium/crypto_verify_32.h> extern "C" { #include "crypto/crypto-ops.h" @@ -81,7 +82,7 @@ namespace rct { unsigned char operator[](int i) const { return bytes[i]; } - bool operator==(const key &k) const { return !memcmp(bytes, k.bytes, sizeof(bytes)); } + bool operator==(const key &k) const { return !crypto_verify_32(bytes, k.bytes); } unsigned char bytes[32]; }; typedef std::vector<key> keyV; //vector of keys @@ -189,6 +190,8 @@ namespace rct { Bulletproof() {} Bulletproof(const rct::key &V, const rct::key &A, const rct::key &S, const rct::key &T1, const rct::key &T2, const rct::key &taux, const rct::key &mu, const rct::keyV &L, const rct::keyV &R, const rct::key &a, const rct::key &b, const rct::key &t): V({V}), A(A), S(S), T1(T1), T2(T2), taux(taux), mu(mu), L(L), R(R), a(a), b(b), t(t) {} + Bulletproof(const rct::keyV &V, const rct::key &A, const rct::key &S, const rct::key &T1, const rct::key &T2, const rct::key &taux, const rct::key &mu, const rct::keyV &L, const rct::keyV &R, const rct::key &a, const rct::key &b, const rct::key &t): + V(V), A(A), S(S), T1(T1), T2(T2), taux(taux), mu(mu), L(L), R(R), a(a), b(b), t(t) {} BEGIN_SERIALIZE_OBJECT() // Commitments aren't saved, they're restored via outPk @@ -210,6 +213,11 @@ namespace rct { END_SERIALIZE() }; + size_t n_bulletproof_amounts(const Bulletproof &proof); + size_t n_bulletproof_max_amounts(const Bulletproof &proof); + size_t n_bulletproof_amounts(const std::vector<Bulletproof> &proofs); + size_t n_bulletproof_max_amounts(const std::vector<Bulletproof> &proofs); + //A container to hold all signatures necessary for RingCT // rangeSigs holds all the rangeproof data of a transaction // MG holds the MLSAG signature of a transaction @@ -221,9 +229,9 @@ namespace rct { RCTTypeNull = 0, RCTTypeFull = 1, RCTTypeSimple = 2, - RCTTypeFullBulletproof = 3, - RCTTypeSimpleBulletproof = 4, + RCTTypeBulletproof = 3, }; + enum RangeProofType { RangeProofBorromean, RangeProofBulletproof, RangeProofMultiOutputBulletproof, RangeProofPaddedBulletproof }; struct rctSigBase { uint8_t type; key message; @@ -240,7 +248,7 @@ namespace rct { FIELD(type) if (type == RCTTypeNull) return true; - if (type != RCTTypeFull && type != RCTTypeFullBulletproof && type != RCTTypeSimple && type != RCTTypeSimpleBulletproof) + if (type != RCTTypeFull && type != RCTTypeSimple && type != RCTTypeBulletproof) return false; VARINT_FIELD(txnFee) // inputs/outputs not saved, only here for serialization help @@ -301,21 +309,25 @@ namespace rct { { if (type == RCTTypeNull) return true; - if (type != RCTTypeFull && type != RCTTypeFullBulletproof && type != RCTTypeSimple && type != RCTTypeSimpleBulletproof) + if (type != RCTTypeFull && type != RCTTypeSimple && type != RCTTypeBulletproof) return false; - if (type == RCTTypeSimpleBulletproof || type == RCTTypeFullBulletproof) + if (type == RCTTypeBulletproof) { ar.tag("bp"); ar.begin_array(); - PREPARE_CUSTOM_VECTOR_SERIALIZATION(outputs, bulletproofs); - if (bulletproofs.size() != outputs) + uint32_t nbp = bulletproofs.size(); + FIELD(nbp) + if (nbp > outputs) return false; - for (size_t i = 0; i < outputs; ++i) + PREPARE_CUSTOM_VECTOR_SERIALIZATION(nbp, bulletproofs); + for (size_t i = 0; i < nbp; ++i) { FIELDS(bulletproofs[i]) - if (outputs - i > 1) + if (nbp - i > 1) ar.delimit_array(); } + if (n_bulletproof_max_amounts(bulletproofs) < outputs) + return false; ar.end_array(); } else @@ -338,7 +350,7 @@ namespace rct { ar.begin_array(); // we keep a byte for size of MGs, because we don't know whether this is // a simple or full rct signature, and it's starting to annoy the hell out of me - size_t mg_elements = (type == RCTTypeSimple || type == RCTTypeSimpleBulletproof) ? inputs : 1; + size_t mg_elements = (type == RCTTypeSimple || type == RCTTypeBulletproof) ? inputs : 1; PREPARE_CUSTOM_VECTOR_SERIALIZATION(mg_elements, MGs); if (MGs.size() != mg_elements) return false; @@ -356,7 +368,7 @@ namespace rct { for (size_t j = 0; j < mixin + 1; ++j) { ar.begin_array(); - size_t mg_ss2_elements = ((type == RCTTypeSimple || type == RCTTypeSimpleBulletproof) ? 1 : inputs) + 1; + size_t mg_ss2_elements = ((type == RCTTypeSimple || type == RCTTypeBulletproof) ? 1 : inputs) + 1; PREPARE_CUSTOM_VECTOR_SERIALIZATION(mg_ss2_elements, MGs[i].ss[j]); if (MGs[i].ss[j].size() != mg_ss2_elements) return false; @@ -382,7 +394,7 @@ namespace rct { ar.delimit_array(); } ar.end_array(); - if (type == RCTTypeSimpleBulletproof) + if (type == RCTTypeBulletproof) { ar.tag("pseudoOuts"); ar.begin_array(); @@ -403,6 +415,16 @@ namespace rct { }; struct rctSig: public rctSigBase { rctSigPrunable p; + + keyV& get_pseudo_outs() + { + return type == RCTTypeBulletproof ? p.pseudoOuts : pseudoOuts; + } + + keyV const& get_pseudo_outs() const + { + return type == RCTTypeBulletproof ? p.pseudoOuts : pseudoOuts; + } }; //other basepoint H = toPoint(cn_fast_hash(G)), G the basepoint @@ -506,24 +528,28 @@ namespace rct { //int[64] to uint long long xmr_amount b2d(bits amountb); - static inline const rct::key pk2rct(const crypto::public_key &pk) { return (const rct::key&)pk; } - static inline const rct::key sk2rct(const crypto::secret_key &sk) { return (const rct::key&)sk; } - static inline const rct::key ki2rct(const crypto::key_image &ki) { return (const rct::key&)ki; } - static inline const rct::key hash2rct(const crypto::hash &h) { return (const rct::key&)h; } - static inline const crypto::public_key rct2pk(const rct::key &k) { return (const crypto::public_key&)k; } - static inline const crypto::secret_key rct2sk(const rct::key &k) { return (const crypto::secret_key&)k; } - static inline const crypto::key_image rct2ki(const rct::key &k) { return (const crypto::key_image&)k; } - static inline const crypto::hash rct2hash(const rct::key &k) { return (const crypto::hash&)k; } - static inline bool operator==(const rct::key &k0, const crypto::public_key &k1) { return !memcmp(&k0, &k1, 32); } - static inline bool operator!=(const rct::key &k0, const crypto::public_key &k1) { return memcmp(&k0, &k1, 32); } + bool is_rct_simple(int type); + bool is_rct_bulletproof(int type); + bool is_rct_borromean(int type); + + static inline const rct::key &pk2rct(const crypto::public_key &pk) { return (const rct::key&)pk; } + static inline const rct::key &sk2rct(const crypto::secret_key &sk) { return (const rct::key&)sk; } + static inline const rct::key &ki2rct(const crypto::key_image &ki) { return (const rct::key&)ki; } + static inline const rct::key &hash2rct(const crypto::hash &h) { return (const rct::key&)h; } + static inline const crypto::public_key &rct2pk(const rct::key &k) { return (const crypto::public_key&)k; } + static inline const crypto::secret_key &rct2sk(const rct::key &k) { return (const crypto::secret_key&)k; } + static inline const crypto::key_image &rct2ki(const rct::key &k) { return (const crypto::key_image&)k; } + static inline const crypto::hash &rct2hash(const rct::key &k) { return (const crypto::hash&)k; } + static inline bool operator==(const rct::key &k0, const crypto::public_key &k1) { return !crypto_verify_32(k0.bytes, (const unsigned char*)&k1); } + static inline bool operator!=(const rct::key &k0, const crypto::public_key &k1) { return crypto_verify_32(k0.bytes, (const unsigned char*)&k1); } } namespace cryptonote { - static inline bool operator==(const crypto::public_key &k0, const rct::key &k1) { return !memcmp(&k0, &k1, 32); } - static inline bool operator!=(const crypto::public_key &k0, const rct::key &k1) { return memcmp(&k0, &k1, 32); } - static inline bool operator==(const crypto::secret_key &k0, const rct::key &k1) { return !memcmp(&k0, &k1, 32); } - static inline bool operator!=(const crypto::secret_key &k0, const rct::key &k1) { return memcmp(&k0, &k1, 32); } + static inline bool operator==(const crypto::public_key &k0, const rct::key &k1) { return !crypto_verify_32((const unsigned char*)&k0, k1.bytes); } + static inline bool operator!=(const crypto::public_key &k0, const rct::key &k1) { return crypto_verify_32((const unsigned char*)&k0, k1.bytes); } + static inline bool operator==(const crypto::secret_key &k0, const rct::key &k1) { return !crypto_verify_32((const unsigned char*)&k0, k1.bytes); } + static inline bool operator!=(const crypto::secret_key &k0, const rct::key &k1) { return crypto_verify_32((const unsigned char*)&k0, k1.bytes); } } namespace rct { diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 0a6daf8f0..4383ad190 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -195,8 +195,8 @@ namespace cryptonote res.stagenet = m_nettype == STAGENET; res.nettype = m_nettype == MAINNET ? "mainnet" : m_nettype == TESTNET ? "testnet" : m_nettype == STAGENET ? "stagenet" : "fakechain"; res.cumulative_difficulty = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.height - 1); - res.block_size_limit = m_core.get_blockchain_storage().get_current_cumulative_blocksize_limit(); - res.block_size_median = m_core.get_blockchain_storage().get_current_cumulative_blocksize_median(); + res.block_size_limit = res.block_weight_limit = m_core.get_blockchain_storage().get_current_cumulative_block_weight_limit(); + res.block_size_median = res.block_weight_median = m_core.get_blockchain_storage().get_current_cumulative_block_weight_median(); res.status = CORE_RPC_STATUS_OK; res.start_time = (uint64_t)m_core.get_start_time(); res.free_space = m_restricted ? std::numeric_limits<uint64_t>::max() : m_core.get_free_space(); @@ -1313,7 +1313,7 @@ namespace cryptonote response.hash = string_tools::pod_to_hex(hash); response.difficulty = m_core.get_blockchain_storage().block_difficulty(height); response.reward = get_block_reward(blk); - response.block_size = m_core.get_blockchain_storage().get_db().get_block_size(height); + response.block_size = response.block_weight = m_core.get_blockchain_storage().get_db().get_block_weight(height); response.num_txes = blk.tx_hashes.size(); response.pow_hash = fill_pow_hash ? string_tools::pod_to_hex(get_block_longhash(blk, height)) : ""; return true; @@ -1646,8 +1646,8 @@ namespace cryptonote res.stagenet = m_nettype == STAGENET; res.nettype = m_nettype == MAINNET ? "mainnet" : m_nettype == TESTNET ? "testnet" : m_nettype == STAGENET ? "stagenet" : "fakechain"; res.cumulative_difficulty = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.height - 1); - res.block_size_limit = m_core.get_blockchain_storage().get_current_cumulative_blocksize_limit(); - res.block_size_median = m_core.get_blockchain_storage().get_current_cumulative_blocksize_median(); + res.block_size_limit = res.block_weight_limit = m_core.get_blockchain_storage().get_current_cumulative_block_weight_limit(); + res.block_size_median = res.block_weight_median = m_core.get_blockchain_storage().get_current_cumulative_block_weight_median(); res.status = CORE_RPC_STATUS_OK; res.start_time = (uint64_t)m_core.get_start_time(); res.free_space = m_restricted ? std::numeric_limits<uint64_t>::max() : m_core.get_free_space(); @@ -1839,14 +1839,15 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_per_kb_fee_estimate(const COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::request& req, COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_get_base_fee_estimate(const COMMAND_RPC_GET_BASE_FEE_ESTIMATE::request& req, COMMAND_RPC_GET_BASE_FEE_ESTIMATE::response& res, epee::json_rpc::error& error_resp) { - PERF_TIMER(on_get_per_kb_fee_estimate); + PERF_TIMER(on_get_base_fee_estimate); bool r; - if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE>(invoke_http_mode::JON_RPC, "get_fee_estimate", req, res, r)) + if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_BASE_FEE_ESTIMATE>(invoke_http_mode::JON_RPC, "get_fee_estimate", req, res, r)) return r; - res.fee = m_core.get_blockchain_storage().get_dynamic_per_kb_fee_estimate(req.grace_blocks); + res.fee = m_core.get_blockchain_storage().get_dynamic_base_fee_estimate(req.grace_blocks); + res.quantization_mask = Blockchain::get_fee_quantization_mask(); res.status = CORE_RPC_STATUS_OK; return true; } diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index 166020d01..5dbe44d24 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -150,7 +150,7 @@ namespace cryptonote MAP_JON_RPC_WE("get_output_histogram", on_get_output_histogram, COMMAND_RPC_GET_OUTPUT_HISTOGRAM) MAP_JON_RPC_WE("get_version", on_get_version, COMMAND_RPC_GET_VERSION) MAP_JON_RPC_WE_IF("get_coinbase_tx_sum", on_get_coinbase_tx_sum, COMMAND_RPC_GET_COINBASE_TX_SUM, !m_restricted) - MAP_JON_RPC_WE("get_fee_estimate", on_get_per_kb_fee_estimate, COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE) + MAP_JON_RPC_WE("get_fee_estimate", on_get_base_fee_estimate, COMMAND_RPC_GET_BASE_FEE_ESTIMATE) MAP_JON_RPC_WE_IF("get_alternate_chains",on_get_alternate_chains, COMMAND_RPC_GET_ALTERNATE_CHAINS, !m_restricted) MAP_JON_RPC_WE_IF("relay_tx", on_relay_tx, COMMAND_RPC_RELAY_TX, !m_restricted) MAP_JON_RPC_WE_IF("sync_info", on_sync_info, COMMAND_RPC_SYNC_INFO, !m_restricted) @@ -214,7 +214,7 @@ namespace cryptonote bool on_get_output_histogram(const COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request& req, COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response& res, epee::json_rpc::error& error_resp); bool on_get_version(const COMMAND_RPC_GET_VERSION::request& req, COMMAND_RPC_GET_VERSION::response& res, epee::json_rpc::error& error_resp); bool on_get_coinbase_tx_sum(const COMMAND_RPC_GET_COINBASE_TX_SUM::request& req, COMMAND_RPC_GET_COINBASE_TX_SUM::response& res, epee::json_rpc::error& error_resp); - bool on_get_per_kb_fee_estimate(const COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::request& req, COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::response& res, epee::json_rpc::error& error_resp); + bool on_get_base_fee_estimate(const COMMAND_RPC_GET_BASE_FEE_ESTIMATE::request& req, COMMAND_RPC_GET_BASE_FEE_ESTIMATE::response& res, epee::json_rpc::error& error_resp); bool on_get_alternate_chains(const COMMAND_RPC_GET_ALTERNATE_CHAINS::request& req, COMMAND_RPC_GET_ALTERNATE_CHAINS::response& res, epee::json_rpc::error& error_resp); bool on_relay_tx(const COMMAND_RPC_RELAY_TX::request& req, COMMAND_RPC_RELAY_TX::response& res, epee::json_rpc::error& error_resp); bool on_sync_info(const COMMAND_RPC_SYNC_INFO::request& req, COMMAND_RPC_SYNC_INFO::response& res, epee::json_rpc::error& error_resp); diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index e2d120cad..da4c8b6cf 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -48,8 +48,8 @@ namespace cryptonote // whether they can talk to a given daemon without having to know in // advance which version they will stop working with // Don't go over 32767 for any of these -#define CORE_RPC_VERSION_MAJOR 1 -#define CORE_RPC_VERSION_MINOR 21 +#define CORE_RPC_VERSION_MAJOR 2 +#define CORE_RPC_VERSION_MINOR 0 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR) @@ -959,7 +959,9 @@ namespace cryptonote std::string top_block_hash; uint64_t cumulative_difficulty; uint64_t block_size_limit; + uint64_t block_weight_limit; uint64_t block_size_median; + uint64_t block_weight_median; uint64_t start_time; uint64_t free_space; bool offline; @@ -990,7 +992,9 @@ namespace cryptonote KV_SERIALIZE(top_block_hash) KV_SERIALIZE(cumulative_difficulty) KV_SERIALIZE(block_size_limit) + KV_SERIALIZE_OPT(block_weight_limit, (uint64_t)0) KV_SERIALIZE(block_size_median) + KV_SERIALIZE_OPT(block_weight_median, (uint64_t)0) KV_SERIALIZE(start_time) KV_SERIALIZE(free_space) KV_SERIALIZE(offline) @@ -1195,6 +1199,7 @@ namespace cryptonote difficulty_type difficulty; uint64_t reward; uint64_t block_size; + uint64_t block_weight; uint64_t num_txes; std::string pow_hash; @@ -1211,6 +1216,7 @@ namespace cryptonote KV_SERIALIZE(difficulty) KV_SERIALIZE(reward) KV_SERIALIZE(block_size) + KV_SERIALIZE_OPT(block_weight, (uint64_t)0) KV_SERIALIZE(num_txes) KV_SERIALIZE(pow_hash) END_KV_SERIALIZE_MAP() @@ -1451,6 +1457,7 @@ namespace cryptonote std::string id_hash; std::string tx_json; // TODO - expose this data directly uint64_t blob_size; + uint64_t weight; uint64_t fee; std::string max_used_block_id_hash; uint64_t max_used_block_height; @@ -1468,6 +1475,7 @@ namespace cryptonote KV_SERIALIZE(id_hash) KV_SERIALIZE(tx_json) KV_SERIALIZE(blob_size) + KV_SERIALIZE_OPT(weight, (uint64_t)0) KV_SERIALIZE(fee) KV_SERIALIZE(max_used_block_id_hash) KV_SERIALIZE(max_used_block_height) @@ -1564,7 +1572,7 @@ namespace cryptonote struct tx_backlog_entry { - uint64_t blob_size; + uint64_t weight; uint64_t fee; uint64_t time_in_pool; }; @@ -2102,7 +2110,7 @@ namespace cryptonote }; }; - struct COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE + struct COMMAND_RPC_GET_BASE_FEE_ESTIMATE { struct request { @@ -2117,11 +2125,13 @@ namespace cryptonote { std::string status; uint64_t fee; + uint64_t quantization_mask; bool untrusted; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(status) KV_SERIALIZE(fee) + KV_SERIALIZE_OPT(quantization_mask, (uint64_t)1) KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; diff --git a/src/rpc/daemon_handler.cpp b/src/rpc/daemon_handler.cpp index 55858cc2a..26f102a8b 100644 --- a/src/rpc/daemon_handler.cpp +++ b/src/rpc/daemon_handler.cpp @@ -86,53 +86,45 @@ namespace rpc res.error_details = "incorrect number of transactions retrieved for block"; return; } - std::vector<transaction> txs; - for (const auto& p : it->second) - { - txs.resize(txs.size() + 1); - if (!parse_and_validate_tx_from_blob(p.second, txs.back())) - { - res.blocks.clear(); - res.output_indices.clear(); - res.status = Message::STATUS_FAILED; - res.error_details = "failed retrieving a requested transaction"; - return; - } - } cryptonote::rpc::block_output_indices& indices = res.output_indices[block_count]; // miner tx output indices { cryptonote::rpc::tx_output_indices tx_indices; - bool r = m_core.get_tx_outputs_gindexs(get_transaction_hash(bwt.block.miner_tx), tx_indices); - if (!r) + if (!m_core.get_tx_outputs_gindexs(get_transaction_hash(bwt.block.miner_tx), tx_indices)) { res.status = Message::STATUS_FAILED; res.error_details = "core::get_tx_outputs_gindexs() returned false"; return; } - indices.push_back(tx_indices); + indices.push_back(std::move(tx_indices)); } - // assume each block returned is returned with all its transactions - // in the correct order. - auto tx_it = txs.begin(); - for (const crypto::hash& h : bwt.block.tx_hashes) + auto hash_it = bwt.block.tx_hashes.begin(); + bwt.transactions.reserve(it->second.size()); + for (const auto& blob : it->second) { - bwt.transactions.emplace(h, *tx_it); - tx_it++; + bwt.transactions.emplace_back(); + if (!parse_and_validate_tx_from_blob(blob.second, bwt.transactions.back())) + { + res.blocks.clear(); + res.output_indices.clear(); + res.status = Message::STATUS_FAILED; + res.error_details = "failed retrieving a requested transaction"; + return; + } cryptonote::rpc::tx_output_indices tx_indices; - bool r = m_core.get_tx_outputs_gindexs(h, tx_indices); - if (!r) + if (!m_core.get_tx_outputs_gindexs(*hash_it, tx_indices)) { res.status = Message::STATUS_FAILED; res.error_details = "core::get_tx_outputs_gindexs() returned false"; return; } - indices.push_back(tx_indices); + indices.push_back(std::move(tx_indices)); + ++hash_it; } it++; @@ -480,7 +472,8 @@ namespace rpc res.info.testnet = m_core.get_nettype() == TESTNET; res.info.stagenet = m_core.get_nettype() == STAGENET; res.info.cumulative_difficulty = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.info.height - 1); - res.info.block_size_limit = m_core.get_blockchain_storage().get_current_cumulative_blocksize_limit(); + res.info.block_size_limit = res.info.block_weight_limit = m_core.get_blockchain_storage().get_current_cumulative_block_weight_limit(); + res.info.block_size_median = res.info.block_weight_median = m_core.get_blockchain_storage().get_current_cumulative_block_weight_median(); res.info.start_time = (uint64_t)m_core.get_start_time(); res.status = Message::STATUS_OK; @@ -771,7 +764,7 @@ namespace rpc void DaemonHandler::handle(const GetPerKBFeeEstimate::Request& req, GetPerKBFeeEstimate::Response& res) { - res.estimated_fee_per_kb = m_core.get_blockchain_storage().get_dynamic_per_kb_fee_estimate(req.num_grace_blocks); + res.estimated_fee_per_kb = m_core.get_blockchain_storage().get_dynamic_base_fee_estimate(req.num_grace_blocks); res.status = Message::STATUS_OK; } diff --git a/src/rpc/message_data_structs.h b/src/rpc/message_data_structs.h index fc1b2329d..cf15ade1c 100644 --- a/src/rpc/message_data_structs.h +++ b/src/rpc/message_data_structs.h @@ -44,7 +44,7 @@ namespace rpc struct block_with_transactions { cryptonote::block block; - std::unordered_map<crypto::hash, cryptonote::transaction> transactions; + std::vector<cryptonote::transaction> transactions; }; typedef std::vector<uint64_t> tx_output_indices; @@ -85,6 +85,7 @@ namespace rpc cryptonote::transaction tx; crypto::hash tx_hash; uint64_t blob_size; + uint64_t weight; uint64_t fee; crypto::hash max_used_block_hash; uint64_t max_used_block_height; @@ -185,6 +186,9 @@ namespace rpc crypto::hash top_block_hash; uint64_t cumulative_difficulty; uint64_t block_size_limit; + uint64_t block_weight_limit; + uint64_t block_size_median; + uint64_t block_weight_median; uint64_t start_time; }; diff --git a/src/serialization/json_object.cpp b/src/serialization/json_object.cpp index c2467b863..89a1dbd23 100644 --- a/src/serialization/json_object.cpp +++ b/src/serialization/json_object.cpp @@ -28,6 +28,8 @@ #include "json_object.h" +#include <boost/range/adaptor/transformed.hpp> +#include <boost/variant/apply_visitor.hpp> #include <limits> #include <type_traits> #include "string_tools.h" @@ -219,11 +221,11 @@ void toJsonValue(rapidjson::Document& doc, const cryptonote::transaction& tx, ra INSERT_INTO_JSON_OBJECT(val, doc, version, tx.version); INSERT_INTO_JSON_OBJECT(val, doc, unlock_time, tx.unlock_time); - INSERT_INTO_JSON_OBJECT(val, doc, vin, tx.vin); - INSERT_INTO_JSON_OBJECT(val, doc, vout, tx.vout); + INSERT_INTO_JSON_OBJECT(val, doc, inputs, tx.vin); + INSERT_INTO_JSON_OBJECT(val, doc, outputs, tx.vout); INSERT_INTO_JSON_OBJECT(val, doc, extra, tx.extra); INSERT_INTO_JSON_OBJECT(val, doc, signatures, tx.signatures); - INSERT_INTO_JSON_OBJECT(val, doc, rct_signatures, tx.rct_signatures); + INSERT_INTO_JSON_OBJECT(val, doc, ringct, tx.rct_signatures); } @@ -236,11 +238,11 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::transaction& tx) GET_FROM_JSON_OBJECT(val, tx.version, version); GET_FROM_JSON_OBJECT(val, tx.unlock_time, unlock_time); - GET_FROM_JSON_OBJECT(val, tx.vin, vin); - GET_FROM_JSON_OBJECT(val, tx.vout, vout); + GET_FROM_JSON_OBJECT(val, tx.vin, inputs); + GET_FROM_JSON_OBJECT(val, tx.vout, outputs); GET_FROM_JSON_OBJECT(val, tx.extra, extra); GET_FROM_JSON_OBJECT(val, tx.signatures, signatures); - GET_FROM_JSON_OBJECT(val, tx.rct_signatures, rct_signatures); + GET_FROM_JSON_OBJECT(val, tx.rct_signatures, ringct); } void toJsonValue(rapidjson::Document& doc, const cryptonote::block& b, rapidjson::Value& val) @@ -277,26 +279,31 @@ void toJsonValue(rapidjson::Document& doc, const cryptonote::txin_v& txin, rapid { val.SetObject(); - if (txin.type() == typeid(cryptonote::txin_gen)) + struct add_input { - val.AddMember("type", "txin_gen", doc.GetAllocator()); - INSERT_INTO_JSON_OBJECT(val, doc, value, boost::get<cryptonote::txin_gen>(txin)); - } - else if (txin.type() == typeid(cryptonote::txin_to_script)) - { - val.AddMember("type", "txin_to_script", doc.GetAllocator()); - INSERT_INTO_JSON_OBJECT(val, doc, value, boost::get<cryptonote::txin_to_script>(txin)); - } - else if (txin.type() == typeid(cryptonote::txin_to_scripthash)) - { - val.AddMember("type", "txin_to_scripthash", doc.GetAllocator()); - INSERT_INTO_JSON_OBJECT(val, doc, value, boost::get<cryptonote::txin_to_scripthash>(txin)); - } - else if (txin.type() == typeid(cryptonote::txin_to_key)) - { - val.AddMember("type", "txin_to_key", doc.GetAllocator()); - INSERT_INTO_JSON_OBJECT(val, doc, value, boost::get<cryptonote::txin_to_key>(txin)); - } + using result_type = void; + + rapidjson::Document& doc; + rapidjson::Value& val; + + void operator()(cryptonote::txin_to_key const& input) const + { + INSERT_INTO_JSON_OBJECT(val, doc, to_key, input); + } + void operator()(cryptonote::txin_gen const& input) const + { + INSERT_INTO_JSON_OBJECT(val, doc, gen, input); + } + void operator()(cryptonote::txin_to_script const& input) const + { + INSERT_INTO_JSON_OBJECT(val, doc, to_script, input); + } + void operator()(cryptonote::txin_to_scripthash const& input) const + { + INSERT_INTO_JSON_OBJECT(val, doc, to_scripthash, input); + } + }; + boost::apply_visitor(add_input{doc, val}, txin); } @@ -307,31 +314,37 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::txin_v& txin) throw WRONG_TYPE("json object"); } - OBJECT_HAS_MEMBER_OR_THROW(val, "type") - OBJECT_HAS_MEMBER_OR_THROW(val, "value") - if (val["type"]== "txin_gen") - { - cryptonote::txin_gen tmpVal; - fromJsonValue(val["value"], tmpVal); - txin = tmpVal; - } - else if (val["type"]== "txin_to_script") + if (val.MemberCount() != 1) { - cryptonote::txin_to_script tmpVal; - fromJsonValue(val["value"], tmpVal); - txin = tmpVal; + throw MISSING_KEY("Invalid input object"); } - else if (val["type"] == "txin_to_scripthash") - { - cryptonote::txin_to_scripthash tmpVal; - fromJsonValue(val["value"], tmpVal); - txin = tmpVal; - } - else if (val["type"] == "txin_to_key") + + for (auto const& elem : val.GetObject()) { - cryptonote::txin_to_key tmpVal; - fromJsonValue(val["value"], tmpVal); - txin = tmpVal; + if (elem.name == "to_key") + { + cryptonote::txin_to_key tmpVal; + fromJsonValue(elem.value, tmpVal); + txin = std::move(tmpVal); + } + else if (elem.name == "gen") + { + cryptonote::txin_gen tmpVal; + fromJsonValue(elem.value, tmpVal); + txin = std::move(tmpVal); + } + else if (elem.name == "to_script") + { + cryptonote::txin_to_script tmpVal; + fromJsonValue(elem.value, tmpVal); + txin = std::move(tmpVal); + } + else if (elem.name == "to_scripthash") + { + cryptonote::txin_to_scripthash tmpVal; + fromJsonValue(elem.value, tmpVal); + txin = std::move(tmpVal); + } } } @@ -405,7 +418,7 @@ void toJsonValue(rapidjson::Document& doc, const cryptonote::txin_to_key& txin, INSERT_INTO_JSON_OBJECT(val, doc, amount, txin.amount); INSERT_INTO_JSON_OBJECT(val, doc, key_offsets, txin.key_offsets); - INSERT_INTO_JSON_OBJECT(val, doc, k_image, txin.k_image); + INSERT_INTO_JSON_OBJECT(val, doc, key_image, txin.k_image); } @@ -418,58 +431,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::txin_to_key& txin) GET_FROM_JSON_OBJECT(val, txin.amount, amount); GET_FROM_JSON_OBJECT(val, txin.key_offsets, key_offsets); - GET_FROM_JSON_OBJECT(val, txin.k_image, k_image); -} - -void toJsonValue(rapidjson::Document& doc, const cryptonote::txout_target_v& txout, rapidjson::Value& val) -{ - val.SetObject(); - - if (txout.type() == typeid(cryptonote::txout_to_script)) - { - val.AddMember("type", "txout_to_script", doc.GetAllocator()); - INSERT_INTO_JSON_OBJECT(val, doc, value, boost::get<cryptonote::txout_to_script>(txout)); - } - else if (txout.type() == typeid(cryptonote::txout_to_scripthash)) - { - val.AddMember("type", "txout_to_scripthash", doc.GetAllocator()); - INSERT_INTO_JSON_OBJECT(val, doc, value, boost::get<cryptonote::txout_to_scripthash>(txout)); - } - else if (txout.type() == typeid(cryptonote::txout_to_key)) - { - val.AddMember("type", "txout_to_key", doc.GetAllocator()); - INSERT_INTO_JSON_OBJECT(val, doc, value, boost::get<cryptonote::txout_to_key>(txout)); - } -} - - -void fromJsonValue(const rapidjson::Value& val, cryptonote::txout_target_v& txout) -{ - if (!val.IsObject()) - { - throw WRONG_TYPE("json object"); - } - - OBJECT_HAS_MEMBER_OR_THROW(val, "type") - OBJECT_HAS_MEMBER_OR_THROW(val, "value") - if (val["type"]== "txout_to_script") - { - cryptonote::txout_to_script tmpVal; - fromJsonValue(val["value"], tmpVal); - txout = tmpVal; - } - else if (val["type"] == "txout_to_scripthash") - { - cryptonote::txout_to_scripthash tmpVal; - fromJsonValue(val["value"], tmpVal); - txout = tmpVal; - } - else if (val["type"] == "txout_to_key") - { - cryptonote::txout_to_key tmpVal; - fromJsonValue(val["value"], tmpVal); - txout = tmpVal; - } + GET_FROM_JSON_OBJECT(val, txin.k_image, key_image); } void toJsonValue(rapidjson::Document& doc, const cryptonote::txout_to_script& txout, rapidjson::Value& val) @@ -533,7 +495,28 @@ void toJsonValue(rapidjson::Document& doc, const cryptonote::tx_out& txout, rapi val.SetObject(); INSERT_INTO_JSON_OBJECT(val, doc, amount, txout.amount); - INSERT_INTO_JSON_OBJECT(val, doc, target, txout.target); + + struct add_output + { + using result_type = void; + + rapidjson::Document& doc; + rapidjson::Value& val; + + void operator()(cryptonote::txout_to_key const& output) const + { + INSERT_INTO_JSON_OBJECT(val, doc, to_key, output); + } + void operator()(cryptonote::txout_to_script const& output) const + { + INSERT_INTO_JSON_OBJECT(val, doc, to_script, output); + } + void operator()(cryptonote::txout_to_scripthash const& output) const + { + INSERT_INTO_JSON_OBJECT(val, doc, to_scripthash, output); + } + }; + boost::apply_visitor(add_output{doc, val}, txout.target); } void fromJsonValue(const rapidjson::Value& val, cryptonote::tx_out& txout) @@ -543,8 +526,37 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::tx_out& txout) throw WRONG_TYPE("json object"); } - GET_FROM_JSON_OBJECT(val, txout.amount, amount); - GET_FROM_JSON_OBJECT(val, txout.target, target); + if (val.MemberCount() != 2) + { + throw MISSING_KEY("Invalid input object"); + } + + for (auto const& elem : val.GetObject()) + { + if (elem.name == "amount") + { + fromJsonValue(elem.value, txout.amount); + } + + if (elem.name == "to_key") + { + cryptonote::txout_to_key tmpVal; + fromJsonValue(elem.value, tmpVal); + txout.target = std::move(tmpVal); + } + else if (elem.name == "to_script") + { + cryptonote::txout_to_script tmpVal; + fromJsonValue(elem.value, tmpVal); + txout.target = std::move(tmpVal); + } + else if (elem.name == "to_scripthash") + { + cryptonote::txout_to_scripthash tmpVal; + fromJsonValue(elem.value, tmpVal); + txout.target = std::move(tmpVal); + } + } } void toJsonValue(rapidjson::Document& doc, const cryptonote::connection_info& info, rapidjson::Value& val) @@ -617,7 +629,7 @@ void toJsonValue(rapidjson::Document& doc, const cryptonote::block_complete_entr val.SetObject(); INSERT_INTO_JSON_OBJECT(val, doc, block, blk.block); - INSERT_INTO_JSON_OBJECT(val, doc, txs, blk.txs); + INSERT_INTO_JSON_OBJECT(val, doc, transactions, blk.txs); } @@ -629,7 +641,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::block_complete_entry } GET_FROM_JSON_OBJECT(val, blk.block, block); - GET_FROM_JSON_OBJECT(val, blk.txs, txs); + GET_FROM_JSON_OBJECT(val, blk.txs, transactions); } void toJsonValue(rapidjson::Document& doc, const cryptonote::rpc::block_with_transactions& blk, rapidjson::Value& val) @@ -745,6 +757,7 @@ void toJsonValue(rapidjson::Document& doc, const cryptonote::rpc::tx_in_pool& tx INSERT_INTO_JSON_OBJECT(val, doc, tx, tx.tx); INSERT_INTO_JSON_OBJECT(val, doc, tx_hash, tx.tx_hash); INSERT_INTO_JSON_OBJECT(val, doc, blob_size, tx.blob_size); + INSERT_INTO_JSON_OBJECT(val, doc, weight, tx.weight); INSERT_INTO_JSON_OBJECT(val, doc, fee, tx.fee); INSERT_INTO_JSON_OBJECT(val, doc, max_used_block_hash, tx.max_used_block_hash); INSERT_INTO_JSON_OBJECT(val, doc, max_used_block_height, tx.max_used_block_height); @@ -768,6 +781,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::tx_in_pool& tx) GET_FROM_JSON_OBJECT(val, tx.tx, tx); GET_FROM_JSON_OBJECT(val, tx.blob_size, blob_size); + GET_FROM_JSON_OBJECT(val, tx.weight, weight); GET_FROM_JSON_OBJECT(val, tx.fee, fee); GET_FROM_JSON_OBJECT(val, tx.max_used_block_hash, max_used_block_hash); GET_FROM_JSON_OBJECT(val, tx.max_used_block_height, max_used_block_height); @@ -936,51 +950,70 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::BlockHeaderResp void toJsonValue(rapidjson::Document& doc, const rct::rctSig& sig, rapidjson::Value& val) { + using boost::adaptors::transform; + val.SetObject(); + const auto just_mask = [] (rct::ctkey const& key) -> rct::key const& + { + return key.mask; + }; + INSERT_INTO_JSON_OBJECT(val, doc, type, sig.type); - INSERT_INTO_JSON_OBJECT(val, doc, message, sig.message); - INSERT_INTO_JSON_OBJECT(val, doc, mixRing, sig.mixRing); - INSERT_INTO_JSON_OBJECT(val, doc, pseudoOuts, sig.pseudoOuts); - INSERT_INTO_JSON_OBJECT(val, doc, ecdhInfo, sig.ecdhInfo); - INSERT_INTO_JSON_OBJECT(val, doc, outPk, sig.outPk); - INSERT_INTO_JSON_OBJECT(val, doc, txnFee, sig.txnFee); - INSERT_INTO_JSON_OBJECT(val, doc, p, sig.p); + INSERT_INTO_JSON_OBJECT(val, doc, encrypted, sig.ecdhInfo); + INSERT_INTO_JSON_OBJECT(val, doc, commitments, transform(sig.outPk, just_mask)); + INSERT_INTO_JSON_OBJECT(val, doc, fee, sig.txnFee); + + // prunable + { + rapidjson::Value prunable; + prunable.SetObject(); + + INSERT_INTO_JSON_OBJECT(prunable, doc, range_proofs, sig.p.rangeSigs); + INSERT_INTO_JSON_OBJECT(prunable, doc, bulletproofs, sig.p.bulletproofs); + INSERT_INTO_JSON_OBJECT(prunable, doc, mlsags, sig.p.MGs); + INSERT_INTO_JSON_OBJECT(prunable, doc, pseudo_outs, sig.get_pseudo_outs()); + + val.AddMember("prunable", prunable, doc.GetAllocator()); + } } void fromJsonValue(const rapidjson::Value& val, rct::rctSig& sig) { + using boost::adaptors::transform; + if (!val.IsObject()) { throw WRONG_TYPE("json object"); } + std::vector<rct::key> commitments; + GET_FROM_JSON_OBJECT(val, sig.type, type); - GET_FROM_JSON_OBJECT(val, sig.message, message); - GET_FROM_JSON_OBJECT(val, sig.mixRing, mixRing); - GET_FROM_JSON_OBJECT(val, sig.pseudoOuts, pseudoOuts); - GET_FROM_JSON_OBJECT(val, sig.ecdhInfo, ecdhInfo); - GET_FROM_JSON_OBJECT(val, sig.outPk, outPk); - GET_FROM_JSON_OBJECT(val, sig.txnFee, txnFee); - GET_FROM_JSON_OBJECT(val, sig.p, p); -} + GET_FROM_JSON_OBJECT(val, sig.ecdhInfo, encrypted); + GET_FROM_JSON_OBJECT(val, commitments, commitments); + GET_FROM_JSON_OBJECT(val, sig.txnFee, fee); -void toJsonValue(rapidjson::Document& doc, const rct::ctkey& key, rapidjson::Value& val) -{ - val.SetObject(); + // prunable + { + OBJECT_HAS_MEMBER_OR_THROW(val, "prunable"); + const auto& prunable = val["prunable"]; - INSERT_INTO_JSON_OBJECT(val, doc, dest, key.dest); - INSERT_INTO_JSON_OBJECT(val, doc, mask, key.mask); -} + rct::keyV pseudo_outs; -void fromJsonValue(const rapidjson::Value& val, rct::ctkey& key) -{ - if (!val.IsObject()) + GET_FROM_JSON_OBJECT(prunable, sig.p.rangeSigs, range_proofs); + GET_FROM_JSON_OBJECT(prunable, sig.p.bulletproofs, bulletproofs); + GET_FROM_JSON_OBJECT(prunable, sig.p.MGs, mlsags); + GET_FROM_JSON_OBJECT(prunable, pseudo_outs, pseudo_outs); + + sig.get_pseudo_outs() = std::move(pseudo_outs); + } + + sig.outPk.reserve(commitments.size()); + for (rct::key const& commitment : commitments) { - throw WRONG_TYPE("json object"); + sig.outPk.push_back({{}, commitment}); } - GET_FROM_JSON_OBJECT(val, key.dest, dest); - GET_FROM_JSON_OBJECT(val, key.mask, mask); } void toJsonValue(rapidjson::Document& doc, const rct::ecdhTuple& tuple, rapidjson::Value& val) @@ -1002,27 +1035,6 @@ void fromJsonValue(const rapidjson::Value& val, rct::ecdhTuple& tuple) GET_FROM_JSON_OBJECT(val, tuple.amount, amount); } -void toJsonValue(rapidjson::Document& doc, const rct::rctSigPrunable& sig, rapidjson::Value& val) -{ - val.SetObject(); - - INSERT_INTO_JSON_OBJECT(val, doc, rangeSigs, sig.rangeSigs); - INSERT_INTO_JSON_OBJECT(val, doc, bulletproofs, sig.bulletproofs); - INSERT_INTO_JSON_OBJECT(val, doc, MGs, sig.MGs); -} - -void fromJsonValue(const rapidjson::Value& val, rct::rctSigPrunable& sig) -{ - if (!val.IsObject()) - { - throw WRONG_TYPE("json object"); - } - - GET_FROM_JSON_OBJECT(val, sig.rangeSigs, rangeSigs); - GET_FROM_JSON_OBJECT(val, sig.bulletproofs, bulletproofs); - GET_FROM_JSON_OBJECT(val, sig.MGs, MGs); -} - void toJsonValue(rapidjson::Document& doc, const rct::rangeSig& sig, rapidjson::Value& val) { val.SetObject(); @@ -1040,10 +1052,16 @@ void fromJsonValue(const rapidjson::Value& val, rct::rangeSig& sig) throw WRONG_TYPE("json object"); } + const auto ci = val.FindMember("Ci"); + if (ci == val.MemberEnd()) + { + throw MISSING_KEY("Ci"); + } + GET_FROM_JSON_OBJECT(val, sig.asig, asig); std::vector<rct::key> keyVector; - cryptonote::json::fromJsonValue(val["Ci"], keyVector); + cryptonote::json::fromJsonValue(ci->value, keyVector); if (!(keyVector.size() == 64)) { throw WRONG_TYPE("key64 (rct::key[64])"); @@ -1098,10 +1116,10 @@ void toJsonValue(rapidjson::Document& doc, const rct::boroSig& sig, rapidjson::V val.SetObject(); std::vector<rct::key> keyVector(sig.s0, std::end(sig.s0)); - INSERT_INTO_JSON_OBJECT(val, doc, s0, sig.s0); + INSERT_INTO_JSON_OBJECT(val, doc, s0, keyVector); keyVector.assign(sig.s1, std::end(sig.s1)); - INSERT_INTO_JSON_OBJECT(val, doc, s1, sig.s1); + INSERT_INTO_JSON_OBJECT(val, doc, s1, keyVector); INSERT_INTO_JSON_OBJECT(val, doc, ee, sig.ee); } @@ -1179,6 +1197,9 @@ void toJsonValue(rapidjson::Document& doc, const cryptonote::rpc::DaemonInfo& in INSERT_INTO_JSON_OBJECT(val, doc, top_block_hash, info.top_block_hash); INSERT_INTO_JSON_OBJECT(val, doc, cumulative_difficulty, info.cumulative_difficulty); INSERT_INTO_JSON_OBJECT(val, doc, block_size_limit, info.block_size_limit); + INSERT_INTO_JSON_OBJECT(val, doc, block_weight_limit, info.block_weight_limit); + INSERT_INTO_JSON_OBJECT(val, doc, block_size_median, info.block_size_median); + INSERT_INTO_JSON_OBJECT(val, doc, block_weight_median, info.block_weight_median); INSERT_INTO_JSON_OBJECT(val, doc, start_time, info.start_time); } @@ -1205,6 +1226,9 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::DaemonInfo& inf GET_FROM_JSON_OBJECT(val, info.top_block_hash, top_block_hash); GET_FROM_JSON_OBJECT(val, info.cumulative_difficulty, cumulative_difficulty); GET_FROM_JSON_OBJECT(val, info.block_size_limit, block_size_limit); + GET_FROM_JSON_OBJECT(val, info.block_weight_limit, block_weight_limit); + GET_FROM_JSON_OBJECT(val, info.block_size_median, block_size_median); + GET_FROM_JSON_OBJECT(val, info.block_weight_median, block_weight_median); GET_FROM_JSON_OBJECT(val, info.start_time, start_time); } diff --git a/src/serialization/json_object.h b/src/serialization/json_object.h index de21ace66..da3351fe3 100644 --- a/src/serialization/json_object.h +++ b/src/serialization/json_object.h @@ -263,15 +263,9 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::BlockHeaderResp void toJsonValue(rapidjson::Document& doc, const rct::rctSig& i, rapidjson::Value& val); void fromJsonValue(const rapidjson::Value& i, rct::rctSig& sig); -void toJsonValue(rapidjson::Document& doc, const rct::ctkey& key, rapidjson::Value& val); -void fromJsonValue(const rapidjson::Value& val, rct::ctkey& key); - void toJsonValue(rapidjson::Document& doc, const rct::ecdhTuple& tuple, rapidjson::Value& val); void fromJsonValue(const rapidjson::Value& val, rct::ecdhTuple& tuple); -void toJsonValue(rapidjson::Document& doc, const rct::rctSigPrunable& sig, rapidjson::Value& val); -void fromJsonValue(const rapidjson::Value& val, rct::rctSigPrunable& sig); - void toJsonValue(rapidjson::Document& doc, const rct::rangeSig& sig, rapidjson::Value& val); void fromJsonValue(const rapidjson::Value& val, rct::rangeSig& sig); diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index e07c7e49b..fc6c4ab43 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -86,26 +86,27 @@ typedef cryptonote::simple_wallet sw; #define EXTENDED_LOGS_FILE "wallet_details.log" -#define DEFAULT_MIX 6 +#define DEFAULT_MIX 10 -#define MIN_RING_SIZE 7 // Used to inform user about min ring size -- does not track actual protocol +#define MIN_RING_SIZE 11 // Used to inform user about min ring size -- does not track actual protocol #define LOCK_IDLE_SCOPE() \ bool auto_refresh_enabled = m_auto_refresh_enabled.load(std::memory_order_relaxed); \ m_auto_refresh_enabled.store(false, std::memory_order_relaxed); \ /* stop any background refresh, and take over */ \ m_wallet->stop(); \ - m_idle_mutex.lock(); \ - while (m_auto_refresh_refreshing) \ - m_idle_cond.notify_one(); \ - m_idle_mutex.unlock(); \ -/* if (auto_refresh_run)*/ \ - /*m_auto_refresh_thread.join();*/ \ boost::unique_lock<boost::mutex> lock(m_idle_mutex); \ + m_idle_cond.notify_all(); \ epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){ \ m_auto_refresh_enabled.store(auto_refresh_enabled, std::memory_order_relaxed); \ }) +#define SCOPED_WALLET_UNLOCK() \ + LOCK_IDLE_SCOPE(); \ + boost::optional<tools::password_container> pwd_container = boost::none; \ + if (m_wallet->ask_password() && !m_wallet->watch_only() && !(pwd_container = get_and_verify_password())) { return true; } \ + tools::wallet_keys_unlocker unlocker(*m_wallet, pwd_container); + enum TransferType { TransferOriginal, TransferNew, @@ -128,8 +129,6 @@ namespace const command_line::arg_descriptor<bool> arg_restore_deterministic_wallet = {"restore-deterministic-wallet", sw::tr("Recover wallet using Electrum-style mnemonic seed"), false}; const command_line::arg_descriptor<bool> arg_restore_multisig_wallet = {"restore-multisig-wallet", sw::tr("Recover multisig wallet using Electrum-style mnemonic seed"), false}; const command_line::arg_descriptor<bool> arg_non_deterministic = {"non-deterministic", sw::tr("Generate non-deterministic view and spend keys"), false}; - const command_line::arg_descriptor<bool> arg_trusted_daemon = {"trusted-daemon", sw::tr("Enable commands which rely on a trusted daemon"), false}; - const command_line::arg_descriptor<bool> arg_untrusted_daemon = {"untrusted-daemon", sw::tr("Disable commands which rely on a trusted daemon"), false}; const command_line::arg_descriptor<bool> arg_allow_mismatched_daemon_version = {"allow-mismatched-daemon-version", sw::tr("Allow communicating with a daemon that uses a different RPC version"), false}; const command_line::arg_descriptor<uint64_t> arg_restore_height = {"restore-height", sw::tr("Restore from specific blockchain height"), 0}; const command_line::arg_descriptor<bool> arg_do_not_relay = {"do-not-relay", sw::tr("The newly created transaction will not be relayed to the monero network"), false}; @@ -152,12 +151,31 @@ namespace // // Note also that input for passwords is NOT translated, to remain compatible with any // passwords containing special characters that predate this switch to UTF-8 support. - static std::string cp850_to_utf8(const std::string &cp850_str) + template<typename T> + static T cp850_to_utf8(const T &cp850_str) { boost::locale::generator gen; gen.locale_cache_enabled(true); std::locale loc = gen("en_US.CP850"); - return boost::locale::conv::to_utf<char>(cp850_str, loc); + const boost::locale::conv::method_type how = boost::locale::conv::default_method; + T result; + const char *begin = cp850_str.data(); + const char *end = begin + cp850_str.size(); + result.reserve(end-begin); + typedef std::back_insert_iterator<T> inserter_type; + inserter_type inserter(result); + boost::locale::utf::code_point c; + while(begin!=end) { + c=boost::locale::utf::utf_traits<char>::template decode<char const *>(begin,end); + if(c==boost::locale::utf::illegal || c==boost::locale::utf::incomplete) { + if(how==boost::locale::conv::stop) + throw boost::locale::conv::conversion_error(); + } + else { + boost::locale::utf::utf_traits<char>::template encode<inserter_type>(c,inserter); + } + } + return result; } #endif @@ -177,6 +195,28 @@ namespace return epee::string_tools::trim(buf); } + epee::wipeable_string input_secure_line(const std::string& prompt) + { +#ifdef HAVE_READLINE + rdln::suspend_readline pause_readline; +#endif + auto pwd_container = tools::password_container::prompt(false, prompt.c_str(), false); + if (!pwd_container) + { + MERROR("Failed to read secure line"); + return ""; + } + + epee::wipeable_string buf = pwd_container->password(); + +#ifdef WIN32 + buf = cp850_to_utf8(buf); +#endif + + buf.trim(); + return buf; + } + boost::optional<tools::password_container> password_prompter(const char *prompt, bool verify) { #ifdef HAVE_READLINE @@ -512,6 +552,18 @@ namespace } return true; } + + void print_secret_key(const crypto::secret_key &k) + { + static constexpr const char hex[] = u8"0123456789abcdef"; + const uint8_t *ptr = (const uint8_t*)k.data; + for (size_t i = 0, sz = sizeof(k); i < sz; ++i) + { + putchar(hex[*ptr >> 4]); + putchar(hex[*ptr & 15]); + ++ptr; + } + } } bool parse_priority(const std::string& arg, uint32_t& priority) @@ -527,6 +579,18 @@ bool parse_priority(const std::string& arg, uint32_t& priority) return false; } +std::string join_priority_strings(const char *delimiter) +{ + std::string s; + for (size_t n = 0; n < allowed_priority_strings.size(); ++n) + { + if (!s.empty()) + s += delimiter; + s += allowed_priority_strings[n]; + } + return s; +} + std::string simple_wallet::get_commands_str() { std::stringstream ss; @@ -561,12 +625,15 @@ std::string simple_wallet::get_command_usage(const std::vector<std::string> &arg bool simple_wallet::viewkey(const std::vector<std::string> &args/* = std::vector<std::string>()*/) { - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + SCOPED_WALLET_UNLOCK(); // don't log + PAUSE_READLINE(); if (m_wallet->key_on_device()) { std::cout << "secret: On device. Not available" << std::endl; } else { - std::cout << "secret: " << string_tools::pod_to_hex(m_wallet->get_account().get_keys().m_view_secret_key) << std::endl; + printf("secret: "); + print_secret_key(m_wallet->get_account().get_keys().m_view_secret_key); + putchar('\n'); } std::cout << "public: " << string_tools::pod_to_hex(m_wallet->get_account().get_keys().m_account_address.m_view_public_key) << std::endl; @@ -580,12 +647,15 @@ bool simple_wallet::spendkey(const std::vector<std::string> &args/* = std::vecto fail_msg_writer() << tr("wallet is watch-only and has no spend key"); return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + SCOPED_WALLET_UNLOCK(); // don't log + PAUSE_READLINE(); if (m_wallet->key_on_device()) { std::cout << "secret: On device. Not available" << std::endl; } else { - std::cout << "secret: " << string_tools::pod_to_hex(m_wallet->get_account().get_keys().m_spend_secret_key) << std::endl; + printf("secret: "); + print_secret_key(m_wallet->get_account().get_keys().m_spend_secret_key); + putchar('\n'); } std::cout << "public: " << string_tools::pod_to_hex(m_wallet->get_account().get_keys().m_account_address.m_spend_public_key) << std::endl; @@ -595,7 +665,7 @@ bool simple_wallet::spendkey(const std::vector<std::string> &args/* = std::vecto bool simple_wallet::print_seed(bool encrypted) { bool success = false; - std::string seed; + epee::wipeable_string seed; bool ready, multisig; if (m_wallet->key_on_device()) @@ -608,7 +678,8 @@ bool simple_wallet::print_seed(bool encrypted) fail_msg_writer() << tr("wallet is watch-only and has no seed"); return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + + SCOPED_WALLET_UNLOCK(); multisig = m_wallet->multisig(&ready); if (multisig) @@ -677,22 +748,36 @@ bool simple_wallet::seed_set_language(const std::vector<std::string> &args/* = s fail_msg_writer() << tr("wallet is watch-only and has no seed"); return true; } - if (!m_wallet->is_deterministic()) - { - fail_msg_writer() << tr("wallet is non-deterministic and has no seed"); - return true; - } - - const auto pwd_container = get_and_verify_password(); - if (pwd_container) + + epee::wipeable_string password; { - std::string mnemonic_language = get_mnemonic_language(); - if (mnemonic_language.empty()) + SCOPED_WALLET_UNLOCK(); + + if (!m_wallet->is_deterministic()) + { + fail_msg_writer() << tr("wallet is non-deterministic and has no seed"); return true; + } - m_wallet->set_seed_language(std::move(mnemonic_language)); - m_wallet->rewrite(m_wallet_file, pwd_container->password()); + // we need the password, even if ask-password is unset + if (!pwd_container) + { + pwd_container = get_and_verify_password(); + if (pwd_container == boost::none) + { + fail_msg_writer() << tr("Incorrect password"); + return true; + } + } + password = pwd_container->password(); } + + std::string mnemonic_language = get_mnemonic_language(); + if (mnemonic_language.empty()) + return true; + + m_wallet->set_seed_language(std::move(mnemonic_language)); + m_wallet->rewrite(m_wallet_file, password); return true; } @@ -713,8 +798,7 @@ bool simple_wallet::change_password(const std::vector<std::string> &args) try { - m_wallet->rewrite(m_wallet_file, pwd_container->password()); - m_wallet->store(); + m_wallet->change_password(m_wallet_file, orig_pwd_container->password(), pwd_container->password()); } catch (const tools::error::wallet_logic_error& e) { @@ -745,21 +829,24 @@ bool simple_wallet::print_fee_info(const std::vector<std::string> &args/* = std: fail_msg_writer() << tr("Cannot connect to daemon"); return true; } - const uint64_t per_kb_fee = m_wallet->get_per_kb_fee(); - const uint64_t typical_size_kb = 13; - message_writer() << (boost::format(tr("Current fee is %s %s per kB")) % print_money(per_kb_fee) % cryptonote::get_unit(cryptonote::get_default_decimal_point())).str(); + const bool per_byte = m_wallet->use_fork_rules(HF_VERSION_PER_BYTE_FEE); + const uint64_t base_fee = m_wallet->get_base_fee(); + const char *base = per_byte ? "byte" : "kB"; + const uint64_t typical_size = per_byte ? 2500 : 13; + const uint64_t size_granularity = per_byte ? 1 : 1024; + message_writer() << (boost::format(tr("Current fee is %s %s per %s")) % print_money(base_fee) % cryptonote::get_unit(cryptonote::get_default_decimal_point()) % base).str(); std::vector<uint64_t> fees; for (uint32_t priority = 1; priority <= 4; ++priority) { uint64_t mult = m_wallet->get_fee_multiplier(priority); - fees.push_back(per_kb_fee * typical_size_kb * mult); + fees.push_back(base_fee * typical_size * mult); } std::vector<std::pair<uint64_t, uint64_t>> blocks; try { - uint64_t base_size = typical_size_kb * 1024; - blocks = m_wallet->estimate_backlog(base_size, base_size + 1023, fees); + uint64_t base_size = typical_size * size_granularity; + blocks = m_wallet->estimate_backlog(base_size, base_size + size_granularity - 1, fees); } catch (const std::exception &e) { @@ -817,12 +904,7 @@ bool simple_wallet::prepare_multisig(const std::vector<std::string> &args) return true; } - const auto orig_pwd_container = get_and_verify_password(); - if(orig_pwd_container == boost::none) - { - fail_msg_writer() << tr("Your password is incorrect."); - return true; - } + SCOPED_WALLET_UNLOCK(); std::string multisig_info = m_wallet->get_multisig_info(); success_msg_writer() << multisig_info; @@ -855,13 +937,6 @@ bool simple_wallet::make_multisig(const std::vector<std::string> &args) return true; } - const auto orig_pwd_container = get_and_verify_password(); - if(orig_pwd_container == boost::none) - { - fail_msg_writer() << tr("Your original password was incorrect."); - return true; - } - if (args.size() < 2) { fail_msg_writer() << tr("usage: make_multisig <threshold> <multisiginfo1> [<multisiginfo2>...]"); @@ -876,6 +951,13 @@ bool simple_wallet::make_multisig(const std::vector<std::string> &args) return true; } + const auto orig_pwd_container = get_and_verify_password(); + if(orig_pwd_container == boost::none) + { + fail_msg_writer() << tr("Your original password was incorrect."); + return true; + } + LOCK_IDLE_SCOPE(); try @@ -917,6 +999,14 @@ bool simple_wallet::finalize_multisig(const std::vector<std::string> &args) fail_msg_writer() << tr("command not supported by HW wallet"); return true; } + + const auto pwd_container = get_and_verify_password(); + if(pwd_container == boost::none) + { + fail_msg_writer() << tr("Your original password was incorrect."); + return true; + } + if (!m_wallet->multisig(&ready)) { fail_msg_writer() << tr("This wallet is not multisig"); @@ -928,12 +1018,7 @@ bool simple_wallet::finalize_multisig(const std::vector<std::string> &args) return true; } - const auto orig_pwd_container = get_and_verify_password(); - if(orig_pwd_container == boost::none) - { - fail_msg_writer() << tr("Your original password was incorrect."); - return true; - } + LOCK_IDLE_SCOPE(); if (args.size() < 2) { @@ -943,7 +1028,7 @@ bool simple_wallet::finalize_multisig(const std::vector<std::string> &args) try { - if (!m_wallet->finalize_multisig(orig_pwd_container->password(), args)) + if (!m_wallet->finalize_multisig(pwd_container->password(), args)) { fail_msg_writer() << tr("Failed to finalize multisig"); return true; @@ -981,8 +1066,8 @@ bool simple_wallet::export_multisig(const std::vector<std::string> &args) fail_msg_writer() << tr("usage: export_multisig_info <filename>"); return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) - return true; + + SCOPED_WALLET_UNLOCK(); const std::string filename = args[0]; if (m_wallet->confirm_export_overwrite() && !check_file_overwrite(filename)) @@ -1033,8 +1118,8 @@ bool simple_wallet::import_multisig(const std::vector<std::string> &args) fail_msg_writer() << tr("usage: import_multisig_info <filename1> [<filename2>...] - one for each other participant"); return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) - return true; + + SCOPED_WALLET_UNLOCK(); std::vector<cryptonote::blobdata> info; for (size_t n = 0; n < args.size(); ++n) @@ -1050,11 +1135,11 @@ bool simple_wallet::import_multisig(const std::vector<std::string> &args) info.push_back(std::move(data)); } - LOCK_IDLE_SCOPE(); - // all read and parsed, actually import try { + m_in_manual_refresh.store(true, std::memory_order_relaxed); + epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){m_in_manual_refresh.store(false, std::memory_order_relaxed);}); size_t n_outputs = m_wallet->import_multisig(info); // Clear line "Height xxx of xxx" std::cout << "\r \r"; @@ -1065,7 +1150,7 @@ bool simple_wallet::import_multisig(const std::vector<std::string> &args) fail_msg_writer() << tr("Failed to import multisig info: ") << e.what(); return true; } - if (is_daemon_trusted()) + if (m_wallet->is_trusted_daemon()) { try { @@ -1112,7 +1197,8 @@ bool simple_wallet::sign_multisig(const std::vector<std::string> &args) fail_msg_writer() << tr("usage: sign_multisig <filename>"); return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + + SCOPED_WALLET_UNLOCK(); std::string filename = args[0]; std::vector<crypto::hash> txids; @@ -1185,7 +1271,8 @@ bool simple_wallet::submit_multisig(const std::vector<std::string> &args) fail_msg_writer() << tr("usage: submit_multisig <filename>"); return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + + SCOPED_WALLET_UNLOCK(); if (!try_connect_to_daemon()) return true; @@ -1217,7 +1304,7 @@ bool simple_wallet::submit_multisig(const std::vector<std::string> &args) } catch (const std::exception &e) { - handle_transfer_exception(std::current_exception(), is_daemon_trusted()); + handle_transfer_exception(std::current_exception(), m_wallet->is_trusted_daemon()); } catch (...) { @@ -1252,7 +1339,8 @@ bool simple_wallet::export_raw_multisig(const std::vector<std::string> &args) fail_msg_writer() << tr("usage: export_raw_multisig <filename>"); return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + + SCOPED_WALLET_UNLOCK(); std::string filename = args[0]; if (m_wallet->confirm_export_overwrite() && !check_file_overwrite(filename)) @@ -1754,6 +1842,8 @@ bool simple_wallet::set_default_ring_size(const std::vector<std::string> &args/* if (ring_size != 0 && ring_size != DEFAULT_MIX+1) message_writer() << tr("WARNING: this is a non default ring size, which may harm your privacy. Default is recommended."); + else if (ring_size == DEFAULT_MIX) + message_writer() << tr("WARNING: from v8, ring size will be fixed and this setting will be ignored."); const auto pwd_container = get_and_verify_password(); if (pwd_container) @@ -1777,12 +1867,12 @@ bool simple_wallet::set_default_ring_size(const std::vector<std::string> &args/* bool simple_wallet::set_default_priority(const std::vector<std::string> &args/* = std::vector<std::string>()*/) { - int priority = 0; + uint32_t priority = 0; try { if (strchr(args[1].c_str(), '-')) { - fail_msg_writer() << tr("priority must be 0, 1, 2, 3, or 4 "); + fail_msg_writer() << tr("priority must be either 0, 1, 2, 3, or 4, or one of: ") << join_priority_strings(", "); return true; } if (args[1] == "0") @@ -1791,11 +1881,23 @@ bool simple_wallet::set_default_priority(const std::vector<std::string> &args/* } else { - priority = boost::lexical_cast<int>(args[1]); - if (priority < 1 || priority > 4) + bool found = false; + for (size_t n = 0; n < allowed_priority_strings.size(); ++n) { - fail_msg_writer() << tr("priority must be 0, 1, 2, 3, or 4"); - return true; + if (allowed_priority_strings[n] == args[1]) + { + found = true; + priority = n; + } + } + if (!found) + { + priority = boost::lexical_cast<int>(args[1]); + if (priority < 1 || priority > 4) + { + fail_msg_writer() << tr("priority must be either 0, 1, 2, 3, or 4, or one of: ") << join_priority_strings(", "); + return true; + } } } @@ -1809,7 +1911,7 @@ bool simple_wallet::set_default_priority(const std::vector<std::string> &args/* } catch(const boost::bad_lexical_cast &) { - fail_msg_writer() << tr("priority must be 0, 1, 2, 3, or 4"); + fail_msg_writer() << tr("priority must be either 0, 1, 2, 3, or 4, or one of: ") << join_priority_strings(", "); return true; } catch(...) @@ -1874,6 +1976,14 @@ bool simple_wallet::set_ask_password(const std::vector<std::string> &args/* = st if (pwd_container) { parse_bool_and_use(args[1], [&](bool r) { + const bool cur_r = m_wallet->ask_password(); + if (!m_wallet->watch_only()) + { + if (cur_r && !r) + m_wallet->decrypt_keys(pwd_container->password()); + else if (!cur_r && r) + m_wallet->encrypt_keys(pwd_container->password()); + } m_wallet->ask_password(r); m_wallet->rewrite(m_wallet_file, pwd_container->password()); }); @@ -2161,15 +2271,15 @@ simple_wallet::simple_wallet() tr("Show the blockchain height.")); m_cmd_binder.set_handler("transfer_original", boost::bind(&simple_wallet::transfer, this, _1), - tr("transfer_original [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> <amount> [<payment_id>]"), - tr("Transfer <amount> to <address> using an older transaction building algorithm. If the parameter \"index=<N1>[,<N2>,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. <priority> is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability. Multiple payments can be made at once by adding <address_2> <amount_2> etcetera (before the payment ID, if it's included)")); + tr("transfer_original [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] (<URI> | <address> <amount>) [<payment_id>]"), + tr("Transfer <amount> to <address> using an older transaction building algorithm. If the parameter \"index=<N1>[,<N2>,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. <priority> is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability. Multiple payments can be made at once by adding URI_2 or <address_2> <amount_2> etcetera (before the payment ID, if it's included)")); m_cmd_binder.set_handler("transfer", boost::bind(&simple_wallet::transfer_new, this, _1), - tr("transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> <amount> [<payment_id>]"), - tr("Transfer <amount> to <address>. If the parameter \"index=<N1>[,<N2>,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. <priority> is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability. Multiple payments can be made at once by adding <address_2> <amount_2> etcetera (before the payment ID, if it's included)")); + tr("transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] (<URI> | <address> <amount>) [<payment_id>]"), + tr("Transfer <amount> to <address>. If the parameter \"index=<N1>[,<N2>,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. <priority> is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability. Multiple payments can be made at once by adding URI_2 or <address_2> <amount_2> etcetera (before the payment ID, if it's included)")); m_cmd_binder.set_handler("locked_transfer", boost::bind(&simple_wallet::locked_transfer, this, _1), - tr("locked_transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <addr> <amount> <lockblocks> [<payment_id>]"), - tr("Transfer <amount> to <address> and lock it for <lockblocks> (max. 1000000). If the parameter \"index=<N1>[,<N2>,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. <priority> is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability. Multiple payments can be made at once by adding <address_2> <amount_2> etcetera (before the payment ID, if it's included)")); + tr("locked_transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] (<URI> | <addr> <amount>) <lockblocks> [<payment_id>]"), + tr("Transfer <amount> to <address> and lock it for <lockblocks> (max. 1000000). If the parameter \"index=<N1>[,<N2>,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. <priority> is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability. Multiple payments can be made at once by adding URI_2 or <address_2> <amount_2> etcetera (before the payment ID, if it's included)")); m_cmd_binder.set_handler("locked_sweep_all", boost::bind(&simple_wallet::locked_sweep_all, this, _1), tr("locked_sweep_all [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> <lockblocks> [<payment_id>]"), @@ -2347,7 +2457,7 @@ simple_wallet::simple_wallet() tr("Show the unspent outputs of a specified address within an optional amount range.")); m_cmd_binder.set_handler("rescan_bc", boost::bind(&simple_wallet::rescan_blockchain, this, _1), - tr("Rescan the blockchain from scratch.")); + tr("Rescan the blockchain from scratch, losing any information which can not be recovered from the blockchain itself.")); m_cmd_binder.set_handler("set_tx_note", boost::bind(&simple_wallet::set_tx_note, this, _1), tr("set_tx_note <txid> [free text note]"), @@ -2476,6 +2586,10 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args) std::string seed_language = m_wallet->get_seed_language(); if (m_use_english_language_names) seed_language = crypto::ElectrumWords::get_english_name_for(seed_language); + std::string priority_string = "invalid"; + uint32_t priority = m_wallet->get_default_priority(); + if (priority < allowed_priority_strings.size()) + priority_string = allowed_priority_strings[priority]; success_msg_writer() << "seed = " << seed_language; success_msg_writer() << "always-confirm-transfers = " << m_wallet->always_confirm_transfers(); success_msg_writer() << "print-ring-members = " << m_wallet->print_ring_members(); @@ -2483,7 +2597,7 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args) success_msg_writer() << "default-ring-size = " << (m_wallet->default_mixin() ? m_wallet->default_mixin() + 1 : 0); success_msg_writer() << "auto-refresh = " << m_wallet->auto_refresh(); success_msg_writer() << "refresh-type = " << get_refresh_type_name(m_wallet->get_refresh_type()); - success_msg_writer() << "priority = " << m_wallet->get_default_priority(); + success_msg_writer() << "priority = " << priority<< " (" << priority_string << ")"; success_msg_writer() << "confirm-missing-payment-id = " << m_wallet->confirm_missing_payment_id(); success_msg_writer() << "ask-password = " << m_wallet->ask_password(); success_msg_writer() << "unit = " << cryptonote::get_unit(cryptonote::get_default_decimal_point()); @@ -2539,7 +2653,7 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args) CHECK_SIMPLE_VARIABLE("default-ring-size", set_default_ring_size, tr("integer >= ") << MIN_RING_SIZE); CHECK_SIMPLE_VARIABLE("auto-refresh", set_auto_refresh, tr("0 or 1")); CHECK_SIMPLE_VARIABLE("refresh-type", set_refresh_type, tr("full (slowest, no assumptions); optimize-coinbase (fast, assumes the whole coinbase is paid to a single address); no-coinbase (fastest, assumes we receive no coinbase transaction), default (same as optimize-coinbase)")); - CHECK_SIMPLE_VARIABLE("priority", set_default_priority, tr("0, 1, 2, 3, or 4")); + CHECK_SIMPLE_VARIABLE("priority", set_default_priority, tr("0, 1, 2, 3, or 4, or one of ") << join_priority_strings(", ")); CHECK_SIMPLE_VARIABLE("confirm-missing-payment-id", set_confirm_missing_payment_id, tr("0 or 1")); CHECK_SIMPLE_VARIABLE("ask-password", set_ask_password, tr("0 or 1")); CHECK_SIMPLE_VARIABLE("unit", set_unit, tr("monero, millinero, micronero, nanonero, piconero")); @@ -2679,28 +2793,45 @@ bool simple_wallet::ask_wallet_create_if_needed() * \brief Prints the seed with a nice message * \param seed seed to print */ -void simple_wallet::print_seed(std::string seed) +void simple_wallet::print_seed(const epee::wipeable_string &seed) { success_msg_writer(true) << "\n" << tr("NOTE: the following 25 words can be used to recover access to your wallet. " "Write them down and store them somewhere safe and secure. Please do not store them in " "your email or on file storage services outside of your immediate control.\n"); - boost::replace_nth(seed, " ", 15, "\n"); - boost::replace_nth(seed, " ", 7, "\n"); // don't log - std::cout << seed << std::endl; + int space_index = 0; + size_t len = seed.size(); + for (const char *ptr = seed.data(); len--; ++ptr) + { + if (*ptr == ' ') + { + if (space_index == 15 || space_index == 7) + putchar('\n'); + else + putchar(*ptr); + ++space_index; + } + else + putchar(*ptr); + } + putchar('\n'); + fflush(stdout); } //---------------------------------------------------------------------------------------------------- -static bool might_be_partial_seed(std::string words) +static bool might_be_partial_seed(const epee::wipeable_string &words) { - std::vector<std::string> seed; + std::vector<epee::wipeable_string> seed; - boost::algorithm::trim(words); - boost::split(seed, words, boost::is_any_of(" "), boost::token_compress_on); + words.split(seed); return seed.size() < 24; } //---------------------------------------------------------------------------------------------------- bool simple_wallet::init(const boost::program_options::variables_map& vm) { + epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){ + m_electrum_seed.wipe(); + }); + const bool testnet = tools::wallet2::has_testnet_option(vm); const bool stagenet = tools::wallet2::has_stagenet_option(vm); if (testnet && stagenet) @@ -2710,7 +2841,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) } const network_type nettype = testnet ? TESTNET : stagenet ? STAGENET : MAINNET; - std::string multisig_keys; + epee::wipeable_string multisig_keys; if (!handle_command_line(vm)) return false; @@ -2752,8 +2883,8 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) { if (m_restore_multisig_wallet) { - const char *prompt = "Specify multisig seed: "; - m_electrum_seed = input_line(prompt); + const char *prompt = "Specify multisig seed"; + m_electrum_seed = input_secure_line(prompt); if (std::cin.eof()) return false; if (m_electrum_seed.empty()) @@ -2767,8 +2898,8 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) m_electrum_seed = ""; do { - const char *prompt = m_electrum_seed.empty() ? "Specify Electrum seed: " : "Electrum seed continued: "; - std::string electrum_seed = input_line(prompt); + const char *prompt = m_electrum_seed.empty() ? "Specify Electrum seed" : "Electrum seed continued"; + epee::wipeable_string electrum_seed = input_secure_line(prompt); if (std::cin.eof()) return false; if (electrum_seed.empty()) @@ -2776,18 +2907,21 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) fail_msg_writer() << tr("specify a recovery parameter with the --electrum-seed=\"words list here\""); return false; } - m_electrum_seed += electrum_seed + " "; + m_electrum_seed += electrum_seed; + m_electrum_seed += ' '; } while (might_be_partial_seed(m_electrum_seed)); } } if (m_restore_multisig_wallet) { - if (!epee::string_tools::parse_hexstr_to_binbuff(m_electrum_seed, multisig_keys)) + const boost::optional<epee::wipeable_string> parsed = m_electrum_seed.parse_hexstr(); + if (!parsed) { fail_msg_writer() << tr("Multisig seed failed verification"); return false; } + multisig_keys = *parsed; } else { @@ -2809,7 +2943,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) crypto::secret_key key; crypto::cn_slow_hash(seed_pass.data(), seed_pass.size(), (crypto::hash&)key); sc_reduce32((unsigned char*)key.data); - multisig_keys = m_wallet->decrypt(multisig_keys, key, true); + multisig_keys = m_wallet->decrypt<epee::wipeable_string>(std::string(multisig_keys.data(), multisig_keys.size()), key, true); } else m_recovery_key = cryptonote::decrypt_key(m_recovery_key, seed_pass); @@ -3112,7 +3246,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) m_wallet_file = m_generate_from_json; try { - m_wallet = tools::wallet2::make_from_json(vm, m_wallet_file, password_prompter); + m_wallet = tools::wallet2::make_from_json(vm, false, m_wallet_file, password_prompter); } catch (const std::exception &e) { @@ -3260,22 +3394,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) return false; } - // set --trusted-daemon if local and not overridden - if (!m_trusted_daemon) - { - try - { - m_trusted_daemon = false; - if (tools::is_local_address(m_wallet->get_daemon_address())) - { - MINFO(tr("Daemon is local, assuming trusted")); - m_trusted_daemon = true; - } - } - catch (const std::exception &e) { } - } - - if (!is_daemon_trusted()) + if (!m_wallet->is_trusted_daemon()) message_writer() << (boost::format(tr("Warning: using an untrusted daemon at %s, privacy will be lessened")) % m_wallet->get_daemon_address()).str(); if (m_wallet->get_ring_database().empty()) @@ -3309,10 +3428,6 @@ bool simple_wallet::handle_command_line(const boost::program_options::variables_ m_restore_deterministic_wallet = command_line::get_arg(vm, arg_restore_deterministic_wallet); m_restore_multisig_wallet = command_line::get_arg(vm, arg_restore_multisig_wallet); m_non_deterministic = command_line::get_arg(vm, arg_non_deterministic); - if (!command_line::is_arg_defaulted(vm, arg_trusted_daemon) || !command_line::is_arg_defaulted(vm, arg_untrusted_daemon)) - m_trusted_daemon = command_line::get_arg(vm, arg_trusted_daemon) && !command_line::get_arg(vm, arg_untrusted_daemon); - if (!command_line::is_arg_defaulted(vm, arg_trusted_daemon) && !command_line::is_arg_defaulted(vm, arg_untrusted_daemon)) - message_writer() << tr("--trusted-daemon and --untrusted-daemon are both seen, assuming untrusted"); m_allow_mismatched_daemon_version = command_line::get_arg(vm, arg_allow_mismatched_daemon_version); m_restore_height = command_line::get_arg(vm, arg_restore_height); m_do_not_relay = command_line::get_arg(vm, arg_do_not_relay); @@ -3414,7 +3529,7 @@ boost::optional<tools::password_container> simple_wallet::get_and_verify_passwor boost::optional<epee::wipeable_string> simple_wallet::new_wallet(const boost::program_options::variables_map& vm, const crypto::secret_key& recovery_key, bool recover, bool two_random, const std::string &old_language) { - auto rc = tools::wallet2::make_new(vm, password_prompter); + auto rc = tools::wallet2::make_new(vm, false, password_prompter); m_wallet = std::move(rc.first); if (!m_wallet) { @@ -3469,7 +3584,10 @@ boost::optional<epee::wipeable_string> simple_wallet::new_wallet(const boost::pr recovery_val = m_wallet->generate(m_wallet_file, std::move(rc.second).password(), recovery_key, recover, two_random, create_address_file); message_writer(console_color_white, true) << tr("Generated new wallet: ") << m_wallet->get_account().get_public_address_str(m_wallet->nettype()); - std::cout << tr("View key: ") << string_tools::pod_to_hex(m_wallet->get_account().get_keys().m_view_secret_key) << ENDL; + PAUSE_READLINE(); + std::cout << tr("View key: "); + print_secret_key(m_wallet->get_account().get_keys().m_view_secret_key); + putchar('\n'); } catch (const std::exception& e) { @@ -3478,7 +3596,7 @@ boost::optional<epee::wipeable_string> simple_wallet::new_wallet(const boost::pr } // convert rng value to electrum-style word list - std::string electrum_words; + epee::wipeable_string electrum_words; crypto::ElectrumWords::bytes_to_words(recovery_val, electrum_words, mnemonic_language); @@ -3506,7 +3624,7 @@ boost::optional<epee::wipeable_string> simple_wallet::new_wallet(const boost::pr const cryptonote::account_public_address& address, const boost::optional<crypto::secret_key>& spendkey, const crypto::secret_key& viewkey) { - auto rc = tools::wallet2::make_new(vm, password_prompter); + auto rc = tools::wallet2::make_new(vm, false, password_prompter); m_wallet = std::move(rc.first); if (!m_wallet) { @@ -3552,7 +3670,7 @@ boost::optional<epee::wipeable_string> simple_wallet::new_wallet(const boost::pr //---------------------------------------------------------------------------------------------------- boost::optional<epee::wipeable_string> simple_wallet::new_wallet(const boost::program_options::variables_map& vm, const std::string &device_name) { - auto rc = tools::wallet2::make_new(vm, password_prompter); + auto rc = tools::wallet2::make_new(vm, false, password_prompter); m_wallet = std::move(rc.first); if (!m_wallet) { @@ -3586,9 +3704,9 @@ boost::optional<epee::wipeable_string> simple_wallet::new_wallet(const boost::pr } //---------------------------------------------------------------------------------------------------- boost::optional<epee::wipeable_string> simple_wallet::new_wallet(const boost::program_options::variables_map& vm, - const std::string &multisig_keys, const std::string &old_language) + const epee::wipeable_string &multisig_keys, const std::string &old_language) { - auto rc = tools::wallet2::make_new(vm, password_prompter); + auto rc = tools::wallet2::make_new(vm, false, password_prompter); m_wallet = std::move(rc.first); if (!m_wallet) { @@ -3659,7 +3777,7 @@ bool simple_wallet::open_wallet(const boost::program_options::variables_map& vm) epee::wipeable_string password; try { - auto rc = tools::wallet2::make_from_file(vm, m_wallet_file, password_prompter); + auto rc = tools::wallet2::make_from_file(vm, false, m_wallet_file, password_prompter); m_wallet = std::move(rc.first); password = std::move(std::move(rc.second).password()); if (!m_wallet) @@ -3686,7 +3804,12 @@ bool simple_wallet::open_wallet(const boost::program_options::variables_map& vm) // NOTE: this is_deprecated() refers to the wallet file format before becoming JSON. It does not refer to the "old english" seed words form of "deprecated" used elsewhere. if (m_wallet->is_deprecated()) { - if (m_wallet->is_deterministic()) + bool is_deterministic; + { + SCOPED_WALLET_UNLOCK(); + is_deterministic = m_wallet->is_deterministic(); + } + if (is_deterministic) { message_writer(console_color_green, false) << "\n" << tr("You had been using " "a deprecated version of the wallet. Please proceed to upgrade your wallet.\n"); @@ -3697,7 +3820,7 @@ bool simple_wallet::open_wallet(const boost::program_options::variables_map& vm) m_wallet->rewrite(m_wallet_file, password); // Display the seed - std::string seed; + epee::wipeable_string seed; m_wallet->get_seed(seed); print_seed(seed); } @@ -3816,7 +3939,7 @@ bool simple_wallet::save_watch_only(const std::vector<std::string> &args/* = std //---------------------------------------------------------------------------------------------------- bool simple_wallet::start_mining(const std::vector<std::string>& args) { - if (!is_daemon_trusted()) + if (!m_wallet->is_trusted_daemon()) { fail_msg_writer() << tr("this command requires a trusted daemon. Enable with --trusted-daemon"); return true; @@ -3924,34 +4047,34 @@ bool simple_wallet::set_daemon(const std::vector<std::string>& args) daemon_url = args[0]; } LOCK_IDLE_SCOPE(); - m_wallet->init(daemon_url); + m_wallet->init(false, daemon_url); if (args.size() == 2) { if (args[1] == "trusted") - m_trusted_daemon = true; + m_wallet->set_trusted_daemon(true); else if (args[1] == "untrusted") - m_trusted_daemon = false; + m_wallet->set_trusted_daemon(false); else { fail_msg_writer() << tr("Expected trusted or untrusted, got ") << args[1] << ": assuming untrusted"; - m_trusted_daemon = false; + m_wallet->set_trusted_daemon(false); } } else { - m_trusted_daemon = false; + m_wallet->set_trusted_daemon(false); try { if (tools::is_local_address(m_wallet->get_daemon_address())) { MINFO(tr("Daemon is local, assuming trusted")); - m_trusted_daemon = true; + m_wallet->set_trusted_daemon(true); } } catch (const std::exception &e) { } } - success_msg_writer() << boost::format("Daemon set to %s, %s") % daemon_url % (*m_trusted_daemon ? tr("trusted") : tr("untrusted")); + success_msg_writer() << boost::format("Daemon set to %s, %s") % daemon_url % (m_wallet->is_trusted_daemon() ? tr("trusted") : tr("untrusted")); } else { fail_msg_writer() << tr("This does not seem to be a valid daemon URL."); } @@ -4038,6 +4161,32 @@ void simple_wallet::on_skip_transaction(uint64_t height, const crypto::hash &txi { } //---------------------------------------------------------------------------------------------------- +boost::optional<epee::wipeable_string> simple_wallet::on_get_password(const char *reason) +{ + // can't ask for password from a background thread + if (!m_in_manual_refresh.load(std::memory_order_relaxed)) + { + message_writer(console_color_red, false) << tr("Password needed - use the refresh command"); + m_cmd_binder.print_prompt(); + return boost::none; + } + +#ifdef HAVE_READLINE + rdln::suspend_readline pause_readline; +#endif + std::string msg = tr("Enter password"); + if (reason && *reason) + msg += std::string(" (") + reason + ")"; + auto pwd_container = tools::password_container::prompt(false, msg.c_str()); + if (!pwd_container) + { + MERROR("Failed to read password"); + return boost::none; + } + + return pwd_container->password(); +} +//---------------------------------------------------------------------------------------------------- bool simple_wallet::refresh_main(uint64_t start_height, bool reset, bool is_init) { if (!try_connect_to_daemon(is_init)) @@ -4061,7 +4210,7 @@ bool simple_wallet::refresh_main(uint64_t start_height, bool reset, bool is_init { m_in_manual_refresh.store(true, std::memory_order_relaxed); epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){m_in_manual_refresh.store(false, std::memory_order_relaxed);}); - m_wallet->refresh(is_daemon_trusted(), start_height, fetched_blocks); + m_wallet->refresh(m_wallet->is_trusted_daemon(), start_height, fetched_blocks); ok = true; // Clear line "Height xxx of xxx" std::cout << "\r \r"; @@ -4352,7 +4501,7 @@ bool simple_wallet::show_blockchain_height(const std::vector<std::string>& args) //---------------------------------------------------------------------------------------------------- bool simple_wallet::rescan_spent(const std::vector<std::string> &args) { - if (!is_daemon_trusted()) + if (!m_wallet->is_trusted_daemon()) { fail_msg_writer() << tr("this command requires a trusted daemon. Enable with --trusted-daemon"); return true; @@ -4513,11 +4662,10 @@ bool simple_wallet::print_ring_members(const std::vector<tools::wallet2::pending bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::string> &args_) { // "transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> <amount> [<payment_id>]" - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } if (!try_connect_to_daemon()) return true; - LOCK_IDLE_SCOPE(); + SCOPED_WALLET_UNLOCK(); std::vector<std::string> local_args = args_; @@ -4561,8 +4709,13 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri fail_msg_writer() << (boost::format(tr("ring size %u is too small, minimum is %u")) % (fake_outs_count+1) % (adjusted_fake_outs_count+1)).str(); return true; } + if (adjusted_fake_outs_count < fake_outs_count) + { + fail_msg_writer() << (boost::format(tr("ring size %u is too large, maximum is %u")) % (fake_outs_count+1) % (adjusted_fake_outs_count+1)).str(); + return true; + } - const size_t min_args = (transfer_type == TransferLocked) ? 3 : 2; + const size_t min_args = (transfer_type == TransferLocked) ? 2 : 1; if(local_args.size() < min_args) { fail_msg_writer() << tr("wrong number of arguments"); @@ -4571,39 +4724,38 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri std::vector<uint8_t> extra; bool payment_id_seen = false; - bool expect_even = (transfer_type == TransferLocked); - if ((expect_even ? 0 : 1) == local_args.size() % 2) + if (!local_args.empty()) { std::string payment_id_str = local_args.back(); - local_args.pop_back(); - crypto::hash payment_id; - bool r = tools::wallet2::parse_long_payment_id(payment_id_str, payment_id); - if(r) + bool r = true; + if (tools::wallet2::parse_long_payment_id(payment_id_str, payment_id)) { std::string extra_nonce; set_payment_id_to_tx_extra_nonce(extra_nonce, payment_id); r = add_extra_nonce_to_tx_extra(extra, extra_nonce); + local_args.pop_back(); + payment_id_seen = true; + message_writer() << tr("Unencrypted payment IDs are bad for privacy: ask the recipient to use subaddresses instead"); } else { crypto::hash8 payment_id8; - r = tools::wallet2::parse_short_payment_id(payment_id_str, payment_id8); - if(r) + if (tools::wallet2::parse_short_payment_id(payment_id_str, payment_id8)) { std::string extra_nonce; set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, payment_id8); r = add_extra_nonce_to_tx_extra(extra, extra_nonce); + local_args.pop_back(); + payment_id_seen = true; } } if(!r) { - fail_msg_writer() << tr("payment id has invalid format, expected 16 or 64 character hex string: ") << payment_id_str; + fail_msg_writer() << tr("payment id failed to encode"); return true; } - payment_id_seen = true; - message_writer() << tr("Unencrypted payment IDs are bad for privacy: ask the recipient to use subaddresses instead"); } uint64_t locked_blocks = 0; @@ -4628,11 +4780,54 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri vector<cryptonote::tx_destination_entry> dsts; size_t num_subaddresses = 0; - for (size_t i = 0; i < local_args.size(); i += 2) + for (size_t i = 0; i < local_args.size(); ) { - cryptonote::address_parse_info info; cryptonote::tx_destination_entry de; - if (!cryptonote::get_account_address_from_str_or_url(info, m_wallet->nettype(), local_args[i], oa_prompter)) + cryptonote::address_parse_info info; + bool r = true; + + // check for a URI + std::string address_uri, payment_id_uri, tx_description, recipient_name, error; + std::vector<std::string> unknown_parameters; + uint64_t amount = 0; + bool has_uri = m_wallet->parse_uri(local_args[i], address_uri, payment_id_uri, amount, tx_description, recipient_name, unknown_parameters, error); + if (has_uri) + { + r = cryptonote::get_account_address_from_str_or_url(info, m_wallet->nettype(), address_uri, oa_prompter); + if (payment_id_uri.size() == 16) + { + if (!tools::wallet2::parse_short_payment_id(payment_id_uri, info.payment_id)) + { + fail_msg_writer() << tr("failed to parse short payment ID from URI"); + return true; + } + info.has_payment_id = true; + } + de.amount = amount; + ++i; + } + else if (i + 1 < local_args.size()) + { + r = cryptonote::get_account_address_from_str_or_url(info, m_wallet->nettype(), local_args[i], oa_prompter); + bool ok = cryptonote::parse_amount(de.amount, local_args[i + 1]); + if(!ok || 0 == de.amount) + { + fail_msg_writer() << tr("amount is wrong: ") << local_args[i] << ' ' << local_args[i + 1] << + ", " << tr("expected number from 0 to ") << print_money(std::numeric_limits<uint64_t>::max()); + return true; + } + i += 2; + } + else + { + if (boost::starts_with(local_args[i], "monero:")) + fail_msg_writer() << tr("Invalid last argument: ") << local_args.back() << ": " << error; + else + fail_msg_writer() << tr("Invalid last argument: ") << local_args.back(); + return true; + } + + if (!r) { fail_msg_writer() << tr("failed to parse address"); return true; @@ -4641,16 +4836,30 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri de.is_subaddress = info.is_subaddress; num_subaddresses += info.is_subaddress; - if (info.has_payment_id) + if (info.has_payment_id || !payment_id_uri.empty()) { if (payment_id_seen) { - fail_msg_writer() << tr("a single transaction cannot use more than one payment id: ") << local_args[i]; + fail_msg_writer() << tr("a single transaction cannot use more than one payment id"); return true; } + crypto::hash payment_id; std::string extra_nonce; - set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, info.payment_id); + if (info.has_payment_id) + { + set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, info.payment_id); + } + else if (tools::wallet2::parse_payment_id(payment_id_uri, payment_id)) + { + set_payment_id_to_tx_extra_nonce(extra_nonce, payment_id); + message_writer() << tr("Unencrypted payment IDs are bad for privacy: ask the recipient to use subaddresses instead"); + } + else + { + fail_msg_writer() << tr("failed to parse payment id, though it was detected"); + return true; + } bool r = add_extra_nonce_to_tx_extra(extra, extra_nonce); if(!r) { @@ -4660,14 +4869,6 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri payment_id_seen = true; } - bool ok = cryptonote::parse_amount(de.amount, local_args[i + 1]); - if(!ok || 0 == de.amount) - { - fail_msg_writer() << tr("amount is wrong: ") << local_args[i] << ' ' << local_args[i + 1] << - ", " << tr("expected number from 0 to ") << print_money(std::numeric_limits<uint64_t>::max()); - return true; - } - dsts.push_back(de); } @@ -4701,16 +4902,16 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri return true; } unlock_block = bc_height + locked_blocks; - ptx_vector = m_wallet->create_transactions_2(dsts, fake_outs_count, unlock_block /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices, is_daemon_trusted()); + ptx_vector = m_wallet->create_transactions_2(dsts, fake_outs_count, unlock_block /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices); break; case TransferNew: - ptx_vector = m_wallet->create_transactions_2(dsts, fake_outs_count, 0 /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices, is_daemon_trusted()); + ptx_vector = m_wallet->create_transactions_2(dsts, fake_outs_count, 0 /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices); break; default: LOG_ERROR("Unknown transfer method, using original"); /* FALLTHRU */ case TransferOriginal: - ptx_vector = m_wallet->create_transactions(dsts, fake_outs_count, 0 /* unlock_time */, priority, extra, is_daemon_trusted()); + ptx_vector = m_wallet->create_transactions(dsts, fake_outs_count, 0 /* unlock_time */, priority, extra); break; } @@ -4886,7 +5087,7 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri } catch (const std::exception &e) { - handle_transfer_exception(std::current_exception(), is_daemon_trusted()); + handle_transfer_exception(std::current_exception(), m_wallet->is_trusted_daemon()); } catch (...) { @@ -4920,15 +5121,15 @@ bool simple_wallet::locked_sweep_all(const std::vector<std::string> &args_) bool simple_wallet::sweep_unmixable(const std::vector<std::string> &args_) { - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } if (!try_connect_to_daemon()) return true; - LOCK_IDLE_SCOPE(); + SCOPED_WALLET_UNLOCK(); + try { // figure out what tx will be necessary - auto ptx_vector = m_wallet->create_unmixable_sweep_transactions(is_daemon_trusted()); + auto ptx_vector = m_wallet->create_unmixable_sweep_transactions(); if (ptx_vector.empty()) { @@ -5007,13 +5208,13 @@ bool simple_wallet::sweep_unmixable(const std::vector<std::string> &args_) { try { - m_wallet->discard_unmixable_outputs(is_daemon_trusted()); + m_wallet->discard_unmixable_outputs(); } catch (...) {} } } catch (const std::exception &e) { - handle_transfer_exception(std::current_exception(), is_daemon_trusted()); + handle_transfer_exception(std::current_exception(), m_wallet->is_trusted_daemon()); } catch (...) { @@ -5037,7 +5238,6 @@ bool simple_wallet::sweep_main(uint64_t below, bool locked, const std::vector<st return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } if (!try_connect_to_daemon()) return true; @@ -5086,6 +5286,11 @@ bool simple_wallet::sweep_main(uint64_t below, bool locked, const std::vector<st fail_msg_writer() << (boost::format(tr("ring size %u is too small, minimum is %u")) % (fake_outs_count+1) % (adjusted_fake_outs_count+1)).str(); return true; } + if (adjusted_fake_outs_count < fake_outs_count) + { + fail_msg_writer() << (boost::format(tr("ring size %u is too large, maximum is %u")) % (fake_outs_count+1) % (adjusted_fake_outs_count+1)).str(); + return true; + } uint64_t unlock_block = 0; if (locked) { @@ -5201,12 +5406,12 @@ bool simple_wallet::sweep_main(uint64_t below, bool locked, const std::vector<st } } - LOCK_IDLE_SCOPE(); + SCOPED_WALLET_UNLOCK(); try { // figure out what tx will be necessary - auto ptx_vector = m_wallet->create_transactions_all(below, info.address, info.is_subaddress, fake_outs_count, unlock_block /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices, is_daemon_trusted()); + auto ptx_vector = m_wallet->create_transactions_all(below, info.address, info.is_subaddress, fake_outs_count, unlock_block /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices); if (ptx_vector.empty()) { @@ -5290,7 +5495,7 @@ bool simple_wallet::sweep_main(uint64_t below, bool locked, const std::vector<st } catch (const std::exception& e) { - handle_transfer_exception(std::current_exception(), is_daemon_trusted()); + handle_transfer_exception(std::current_exception(), m_wallet->is_trusted_daemon()); } catch (...) { @@ -5303,7 +5508,7 @@ bool simple_wallet::sweep_main(uint64_t below, bool locked, const std::vector<st //---------------------------------------------------------------------------------------------------- bool simple_wallet::sweep_single(const std::vector<std::string> &args_) { - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + SCOPED_WALLET_UNLOCK(); if (!try_connect_to_daemon()) return true; @@ -5324,12 +5529,28 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) if (fake_outs_count == 0) fake_outs_count = DEFAULT_MIX; } + else if (ring_size == 0) + { + fail_msg_writer() << tr("Ring size must not be 0"); + return true; + } else { fake_outs_count = ring_size - 1; local_args.erase(local_args.begin()); } } + uint64_t adjusted_fake_outs_count = m_wallet->adjust_mixin(fake_outs_count); + if (adjusted_fake_outs_count > fake_outs_count) + { + fail_msg_writer() << (boost::format(tr("ring size %u is too small, minimum is %u")) % (fake_outs_count+1) % (adjusted_fake_outs_count+1)).str(); + return true; + } + if (adjusted_fake_outs_count < fake_outs_count) + { + fail_msg_writer() << (boost::format(tr("ring size %u is too large, maximum is %u")) % (fake_outs_count+1) % (adjusted_fake_outs_count+1)).str(); + return true; + } std::vector<uint8_t> extra; bool payment_id_seen = false; @@ -5419,7 +5640,7 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) try { // figure out what tx will be necessary - auto ptx_vector = m_wallet->create_transactions_single(ki, info.address, info.is_subaddress, fake_outs_count, 0 /* unlock_time */, priority, extra, is_daemon_trusted()); + auto ptx_vector = m_wallet->create_transactions_single(ki, info.address, info.is_subaddress, fake_outs_count, 0 /* unlock_time */, priority, extra); if (ptx_vector.empty()) { @@ -5489,7 +5710,7 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) } catch (const std::exception& e) { - handle_transfer_exception(std::current_exception(), is_daemon_trusted()); + handle_transfer_exception(std::current_exception(), m_wallet->is_trusted_daemon()); } catch (...) { @@ -5729,7 +5950,8 @@ bool simple_wallet::sign_transfer(const std::vector<std::string> &args_) fail_msg_writer() << tr("usage: sign_transfer [export_raw]"); return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + + SCOPED_WALLET_UNLOCK(); const bool export_raw = args_.size() == 1; std::vector<tools::wallet2::pending_tx> ptx; @@ -5794,7 +6016,7 @@ bool simple_wallet::submit_transfer(const std::vector<std::string> &args_) } catch (const std::exception& e) { - handle_transfer_exception(std::current_exception(), is_daemon_trusted()); + handle_transfer_exception(std::current_exception(), m_wallet->is_trusted_daemon()); } catch (...) { @@ -5818,7 +6040,6 @@ bool simple_wallet::get_tx_key(const std::vector<std::string> &args_) fail_msg_writer() << tr("usage: get_tx_key <txid>"); return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } crypto::hash txid; if (!epee::string_tools::hex_to_pod(local_args[0], txid)) @@ -5827,7 +6048,7 @@ bool simple_wallet::get_tx_key(const std::vector<std::string> &args_) return true; } - LOCK_IDLE_SCOPE(); + SCOPED_WALLET_UNLOCK(); crypto::secret_key tx_key; std::vector<crypto::secret_key> additional_tx_keys; @@ -5932,7 +6153,7 @@ bool simple_wallet::get_tx_proof(const std::vector<std::string> &args) return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + SCOPED_WALLET_UNLOCK(); try { @@ -6147,7 +6368,7 @@ bool simple_wallet::get_spend_proof(const std::vector<std::string> &args) return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + SCOPED_WALLET_UNLOCK(); try { @@ -6242,9 +6463,7 @@ bool simple_wallet::get_reserve_proof(const std::vector<std::string> &args) return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } - - LOCK_IDLE_SCOPE(); + SCOPED_WALLET_UNLOCK(); try { @@ -6345,7 +6564,7 @@ static std::string get_human_readable_timespan(std::chrono::seconds seconds) if (ts < 3600 * 24 * 30.5) return std::to_string((uint64_t)(ts / (3600 * 24))) + tr(" days"); if (ts < 3600 * 24 * 365.25) - return std::to_string((uint64_t)(ts / (3600 * 24 * 365.25))) + tr(" months"); + return std::to_string((uint64_t)(ts / (3600 * 24 * 30.5))) + tr(" months"); return tr("a long time"); } //---------------------------------------------------------------------------------------------------- @@ -6497,6 +6716,9 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_) if (pool) { try { + m_in_manual_refresh.store(true, std::memory_order_relaxed); + epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){m_in_manual_refresh.store(false, std::memory_order_relaxed);}); + m_wallet->update_pool_state(); std::list<std::pair<crypto::hash, tools::wallet2::pool_payment_details>> payments; m_wallet->get_unconfirmed_payments(payments, m_current_subaddress_account, subaddr_indices); @@ -6680,6 +6902,14 @@ bool simple_wallet::unspent_outputs(const std::vector<std::string> &args_) //---------------------------------------------------------------------------------------------------- bool simple_wallet::rescan_blockchain(const std::vector<std::string> &args_) { + message_writer() << tr("Warning: this will lose any information which can not be recovered from the blockchain."); + message_writer() << tr("This includes destination addresses, tx secret keys, tx notes, etc"); + std::string confirm = input_line(tr("Rescan anyway ? (Y/Yes/N/No): ")); + if(!std::cin.eof()) + { + if (!command_line::is_yes(confirm)) + return true; + } return refresh_main(0, true); } //---------------------------------------------------------------------------------------------------- @@ -6699,7 +6929,7 @@ void simple_wallet::wallet_idle_thread() { uint64_t fetched_blocks; if (try_connect_to_daemon(true)) - m_wallet->refresh(is_daemon_trusted(), 0, fetched_blocks); + m_wallet->refresh(m_wallet->is_trusted_daemon(), 0, fetched_blocks); } catch(...) {} m_auto_refresh_refreshing = false; @@ -7345,7 +7575,8 @@ bool simple_wallet::sign(const std::vector<std::string> &args) fail_msg_writer() << tr("This wallet is multisig and cannot sign"); return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + + SCOPED_WALLET_UNLOCK(); std::string filename = args[0]; std::string data; bool r = epee::file_io_utils::load_file_to_string(filename, data); @@ -7414,14 +7645,14 @@ bool simple_wallet::export_key_images(const std::vector<std::string> &args) fail_msg_writer() << tr("wallet is watch-only and cannot export key images"); return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + + SCOPED_WALLET_UNLOCK(); std::string filename = args[0]; if (m_wallet->confirm_export_overwrite() && !check_file_overwrite(filename)) return true; try { - LOCK_IDLE_SCOPE(); if (!m_wallet->export_key_images(filename)) { fail_msg_writer() << tr("failed to save file ") << filename; @@ -7446,7 +7677,7 @@ bool simple_wallet::import_key_images(const std::vector<std::string> &args) fail_msg_writer() << tr("command not supported by HW wallet"); return true; } - if (!is_daemon_trusted()) + if (!m_wallet->is_trusted_daemon()) { fail_msg_writer() << tr("this command requires a trusted daemon. Enable with --trusted-daemon"); return true; @@ -7493,12 +7724,12 @@ bool simple_wallet::export_outputs(const std::vector<std::string> &args) fail_msg_writer() << tr("usage: export_outputs <filename>"); return true; } - if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } + + SCOPED_WALLET_UNLOCK(); std::string filename = args[0]; if (m_wallet->confirm_export_overwrite() && !check_file_overwrite(filename)) return true; - LOCK_IDLE_SCOPE(); try { std::string data = m_wallet->export_outputs_to_str(); @@ -7544,7 +7775,7 @@ bool simple_wallet::import_outputs(const std::vector<std::string> &args) try { - LOCK_IDLE_SCOPE(); + SCOPED_WALLET_UNLOCK(); size_t n_outputs = m_wallet->import_outputs_from_str(data); success_msg_writer() << boost::lexical_cast<std::string>(n_outputs) << " outputs imported"; } @@ -7777,8 +8008,6 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_params, arg_restore_multisig_wallet ); command_line::add_arg(desc_params, arg_non_deterministic ); command_line::add_arg(desc_params, arg_electrum_seed ); - command_line::add_arg(desc_params, arg_trusted_daemon); - command_line::add_arg(desc_params, arg_untrusted_daemon); command_line::add_arg(desc_params, arg_allow_mismatched_daemon_version); command_line::add_arg(desc_params, arg_restore_height); command_line::add_arg(desc_params, arg_do_not_relay); diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h index 472ad0c00..bfbe633ac 100644 --- a/src/simplewallet/simplewallet.h +++ b/src/simplewallet/simplewallet.h @@ -44,6 +44,7 @@ #include "cryptonote_basic/cryptonote_basic_impl.h" #include "wallet/wallet2.h" #include "console_handler.h" +#include "wipeable_string.h" #include "common/i18n.h" #include "common/password.h" #include "crypto/crypto.h" // for definition of crypto::secret_key @@ -96,7 +97,7 @@ namespace cryptonote boost::optional<epee::wipeable_string> new_wallet(const boost::program_options::variables_map& vm, const cryptonote::account_public_address& address, const boost::optional<crypto::secret_key>& spendkey, const crypto::secret_key& viewkey); boost::optional<epee::wipeable_string> new_wallet(const boost::program_options::variables_map& vm, - const std::string &multisig_keys, const std::string &old_language); + const epee::wipeable_string &multisig_keys, const std::string &old_language); boost::optional<epee::wipeable_string> new_wallet(const boost::program_options::variables_map& vm, const std::string& device_name); bool open_wallet(const boost::program_options::variables_map& vm); bool close_wallet(); @@ -232,13 +233,12 @@ namespace cryptonote bool print_ring_members(const std::vector<tools::wallet2::pending_tx>& ptx_vector, std::ostream& ostr); std::string get_prompt() const; bool print_seed(bool encrypted); - bool is_daemon_trusted() const { return *m_trusted_daemon; } /*! * \brief Prints the seed with a nice message * \param seed seed to print */ - void print_seed(std::string seed); + void print_seed(const epee::wipeable_string &seed); /*! * \brief Gets the word seed language from the user. @@ -261,6 +261,7 @@ namespace cryptonote virtual void on_unconfirmed_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index); virtual void on_money_spent(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& in_tx, uint64_t amount, const cryptonote::transaction& spend_tx, const cryptonote::subaddress_index& subaddr_index); virtual void on_skip_transaction(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx); + virtual boost::optional<epee::wipeable_string> on_get_password(const char *reason); //---------------------------------------------------------- friend class refresh_progress_reporter_t; @@ -329,13 +330,12 @@ namespace cryptonote std::string m_import_path; std::string m_subaddress_lookahead; - std::string m_electrum_seed; // electrum-style seed parameter + epee::wipeable_string m_electrum_seed; // electrum-style seed parameter crypto::secret_key m_recovery_key; // recovery key (used as random for wallet gen) bool m_restore_deterministic_wallet; // recover flag bool m_restore_multisig_wallet; // recover flag bool m_non_deterministic; // old 2-random generation - boost::optional<bool> m_trusted_daemon; bool m_allow_mismatched_daemon_version; bool m_restoring; // are we restoring, by whatever method? uint64_t m_restore_height; // optional diff --git a/src/wallet/api/wallet.cpp b/src/wallet/api/wallet.cpp index b48bf07e0..3780d7271 100644 --- a/src/wallet/api/wallet.cpp +++ b/src/wallet/api/wallet.cpp @@ -369,7 +369,6 @@ void Wallet::error(const std::string &category, const std::string &str) { WalletImpl::WalletImpl(NetworkType nettype, uint64_t kdf_rounds) :m_wallet(nullptr) , m_status(Wallet::Status_Ok) - , m_trustedDaemon(false) , m_wallet2Callback(nullptr) , m_recoveringFromSeed(false) , m_recoveringFromDevice(false) @@ -733,10 +732,10 @@ bool WalletImpl::close(bool store) std::string WalletImpl::seed() const { - std::string seed; + epee::wipeable_string seed; if (m_wallet) m_wallet->get_seed(seed); - return seed; + return std::string(seed.data(), seed.size()); // TODO } std::string WalletImpl::getSeedLanguage() const @@ -1298,6 +1297,7 @@ PendingTransaction *WalletImpl::createTransaction(const string &dst_addr, const size_t fake_outs_count = mixin_count > 0 ? mixin_count : m_wallet->default_mixin(); if (fake_outs_count == 0) fake_outs_count = DEFAULT_MIXIN; + fake_outs_count = m_wallet->adjust_mixin(fake_outs_count); uint32_t adjusted_priority = m_wallet->adjust_priority(static_cast<uint32_t>(priority)); @@ -1358,7 +1358,7 @@ PendingTransaction *WalletImpl::createTransaction(const string &dst_addr, const dsts.push_back(de); transaction->m_pending_tx = m_wallet->create_transactions_2(dsts, fake_outs_count, 0 /* unlock_time */, adjusted_priority, - extra, subaddr_account, subaddr_indices, m_trustedDaemon); + extra, subaddr_account, subaddr_indices); } else { // for the GUI, sweep_all (i.e. amount set as "(all)") will always sweep all the funds in all the addresses if (subaddr_indices.empty()) @@ -1368,7 +1368,7 @@ PendingTransaction *WalletImpl::createTransaction(const string &dst_addr, const } transaction->m_pending_tx = m_wallet->create_transactions_all(0, info.address, info.is_subaddress, fake_outs_count, 0 /* unlock_time */, adjusted_priority, - extra, subaddr_account, subaddr_indices, m_trustedDaemon); + extra, subaddr_account, subaddr_indices); } if (multisig().isMultisig) { @@ -1454,7 +1454,7 @@ PendingTransaction *WalletImpl::createSweepUnmixableTransaction() do { try { - transaction->m_pending_tx = m_wallet->create_unmixable_sweep_transactions(m_trustedDaemon); + transaction->m_pending_tx = m_wallet->create_unmixable_sweep_transactions(); } catch (const tools::error::daemon_busy&) { // TODO: make it translatable with "tr"? @@ -1891,12 +1891,12 @@ Wallet::ConnectionStatus WalletImpl::connected() const void WalletImpl::setTrustedDaemon(bool arg) { - m_trustedDaemon = arg; + m_wallet->set_trusted_daemon(arg); } bool WalletImpl::trustedDaemon() const { - return m_trustedDaemon; + return m_wallet->is_trusted_daemon(); } bool WalletImpl::watchOnly() const @@ -2032,7 +2032,8 @@ bool WalletImpl::isNewWallet() const bool WalletImpl::doInit(const string &daemon_address, uint64_t upper_transaction_size_limit, bool ssl) { - if (!m_wallet->init(daemon_address, m_daemon_login, upper_transaction_size_limit, ssl)) + // claim RPC so there's no in-memory encryption for now + if (!m_wallet->init(true, daemon_address, m_daemon_login, upper_transaction_size_limit, ssl)) return false; // in case new wallet, this will force fast-refresh (pulling hashes instead of blocks) diff --git a/src/wallet/api/wallet.h b/src/wallet/api/wallet.h index 9218d3ad5..0f3b1ce04 100644 --- a/src/wallet/api/wallet.h +++ b/src/wallet/api/wallet.h @@ -219,7 +219,6 @@ private: mutable std::string m_errorString; std::string m_password; TransactionHistoryImpl * m_history; - bool m_trustedDaemon; Wallet2CallbackImpl * m_wallet2Callback; AddressBookImpl * m_addressBook; SubaddressImpl * m_subaddress; diff --git a/src/wallet/node_rpc_proxy.cpp b/src/wallet/node_rpc_proxy.cpp index 401ada61b..21f75371b 100644 --- a/src/wallet/node_rpc_proxy.cpp +++ b/src/wallet/node_rpc_proxy.cpp @@ -50,12 +50,13 @@ void NodeRPCProxy::invalidate() m_height = 0; for (size_t n = 0; n < 256; ++n) m_earliest_height[n] = 0; - m_dynamic_per_kb_fee_estimate = 0; - m_dynamic_per_kb_fee_estimate_cached_height = 0; - m_dynamic_per_kb_fee_estimate_grace_blocks = 0; + m_dynamic_base_fee_estimate = 0; + m_dynamic_base_fee_estimate_cached_height = 0; + m_dynamic_base_fee_estimate_grace_blocks = 0; + m_fee_quantization_mask = 1; m_rpc_version = 0; m_target_height = 0; - m_block_size_limit = 0; + m_block_weight_limit = 0; m_get_info_time = 0; } @@ -99,7 +100,7 @@ boost::optional<std::string> NodeRPCProxy::get_info() const CHECK_AND_ASSERT_MES(resp_t.status == CORE_RPC_STATUS_OK, resp_t.status, "Failed to get target blockchain height"); m_height = resp_t.height; m_target_height = resp_t.target_height; - m_block_size_limit = resp_t.block_size_limit; + m_block_weight_limit = resp_t.block_weight_limit ? resp_t.block_weight_limit : resp_t.block_size_limit; m_get_info_time = now; } return boost::optional<std::string>(); @@ -123,12 +124,12 @@ boost::optional<std::string> NodeRPCProxy::get_target_height(uint64_t &height) c return boost::optional<std::string>(); } -boost::optional<std::string> NodeRPCProxy::get_block_size_limit(uint64_t &block_size_limit) const +boost::optional<std::string> NodeRPCProxy::get_block_weight_limit(uint64_t &block_weight_limit) const { auto res = get_info(); if (res) return res; - block_size_limit = m_block_size_limit; + block_weight_limit = m_block_weight_limit; return boost::optional<std::string>(); } @@ -146,14 +147,14 @@ boost::optional<std::string> NodeRPCProxy::get_earliest_height(uint8_t version, CHECK_AND_ASSERT_MES(r, std::string(), "Failed to connect to daemon"); CHECK_AND_ASSERT_MES(resp_t.status != CORE_RPC_STATUS_BUSY, resp_t.status, "Failed to connect to daemon"); CHECK_AND_ASSERT_MES(resp_t.status == CORE_RPC_STATUS_OK, resp_t.status, "Failed to get hard fork status"); - m_earliest_height[version] = resp_t.enabled ? resp_t.earliest_height : std::numeric_limits<uint64_t>::max(); + m_earliest_height[version] = resp_t.earliest_height; } earliest_height = m_earliest_height[version]; return boost::optional<std::string>(); } -boost::optional<std::string> NodeRPCProxy::get_dynamic_per_kb_fee_estimate(uint64_t grace_blocks, uint64_t &fee) const +boost::optional<std::string> NodeRPCProxy::get_dynamic_base_fee_estimate(uint64_t grace_blocks, uint64_t &fee) const { uint64_t height; @@ -161,10 +162,10 @@ boost::optional<std::string> NodeRPCProxy::get_dynamic_per_kb_fee_estimate(uint6 if (result) return result; - if (m_dynamic_per_kb_fee_estimate_cached_height != height || m_dynamic_per_kb_fee_estimate_grace_blocks != grace_blocks) + if (m_dynamic_base_fee_estimate_cached_height != height || m_dynamic_base_fee_estimate_grace_blocks != grace_blocks) { - cryptonote::COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::request req_t = AUTO_VAL_INIT(req_t); - cryptonote::COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::response resp_t = AUTO_VAL_INIT(resp_t); + cryptonote::COMMAND_RPC_GET_BASE_FEE_ESTIMATE::request req_t = AUTO_VAL_INIT(req_t); + cryptonote::COMMAND_RPC_GET_BASE_FEE_ESTIMATE::response resp_t = AUTO_VAL_INIT(resp_t); m_daemon_rpc_mutex.lock(); req_t.grace_blocks = grace_blocks; @@ -173,12 +174,47 @@ boost::optional<std::string> NodeRPCProxy::get_dynamic_per_kb_fee_estimate(uint6 CHECK_AND_ASSERT_MES(r, std::string(), "Failed to connect to daemon"); CHECK_AND_ASSERT_MES(resp_t.status != CORE_RPC_STATUS_BUSY, resp_t.status, "Failed to connect to daemon"); CHECK_AND_ASSERT_MES(resp_t.status == CORE_RPC_STATUS_OK, resp_t.status, "Failed to get fee estimate"); - m_dynamic_per_kb_fee_estimate = resp_t.fee; - m_dynamic_per_kb_fee_estimate_cached_height = height; - m_dynamic_per_kb_fee_estimate_grace_blocks = grace_blocks; + m_dynamic_base_fee_estimate = resp_t.fee; + m_dynamic_base_fee_estimate_cached_height = height; + m_dynamic_base_fee_estimate_grace_blocks = grace_blocks; + m_fee_quantization_mask = resp_t.quantization_mask; } - fee = m_dynamic_per_kb_fee_estimate; + fee = m_dynamic_base_fee_estimate; + return boost::optional<std::string>(); +} + +boost::optional<std::string> NodeRPCProxy::get_fee_quantization_mask(uint64_t &fee_quantization_mask) const +{ + uint64_t height; + + boost::optional<std::string> result = get_height(height); + if (result) + return result; + + if (m_dynamic_base_fee_estimate_cached_height != height) + { + cryptonote::COMMAND_RPC_GET_BASE_FEE_ESTIMATE::request req_t = AUTO_VAL_INIT(req_t); + cryptonote::COMMAND_RPC_GET_BASE_FEE_ESTIMATE::response resp_t = AUTO_VAL_INIT(resp_t); + + m_daemon_rpc_mutex.lock(); + req_t.grace_blocks = m_dynamic_base_fee_estimate_grace_blocks; + bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_fee_estimate", req_t, resp_t, m_http_client, rpc_timeout); + m_daemon_rpc_mutex.unlock(); + CHECK_AND_ASSERT_MES(r, std::string(), "Failed to connect to daemon"); + CHECK_AND_ASSERT_MES(resp_t.status != CORE_RPC_STATUS_BUSY, resp_t.status, "Failed to connect to daemon"); + CHECK_AND_ASSERT_MES(resp_t.status == CORE_RPC_STATUS_OK, resp_t.status, "Failed to get fee estimate"); + m_dynamic_base_fee_estimate = resp_t.fee; + m_dynamic_base_fee_estimate_cached_height = height; + m_fee_quantization_mask = resp_t.quantization_mask; + } + + fee_quantization_mask = m_fee_quantization_mask; + if (fee_quantization_mask == 0) + { + MERROR("Fee quantization mask is 0, forcing to 1"); + fee_quantization_mask = 1; + } return boost::optional<std::string>(); } diff --git a/src/wallet/node_rpc_proxy.h b/src/wallet/node_rpc_proxy.h index 8a65884f7..65f13eaaa 100644 --- a/src/wallet/node_rpc_proxy.h +++ b/src/wallet/node_rpc_proxy.h @@ -47,9 +47,10 @@ public: boost::optional<std::string> get_height(uint64_t &height) const; void set_height(uint64_t h); boost::optional<std::string> get_target_height(uint64_t &height) const; - boost::optional<std::string> get_block_size_limit(uint64_t &block_size_limit) const; + boost::optional<std::string> get_block_weight_limit(uint64_t &block_weight_limit) const; boost::optional<std::string> get_earliest_height(uint8_t version, uint64_t &earliest_height) const; - boost::optional<std::string> get_dynamic_per_kb_fee_estimate(uint64_t grace_blocks, uint64_t &fee) const; + boost::optional<std::string> get_dynamic_base_fee_estimate(uint64_t grace_blocks, uint64_t &fee) const; + boost::optional<std::string> get_fee_quantization_mask(uint64_t &fee_quantization_mask) const; private: boost::optional<std::string> get_info() const; @@ -59,12 +60,13 @@ private: mutable uint64_t m_height; mutable uint64_t m_earliest_height[256]; - mutable uint64_t m_dynamic_per_kb_fee_estimate; - mutable uint64_t m_dynamic_per_kb_fee_estimate_cached_height; - mutable uint64_t m_dynamic_per_kb_fee_estimate_grace_blocks; + mutable uint64_t m_dynamic_base_fee_estimate; + mutable uint64_t m_dynamic_base_fee_estimate_cached_height; + mutable uint64_t m_dynamic_base_fee_estimate_grace_blocks; + mutable uint64_t m_fee_quantization_mask; mutable uint32_t m_rpc_version; mutable uint64_t m_target_height; - mutable uint64_t m_block_size_limit; + mutable uint64_t m_block_weight_limit; mutable time_t m_get_info_time; }; diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index f4b6708fc..81fe3892c 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -84,11 +84,12 @@ using namespace cryptonote; // used to choose when to stop adding outputs to a tx #define APPROXIMATE_INPUT_BYTES 80 -// used to target a given block size (additional outputs may be added on top to build fee) -#define TX_SIZE_TARGET(bytes) (bytes*2/3) +// used to target a given block weight (additional outputs may be added on top to build fee) +#define TX_WEIGHT_TARGET(bytes) (bytes*2/3) // arbitrary, used to generate different hashes from the same input #define CHACHA8_KEY_TAIL 0x8c +#define CACHE_KEY_TAIL 0x8d #define UNSIGNED_TX_PREFIX "Monero unsigned tx set\004" #define SIGNED_TX_PREFIX "Monero signed tx set\004" @@ -121,8 +122,6 @@ using namespace cryptonote; static const std::string MULTISIG_SIGNATURE_MAGIC = "SigMultisigPkV1"; -std::atomic<unsigned int> tools::wallet2::key_ref::refs(0); - namespace { std::string get_default_ringdb_path() @@ -141,6 +140,8 @@ namespace struct options { const command_line::arg_descriptor<std::string> daemon_address = {"daemon-address", tools::wallet2::tr("Use daemon instance at <host>:<port>"), ""}; const command_line::arg_descriptor<std::string> daemon_host = {"daemon-host", tools::wallet2::tr("Use daemon instance at host <arg> instead of localhost"), ""}; + const command_line::arg_descriptor<bool> trusted_daemon = {"trusted-daemon", tools::wallet2::tr("Enable commands which rely on a trusted daemon"), false}; + const command_line::arg_descriptor<bool> untrusted_daemon = {"untrusted-daemon", tools::wallet2::tr("Disable commands which rely on a trusted daemon"), false}; const command_line::arg_descriptor<std::string> password = {"password", tools::wallet2::tr("Wallet password (escape/quote as needed)"), "", true}; const command_line::arg_descriptor<std::string> password_file = {"password-file", tools::wallet2::tr("Wallet password file"), "", true}; const command_line::arg_descriptor<int> daemon_port = {"daemon-port", tools::wallet2::tr("Use daemon instance at port <arg> instead of 18081"), 0}; @@ -182,22 +183,24 @@ uint64_t calculate_fee(uint64_t fee_per_kb, size_t bytes, uint64_t fee_multiplie return kB * fee_per_kb * fee_multiplier; } -uint64_t calculate_fee(uint64_t fee_per_kb, const cryptonote::blobdata &blob, uint64_t fee_multiplier) +uint64_t calculate_fee_from_weight(uint64_t base_fee, uint64_t weight, uint64_t fee_multiplier, uint64_t fee_quantization_mask) { - return calculate_fee(fee_per_kb, blob.size(), fee_multiplier); + uint64_t fee = weight * base_fee * fee_multiplier; + fee = (fee + fee_quantization_mask - 1) / fee_quantization_mask * fee_quantization_mask; + return fee; } -std::string get_size_string(size_t sz) +std::string get_weight_string(size_t weight) { - return std::to_string(sz) + " bytes (" + std::to_string((sz + 1023) / 1024) + " kB)"; + return std::to_string(weight) + " weight"; } -std::string get_size_string(const cryptonote::blobdata &tx) +std::string get_weight_string(const cryptonote::transaction &tx, size_t blob_size) { - return get_size_string(tx.size()); + return get_weight_string(get_transaction_weight(tx, blob_size)); } -std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variables_map& vm, const options& opts, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter) +std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variables_map& vm, bool rpc, const options& opts, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter) { const bool testnet = command_line::get_arg(vm, opts.testnet); const bool stagenet = command_line::get_arg(vm, opts.stagenet); @@ -237,8 +240,29 @@ std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variabl if (daemon_address.empty()) daemon_address = std::string("http://") + daemon_host + ":" + std::to_string(daemon_port); + boost::optional<bool> trusted_daemon; + if (!command_line::is_arg_defaulted(vm, opts.trusted_daemon) || !command_line::is_arg_defaulted(vm, opts.untrusted_daemon)) + trusted_daemon = command_line::get_arg(vm, opts.trusted_daemon) && !command_line::get_arg(vm, opts.untrusted_daemon); + THROW_WALLET_EXCEPTION_IF(!command_line::is_arg_defaulted(vm, opts.trusted_daemon) && !command_line::is_arg_defaulted(vm, opts.untrusted_daemon), + tools::error::wallet_internal_error, tools::wallet2::tr("--trusted-daemon and --untrusted-daemon are both seen, assuming untrusted")); + + // set --trusted-daemon if local and not overridden + if (!trusted_daemon) + { + try + { + trusted_daemon = false; + if (tools::is_local_address(daemon_address)) + { + MINFO(tr("Daemon is local, assuming trusted")); + trusted_daemon = true; + } + } + catch (const std::exception &e) { } + } + std::unique_ptr<tools::wallet2> wallet(new tools::wallet2(nettype, kdf_rounds)); - wallet->init(std::move(daemon_address), std::move(login)); + wallet->init(rpc, std::move(daemon_address), std::move(login), 0, false, *trusted_daemon); boost::filesystem::path ringdb_path = command_line::get_arg(vm, opts.shared_ringdb_dir); wallet->set_ring_database(ringdb_path.string()); return wallet; @@ -273,7 +297,7 @@ boost::optional<tools::password_container> get_password(const boost::program_opt return password_prompter(verify ? tr("Enter a new password for the wallet") : tr("Wallet password"), verify); } -std::unique_ptr<tools::wallet2> generate_from_json(const std::string& json_file, const boost::program_options::variables_map& vm, const options& opts, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter) +std::unique_ptr<tools::wallet2> generate_from_json(const std::string& json_file, const boost::program_options::variables_map& vm, bool rpc, const options& opts, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter) { const bool testnet = command_line::get_arg(vm, opts.testnet); const bool stagenet = command_line::get_arg(vm, opts.stagenet); @@ -411,7 +435,7 @@ std::unique_ptr<tools::wallet2> generate_from_json(const std::string& json_file, THROW_WALLET_EXCEPTION_IF(deprecated_wallet, tools::error::wallet_internal_error, tools::wallet2::tr("Cannot generate deprecated wallets from JSON")); - wallet.reset(make_basic(vm, opts, password_prompter).release()); + wallet.reset(make_basic(vm, rpc, opts, password_prompter).release()); wallet->set_refresh_from_block_height(field_scan_from_height); wallet->explicit_refresh_from_block_height(field_scan_from_height_found); @@ -554,7 +578,12 @@ size_t estimate_rct_tx_size(int n_inputs, int mixin, int n_outputs, size_t extra // rangeSigs if (bulletproof) - size += ((2*6 + 4 + 5)*32 + 3) * n_outputs; + { + size_t log_padded_outputs = 0; + while ((1<<log_padded_outputs) < n_outputs) + ++log_padded_outputs; + size += (2 * (6 + log_padded_outputs) + 4 + 5) * 32 + 3; + } else size += (2*64*32+32+64*32) * n_outputs; @@ -573,23 +602,63 @@ size_t estimate_rct_tx_size(int n_inputs, int mixin, int n_outputs, size_t extra // txnFee size += 4; - LOG_PRINT_L2("estimated rct tx size for " << n_inputs << " with ring size " << (mixin+1) << " and " << n_outputs << ": " << size << " (" << ((32 * n_inputs/*+1*/) + 2 * 32 * (mixin+1) * n_inputs + 32 * n_outputs) << " saved)"); + LOG_PRINT_L2("estimated " << (bulletproof ? "bulletproof" : "borromean") << " rct tx size for " << n_inputs << " inputs with ring size " << (mixin+1) << " and " << n_outputs << " outputs: " << size << " (" << ((32 * n_inputs/*+1*/) + 2 * 32 * (mixin+1) * n_inputs + 32 * n_outputs) << " saved)"); return size; } size_t estimate_tx_size(bool use_rct, int n_inputs, int mixin, int n_outputs, size_t extra_size, bool bulletproof) { if (use_rct) - return estimate_rct_tx_size(n_inputs, mixin, n_outputs + 1, extra_size, bulletproof); + return estimate_rct_tx_size(n_inputs, mixin, n_outputs, extra_size, bulletproof); else return n_inputs * (mixin+1) * APPROXIMATE_INPUT_BYTES + extra_size; } +uint64_t estimate_tx_weight(bool use_rct, int n_inputs, int mixin, int n_outputs, size_t extra_size, bool bulletproof) +{ + size_t size = estimate_tx_size(use_rct, n_inputs, mixin, n_outputs, extra_size, bulletproof); + if (use_rct && bulletproof && n_outputs > 2) + { + const uint64_t bp_base = 368; + size_t log_padded_outputs = 2; + while ((1<<log_padded_outputs) < n_outputs) + ++log_padded_outputs; + uint64_t nlr = 2 * (6 + log_padded_outputs); + const uint64_t bp_size = 32 * (9 + nlr); + const uint64_t bp_clawback = (bp_base * (1<<log_padded_outputs) - bp_size) * 4 / 5; + MDEBUG("clawback on size " << size << ": " << bp_clawback); + size += bp_clawback; + } + return size; +} + uint8_t get_bulletproof_fork() { return 8; } +uint64_t estimate_fee(bool use_per_byte_fee, bool use_rct, int n_inputs, int mixin, int n_outputs, size_t extra_size, bool bulletproof, uint64_t base_fee, uint64_t fee_multiplier, uint64_t fee_quantization_mask) +{ + if (use_per_byte_fee) + { + const size_t estimated_tx_weight = estimate_tx_weight(use_rct, n_inputs, mixin, n_outputs, extra_size, bulletproof); + return calculate_fee_from_weight(base_fee, estimated_tx_weight, fee_multiplier, fee_quantization_mask); + } + else + { + const size_t estimated_tx_size = estimate_tx_size(use_rct, n_inputs, mixin, n_outputs, extra_size, bulletproof); + return calculate_fee(base_fee, estimated_tx_size, fee_multiplier); + } +} + +uint64_t calculate_fee(bool use_per_byte_fee, const cryptonote::transaction &tx, size_t blob_size, uint64_t base_fee, uint64_t fee_multiplier, uint64_t fee_quantization_mask) +{ + if (use_per_byte_fee) + return calculate_fee_from_weight(base_fee, cryptonote::get_transaction_weight(tx, blob_size), fee_multiplier, fee_quantization_mask); + else + return calculate_fee(base_fee, blob_size, fee_multiplier); +} + crypto::hash8 get_short_payment_id(const tools::wallet2::pending_tx &ptx, hw::device &hwdev) { crypto::hash8 payment_id8 = null_hash8; @@ -648,11 +717,40 @@ const size_t MAX_SPLIT_ATTEMPTS = 30; constexpr const std::chrono::seconds wallet2::rpc_timeout; const char* wallet2::tr(const char* str) { return i18n_translate(str, "tools::wallet2"); } +wallet_keys_unlocker::wallet_keys_unlocker(wallet2 &w, const boost::optional<tools::password_container> &password): + w(w), + locked(password != boost::none) +{ + if (!locked || w.is_rpc()) + return; + const epee::wipeable_string pass = password->password(); + w.generate_chacha_key_from_password(pass, key); + w.decrypt_keys(key); +} + +wallet_keys_unlocker::wallet_keys_unlocker(wallet2 &w, bool locked, const epee::wipeable_string &password): + w(w), + locked(locked) +{ + if (!locked) + return; + w.generate_chacha_key_from_password(password, key); + w.decrypt_keys(key); +} + +wallet_keys_unlocker::~wallet_keys_unlocker() +{ + if (!locked) + return; + w.encrypt_keys(key); +} + wallet2::wallet2(network_type nettype, uint64_t kdf_rounds): m_multisig_rescan_info(NULL), m_multisig_rescan_k(NULL), m_run(true), m_callback(0), + m_trusted_daemon(false), m_nettype(nettype), m_always_confirm_transfers(true), m_print_ring_members(false), @@ -693,7 +791,9 @@ wallet2::wallet2(network_type nettype, uint64_t kdf_rounds): m_key_on_device(false), m_ring_history_saved(false), m_ringdb(), - m_last_block_reward(0) + m_last_block_reward(0), + m_encrypt_keys_after_refresh(boost::none), + m_rpc(false) { } @@ -716,6 +816,8 @@ void wallet2::init_options(boost::program_options::options_description& desc_par const options opts{}; command_line::add_arg(desc_params, opts.daemon_address); command_line::add_arg(desc_params, opts.daemon_host); + command_line::add_arg(desc_params, opts.trusted_daemon); + command_line::add_arg(desc_params, opts.untrusted_daemon); command_line::add_arg(desc_params, opts.password); command_line::add_arg(desc_params, opts.password_file); command_line::add_arg(desc_params, opts.daemon_port); @@ -726,14 +828,14 @@ void wallet2::init_options(boost::program_options::options_description& desc_par command_line::add_arg(desc_params, opts.kdf_rounds); } -std::unique_ptr<wallet2> wallet2::make_from_json(const boost::program_options::variables_map& vm, const std::string& json_file, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter) +std::unique_ptr<wallet2> wallet2::make_from_json(const boost::program_options::variables_map& vm, bool rpc, const std::string& json_file, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter) { const options opts{}; - return generate_from_json(json_file, vm, opts, password_prompter); + return generate_from_json(json_file, vm, rpc, opts, password_prompter); } std::pair<std::unique_ptr<wallet2>, password_container> wallet2::make_from_file( - const boost::program_options::variables_map& vm, const std::string& wallet_file, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter) + const boost::program_options::variables_map& vm, bool rpc, const std::string& wallet_file, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter) { const options opts{}; auto pwd = get_password(vm, opts, password_prompter, false); @@ -741,7 +843,7 @@ std::pair<std::unique_ptr<wallet2>, password_container> wallet2::make_from_file( { return {nullptr, password_container{}}; } - auto wallet = make_basic(vm, opts, password_prompter); + auto wallet = make_basic(vm, rpc, opts, password_prompter); if (wallet) { wallet->load(wallet_file, pwd->password()); @@ -749,7 +851,7 @@ std::pair<std::unique_ptr<wallet2>, password_container> wallet2::make_from_file( return {std::move(wallet), std::move(*pwd)}; } -std::pair<std::unique_ptr<wallet2>, password_container> wallet2::make_new(const boost::program_options::variables_map& vm, const std::function<boost::optional<password_container>(const char *, bool)> &password_prompter) +std::pair<std::unique_ptr<wallet2>, password_container> wallet2::make_new(const boost::program_options::variables_map& vm, bool rpc, const std::function<boost::optional<password_container>(const char *, bool)> &password_prompter) { const options opts{}; auto pwd = get_password(vm, opts, password_prompter, true); @@ -757,25 +859,27 @@ std::pair<std::unique_ptr<wallet2>, password_container> wallet2::make_new(const { return {nullptr, password_container{}}; } - return {make_basic(vm, opts, password_prompter), std::move(*pwd)}; + return {make_basic(vm, rpc, opts, password_prompter), std::move(*pwd)}; } -std::unique_ptr<wallet2> wallet2::make_dummy(const boost::program_options::variables_map& vm, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter) +std::unique_ptr<wallet2> wallet2::make_dummy(const boost::program_options::variables_map& vm, bool rpc, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter) { const options opts{}; - return make_basic(vm, opts, password_prompter); + return make_basic(vm, rpc, opts, password_prompter); } //---------------------------------------------------------------------------------------------------- -bool wallet2::init(std::string daemon_address, boost::optional<epee::net_utils::http::login> daemon_login, uint64_t upper_transaction_size_limit, bool ssl) +bool wallet2::init(bool rpc, std::string daemon_address, boost::optional<epee::net_utils::http::login> daemon_login, uint64_t upper_transaction_weight_limit, bool ssl, bool trusted_daemon) { + m_rpc = rpc; m_checkpoints.init_default_checkpoints(m_nettype); if(m_http_client.is_connected()) m_http_client.disconnect(); m_is_initialized = true; - m_upper_transaction_size_limit = upper_transaction_size_limit; + m_upper_transaction_weight_limit = upper_transaction_weight_limit; m_daemon_address = std::move(daemon_address); m_daemon_login = std::move(daemon_login); + m_trusted_daemon = trusted_daemon; // When switching from light wallet to full wallet, we need to reset the height we got from lw node. return m_http_client.set_server(get_daemon_address(), get_daemon_login(), ssl); } @@ -785,11 +889,10 @@ bool wallet2::is_deterministic() const crypto::secret_key second; keccak((uint8_t *)&get_account().get_keys().m_spend_secret_key, sizeof(crypto::secret_key), (uint8_t *)&second, sizeof(crypto::secret_key)); sc_reduce32((uint8_t *)&second); - bool keys_deterministic = memcmp(second.data,get_account().get_keys().m_view_secret_key.data, sizeof(crypto::secret_key)) == 0; - return keys_deterministic; + return memcmp(second.data,get_account().get_keys().m_view_secret_key.data, sizeof(crypto::secret_key)) == 0; } //---------------------------------------------------------------------------------------------------- -bool wallet2::get_seed(std::string& electrum_words, const epee::wipeable_string &passphrase) const +bool wallet2::get_seed(epee::wipeable_string& electrum_words, const epee::wipeable_string &passphrase) const { bool keys_deterministic = is_deterministic(); if (!keys_deterministic) @@ -815,7 +918,7 @@ bool wallet2::get_seed(std::string& electrum_words, const epee::wipeable_string return true; } //---------------------------------------------------------------------------------------------------- -bool wallet2::get_multisig_seed(std::string& seed, const epee::wipeable_string &passphrase, bool raw) const +bool wallet2::get_multisig_seed(epee::wipeable_string& seed, const epee::wipeable_string &passphrase, bool raw) const { bool ready; uint32_t threshold, total; @@ -838,7 +941,7 @@ bool wallet2::get_multisig_seed(std::string& seed, const epee::wipeable_string & crypto::secret_key skey; crypto::public_key pkey; const account_keys &keys = get_account().get_keys(); - std::string data; + epee::wipeable_string data; data.append((const char*)&threshold, sizeof(uint32_t)); data.append((const char*)&total, sizeof(uint32_t)); skey = keys.m_spend_secret_key; @@ -864,7 +967,7 @@ bool wallet2::get_multisig_seed(std::string& seed, const epee::wipeable_string & if (raw) { - seed = epee::string_tools::buff_to_hex_nodelimer(data); + seed = epee::to_hex::wipeable_string({(const unsigned char*)data.data(), data.size()}); } else { @@ -1086,10 +1189,9 @@ static uint64_t decodeRct(const rct::rctSig & rv, const crypto::key_derivation & switch (rv.type) { case rct::RCTTypeSimple: - case rct::RCTTypeSimpleBulletproof: + case rct::RCTTypeBulletproof: return rct::decodeRctSimple(rv, rct::sk2rct(scalar1), i, mask, hwdev); case rct::RCTTypeFull: - case rct::RCTTypeFullBulletproof: return rct::decodeRct(rv, rct::sk2rct(scalar1), i, mask, hwdev); default: LOG_ERROR("Unsupported rct type: " << rv.type); @@ -1103,9 +1205,25 @@ static uint64_t decodeRct(const rct::rctSig & rv, const crypto::key_derivation & } } //---------------------------------------------------------------------------------------------------- -void wallet2::scan_output(const cryptonote::transaction &tx, const crypto::public_key &tx_pub_key, size_t i, tx_scan_info_t &tx_scan_info, int &num_vouts_received, std::unordered_map<cryptonote::subaddress_index, uint64_t> &tx_money_got_in_outs, std::vector<size_t> &outs) const +void wallet2::scan_output(const cryptonote::transaction &tx, const crypto::public_key &tx_pub_key, size_t i, tx_scan_info_t &tx_scan_info, int &num_vouts_received, std::unordered_map<cryptonote::subaddress_index, uint64_t> &tx_money_got_in_outs, std::vector<size_t> &outs) { THROW_WALLET_EXCEPTION_IF(i >= tx.vout.size(), error::wallet_internal_error, "Invalid vout index"); + + // if keys are encrypted, ask for password + if (m_ask_password && !m_rpc && !m_watch_only && !m_multisig_rescan_k) + { + static critical_section password_lock; + CRITICAL_REGION_LOCAL(password_lock); + if (!m_encrypt_keys_after_refresh) + { + boost::optional<epee::wipeable_string> pwd = m_callback->on_get_password("output received"); + THROW_WALLET_EXCEPTION_IF(!pwd, error::password_needed, tr("Password is needed to compute key image for incoming monero")); + THROW_WALLET_EXCEPTION_IF(!verify_password(*pwd), error::password_needed, tr("Invalid password: password is needed to compute key image for incoming monero")); + decrypt_keys(*pwd); + m_encrypt_keys_after_refresh = *pwd; + } + } + if (m_multisig) { tx_scan_info.in_ephemeral.pub = boost::get<cryptonote::txout_to_key>(tx.vout[i].target).key; @@ -1763,34 +1881,7 @@ void wallet2::pull_blocks(uint64_t start_height, uint64_t &blocks_start_height, cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::response res = AUTO_VAL_INIT(res); req.block_ids = short_chain_history; - uint32_t rpc_version; - boost::optional<std::string> result = m_node_rpc_proxy.get_rpc_version(rpc_version); - // no error - if (!!result) - { - // empty string -> not connection - THROW_WALLET_EXCEPTION_IF(result->empty(), tools::error::no_connection_to_daemon, "getversion"); - THROW_WALLET_EXCEPTION_IF(*result == CORE_RPC_STATUS_BUSY, tools::error::daemon_busy, "getversion"); - if (*result != CORE_RPC_STATUS_OK) - { - MDEBUG("Cannot determine daemon RPC version, not asking for pruned blocks"); - req.prune = false; // old daemon - } - } - else - { - if (rpc_version >= MAKE_CORE_RPC_VERSION(1, 7)) - { - MDEBUG("Daemon is recent enough, asking for pruned blocks"); - req.prune = true; - } - else - { - MDEBUG("Daemon is too old, not asking for pruned blocks"); - req.prune = false; - } - } - + req.prune = true; req.start_height = start_height; req.no_miner_tx = m_refresh_type == RefreshNoCoinbase; m_daemon_rpc_mutex.lock(); @@ -2065,6 +2156,14 @@ void wallet2::update_pool_state(bool refreshed) { MDEBUG("update_pool_state start"); + auto keys_reencryptor = epee::misc_utils::create_scope_leave_handler([&, this]() { + if (m_encrypt_keys_after_refresh) + { + encrypt_keys(*m_encrypt_keys_after_refresh); + m_encrypt_keys_after_refresh = boost::none; + } + }); + // get the pool state cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL_HASHES_BIN::request req; cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL_HASHES_BIN::response res; @@ -2372,8 +2471,6 @@ bool wallet2::delete_address_book_row(std::size_t row_id) { //---------------------------------------------------------------------------------------------------- void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched, bool& received_money) { - key_ref kref(*this); - if(m_light_wallet) { // MyMonero get_address_info needs to be called occasionally to trigger wallet sync. @@ -2441,6 +2538,14 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo // subsequent pulls in this refresh. start_height = 0; + auto keys_reencryptor = epee::misc_utils::create_scope_leave_handler([&, this]() { + if (m_encrypt_keys_after_refresh) + { + encrypt_keys(*m_encrypt_keys_after_refresh); + m_encrypt_keys_after_refresh = boost::none; + } + }); + bool first = true; while(m_run.load(std::memory_order_relaxed)) { @@ -2511,6 +2616,12 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo blocks = std::move(next_blocks); parsed_blocks = std::move(next_parsed_blocks); } + catch (const tools::error::password_needed&) + { + blocks_fetched += added_blocks; + waiter.wait(&tpool); + throw; + } catch (const std::exception&) { blocks_fetched += added_blocks; @@ -2595,6 +2706,7 @@ bool wallet2::get_rct_distribution(uint64_t &start_height, std::vector<uint64_t> req.amounts.push_back(0); req.from_height = 0; req.cumulative = true; + req.binary = true; m_daemon_rpc_mutex.lock(); bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_distribution", req, res, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); @@ -2732,8 +2844,20 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable std::string multisig_signers; cryptonote::account_base account = m_account; + crypto::chacha_key key; + crypto::generate_chacha_key(password.data(), password.size(), key, m_kdf_rounds); + + if (m_ask_password && !m_rpc && !m_watch_only) + { + account.encrypt_viewkey(key); + account.decrypt_keys(key); + } + if (watch_only) account.forget_spend_key(); + + account.encrypt_keys(key); + bool r = epee::serialization::store_t_to_binary(account, account_data); CHECK_AND_ASSERT_MES(r, false, "failed to serialize wallet keys"); wallet2::keys_file_data keys_file_data = boost::value_initialized<wallet2::keys_file_data>(); @@ -2850,6 +2974,9 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable value2.SetUint(m_subaddress_lookahead_minor); json.AddMember("subaddress_lookahead_minor", value2, json.GetAllocator()); + value2.SetUint(1); + json.AddMember("encrypted_secret_keys", value2, json.GetAllocator()); + // Serialize the JSON object rapidjson::StringBuffer buffer; rapidjson::Writer<rapidjson::StringBuffer> writer(buffer); @@ -2857,7 +2984,6 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable account_data = buffer.GetString(); // Encrypt the entire JSON object. - crypto::chacha_key key; crypto::generate_chacha_key(password.data(), password.size(), key, m_kdf_rounds); std::string cipher; cipher.resize(account_data.size()); @@ -2875,6 +3001,35 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable return true; } //---------------------------------------------------------------------------------------------------- +void wallet2::setup_keys(const epee::wipeable_string &password) +{ + crypto::chacha_key key; + crypto::generate_chacha_key(password.data(), password.size(), key, m_kdf_rounds); + + // re-encrypt, but keep viewkey unencrypted + if (m_ask_password && !m_rpc && !m_watch_only) + { + m_account.encrypt_keys(key); + m_account.decrypt_viewkey(key); + } + + static_assert(HASH_SIZE == sizeof(crypto::chacha_key), "Mismatched sizes of hash and chacha key"); + epee::mlocked<tools::scrubbed_arr<char, HASH_SIZE+1>> cache_key_data; + memcpy(cache_key_data.data(), &key, HASH_SIZE); + cache_key_data[HASH_SIZE] = CACHE_KEY_TAIL; + cn_fast_hash(cache_key_data.data(), HASH_SIZE+1, (crypto::hash&)m_cache_key); + get_ringdb_key(); +} +//---------------------------------------------------------------------------------------------------- +void wallet2::change_password(const std::string &filename, const epee::wipeable_string &original_password, const epee::wipeable_string &new_password) +{ + if (m_ask_password && !m_rpc && !m_watch_only) + decrypt_keys(original_password); + setup_keys(new_password); + rewrite(filename, new_password); + store(); +} +//---------------------------------------------------------------------------------------------------- /*! * \brief Load wallet information from wallet file. * \param keys_file_name Name of wallet file @@ -2885,6 +3040,7 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ rapidjson::Document json; wallet2::keys_file_data keys_file_data; std::string buf; + bool encrypted_secret_keys = false; bool r = epee::file_io_utils::load_file_to_string(keys_file_name, buf); THROW_WALLET_EXCEPTION_IF(!r, error::file_read_error, keys_file_name); @@ -2930,6 +3086,7 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ m_subaddress_lookahead_major = SUBADDRESS_LOOKAHEAD_MAJOR; m_subaddress_lookahead_minor = SUBADDRESS_LOOKAHEAD_MINOR; m_key_on_device = false; + encrypted_secret_keys = false; } else if(json.IsObject()) { @@ -3059,6 +3216,8 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ m_subaddress_lookahead_major = field_subaddress_lookahead_major; GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, subaddress_lookahead_minor, uint32_t, Uint, false, SUBADDRESS_LOOKAHEAD_MINOR); m_subaddress_lookahead_minor = field_subaddress_lookahead_minor; + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, encrypted_secret_keys, uint32_t, Uint, false, false); + encrypted_secret_keys = field_encrypted_secret_keys; } else { @@ -3075,12 +3234,39 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ m_account.set_device(hwdev); LOG_PRINT_L0("Device inited..."); } + + if (r) + { + if (encrypted_secret_keys) + { + m_account.decrypt_keys(key); + } + else + { + // rewrite with encrypted keys, ignore errors + if (m_ask_password && !m_rpc && !m_watch_only) + encrypt_keys(key); + bool saved_ret = store_keys(keys_file_name, password, m_watch_only); + if (!saved_ret) + { + // just moan a bit, but not fatal + MERROR("Error saving keys file with encrypted keys, not fatal"); + } + if (m_ask_password && !m_rpc && !m_watch_only) + decrypt_keys(key); + m_keys_file_locker.reset(); + } + } const cryptonote::account_keys& keys = m_account.get_keys(); hw::device &hwdev = m_account.get_device(); r = r && hwdev.verify_keys(keys.m_view_secret_key, keys.m_account_address.m_view_public_key); if(!m_watch_only && !m_multisig) r = r && hwdev.verify_keys(keys.m_spend_secret_key, keys.m_account_address.m_spend_public_key); THROW_WALLET_EXCEPTION_IF(!r, error::invalid_password); + + if (r) + setup_keys(password); + return true; } @@ -3121,6 +3307,7 @@ bool wallet2::verify_password(const std::string& keys_file_name, const epee::wip rapidjson::Document json; wallet2::keys_file_data keys_file_data; std::string buf; + bool encrypted_secret_keys = false; bool r = epee::file_io_utils::load_file_to_string(keys_file_name, buf); THROW_WALLET_EXCEPTION_IF(!r, error::file_read_error, keys_file_name); @@ -3144,19 +3331,50 @@ bool wallet2::verify_password(const std::string& keys_file_name, const epee::wip { account_data = std::string(json["key_data"].GetString(), json["key_data"].GetString() + json["key_data"].GetStringLength()); + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, encrypted_secret_keys, uint32_t, Uint, false, false); + encrypted_secret_keys = field_encrypted_secret_keys; } cryptonote::account_base account_data_check; r = epee::serialization::load_t_from_binary(account_data_check, account_data); - const cryptonote::account_keys& keys = account_data_check.get_keys(); + if (encrypted_secret_keys) + account_data_check.decrypt_keys(key); + + const cryptonote::account_keys& keys = account_data_check.get_keys(); r = r && hwdev.verify_keys(keys.m_view_secret_key, keys.m_account_address.m_view_public_key); if(!no_spend_key) r = r && hwdev.verify_keys(keys.m_spend_secret_key, keys.m_account_address.m_spend_public_key); return r; } +void wallet2::encrypt_keys(const crypto::chacha_key &key) +{ + m_account.encrypt_keys(key); + m_account.decrypt_viewkey(key); +} + +void wallet2::decrypt_keys(const crypto::chacha_key &key) +{ + m_account.encrypt_viewkey(key); + m_account.decrypt_keys(key); +} + +void wallet2::encrypt_keys(const epee::wipeable_string &password) +{ + crypto::chacha_key key; + crypto::generate_chacha_key(password.data(), password.size(), key, m_kdf_rounds); + encrypt_keys(key); +} + +void wallet2::decrypt_keys(const epee::wipeable_string &password) +{ + crypto::chacha_key key; + crypto::generate_chacha_key(password.data(), password.size(), key, m_kdf_rounds); + decrypt_keys(key); +} + /*! * \brief Generates a wallet or restores one. * \param wallet_ Name of wallet file @@ -3165,7 +3383,7 @@ bool wallet2::verify_password(const std::string& keys_file_name, const epee::wip * \param create_address_file Whether to create an address file */ void wallet2::generate(const std::string& wallet_, const epee::wipeable_string& password, - const std::string& multisig_data, bool create_address_file) + const epee::wipeable_string& multisig_data, bool create_address_file) { clear(); prepare_file_names(wallet_); @@ -3221,6 +3439,7 @@ void wallet2::generate(const std::string& wallet_, const epee::wipeable_string& for (const auto &msk: multisig_keys) sc_add(skey.bytes, skey.bytes, rct::sk2rct(msk).bytes); THROW_WALLET_EXCEPTION_IF(!(rct::rct2sk(skey) == spend_secret_key), error::invalid_multisig_seed); + memwipe(&skey, sizeof(rct::key)); m_account.make_multisig(view_secret_key, spend_secret_key, spend_public_key, multisig_keys); m_account.finalize_multisig(spend_public_key); @@ -3231,6 +3450,7 @@ void wallet2::generate(const std::string& wallet_, const epee::wipeable_string& m_multisig_threshold = threshold; m_multisig_signers = multisig_signers; m_key_on_device = false; + setup_keys(password); if (!wallet_.empty()) { @@ -3285,6 +3505,7 @@ crypto::secret_key wallet2::generate(const std::string& wallet_, const epee::wip m_multisig_threshold = 0; m_multisig_signers.clear(); m_key_on_device = false; + setup_keys(password); // calculate a starting refresh height if(m_refresh_from_block_height == 0 && !recover){ @@ -3386,6 +3607,7 @@ void wallet2::generate(const std::string& wallet_, const epee::wipeable_string& m_multisig_threshold = 0; m_multisig_signers.clear(); m_key_on_device = false; + setup_keys(password); if (!wallet_.empty()) { @@ -3439,6 +3661,7 @@ void wallet2::generate(const std::string& wallet_, const epee::wipeable_string& m_multisig_threshold = 0; m_multisig_signers.clear(); m_key_on_device = false; + setup_keys(password); if (!wallet_.empty()) { @@ -3485,6 +3708,7 @@ void wallet2::restore(const std::string& wallet_, const epee::wipeable_string& p m_multisig = false; m_multisig_threshold = 0; m_multisig_signers.clear(); + setup_keys(password); if (!wallet_.empty()) { bool r = store_keys(m_keys_file, password, false); @@ -3524,6 +3748,17 @@ std::string wallet2::make_multisig(const epee::wipeable_string &password, clear(); + // decrypt keys + epee::misc_utils::auto_scope_leave_caller keys_reencryptor; + if (m_ask_password && !m_rpc && !m_watch_only) + { + crypto::chacha_key chacha_key; + crypto::generate_chacha_key(password.data(), password.size(), chacha_key, m_kdf_rounds); + m_account.encrypt_viewkey(chacha_key); + m_account.decrypt_keys(chacha_key); + keys_reencryptor = epee::misc_utils::create_scope_leave_handler([&, this, chacha_key]() { m_account.encrypt_keys(chacha_key); m_account.decrypt_viewkey(chacha_key); }); + } + MINFO("Creating spend key..."); std::vector<crypto::secret_key> multisig_keys; rct::key spend_pkey, spend_skey; @@ -3567,6 +3802,7 @@ std::string wallet2::make_multisig(const epee::wipeable_string &password, MINFO("Creating multisig address..."); CHECK_AND_ASSERT_THROW_MES(m_account.make_multisig(view_skey, rct::rct2sk(spend_skey), rct::rct2pk(spend_pkey), multisig_keys), "Failed to create multisig wallet due to bad keys"); + memwipe(&spend_skey, sizeof(rct::key)); m_account_public_address = m_account.get_keys().m_account_address; m_watch_only = false; @@ -3584,6 +3820,9 @@ std::string wallet2::make_multisig(const epee::wipeable_string &password, m_multisig_signers = std::vector<crypto::public_key>(spend_keys.size() + 1, crypto::null_pkey); } + // re-encrypt keys + keys_reencryptor = epee::misc_utils::auto_scope_leave_caller(); + if (!m_wallet_file.empty()) { bool r = store_keys(m_keys_file, password, false); @@ -3666,6 +3905,17 @@ bool wallet2::finalize_multisig(const epee::wipeable_string &password, std::unor { CHECK_AND_ASSERT_THROW_MES(!pkeys.empty(), "empty pkeys"); + // keys are decrypted + epee::misc_utils::auto_scope_leave_caller keys_reencryptor; + if (m_ask_password && !m_rpc && !m_watch_only) + { + crypto::chacha_key chacha_key; + crypto::generate_chacha_key(password.data(), password.size(), chacha_key, m_kdf_rounds); + m_account.encrypt_viewkey(chacha_key); + m_account.decrypt_keys(chacha_key); + keys_reencryptor = epee::misc_utils::create_scope_leave_handler([&, this, chacha_key]() { m_account.encrypt_keys(chacha_key); m_account.decrypt_viewkey(chacha_key); }); + } + // add ours if not included crypto::public_key local_signer; CHECK_AND_ASSERT_THROW_MES(crypto::secret_key_to_public_key(get_account().get_keys().m_spend_secret_key, local_signer), @@ -3688,6 +3938,9 @@ bool wallet2::finalize_multisig(const epee::wipeable_string &password, std::unor m_multisig_signers = signers; std::sort(m_multisig_signers.begin(), m_multisig_signers.end(), [](const crypto::public_key &e0, const crypto::public_key &e1){ return memcmp(&e0, &e1, sizeof(e0)); }); + // keys are encrypted again + keys_reencryptor = epee::misc_utils::auto_scope_leave_caller(); + if (!m_wallet_file.empty()) { bool r = store_keys(m_keys_file, password, false); @@ -3999,6 +4252,11 @@ bool wallet2::generate_chacha_key_from_secret_keys(crypto::chacha_key &key) cons return hwdev.generate_chacha_key(m_account.get_keys(), key, m_kdf_rounds); } //---------------------------------------------------------------------------------------------------- +void wallet2::generate_chacha_key_from_password(const epee::wipeable_string &pass, crypto::chacha_key &key) const +{ + crypto::generate_chacha_key(pass.data(), pass.size(), key, m_kdf_rounds); +} +//---------------------------------------------------------------------------------------------------- void wallet2::load(const std::string& wallet_, const epee::wipeable_string& password) { clear(); @@ -4019,6 +4277,8 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass LOG_PRINT_L0("Loaded wallet keys file, with public address: " << m_account.get_public_address_str(m_nettype)); lock_keys_file(); + wallet_keys_unlocker unlocker(*this, m_ask_password && !m_rpc && !m_watch_only, password); + //keys loaded ok! //try to load wallet file. but even if we failed, it is not big problem if(!boost::filesystem::exists(m_wallet_file, e) || e) @@ -4040,11 +4300,9 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass r = ::serialization::parse_binary(buf, cache_file_data); THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "internal error: failed to deserialize \"" + m_wallet_file + '\"'); - crypto::chacha_key key; - generate_chacha_key_from_secret_keys(key); std::string cache_data; cache_data.resize(cache_file_data.cache_data.size()); - crypto::chacha20(cache_file_data.cache_data.data(), cache_file_data.cache_data.size(), key, cache_file_data.iv, &cache_data[0]); + crypto::chacha20(cache_file_data.cache_data.data(), cache_file_data.cache_data.size(), m_cache_key, cache_file_data.iv, &cache_data[0]); try { std::stringstream iss; @@ -4052,11 +4310,13 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass boost::archive::portable_binary_iarchive ar(iss); ar >> *this; } - catch (...) + catch(...) { - crypto::chacha8(cache_file_data.cache_data.data(), cache_file_data.cache_data.size(), key, cache_file_data.iv, &cache_data[0]); - try - { + // try with previous scheme: direct from keys + crypto::chacha_key key; + generate_chacha_key_from_secret_keys(key); + crypto::chacha20(cache_file_data.cache_data.data(), cache_file_data.cache_data.size(), key, cache_file_data.iv, &cache_data[0]); + try { std::stringstream iss; iss << cache_data; boost::archive::portable_binary_iarchive ar(iss); @@ -4064,13 +4324,24 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass } catch (...) { - LOG_PRINT_L0("Failed to open portable binary, trying unportable"); - boost::filesystem::copy_file(m_wallet_file, m_wallet_file + ".unportable", boost::filesystem::copy_option::overwrite_if_exists); - std::stringstream iss; - iss.str(""); - iss << cache_data; - boost::archive::binary_iarchive ar(iss); - ar >> *this; + crypto::chacha8(cache_file_data.cache_data.data(), cache_file_data.cache_data.size(), key, cache_file_data.iv, &cache_data[0]); + try + { + std::stringstream iss; + iss << cache_data; + boost::archive::portable_binary_iarchive ar(iss); + ar >> *this; + } + catch (...) + { + LOG_PRINT_L0("Failed to open portable binary, trying unportable"); + boost::filesystem::copy_file(m_wallet_file, m_wallet_file + ".unportable", boost::filesystem::copy_option::overwrite_if_exists); + std::stringstream iss; + iss.str(""); + iss << cache_data; + boost::archive::binary_iarchive ar(iss); + ar >> *this; + } } } } @@ -4222,12 +4493,10 @@ void wallet2::store_to(const std::string &path, const epee::wipeable_string &pas wallet2::cache_file_data cache_file_data = boost::value_initialized<wallet2::cache_file_data>(); cache_file_data.cache_data = oss.str(); - crypto::chacha_key key; - generate_chacha_key_from_secret_keys(key); std::string cipher; cipher.resize(cache_file_data.cache_data.size()); cache_file_data.iv = crypto::rand<crypto::chacha_iv>(); - crypto::chacha20(cache_file_data.cache_data.data(), cache_file_data.cache_data.size(), key, cache_file_data.iv, &cipher[0]); + crypto::chacha20(cache_file_data.cache_data.data(), cache_file_data.cache_data.size(), m_cache_key, cache_file_data.iv, &cipher[0]); cache_file_data.cache_data = cipher; const std::string new_file = same_file ? m_wallet_file + ".new" : path; @@ -4685,7 +4954,7 @@ size_t wallet2::pop_best_value(std::vector<size_t> &unused_indices, const std::v // returns: // direct return: amount of money found // modified reference: selected_transfers, a list of iterators/indices of input sources -uint64_t wallet2::select_transfers(uint64_t needed_money, std::vector<size_t> unused_transfers_indices, std::vector<size_t>& selected_transfers, bool trusted_daemon) const +uint64_t wallet2::select_transfers(uint64_t needed_money, std::vector<size_t> unused_transfers_indices, std::vector<size_t>& selected_transfers) const { uint64_t found_money = 0; selected_transfers.reserve(unused_transfers_indices.size()); @@ -4729,17 +4998,17 @@ void wallet2::add_unconfirmed_tx(const cryptonote::transaction& tx, uint64_t amo //---------------------------------------------------------------------------------------------------- void wallet2::transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outs_count, const std::vector<size_t> &unused_transfers_indices, - uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx& ptx, bool trusted_daemon) + uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx& ptx) { - transfer(dsts, fake_outs_count, unused_transfers_indices, unlock_time, fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), tx, ptx, trusted_daemon); + transfer(dsts, fake_outs_count, unused_transfers_indices, unlock_time, fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), tx, ptx); } //---------------------------------------------------------------------------------------------------- void wallet2::transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outs_count, const std::vector<size_t> &unused_transfers_indices, - uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, bool trusted_daemon) + uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra) { cryptonote::transaction tx; pending_tx ptx; - transfer(dsts, fake_outs_count, unused_transfers_indices, unlock_time, fee, extra, tx, ptx, trusted_daemon); + transfer(dsts, fake_outs_count, unused_transfers_indices, unlock_time, fee, extra, tx, ptx); } namespace { @@ -5049,10 +5318,18 @@ bool wallet2::sign_tx(unsigned_tx_set &exported_txs, std::vector<wallet2::pendin LOG_PRINT_L1(" " << (n+1) << ": " << sd.sources.size() << " inputs, ring size " << sd.sources[0].outputs.size()); signed_txes.ptx.push_back(pending_tx()); tools::wallet2::pending_tx &ptx = signed_txes.ptx.back(); + rct::RangeProofType range_proof_type = rct::RangeProofBorromean; + if (sd.use_bulletproofs) + { + range_proof_type = rct::RangeProofBulletproof; + for (const rct::Bulletproof &proof: ptx.tx.rct_signatures.p.bulletproofs) + if (proof.V.size() > 1) + range_proof_type = rct::RangeProofPaddedBulletproof; + } crypto::secret_key tx_key; std::vector<crypto::secret_key> additional_tx_keys; rct::multisig_out msout; - bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sd.sources, sd.splitted_dsts, sd.change_dts.addr, sd.extra, ptx.tx, sd.unlock_time, tx_key, additional_tx_keys, sd.use_rct, sd.use_bulletproofs, m_multisig ? &msout : NULL); + bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sd.sources, sd.splitted_dsts, sd.change_dts.addr, sd.extra, ptx.tx, sd.unlock_time, tx_key, additional_tx_keys, sd.use_rct, range_proof_type, m_multisig ? &msout : NULL); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sd.sources, sd.splitted_dsts, sd.unlock_time, m_nettype); // we don't test tx size, because we don't know the current limit, due to not having a blockchain, // and it's a bit pointless to fail there anyway, since it'd be a (good) guess only. We sign anyway, @@ -5465,7 +5742,15 @@ bool wallet2::sign_multisig_tx(multisig_tx_set &exported_txs, std::vector<crypto cryptonote::transaction tx; rct::multisig_out msout = ptx.multisig_sigs.front().msout; auto sources = sd.sources; - bool r = cryptonote::construct_tx_with_tx_key(m_account.get_keys(), m_subaddresses, sources, sd.splitted_dsts, ptx.change_dts.addr, sd.extra, tx, sd.unlock_time, ptx.tx_key, ptx.additional_tx_keys, sd.use_rct, sd.use_bulletproofs, &msout, false); + rct::RangeProofType range_proof_type = rct::RangeProofBorromean; + if (sd.use_bulletproofs) + { + range_proof_type = rct::RangeProofBulletproof; + for (const rct::Bulletproof &proof: ptx.tx.rct_signatures.p.bulletproofs) + if (proof.V.size() > 1) + range_proof_type = rct::RangeProofPaddedBulletproof; + } + bool r = cryptonote::construct_tx_with_tx_key(m_account.get_keys(), m_subaddresses, sources, sd.splitted_dsts, ptx.change_dts.addr, sd.extra, tx, sd.unlock_time, ptx.tx_key, ptx.additional_tx_keys, sd.use_rct, range_proof_type, &msout, false); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sd.sources, sd.splitted_dsts, sd.unlock_time, m_nettype); THROW_WALLET_EXCEPTION_IF(get_transaction_prefix_hash (tx) != get_transaction_prefix_hash(ptx.tx), @@ -5565,9 +5850,18 @@ bool wallet2::sign_multisig_tx_from_file(const std::string &filename, std::vecto //---------------------------------------------------------------------------------------------------- uint64_t wallet2::get_fee_multiplier(uint32_t priority, int fee_algorithm) const { - static const uint64_t old_multipliers[3] = {1, 2, 3}; - static const uint64_t new_multipliers[3] = {1, 20, 166}; - static const uint64_t newer_multipliers[4] = {1, 4, 20, 166}; + static const struct + { + size_t count; + uint64_t multipliers[4]; + } + multipliers[] = + { + { 3, {1, 2, 3} }, + { 3, {1, 20, 166} }, + { 4, {1, 4, 20, 166} }, + { 4, {1, 5, 25, 1000} }, + }; if (fee_algorithm == -1) fee_algorithm = get_fee_algorithm(); @@ -5583,47 +5877,68 @@ uint64_t wallet2::get_fee_multiplier(uint32_t priority, int fee_algorithm) const priority = 1; } + THROW_WALLET_EXCEPTION_IF(fee_algorithm < 0 || fee_algorithm > 3, error::invalid_priority); + // 1 to 3/4 are allowed as priorities - uint32_t max_priority = (fee_algorithm >= 2) ? 4 : 3; + const uint32_t max_priority = multipliers[fee_algorithm].count; if (priority >= 1 && priority <= max_priority) { - switch (fee_algorithm) - { - case 0: return old_multipliers[priority-1]; - case 1: return new_multipliers[priority-1]; - case 2: return newer_multipliers[priority-1]; - default: THROW_WALLET_EXCEPTION_IF (true, error::invalid_priority); - } + return multipliers[fee_algorithm].multipliers[priority-1]; } THROW_WALLET_EXCEPTION_IF (false, error::invalid_priority); return 1; } //---------------------------------------------------------------------------------------------------- -uint64_t wallet2::get_dynamic_per_kb_fee_estimate() const +uint64_t wallet2::get_dynamic_base_fee_estimate() const { uint64_t fee; - boost::optional<std::string> result = m_node_rpc_proxy.get_dynamic_per_kb_fee_estimate(FEE_ESTIMATE_GRACE_BLOCKS, fee); + boost::optional<std::string> result = m_node_rpc_proxy.get_dynamic_base_fee_estimate(FEE_ESTIMATE_GRACE_BLOCKS, fee); if (!result) return fee; - LOG_PRINT_L1("Failed to query per kB fee, using " << print_money(FEE_PER_KB)); - return FEE_PER_KB; + const uint64_t base_fee = use_fork_rules(HF_VERSION_PER_BYTE_FEE) ? FEE_PER_BYTE : FEE_PER_KB; + LOG_PRINT_L1("Failed to query base fee, using " << print_money(base_fee)); + return base_fee; } //---------------------------------------------------------------------------------------------------- -uint64_t wallet2::get_per_kb_fee() const +uint64_t wallet2::get_base_fee() const { if(m_light_wallet) - return m_light_wallet_per_kb_fee; + { + if (use_fork_rules(HF_VERSION_PER_BYTE_FEE)) + return m_light_wallet_per_kb_fee / 1024; + else + return m_light_wallet_per_kb_fee; + } bool use_dyn_fee = use_fork_rules(HF_VERSION_DYNAMIC_FEE, -720 * 1); if (!use_dyn_fee) return FEE_PER_KB; - return get_dynamic_per_kb_fee_estimate(); + return get_dynamic_base_fee_estimate(); +} +//---------------------------------------------------------------------------------------------------- +uint64_t wallet2::get_fee_quantization_mask() const +{ + if(m_light_wallet) + { + return 1; // TODO + } + bool use_per_byte_fee = use_fork_rules(HF_VERSION_PER_BYTE_FEE, 0); + if (!use_per_byte_fee) + return 1; + + uint64_t fee_quantization_mask; + boost::optional<std::string> result = m_node_rpc_proxy.get_fee_quantization_mask(fee_quantization_mask); + if (result) + return 1; + return fee_quantization_mask; } //---------------------------------------------------------------------------------------------------- int wallet2::get_fee_algorithm() const { - // changes at v3 and v5 + // changes at v3, v5, v8 + if (use_fork_rules(HF_VERSION_PER_BYTE_FEE, 0)) + return 3; if (use_fork_rules(5, 0)) return 2; if (use_fork_rules(3, -720 * 14)) @@ -5631,19 +5946,39 @@ int wallet2::get_fee_algorithm() const return 0; } //------------------------------------------------------------------------------------------------------------------------------ +uint64_t wallet2::get_min_ring_size() const +{ + if (use_fork_rules(8, 10)) + return 11; + if (use_fork_rules(7, 10)) + return 7; + if (use_fork_rules(6, 10)) + return 5; + if (use_fork_rules(2, 10)) + return 3; + return 0; +} +//------------------------------------------------------------------------------------------------------------------------------ +uint64_t wallet2::get_max_ring_size() const +{ + if (use_fork_rules(8, 10)) + return 11; + return 0; +} +//------------------------------------------------------------------------------------------------------------------------------ uint64_t wallet2::adjust_mixin(uint64_t mixin) const { - if (mixin < 6 && use_fork_rules(7, 10)) { - MWARNING("Requested ring size " << (mixin + 1) << " too low for hard fork 7, using 7"); - mixin = 6; - } - else if (mixin < 4 && use_fork_rules(6, 10)) { - MWARNING("Requested ring size " << (mixin + 1) << " too low for hard fork 6, using 5"); - mixin = 4; + const uint64_t min_ring_size = get_min_ring_size(); + if (mixin + 1 < min_ring_size) + { + MWARNING("Requested ring size " << (mixin + 1) << " too low, using " << min_ring_size); + mixin = min_ring_size-1; } - else if (mixin < 2 && use_fork_rules(2, 10)) { - MWARNING("Requested ring size " << (mixin + 1) << " too low for hard fork 2, using 3"); - mixin = 2; + const uint64_t max_ring_size = get_max_ring_size(); + if (max_ring_size && mixin + 1 > max_ring_size) + { + MWARNING("Requested ring size " << (mixin + 1) << " too high, using " << max_ring_size); + mixin = max_ring_size-1; } return mixin; } @@ -5655,7 +5990,10 @@ uint32_t wallet2::adjust_priority(uint32_t priority) try { // check if there's a backlog in the tx pool - const double fee_level = get_fee_multiplier(1) * get_per_kb_fee() * (12/(double)13) / (double)1024; + const bool use_per_byte_fee = use_fork_rules(HF_VERSION_PER_BYTE_FEE, 0); + const uint64_t base_fee = get_base_fee(); + const uint64_t fee_multiplier = get_fee_multiplier(1); + const double fee_level = fee_multiplier * base_fee * (use_per_byte_fee ? 1 : (12/(double)13 / (double)1024)); const std::vector<std::pair<uint64_t, uint64_t>> blocks = estimate_backlog({std::make_pair(fee_level, fee_level)}); if (blocks.size() != 1) { @@ -5669,10 +6007,10 @@ uint32_t wallet2::adjust_priority(uint32_t priority) } // get the current full reward zone - uint64_t block_size_limit = 0; - const auto result = m_node_rpc_proxy.get_block_size_limit(block_size_limit); + uint64_t block_weight_limit = 0; + const auto result = m_node_rpc_proxy.get_block_weight_limit(block_weight_limit); throw_on_rpc_response_error(result, "get_info"); - const uint64_t full_reward_zone = block_size_limit / 2; + const uint64_t full_reward_zone = block_weight_limit / 2; // get the last N block headers and sum the block sizes const size_t N = 10; @@ -5696,14 +6034,14 @@ uint32_t wallet2::adjust_priority(uint32_t priority) MERROR("Bad blockheaders size"); return priority; } - size_t block_size_sum = 0; + size_t block_weight_sum = 0; for (const cryptonote::block_header_response &i : getbh_res.headers) { - block_size_sum += i.block_size; + block_weight_sum += i.block_weight; } // estimate how 'full' the last N blocks are - const size_t P = 100 * block_size_sum / (N * full_reward_zone); + const size_t P = 100 * block_weight_sum / (N * full_reward_zone); MINFO((boost::format("The last %d blocks fill roughly %d%% of the full reward zone.") % N % P).str()); if (P > 80) { @@ -5725,12 +6063,14 @@ uint32_t wallet2::adjust_priority(uint32_t priority) // // this function will make multiple calls to wallet2::transfer if multiple // transactions will be required -std::vector<wallet2::pending_tx> wallet2::create_transactions(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, bool trusted_daemon) +std::vector<wallet2::pending_tx> wallet2::create_transactions(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra) { - const std::vector<size_t> unused_transfers_indices = select_available_outputs_from_histogram(fake_outs_count + 1, true, true, true, trusted_daemon); + const std::vector<size_t> unused_transfers_indices = select_available_outputs_from_histogram(fake_outs_count + 1, true, true, true); - const uint64_t fee_per_kb = get_per_kb_fee(); + const uint64_t base_fee = get_base_fee(); const uint64_t fee_multiplier = get_fee_multiplier(priority, get_fee_algorithm()); + const bool use_per_byte_fee = use_fork_rules(HF_VERSION_PER_BYTE_FEE); + const uint64_t fee_quantization_mask = get_fee_quantization_mask(); // failsafe split attempt counter size_t attempt_count = 0; @@ -5757,13 +6097,12 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions(std::vector<crypto pending_tx ptx; // loop until fee is met without increasing tx size to next KB boundary. - const size_t estimated_tx_size = estimate_tx_size(false, unused_transfers_indices.size(), fake_outs_count, dst_vector.size(), extra.size(), false); - uint64_t needed_fee = calculate_fee(fee_per_kb, estimated_tx_size, fee_multiplier); + uint64_t needed_fee = estimate_fee(use_per_byte_fee, false, unused_transfers_indices.size(), fake_outs_count, dst_vector.size()+1, extra.size(), false, base_fee, fee_multiplier, fee_quantization_mask); do { - transfer(dst_vector, fake_outs_count, unused_transfers_indices, unlock_time, needed_fee, extra, tx, ptx, trusted_daemon); + transfer(dst_vector, fake_outs_count, unused_transfers_indices, unlock_time, needed_fee, extra, tx, ptx); auto txBlob = t_serializable_object_to_blob(ptx.tx); - needed_fee = calculate_fee(fee_per_kb, txBlob, fee_multiplier); + needed_fee = calculate_fee(use_per_byte_fee, ptx.tx, txBlob.size(), base_fee, fee_multiplier, fee_quantization_mask); } while (ptx.fee < needed_fee); ptx_vector.push_back(ptx); @@ -5865,12 +6204,6 @@ crypto::chacha_key wallet2::get_ringdb_key() return *m_ringdb_key; } -void wallet2::clear_ringdb_key() -{ - MINFO("clearing ringdb key"); - m_ringdb_key = boost::none; -} - bool wallet2::add_rings(const crypto::chacha_key &key, const cryptonote::transaction_prefix &tx) { if (!m_ringdb) @@ -5881,7 +6214,6 @@ bool wallet2::add_rings(const crypto::chacha_key &key, const cryptonote::transac bool wallet2::add_rings(const cryptonote::transaction_prefix &tx) { - key_ref kref(*this); try { return add_rings(get_ringdb_key(), tx); } catch (const std::exception &e) { return false; } } @@ -5890,7 +6222,6 @@ bool wallet2::remove_rings(const cryptonote::transaction_prefix &tx) { if (!m_ringdb) return false; - key_ref kref(*this); try { return m_ringdb->remove_rings(get_ringdb_key(), tx); } catch (const std::exception &e) { return false; } } @@ -5928,7 +6259,6 @@ bool wallet2::get_rings(const crypto::hash &txid, std::vector<std::pair<crypto:: bool wallet2::get_ring(const crypto::key_image &key_image, std::vector<uint64_t> &outs) { - key_ref kref(*this); try { return get_ring(get_ringdb_key(), key_image, outs); } catch (const std::exception &e) { return false; } } @@ -5938,7 +6268,6 @@ bool wallet2::set_ring(const crypto::key_image &key_image, const std::vector<uin if (!m_ringdb) return false; - key_ref kref(*this); try { return m_ringdb->set_ring(get_ringdb_key(), key_image, outs, relative); } catch (const std::exception &e) { return false; } } @@ -5950,7 +6279,6 @@ bool wallet2::find_and_save_rings(bool force) if (!m_ringdb) return false; - key_ref kref(*this); COMMAND_RPC_GET_TRANSACTIONS::request req = AUTO_VAL_INIT(req); COMMAND_RPC_GET_TRANSACTIONS::response res = AUTO_VAL_INIT(res); @@ -6271,6 +6599,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> req_t.from_height = std::max<uint64_t>(segregation_fork_height, RECENT_OUTPUT_BLOCKS) - RECENT_OUTPUT_BLOCKS; req_t.to_height = segregation_fork_height + 1; req_t.cumulative = true; + req_t.binary = true; m_daemon_rpc_mutex.lock(); bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_distribution", req_t, resp_t, m_http_client, rpc_timeout * 1000); m_daemon_rpc_mutex.unlock(); @@ -6743,7 +7072,7 @@ void wallet2::transfer_selected(const std::vector<cryptonote::tx_destination_ent THROW_WALLET_EXCEPTION_IF(m_multisig, error::wallet_internal_error, "Multisig wallets cannot spend non rct outputs"); - uint64_t upper_transaction_size_limit = get_upper_transaction_size_limit(); + uint64_t upper_transaction_weight_limit = get_upper_transaction_weight_limit(); uint64_t needed_money = fee; LOG_PRINT_L2("transfer: starting with fee " << print_money (needed_money)); @@ -6845,10 +7174,10 @@ void wallet2::transfer_selected(const std::vector<cryptonote::tx_destination_ent std::vector<crypto::secret_key> additional_tx_keys; rct::multisig_out msout; LOG_PRINT_L2("constructing tx"); - bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sources, splitted_dsts, change_dts.addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, false, m_multisig ? &msout : NULL); + bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sources, splitted_dsts, change_dts.addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, rct::RangeProofBulletproof, m_multisig ? &msout : NULL); LOG_PRINT_L2("constructed tx, r="<<r); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sources, splitted_dsts, unlock_time, m_nettype); - THROW_WALLET_EXCEPTION_IF(upper_transaction_size_limit <= get_object_blobsize(tx), error::tx_too_big, tx, upper_transaction_size_limit); + THROW_WALLET_EXCEPTION_IF(upper_transaction_weight_limit <= get_transaction_weight(tx), error::tx_too_big, tx, upper_transaction_weight_limit); std::string key_images; bool all_are_txin_to_key = std::all_of(tx.vin.begin(), tx.vin.end(), [&](const txin_v& s_e) -> bool @@ -6894,13 +7223,13 @@ void wallet2::transfer_selected(const std::vector<cryptonote::tx_destination_ent void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry> dsts, const std::vector<size_t>& selected_transfers, size_t fake_outputs_count, std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, - uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx &ptx, bool bulletproof) + uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx &ptx, rct::RangeProofType range_proof_type) { using namespace cryptonote; // throw if attempting a transaction with no destinations THROW_WALLET_EXCEPTION_IF(dsts.empty(), error::zero_destination); - uint64_t upper_transaction_size_limit = get_upper_transaction_size_limit(); + uint64_t upper_transaction_weight_limit = get_upper_transaction_weight_limit(); uint64_t needed_money = fee; LOG_PRINT_L2("transfer_selected_rct: starting with fee " << print_money (needed_money)); LOG_PRINT_L2("selected transfers: " << strjoin(selected_transfers, " ")); @@ -7050,10 +7379,10 @@ void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry rct::multisig_out msout; LOG_PRINT_L2("constructing tx"); auto sources_copy = sources; - bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sources, splitted_dsts, change_dts.addr, extra, tx, unlock_time, tx_key, additional_tx_keys, true, bulletproof, m_multisig ? &msout : NULL); + bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sources, splitted_dsts, change_dts.addr, extra, tx, unlock_time, tx_key, additional_tx_keys, true, range_proof_type, m_multisig ? &msout : NULL); LOG_PRINT_L2("constructed tx, r="<<r); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sources, dsts, unlock_time, m_nettype); - THROW_WALLET_EXCEPTION_IF(upper_transaction_size_limit <= get_object_blobsize(tx), error::tx_too_big, tx, upper_transaction_size_limit); + THROW_WALLET_EXCEPTION_IF(upper_transaction_weight_limit <= get_transaction_weight(tx), error::tx_too_big, tx, upper_transaction_weight_limit); // work out the permutation done on sources std::vector<size_t> ins_order; @@ -7095,10 +7424,10 @@ void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry LOG_PRINT_L2("Creating supplementary multisig transaction"); cryptonote::transaction ms_tx; auto sources_copy_copy = sources_copy; - bool r = cryptonote::construct_tx_with_tx_key(m_account.get_keys(), m_subaddresses, sources_copy_copy, splitted_dsts, change_dts.addr, extra, ms_tx, unlock_time,tx_key, additional_tx_keys, true, bulletproof, &msout, false); + bool r = cryptonote::construct_tx_with_tx_key(m_account.get_keys(), m_subaddresses, sources_copy_copy, splitted_dsts, change_dts.addr, extra, ms_tx, unlock_time,tx_key, additional_tx_keys, true, range_proof_type, &msout, false); LOG_PRINT_L2("constructed tx, r="<<r); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sources, splitted_dsts, unlock_time, m_nettype); - THROW_WALLET_EXCEPTION_IF(upper_transaction_size_limit <= get_object_blobsize(tx), error::tx_too_big, tx, upper_transaction_size_limit); + THROW_WALLET_EXCEPTION_IF(upper_transaction_weight_limit <= get_transaction_weight(tx), error::tx_too_big, tx, upper_transaction_weight_limit); THROW_WALLET_EXCEPTION_IF(cryptonote::get_transaction_prefix_hash(ms_tx) != prefix_hash, error::wallet_internal_error, "Multisig txes do not share prefix"); multisig_sigs.push_back({ms_tx.rct_signatures, multisig_signers[signer_index], new_used_L, std::unordered_set<crypto::public_key>(), msout}); @@ -7746,7 +8075,7 @@ bool wallet2::light_wallet_key_image_is_ours(const crypto::key_image& key_image, // This system allows for sending (almost) the entire balance, since it does // not generate spurious change in all txes, thus decreasing the instantaneous // usable balance. -std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices, bool trusted_daemon) +std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices) { //ensure device is let in NONE mode in any case hw::device &hwdev = m_account.get_device(); @@ -7766,11 +8095,11 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp std::vector<cryptonote::tx_destination_entry> dsts; cryptonote::transaction tx; pending_tx ptx; - size_t bytes; + size_t weight; uint64_t needed_fee; std::vector<std::vector<tools::wallet2::get_outs_entry>> outs; - TX() : bytes(0), needed_fee(0) {} + TX() : weight(0), needed_fee(0) {} void add(const account_public_address &addr, bool is_subaddress, uint64_t amount, unsigned int original_output_index, bool merge_destinations) { if (merge_destinations) @@ -7798,12 +8127,15 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp std::vector<TX> txes; bool adding_fee; // true if new outputs go towards fee, rather than destinations uint64_t needed_fee, available_for_fee = 0; - uint64_t upper_transaction_size_limit = get_upper_transaction_size_limit(); + uint64_t upper_transaction_weight_limit = get_upper_transaction_weight_limit(); + const bool use_per_byte_fee = use_fork_rules(HF_VERSION_PER_BYTE_FEE, 0); const bool use_rct = use_fork_rules(4, 0); const bool bulletproof = use_fork_rules(get_bulletproof_fork(), 0); + const rct::RangeProofType range_proof_type = bulletproof ? rct::RangeProofPaddedBulletproof : rct::RangeProofBorromean; - const uint64_t fee_per_kb = get_per_kb_fee(); + const uint64_t base_fee = get_base_fee(); const uint64_t fee_multiplier = get_fee_multiplier(priority, get_fee_algorithm()); + const uint64_t fee_quantization_mask = get_fee_quantization_mask(); // throw if attempting a transaction with no destinations THROW_WALLET_EXCEPTION_IF(dsts.empty(), error::zero_destination); @@ -7834,7 +8166,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp // early out if we know we can't make it anyway // we could also check for being within FEE_PER_KB, but if the fee calculation // ever changes, this might be missed, so let this go through - const uint64_t min_fee = (fee_multiplier * fee_per_kb * estimate_tx_size(use_rct, 1, fake_outs_count, 2, extra.size(), bulletproof)) / 1024; + const uint64_t min_fee = (fee_multiplier * base_fee * estimate_tx_size(use_rct, 1, fake_outs_count, 2, extra.size(), bulletproof)) / 1024; uint64_t balance_subtotal = 0; uint64_t unlocked_balance_subtotal = 0; for (uint32_t index_minor : subaddr_indices) @@ -7852,11 +8184,11 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp LOG_PRINT_L2("Candidate subaddress index for spending: " << i); // determine threshold for fractional amount - const size_t tx_size_one_ring = estimate_tx_size(use_rct, 1, fake_outs_count, 2, 0, bulletproof); - const size_t tx_size_two_rings = estimate_tx_size(use_rct, 2, fake_outs_count, 2, 0, bulletproof); - THROW_WALLET_EXCEPTION_IF(tx_size_one_ring > tx_size_two_rings, error::wallet_internal_error, "Estimated tx size with 1 input is larger than with 2 inputs!"); - const size_t tx_size_per_ring = tx_size_two_rings - tx_size_one_ring; - const uint64_t fractional_threshold = (fee_multiplier * fee_per_kb * tx_size_per_ring) / 1024; + const size_t tx_weight_one_ring = estimate_tx_weight(use_rct, 1, fake_outs_count, 2, 0, bulletproof); + const size_t tx_weight_two_rings = estimate_tx_weight(use_rct, 2, fake_outs_count, 2, 0, bulletproof); + THROW_WALLET_EXCEPTION_IF(tx_weight_one_ring > tx_weight_two_rings, error::wallet_internal_error, "Estimated tx weight with 1 input is larger than with 2 inputs!"); + const size_t tx_weight_per_ring = tx_weight_two_rings - tx_weight_one_ring; + const uint64_t fractional_threshold = (fee_multiplier * base_fee * tx_weight_per_ring) / (use_per_byte_fee ? 1 : 1024); // gather all dust and non-dust outputs belonging to specified subaddresses size_t num_nondust_outputs = 0; @@ -7949,7 +8281,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp { // this is used to build a tx that's 1 or 2 inputs, and 2 outputs, which // will get us a known fee. - uint64_t estimated_fee = calculate_fee(fee_per_kb, estimate_rct_tx_size(2, fake_outs_count, 2, extra.size(), bulletproof), fee_multiplier); + uint64_t estimated_fee = estimate_fee(use_per_byte_fee, use_rct, 2, fake_outs_count, 2, extra.size(), bulletproof, base_fee, fee_multiplier, fee_quantization_mask); preferred_inputs = pick_preferred_rct_inputs(needed_money + estimated_fee, subaddr_account, subaddr_indices); if (!preferred_inputs.empty()) { @@ -7991,7 +8323,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp while ((!dsts.empty() && dsts[0].amount > 0) || adding_fee || !preferred_inputs.empty() || should_pick_a_second_output(use_rct, txes.back().selected_transfers.size(), *unused_transfers_indices, *unused_dust_indices)) { TX &tx = txes.back(); - LOG_PRINT_L2("Start of loop with " << unused_transfers_indices->size() << " " << unused_dust_indices->size()); + LOG_PRINT_L2("Start of loop with " << unused_transfers_indices->size() << " " << unused_dust_indices->size() << ", tx.dsts.size() " << tx.dsts.size()); LOG_PRINT_L2("unused_transfers_indices: " << strjoin(*unused_transfers_indices, " ")); LOG_PRINT_L2("unused_dust_indices: " << strjoin(*unused_dust_indices, " ")); LOG_PRINT_L2("dsts size " << dsts.size() << ", first " << (dsts.empty() ? "-" : cryptonote::print_money(dsts[0].amount))); @@ -8054,7 +8386,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp } else { - while (!dsts.empty() && dsts[0].amount <= available_amount && estimate_tx_size(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size(), extra.size(), bulletproof) < TX_SIZE_TARGET(upper_transaction_size_limit)) + while (!dsts.empty() && dsts[0].amount <= available_amount && estimate_tx_weight(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size()+1, extra.size(), bulletproof) < TX_WEIGHT_TARGET(upper_transaction_weight_limit)) { // we can fully pay that destination LOG_PRINT_L2("We can fully pay " << get_account_address_as_str(m_nettype, dsts[0].is_subaddress, dsts[0].addr) << @@ -8066,7 +8398,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp ++original_output_index; } - if (available_amount > 0 && !dsts.empty() && estimate_tx_size(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size(), extra.size(), bulletproof) < TX_SIZE_TARGET(upper_transaction_size_limit)) { + if (available_amount > 0 && !dsts.empty() && estimate_tx_weight(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size()+1, extra.size(), bulletproof) < TX_WEIGHT_TARGET(upper_transaction_weight_limit)) { // we can partially fill that destination LOG_PRINT_L2("We can partially pay " << get_account_address_as_str(m_nettype, dsts[0].is_subaddress, dsts[0].addr) << " for " << print_money(available_amount) << "/" << print_money(dsts[0].amount)); @@ -8078,7 +8410,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp // here, check if we need to sent tx and start a new one LOG_PRINT_L2("Considering whether to create a tx now, " << tx.selected_transfers.size() << " inputs, tx limit " - << upper_transaction_size_limit); + << upper_transaction_weight_limit); bool try_tx = false; // if we have preferred picks, but haven't yet used all of them, continue if (preferred_inputs.empty()) @@ -8090,8 +8422,8 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp } else { - const size_t estimated_rct_tx_size = estimate_tx_size(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size(), extra.size(), bulletproof); - try_tx = dsts.empty() || (estimated_rct_tx_size >= TX_SIZE_TARGET(upper_transaction_size_limit)); + const size_t estimated_rct_tx_weight = estimate_tx_weight(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size()+1, extra.size(), bulletproof); + try_tx = dsts.empty() || (estimated_rct_tx_weight >= TX_WEIGHT_TARGET(upper_transaction_weight_limit)); } } @@ -8099,8 +8431,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp cryptonote::transaction test_tx; pending_tx test_ptx; - const size_t estimated_tx_size = estimate_tx_size(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size(), extra.size(), bulletproof); - needed_fee = calculate_fee(fee_per_kb, estimated_tx_size, fee_multiplier); + needed_fee = estimate_fee(use_per_byte_fee, use_rct ,tx.selected_transfers.size(), fake_outs_count, tx.dsts.size()+1, extra.size(), bulletproof, base_fee, fee_multiplier, fee_multiplier); uint64_t inputs = 0, outputs = needed_fee; for (size_t idx: tx.selected_transfers) inputs += m_transfers[idx].amount(); @@ -8117,14 +8448,14 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp tx.selected_transfers.size() << " inputs"); if (use_rct) transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, - test_tx, test_ptx, bulletproof); + test_tx, test_ptx, range_proof_type); else transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); auto txBlob = t_serializable_object_to_blob(test_ptx.tx); - needed_fee = calculate_fee(fee_per_kb, txBlob, fee_multiplier); + needed_fee = calculate_fee(use_per_byte_fee, test_ptx.tx, txBlob.size(), base_fee, fee_multiplier, fee_quantization_mask); available_for_fee = test_ptx.fee + test_ptx.change_dts.amount + (!test_ptx.dust_added_to_fee ? test_ptx.dust : 0); - LOG_PRINT_L2("Made a " << get_size_string(txBlob) << " tx, with " << print_money(available_for_fee) << " available for fee (" << + LOG_PRINT_L2("Made a " << get_weight_string(test_ptx.tx, txBlob.size()) << " tx, with " << print_money(available_for_fee) << " available for fee (" << print_money(needed_fee) << " needed)"); if (needed_fee > available_for_fee && !dsts.empty() && dsts[0].amount > 0) @@ -8160,22 +8491,22 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp while (needed_fee > test_ptx.fee) { if (use_rct) transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, - test_tx, test_ptx, bulletproof); + test_tx, test_ptx, range_proof_type); else transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); txBlob = t_serializable_object_to_blob(test_ptx.tx); - needed_fee = calculate_fee(fee_per_kb, txBlob, fee_multiplier); - LOG_PRINT_L2("Made an attempt at a final " << get_size_string(txBlob) << " tx, with " << print_money(test_ptx.fee) << + needed_fee = calculate_fee(use_per_byte_fee, test_ptx.tx, txBlob.size(), base_fee, fee_multiplier, fee_quantization_mask); + LOG_PRINT_L2("Made an attempt at a final " << get_weight_string(test_ptx.tx, txBlob.size()) << " tx, with " << print_money(test_ptx.fee) << " fee and " << print_money(test_ptx.change_dts.amount) << " change"); } - LOG_PRINT_L2("Made a final " << get_size_string(txBlob) << " tx, with " << print_money(test_ptx.fee) << + LOG_PRINT_L2("Made a final " << get_weight_string(test_ptx.tx, txBlob.size()) << " tx, with " << print_money(test_ptx.fee) << " fee and " << print_money(test_ptx.change_dts.amount) << " change"); tx.tx = test_tx; tx.ptx = test_ptx; - tx.bytes = txBlob.size(); + tx.weight = get_transaction_weight(test_tx, txBlob.size()); tx.outs = outs; tx.needed_fee = needed_fee; accumulated_fee += test_ptx.fee; @@ -8233,7 +8564,7 @@ skip_tx: extra, /* const std::vector<uint8_t>& extra, */ test_tx, /* OUT cryptonote::transaction& tx, */ test_ptx, /* OUT cryptonote::transaction& tx, */ - bulletproof); + range_proof_type); } else { transfer_selected(tx.dsts, tx.selected_transfers, @@ -8250,7 +8581,7 @@ skip_tx: auto txBlob = t_serializable_object_to_blob(test_ptx.tx); tx.tx = test_tx; tx.ptx = test_ptx; - tx.bytes = txBlob.size(); + tx.weight = get_transaction_weight(test_tx, txBlob.size()); } std::vector<wallet2::pending_tx> ptx_vector; @@ -8261,7 +8592,7 @@ skip_tx: for (size_t idx: tx.selected_transfers) tx_money += m_transfers[idx].amount(); LOG_PRINT_L1(" Transaction " << (1+std::distance(txes.begin(), i)) << "/" << txes.size() << - " " << get_transaction_hash(tx.ptx.tx) << ": " << get_size_string(tx.bytes) << ", sending " << print_money(tx_money) << " in " << tx.selected_transfers.size() << + " " << get_transaction_hash(tx.ptx.tx) << ": " << get_weight_string(tx.weight) << ", sending " << print_money(tx_money) << " in " << tx.selected_transfers.size() << " outputs to " << tx.dsts.size() << " destination(s), including " << print_money(tx.ptx.fee) << " fee, " << print_money(tx.ptx.change_dts.amount) << " change"); ptx_vector.push_back(tx.ptx); @@ -8271,7 +8602,7 @@ skip_tx: return ptx_vector; } -std::vector<wallet2::pending_tx> wallet2::create_transactions_all(uint64_t below, const cryptonote::account_public_address &address, bool is_subaddress, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices, bool trusted_daemon) +std::vector<wallet2::pending_tx> wallet2::create_transactions_all(uint64_t below, const cryptonote::account_public_address &address, bool is_subaddress, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices) { std::vector<size_t> unused_transfers_indices; std::vector<size_t> unused_dust_indices; @@ -8322,10 +8653,10 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_all(uint64_t below } } - return create_transactions_from(address, is_subaddress, unused_transfers_indices, unused_dust_indices, fake_outs_count, unlock_time, priority, extra, trusted_daemon); + return create_transactions_from(address, is_subaddress, unused_transfers_indices, unused_dust_indices, fake_outs_count, unlock_time, priority, extra); } -std::vector<wallet2::pending_tx> wallet2::create_transactions_single(const crypto::key_image &ki, const cryptonote::account_public_address &address, bool is_subaddress, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, bool trusted_daemon) +std::vector<wallet2::pending_tx> wallet2::create_transactions_single(const crypto::key_image &ki, const cryptonote::account_public_address &address, bool is_subaddress, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra) { std::vector<size_t> unused_transfers_indices; std::vector<size_t> unused_dust_indices; @@ -8343,10 +8674,10 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_single(const crypt break; } } - return create_transactions_from(address, is_subaddress, unused_transfers_indices, unused_dust_indices, fake_outs_count, unlock_time, priority, extra, trusted_daemon); + return create_transactions_from(address, is_subaddress, unused_transfers_indices, unused_dust_indices, fake_outs_count, unlock_time, priority, extra); } -std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const cryptonote::account_public_address &address, bool is_subaddress, std::vector<size_t> unused_transfers_indices, std::vector<size_t> unused_dust_indices, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, bool trusted_daemon) +std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const cryptonote::account_public_address &address, bool is_subaddress, std::vector<size_t> unused_transfers_indices, std::vector<size_t> unused_dust_indices, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra) { //ensure device is let in NONE mode in any case hw::device &hwdev = m_account.get_device(); @@ -8359,21 +8690,24 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton std::vector<cryptonote::tx_destination_entry> dsts; cryptonote::transaction tx; pending_tx ptx; - size_t bytes; + size_t weight; uint64_t needed_fee; std::vector<std::vector<get_outs_entry>> outs; - TX() : bytes(0), needed_fee(0) {} + TX() : weight(0), needed_fee(0) {} }; std::vector<TX> txes; uint64_t needed_fee, available_for_fee = 0; - uint64_t upper_transaction_size_limit = get_upper_transaction_size_limit(); + uint64_t upper_transaction_weight_limit = get_upper_transaction_weight_limit(); std::vector<std::vector<get_outs_entry>> outs; + const bool use_per_byte_fee = use_fork_rules(HF_VERSION_PER_BYTE_FEE); const bool use_rct = fake_outs_count > 0 && use_fork_rules(4, 0); const bool bulletproof = use_fork_rules(get_bulletproof_fork(), 0); - const uint64_t fee_per_kb = get_per_kb_fee(); + const rct::RangeProofType range_proof_type = bulletproof ? rct::RangeProofPaddedBulletproof : rct::RangeProofBorromean; + const uint64_t base_fee = get_base_fee(); const uint64_t fee_multiplier = get_fee_multiplier(priority, get_fee_algorithm()); + const uint64_t fee_quantization_mask = get_fee_quantization_mask(); LOG_PRINT_L2("Starting with " << unused_transfers_indices.size() << " non-dust outputs and " << unused_dust_indices.size() << " dust outputs"); @@ -8395,7 +8729,25 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton // get a random unspent output and use it to pay next chunk. We try to alternate // dust and non dust to ensure we never get with only dust, from which we might // get a tx that can't pay for itself - size_t idx = unused_transfers_indices.empty() ? pop_best_value(unused_dust_indices, tx.selected_transfers) : unused_dust_indices.empty() ? pop_best_value(unused_transfers_indices, tx.selected_transfers) : ((tx.selected_transfers.size() & 1) || accumulated_outputs > fee_per_kb * fee_multiplier * (upper_transaction_size_limit + 1023) / 1024) ? pop_best_value(unused_dust_indices, tx.selected_transfers) : pop_best_value(unused_transfers_indices, tx.selected_transfers); + uint64_t fee_dust_threshold; + if (use_fork_rules(HF_VERSION_PER_BYTE_FEE)) + { + const uint64_t estimated_tx_weight_with_one_extra_output = estimate_tx_weight(use_rct, tx.selected_transfers.size() + 1, fake_outs_count, tx.dsts.size()+1, extra.size(), bulletproof); + fee_dust_threshold = calculate_fee_from_weight(base_fee, estimated_tx_weight_with_one_extra_output, fee_multiplier, fee_quantization_mask); + } + else + { + fee_dust_threshold = base_fee * fee_multiplier * (upper_transaction_weight_limit + 1023) / 1024; + } + + size_t idx = + unused_transfers_indices.empty() + ? pop_best_value(unused_dust_indices, tx.selected_transfers) + : unused_dust_indices.empty() + ? pop_best_value(unused_transfers_indices, tx.selected_transfers) + : ((tx.selected_transfers.size() & 1) || accumulated_outputs > fee_dust_threshold) + ? pop_best_value(unused_dust_indices, tx.selected_transfers) + : pop_best_value(unused_transfers_indices, tx.selected_transfers); const transfer_details &td = m_transfers[idx]; LOG_PRINT_L2("Picking output " << idx << ", amount " << print_money(td.amount())); @@ -8410,16 +8762,15 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton // here, check if we need to sent tx and start a new one LOG_PRINT_L2("Considering whether to create a tx now, " << tx.selected_transfers.size() << " inputs, tx limit " - << upper_transaction_size_limit); - const size_t estimated_rct_tx_size = estimate_tx_size(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size() + 1, extra.size(), bulletproof); - bool try_tx = (unused_dust_indices.empty() && unused_transfers_indices.empty()) || ( estimated_rct_tx_size >= TX_SIZE_TARGET(upper_transaction_size_limit)); + << upper_transaction_weight_limit); + const size_t estimated_rct_tx_weight = estimate_tx_weight(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size() + 2, extra.size(), bulletproof); + bool try_tx = (unused_dust_indices.empty() && unused_transfers_indices.empty()) || ( estimated_rct_tx_weight >= TX_WEIGHT_TARGET(upper_transaction_weight_limit)); if (try_tx) { cryptonote::transaction test_tx; pending_tx test_ptx; - const size_t estimated_tx_size = estimate_tx_size(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size(), extra.size(), bulletproof); - needed_fee = calculate_fee(fee_per_kb, estimated_tx_size, fee_multiplier); + needed_fee = estimate_fee(use_per_byte_fee, use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size()+1, extra.size(), bulletproof, base_fee, fee_multiplier, fee_quantization_mask); tx.dsts.push_back(tx_destination_entry(1, address, is_subaddress)); @@ -8427,14 +8778,14 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton tx.selected_transfers.size() << " outputs"); if (use_rct) transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, - test_tx, test_ptx, bulletproof); + test_tx, test_ptx, range_proof_type); else transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); auto txBlob = t_serializable_object_to_blob(test_ptx.tx); - needed_fee = calculate_fee(fee_per_kb, txBlob, fee_multiplier); + needed_fee = calculate_fee(use_per_byte_fee, test_ptx.tx, txBlob.size(), base_fee, fee_multiplier, fee_quantization_mask); available_for_fee = test_ptx.fee + test_ptx.dests[0].amount + test_ptx.change_dts.amount; - LOG_PRINT_L2("Made a " << get_size_string(txBlob) << " tx, with " << print_money(available_for_fee) << " available for fee (" << + LOG_PRINT_L2("Made a " << get_weight_string(test_ptx.tx, txBlob.size()) << " tx, with " << print_money(available_for_fee) << " available for fee (" << print_money(needed_fee) << " needed)"); THROW_WALLET_EXCEPTION_IF(needed_fee > available_for_fee, error::wallet_internal_error, "Transaction cannot pay for itself"); @@ -8444,22 +8795,22 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton tx.dsts[0].amount = available_for_fee - needed_fee; if (use_rct) transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, - test_tx, test_ptx, bulletproof); + test_tx, test_ptx, range_proof_type); else transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); txBlob = t_serializable_object_to_blob(test_ptx.tx); - needed_fee = calculate_fee(fee_per_kb, txBlob, fee_multiplier); - LOG_PRINT_L2("Made an attempt at a final " << get_size_string(txBlob) << " tx, with " << print_money(test_ptx.fee) << + needed_fee = calculate_fee(use_per_byte_fee, test_ptx.tx, txBlob.size(), base_fee, fee_multiplier, fee_quantization_mask); + LOG_PRINT_L2("Made an attempt at a final " << get_weight_string(test_ptx.tx, txBlob.size()) << " tx, with " << print_money(test_ptx.fee) << " fee and " << print_money(test_ptx.change_dts.amount) << " change"); } while (needed_fee > test_ptx.fee); - LOG_PRINT_L2("Made a final " << get_size_string(txBlob) << " tx, with " << print_money(test_ptx.fee) << + LOG_PRINT_L2("Made a final " << get_weight_string(test_ptx.tx, txBlob.size()) << " tx, with " << print_money(test_ptx.fee) << " fee and " << print_money(test_ptx.change_dts.amount) << " change"); tx.tx = test_tx; tx.ptx = test_ptx; - tx.bytes = txBlob.size(); + tx.weight = get_transaction_weight(test_tx, txBlob.size()); tx.outs = outs; tx.needed_fee = needed_fee; accumulated_fee += test_ptx.fee; @@ -8483,7 +8834,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton pending_tx test_ptx; if (use_rct) { transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, tx.outs, unlock_time, tx.needed_fee, extra, - test_tx, test_ptx, bulletproof); + test_tx, test_ptx, range_proof_type); } else { transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, tx.outs, unlock_time, tx.needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); @@ -8491,7 +8842,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton auto txBlob = t_serializable_object_to_blob(test_ptx.tx); tx.tx = test_tx; tx.ptx = test_ptx; - tx.bytes = txBlob.size(); + tx.weight = get_transaction_weight(test_tx, txBlob.size()); } std::vector<wallet2::pending_tx> ptx_vector; @@ -8502,7 +8853,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton for (size_t idx: tx.selected_transfers) tx_money += m_transfers[idx].amount(); LOG_PRINT_L1(" Transaction " << (1+std::distance(txes.begin(), i)) << "/" << txes.size() << - " " << get_transaction_hash(tx.ptx.tx) << ": " << get_size_string(tx.bytes) << ", sending " << print_money(tx_money) << " in " << tx.selected_transfers.size() << + " " << get_transaction_hash(tx.ptx.tx) << ": " << get_weight_string(tx.weight) << ", sending " << print_money(tx_money) << " in " << tx.selected_transfers.size() << " outputs to " << tx.dsts.size() << " destination(s), including " << print_money(tx.ptx.fee) << " fee, " << print_money(tx.ptx.change_dts.amount) << " change"); ptx_vector.push_back(tx.ptx); @@ -8529,7 +8880,7 @@ bool wallet2::use_fork_rules(uint8_t version, int64_t early_blocks) const result = m_node_rpc_proxy.get_earliest_height(version, earliest_height); throw_on_rpc_response_error(result, "get_hard_fork_info"); - bool close_enough = height >= earliest_height - early_blocks && earliest_height != std::numeric_limits<uint64_t>::max(); // start using the rules that many blocks beforehand + bool close_enough = height >= earliest_height - early_blocks; // start using the rules that many blocks beforehand if (close_enough) LOG_PRINT_L2("Using v" << (unsigned)version << " rules"); else @@ -8537,12 +8888,15 @@ bool wallet2::use_fork_rules(uint8_t version, int64_t early_blocks) const return close_enough; } //---------------------------------------------------------------------------------------------------- -uint64_t wallet2::get_upper_transaction_size_limit() const +uint64_t wallet2::get_upper_transaction_weight_limit() const { - if (m_upper_transaction_size_limit > 0) - return m_upper_transaction_size_limit; + if (m_upper_transaction_weight_limit > 0) + return m_upper_transaction_weight_limit; uint64_t full_reward_zone = use_fork_rules(5, 10) ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 : use_fork_rules(2, 10) ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V2 : CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V1; - return full_reward_zone - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; + if (use_fork_rules(8, 10)) + return full_reward_zone / 2 - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; + else + return full_reward_zone - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; } //---------------------------------------------------------------------------------------------------- std::vector<size_t> wallet2::select_available_outputs(const std::function<bool(const transfer_details &td)> &f) const @@ -8580,12 +8934,12 @@ std::vector<uint64_t> wallet2::get_unspent_amounts_vector() const return vector; } //---------------------------------------------------------------------------------------------------- -std::vector<size_t> wallet2::select_available_outputs_from_histogram(uint64_t count, bool atleast, bool unlocked, bool allow_rct, bool trusted_daemon) +std::vector<size_t> wallet2::select_available_outputs_from_histogram(uint64_t count, bool atleast, bool unlocked, bool allow_rct) { cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request req_t = AUTO_VAL_INIT(req_t); cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response resp_t = AUTO_VAL_INIT(resp_t); m_daemon_rpc_mutex.lock(); - if (trusted_daemon) + if (is_trusted_daemon()) req_t.amounts = get_unspent_amounts_vector(); req_t.min_count = count; req_t.max_count = 0; @@ -8646,30 +9000,28 @@ const wallet2::transfer_details &wallet2::get_transfer_details(size_t idx) const return m_transfers[idx]; } //---------------------------------------------------------------------------------------------------- -std::vector<size_t> wallet2::select_available_unmixable_outputs(bool trusted_daemon) +std::vector<size_t> wallet2::select_available_unmixable_outputs() { - // request all outputs with less than 3 instances - const size_t min_mixin = use_fork_rules(7, 10) ? 6 : use_fork_rules(6, 10) ? 4 : 2; // v6 increases min mixin from 2 to 4, v7 to 6 - return select_available_outputs_from_histogram(min_mixin + 1, false, true, false, trusted_daemon); + // request all outputs with less instances than the min ring size + return select_available_outputs_from_histogram(get_min_ring_size(), false, true, false); } //---------------------------------------------------------------------------------------------------- -std::vector<size_t> wallet2::select_available_mixable_outputs(bool trusted_daemon) +std::vector<size_t> wallet2::select_available_mixable_outputs() { - // request all outputs with at least 3 instances, so we can use mixin 2 with - const size_t min_mixin = use_fork_rules(7, 10) ? 6 : use_fork_rules(6, 10) ? 4 : 2; // v6 increases min mixin from 2 to 4, v7 to 6 - return select_available_outputs_from_histogram(min_mixin + 1, true, true, true, trusted_daemon); + // request all outputs with at least as many instances as the min ring size + return select_available_outputs_from_histogram(get_min_ring_size(), true, true, true); } //---------------------------------------------------------------------------------------------------- -std::vector<wallet2::pending_tx> wallet2::create_unmixable_sweep_transactions(bool trusted_daemon) +std::vector<wallet2::pending_tx> wallet2::create_unmixable_sweep_transactions() { // From hard fork 1, we don't consider small amounts to be dust anymore const bool hf1_rules = use_fork_rules(2, 10); // first hard fork has version 2 tx_dust_policy dust_policy(hf1_rules ? 0 : ::config::DEFAULT_DUST_THRESHOLD); - const uint64_t fee_per_kb = get_per_kb_fee(); + const uint64_t base_fee = get_base_fee(); // may throw - std::vector<size_t> unmixable_outputs = select_available_unmixable_outputs(trusted_daemon); + std::vector<size_t> unmixable_outputs = select_available_unmixable_outputs(); size_t num_dust_outputs = unmixable_outputs.size(); if (num_dust_outputs == 0) @@ -8681,19 +9033,19 @@ std::vector<wallet2::pending_tx> wallet2::create_unmixable_sweep_transactions(bo std::vector<size_t> unmixable_transfer_outputs, unmixable_dust_outputs; for (auto n: unmixable_outputs) { - if (m_transfers[n].amount() < fee_per_kb) + if (m_transfers[n].amount() < base_fee) unmixable_dust_outputs.push_back(n); else unmixable_transfer_outputs.push_back(n); } - return create_transactions_from(m_account_public_address, false, unmixable_transfer_outputs, unmixable_dust_outputs, 0 /*fake_outs_count */, 0 /* unlock_time */, 1 /*priority */, std::vector<uint8_t>(), trusted_daemon); + return create_transactions_from(m_account_public_address, false, unmixable_transfer_outputs, unmixable_dust_outputs, 0 /*fake_outs_count */, 0 /* unlock_time */, 1 /*priority */, std::vector<uint8_t>()); } //---------------------------------------------------------------------------------------------------- -void wallet2::discard_unmixable_outputs(bool trusted_daemon) +void wallet2::discard_unmixable_outputs() { // may throw - std::vector<size_t> unmixable_outputs = select_available_unmixable_outputs(trusted_daemon); + std::vector<size_t> unmixable_outputs = select_available_unmixable_outputs(); for (size_t idx : unmixable_outputs) { m_transfers[idx].m_spent = true; @@ -9074,6 +9426,8 @@ void wallet2::check_tx_key_helper(const crypto::hash &txid, const crypto::key_de hwdev.ecdhDecode(ecdh_info, rct::sk2rct(scalar1)); const rct::key C = tx.rct_signatures.outPk[n].mask; rct::key Ctmp; + THROW_WALLET_EXCEPTION_IF(sc_check(ecdh_info.mask.bytes) != 0, error::wallet_internal_error, "Bad ECDH input mask"); + THROW_WALLET_EXCEPTION_IF(sc_check(ecdh_info.amount.bytes) != 0, error::wallet_internal_error, "Bad ECDH input amount"); rct::addKeys2(Ctmp, ecdh_info.mask, ecdh_info.amount, rct::H); if (rct::equalKeys(C, Ctmp)) amount = rct::h2d(ecdh_info.amount); @@ -9644,7 +9998,7 @@ uint64_t wallet2::get_approximate_blockchain_height() const // Calculated blockchain height uint64_t approx_blockchain_height = fork_block + (time(NULL) - fork_time)/seconds_per_block; // testnet got some huge rollbacks, so the estimation is way off - static const uint64_t approximate_testnet_rolled_back_blocks = 148540; + static const uint64_t approximate_testnet_rolled_back_blocks = 303967; if (m_nettype == TESTNET && approx_blockchain_height > approximate_testnet_rolled_back_blocks) approx_blockchain_height -= approximate_testnet_rolled_back_blocks; LOG_PRINT_L2("Calculated blockchain height: " << approx_blockchain_height); @@ -10666,6 +11020,7 @@ size_t wallet2::import_multisig(std::vector<cryptonote::blobdata> blobs) m_multisig_rescan_info = &info; try { + refresh(false); } catch (...) {} @@ -10675,14 +11030,14 @@ size_t wallet2::import_multisig(std::vector<cryptonote::blobdata> blobs) return n_outputs; } //---------------------------------------------------------------------------------------------------- -std::string wallet2::encrypt(const std::string &plaintext, const crypto::secret_key &skey, bool authenticated) const +std::string wallet2::encrypt(const char *plaintext, size_t len, const crypto::secret_key &skey, bool authenticated) const { crypto::chacha_key key; crypto::generate_chacha_key(&skey, sizeof(skey), key, m_kdf_rounds); std::string ciphertext; crypto::chacha_iv iv = crypto::rand<crypto::chacha_iv>(); - ciphertext.resize(plaintext.size() + sizeof(iv) + (authenticated ? sizeof(crypto::signature) : 0)); - crypto::chacha20(plaintext.data(), plaintext.size(), key, iv, &ciphertext[sizeof(iv)]); + ciphertext.resize(len + sizeof(iv) + (authenticated ? sizeof(crypto::signature) : 0)); + crypto::chacha20(plaintext, len, key, iv, &ciphertext[sizeof(iv)]); memcpy(&ciphertext[0], &iv, sizeof(iv)); if (authenticated) { @@ -10696,12 +11051,28 @@ std::string wallet2::encrypt(const std::string &plaintext, const crypto::secret_ return ciphertext; } //---------------------------------------------------------------------------------------------------- +std::string wallet2::encrypt(const epee::span<char> &plaintext, const crypto::secret_key &skey, bool authenticated) const +{ + return encrypt(plaintext.data(), plaintext.size(), skey, authenticated); +} +//---------------------------------------------------------------------------------------------------- +std::string wallet2::encrypt(const std::string &plaintext, const crypto::secret_key &skey, bool authenticated) const +{ + return encrypt(plaintext.data(), plaintext.size(), skey, authenticated); +} +//---------------------------------------------------------------------------------------------------- +std::string wallet2::encrypt(const epee::wipeable_string &plaintext, const crypto::secret_key &skey, bool authenticated) const +{ + return encrypt(plaintext.data(), plaintext.size(), skey, authenticated); +} +//---------------------------------------------------------------------------------------------------- std::string wallet2::encrypt_with_view_secret_key(const std::string &plaintext, bool authenticated) const { return encrypt(plaintext, get_account().get_keys().m_view_secret_key, authenticated); } //---------------------------------------------------------------------------------------------------- -std::string wallet2::decrypt(const std::string &ciphertext, const crypto::secret_key &skey, bool authenticated) const +template<typename T> +T wallet2::decrypt(const std::string &ciphertext, const crypto::secret_key &skey, bool authenticated) const { const size_t prefix_size = sizeof(chacha_iv) + (authenticated ? sizeof(crypto::signature) : 0); THROW_WALLET_EXCEPTION_IF(ciphertext.size() < prefix_size, @@ -10710,8 +11081,6 @@ std::string wallet2::decrypt(const std::string &ciphertext, const crypto::secret crypto::chacha_key key; crypto::generate_chacha_key(&skey, sizeof(skey), key, m_kdf_rounds); const crypto::chacha_iv &iv = *(const crypto::chacha_iv*)&ciphertext[0]; - std::string plaintext; - plaintext.resize(ciphertext.size() - prefix_size); if (authenticated) { crypto::hash hash; @@ -10722,10 +11091,14 @@ std::string wallet2::decrypt(const std::string &ciphertext, const crypto::secret THROW_WALLET_EXCEPTION_IF(!crypto::check_signature(hash, pkey, signature), error::wallet_internal_error, "Failed to authenticate ciphertext"); } - crypto::chacha20(ciphertext.data() + sizeof(iv), ciphertext.size() - prefix_size, key, iv, &plaintext[0]); - return plaintext; + std::unique_ptr<char[]> buffer{new char[ciphertext.size() - prefix_size]}; + auto wiper = epee::misc_utils::create_scope_leave_handler([&]() { memwipe(buffer.get(), ciphertext.size() - prefix_size); }); + crypto::chacha20(ciphertext.data() + sizeof(iv), ciphertext.size() - prefix_size, key, iv, buffer.get()); + return T(buffer.get(), ciphertext.size() - prefix_size); } //---------------------------------------------------------------------------------------------------- +template epee::wipeable_string wallet2::decrypt(const std::string &ciphertext, const crypto::secret_key &skey, bool authenticated) const; +//---------------------------------------------------------------------------------------------------- std::string wallet2::decrypt_with_view_secret_key(const std::string &ciphertext, bool authenticated) const { return decrypt(ciphertext, get_account().get_keys().m_view_secret_key, authenticated); @@ -10983,45 +11356,46 @@ std::vector<std::pair<uint64_t, uint64_t>> wallet2::estimate_backlog(const std:: THROW_WALLET_EXCEPTION_IF(res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_txpool_backlog"); THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::get_tx_pool_error); - uint64_t block_size_limit = 0; - const auto result = m_node_rpc_proxy.get_block_size_limit(block_size_limit); + uint64_t block_weight_limit = 0; + const auto result = m_node_rpc_proxy.get_block_weight_limit(block_weight_limit); throw_on_rpc_response_error(result, "get_info"); - uint64_t full_reward_zone = block_size_limit / 2; + uint64_t full_reward_zone = block_weight_limit / 2; + THROW_WALLET_EXCEPTION_IF(full_reward_zone == 0, error::wallet_internal_error, "Invalid block weight limit from daemon"); std::vector<std::pair<uint64_t, uint64_t>> blocks; for (const auto &fee_level: fee_levels) { const double our_fee_byte_min = fee_level.first; const double our_fee_byte_max = fee_level.second; - uint64_t priority_size_min = 0, priority_size_max = 0; + uint64_t priority_weight_min = 0, priority_weight_max = 0; for (const auto &i: res.backlog) { - if (i.blob_size == 0) + if (i.weight == 0) { - MWARNING("Got 0 sized blob from txpool, ignored"); + MWARNING("Got 0 weight tx from txpool, ignored"); continue; } - double this_fee_byte = i.fee / (double)i.blob_size; + double this_fee_byte = i.fee / (double)i.weight; if (this_fee_byte >= our_fee_byte_min) - priority_size_min += i.blob_size; + priority_weight_min += i.weight; if (this_fee_byte >= our_fee_byte_max) - priority_size_max += i.blob_size; + priority_weight_max += i.weight; } - uint64_t nblocks_min = priority_size_min / full_reward_zone; - uint64_t nblocks_max = priority_size_max / full_reward_zone; - MDEBUG("estimate_backlog: priority_size " << priority_size_min << " - " << priority_size_max << " for " + uint64_t nblocks_min = priority_weight_min / full_reward_zone; + uint64_t nblocks_max = priority_weight_max / full_reward_zone; + MDEBUG("estimate_backlog: priority_weight " << priority_weight_min << " - " << priority_weight_max << " for " << our_fee_byte_min << " - " << our_fee_byte_max << " piconero byte fee, " - << nblocks_min << " - " << nblocks_max << " blocks at block size " << full_reward_zone); + << nblocks_min << " - " << nblocks_max << " blocks at block weight " << full_reward_zone); blocks.push_back(std::make_pair(nblocks_min, nblocks_max)); } return blocks; } //---------------------------------------------------------------------------------------------------- -std::vector<std::pair<uint64_t, uint64_t>> wallet2::estimate_backlog(uint64_t min_blob_size, uint64_t max_blob_size, const std::vector<uint64_t> &fees) +std::vector<std::pair<uint64_t, uint64_t>> wallet2::estimate_backlog(uint64_t min_tx_weight, uint64_t max_tx_weight, const std::vector<uint64_t> &fees) { - THROW_WALLET_EXCEPTION_IF(min_blob_size == 0, error::wallet_internal_error, "Invalid 0 fee"); - THROW_WALLET_EXCEPTION_IF(max_blob_size == 0, error::wallet_internal_error, "Invalid 0 fee"); + THROW_WALLET_EXCEPTION_IF(min_tx_weight == 0, error::wallet_internal_error, "Invalid 0 fee"); + THROW_WALLET_EXCEPTION_IF(max_tx_weight == 0, error::wallet_internal_error, "Invalid 0 fee"); for (uint64_t fee: fees) { THROW_WALLET_EXCEPTION_IF(fee == 0, error::wallet_internal_error, "Invalid 0 fee"); @@ -11029,7 +11403,7 @@ std::vector<std::pair<uint64_t, uint64_t>> wallet2::estimate_backlog(uint64_t mi std::vector<std::pair<double, double>> fee_levels; for (uint64_t fee: fees) { - double our_fee_byte_min = fee / (double)min_blob_size, our_fee_byte_max = fee / (double)max_blob_size; + double our_fee_byte_min = fee / (double)min_tx_weight, our_fee_byte_max = fee / (double)max_tx_weight; fee_levels.emplace_back(our_fee_byte_min, our_fee_byte_max); } return estimate_backlog(fee_levels); diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index d04156461..f9b516bff 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -67,6 +67,19 @@ class Serialization_portability_wallet_Test; namespace tools { class ringdb; + class wallet2; + + class wallet_keys_unlocker + { + public: + wallet_keys_unlocker(wallet2 &w, const boost::optional<tools::password_container> &password); + wallet_keys_unlocker(wallet2 &w, bool locked, const epee::wipeable_string &password); + ~wallet_keys_unlocker(); + private: + wallet2 &w; + bool locked; + crypto::chacha_key key; + }; class i_wallet2_callback { @@ -77,6 +90,7 @@ namespace tools virtual void on_unconfirmed_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index) {} virtual void on_money_spent(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& in_tx, uint64_t amount, const cryptonote::transaction& spend_tx, const cryptonote::subaddress_index& subaddr_index) {} virtual void on_skip_transaction(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx) {} + virtual boost::optional<epee::wipeable_string> on_get_password(const char *reason) { return boost::none; } // Light wallet callbacks virtual void on_lw_new_block(uint64_t height) {} virtual void on_lw_money_received(uint64_t height, const crypto::hash &txid, uint64_t amount) {} @@ -133,9 +147,11 @@ namespace tools std::deque<crypto::hash> m_blockchain; }; + class wallet_keys_unlocker; class wallet2 { friend class ::Serialization_portability_wallet_Test; + friend class wallet_keys_unlocker; public: static constexpr const std::chrono::seconds rpc_timeout = std::chrono::minutes(3) + std::chrono::seconds(30); @@ -153,17 +169,17 @@ namespace tools static void init_options(boost::program_options::options_description& desc_params); //! Uses stdin and stdout. Returns a wallet2 if no errors. - static std::unique_ptr<wallet2> make_from_json(const boost::program_options::variables_map& vm, const std::string& json_file, const std::function<boost::optional<password_container>(const char *, bool)> &password_prompter); + static std::unique_ptr<wallet2> make_from_json(const boost::program_options::variables_map& vm, bool rpc, const std::string& json_file, const std::function<boost::optional<password_container>(const char *, bool)> &password_prompter); //! Uses stdin and stdout. Returns a wallet2 and password for `wallet_file` if no errors. static std::pair<std::unique_ptr<wallet2>, password_container> - make_from_file(const boost::program_options::variables_map& vm, const std::string& wallet_file, const std::function<boost::optional<password_container>(const char *, bool)> &password_prompter); + make_from_file(const boost::program_options::variables_map& vm, bool rpc, const std::string& wallet_file, const std::function<boost::optional<password_container>(const char *, bool)> &password_prompter); //! Uses stdin and stdout. Returns a wallet2 and password for wallet with no file if no errors. - static std::pair<std::unique_ptr<wallet2>, password_container> make_new(const boost::program_options::variables_map& vm, const std::function<boost::optional<password_container>(const char *, bool)> &password_prompter); + static std::pair<std::unique_ptr<wallet2>, password_container> make_new(const boost::program_options::variables_map& vm, bool rpc, const std::function<boost::optional<password_container>(const char *, bool)> &password_prompter); //! Just parses variables. - static std::unique_ptr<wallet2> make_dummy(const boost::program_options::variables_map& vm, const std::function<boost::optional<password_container>(const char *, bool)> &password_prompter); + static std::unique_ptr<wallet2> make_dummy(const boost::program_options::variables_map& vm, bool rpc, const std::function<boost::optional<password_container>(const char *, bool)> &password_prompter); static bool verify_password(const std::string& keys_file_name, const epee::wipeable_string& password, bool no_spend_key, hw::device &hwdev, uint64_t kdf_rounds); @@ -477,16 +493,6 @@ namespace tools std::vector<is_out_data> additional; }; - struct key_ref - { - key_ref(tools::wallet2 &w): wallet(w) { ++refs; } - ~key_ref() { if (!--refs) wallet.clear_ringdb_key(); } - - private: - tools::wallet2 &wallet; - static std::atomic<unsigned int> refs; - }; - /*! * \brief Generates a wallet or restores one. * \param wallet_ Name of wallet file @@ -495,7 +501,7 @@ namespace tools * \param create_address_file Whether to create an address file */ void generate(const std::string& wallet_, const epee::wipeable_string& password, - const std::string& multisig_data, bool create_address_file = false); + const epee::wipeable_string& multisig_data, bool create_address_file = false); /*! * \brief Generates a wallet or restores one. @@ -613,31 +619,34 @@ namespace tools cryptonote::account_base& get_account(){return m_account;} const cryptonote::account_base& get_account()const{return m_account;} + void encrypt_keys(const crypto::chacha_key &key); + void encrypt_keys(const epee::wipeable_string &password); + void decrypt_keys(const crypto::chacha_key &key); + void decrypt_keys(const epee::wipeable_string &password); + void set_refresh_from_block_height(uint64_t height) {m_refresh_from_block_height = height;} uint64_t get_refresh_from_block_height() const {return m_refresh_from_block_height;} void explicit_refresh_from_block_height(bool expl) {m_explicit_refresh_from_block_height = expl;} bool explicit_refresh_from_block_height() const {return m_explicit_refresh_from_block_height;} - // upper_transaction_size_limit as defined below is set to - // approximately 125% of the fixed minimum allowable penalty - // free block size. TODO: fix this so that it actually takes - // into account the current median block size rather than - // the minimum block size. bool deinit(); - bool init(std::string daemon_address = "http://localhost:8080", - boost::optional<epee::net_utils::http::login> daemon_login = boost::none, uint64_t upper_transaction_size_limit = 0, bool ssl = false); + bool init(bool rpc, std::string daemon_address = "http://localhost:8080", + boost::optional<epee::net_utils::http::login> daemon_login = boost::none, uint64_t upper_transaction_weight_limit = 0, bool ssl = false, bool trusted_daemon = false); void stop() { m_run.store(false, std::memory_order_relaxed); } i_wallet2_callback* callback() const { return m_callback; } void callback(i_wallet2_callback* callback) { m_callback = callback; } + bool is_trusted_daemon() const { return m_trusted_daemon; } + void set_trusted_daemon(bool trusted) { m_trusted_daemon = trusted; } + /*! * \brief Checks if deterministic wallet */ bool is_deterministic() const; - bool get_seed(std::string& electrum_words, const epee::wipeable_string &passphrase = epee::wipeable_string()) const; + bool get_seed(epee::wipeable_string& electrum_words, const epee::wipeable_string &passphrase = epee::wipeable_string()) const; /*! * \brief Checks if light wallet. A light wallet sends view key to a server where the blockchain is scanned. @@ -691,7 +700,7 @@ namespace tools bool multisig(bool *ready = NULL, uint32_t *threshold = NULL, uint32_t *total = NULL) const; bool has_multisig_partial_key_images() const; bool has_unknown_key_images() const; - bool get_multisig_seed(std::string& seed, const epee::wipeable_string &passphrase = std::string(), bool raw = true) const; + bool get_multisig_seed(epee::wipeable_string& seed, const epee::wipeable_string &passphrase = std::string(), bool raw = true) const; bool key_on_device() const { return m_key_on_device; } // locked & unlocked balance of given or current subaddress account @@ -704,18 +713,18 @@ namespace tools uint64_t balance_all() const; uint64_t unlocked_balance_all() const; template<typename T> - void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, bool trusted_daemon); + void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy); template<typename T> - void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, cryptonote::transaction& tx, pending_tx& ptx, bool trusted_daemon); - void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, bool trusted_daemon); - void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx& ptx, bool trusted_daemon); + void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, cryptonote::transaction& tx, pending_tx& ptx); + void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra); + void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx& ptx); template<typename T> void transfer_selected(const std::vector<cryptonote::tx_destination_entry>& dsts, const std::vector<size_t>& selected_transfers, size_t fake_outputs_count, std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, cryptonote::transaction& tx, pending_tx &ptx); void transfer_selected_rct(std::vector<cryptonote::tx_destination_entry> dsts, const std::vector<size_t>& selected_transfers, size_t fake_outputs_count, std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, - uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx &ptx, bool bulletproof); + uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx &ptx, rct::RangeProofType range_proof_type); void commit_tx(pending_tx& ptx_vector); void commit_tx(std::vector<pending_tx>& ptx_vector); @@ -737,18 +746,18 @@ namespace tools bool parse_unsigned_tx_from_str(const std::string &unsigned_tx_st, unsigned_tx_set &exported_txs) const; bool load_tx(const std::string &signed_filename, std::vector<tools::wallet2::pending_tx> &ptx, std::function<bool(const signed_tx_set&)> accept_func = NULL); bool parse_tx_from_str(const std::string &signed_tx_st, std::vector<tools::wallet2::pending_tx> &ptx, std::function<bool(const signed_tx_set &)> accept_func); - std::vector<pending_tx> create_transactions(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, bool trusted_daemon); - std::vector<wallet2::pending_tx> create_transactions_2(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices, bool trusted_daemon); // pass subaddr_indices by value on purpose - std::vector<wallet2::pending_tx> create_transactions_all(uint64_t below, const cryptonote::account_public_address &address, bool is_subaddress, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices, bool trusted_daemon); - std::vector<wallet2::pending_tx> create_transactions_single(const crypto::key_image &ki, const cryptonote::account_public_address &address, bool is_subaddress, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, bool trusted_daemon); - std::vector<wallet2::pending_tx> create_transactions_from(const cryptonote::account_public_address &address, bool is_subaddress, std::vector<size_t> unused_transfers_indices, std::vector<size_t> unused_dust_indices, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, bool trusted_daemon); + std::vector<pending_tx> create_transactions(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra); + std::vector<wallet2::pending_tx> create_transactions_2(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices); // pass subaddr_indices by value on purpose + std::vector<wallet2::pending_tx> create_transactions_all(uint64_t below, const cryptonote::account_public_address &address, bool is_subaddress, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices); + std::vector<wallet2::pending_tx> create_transactions_single(const crypto::key_image &ki, const cryptonote::account_public_address &address, bool is_subaddress, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra); + std::vector<wallet2::pending_tx> create_transactions_from(const cryptonote::account_public_address &address, bool is_subaddress, std::vector<size_t> unused_transfers_indices, std::vector<size_t> unused_dust_indices, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra); bool load_multisig_tx(cryptonote::blobdata blob, multisig_tx_set &exported_txs, std::function<bool(const multisig_tx_set&)> accept_func = NULL); bool load_multisig_tx_from_file(const std::string &filename, multisig_tx_set &exported_txs, std::function<bool(const multisig_tx_set&)> accept_func = NULL); bool sign_multisig_tx_from_file(const std::string &filename, std::vector<crypto::hash> &txids, std::function<bool(const multisig_tx_set&)> accept_func); bool sign_multisig_tx(multisig_tx_set &exported_txs, std::vector<crypto::hash> &txids); bool sign_multisig_tx_to_file(multisig_tx_set &exported_txs, const std::string &filename, std::vector<crypto::hash> &txids); - std::vector<pending_tx> create_unmixable_sweep_transactions(bool trusted_daemon); - void discard_unmixable_outputs(bool trusted_daemon); + std::vector<pending_tx> create_unmixable_sweep_transactions(); + void discard_unmixable_outputs(); bool check_connection(uint32_t *version = NULL, uint32_t timeout = 200000); void get_transfers(wallet2::transfer_container& incoming_transfers) const; void get_payments(const crypto::hash& payment_id, std::list<wallet2::payment_details>& payments, uint64_t min_height = 0, const boost::optional<uint32_t>& subaddr_account = boost::none, const std::set<uint32_t>& subaddr_indices = {}) const; @@ -985,10 +994,10 @@ namespace tools */ uint64_t get_approximate_blockchain_height() const; uint64_t estimate_blockchain_height(); - std::vector<size_t> select_available_outputs_from_histogram(uint64_t count, bool atleast, bool unlocked, bool allow_rct, bool trusted_daemon); + std::vector<size_t> select_available_outputs_from_histogram(uint64_t count, bool atleast, bool unlocked, bool allow_rct); std::vector<size_t> select_available_outputs(const std::function<bool(const transfer_details &td)> &f) const; - std::vector<size_t> select_available_unmixable_outputs(bool trusted_daemon); - std::vector<size_t> select_available_mixable_outputs(bool trusted_daemon); + std::vector<size_t> select_available_unmixable_outputs(); + std::vector<size_t> select_available_mixable_outputs(); size_t pop_best_value_from(const transfer_container &transfers, std::vector<size_t> &unused_dust_indices, const std::vector<size_t>& selected_transfers, bool smallest = false) const; size_t pop_best_value(std::vector<size_t> &unused_dust_indices, const std::vector<size_t>& selected_transfers, bool smallest = false) const; @@ -1054,9 +1063,12 @@ namespace tools void update_pool_state(bool refreshed = false); void remove_obsolete_pool_txs(const std::vector<crypto::hash> &tx_hashes); + std::string encrypt(const char *plaintext, size_t len, const crypto::secret_key &skey, bool authenticated = true) const; + std::string encrypt(const epee::span<char> &span, const crypto::secret_key &skey, bool authenticated = true) const; std::string encrypt(const std::string &plaintext, const crypto::secret_key &skey, bool authenticated = true) const; + std::string encrypt(const epee::wipeable_string &plaintext, const crypto::secret_key &skey, bool authenticated = true) const; std::string encrypt_with_view_secret_key(const std::string &plaintext, bool authenticated = true) const; - std::string decrypt(const std::string &ciphertext, const crypto::secret_key &skey, bool authenticated = true) const; + template<typename T=std::string> T decrypt(const std::string &ciphertext, const crypto::secret_key &skey, bool authenticated = true) const; std::string decrypt_with_view_secret_key(const std::string &ciphertext, bool authenticated = true) const; std::string make_uri(const std::string &address, const std::string &payment_id, uint64_t amount, const std::string &tx_description, const std::string &recipient_name, std::string &error) const; @@ -1067,13 +1079,18 @@ namespace tools bool is_synced() const; std::vector<std::pair<uint64_t, uint64_t>> estimate_backlog(const std::vector<std::pair<double, double>> &fee_levels); - std::vector<std::pair<uint64_t, uint64_t>> estimate_backlog(uint64_t min_blob_size, uint64_t max_blob_size, const std::vector<uint64_t> &fees); + std::vector<std::pair<uint64_t, uint64_t>> estimate_backlog(uint64_t min_tx_weight, uint64_t max_tx_weight, const std::vector<uint64_t> &fees); uint64_t get_fee_multiplier(uint32_t priority, int fee_algorithm = -1) const; - uint64_t get_per_kb_fee() const; + uint64_t get_base_fee() const; + uint64_t get_fee_quantization_mask() const; + uint64_t get_min_ring_size() const; + uint64_t get_max_ring_size() const; uint64_t adjust_mixin(uint64_t mixin) const; uint32_t adjust_priority(uint32_t priority); + bool is_rpc() const { return m_rpc; } + // Light wallet specific functions // fetch unspent outs from lw node and store in m_transfers void light_wallet_get_unspent_outs(); @@ -1150,6 +1167,9 @@ namespace tools bool lock_keys_file(); bool unlock_keys_file(); bool is_keys_file_locked() const; + + void change_password(const std::string &filename, const epee::wipeable_string &original_password, const epee::wipeable_string &new_password); + private: /*! * \brief Stores wallet information to wallet file. @@ -1176,7 +1196,7 @@ namespace tools void fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, bool force = false); void pull_and_parse_next_blocks(uint64_t start_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, const std::vector<cryptonote::block_complete_entry> &prev_blocks, const std::vector<parsed_block> &prev_parsed_blocks, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<parsed_block> &parsed_blocks, bool &error); void process_parsed_blocks(uint64_t start_height, const std::vector<cryptonote::block_complete_entry> &blocks, const std::vector<parsed_block> &parsed_blocks, uint64_t& blocks_added); - uint64_t select_transfers(uint64_t needed_money, std::vector<size_t> unused_transfers_indices, std::vector<size_t>& selected_transfers, bool trusted_daemon) const; + uint64_t select_transfers(uint64_t needed_money, std::vector<size_t> unused_transfers_indices, std::vector<size_t>& selected_transfers) const; bool prepare_file_names(const std::string& file_path); void process_unconfirmed(const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t height); void process_outgoing(const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t height, uint64_t ts, uint64_t spent, uint64_t received, uint32_t subaddr_account, const std::set<uint32_t>& subaddr_indices); @@ -1184,14 +1204,15 @@ namespace tools void generate_genesis(cryptonote::block& b) const; void check_genesis(const crypto::hash& genesis_hash) const; //throws bool generate_chacha_key_from_secret_keys(crypto::chacha_key &key) const; + void generate_chacha_key_from_password(const epee::wipeable_string &pass, crypto::chacha_key &key) const; crypto::hash get_payment_id(const pending_tx &ptx) const; void check_acc_out_precomp(const cryptonote::tx_out &o, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, size_t i, tx_scan_info_t &tx_scan_info) const; void check_acc_out_precomp(const cryptonote::tx_out &o, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, size_t i, const is_out_data *is_out_data, tx_scan_info_t &tx_scan_info) const; void check_acc_out_precomp_once(const cryptonote::tx_out &o, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, size_t i, const is_out_data *is_out_data, tx_scan_info_t &tx_scan_info, bool &already_seen) const; void parse_block_round(const cryptonote::blobdata &blob, cryptonote::block &bl, crypto::hash &bl_id, bool &error) const; - uint64_t get_upper_transaction_size_limit() const; + uint64_t get_upper_transaction_weight_limit() const; std::vector<uint64_t> get_unspent_amounts_vector() const; - uint64_t get_dynamic_per_kb_fee_estimate() const; + uint64_t get_dynamic_base_fee_estimate() const; float get_output_relatedness(const transfer_details &td0, const transfer_details &td1) const; std::vector<size_t> pick_preferred_rct_inputs(uint64_t needed_money, uint32_t subaddr_account, const std::set<uint32_t> &subaddr_indices) const; void set_spent(size_t idx, uint64_t height); @@ -1201,7 +1222,7 @@ namespace tools crypto::public_key get_tx_pub_key_from_received_outs(const tools::wallet2::transfer_details &td) const; bool should_pick_a_second_output(bool use_rct, size_t n_transfers, const std::vector<size_t> &unused_transfers_indices, const std::vector<size_t> &unused_dust_indices) const; std::vector<size_t> get_only_rct(const std::vector<size_t> &unused_dust_indices, const std::vector<size_t> &unused_transfers_indices) const; - void scan_output(const cryptonote::transaction &tx, const crypto::public_key &tx_pub_key, size_t i, tx_scan_info_t &tx_scan_info, int &num_vouts_received, std::unordered_map<cryptonote::subaddress_index, uint64_t> &tx_money_got_in_outs, std::vector<size_t> &outs) const; + void scan_output(const cryptonote::transaction &tx, const crypto::public_key &tx_pub_key, size_t i, tx_scan_info_t &tx_scan_info, int &num_vouts_received, std::unordered_map<cryptonote::subaddress_index, uint64_t> &tx_money_got_in_outs, std::vector<size_t> &outs); void trim_hashchain(); crypto::key_image get_multisig_composite_key_image(size_t n) const; rct::multisig_kLRki get_multisig_composite_kLRki(size_t n, const crypto::public_key &ignore, std::unordered_set<rct::key> &used_L, std::unordered_set<rct::key> &new_used_L) const; @@ -1213,8 +1234,7 @@ namespace tools bool remove_rings(const cryptonote::transaction_prefix &tx); bool get_ring(const crypto::chacha_key &key, const crypto::key_image &key_image, std::vector<uint64_t> &outs); crypto::chacha_key get_ringdb_key(); - void cache_ringdb_key(); - void clear_ringdb_key(); + void setup_keys(const epee::wipeable_string &password); bool get_rct_distribution(uint64_t &start_height, std::vector<uint64_t> &distribution); @@ -1247,7 +1267,7 @@ namespace tools std::unordered_map<std::string, std::string> m_attributes; std::vector<tools::wallet2::address_book_row> m_address_book; std::pair<std::map<std::string, std::string>, std::vector<std::string>> m_account_tags; - uint64_t m_upper_transaction_size_limit; //TODO: auto-calc this value or request from daemon, now use some fixed value + uint64_t m_upper_transaction_weight_limit; //TODO: auto-calc this value or request from daemon, now use some fixed value const std::vector<std::vector<tools::wallet2::multisig_info>> *m_multisig_rescan_info; const std::vector<std::vector<rct::key>> *m_multisig_rescan_k; @@ -1255,6 +1275,7 @@ namespace tools boost::mutex m_daemon_rpc_mutex; + bool m_trusted_daemon; i_wallet2_callback* m_callback; bool m_key_on_device; cryptonote::network_type m_nettype; @@ -1317,6 +1338,11 @@ namespace tools uint64_t m_last_block_reward; std::unique_ptr<tools::file_locker> m_keys_file_locker; + + crypto::chacha_key m_cache_key; + boost::optional<epee::wipeable_string> m_encrypt_keys_after_refresh; + + bool m_rpc; }; } BOOST_CLASS_VERSION(tools::wallet2, 25) @@ -1793,16 +1819,16 @@ namespace tools //---------------------------------------------------------------------------------------------------- template<typename T> void wallet2::transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outs_count, const std::vector<size_t> &unused_transfers_indices, - uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, bool trusted_daemon) + uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy) { pending_tx ptx; cryptonote::transaction tx; - transfer(dsts, fake_outs_count, unused_transfers_indices, unlock_time, fee, extra, destination_split_strategy, dust_policy, tx, ptx, trusted_daemon); + transfer(dsts, fake_outs_count, unused_transfers_indices, unlock_time, fee, extra, destination_split_strategy, dust_policy, tx, ptx); } template<typename T> void wallet2::transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices, - uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, cryptonote::transaction& tx, pending_tx &ptx, bool trusted_daemon) + uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, cryptonote::transaction& tx, pending_tx &ptx) { using namespace cryptonote; // throw if attempting a transaction with no destinations @@ -1810,7 +1836,7 @@ namespace tools THROW_WALLET_EXCEPTION_IF(m_multisig, error::wallet_internal_error, "Multisig wallets cannot spend non rct outputs"); - uint64_t upper_transaction_size_limit = get_upper_transaction_size_limit(); + uint64_t upper_transaction_weight_limit = get_upper_transaction_weight_limit(); uint64_t needed_money = fee; // calculate total amount being sent to all destinations @@ -1825,7 +1851,7 @@ namespace tools // randomly select inputs for transaction // throw if requested send amount is greater than (unlocked) amount available to send std::vector<size_t> selected_transfers; - uint64_t found_money = select_transfers(needed_money, unused_transfers_indices, selected_transfers, trusted_daemon); + uint64_t found_money = select_transfers(needed_money, unused_transfers_indices, selected_transfers); THROW_WALLET_EXCEPTION_IF(found_money < needed_money, error::not_enough_unlocked_money, found_money, needed_money - fee, fee); uint32_t subaddr_account = m_transfers[*selected_transfers.begin()].m_subaddr_index.major; @@ -1940,9 +1966,9 @@ namespace tools crypto::secret_key tx_key; std::vector<crypto::secret_key> additional_tx_keys; rct::multisig_out msout; - bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sources, splitted_dsts, change_dts.addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, false, m_multisig ? &msout : NULL); + bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sources, splitted_dsts, change_dts.addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, rct::RangeProofBorromean, m_multisig ? &msout : NULL); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sources, splitted_dsts, unlock_time, m_nettype); - THROW_WALLET_EXCEPTION_IF(upper_transaction_size_limit <= get_object_blobsize(tx), error::tx_too_big, tx, upper_transaction_size_limit); + THROW_WALLET_EXCEPTION_IF(upper_transaction_weight_limit <= get_transaction_weight(tx), error::tx_too_big, tx, upper_transaction_weight_limit); std::string key_images; bool all_are_txin_to_key = std::all_of(tx.vin.begin(), tx.vin.end(), [&](const txin_v& s_e) -> bool diff --git a/src/wallet/wallet_errors.h b/src/wallet/wallet_errors.h index e80652750..a30e807b1 100644 --- a/src/wallet/wallet_errors.h +++ b/src/wallet/wallet_errors.h @@ -53,6 +53,7 @@ namespace tools // wallet_not_initialized // multisig_export_needed // multisig_import_needed + // password_needed // std::logic_error // wallet_logic_error * // file_exists @@ -209,6 +210,14 @@ namespace tools } }; //---------------------------------------------------------------------------------------------------- + struct password_needed : public wallet_runtime_error + { + explicit password_needed(std::string&& loc, const std::string &msg = "Password needed") + : wallet_runtime_error(std::move(loc), msg) + { + } + }; + //---------------------------------------------------------------------------------------------------- const char* const file_error_messages[] = { "file already exists", "file not found", @@ -670,30 +679,30 @@ namespace tools //---------------------------------------------------------------------------------------------------- struct tx_too_big : public transfer_error { - explicit tx_too_big(std::string&& loc, const cryptonote::transaction& tx, uint64_t tx_size_limit) + explicit tx_too_big(std::string&& loc, const cryptonote::transaction& tx, uint64_t tx_weight_limit) : transfer_error(std::move(loc), "transaction is too big") , m_tx(tx) - , m_tx_size_limit(tx_size_limit) + , m_tx_weight_limit(tx_weight_limit) { } const cryptonote::transaction& tx() const { return m_tx; } - uint64_t tx_size_limit() const { return m_tx_size_limit; } + uint64_t tx_weight_limit() const { return m_tx_weight_limit; } std::string to_string() const { std::ostringstream ss; cryptonote::transaction tx = m_tx; ss << transfer_error::to_string() << - ", tx_size_limit = " << m_tx_size_limit << - ", tx size = " << get_object_blobsize(m_tx) << + ", tx_weight_limit = " << m_tx_weight_limit << + ", tx weight = " << get_transaction_weight(m_tx) << ", tx:\n" << cryptonote::obj_to_json_str(tx); return ss.str(); } private: cryptonote::transaction m_tx; - uint64_t m_tx_size_limit; + uint64_t m_tx_weight_limit; }; //---------------------------------------------------------------------------------------------------- struct zero_destination : public transfer_error diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index 510cb3e58..0e0b2e4eb 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -59,7 +59,6 @@ namespace { const command_line::arg_descriptor<std::string, true> arg_rpc_bind_port = {"rpc-bind-port", "Sets bind port for server"}; const command_line::arg_descriptor<bool> arg_disable_rpc_login = {"disable-rpc-login", "Disable HTTP authentication for RPC connections served by this process"}; - const command_line::arg_descriptor<bool> arg_trusted_daemon = {"trusted-daemon", "Enable commands which rely on a trusted daemon", false}; const command_line::arg_descriptor<bool> arg_restricted = {"restricted-rpc", "Restricts to view-only commands", false}; const command_line::arg_descriptor<std::string> arg_wallet_dir = {"wallet-dir", "Directory for newly created wallets"}; const command_line::arg_descriptor<bool> arg_prompt_for_password = {"prompt-for-password", "Prompts for password when not provided", false}; @@ -100,7 +99,7 @@ namespace tools } //------------------------------------------------------------------------------------------------------------------------------ - wallet_rpc_server::wallet_rpc_server():m_wallet(NULL), rpc_login_file(), m_stop(false), m_trusted_daemon(false), m_restricted(false), m_vm(NULL) + wallet_rpc_server::wallet_rpc_server():m_wallet(NULL), rpc_login_file(), m_stop(false), m_restricted(false), m_vm(NULL) { } //------------------------------------------------------------------------------------------------------------------------------ @@ -120,7 +119,7 @@ namespace tools m_stop = false; m_net_server.add_idle_handler([this](){ try { - if (m_wallet) m_wallet->refresh(m_trusted_daemon); + if (m_wallet) m_wallet->refresh(m_wallet->is_trusted_daemon()); } catch (const std::exception& ex) { LOG_ERROR("Exception at while refreshing, what=" << ex.what()); } @@ -156,28 +155,10 @@ namespace tools return false; m_vm = vm; - tools::wallet2 *walvars; - std::unique_ptr<tools::wallet2> tmpwal; - if (m_wallet) - walvars = m_wallet; - else - { - tmpwal = tools::wallet2::make_dummy(*m_vm, password_prompter); - walvars = tmpwal.get(); - } boost::optional<epee::net_utils::http::login> http_login{}; std::string bind_port = command_line::get_arg(*m_vm, arg_rpc_bind_port); const bool disable_auth = command_line::get_arg(*m_vm, arg_disable_rpc_login); - m_trusted_daemon = command_line::get_arg(*m_vm, arg_trusted_daemon); - if (!command_line::has_arg(*m_vm, arg_trusted_daemon)) - { - if (tools::is_local_address(walvars->get_daemon_address())) - { - MINFO(tr("Daemon is local, assuming trusted")); - m_trusted_daemon = true; - } - } m_restricted = command_line::get_arg(*m_vm, arg_restricted); if (command_line::has_arg(*m_vm, arg_wallet_dir)) { @@ -857,7 +838,7 @@ namespace tools mixin = m_wallet->adjust_mixin(req.mixin); } uint32_t priority = m_wallet->adjust_priority(req.priority); - std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_2(dsts, mixin, req.unlock_time, priority, extra, req.account_index, req.subaddr_indices, m_trusted_daemon); + std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_2(dsts, mixin, req.unlock_time, priority, extra, req.account_index, req.subaddr_indices); if (ptx_vector.empty()) { @@ -918,7 +899,7 @@ namespace tools } uint32_t priority = m_wallet->adjust_priority(req.priority); LOG_PRINT_L2("on_transfer_split calling create_transactions_2"); - std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_2(dsts, mixin, req.unlock_time, priority, extra, req.account_index, req.subaddr_indices, m_trusted_daemon); + std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_2(dsts, mixin, req.unlock_time, priority, extra, req.account_index, req.subaddr_indices); LOG_PRINT_L2("on_transfer_split called create_transactions_2"); return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay, @@ -1079,7 +1060,7 @@ namespace tools try { - std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_unmixable_sweep_transactions(m_trusted_daemon); + std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_unmixable_sweep_transactions(); return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay, res.tx_hash_list, req.get_tx_hex, res.tx_blob_list, req.get_tx_metadata, res.tx_metadata_list, er); @@ -1127,7 +1108,7 @@ namespace tools mixin = m_wallet->adjust_mixin(req.mixin); } uint32_t priority = m_wallet->adjust_priority(req.priority); - std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_all(req.below_amount, dsts[0].addr, dsts[0].is_subaddress, mixin, req.unlock_time, priority, extra, req.account_index, req.subaddr_indices, m_trusted_daemon); + std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_all(req.below_amount, dsts[0].addr, dsts[0].is_subaddress, mixin, req.unlock_time, priority, extra, req.account_index, req.subaddr_indices); return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay, res.tx_hash_list, req.get_tx_hex, res.tx_blob_list, req.get_tx_metadata, res.tx_metadata_list, er); @@ -1183,7 +1164,7 @@ namespace tools mixin = m_wallet->adjust_mixin(req.mixin); } uint32_t priority = m_wallet->adjust_priority(req.priority); - std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_single(ki, dsts[0].addr, dsts[0].is_subaddress, mixin, req.unlock_time, priority, extra, m_trusted_daemon); + std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_single(ki, dsts[0].addr, dsts[0].is_subaddress, mixin, req.unlock_time, priority, extra); if (ptx_vector.empty()) { @@ -1552,7 +1533,6 @@ namespace tools rpc_transfers.spent = td.m_spent; rpc_transfers.global_index = td.m_global_output_index; rpc_transfers.tx_hash = epee::string_tools::pod_to_hex(td.m_txid); - rpc_transfers.tx_size = txBlob.size(); rpc_transfers.subaddr_index = td.m_subaddr_index.minor; rpc_transfers.key_image = req.verbose && td.m_key_image_known ? epee::string_tools::pod_to_hex(td.m_key_image) : ""; res.transfers.push_back(rpc_transfers); @@ -1574,11 +1554,13 @@ namespace tools if (req.key_type.compare("mnemonic") == 0) { - if (!m_wallet->get_seed(res.key)) + epee::wipeable_string seed; + if (!m_wallet->get_seed(seed)) { er.message = "The wallet is non-deterministic. Cannot display seed."; return false; } + res.key = std::string(seed.data(), seed.size()); // send to the network, then wipe RAM :D } else if(req.key_type.compare("view_key") == 0) { @@ -2302,7 +2284,7 @@ namespace tools er.message = "Command unavailable in restricted mode."; return false; } - if (!m_trusted_daemon) + if (!m_wallet->is_trusted_daemon()) { er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; er.message = "This command requires a trusted daemon."; @@ -2505,6 +2487,28 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_refresh(const wallet_rpc::COMMAND_RPC_REFRESH::request& req, wallet_rpc::COMMAND_RPC_REFRESH::response& res, epee::json_rpc::error& er) + { + if (!m_wallet) return not_open(er); + if (m_restricted) + { + er.code = WALLET_RPC_ERROR_CODE_DENIED; + er.message = "Command unavailable in restricted mode."; + return false; + } + try + { + m_wallet->refresh(m_wallet->is_trusted_daemon(), req.start_height, res.blocks_fetched, res.received_money); + return true; + } + catch (const std::exception& e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_rescan_spent(const wallet_rpc::COMMAND_RPC_RESCAN_SPENT::request& req, wallet_rpc::COMMAND_RPC_RESCAN_SPENT::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); @@ -2530,7 +2534,7 @@ namespace tools bool wallet_rpc_server::on_start_mining(const wallet_rpc::COMMAND_RPC_START_MINING::request& req, wallet_rpc::COMMAND_RPC_START_MINING::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (!m_trusted_daemon) + if (!m_wallet->is_trusted_daemon()) { er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; er.message = "This command requires a trusted daemon."; @@ -2636,7 +2640,7 @@ namespace tools command_line::add_arg(desc, arg_password); po::store(po::parse_command_line(argc, argv, desc), vm2); } - std::unique_ptr<tools::wallet2> wal = tools::wallet2::make_new(vm2, nullptr).first; + std::unique_ptr<tools::wallet2> wal = tools::wallet2::make_new(vm2, true, nullptr).first; if (!wal) { er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; @@ -2664,8 +2668,20 @@ namespace tools er.message = "Failed to generate wallet"; return false; } + if (m_wallet) + { + try + { + m_wallet->store(); + } + catch (const std::exception& e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } delete m_wallet; + } m_wallet = wal.release(); return true; } @@ -2710,7 +2726,7 @@ namespace tools } std::unique_ptr<tools::wallet2> wal = nullptr; try { - wal = tools::wallet2::make_from_file(vm2, wallet_file, nullptr).first; + wal = tools::wallet2::make_from_file(vm2, true, wallet_file, nullptr).first; } catch (const std::exception& e) { @@ -2722,12 +2738,74 @@ namespace tools er.message = "Failed to open wallet"; return false; } + if (m_wallet) + { + try + { + m_wallet->store(); + } + catch (const std::exception& e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } delete m_wallet; + } m_wallet = wal.release(); return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_close_wallet(const wallet_rpc::COMMAND_RPC_CLOSE_WALLET::request& req, wallet_rpc::COMMAND_RPC_CLOSE_WALLET::response& res, epee::json_rpc::error& er) + { + if (!m_wallet) return not_open(er); + + try + { + m_wallet->store(); + } + catch (const std::exception& e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + delete m_wallet; + m_wallet = NULL; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_change_wallet_password(const wallet_rpc::COMMAND_RPC_CHANGE_WALLET_PASSWORD::request& req, wallet_rpc::COMMAND_RPC_CHANGE_WALLET_PASSWORD::response& res, epee::json_rpc::error& er) + { + if (!m_wallet) return not_open(er); + if (m_restricted) + { + er.code = WALLET_RPC_ERROR_CODE_DENIED; + er.message = "Command unavailable in restricted mode."; + return false; + } + if (m_wallet->verify_password(req.old_password)) + { + try + { + m_wallet->rewrite(m_wallet->get_wallet_file(), req.new_password); + m_wallet->store(); + LOG_PRINT_L0("Wallet password changed."); + } + catch (const std::exception& e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + } + else + { + er.code = WALLET_RPC_ERROR_CODE_INVALID_PASSWORD; + er.message = "Invalid original password."; + return false; + } + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ void wallet_rpc_server::handle_rpc_exception(const std::exception_ptr& e, epee::json_rpc::error& er, int default_error_code) { try { @@ -2969,7 +3047,7 @@ namespace tools return false; } - if (m_trusted_daemon) + if (m_wallet->is_trusted_daemon()) { try { @@ -3192,7 +3270,6 @@ int main(int argc, char** argv) { tools::wallet2::init_options(desc_params); command_line::add_arg(desc_params, arg_rpc_bind_port); command_line::add_arg(desc_params, arg_disable_rpc_login); - command_line::add_arg(desc_params, arg_trusted_daemon); command_line::add_arg(desc_params, arg_restricted); cryptonote::rpc_args::init_options(desc_params); command_line::add_arg(desc_params, arg_wallet_file); @@ -3259,13 +3336,13 @@ int main(int argc, char** argv) { LOG_PRINT_L0(tools::wallet_rpc_server::tr("Loading wallet...")); if(!wallet_file.empty()) { - wal = tools::wallet2::make_from_file(*vm, wallet_file, password_prompt).first; + wal = tools::wallet2::make_from_file(*vm, true, wallet_file, password_prompt).first; } else { try { - wal = tools::wallet2::make_from_json(*vm, from_json, password_prompt); + wal = tools::wallet2::make_from_json(*vm, true, from_json, password_prompt); } catch (const std::exception &e) { @@ -3285,7 +3362,7 @@ int main(int argc, char** argv) { wal->stop(); }); - wal->refresh(command_line::get_arg(*vm, arg_trusted_daemon)); + wal->refresh(wal->is_trusted_daemon()); // if we ^C during potentially length load/refresh, there's no server loop yet if (quit) { diff --git a/src/wallet/wallet_rpc_server.h b/src/wallet/wallet_rpc_server.h index 25eb01ba9..ab7917a78 100644 --- a/src/wallet/wallet_rpc_server.h +++ b/src/wallet/wallet_rpc_server.h @@ -126,12 +126,15 @@ namespace tools MAP_JON_RPC_WE("get_address_book", on_get_address_book, wallet_rpc::COMMAND_RPC_GET_ADDRESS_BOOK_ENTRY) MAP_JON_RPC_WE("add_address_book", on_add_address_book, wallet_rpc::COMMAND_RPC_ADD_ADDRESS_BOOK_ENTRY) MAP_JON_RPC_WE("delete_address_book",on_delete_address_book,wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY) + MAP_JON_RPC_WE("refresh", on_refresh, wallet_rpc::COMMAND_RPC_REFRESH) MAP_JON_RPC_WE("rescan_spent", on_rescan_spent, wallet_rpc::COMMAND_RPC_RESCAN_SPENT) MAP_JON_RPC_WE("start_mining", on_start_mining, wallet_rpc::COMMAND_RPC_START_MINING) MAP_JON_RPC_WE("stop_mining", on_stop_mining, wallet_rpc::COMMAND_RPC_STOP_MINING) MAP_JON_RPC_WE("get_languages", on_get_languages, wallet_rpc::COMMAND_RPC_GET_LANGUAGES) MAP_JON_RPC_WE("create_wallet", on_create_wallet, wallet_rpc::COMMAND_RPC_CREATE_WALLET) MAP_JON_RPC_WE("open_wallet", on_open_wallet, wallet_rpc::COMMAND_RPC_OPEN_WALLET) + MAP_JON_RPC_WE("close_wallet", on_close_wallet, wallet_rpc::COMMAND_RPC_CLOSE_WALLET) + MAP_JON_RPC_WE("change_wallet_password", on_change_wallet_password, wallet_rpc::COMMAND_RPC_CHANGE_WALLET_PASSWORD) MAP_JON_RPC_WE("is_multisig", on_is_multisig, wallet_rpc::COMMAND_RPC_IS_MULTISIG) MAP_JON_RPC_WE("prepare_multisig", on_prepare_multisig, wallet_rpc::COMMAND_RPC_PREPARE_MULTISIG) MAP_JON_RPC_WE("make_multisig", on_make_multisig, wallet_rpc::COMMAND_RPC_MAKE_MULTISIG) @@ -200,12 +203,15 @@ namespace tools bool on_get_address_book(const wallet_rpc::COMMAND_RPC_GET_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_GET_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er); bool on_add_address_book(const wallet_rpc::COMMAND_RPC_ADD_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_ADD_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er); bool on_delete_address_book(const wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er); + bool on_refresh(const wallet_rpc::COMMAND_RPC_REFRESH::request& req, wallet_rpc::COMMAND_RPC_REFRESH::response& res, epee::json_rpc::error& er); bool on_rescan_spent(const wallet_rpc::COMMAND_RPC_RESCAN_SPENT::request& req, wallet_rpc::COMMAND_RPC_RESCAN_SPENT::response& res, epee::json_rpc::error& er); bool on_start_mining(const wallet_rpc::COMMAND_RPC_START_MINING::request& req, wallet_rpc::COMMAND_RPC_START_MINING::response& res, epee::json_rpc::error& er); bool on_stop_mining(const wallet_rpc::COMMAND_RPC_STOP_MINING::request& req, wallet_rpc::COMMAND_RPC_STOP_MINING::response& res, epee::json_rpc::error& er); bool on_get_languages(const wallet_rpc::COMMAND_RPC_GET_LANGUAGES::request& req, wallet_rpc::COMMAND_RPC_GET_LANGUAGES::response& res, epee::json_rpc::error& er); bool on_create_wallet(const wallet_rpc::COMMAND_RPC_CREATE_WALLET::request& req, wallet_rpc::COMMAND_RPC_CREATE_WALLET::response& res, epee::json_rpc::error& er); bool on_open_wallet(const wallet_rpc::COMMAND_RPC_OPEN_WALLET::request& req, wallet_rpc::COMMAND_RPC_OPEN_WALLET::response& res, epee::json_rpc::error& er); + bool on_close_wallet(const wallet_rpc::COMMAND_RPC_CLOSE_WALLET::request& req, wallet_rpc::COMMAND_RPC_CLOSE_WALLET::response& res, epee::json_rpc::error& er); + bool on_change_wallet_password(const wallet_rpc::COMMAND_RPC_CHANGE_WALLET_PASSWORD::request& req, wallet_rpc::COMMAND_RPC_CHANGE_WALLET_PASSWORD::response& res, epee::json_rpc::error& er); bool on_is_multisig(const wallet_rpc::COMMAND_RPC_IS_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_IS_MULTISIG::response& res, epee::json_rpc::error& er); bool on_prepare_multisig(const wallet_rpc::COMMAND_RPC_PREPARE_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_PREPARE_MULTISIG::response& res, epee::json_rpc::error& er); bool on_make_multisig(const wallet_rpc::COMMAND_RPC_MAKE_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_MAKE_MULTISIG::response& res, epee::json_rpc::error& er); @@ -236,7 +242,6 @@ namespace tools std::string m_wallet_dir; tools::private_file rpc_login_file; std::atomic<bool> m_stop; - bool m_trusted_daemon; bool m_restricted; const boost::program_options::variables_map *m_vm; }; diff --git a/src/wallet/wallet_rpc_server_commands_defs.h b/src/wallet/wallet_rpc_server_commands_defs.h index 48d881c4c..ce10e2917 100644 --- a/src/wallet/wallet_rpc_server_commands_defs.h +++ b/src/wallet/wallet_rpc_server_commands_defs.h @@ -47,7 +47,7 @@ // advance which version they will stop working with // Don't go over 32767 for any of these #define WALLET_RPC_VERSION_MAJOR 1 -#define WALLET_RPC_VERSION_MINOR 2 +#define WALLET_RPC_VERSION_MINOR 3 #define MAKE_WALLET_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define WALLET_RPC_VERSION MAKE_WALLET_RPC_VERSION(WALLET_RPC_VERSION_MAJOR, WALLET_RPC_VERSION_MINOR) namespace tools @@ -445,7 +445,6 @@ namespace wallet_rpc { std::string tx_hash; std::string tx_key; - std::list<std::string> amount_keys; uint64_t amount; uint64_t fee; std::string tx_blob; @@ -456,7 +455,6 @@ namespace wallet_rpc BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tx_hash) KV_SERIALIZE(tx_key) - KV_SERIALIZE(amount_keys) KV_SERIALIZE(amount) KV_SERIALIZE(fee) KV_SERIALIZE(tx_blob) @@ -867,7 +865,6 @@ namespace wallet_rpc bool spent; uint64_t global_index; std::string tx_hash; - uint64_t tx_size; uint32_t subaddr_index; std::string key_image; @@ -876,7 +873,6 @@ namespace wallet_rpc KV_SERIALIZE(spent) KV_SERIALIZE(global_index) KV_SERIALIZE(tx_hash) - KV_SERIALIZE(tx_size) KV_SERIALIZE(subaddr_index) KV_SERIALIZE(key_image) END_KV_SERIALIZE_MAP() @@ -1714,6 +1710,29 @@ namespace wallet_rpc }; }; + struct COMMAND_RPC_REFRESH + { + struct request + { + uint64_t start_height; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_OPT(start_height, (uint64_t) 0) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + uint64_t blocks_fetched; + bool received_money; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(blocks_fetched); + KV_SERIALIZE(received_money); + END_KV_SERIALIZE_MAP() + }; + }; + struct COMMAND_RPC_START_MINING { struct request @@ -1808,6 +1827,40 @@ namespace wallet_rpc }; }; + struct COMMAND_RPC_CLOSE_WALLET + { + struct request + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + + struct response + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + }; + + struct COMMAND_RPC_CHANGE_WALLET_PASSWORD + { + struct request + { + std::string old_password; + std::string new_password; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(old_password) + KV_SERIALIZE(new_password) + END_KV_SERIALIZE_MAP() + }; + struct response + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + }; + struct COMMAND_RPC_IS_MULTISIG { struct request |