diff options
Diffstat (limited to 'src')
84 files changed, 6530 insertions, 2582 deletions
diff --git a/src/blockchain_db/berkeleydb/db_bdb.cpp b/src/blockchain_db/berkeleydb/db_bdb.cpp index f827ab7c3..6c79120e8 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.cpp +++ b/src/blockchain_db/berkeleydb/db_bdb.cpp @@ -224,7 +224,7 @@ struct Dbt_safe : public Dbt namespace cryptonote { -void BlockchainBDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const crypto::hash& blk_hash) +void BlockchainBDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const crypto::hash& blk_hash) { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); @@ -255,7 +255,7 @@ void BlockchainBDB::add_block(const block& blk, const size_t& block_size, const if (res) throw0(DB_ERROR("Failed to add block blob to db transaction.")); - Dbt_copy<size_t> sz(block_size); + Dbt_copy<size_t> sz(block_weight); if (m_block_sizes->put(DB_DEFAULT_TX, &key, &sz, 0)) throw0(DB_ERROR("Failed to add block size to db transaction.")); @@ -1353,7 +1353,7 @@ uint64_t BlockchainBDB::get_top_block_timestamp() const return get_block_timestamp(m_height - 1); } -size_t BlockchainBDB::get_block_size(const uint64_t& height) const +size_t BlockchainBDB::get_block_weight(const uint64_t& height) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); @@ -1861,7 +1861,7 @@ void BlockchainBDB::block_txn_abort() // TODO } -uint64_t BlockchainBDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const std::vector<transaction>& txs) +uint64_t BlockchainBDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const std::vector<transaction>& txs) { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); @@ -1874,7 +1874,7 @@ uint64_t BlockchainBDB::add_block(const block& blk, const size_t& block_size, co uint64_t num_outputs = m_num_outputs; try { - BlockchainDB::add_block(blk, block_size, cumulative_difficulty, coins_generated, txs); + BlockchainDB::add_block(blk, block_weight, cumulative_difficulty, coins_generated, txs); m_write_txn = NULL; TIME_MEASURE_START(time1); diff --git a/src/blockchain_db/berkeleydb/db_bdb.h b/src/blockchain_db/berkeleydb/db_bdb.h index 8c99ab255..76d0a0517 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.h +++ b/src/blockchain_db/berkeleydb/db_bdb.h @@ -266,7 +266,7 @@ public: virtual uint64_t get_top_block_timestamp() const; - virtual size_t get_block_size(const uint64_t& height) const; + virtual size_t get_block_weight(const uint64_t& height) const; virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const; @@ -318,7 +318,7 @@ public: virtual bool has_key_image(const crypto::key_image& img) const; virtual uint64_t add_block( const block& blk - , const size_t& block_size + , size_t block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , const std::vector<transaction>& txs @@ -353,7 +353,7 @@ public: private: virtual void add_block( const block& blk - , const size_t& block_size + , size_t block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , const crypto::hash& block_hash diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp index 78d63fdcf..be0ffeac3 100644 --- a/src/blockchain_db/blockchain_db.cpp +++ b/src/blockchain_db/blockchain_db.cpp @@ -196,7 +196,7 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti } uint64_t BlockchainDB::add_block( const block& blk - , const size_t& block_size + , size_t block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , const std::vector<transaction>& txs @@ -241,7 +241,7 @@ uint64_t BlockchainDB::add_block( const block& blk // call out to subclass implementation to add the block & metadata time1 = epee::misc_utils::get_tick_count(); - add_block(blk, block_size, cumulative_difficulty, coins_generated, num_rct_outs, blk_hash); + add_block(blk, block_weight, cumulative_difficulty, coins_generated, num_rct_outs, blk_hash); TIME_MEASURE_FINISH(time1); time_add_block1 += time1; diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 4431ca44c..396ae7544 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -137,7 +137,7 @@ struct txpool_tx_meta_t { crypto::hash max_used_block_id; crypto::hash last_failed_id; - uint64_t blob_size; + uint64_t weight; uint64_t fee; uint64_t max_used_block_height; uint64_t last_failed_height; @@ -358,13 +358,13 @@ private: * subclass of DB_EXCEPTION * * @param blk the block to be added - * @param block_size the size of the block (transactions and all) + * @param block_weight the weight of the block (transactions and all) * @param cumulative_difficulty the accumulated difficulty after this block * @param coins_generated the number of coins generated total after this block * @param blk_hash the hash of the block */ virtual void add_block( const block& blk - , const size_t& block_size + , size_t block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , uint64_t num_rct_outs @@ -376,7 +376,7 @@ private: * * The subclass implementing this will remove the block data from the top * block in the chain. The data to be removed is that which was added in - * BlockchainDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const crypto::hash& blk_hash) + * BlockchainDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const crypto::hash& blk_hash) * * If any of this cannot be done, the subclass should throw the corresponding * subclass of DB_EXCEPTION @@ -789,7 +789,7 @@ public: * subclass of DB_EXCEPTION * * @param blk the block to be added - * @param block_size the size of the block (transactions and all) + * @param block_weight the size of the block (transactions and all) * @param cumulative_difficulty the accumulated difficulty after this block * @param coins_generated the number of coins generated total after this block * @param txs the transactions in the block @@ -797,7 +797,7 @@ public: * @return the height of the chain post-addition */ virtual uint64_t add_block( const block& blk - , const size_t& block_size + , size_t block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , const std::vector<transaction>& txs @@ -930,18 +930,18 @@ public: virtual uint64_t get_top_block_timestamp() const = 0; /** - * @brief fetch a block's size + * @brief fetch a block's weight * - * The subclass should return the size of the block with the + * The subclass should return the weight of the block with the * given height. * * If the block does not exist, the subclass should throw BLOCK_DNE * * @param height the height requested * - * @return the size + * @return the weight */ - virtual size_t get_block_size(const uint64_t& height) const = 0; + virtual size_t get_block_weight(const uint64_t& height) const = 0; /** * @brief fetch a block's cumulative difficulty diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 983a0168a..9e22e2e4b 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -256,7 +256,7 @@ typedef struct mdb_block_info_old uint64_t bi_height; uint64_t bi_timestamp; uint64_t bi_coins; - uint64_t bi_size; // a size_t really but we need 32-bit compat + uint64_t bi_weight; // a size_t really but we need 32-bit compat difficulty_type bi_diff; crypto::hash bi_hash; } mdb_block_info_old; @@ -266,7 +266,7 @@ typedef struct mdb_block_info uint64_t bi_height; uint64_t bi_timestamp; uint64_t bi_coins; - uint64_t bi_size; // a size_t really but we need 32-bit compat + uint64_t bi_weight; // a size_t really but we need 32-bit compat difficulty_type bi_diff; crypto::hash bi_hash; uint64_t bi_cum_rct; @@ -545,8 +545,8 @@ bool BlockchainLMDB::need_resize(uint64_t threshold_size) const LOG_PRINT_L1("Space used: " << size_used); LOG_PRINT_L1("Space remaining: " << mei.me_mapsize - size_used); LOG_PRINT_L1("Size threshold: " << threshold_size); - float resize_percent_old = RESIZE_PERCENT; - LOG_PRINT_L1(boost::format("Percent used: %.04f Percent threshold: %.04f") % ((double)size_used/mei.me_mapsize) % resize_percent_old); + float resize_percent = RESIZE_PERCENT; + LOG_PRINT_L1(boost::format("Percent used: %.04f Percent threshold: %.04f") % ((double)size_used/mei.me_mapsize) % resize_percent); if (threshold_size > 0) { @@ -559,10 +559,6 @@ bool BlockchainLMDB::need_resize(uint64_t threshold_size) const return false; } - std::mt19937 engine(std::random_device{}()); - std::uniform_real_distribution<double> fdis(0.6, 0.9); - double resize_percent = fdis(engine); - if ((double)size_used / mei.me_mapsize > resize_percent) { LOG_PRINT_L1("Threshold met (percent-based)"); @@ -656,8 +652,11 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks, uin block_rtxn_start(&rtxn, &rcurs); for (uint64_t block_num = block_start; block_num <= block_stop; ++block_num) { - uint32_t block_size = get_block_size(block_num); - total_block_size += block_size; + // we have access to block weight, which will be greater or equal to block size, + // so use this as a proxy. If it's too much off, we might have to check actual size, + // which involves reading more data, so is not really wanted + size_t block_weight = get_block_weight(block_num); + total_block_size += block_weight; // Track number of blocks being totalled here instead of assuming, in case // some blocks were to be skipped for being outliers. ++num_blocks_used; @@ -678,7 +677,7 @@ estim: return threshold_size; } -void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, +void BlockchainLMDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, uint64_t num_rct_outs, const crypto::hash& blk_hash) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -724,7 +723,7 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const bi.bi_height = m_height; bi.bi_timestamp = blk.timestamp; bi.bi_coins = coins_generated; - bi.bi_size = block_size; + bi.bi_weight = block_weight; bi.bi_diff = cumulative_difficulty; bi.bi_hash = blk_hash; bi.bi_cum_rct = num_rct_outs; @@ -747,7 +746,9 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const if (result) throw0(DB_ERROR(lmdb_error("Failed to add block height by hash to db transaction: ", result).c_str())); - m_cum_size += block_size; + // we use weight as a proxy for size, since we don't have size but weight is >= size + // and often actually equal + m_cum_size += block_weight; m_cum_count++; } @@ -1200,8 +1201,12 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) throw DB_ERROR("Database could not be opened"); } - if (tools::is_hdd(filename.c_str())) - MCLOG_RED(el::Level::Warning, "global", "The blockchain is on a rotating drive: this will be very slow, use a SSD if possible"); + boost::optional<bool> is_hdd_result = tools::is_hdd(filename.c_str()); + if (is_hdd_result) + { + if (is_hdd_result.value()) + MCLOG_RED(el::Level::Warning, "global", "The blockchain is on a rotating drive: this will be very slow, use a SSD if possible"); + } m_folder = filename; @@ -1971,14 +1976,25 @@ std::vector<uint64_t> BlockchainLMDB::get_block_cumulative_rct_outputs(const std MDB_val v; + uint64_t prev_height = heights[0]; for (uint64_t height: heights) { - MDB_val_set(v, height); - result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); + if (height == prev_height + 1) + { + MDB_val k2; + result = mdb_cursor_get(m_cur_block_info, &k2, &v, MDB_NEXT); + } + else + { + v.mv_size = sizeof(uint64_t); + v.mv_data = (void*)&height; + result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); + } if (result) throw0(DB_ERROR(lmdb_error("Error attempting to retrieve rct distribution from the db: ", result).c_str())); const mdb_block_info *bi = (const mdb_block_info *)v.mv_data; res.push_back(bi->bi_cum_rct); + prev_height = height; } TXN_POSTFIX_RDONLY(); @@ -2000,7 +2016,7 @@ uint64_t BlockchainLMDB::get_top_block_timestamp() const return get_block_timestamp(m_height - 1); } -size_t BlockchainLMDB::get_block_size(const uint64_t& height) const +size_t BlockchainLMDB::get_block_weight(const uint64_t& height) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -2018,7 +2034,7 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) const throw0(DB_ERROR("Error attempting to retrieve a block size from the db")); mdb_block_info *bi = (mdb_block_info *)result.mv_data; - size_t ret = bi->bi_size; + size_t ret = bi->bi_weight; TXN_POSTFIX_RDONLY(); return ret; } @@ -3079,7 +3095,7 @@ void BlockchainLMDB::block_txn_abort() } } -uint64_t BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, +uint64_t BlockchainLMDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const std::vector<transaction>& txs) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -3098,7 +3114,7 @@ uint64_t BlockchainLMDB::add_block(const block& blk, const size_t& block_size, c try { - BlockchainDB::add_block(blk, block_size, cumulative_difficulty, coins_generated, txs); + BlockchainDB::add_block(blk, block_weight, cumulative_difficulty, coins_generated, txs); } catch (const DB_ERROR_TXN_START& e) { @@ -3687,9 +3703,9 @@ void BlockchainLMDB::migrate_0_1() if (result) throw0(DB_ERROR(lmdb_error("Failed to get a record from block_sizes: ", result).c_str())); if (v.mv_size == sizeof(uint32_t)) - bi.bi_size = *(uint32_t *)v.mv_data; + bi.bi_weight = *(uint32_t *)v.mv_data; else - bi.bi_size = *(uint64_t *)v.mv_data; // this is a 32/64 compat bug in version 0 + bi.bi_weight = *(uint64_t *)v.mv_data; // this is a 32/64 compat bug in version 0 result = mdb_cursor_get(c_timestamps, &k, &v, MDB_NEXT); if (result) throw0(DB_ERROR(lmdb_error("Failed to get a record from block_timestamps: ", result).c_str())); @@ -4248,7 +4264,7 @@ void BlockchainLMDB::migrate_2_3() bi.bi_height = bi_old->bi_height; bi.bi_timestamp = bi_old->bi_timestamp; bi.bi_coins = bi_old->bi_coins; - bi.bi_size = bi_old->bi_size; + bi.bi_weight = bi_old->bi_weight; bi.bi_diff = bi_old->bi_diff; bi.bi_hash = bi_old->bi_hash; if (bi_old->bi_height >= distribution.size()) diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 8ff2073da..e1f748ed8 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -205,7 +205,7 @@ public: virtual uint64_t get_top_block_timestamp() const; - virtual size_t get_block_size(const uint64_t& height) const; + virtual size_t get_block_weight(const uint64_t& height) const; virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const; @@ -273,7 +273,7 @@ public: virtual bool for_all_outputs(uint64_t amount, const std::function<bool(uint64_t height)> &f) const; virtual uint64_t add_block( const block& blk - , const size_t& block_size + , size_t block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , const std::vector<transaction>& txs @@ -317,7 +317,7 @@ private: uint64_t get_estimated_batch_size(uint64_t batch_num_blocks, uint64_t batch_bytes) const; virtual void add_block( const block& blk - , const size_t& block_size + , size_t block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , uint64_t num_rct_outs @@ -449,7 +449,7 @@ private: #endif #endif - constexpr static float RESIZE_PERCENT = 0.8f; + constexpr static float RESIZE_PERCENT = 0.9f; }; } // namespace cryptonote diff --git a/src/blockchain_utilities/CMakeLists.txt b/src/blockchain_utilities/CMakeLists.txt index 338ec3e4b..fe57895e1 100644 --- a/src/blockchain_utilities/CMakeLists.txt +++ b/src/blockchain_utilities/CMakeLists.txt @@ -91,6 +91,28 @@ monero_private_headers(blockchain_usage +set(blockchain_ancestry_sources + blockchain_ancestry.cpp + ) + +set(blockchain_ancestry_private_headers) + +monero_private_headers(blockchain_ancestry + ${blockchain_ancestry_private_headers}) + + + +set(blockchain_depth_sources + blockchain_depth.cpp + ) + +set(blockchain_depth_private_headers) + +monero_private_headers(blockchain_depth + ${blockchain_depth_private_headers}) + + + monero_add_executable(blockchain_import ${blockchain_import_sources} ${blockchain_import_private_headers} @@ -183,3 +205,45 @@ set_property(TARGET blockchain_usage OUTPUT_NAME "monero-blockchain-usage") install(TARGETS blockchain_usage DESTINATION bin) +monero_add_executable(blockchain_ancestry + ${blockchain_ancestry_sources} + ${blockchain_ancestry_private_headers}) + +target_link_libraries(blockchain_ancestry + PRIVATE + cryptonote_core + blockchain_db + version + epee + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${Boost_THREAD_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES}) + +set_property(TARGET blockchain_ancestry + PROPERTY + OUTPUT_NAME "monero-blockchain-ancestry") +install(TARGETS blockchain_ancestry DESTINATION bin) + +monero_add_executable(blockchain_depth + ${blockchain_depth_sources} + ${blockchain_depth_private_headers}) + +target_link_libraries(blockchain_depth + PRIVATE + cryptonote_core + blockchain_db + version + epee + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${Boost_THREAD_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES}) + +set_property(TARGET blockchain_depth + PROPERTY + OUTPUT_NAME "monero-blockchain-depth") +install(TARGETS blockchain_depth DESTINATION bin) + diff --git a/src/blockchain_utilities/blockchain_ancestry.cpp b/src/blockchain_utilities/blockchain_ancestry.cpp new file mode 100644 index 000000000..2f0bbffd6 --- /dev/null +++ b/src/blockchain_utilities/blockchain_ancestry.cpp @@ -0,0 +1,772 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <unordered_map> +#include <unordered_set> +#include <boost/range/adaptor/transformed.hpp> +#include <boost/algorithm/string.hpp> +#include <boost/archive/portable_binary_iarchive.hpp> +#include <boost/archive/portable_binary_oarchive.hpp> +#include "common/unordered_containers_boost_serialization.h" +#include "common/command_line.h" +#include "common/varint.h" +#include "cryptonote_basic/cryptonote_boost_serialization.h" +#include "cryptonote_core/tx_pool.h" +#include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_core/blockchain.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/db_types.h" +#include "version.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "bcutil" + +namespace po = boost::program_options; +using namespace epee; +using namespace cryptonote; + +static bool stop_requested = false; + +struct ancestor +{ + uint64_t amount; + uint64_t offset; + + bool operator==(const ancestor &other) const { return amount == other.amount && offset == other.offset; } + + template <typename t_archive> void serialize(t_archive &a, const unsigned int ver) + { + a & amount; + a & offset; + } +}; +BOOST_CLASS_VERSION(ancestor, 0) + +namespace std +{ + template<> struct hash<ancestor> + { + size_t operator()(const ancestor &a) const + { + return a.amount ^ a.offset; // not that bad, since amount almost always have a high bit set, and offset doesn't + } + }; +} + +struct tx_data_t +{ + std::vector<std::pair<uint64_t, std::vector<uint64_t>>> vin; + std::vector<crypto::public_key> vout; + bool coinbase; + + tx_data_t(): coinbase(false) {} + tx_data_t(const cryptonote::transaction &tx) + { + coinbase = tx.vin.size() == 1 && tx.vin[0].type() == typeid(cryptonote::txin_gen); + if (!coinbase) + { + vin.reserve(tx.vin.size()); + for (size_t ring = 0; ring < tx.vin.size(); ++ring) + { + if (tx.vin[ring].type() == typeid(cryptonote::txin_to_key)) + { + const cryptonote::txin_to_key &txin = boost::get<cryptonote::txin_to_key>(tx.vin[ring]); + vin.push_back(std::make_pair(txin.amount, cryptonote::relative_output_offsets_to_absolute(txin.key_offsets))); + } + else + { + LOG_PRINT_L0("Bad vin type in txid " << get_transaction_hash(tx)); + throw std::runtime_error("Bad vin type"); + } + } + } + vout.reserve(tx.vout.size()); + for (size_t out = 0; out < tx.vout.size(); ++out) + { + if (tx.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get<cryptonote::txout_to_key>(tx.vout[out].target); + vout.push_back(txout.key); + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << get_transaction_hash(tx)); + throw std::runtime_error("Bad vout type"); + } + } + } + + template <typename t_archive> void serialize(t_archive &a, const unsigned int ver) + { + a & coinbase; + a & vin; + a & vout; + } +}; + +struct ancestry_state_t +{ + uint64_t height; + std::unordered_map<crypto::hash, std::unordered_set<ancestor>> ancestry; + std::unordered_map<ancestor, crypto::hash> output_cache; + std::unordered_map<crypto::hash, ::tx_data_t> tx_cache; + std::vector<cryptonote::block> block_cache; + + template <typename t_archive> void serialize(t_archive &a, const unsigned int ver) + { + a & height; + a & ancestry; + a & output_cache; + if (ver < 1) + { + std::unordered_map<crypto::hash, cryptonote::transaction> old_tx_cache; + a & old_tx_cache; + for (const auto i: old_tx_cache) + tx_cache.insert(std::make_pair(i.first, ::tx_data_t(i.second))); + } + else + { + a & tx_cache; + } + if (ver < 2) + { + std::unordered_map<uint64_t, cryptonote::block> old_block_cache; + a & old_block_cache; + block_cache.resize(old_block_cache.size()); + for (const auto i: old_block_cache) + block_cache[i.first] = i.second; + } + else + { + a & block_cache; + } + } +}; +BOOST_CLASS_VERSION(ancestry_state_t, 2) + +static void add_ancestor(std::unordered_map<ancestor, unsigned int> &ancestry, uint64_t amount, uint64_t offset) +{ + std::pair<std::unordered_map<ancestor, unsigned int>::iterator, bool> p = ancestry.insert(std::make_pair(ancestor{amount, offset}, 1)); + if (!p.second) + { + ++p.first->second; + } +} + +static size_t get_full_ancestry(const std::unordered_map<ancestor, unsigned int> &ancestry) +{ + size_t count = 0; + for (const auto &i: ancestry) + count += i.second; + return count; +} + +static size_t get_deduplicated_ancestry(const std::unordered_map<ancestor, unsigned int> &ancestry) +{ + return ancestry.size(); +} + +static void add_ancestry(std::unordered_map<crypto::hash, std::unordered_set<ancestor>> &ancestry, const crypto::hash &txid, const std::unordered_set<ancestor> &ancestors) +{ + std::pair<std::unordered_map<crypto::hash, std::unordered_set<ancestor>>::iterator, bool> p = ancestry.insert(std::make_pair(txid, ancestors)); + if (!p.second) + { + for (const auto &e: ancestors) + p.first->second.insert(e); + } +} + +static void add_ancestry(std::unordered_map<crypto::hash, std::unordered_set<ancestor>> &ancestry, const crypto::hash &txid, const ancestor &new_ancestor) +{ + std::pair<std::unordered_map<crypto::hash, std::unordered_set<ancestor>>::iterator, bool> p = ancestry.insert(std::make_pair(txid, std::unordered_set<ancestor>())); + p.first->second.insert(new_ancestor); +} + +static std::unordered_set<ancestor> get_ancestry(const std::unordered_map<crypto::hash, std::unordered_set<ancestor>> &ancestry, const crypto::hash &txid) +{ + std::unordered_map<crypto::hash, std::unordered_set<ancestor>>::const_iterator i = ancestry.find(txid); + if (i == ancestry.end()) + { + //MERROR("txid ancestry not found: " << txid); + //throw std::runtime_error("txid ancestry not found"); + return std::unordered_set<ancestor>(); + } + return i->second; +} + +int main(int argc, char* argv[]) +{ + TRY_ENTRY(); + + epee::string_tools::set_module_name_and_folder(argv[0]); + + std::string default_db_type = "lmdb"; + + std::string available_dbs = cryptonote::blockchain_db_types(", "); + available_dbs = "available: " + available_dbs; + + uint32_t log_level = 0; + + tools::on_startup(); + + boost::filesystem::path output_file_path; + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""}; + const command_line::arg_descriptor<std::string> arg_database = { + "database", available_dbs.c_str(), default_db_type + }; + const command_line::arg_descriptor<std::string> arg_txid = {"txid", "Get ancestry for this txid", ""}; + const command_line::arg_descriptor<uint64_t> arg_height = {"height", "Get ancestry for all txes at this height", 0}; + const command_line::arg_descriptor<bool> arg_all = {"all", "Include the whole chain", false}; + const command_line::arg_descriptor<bool> arg_cache_outputs = {"cache-outputs", "Cache outputs (memory hungry)", false}; + const command_line::arg_descriptor<bool> arg_cache_txes = {"cache-txes", "Cache txes (memory hungry)", false}; + const command_line::arg_descriptor<bool> arg_cache_blocks = {"cache-blocks", "Cache blocks (memory hungry)", false}; + const command_line::arg_descriptor<bool> arg_include_coinbase = {"include-coinbase", "Including coinbase tx", false}; + const command_line::arg_descriptor<bool> arg_show_cache_stats = {"show-cache-stats", "Show cache statistics", false}; + + command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_database); + command_line::add_arg(desc_cmd_sett, arg_txid); + command_line::add_arg(desc_cmd_sett, arg_height); + command_line::add_arg(desc_cmd_sett, arg_all); + command_line::add_arg(desc_cmd_sett, arg_cache_outputs); + command_line::add_arg(desc_cmd_sett, arg_cache_txes); + command_line::add_arg(desc_cmd_sett, arg_cache_blocks); + command_line::add_arg(desc_cmd_sett, arg_include_coinbase); + command_line::add_arg(desc_cmd_sett, arg_show_cache_stats); + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + auto parser = po::command_line_parser(argc, argv).options(desc_options); + po::store(parser.run(), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + mlog_configure(mlog_get_default_log_path("monero-blockchain-ancestry.log"), true); + if (!command_line::is_arg_defaulted(vm, arg_log_level)) + mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); + else + mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str()); + + LOG_PRINT_L0("Starting..."); + + std::string opt_data_dir = command_line::get_arg(vm, cryptonote::arg_data_dir); + bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); + bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); + network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; + std::string opt_txid_string = command_line::get_arg(vm, arg_txid); + uint64_t opt_height = command_line::get_arg(vm, arg_height); + bool opt_all = command_line::get_arg(vm, arg_all); + bool opt_cache_outputs = command_line::get_arg(vm, arg_cache_outputs); + bool opt_cache_txes = command_line::get_arg(vm, arg_cache_txes); + bool opt_cache_blocks = command_line::get_arg(vm, arg_cache_blocks); + bool opt_include_coinbase = command_line::get_arg(vm, arg_include_coinbase); + bool opt_show_cache_stats = command_line::get_arg(vm, arg_show_cache_stats); + + if ((!opt_txid_string.empty()) + !!opt_height + !!opt_all > 1) + { + std::cerr << "Only one of --txid, --height and --all can be given" << std::endl; + return 1; + } + crypto::hash opt_txid = crypto::null_hash; + if (!opt_txid_string.empty()) + { + if (!epee::string_tools::hex_to_pod(opt_txid_string, opt_txid)) + { + std::cerr << "Invalid txid" << std::endl; + return 1; + } + } + + std::string db_type = command_line::get_arg(vm, arg_database); + if (!cryptonote::blockchain_valid_db_type(db_type)) + { + std::cerr << "Invalid database type: " << db_type << std::endl; + return 1; + } + + // If we wanted to use the memory pool, we would set up a fake_core. + + // Use Blockchain instead of lower-level BlockchainDB for two reasons: + // 1. Blockchain has the init() method for easy setup + // 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash() + // + // cannot match blockchain_storage setup above with just one line, + // e.g. + // Blockchain* core_storage = new Blockchain(NULL); + // because unlike blockchain_storage constructor, which takes a pointer to + // tx_memory_pool, Blockchain's constructor takes tx_memory_pool object. + LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); + std::unique_ptr<Blockchain> core_storage; + tx_memory_pool m_mempool(*core_storage); + core_storage.reset(new Blockchain(m_mempool)); + BlockchainDB *db = new_db(db_type); + if (db == NULL) + { + LOG_ERROR("Attempted to use non-existent database type: " << db_type); + throw std::runtime_error("Attempting to use non-existent database type"); + } + LOG_PRINT_L0("database: " << db_type); + + const std::string filename = (boost::filesystem::path(opt_data_dir) / db->get_db_name()).string(); + LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); + + try + { + db->open(filename, DBF_RDONLY); + } + catch (const std::exception& e) + { + LOG_PRINT_L0("Error opening database: " << e.what()); + return 1; + } + r = core_storage->init(db, net_type); + + CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage"); + LOG_PRINT_L0("Source blockchain storage initialized OK"); + + std::vector<crypto::hash> start_txids; + + // forward method + if (opt_all) + { + uint64_t cached_txes = 0, cached_blocks = 0, cached_outputs = 0, total_txes = 0, total_blocks = 0, total_outputs = 0; + ancestry_state_t state; + + const std::string state_file_path = (boost::filesystem::path(opt_data_dir) / "ancestry-state.bin").string(); + LOG_PRINT_L0("Loading state data from " << state_file_path); + std::ifstream state_data_in; + state_data_in.open(state_file_path, std::ios_base::binary | std::ios_base::in); + if (!state_data_in.fail()) + { + try + { + boost::archive::portable_binary_iarchive a(state_data_in); + a >> state; + } + catch (const std::exception &e) + { + MERROR("Failed to load state data from " << state_file_path << ", restarting from scratch"); + state = ancestry_state_t(); + } + state_data_in.close(); + } + + tools::signal_handler::install([](int type) { + stop_requested = true; + }); + + MINFO("Starting from height " << state.height); + const uint64_t db_height = db->height(); + state.block_cache.reserve(db_height); + for (uint64_t h = state.height; h < db_height; ++h) + { + size_t block_ancestry_size = 0; + const crypto::hash block_hash = db->get_block_hash_from_height(h); + const cryptonote::blobdata bd = db->get_block_blob(block_hash); + ++total_blocks; + cryptonote::block b; + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + if (opt_cache_blocks) + { + state.block_cache.resize(h + 1); + state.block_cache[h] = b; + } + std::vector<crypto::hash> txids; + txids.reserve(1 + b.tx_hashes.size()); + if (opt_include_coinbase) + txids.push_back(cryptonote::get_transaction_hash(b.miner_tx)); + for (const auto &h: b.tx_hashes) + txids.push_back(h); + for (const crypto::hash &txid: txids) + { + printf("%lu/%lu \r", (unsigned long)h, (unsigned long)db_height); + fflush(stdout); + ::tx_data_t tx_data; + std::unordered_map<crypto::hash, ::tx_data_t>::const_iterator i = state.tx_cache.find(txid); + ++total_txes; + if (i != state.tx_cache.end()) + { + ++cached_txes; + tx_data = i->second; + } + else + { + cryptonote::blobdata bd; + if (!db->get_pruned_tx_blob(txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << txid << " from db"); + return 1; + } + cryptonote::transaction tx; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) + { + LOG_PRINT_L0("Bad tx: " << txid); + return 1; + } + tx_data = ::tx_data_t(tx); + if (opt_cache_txes) + state.tx_cache.insert(std::make_pair(txid, tx_data)); + } + if (tx_data.coinbase) + { + add_ancestry(state.ancestry, txid, std::unordered_set<ancestor>()); + } + else + { + for (size_t ring = 0; ring < tx_data.vin.size(); ++ring) + { + if (1) + { + const uint64_t amount = tx_data.vin[ring].first; + const std::vector<uint64_t> &absolute_offsets = tx_data.vin[ring].second; + for (uint64_t offset: absolute_offsets) + { + const output_data_t od = db->get_output_key(amount, offset); + add_ancestry(state.ancestry, txid, ancestor{amount, offset}); + cryptonote::block b; + ++total_blocks; + if (state.block_cache.size() > od.height && !state.block_cache[od.height].miner_tx.vin.empty()) + { + ++cached_blocks; + b = state.block_cache[od.height]; + } + else + { + const crypto::hash block_hash = db->get_block_hash_from_height(od.height); + cryptonote::blobdata bd = db->get_block_blob(block_hash); + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + if (opt_cache_blocks) + { + state.block_cache.resize(od.height + 1); + state.block_cache[od.height] = b; + } + } + // find the tx which created this output + bool found = false; + std::unordered_map<ancestor, crypto::hash>::const_iterator i = state.output_cache.find({amount, offset}); + ++total_outputs; + if (i != state.output_cache.end()) + { + ++cached_outputs; + add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, i->second)); + found = true; + } + else for (size_t out = 0; out < b.miner_tx.vout.size(); ++out) + { + if (b.miner_tx.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get<cryptonote::txout_to_key>(b.miner_tx.vout[out].target); + if (txout.key == od.pubkey) + { + found = true; + add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, cryptonote::get_transaction_hash(b.miner_tx))); + if (opt_cache_outputs) + state.output_cache.insert(std::make_pair(ancestor{amount, offset}, cryptonote::get_transaction_hash(b.miner_tx))); + break; + } + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << cryptonote::get_transaction_hash(b.miner_tx)); + return 1; + } + } + for (const crypto::hash &block_txid: b.tx_hashes) + { + if (found) + break; + ::tx_data_t tx_data2; + std::unordered_map<crypto::hash, ::tx_data_t>::const_iterator i = state.tx_cache.find(block_txid); + ++total_txes; + if (i != state.tx_cache.end()) + { + ++cached_txes; + tx_data2 = i->second; + } + else + { + cryptonote::blobdata bd; + if (!db->get_pruned_tx_blob(block_txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << block_txid << " from db"); + return 1; + } + cryptonote::transaction tx; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) + { + LOG_PRINT_L0("Bad tx: " << block_txid); + return 1; + } + tx_data2 = ::tx_data_t(tx); + if (opt_cache_txes) + state.tx_cache.insert(std::make_pair(block_txid, tx_data2)); + } + for (size_t out = 0; out < tx_data2.vout.size(); ++out) + { + if (tx_data2.vout[out] == od.pubkey) + { + found = true; + add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, block_txid)); + if (opt_cache_outputs) + state.output_cache.insert(std::make_pair(ancestor{amount, offset}, block_txid)); + break; + } + } + } + if (!found) + { + LOG_PRINT_L0("Output originating transaction not found"); + return 1; + } + } + } + } + } + const size_t ancestry_size = get_ancestry(state.ancestry, txid).size(); + block_ancestry_size += ancestry_size; + MINFO(txid << ": " << ancestry_size); + } + if (!txids.empty()) + { + std::string stats_msg; + if (opt_show_cache_stats) + stats_msg = std::string(", cache: txes ") + std::to_string(cached_txes*100./total_txes) + + ", blocks " + std::to_string(cached_blocks*100./total_blocks) + ", outputs " + + std::to_string(cached_outputs*100./total_outputs); + MINFO("Height " << h << ": " << (block_ancestry_size / txids.size()) << " average over " << txids.size() << stats_msg); + } + state.height = h; + if (stop_requested) + break; + } + + LOG_PRINT_L0("Saving state data to " << state_file_path); + std::ofstream state_data_out; + state_data_out.open(state_file_path, std::ios_base::binary | std::ios_base::out | std::ios::trunc); + if (!state_data_out.fail()) + { + try + { + boost::archive::portable_binary_oarchive a(state_data_out); + a << state; + } + catch (const std::exception &e) + { + MERROR("Failed to save state data to " << state_file_path); + } + state_data_out.close(); + } + + goto done; + } + + if (!opt_txid_string.empty()) + { + start_txids.push_back(opt_txid); + } + else + { + const crypto::hash block_hash = db->get_block_hash_from_height(opt_height); + const cryptonote::blobdata bd = db->get_block_blob(block_hash); + cryptonote::block b; + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + for (const crypto::hash &txid: b.tx_hashes) + start_txids.push_back(txid); + } + + if (start_txids.empty()) + { + LOG_PRINT_L0("No transaction(s) to check"); + return 1; + } + + for (const crypto::hash &start_txid: start_txids) + { + LOG_PRINT_L0("Checking ancestry for txid " << start_txid); + + std::unordered_map<ancestor, unsigned int> ancestry; + + std::list<crypto::hash> txids; + txids.push_back(start_txid); + while (!txids.empty()) + { + const crypto::hash txid = txids.front(); + txids.pop_front(); + + cryptonote::blobdata bd; + if (!db->get_pruned_tx_blob(txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << txid << " from db"); + return 1; + } + cryptonote::transaction tx; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) + { + LOG_PRINT_L0("Bad tx: " << txid); + return 1; + } + const bool coinbase = tx.vin.size() == 1 && tx.vin[0].type() == typeid(cryptonote::txin_gen); + if (coinbase) + continue; + + for (size_t ring = 0; ring < tx.vin.size(); ++ring) + { + if (tx.vin[ring].type() == typeid(cryptonote::txin_to_key)) + { + const cryptonote::txin_to_key &txin = boost::get<cryptonote::txin_to_key>(tx.vin[ring]); + const uint64_t amount = txin.amount; + auto absolute_offsets = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets); + for (uint64_t offset: absolute_offsets) + { + add_ancestor(ancestry, amount, offset); + const output_data_t od = db->get_output_key(amount, offset); + const crypto::hash block_hash = db->get_block_hash_from_height(od.height); + bd = db->get_block_blob(block_hash); + cryptonote::block b; + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + // find the tx which created this output + bool found = false; + for (size_t out = 0; out < b.miner_tx.vout.size(); ++out) + { + if (b.miner_tx.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get<cryptonote::txout_to_key>(b.miner_tx.vout[out].target); + if (txout.key == od.pubkey) + { + found = true; + txids.push_back(cryptonote::get_transaction_hash(b.miner_tx)); + MDEBUG("adding txid: " << cryptonote::get_transaction_hash(b.miner_tx)); + break; + } + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << cryptonote::get_transaction_hash(b.miner_tx)); + return 1; + } + } + for (const crypto::hash &block_txid: b.tx_hashes) + { + if (found) + break; + if (!db->get_pruned_tx_blob(block_txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << block_txid << " from db"); + return 1; + } + cryptonote::transaction tx2; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx2)) + { + LOG_PRINT_L0("Bad tx: " << block_txid); + return 1; + } + for (size_t out = 0; out < tx2.vout.size(); ++out) + { + if (tx2.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get<cryptonote::txout_to_key>(tx2.vout[out].target); + if (txout.key == od.pubkey) + { + found = true; + txids.push_back(block_txid); + MDEBUG("adding txid: " << block_txid); + break; + } + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << block_txid); + return 1; + } + } + } + if (!found) + { + LOG_PRINT_L0("Output originating transaction not found"); + return 1; + } + } + } + else + { + LOG_PRINT_L0("Bad vin type in txid " << txid); + return 1; + } + } + } + + MINFO("Ancestry for " << start_txid << ": " << get_deduplicated_ancestry(ancestry) << " / " << get_full_ancestry(ancestry)); + for (const auto &i: ancestry) + { + MINFO(cryptonote::print_money(i.first.amount) << "/" << i.first.offset << ": " << i.second); + } + } + +done: + core_storage->deinit(); + return 0; + + CATCH_ENTRY("Depth query error", 1); +} diff --git a/src/blockchain_utilities/blockchain_blackball.cpp b/src/blockchain_utilities/blockchain_blackball.cpp index 1653910fc..d9a179f64 100644 --- a/src/blockchain_utilities/blockchain_blackball.cpp +++ b/src/blockchain_utilities/blockchain_blackball.cpp @@ -50,64 +50,86 @@ namespace po = boost::program_options; using namespace epee; using namespace cryptonote; +static const char zerokey[8] = {0}; +static const MDB_val zerokval = { sizeof(zerokey), (void *)zerokey }; + +static uint64_t records_per_sync = 200; +static uint64_t db_flags = 0; +static MDB_dbi dbi_relative_rings; +static MDB_dbi dbi_outputs; +static MDB_dbi dbi_processed_txidx; +static MDB_dbi dbi_spent; +static MDB_dbi dbi_ring_instances; +static MDB_dbi dbi_stats; +static MDB_env *env = NULL; + struct output_data { uint64_t amount; - uint64_t index; - output_data(): amount(0), index(0) {} - output_data(uint64_t a, uint64_t i): amount(a), index(i) {} - bool operator==(const output_data &other) const { return other.amount == amount && other.index == index; } - template <typename t_archive> void serialize(t_archive &a, const unsigned int ver) - { - a & amount; - a & index; - } + uint64_t offset; + output_data(): amount(0), offset(0) {} + output_data(uint64_t a, uint64_t i): amount(a), offset(i) {} + bool operator==(const output_data &other) const { return other.amount == amount && other.offset == offset; } }; -BOOST_CLASS_VERSION(output_data, 0) -namespace std +// +// relative_rings: key_image -> vector<uint64_t> +// outputs: 128 bits -> set of key images +// processed_txidx: string -> uint64_t +// spent: amount -> offset +// ring_instances: vector<uint64_t> -> uint64_t +// stats: string -> arbitrary +// + +static bool parse_db_sync_mode(std::string db_sync_mode) { - template<> struct hash<output_data> + std::vector<std::string> options; + boost::trim(db_sync_mode); + boost::split(options, db_sync_mode, boost::is_any_of(" :")); + + for(const auto &option : options) + MDEBUG("option: " << option); + + // default to fast:async:1 + uint64_t DEFAULT_FLAGS = DBF_FAST; + + if(options.size() == 0) + { + // default to fast:async:1 + db_flags = DEFAULT_FLAGS; + } + + bool safemode = false; + if(options.size() >= 1) { - size_t operator()(const output_data &od) const + if(options[0] == "safe") { - const uint64_t data[2] = {od.amount, od.index}; - crypto::hash h; - crypto::cn_fast_hash(data, 2 * sizeof(uint64_t), h); - return reinterpret_cast<const std::size_t &>(h); + safemode = true; + db_flags = DBF_SAFE; } - }; - template<> struct hash<std::vector<uint64_t>> - { - size_t operator()(const std::vector<uint64_t> &v) const + else if(options[0] == "fast") { - crypto::hash h; - crypto::cn_fast_hash(v.data(), v.size() * sizeof(uint64_t), h); - return reinterpret_cast<const std::size_t &>(h); + db_flags = DBF_FAST; } - }; -} - -struct blackball_state_t -{ - std::unordered_map<crypto::key_image, std::vector<uint64_t>> relative_rings; - std::unordered_map<output_data, std::unordered_set<crypto::key_image>> outputs; - std::unordered_map<std::string, uint64_t> processed_heights; - std::unordered_set<output_data> spent; - std::unordered_map<std::vector<uint64_t>, size_t> ring_instances; + else if(options[0] == "fastest") + { + db_flags = DBF_FASTEST; + records_per_sync = 1000; // default to fastest:async:1000 + } + else + db_flags = DEFAULT_FLAGS; + } - template <typename t_archive> void serialize(t_archive &a, const unsigned int ver) + if(options.size() >= 2 && !safemode) { - a & relative_rings; - a & outputs; - a & processed_heights; - a & spent; - if (ver < 1) - return; - a & ring_instances; + char *endptr; + uint64_t bps = strtoull(options[1].c_str(), &endptr, 0); + if (*endptr == '\0') + records_per_sync = bps; } -}; -BOOST_CLASS_VERSION(blackball_state_t, 1) + + return true; +} static std::string get_default_db_path() { @@ -118,7 +140,195 @@ static std::string get_default_db_path() return dir.string(); } -static bool for_all_transactions(const std::string &filename, uint64_t &start_idx, const std::function<bool(const cryptonote::transaction_prefix&)> &f) +static std::string get_cache_filename(boost::filesystem::path filename) +{ + if (!boost::filesystem::is_directory(filename)) + filename.remove_filename(); + return filename.string(); +} + +static int compare_hash32(const MDB_val *a, const MDB_val *b) +{ + const uint32_t *va = (const uint32_t*) a->mv_data; + const uint32_t *vb = (const uint32_t*) b->mv_data; + for (int n = 7; n >= 0; n--) + { + if (va[n] == vb[n]) + continue; + return va[n] < vb[n] ? -1 : 1; + } + + return 0; +} + +int compare_uint64(const MDB_val *a, const MDB_val *b) +{ + const uint64_t va = *(const uint64_t *)a->mv_data; + const uint64_t vb = *(const uint64_t *)b->mv_data; + return (va < vb) ? -1 : va > vb; +} + +static int compare_double64(const MDB_val *a, const MDB_val *b) +{ + const uint64_t va = *(const uint64_t*) a->mv_data; + const uint64_t vb = *(const uint64_t*) b->mv_data; + if (va == vb) + { + const uint64_t va = ((const uint64_t*) a->mv_data)[1]; + const uint64_t vb = ((const uint64_t*) b->mv_data)[1]; + return va < vb ? -1 : va > vb; + } + return va < vb ? -1 : va > vb; +} + +static int resize_env(const char *db_path) +{ + MDB_envinfo mei; + MDB_stat mst; + int ret; + + size_t needed = 1000ul * 1024 * 1024; // at least 1000 MB + + ret = mdb_env_info(env, &mei); + if (ret) + return ret; + ret = mdb_env_stat(env, &mst); + if (ret) + return ret; + uint64_t size_used = mst.ms_psize * mei.me_last_pgno; + uint64_t mapsize = mei.me_mapsize; + if (size_used + needed > mei.me_mapsize) + { + try + { + boost::filesystem::path path(db_path); + boost::filesystem::space_info si = boost::filesystem::space(path); + if(si.available < needed) + { + MERROR("!! WARNING: Insufficient free space to extend database !!: " << (si.available >> 20L) << " MB available"); + return ENOSPC; + } + } + catch(...) + { + // print something but proceed. + MWARNING("Unable to query free disk space."); + } + + mapsize += needed; + } + return mdb_env_set_mapsize(env, mapsize); +} + +static void init(std::string cache_filename) +{ + MDB_txn *txn; + bool tx_active = false; + int dbr; + + MINFO("Creating blackball cache in " << cache_filename); + + tools::create_directories_if_necessary(cache_filename); + + int flags = 0; + if (db_flags & DBF_FAST) + flags |= MDB_NOSYNC; + if (db_flags & DBF_FASTEST) + flags |= MDB_NOSYNC | MDB_WRITEMAP | MDB_MAPASYNC; + + dbr = mdb_env_create(&env); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); + dbr = mdb_env_set_maxdbs(env, 6); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); + const std::string actual_filename = get_cache_filename(cache_filename); + dbr = mdb_env_open(env, actual_filename.c_str(), flags, 0664); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open rings database file '" + + actual_filename + "': " + std::string(mdb_strerror(dbr))); + + dbr = mdb_txn_begin(env, NULL, 0, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + dbr = mdb_dbi_open(txn, "relative_rings", MDB_CREATE | MDB_INTEGERKEY, &dbi_relative_rings); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_compare(txn, dbi_relative_rings, compare_hash32); + + dbr = mdb_dbi_open(txn, "outputs", MDB_CREATE | MDB_INTEGERKEY, &dbi_outputs); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_compare(txn, dbi_outputs, compare_double64); + + dbr = mdb_dbi_open(txn, "processed_txidx", MDB_CREATE, &dbi_processed_txidx); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_dbi_open(txn, "spent", MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi_spent); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_dupsort(txn, dbi_spent, compare_uint64); + + dbr = mdb_dbi_open(txn, "ring_instances", MDB_CREATE, &dbi_ring_instances); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_dbi_open(txn, "stats", MDB_CREATE, &dbi_stats); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); + tx_active = false; +} + +static void close() +{ + if (env) + { + mdb_dbi_close(env, dbi_relative_rings); + mdb_dbi_close(env, dbi_outputs); + mdb_dbi_close(env, dbi_processed_txidx); + mdb_dbi_close(env, dbi_spent); + mdb_dbi_close(env, dbi_ring_instances); + mdb_dbi_close(env, dbi_stats); + mdb_env_close(env); + env = NULL; + } +} + +static std::string compress_ring(const std::vector<uint64_t> &ring, std::string s = "") +{ + const size_t sz = s.size(); + s.resize(s.size() + 12 * ring.size()); + char *ptr = (char*)s.data() + sz; + for (uint64_t out: ring) + tools::write_varint(ptr, out); + if (ptr > s.data() + sz + 12 * ring.size()) + throw std::runtime_error("varint output overflow"); + s.resize(ptr - s.data()); + return s; +} + +static std::string compress_ring(uint64_t amount, const std::vector<uint64_t> &ring) +{ + char s[12], *ptr = s; + tools::write_varint(ptr, amount); + if (ptr > s + sizeof(s)) + throw std::runtime_error("varint output overflow"); + return compress_ring(ring, std::string(s, ptr-s)); +} + +static std::vector<uint64_t> decompress_ring(const std::string &s) +{ + std::vector<uint64_t> ring; + int read = 0; + for (std::string::const_iterator i = s.begin(); i != s.cend(); std::advance(i, read)) + { + uint64_t out; + std::string tmp(i, s.cend()); + read = tools::read_varint(tmp.begin(), tmp.end(), out); + CHECK_AND_ASSERT_THROW_MES(read > 0 && read <= 256, "Internal error decompressing ring"); + ring.push_back(out); + } + return ring; +} + +static bool for_all_transactions(const std::string &filename, uint64_t &start_idx, uint64_t &n_txes, const std::function<bool(const cryptonote::transaction_prefix&)> &f) { MDB_env *env; MDB_dbi dbi; @@ -126,6 +336,8 @@ static bool for_all_transactions(const std::string &filename, uint64_t &start_id MDB_cursor *cur; int dbr; bool tx_active = false; + MDB_val k; + MDB_val v; dbr = mdb_env_create(&env); if (dbr) throw std::runtime_error("Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); @@ -136,7 +348,7 @@ static bool for_all_transactions(const std::string &filename, uint64_t &start_id if (dbr) throw std::runtime_error("Failed to open rings database file '" + actual_filename + "': " + std::string(mdb_strerror(dbr))); - dbr = mdb_txn_begin(env, NULL, 0, &txn); + dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); tx_active = true; @@ -147,9 +359,11 @@ static bool for_all_transactions(const std::string &filename, uint64_t &start_id if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); dbr = mdb_cursor_open(txn, dbi, &cur); if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + MDB_stat stat; + dbr = mdb_stat(txn, dbi, &stat); + if (dbr) throw std::runtime_error("Failed to query m_block_info: " + std::string(mdb_strerror(dbr))); + n_txes = stat.ms_entries; - MDB_val k; - MDB_val v; bool fret = true; k.mv_size = sizeof(uint64_t); @@ -194,6 +408,77 @@ static bool for_all_transactions(const std::string &filename, uint64_t &start_id return fret; } +static uint64_t find_first_diverging_transaction(const std::string &first_filename, const std::string &second_filename) +{ + MDB_env *env[2]; + MDB_dbi dbi[2]; + MDB_txn *txn[2]; + MDB_cursor *cur[2]; + int dbr; + bool tx_active[2] = { false, false }; + uint64_t n_txes[2]; + MDB_val k; + MDB_val v[2]; + + epee::misc_utils::auto_scope_leave_caller txn_dtor[2]; + for (int i = 0; i < 2; ++i) + { + dbr = mdb_env_create(&env[i]); + if (dbr) throw std::runtime_error("Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); + dbr = mdb_env_set_maxdbs(env[i], 2); + if (dbr) throw std::runtime_error("Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); + const std::string actual_filename = i ? second_filename : first_filename; + dbr = mdb_env_open(env[i], actual_filename.c_str(), 0, 0664); + if (dbr) throw std::runtime_error("Failed to open rings database file '" + + actual_filename + "': " + std::string(mdb_strerror(dbr))); + + dbr = mdb_txn_begin(env[i], NULL, MDB_RDONLY, &txn[i]); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + txn_dtor[i] = epee::misc_utils::create_scope_leave_handler([&, i](){if (tx_active[i]) mdb_txn_abort(txn[i]);}); + tx_active[i] = true; + + dbr = mdb_dbi_open(txn[i], "txs_pruned", MDB_INTEGERKEY, &dbi[i]); + if (dbr) + dbr = mdb_dbi_open(txn[i], "txs", MDB_INTEGERKEY, &dbi[i]); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_open(txn[i], dbi[i], &cur[i]); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + MDB_stat stat; + dbr = mdb_stat(txn[i], dbi[i], &stat); + if (dbr) throw std::runtime_error("Failed to query m_block_info: " + std::string(mdb_strerror(dbr))); + n_txes[i] = stat.ms_entries; + } + + if (n_txes[0] == 0 || n_txes[1] == 0) + throw std::runtime_error("No transaction in the database"); + uint64_t lo = 0, hi = std::min(n_txes[0], n_txes[1]) - 1; + while (lo <= hi) + { + uint64_t mid = (lo + hi) / 2; + + k.mv_size = sizeof(uint64_t); + k.mv_data = (void*)∣ + dbr = mdb_cursor_get(cur[0], &k, &v[0], MDB_SET); + if (dbr) throw std::runtime_error("Failed to query transaction: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_get(cur[1], &k, &v[1], MDB_SET); + if (dbr) throw std::runtime_error("Failed to query transaction: " + std::string(mdb_strerror(dbr))); + if (v[0].mv_size == v[1].mv_size && !memcmp(v[0].mv_data, v[1].mv_data, v[0].mv_size)) + lo = mid + 1; + else + hi = mid - 1; + } + + for (int i = 0; i < 2; ++i) + { + mdb_cursor_close(cur[i]); + mdb_txn_commit(txn[i]); + tx_active[i] = false; + mdb_dbi_close(env[i], dbi[i]); + mdb_env_close(env[i]); + } + return hi; +} + static std::vector<uint64_t> canonicalize(const std::vector<uint64_t> &v) { std::vector<uint64_t> c; @@ -212,6 +497,499 @@ static std::vector<uint64_t> canonicalize(const std::vector<uint64_t> &v) return c; } +static uint64_t get_num_spent_outputs() +{ + MDB_txn *txn; + bool tx_active = false; + + int dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + MDB_cursor *cur; + dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for spent outputs: " + std::string(mdb_strerror(dbr))); + MDB_val k, v; + mdb_size_t count = 0, tmp; + + MDB_cursor_op op = MDB_FIRST; + while (1) + { + dbr = mdb_cursor_get(cur, &k, &v, op); + op = MDB_NEXT_NODUP; + if (dbr == MDB_NOTFOUND) + break; + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get first/next spent output: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_count(cur, &tmp); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to count entries: " + std::string(mdb_strerror(dbr))); + count += tmp; + } + + mdb_cursor_close(cur); + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn: " + std::string(mdb_strerror(dbr))); + tx_active = false; + + return count; +} + +static void add_spent_output(MDB_cursor *cur, const output_data &od) +{ + MDB_val k = {sizeof(od.amount), (void*)&od.amount}; + MDB_val v = {sizeof(od.offset), (void*)&od.offset}; + int dbr = mdb_cursor_put(cur, &k, &v, 0); + CHECK_AND_ASSERT_THROW_MES(!dbr || dbr == MDB_KEYEXIST, "Failed to add spent output: " + std::string(mdb_strerror(dbr))); +} + +static bool is_output_spent(MDB_cursor *cur, const output_data &od) +{ + MDB_val k = {sizeof(od.amount), (void*)&od.amount}; + MDB_val v = {sizeof(od.offset), (void*)&od.offset}; + int dbr = mdb_cursor_get(cur, &k, &v, MDB_GET_BOTH); + CHECK_AND_ASSERT_THROW_MES(!dbr || dbr == MDB_NOTFOUND, "Failed to get spent output: " + std::string(mdb_strerror(dbr))); + bool spent = dbr == 0; + return spent; +} + +static std::vector<output_data> get_spent_outputs(MDB_txn *txn) +{ + MDB_cursor *cur; + int dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for spent outputs: " + std::string(mdb_strerror(dbr))); + MDB_val k, v; + uint64_t count = 0; + dbr = mdb_cursor_get(cur, &k, &v, MDB_FIRST); + if (dbr != MDB_NOTFOUND) + { + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get first spent output: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_count(cur, &count); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to count entries: " + std::string(mdb_strerror(dbr))); + } + std::vector<output_data> outs; + outs.reserve(count); + while (1) + { + outs.push_back({*(const uint64_t*)k.mv_data, *(const uint64_t*)v.mv_data}); + dbr = mdb_cursor_get(cur, &k, &v, MDB_NEXT); + if (dbr == MDB_NOTFOUND) + break; + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get next spent output: " + std::string(mdb_strerror(dbr))); + } + mdb_cursor_close(cur); + return outs; +} + +static uint64_t get_processed_txidx(const std::string &name) +{ + MDB_txn *txn; + bool tx_active = false; + + int dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + uint64_t height = 0; + MDB_val k, v; + k.mv_data = (void*)name.c_str(); + k.mv_size = name.size(); + dbr = mdb_get(txn, dbi_processed_txidx, &k, &v); + if (dbr != MDB_NOTFOUND) + { + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get processed height: " + std::string(mdb_strerror(dbr))); + height = *(const uint64_t*)v.mv_data; + } + + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn: " + std::string(mdb_strerror(dbr))); + tx_active = false; + + return height; +} + +static void set_processed_txidx(MDB_txn *txn, const std::string &name, uint64_t height) +{ + MDB_val k, v; + k.mv_data = (void*)name.c_str(); + k.mv_size = name.size(); + v.mv_data = (void*)&height; + v.mv_size = sizeof(height); + int dbr = mdb_put(txn, dbi_processed_txidx, &k, &v, 0); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set processed height: " + std::string(mdb_strerror(dbr))); +} + +static bool get_relative_ring(MDB_txn *txn, const crypto::key_image &ki, std::vector<uint64_t> &ring) +{ + MDB_val k, v; + k.mv_data = (void*)&ki; + k.mv_size = sizeof(ki); + int dbr = mdb_get(txn, dbi_relative_rings, &k, &v); + if (dbr == MDB_NOTFOUND) + return false; + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get relative ring: " + std::string(mdb_strerror(dbr))); + ring = decompress_ring(std::string((const char*)v.mv_data, v.mv_size)); + return true; +} + +static void set_relative_ring(MDB_txn *txn, const crypto::key_image &ki, const std::vector<uint64_t> &ring) +{ + const std::string sring = compress_ring(ring); + MDB_val k, v; + k.mv_data = (void*)&ki; + k.mv_size = sizeof(ki); + v.mv_data = (void*)sring.c_str(); + v.mv_size = sring.size(); + int dbr = mdb_put(txn, dbi_relative_rings, &k, &v, 0); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set relative ring: " + std::string(mdb_strerror(dbr))); +} + +static std::string keep_under_511(const std::string &s) +{ + if (s.size() <= 511) + return s; + crypto::hash hash; + crypto::cn_fast_hash(s.data(), s.size(), hash); + return std::string((const char*)&hash, 32); +} + +static uint64_t get_ring_instances(MDB_txn *txn, uint64_t amount, const std::vector<uint64_t> &ring) +{ + const std::string sring = keep_under_511(compress_ring(amount, ring)); + MDB_val k, v; + k.mv_data = (void*)sring.data(); + k.mv_size = sring.size(); + int dbr = mdb_get(txn, dbi_ring_instances, &k, &v); + if (dbr == MDB_NOTFOUND) + return 0; + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get ring instances: " + std::string(mdb_strerror(dbr))); + return *(const uint64_t*)v.mv_data; +} + +static uint64_t get_ring_subset_instances(MDB_txn *txn, uint64_t amount, const std::vector<uint64_t> &ring) +{ + uint64_t instances = get_ring_instances(txn, amount, ring); + if (ring.size() > 11) + return instances; + + uint64_t extra = 0; + std::vector<uint64_t> subset; + subset.reserve(ring.size()); + for (uint64_t mask = 1; mask < (1u << ring.size()) - 1; ++mask) + { + subset.resize(0); + for (size_t i = 0; i < ring.size(); ++i) + if ((mask >> i) & 1) + subset.push_back(ring[i]); + extra += get_ring_instances(txn, amount, subset); + } + return instances + extra; +} + +static uint64_t inc_ring_instances(MDB_txn *txn, uint64_t amount, const std::vector<uint64_t> &ring) +{ + const std::string sring = keep_under_511(compress_ring(amount, ring)); + MDB_val k, v; + k.mv_data = (void*)sring.data(); + k.mv_size = sring.size(); + + int dbr = mdb_get(txn, dbi_ring_instances, &k, &v); + CHECK_AND_ASSERT_THROW_MES(!dbr || dbr == MDB_NOTFOUND, "Failed to get ring instances: " + std::string(mdb_strerror(dbr))); + + uint64_t count; + if (dbr == MDB_NOTFOUND) + count = 1; + else + count = 1 + *(const uint64_t*)v.mv_data; + + v.mv_data = &count; + v.mv_size = sizeof(count); + dbr = mdb_put(txn, dbi_ring_instances, &k, &v, 0); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set ring instances: " + std::string(mdb_strerror(dbr))); + + return count; +} + +static std::vector<crypto::key_image> get_key_images(MDB_txn *txn, const output_data &od) +{ + MDB_val k, v; + k.mv_data = (void*)&od; + k.mv_size = sizeof(od); + int dbr = mdb_get(txn, dbi_outputs, &k, &v); + CHECK_AND_ASSERT_THROW_MES(!dbr || dbr == MDB_NOTFOUND, "Failed to get output: " + std::string(mdb_strerror(dbr))); + if (dbr == MDB_NOTFOUND) + return {}; + CHECK_AND_ASSERT_THROW_MES(v.mv_size % 32 == 0, "Unexpected record size"); + std::vector<crypto::key_image> key_images; + key_images.reserve(v.mv_size / 32); + const crypto::key_image *ki = (const crypto::key_image*)v.mv_data; + for (size_t n = 0; n < v.mv_size / 32; ++n) + key_images.push_back(*ki++); + return key_images; +} + +static void add_key_image(MDB_txn *txn, const output_data &od, const crypto::key_image &ki) +{ + MDB_val k, v; + k.mv_data = (void*)&od; + k.mv_size = sizeof(od); + int dbr = mdb_get(txn, dbi_outputs, &k, &v); + CHECK_AND_ASSERT_THROW_MES(!dbr || dbr == MDB_NOTFOUND, "Failed to get output"); + std::string data; + if (!dbr) + { + CHECK_AND_ASSERT_THROW_MES(v.mv_size % 32 == 0, "Unexpected record size"); + data = std::string((const char*)v.mv_data, v.mv_size); + } + data += std::string((const char*)&ki, sizeof(ki)); + + v.mv_data = (void*)data.data(); + v.mv_size = data.size(); + dbr = mdb_put(txn, dbi_outputs, &k, &v, 0); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set outputs: " + std::string(mdb_strerror(dbr))); +} + +static bool get_stat(MDB_txn *txn, const char *key, uint64_t &data) +{ + MDB_val k, v; + k.mv_data = (void*)key; + k.mv_size = strlen(key); + int dbr = mdb_get(txn, dbi_stats, &k, &v); + if (dbr == MDB_NOTFOUND) + return false; + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get stat record"); + CHECK_AND_ASSERT_THROW_MES(v.mv_size == sizeof(uint64_t), "Unexpected record size"); + data = *(const uint64_t*)v.mv_data; + return true; +} + +static void set_stat(MDB_txn *txn, const char *key, uint64_t data) +{ + MDB_val k, v; + k.mv_data = (void*)key; + k.mv_size = strlen(key); + v.mv_data = (void*)&data; + v.mv_size = sizeof(uint64_t); + int dbr = mdb_put(txn, dbi_stats, &k, &v, 0); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set stat record"); +} + +static void inc_stat(MDB_txn *txn, const char *key) +{ + uint64_t data; + if (!get_stat(txn, key, data)) + data = 0; + ++data; + set_stat(txn, key, data); +} + +static void open_db(const std::string &filename, MDB_env **env, MDB_txn **txn, MDB_cursor **cur, MDB_dbi *dbi) +{ + tools::create_directories_if_necessary(filename); + + int flags = MDB_RDONLY; + if (db_flags & DBF_FAST) + flags |= MDB_NOSYNC; + if (db_flags & DBF_FASTEST) + flags |= MDB_NOSYNC | MDB_WRITEMAP | MDB_MAPASYNC; + + int dbr = mdb_env_create(env); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); + dbr = mdb_env_set_maxdbs(*env, 1); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); + const std::string actual_filename = filename; + MINFO("Opening monero blockchain at " << actual_filename); + dbr = mdb_env_open(*env, actual_filename.c_str(), flags, 0664); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open rings database file '" + + actual_filename + "': " + std::string(mdb_strerror(dbr))); + + dbr = mdb_txn_begin(*env, NULL, MDB_RDONLY, txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_dbi_open(*txn, "output_amounts", MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, dbi); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_dupsort(*txn, *dbi, compare_uint64); + + dbr = mdb_cursor_open(*txn, *dbi, cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); +} + +static void close_db(MDB_env *env, MDB_txn *txn, MDB_cursor *cur, MDB_dbi dbi) +{ + mdb_txn_abort(txn); + mdb_cursor_close(cur); + mdb_dbi_close(env, dbi); + mdb_env_close(env); +} + +static void get_num_outputs(MDB_txn *txn, MDB_cursor *cur, MDB_dbi dbi, uint64_t &pre_rct, uint64_t &rct) +{ + uint64_t amount = 0; + MDB_val k = { sizeof(amount), (void*)&amount }, v; + int dbr = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (dbr == MDB_NOTFOUND) + { + rct = 0; + } + else + { + if (dbr) throw std::runtime_error("Record 0 not found: " + std::string(mdb_strerror(dbr))); + mdb_size_t count = 0; + dbr = mdb_cursor_count(cur, &count); + if (dbr) throw std::runtime_error("Failed to count records: " + std::string(mdb_strerror(dbr))); + rct = count; + } + MDB_stat s; + dbr = mdb_stat(txn, dbi, &s); + if (dbr) throw std::runtime_error("Failed to count records: " + std::string(mdb_strerror(dbr))); + if (s.ms_entries < rct) throw std::runtime_error("Inconsistent records: " + std::string(mdb_strerror(dbr))); + pre_rct = s.ms_entries - rct; +} + +static crypto::hash get_genesis_block_hash(const std::string &filename) +{ + MDB_env *env; + MDB_dbi dbi; + MDB_txn *txn; + int dbr; + bool tx_active = false; + + dbr = mdb_env_create(&env); + if (dbr) throw std::runtime_error("Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); + dbr = mdb_env_set_maxdbs(env, 1); + if (dbr) throw std::runtime_error("Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); + const std::string actual_filename = filename; + dbr = mdb_env_open(env, actual_filename.c_str(), 0, 0664); + if (dbr) throw std::runtime_error("Failed to open rings database file '" + + actual_filename + "': " + std::string(mdb_strerror(dbr))); + + dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + dbr = mdb_dbi_open(txn, "block_info", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi); + mdb_set_dupsort(txn, dbi, compare_uint64); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + uint64_t zero = 0; + MDB_val k = { sizeof(uint64_t), (void*)&zero}, v; + dbr = mdb_get(txn, dbi, &k, &v); + if (dbr) throw std::runtime_error("Failed to retrieve genesis block: " + std::string(mdb_strerror(dbr))); + crypto::hash genesis_block_hash = *(const crypto::hash*)(((const uint64_t*)v.mv_data) + 5); + mdb_dbi_close(env, dbi); + mdb_txn_abort(txn); + mdb_env_close(env); + tx_active = false; + return genesis_block_hash; +} + +static std::vector<std::pair<uint64_t, uint64_t>> load_outputs(const std::string &filename) +{ + std::vector<std::pair<uint64_t, uint64_t>> outputs; + uint64_t amount = std::numeric_limits<uint64_t>::max(); + FILE *f; + + f = fopen(filename.c_str(), "r"); + if (!f) + { + MERROR("Failed to load outputs from " << filename << ": " << strerror(errno)); + return {}; + } + while (1) + { + char s[256]; + fgets(s, sizeof(s), f); + if (feof(f)) + break; + const size_t len = strlen(s); + if (len > 0 && s[len - 1] == '\n') + s[len - 1] = 0; + if (!s[0]) + continue; + std::pair<uint64_t, uint64_t> output; + uint64_t offset, num_offsets; + if (sscanf(s, "@%" PRIu64, &amount) == 1) + { + continue; + } + if (amount == std::numeric_limits<uint64_t>::max()) + { + MERROR("Bad format in " << filename); + continue; + } + if (sscanf(s, "%" PRIu64 "*%" PRIu64, &offset, &num_offsets) == 2 && num_offsets < std::numeric_limits<uint64_t>::max() - offset) + { + while (num_offsets-- > 0) + outputs.push_back(std::make_pair(amount, offset++)); + } + else if (sscanf(s, "%" PRIu64, &offset) == 1) + { + outputs.push_back(std::make_pair(amount, offset)); + } + else + { + MERROR("Bad format in " << filename); + continue; + } + } + fclose(f); + return outputs; +} + +static bool export_spent_outputs(MDB_cursor *cur, const std::string &filename) +{ + FILE *f = fopen(filename.c_str(), "w"); + if (!f) + { + MERROR("Failed to open " << filename << ": " << strerror(errno)); + return false; + } + + uint64_t pending_amount = std::numeric_limits<uint64_t>::max(); + std::vector<uint64_t> pending_offsets; + MDB_val k, v; + MDB_cursor_op op = MDB_FIRST; + while (1) + { + int dbr = mdb_cursor_get(cur, &k, &v, op); + if (dbr == MDB_NOTFOUND) + break; + op = MDB_NEXT; + if (dbr) + { + fclose(f); + MERROR("Failed to enumerate spent outputs: " << mdb_strerror(dbr)); + return false; + } + const uint64_t amount = *(const uint64_t*)k.mv_data; + const uint64_t offset = *(const uint64_t*)v.mv_data; + if (!pending_offsets.empty() && (amount != pending_amount || pending_offsets.back()+1 != offset)) + { + if (pending_offsets.size() == 1) + fprintf(f, "%" PRIu64 "\n", pending_offsets.front()); + else + fprintf(f, "%" PRIu64 "*%" PRIu64 "\n", pending_offsets.front(), pending_offsets.size()); + pending_offsets.clear(); + } + if (pending_amount != amount) + { + fprintf(f, "@%" PRIu64 "\n", amount); + pending_amount = amount; + } + pending_offsets.push_back(offset); + } + if (!pending_offsets.empty()) + { + if (pending_offsets.size() == 1) + fprintf(f, "%" PRIu64 "\n", pending_offsets.front()); + else + fprintf(f, "%" PRIu64 "*%" PRIu64 "\n", pending_offsets.front(), pending_offsets.size()); + pending_offsets.clear(); + } + fclose(f); + return true; +} + int main(int argc, char* argv[]) { TRY_ENTRY(); @@ -231,31 +1009,37 @@ int main(int argc, char* argv[]) po::options_description desc_cmd_only("Command line options"); po::options_description desc_cmd_sett("Command line options and settings options"); - const command_line::arg_descriptor<std::string, false, true, 2> arg_blackball_db_dir = { + const command_line::arg_descriptor<std::string> arg_blackball_db_dir = { "blackball-db-dir", "Specify blackball database directory", get_default_db_path(), - {{ &arg_testnet_on, &arg_stagenet_on }}, - [](std::array<bool, 2> testnet_stagenet, bool defaulted, std::string val)->std::string { - if (testnet_stagenet[0]) - return (boost::filesystem::path(val) / "testnet").string(); - else if (testnet_stagenet[1]) - return (boost::filesystem::path(val) / "stagenet").string(); - return val; - } }; const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""}; const command_line::arg_descriptor<std::string> arg_database = { "database", available_dbs.c_str(), default_db_type }; const command_line::arg_descriptor<bool> arg_rct_only = {"rct-only", "Only work on ringCT outputs", false}; + const command_line::arg_descriptor<bool> arg_check_subsets = {"check-subsets", "Check ring subsets (very expensive)", false}; + const command_line::arg_descriptor<bool> arg_verbose = {"verbose", "Verbose output)", false}; const command_line::arg_descriptor<std::vector<std::string> > arg_inputs = {"inputs", "Path to Monero DB, and path to any fork DBs"}; + const command_line::arg_descriptor<std::string> arg_db_sync_mode = { + "db-sync-mode" + , "Specify sync option, using format [safe|fast|fastest]:[nrecords_per_sync]." + , "fast:1000" + }; + const command_line::arg_descriptor<std::string> arg_extra_spent_list = {"extra-spent-list", "Optional list of known spent outputs",""}; + const command_line::arg_descriptor<std::string> arg_export = {"export", "Filename to export the backball list to"}; + const command_line::arg_descriptor<bool> arg_force_chain_reaction_pass = {"force-chain-reaction-pass", "Run the chain reaction pass even if no new blockchain data was processed"}; command_line::add_arg(desc_cmd_sett, arg_blackball_db_dir); - command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); - command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); command_line::add_arg(desc_cmd_sett, arg_log_level); command_line::add_arg(desc_cmd_sett, arg_database); command_line::add_arg(desc_cmd_sett, arg_rct_only); + command_line::add_arg(desc_cmd_sett, arg_check_subsets); + command_line::add_arg(desc_cmd_sett, arg_verbose); + command_line::add_arg(desc_cmd_sett, arg_db_sync_mode); + command_line::add_arg(desc_cmd_sett, arg_extra_spent_list); + command_line::add_arg(desc_cmd_sett, arg_export); + command_line::add_arg(desc_cmd_sett, arg_force_chain_reaction_pass); command_line::add_arg(desc_cmd_sett, arg_inputs); command_line::add_arg(desc_cmd_only, command_line::arg_help); @@ -291,11 +1075,14 @@ int main(int argc, char* argv[]) LOG_PRINT_L0("Starting..."); - bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); - bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); - network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; output_file_path = command_line::get_arg(vm, arg_blackball_db_dir); bool opt_rct_only = command_line::get_arg(vm, arg_rct_only); + bool opt_check_subsets = command_line::get_arg(vm, arg_check_subsets); + bool opt_verbose = command_line::get_arg(vm, arg_verbose); + bool opt_force_chain_reaction_pass = command_line::get_arg(vm, arg_force_chain_reaction_pass); + std::string opt_export = command_line::get_arg(vm, arg_export); + std::string extra_spent_list = command_line::get_arg(vm, arg_extra_spent_list); + std::vector<std::pair<uint64_t, uint64_t>> extra_spent_outputs = extra_spent_list.empty() ? std::vector<std::pair<uint64_t, uint64_t>>() : load_outputs(extra_spent_list); std::string db_type = command_line::get_arg(vm, arg_database); if (!cryptonote::blockchain_valid_db_type(db_type)) @@ -304,121 +1091,99 @@ int main(int argc, char* argv[]) return 1; } - // If we wanted to use the memory pool, we would set up a fake_core. + std::string db_sync_mode = command_line::get_arg(vm, arg_db_sync_mode); + if (!parse_db_sync_mode(db_sync_mode)) + { + MERROR("Invalid db sync mode: " << db_sync_mode); + return 1; + } - // Use Blockchain instead of lower-level BlockchainDB for two reasons: - // 1. Blockchain has the init() method for easy setup - // 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash() - // - // cannot match blockchain_storage setup above with just one line, - // e.g. - // Blockchain* core_storage = new Blockchain(NULL); - // because unlike blockchain_storage constructor, which takes a pointer to - // tx_memory_pool, Blockchain's constructor takes tx_memory_pool object. - LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); const std::vector<std::string> inputs = command_line::get_arg(vm, arg_inputs); if (inputs.empty()) { LOG_PRINT_L0("No inputs given"); return 1; } - std::vector<std::unique_ptr<Blockchain>> core_storage(inputs.size()); - Blockchain *blockchain = NULL; - tx_memory_pool m_mempool(*blockchain); - for (size_t n = 0; n < inputs.size(); ++n) - { - core_storage[n].reset(new Blockchain(m_mempool)); - BlockchainDB* db = new_db(db_type); - if (db == NULL) - { - LOG_ERROR("Attempted to use non-existent database type: " << db_type); - throw std::runtime_error("Attempting to use non-existent database type"); - } - LOG_PRINT_L0("database: " << db_type); + const std::string cache_dir = (output_file_path / "blackball-cache").string(); + init(cache_dir); - std::string filename = inputs[n]; - while (boost::ends_with(filename, "/") || boost::ends_with(filename, "\\")) - filename.pop_back(); - LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); + LOG_PRINT_L0("Scanning for blackballable outputs..."); - try - { - db->open(filename, DBF_RDONLY); - } - catch (const std::exception& e) - { - LOG_PRINT_L0("Error opening database: " << e.what()); - return 1; - } - r = core_storage[n]->init(db, net_type); + size_t done = 0; - CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage"); - LOG_PRINT_L0("Source blockchain storage initialized OK"); - } + const uint64_t start_blackballed_outputs = get_num_spent_outputs(); - boost::filesystem::path direc(output_file_path.string()); - if (boost::filesystem::exists(direc)) - { - if (!boost::filesystem::is_directory(direc)) - { - MERROR("LMDB needs a directory path, but a file was passed: " << output_file_path.string()); - return 1; - } - } - else - { - if (!boost::filesystem::create_directories(direc)) - { - MERROR("Failed to create directory: " << output_file_path.string()); - return 1; - } - } + tools::ringdb ringdb(output_file_path.string(), epee::string_tools::pod_to_hex(get_genesis_block_hash(inputs[0]))); - LOG_PRINT_L0("Scanning for blackballable outputs..."); + bool stop_requested = false; + tools::signal_handler::install([&stop_requested](int type) { + stop_requested = true; + }); - size_t done = 0; - blackball_state_t state; - std::unordered_set<output_data> newly_spent; - const std::string state_file_path = (boost::filesystem::path(output_file_path) / "blackball-state.bin").string(); + int dbr = resize_env(cache_dir.c_str()); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to resize LMDB database: " + std::string(mdb_strerror(dbr))); + + // open first db + MDB_env *env0; + MDB_txn *txn0; + MDB_dbi dbi0; + MDB_cursor *cur0; + open_db(inputs[0], &env0, &txn0, &cur0, &dbi0); - LOG_PRINT_L0("Loading state data from " << state_file_path); - std::ifstream state_data_in; - state_data_in.open(state_file_path, std::ios_base::binary | std::ios_base::in); - if (!state_data_in.fail()) + if (!extra_spent_outputs.empty()) { - try + MINFO("Adding " << extra_spent_outputs.size() << " extra spent outputs"); + MDB_txn *txn; + int dbr = mdb_txn_begin(env, NULL, 0, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + MDB_cursor *cur; + dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr))); + + std::vector<std::pair<uint64_t, uint64_t>> blackballs; + for (const std::pair<uint64_t, uint64_t> &output: extra_spent_outputs) { - boost::archive::portable_binary_iarchive a(state_data_in); - a >> state; + if (!is_output_spent(cur, output_data(output.first, output.second))) + { + blackballs.push_back(output); + add_spent_output(cur, output_data(output.first, output.second)); + inc_stat(txn, output.first ? "pre-rct-extra" : "rct-ring-extra"); + } } - catch (const std::exception &e) + if (!blackballs.empty()) { - MERROR("Failed to load state data from " << state_file_path << ", restarting from scratch"); - state = blackball_state_t(); + ringdb.blackball(blackballs); + blackballs.clear(); } - state_data_in.close(); + mdb_cursor_close(cur); + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); } - uint64_t start_blackballed_outputs = state.spent.size(); - - cryptonote::block b = core_storage[0]->get_db().get_block_from_height(0); - tools::ringdb ringdb(output_file_path.string(), epee::string_tools::pod_to_hex(get_block_hash(b))); - - bool stop_requested = false; - tools::signal_handler::install([&stop_requested](int type) { - stop_requested = true; - }); for (size_t n = 0; n < inputs.size(); ++n) { const std::string canonical = boost::filesystem::canonical(inputs[n]).string(); - uint64_t start_idx = 0; - auto it = state.processed_heights.find(canonical); - if (it != state.processed_heights.end()) - start_idx = it->second; + uint64_t start_idx = get_processed_txidx(canonical); + if (n > 0 && start_idx == 0) + { + start_idx = find_first_diverging_transaction(inputs[0], inputs[n]); + LOG_PRINT_L0("First diverging transaction at " << start_idx); + } LOG_PRINT_L0("Reading blockchain from " << inputs[n] << " from " << start_idx); - for_all_transactions(inputs[n], start_idx, [&](const cryptonote::transaction_prefix &tx)->bool + MDB_txn *txn; + int dbr = mdb_txn_begin(env, NULL, 0, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + MDB_cursor *cur; + dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr))); + size_t records = 0; + const std::string filename = inputs[n]; + std::vector<std::pair<uint64_t, uint64_t>> blackballs; + uint64_t n_txes; + for_all_transactions(filename, start_idx, n_txes, [&](const cryptonote::transaction_prefix &tx)->bool { + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; for (const auto &in: tx.vin) { if (in.type() != typeid(txin_to_key)) @@ -430,37 +1195,65 @@ int main(int argc, char* argv[]) const std::vector<uint64_t> absolute = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets); if (n == 0) for (uint64_t out: absolute) - state.outputs[output_data(txin.amount, out)].insert(txin.k_image); + add_key_image(txn, output_data(txin.amount, out), txin.k_image); + std::vector<uint64_t> relative_ring; std::vector<uint64_t> new_ring = canonicalize(txin.key_offsets); const uint32_t ring_size = txin.key_offsets.size(); - state.ring_instances[new_ring] += 1; - if (ring_size == 1) + const uint64_t instances = inc_ring_instances(txn, txin.amount, new_ring); + if (n == 0 && ring_size == 1) { - const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, absolute[0]); - MINFO("Blackballing output " << pkey << ", due to being used in a 1-ring"); - ringdb.blackball(pkey); - newly_spent.insert(output_data(txin.amount, absolute[0])); + const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, absolute[0]); + if (opt_verbose) + { + MINFO("Blackballing output " << output.first << "/" << output.second << ", due to being used in a 1-ring"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + } + blackballs.push_back(output); + add_spent_output(cur, output_data(txin.amount, absolute[0])); + inc_stat(txn, txin.amount ? "pre-rct-ring-size-1" : "rct-ring-size-1"); } - else if (state.ring_instances[new_ring] == new_ring.size()) + else if (n == 0 && instances == new_ring.size()) { for (size_t o = 0; o < new_ring.size(); ++o) { - const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, absolute[o]); - MINFO("Blackballing output " << pkey << ", due to being used in " << new_ring.size() << " identical " << new_ring.size() << "-rings"); - ringdb.blackball(pkey); - newly_spent.insert(output_data(txin.amount, absolute[o])); + const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, absolute[o]); + if (opt_verbose) + { + MINFO("Blackballing output " << output.first << "/" << output.second << ", due to being used in " << new_ring.size() << " identical " << new_ring.size() << "-rings"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + } + blackballs.push_back(output); + add_spent_output(cur, output_data(txin.amount, absolute[o])); + inc_stat(txn, txin.amount ? "pre-rct-duplicate-rings" : "rct-duplicate-rings"); } } - else if (state.relative_rings.find(txin.k_image) != state.relative_rings.end()) + else if (n == 0 && opt_check_subsets && get_ring_subset_instances(txn, txin.amount, new_ring) >= new_ring.size()) { - MINFO("Key image " << txin.k_image << " already seen: rings " << - boost::join(state.relative_rings[txin.k_image] | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " ") << + for (size_t o = 0; o < new_ring.size(); ++o) + { + const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, absolute[o]); + if (opt_verbose) + { + MINFO("Blackballing output " << output.first << "/" << output.second << ", due to being used in " << new_ring.size() << " subsets of " << new_ring.size() << "-rings"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + } + blackballs.push_back(output); + add_spent_output(cur, output_data(txin.amount, absolute[o])); + inc_stat(txn, txin.amount ? "pre-rct-subset-rings" : "rct-subset-rings"); + } + } + else if (n > 0 && get_relative_ring(txn, txin.k_image, relative_ring)) + { + MDEBUG("Key image " << txin.k_image << " already seen: rings " << + boost::join(relative_ring | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " ") << ", " << boost::join(txin.key_offsets | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " ")); - if (state.relative_rings[txin.k_image] != txin.key_offsets) + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + if (relative_ring != txin.key_offsets) { - MINFO("Rings are different"); - const std::vector<uint64_t> r0 = cryptonote::relative_output_offsets_to_absolute(state.relative_rings[txin.k_image]); + MDEBUG("Rings are different"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + const std::vector<uint64_t> r0 = cryptonote::relative_output_offsets_to_absolute(relative_ring); const std::vector<uint64_t> r1 = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets); std::vector<uint64_t> common; for (uint64_t out: r0) @@ -471,17 +1264,24 @@ int main(int argc, char* argv[]) if (common.empty()) { MERROR("Rings for the same key image are disjoint"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; } else if (common.size() == 1) { - const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, common[0]); - MINFO("Blackballing output " << pkey << ", due to being used in rings with a single common element"); - ringdb.blackball(pkey); - newly_spent.insert(output_data(txin.amount, common[0])); + const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, common[0]); + if (opt_verbose) + { + MINFO("Blackballing output " << output.first << "/" << output.second << ", due to being used in rings with a single common element"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + } + blackballs.push_back(output); + add_spent_output(cur, output_data(txin.amount, common[0])); + inc_stat(txn, txin.amount ? "pre-rct-key-image-attack" : "rct-key-image-attack"); } else { - MINFO("The intersection has more than one element, it's still ok"); + MDEBUG("The intersection has more than one element, it's still ok"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; for (const auto &out: r0) if (std::find(common.begin(), common.end(), out) != common.end()) new_ring.push_back(out); @@ -489,78 +1289,172 @@ int main(int argc, char* argv[]) } } } - state.relative_rings[txin.k_image] = new_ring; + if (n == 0) + set_relative_ring(txn, txin.k_image, new_ring); + } + set_processed_txidx(txn, canonical, start_idx+1); + + ++records; + if (records >= records_per_sync) + { + if (!blackballs.empty()) + { + ringdb.blackball(blackballs); + blackballs.clear(); + } + mdb_cursor_close(cur); + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); + int dbr = resize_env(cache_dir.c_str()); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to resize LMDB database: " + std::string(mdb_strerror(dbr))); + dbr = mdb_txn_begin(env, NULL, 0, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr))); + records = 0; } + if (stop_requested) { - MINFO("Stopping scan, secondary passes will still happen..."); + MINFO("Stopping scan..."); return false; } return true; }); - LOG_PRINT_L0("blockchain from " << inputs[n] << " processed still height " << start_idx); - state.processed_heights[canonical] = start_idx; + mdb_cursor_close(cur); + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); + LOG_PRINT_L0("blockchain from " << inputs[n] << " processed till tx idx " << start_idx); if (stop_requested) break; } - while (!newly_spent.empty()) + std::vector<output_data> work_spent; + + if (stop_requested) + goto skip_secondary_passes; + + if (opt_force_chain_reaction_pass || get_num_spent_outputs() > start_blackballed_outputs) { - LOG_PRINT_L0("Secondary pass due to " << newly_spent.size() << " newly found spent outputs"); - std::unordered_set<output_data> work_spent = std::move(newly_spent); - newly_spent.clear(); + MDB_txn *txn; + dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + work_spent = get_spent_outputs(txn); + mdb_txn_abort(txn); + } + + while (!work_spent.empty()) + { + LOG_PRINT_L0("Secondary pass on " << work_spent.size() << " spent outputs"); - for (const auto &e: work_spent) - state.spent.insert(e); + int dbr = resize_env(cache_dir.c_str()); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to resize LMDB database: " + std::string(mdb_strerror(dbr))); - for (const output_data &od: work_spent) + MDB_txn *txn; + dbr = mdb_txn_begin(env, NULL, 0, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + MDB_cursor *cur; + dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr))); + + std::vector<std::pair<uint64_t, uint64_t>> blackballs; + std::vector<output_data> scan_spent = std::move(work_spent); + work_spent.clear(); + for (const output_data &od: scan_spent) { - for (const crypto::key_image &ki: state.outputs[od]) + std::vector<crypto::key_image> key_images = get_key_images(txn, od); + for (const crypto::key_image &ki: key_images) { - std::vector<uint64_t> absolute = cryptonote::relative_output_offsets_to_absolute(state.relative_rings[ki]); + std::vector<uint64_t> relative_ring; + CHECK_AND_ASSERT_THROW_MES(get_relative_ring(txn, ki, relative_ring), "Relative ring not found"); + std::vector<uint64_t> absolute = cryptonote::relative_output_offsets_to_absolute(relative_ring); size_t known = 0; uint64_t last_unknown = 0; for (uint64_t out: absolute) { output_data new_od(od.amount, out); - if (state.spent.find(new_od) != state.spent.end()) + if (is_output_spent(cur, new_od)) ++known; else last_unknown = out; } if (known == absolute.size() - 1) { - const crypto::public_key pkey = core_storage[0]->get_output_key(od.amount, last_unknown); - MINFO("Blackballing output " << pkey << ", due to being used in a " << - absolute.size() << "-ring where all other outputs are known to be spent"); - ringdb.blackball(pkey); - newly_spent.insert(output_data(od.amount, last_unknown)); + const std::pair<uint64_t, uint64_t> output = std::make_pair(od.amount, last_unknown); + if (opt_verbose) + { + MINFO("Blackballing output " << output.first << "/" << output.second << ", due to being used in a " << + absolute.size() << "-ring where all other outputs are known to be spent"); + } + blackballs.push_back(output); + add_spent_output(cur, output_data(od.amount, last_unknown)); + work_spent.push_back(output_data(od.amount, last_unknown)); + inc_stat(txn, od.amount ? "pre-rct-chain-reaction" : "rct-chain-reaction"); } } - } - } - LOG_PRINT_L0("Saving state data to " << state_file_path); - std::ofstream state_data_out; - state_data_out.open(state_file_path, std::ios_base::binary | std::ios_base::out | std::ios::trunc); - if (!state_data_out.fail()) - { - try - { - boost::archive::portable_binary_oarchive a(state_data_out); - a << state; + if (stop_requested) + { + MINFO("Stopping secondary passes. Secondary passes are not incremental, they will re-run fully."); + return false; + } } - catch (const std::exception &e) + if (!blackballs.empty()) { - MERROR("Failed to save state data to " << state_file_path); + ringdb.blackball(blackballs); + blackballs.clear(); } - state_data_out.close(); + mdb_cursor_close(cur); + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); + } + +skip_secondary_passes: + uint64_t diff = get_num_spent_outputs() - start_blackballed_outputs; + LOG_PRINT_L0(std::to_string(diff) << " new outputs blackballed, " << get_num_spent_outputs() << " total outputs blackballed"); + + MDB_txn *txn; + dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + uint64_t pre_rct = 0, rct = 0; + get_num_outputs(txn0, cur0, dbi0, pre_rct, rct); + MINFO("Total pre-rct outputs: " << pre_rct); + MINFO("Total rct outputs: " << rct); + static const struct { const char *key; uint64_t base; } stat_keys[] = { + { "pre-rct-ring-size-1", pre_rct }, { "rct-ring-size-1", rct }, + { "pre-rct-duplicate-rings", pre_rct }, { "rct-duplicate-rings", rct }, + { "pre-rct-subset-rings", pre_rct }, { "rct-subset-rings", rct }, + { "pre-rct-key-image-attack", pre_rct }, { "rct-key-image-attack", rct }, + { "pre-rct-extra", pre_rct }, { "rct-ring-extra", rct }, + { "pre-rct-chain-reaction", pre_rct }, { "rct-chain-reaction", rct }, + }; + for (const auto &key: stat_keys) + { + uint64_t data; + if (!get_stat(txn, key.key, data)) + data = 0; + float percent = key.base ? 100.0f * data / key.base : 0.0f; + MINFO(key.key << ": " << data << " (" << percent << "%)"); + } + mdb_txn_abort(txn); + + if (!opt_export.empty()) + { + MDB_txn *txn; + int dbr = mdb_txn_begin(env, NULL, 0, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + MDB_cursor *cur; + dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr))); + export_spent_outputs(cur, opt_export); + mdb_cursor_close(cur); + mdb_txn_abort(txn); } - uint64_t diff = state.spent.size() - start_blackballed_outputs; - LOG_PRINT_L0(std::to_string(diff) << " new outputs blackballed, " << state.spent.size() << " total outputs blackballed"); LOG_PRINT_L0("Blockchain blackball data exported OK"); + close_db(env0, txn0, cur0, dbi0); + close(); return 0; - CATCH_ENTRY("Export error", 1); + CATCH_ENTRY("Error", 1); } diff --git a/src/blockchain_utilities/blockchain_depth.cpp b/src/blockchain_utilities/blockchain_depth.cpp new file mode 100644 index 000000000..dd2387e5b --- /dev/null +++ b/src/blockchain_utilities/blockchain_depth.cpp @@ -0,0 +1,351 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <boost/range/adaptor/transformed.hpp> +#include <boost/algorithm/string.hpp> +#include "common/command_line.h" +#include "common/varint.h" +#include "cryptonote_core/tx_pool.h" +#include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_core/blockchain.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/db_types.h" +#include "version.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "bcutil" + +namespace po = boost::program_options; +using namespace epee; +using namespace cryptonote; + +int main(int argc, char* argv[]) +{ + TRY_ENTRY(); + + epee::string_tools::set_module_name_and_folder(argv[0]); + + std::string default_db_type = "lmdb"; + + std::string available_dbs = cryptonote::blockchain_db_types(", "); + available_dbs = "available: " + available_dbs; + + uint32_t log_level = 0; + + tools::on_startup(); + + boost::filesystem::path output_file_path; + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""}; + const command_line::arg_descriptor<std::string> arg_database = { + "database", available_dbs.c_str(), default_db_type + }; + const command_line::arg_descriptor<std::string> arg_txid = {"txid", "Get min depth for this txid", ""}; + const command_line::arg_descriptor<uint64_t> arg_height = {"height", "Get min depth for all txes at this height", 0}; + const command_line::arg_descriptor<bool> arg_include_coinbase = {"include-coinbase", "Include coinbase in the average", false}; + + command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_database); + command_line::add_arg(desc_cmd_sett, arg_txid); + command_line::add_arg(desc_cmd_sett, arg_height); + command_line::add_arg(desc_cmd_sett, arg_include_coinbase); + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + auto parser = po::command_line_parser(argc, argv).options(desc_options); + po::store(parser.run(), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + mlog_configure(mlog_get_default_log_path("monero-blockchain-depth.log"), true); + if (!command_line::is_arg_defaulted(vm, arg_log_level)) + mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); + else + mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str()); + + LOG_PRINT_L0("Starting..."); + + std::string opt_data_dir = command_line::get_arg(vm, cryptonote::arg_data_dir); + bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); + bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); + network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; + std::string opt_txid_string = command_line::get_arg(vm, arg_txid); + uint64_t opt_height = command_line::get_arg(vm, arg_height); + bool opt_include_coinbase = command_line::get_arg(vm, arg_include_coinbase); + + if (!opt_txid_string.empty() && opt_height) + { + std::cerr << "txid and height cannot be given at the same time" << std::endl; + return 1; + } + crypto::hash opt_txid = crypto::null_hash; + if (!opt_txid_string.empty()) + { + if (!epee::string_tools::hex_to_pod(opt_txid_string, opt_txid)) + { + std::cerr << "Invalid txid" << std::endl; + return 1; + } + } + + std::string db_type = command_line::get_arg(vm, arg_database); + if (!cryptonote::blockchain_valid_db_type(db_type)) + { + std::cerr << "Invalid database type: " << db_type << std::endl; + return 1; + } + + // If we wanted to use the memory pool, we would set up a fake_core. + + // Use Blockchain instead of lower-level BlockchainDB for two reasons: + // 1. Blockchain has the init() method for easy setup + // 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash() + // + // cannot match blockchain_storage setup above with just one line, + // e.g. + // Blockchain* core_storage = new Blockchain(NULL); + // because unlike blockchain_storage constructor, which takes a pointer to + // tx_memory_pool, Blockchain's constructor takes tx_memory_pool object. + LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); + std::unique_ptr<Blockchain> core_storage; + tx_memory_pool m_mempool(*core_storage); + core_storage.reset(new Blockchain(m_mempool)); + BlockchainDB *db = new_db(db_type); + if (db == NULL) + { + LOG_ERROR("Attempted to use non-existent database type: " << db_type); + throw std::runtime_error("Attempting to use non-existent database type"); + } + LOG_PRINT_L0("database: " << db_type); + + const std::string filename = (boost::filesystem::path(opt_data_dir) / db->get_db_name()).string(); + LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); + + try + { + db->open(filename, DBF_RDONLY); + } + catch (const std::exception& e) + { + LOG_PRINT_L0("Error opening database: " << e.what()); + return 1; + } + r = core_storage->init(db, net_type); + + CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage"); + LOG_PRINT_L0("Source blockchain storage initialized OK"); + + std::vector<crypto::hash> start_txids; + if (!opt_txid_string.empty()) + { + start_txids.push_back(opt_txid); + } + else + { + const crypto::hash block_hash = db->get_block_hash_from_height(opt_height); + const cryptonote::blobdata bd = db->get_block_blob(block_hash); + cryptonote::block b; + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + for (const crypto::hash &txid: b.tx_hashes) + start_txids.push_back(txid); + if (opt_include_coinbase) + start_txids.push_back(cryptonote::get_transaction_hash(b.miner_tx)); + } + + if (start_txids.empty()) + { + LOG_PRINT_L0("No transaction(s) to check"); + return 1; + } + + std::vector<uint64_t> depths; + for (const crypto::hash &start_txid: start_txids) + { + uint64_t depth = 0; + bool coinbase = false; + + LOG_PRINT_L0("Checking depth for txid " << start_txid); + std::vector<crypto::hash> txids(1, start_txid); + while (!coinbase) + { + LOG_PRINT_L0("Considering "<< txids.size() << " transaction(s) at depth " << depth); + std::vector<crypto::hash> new_txids; + for (const crypto::hash &txid: txids) + { + cryptonote::blobdata bd; + if (!db->get_pruned_tx_blob(txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << txid << " from db"); + return 1; + } + cryptonote::transaction tx; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) + { + LOG_PRINT_L0("Bad tx: " << txid); + return 1; + } + for (size_t ring = 0; ring < tx.vin.size(); ++ring) + { + if (tx.vin[ring].type() == typeid(cryptonote::txin_gen)) + { + MDEBUG(txid << " is a coinbase transaction"); + coinbase = true; + goto done; + } + if (tx.vin[ring].type() == typeid(cryptonote::txin_to_key)) + { + const cryptonote::txin_to_key &txin = boost::get<cryptonote::txin_to_key>(tx.vin[ring]); + const uint64_t amount = txin.amount; + auto absolute_offsets = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets); + for (uint64_t offset: absolute_offsets) + { + const output_data_t od = db->get_output_key(amount, offset); + const crypto::hash block_hash = db->get_block_hash_from_height(od.height); + bd = db->get_block_blob(block_hash); + cryptonote::block b; + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + // find the tx which created this output + bool found = false; + for (size_t out = 0; out < b.miner_tx.vout.size(); ++out) + { + if (b.miner_tx.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get<cryptonote::txout_to_key>(b.miner_tx.vout[out].target); + if (txout.key == od.pubkey) + { + found = true; + new_txids.push_back(cryptonote::get_transaction_hash(b.miner_tx)); + MDEBUG("adding txid: " << cryptonote::get_transaction_hash(b.miner_tx)); + break; + } + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << cryptonote::get_transaction_hash(b.miner_tx)); + return 1; + } + } + for (const crypto::hash &block_txid: b.tx_hashes) + { + if (found) + break; + if (!db->get_pruned_tx_blob(block_txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << block_txid << " from db"); + return 1; + } + cryptonote::transaction tx2; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx2)) + { + LOG_PRINT_L0("Bad tx: " << block_txid); + return 1; + } + for (size_t out = 0; out < tx2.vout.size(); ++out) + { + if (tx2.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get<cryptonote::txout_to_key>(tx2.vout[out].target); + if (txout.key == od.pubkey) + { + found = true; + new_txids.push_back(block_txid); + MDEBUG("adding txid: " << block_txid); + break; + } + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << block_txid); + return 1; + } + } + } + if (!found) + { + LOG_PRINT_L0("Output originating transaction not found"); + return 1; + } + } + } + else + { + LOG_PRINT_L0("Bad vin type in txid " << txid); + return 1; + } + } + } + if (!coinbase) + { + std::swap(txids, new_txids); + ++depth; + } + } +done: + LOG_PRINT_L0("Min depth for txid " << start_txid << ": " << depth); + depths.push_back(depth); + } + + uint64_t cumulative_depth = 0; + for (uint64_t depth: depths) + cumulative_depth += depth; + LOG_PRINT_L0("Average min depth for " << start_txids.size() << " transaction(s): " << cumulative_depth/(float)depths.size()); + LOG_PRINT_L0("Median min depth for " << start_txids.size() << " transaction(s): " << epee::misc_utils::median(depths)); + + core_storage->deinit(); + return 0; + + CATCH_ENTRY("Depth query error", 1); +} diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp index 7291dbd68..9ec768d26 100644 --- a/src/blockchain_utilities/blockchain_import.cpp +++ b/src/blockchain_utilities/blockchain_import.cpp @@ -474,17 +474,17 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path txs.push_back(tx); } - size_t block_size; + size_t block_weight; difficulty_type cumulative_difficulty; uint64_t coins_generated; - block_size = bp.block_size; + block_weight = bp.block_weight; cumulative_difficulty = bp.cumulative_difficulty; coins_generated = bp.coins_generated; try { - core.get_blockchain_storage().get_db().add_block(b, block_size, cumulative_difficulty, coins_generated, txs); + core.get_blockchain_storage().get_db().add_block(b, block_weight, cumulative_difficulty, coins_generated, txs); } catch (const std::exception& e) { diff --git a/src/blockchain_utilities/bootstrap_file.cpp b/src/blockchain_utilities/bootstrap_file.cpp index ba2697226..beaad2abc 100644 --- a/src/blockchain_utilities/bootstrap_file.cpp +++ b/src/blockchain_utilities/bootstrap_file.cpp @@ -236,11 +236,11 @@ void BootstrapFile::write_block(block& block) bool include_extra_block_data = true; if (include_extra_block_data) { - size_t block_size = m_blockchain_storage->get_db().get_block_size(block_height); + size_t block_weight = m_blockchain_storage->get_db().get_block_weight(block_height); difficulty_type cumulative_difficulty = m_blockchain_storage->get_db().get_block_cumulative_difficulty(block_height); uint64_t coins_generated = m_blockchain_storage->get_db().get_block_already_generated_coins(block_height); - bp.block_size = block_size; + bp.block_weight = block_weight; bp.cumulative_difficulty = cumulative_difficulty; bp.coins_generated = coins_generated; } diff --git a/src/blockchain_utilities/bootstrap_serialization.h b/src/blockchain_utilities/bootstrap_serialization.h index 8755d3fe3..278a7b40f 100644 --- a/src/blockchain_utilities/bootstrap_serialization.h +++ b/src/blockchain_utilities/bootstrap_serialization.h @@ -70,14 +70,14 @@ namespace cryptonote { cryptonote::block block; std::vector<transaction> txs; - size_t block_size; + size_t block_weight; difficulty_type cumulative_difficulty; uint64_t coins_generated; BEGIN_SERIALIZE() FIELD(block) FIELD(txs) - VARINT_FIELD(block_size) + VARINT_FIELD(block_weight) VARINT_FIELD(cumulative_difficulty) VARINT_FIELD(coins_generated) END_SERIALIZE() diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index f0df05b0d..c6bac2199 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -38,7 +38,8 @@ set(common_sources password.cpp perf_timer.cpp threadpool.cpp - updates.cpp) + updates.cpp + aligned.c) if (STACK_TRACE) list(APPEND common_sources stack_trace.cpp) @@ -67,7 +68,8 @@ set(common_private_headers perf_timer.h stack_trace.h threadpool.h - updates.h) + updates.h + aligned.h) monero_private_headers(common ${common_private_headers}) diff --git a/src/common/aligned.c b/src/common/aligned.c new file mode 100644 index 000000000..763dfd0e7 --- /dev/null +++ b/src/common/aligned.c @@ -0,0 +1,139 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <stdint.h> +#include <string.h> +#include "aligned.h" + +static inline int is_power_of_2(size_t n) { return n && (n & (n-1)) == 0; } + +#define MAGIC 0xaa0817161500ff81 +#define MAGIC_FREED 0xaa0817161500ff82 + +static void local_abort(const char *msg) +{ + fprintf(stderr, "%s\n", msg); +#ifdef NDEBUG + _exit(1); +#else + abort(); +#endif +} + +typedef struct +{ + uint64_t magic; + void *raw; + size_t bytes; + size_t align; +} control; + +void *aligned_malloc(size_t bytes, size_t align) +{ + void *raw, *ptr; + control *ctrl; + + if (!is_power_of_2(align)) + return NULL; + if (bytes > (size_t)-1 - align) + return NULL; + if (bytes + align > (size_t)-1 - sizeof(control)) + return NULL; + + raw = malloc(bytes + sizeof(control) + align); + if (!raw) + return NULL; + ptr = (void*)(((uintptr_t)raw + align + sizeof(control) - 1) & ~(align-1)); + ctrl = ((control*)ptr) - 1; + ctrl->magic = MAGIC; + ctrl->raw = raw; + ctrl->bytes = bytes; + ctrl->align = align; + return ptr; +} + +void *aligned_realloc(void *ptr, size_t bytes, size_t align) +{ + void *raw, *ptr2; + control *ctrl, *ctrl2; + + if (!ptr) + return aligned_malloc(bytes, align); + if (!bytes) + { + aligned_free(ptr); + return NULL; + } + if (!is_power_of_2(align)) + return NULL; + + ctrl = ((control*)ptr) - 1; + if (ctrl->magic == MAGIC_FREED) + local_abort("Double free detected"); + if (ctrl->magic != MAGIC) + local_abort("Freeing unallocated memory"); + if (ctrl->align != align) + return NULL; + if (ctrl->bytes >= bytes) + return ptr; + + if (ctrl->bytes > (size_t)-1 - ctrl->align) + return NULL; + if (ctrl->bytes + ctrl->align > (size_t)-1 - sizeof(control)) + return NULL; + + raw = malloc(bytes + sizeof(control) + ctrl->align); + if (!raw) + return NULL; + ptr2 = (void*)(((uintptr_t)raw + ctrl->align + sizeof(control) - 1) & ~(ctrl->align-1)); + memcpy(ptr2, ptr, ctrl->bytes); + ctrl2 = ((control*)ptr2) - 1; + ctrl2->magic = MAGIC; + ctrl2->raw = raw; + ctrl2->bytes = bytes; + ctrl2->align = ctrl->align; + ctrl->magic = MAGIC_FREED; + free(ctrl->raw); + return ptr2; +} + +void aligned_free(void *ptr) +{ + if (!ptr) + return; + control *ctrl = ((control*)ptr) - 1; + if (ctrl->magic == MAGIC_FREED) + local_abort("Double free detected"); + if (ctrl->magic != MAGIC) + local_abort("Freeing unallocated memory"); + ctrl->magic = MAGIC_FREED; + free(ctrl->raw); +} diff --git a/src/common/aligned.h b/src/common/aligned.h new file mode 100644 index 000000000..fed3ccb36 --- /dev/null +++ b/src/common/aligned.h @@ -0,0 +1,41 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +void *aligned_malloc(size_t bytes, size_t align); +void *aligned_realloc(void *ptr, size_t bytes, size_t align); +void aligned_free(void *ptr); + +#ifdef __cplusplus +} +#endif diff --git a/src/common/dns_utils.cpp b/src/common/dns_utils.cpp index 33f60bc3c..3f2bde620 100644 --- a/src/common/dns_utils.cpp +++ b/src/common/dns_utils.cpp @@ -97,11 +97,16 @@ get_builtin_cert(void) */ /** return the built in root DS trust anchor */ -static const char* +static const char* const* get_builtin_ds(void) { - return -". IN DS 19036 8 2 49AAC11D7B6F6446702E54A1607371607A1A41855200FD2CE1CDDE32F24E8FB5\n"; + static const char * const ds[] = + { + ". IN DS 19036 8 2 49AAC11D7B6F6446702E54A1607371607A1A41855200FD2CE1CDDE32F24E8FB5\n", + ". IN DS 20326 8 2 E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC683457104237C7F8EC8D\n", + NULL + }; + return ds; } /************************************************************ @@ -240,7 +245,12 @@ DNSResolver::DNSResolver() : m_data(new DNSResolverData()) ub_ctx_hosts(m_data->m_ub_context, NULL); } - ub_ctx_add_ta(m_data->m_ub_context, string_copy(::get_builtin_ds())); + const char * const *ds = ::get_builtin_ds(); + while (*ds) + { + MINFO("adding trust anchor: " << *ds); + ub_ctx_add_ta(m_data->m_ub_context, string_copy(*ds++)); + } } DNSResolver::~DNSResolver() diff --git a/src/common/perf_timer.cpp b/src/common/perf_timer.cpp index 16abdfd99..6910ebdd4 100644 --- a/src/common/perf_timer.cpp +++ b/src/common/perf_timer.cpp @@ -33,7 +33,7 @@ #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "perf" -namespace +namespace tools { uint64_t get_tick_count() { @@ -83,7 +83,7 @@ namespace tools el::Level performance_timer_log_level = el::Level::Debug; -static __thread std::vector<PerformanceTimer*> *performance_timers = NULL; +static __thread std::vector<LoggingPerformanceTimer*> *performance_timers = NULL; void set_performance_timer_log_level(el::Level level) { @@ -96,17 +96,24 @@ void set_performance_timer_log_level(el::Level level) performance_timer_log_level = level; } -PerformanceTimer::PerformanceTimer(const std::string &s, uint64_t unit, el::Level l): name(s), unit(unit), level(l), started(false), paused(false) +PerformanceTimer::PerformanceTimer(bool paused): started(true), paused(paused) +{ + if (paused) + ticks = 0; + else + ticks = get_tick_count(); +} + +LoggingPerformanceTimer::LoggingPerformanceTimer(const std::string &s, uint64_t unit, el::Level l): PerformanceTimer(), name(s), unit(unit), level(l) { - ticks = get_tick_count(); if (!performance_timers) { MLOG(level, "PERF ----------"); - performance_timers = new std::vector<PerformanceTimer*>(); + performance_timers = new std::vector<LoggingPerformanceTimer*>(); } else { - PerformanceTimer *pt = performance_timers->back(); + LoggingPerformanceTimer *pt = performance_timers->back(); if (!pt->started && !pt->paused) { size_t size = 0; for (const auto *tmp: *performance_timers) if (!tmp->paused) ++size; @@ -119,9 +126,14 @@ PerformanceTimer::PerformanceTimer(const std::string &s, uint64_t unit, el::Leve PerformanceTimer::~PerformanceTimer() { - performance_timers->pop_back(); if (!paused) ticks = get_tick_count() - ticks; +} + +LoggingPerformanceTimer::~LoggingPerformanceTimer() +{ + pause(); + performance_timers->pop_back(); char s[12]; snprintf(s, sizeof(s), "%8llu ", (unsigned long long)(ticks_to_ns(ticks) / (1000000000 / unit))); size_t size = 0; for (const auto *tmp: *performance_timers) if (!tmp->paused || tmp==this) ++size; diff --git a/src/common/perf_timer.h b/src/common/perf_timer.h index 0e910caf9..675d6234d 100644 --- a/src/common/perf_timer.h +++ b/src/common/perf_timer.h @@ -43,30 +43,46 @@ class PerformanceTimer; extern el::Level performance_timer_log_level; +uint64_t get_tick_count(); +uint64_t get_ticks_per_ns(); +uint64_t ticks_to_ns(uint64_t ticks); + class PerformanceTimer { public: - PerformanceTimer(const std::string &s, uint64_t unit, el::Level l = el::Level::Debug); + PerformanceTimer(bool paused = false); ~PerformanceTimer(); void pause(); void resume(); + uint64_t value() const { return ticks; } +void set(uint64_t v){ticks=v;} + +protected: + uint64_t ticks; + bool started; + bool paused; +}; + +class LoggingPerformanceTimer: public PerformanceTimer +{ +public: + LoggingPerformanceTimer(const std::string &s, uint64_t unit, el::Level l = el::Level::Debug); + ~LoggingPerformanceTimer(); + private: std::string name; uint64_t unit; el::Level level; - uint64_t ticks; - bool started; - bool paused; }; void set_performance_timer_log_level(el::Level level); -#define PERF_TIMER_UNIT(name, unit) tools::PerformanceTimer pt_##name(#name, unit, tools::performance_timer_log_level) -#define PERF_TIMER_UNIT_L(name, unit, l) tools::PerformanceTimer pt_##name(#name, unit, l) +#define PERF_TIMER_UNIT(name, unit) tools::LoggingPerformanceTimer pt_##name(#name, unit, tools::performance_timer_log_level) +#define PERF_TIMER_UNIT_L(name, unit, l) tools::LoggingPerformanceTimer pt_##name(#name, unit, l) #define PERF_TIMER(name) PERF_TIMER_UNIT(name, 1000) #define PERF_TIMER_L(name, l) PERF_TIMER_UNIT_L(name, 1000, l) -#define PERF_TIMER_START_UNIT(name, unit) std::unique_ptr<tools::PerformanceTimer> pt_##name(new tools::PerformanceTimer(#name, unit, el::Level::Info)) +#define PERF_TIMER_START_UNIT(name, unit) std::unique_ptr<tools::LoggingPerformanceTimer> pt_##name(new tools::LoggingPerformanceTimer(#name, unit, el::Level::Info)) #define PERF_TIMER_START(name) PERF_TIMER_START_UNIT(name, 1000) #define PERF_TIMER_STOP(name) do { pt_##name.reset(NULL); } while(0) #define PERF_TIMER_PAUSE(name) pt_##name->pause() diff --git a/src/common/util.cpp b/src/common/util.cpp index 5e0d2726e..c56c77505 100644 --- a/src/common/util.cpp +++ b/src/common/util.cpp @@ -38,7 +38,6 @@ #include <sys/types.h> #include <sys/stat.h> #include <sys/resource.h> -#include <ustat.h> #include <unistd.h> #include <dirent.h> #include <string.h> @@ -46,6 +45,13 @@ #include <string> #endif +//tools::is_hdd +#ifdef __GLIBC__ + #include <sstream> + #include <sys/sysmacros.h> + #include <fstream> +#endif + #include "unbound.h" #include "include_base_utils.h" @@ -733,62 +739,41 @@ std::string get_nix_version_display_string() #endif } - bool is_hdd(const char *path) + boost::optional<bool> is_hdd(const char *file_path) { #ifdef __GLIBC__ - std::string device = ""; - struct stat st, dst; - if (stat(path, &st) < 0) - return 0; - - DIR *dir = opendir("/dev/block"); - if (!dir) - return 0; - struct dirent *de; - while ((de = readdir(dir))) - { - if (strcmp(de->d_name, ".") && strcmp(de->d_name, "..")) + struct stat st; + std::string prefix; + if(stat(file_path, &st) == 0) + { + std::ostringstream s; + s << "/sys/dev/block/" << major(st.st_dev) << ":" << minor(st.st_dev); + prefix = s.str(); + } + else + { + return boost::none; + } + std::string attr_path = prefix + "/queue/rotational"; + std::ifstream f(attr_path, std::ios_base::in); + if(not f.is_open()) + { + attr_path = prefix + "/../queue/rotational"; + f.open(attr_path, std::ios_base::in); + if(not f.is_open()) { - std::string dev_path = std::string("/dev/block/") + de->d_name; - char resolved[PATH_MAX]; - if (realpath(dev_path.c_str(), resolved) && !strncmp(resolved, "/dev/", 5)) - { - if (stat(resolved, &dst) == 0) - { - if (dst.st_rdev == st.st_dev) - { - // take out trailing digits (eg, sda1 -> sda) - char *ptr = resolved; - while (*ptr) - ++ptr; - while (ptr > resolved && isdigit(*--ptr)) - *ptr = 0; - device = resolved + 5; - break; - } - } - } + return boost::none; } } - closedir(dir); - - if (device.empty()) - return 0; - - std::string sys_path = "/sys/block/" + device + "/queue/rotational"; - FILE *f = fopen(sys_path.c_str(), "r"); - if (!f) - return false; - char s[8]; - char *ptr = fgets(s, sizeof(s), f); - fclose(f); - if (!ptr) - return 0; - s[sizeof(s) - 1] = 0; - int n = atoi(s); // returns 0 on parse error - return n == 1; + unsigned short val = 0xdead; + f >> val; + if(not f.fail()) + { + return (val == 1); + } + return boost::none; #else - return 0; + return boost::none; #endif } diff --git a/src/common/util.h b/src/common/util.h index 8815232e2..0e0b50520 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -230,7 +230,7 @@ namespace tools bool sha256sum(const uint8_t *data, size_t len, crypto::hash &hash); bool sha256sum(const std::string &filename, crypto::hash &hash); - bool is_hdd(const char *path); + boost::optional<bool> is_hdd(const char *path); boost::optional<std::pair<uint32_t, uint32_t>> parse_subaddress_lookahead(const std::string& str); diff --git a/src/crypto/crypto-ops-data.c b/src/crypto/crypto-ops-data.c index 4ff4310de..1f77513ca 100644 --- a/src/crypto/crypto-ops-data.c +++ b/src/crypto/crypto-ops-data.c @@ -871,3 +871,9 @@ const fe fe_fffb2 = {8166131, -6741800, -17040804, 3154616, 21461005, 1466302, - const fe fe_fffb3 = {-13620103, 14639558, 4532995, 7679154, 16815101, -15883539, -22863840, -14813421, 13716513, -6477756}; /* sqrt(-sqrt(-1) * A * (A + 2)) */ const fe fe_fffb4 = {-21786234, -12173074, 21573800, 4524538, -4645904, 16204591, 8012863, -8444712, 3212926, 6885324}; /* sqrt(sqrt(-1) * A * (A + 2)) */ const ge_p3 ge_p3_identity = { {0}, {1, 0}, {1, 0}, {0} }; +const ge_p3 ge_p3_H = { + {7329926, -15101362, 31411471, 7614783, 27996851, -3197071, -11157635, -6878293, 466949, -7986503}, + {5858699, 5096796, 21321203, -7536921, -5553480, -11439507, -5627669, 15045946, 19977121, 5275251}, + {1, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + {23443568, -5110398, -8776029, -4345135, 6889568, -14710814, 7474843, 3279062, 14550766, -7453428} +}; diff --git a/src/crypto/crypto-ops.c b/src/crypto/crypto-ops.c index 45d412ac6..09296d6f9 100644 --- a/src/crypto/crypto-ops.c +++ b/src/crypto/crypto-ops.c @@ -3707,9 +3707,8 @@ void sc_muladd(unsigned char *s, const unsigned char *a, const unsigned char *b, s[31] = s11 >> 17; } -/* Assumes that a != INT64_MIN */ static int64_t signum(int64_t a) { - return (a >> 63) - ((-a) >> 63); + return a > 0 ? 1 : a < 0 ? -1 : 0; } int sc_check(const unsigned char *s) { @@ -3730,3 +3729,16 @@ int sc_isnonzero(const unsigned char *s) { s[18] | s[19] | s[20] | s[21] | s[22] | s[23] | s[24] | s[25] | s[26] | s[27] | s[28] | s[29] | s[30] | s[31]) - 1) >> 8) + 1; } + +int ge_p3_is_point_at_infinity(const ge_p3 *p) { + // X = 0 and Y == Z + int n; + for (n = 0; n < 10; ++n) + { + if (p->X[n] | p->T[n]) + return 0; + if (p->Y[n] != p->Z[n]) + return 0; + } + return 1; +} diff --git a/src/crypto/crypto-ops.h b/src/crypto/crypto-ops.h index dc3c60794..2910dafd4 100644 --- a/src/crypto/crypto-ops.h +++ b/src/crypto/crypto-ops.h @@ -140,6 +140,7 @@ extern const fe fe_fffb2; extern const fe fe_fffb3; extern const fe fe_fffb4; extern const ge_p3 ge_p3_identity; +extern const ge_p3 ge_p3_H; void ge_fromfe_frombytes_vartime(ge_p2 *, const unsigned char *); void sc_0(unsigned char *); void sc_reduce32(unsigned char *); @@ -158,3 +159,5 @@ void ge_sub(ge_p1p1 *r, const ge_p3 *p, const ge_cached *q); void fe_add(fe h, const fe f, const fe g); void fe_tobytes(unsigned char *, const fe); void fe_invert(fe out, const fe z); + +int ge_p3_is_point_at_infinity(const ge_p3 *p); diff --git a/src/crypto/crypto.cpp b/src/crypto/crypto.cpp index 4243c71fd..ad7721cf0 100644 --- a/src/crypto/crypto.cpp +++ b/src/crypto/crypto.cpp @@ -116,7 +116,7 @@ namespace crypto { do { generate_random_bytes_thread_safe(32, bytes); - } while (!less32(bytes, limit)); // should be good about 15/16 of the time + } while (!sc_isnonzero(bytes) && !less32(bytes, limit)); // should be good about 15/16 of the time sc_reduce32(bytes); } /* generate a random 32-byte (256-bit) integer and copy it to res */ @@ -274,11 +274,18 @@ namespace crypto { #endif buf.h = prefix_hash; buf.key = pub; + try_again: random_scalar(k); + if (((const uint32_t*)(&k))[7] == 0) // we don't want tiny numbers here + goto try_again; ge_scalarmult_base(&tmp3, &k); ge_p3_tobytes(&buf.comm, &tmp3); hash_to_scalar(&buf, sizeof(s_comm), sig.c); + if (!sc_isnonzero((const unsigned char*)sig.c.data)) + goto try_again; sc_mulsub(&sig.r, &sig.c, &unwrap(sec), &k); + if (!sc_isnonzero((const unsigned char*)sig.r.data)) + goto try_again; } bool crypto_ops::check_signature(const hash &prefix_hash, const public_key &pub, const signature &sig) { @@ -292,11 +299,14 @@ namespace crypto { if (ge_frombytes_vartime(&tmp3, &pub) != 0) { return false; } - if (sc_check(&sig.c) != 0 || sc_check(&sig.r) != 0) { + if (sc_check(&sig.c) != 0 || sc_check(&sig.r) != 0 || !sc_isnonzero(&sig.c)) { return false; } ge_double_scalarmult_base_vartime(&tmp2, &sig.c, &tmp3, &sig.r); ge_tobytes(&buf.comm, &tmp2); + static const ec_point infinity = {{ 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}; + if (memcmp(&buf.comm, &infinity, 32) == 0) + return false; hash_to_scalar(&buf, sizeof(s_comm), c); sc_sub(&c, &c, &sig.c); return sc_isnonzero(&c) == 0; diff --git a/src/crypto/keccak.c b/src/crypto/keccak.c index de8e2a5b3..8fcd2138e 100644 --- a/src/crypto/keccak.c +++ b/src/crypto/keccak.c @@ -132,3 +132,77 @@ void keccak1600(const uint8_t *in, size_t inlen, uint8_t *md) { keccak(in, inlen, md, sizeof(state_t)); } + +#define KECCAK_FINALIZED 0x80000000 +#define KECCAK_BLOCKLEN 136 +#define KECCAK_WORDS 17 +#define KECCAK_DIGESTSIZE 32 +#define IS_ALIGNED_64(p) (0 == (7 & ((const char*)(p) - (const char*)0))) +#define KECCAK_PROCESS_BLOCK(st, block) { \ + for (int i_ = 0; i_ < KECCAK_WORDS; i_++){ \ + ((st))[i_] ^= ((block))[i_]; \ + }; \ + keccakf(st, KECCAK_ROUNDS); } + + +void keccak_init(KECCAK_CTX * ctx){ + memset(ctx, 0, sizeof(KECCAK_CTX)); +} + +void keccak_update(KECCAK_CTX * ctx, const uint8_t *in, size_t inlen){ + if (ctx->rest & KECCAK_FINALIZED) { + local_abort("Bad keccak use"); + } + + const size_t idx = ctx->rest; + ctx->rest = (ctx->rest + inlen) % KECCAK_BLOCKLEN; + + // fill partial block + if (idx) { + size_t left = KECCAK_BLOCKLEN - idx; + memcpy((char*)ctx->message + idx, in, (inlen < left ? inlen : left)); + if (inlen < left) return; + + KECCAK_PROCESS_BLOCK(ctx->hash, ctx->message); + + in += left; + inlen -= left; + } + + const bool is_aligned = IS_ALIGNED_64(in); + while (inlen >= KECCAK_BLOCKLEN) { + const uint64_t* aligned_message_block; + if (is_aligned) { + aligned_message_block = (uint64_t*)in; + } else { + memcpy(ctx->message, in, KECCAK_BLOCKLEN); + aligned_message_block = ctx->message; + } + + KECCAK_PROCESS_BLOCK(ctx->hash, aligned_message_block); + in += KECCAK_BLOCKLEN; + inlen -= KECCAK_BLOCKLEN; + } + if (inlen) { + memcpy(ctx->message, in, inlen); + } +} + +void keccak_finish(KECCAK_CTX * ctx, uint8_t *md){ + if (!(ctx->rest & KECCAK_FINALIZED)) + { + // clear the rest of the data queue + memset((char*)ctx->message + ctx->rest, 0, KECCAK_BLOCKLEN - ctx->rest); + ((char*)ctx->message)[ctx->rest] |= 0x01; + ((char*)ctx->message)[KECCAK_BLOCKLEN - 1] |= 0x80; + + // process final block + KECCAK_PROCESS_BLOCK(ctx->hash, ctx->message); + ctx->rest = KECCAK_FINALIZED; // mark context as finalized + } + + static_assert(KECCAK_BLOCKLEN > KECCAK_DIGESTSIZE, ""); + if (md) { + memcpy(md, ctx->hash, KECCAK_DIGESTSIZE); + } +} diff --git a/src/crypto/keccak.h b/src/crypto/keccak.h index fb9d8bd04..9123c7a3b 100644 --- a/src/crypto/keccak.h +++ b/src/crypto/keccak.h @@ -15,6 +15,17 @@ #define ROTL64(x, y) (((x) << (y)) | ((x) >> (64 - (y)))) #endif +// SHA3 Algorithm context. +typedef struct KECCAK_CTX +{ + // 1600 bits algorithm hashing state + uint64_t hash[25]; + // 1088-bit buffer for leftovers, block size = 136 B for 256-bit keccak + uint64_t message[17]; + // count of bytes in the message[] buffer + size_t rest; +} KECCAK_CTX; + // compute a keccak hash (md) of given byte length from "in" void keccak(const uint8_t *in, size_t inlen, uint8_t *md, int mdlen); @@ -23,4 +34,7 @@ void keccakf(uint64_t st[25], int norounds); void keccak1600(const uint8_t *in, size_t inlen, uint8_t *md); +void keccak_init(KECCAK_CTX * ctx); +void keccak_update(KECCAK_CTX * ctx, const uint8_t *in, size_t inlen); +void keccak_finish(KECCAK_CTX * ctx, uint8_t *md); #endif diff --git a/src/crypto/oaes_lib.c b/src/crypto/oaes_lib.c index 9e31ebf46..210f5d43a 100644 --- a/src/crypto/oaes_lib.c +++ b/src/crypto/oaes_lib.c @@ -33,14 +33,15 @@ #include <stdlib.h> #include <stdio.h> -// OS X, FreeBSD, and OpenBSD don't need malloc.h +// OS X, FreeBSD, OpenBSD and NetBSD don't need malloc.h #if !defined(__APPLE__) && !defined(__FreeBSD__) && !defined(__OpenBSD__) \ - && !defined(__DragonFly__) + && !defined(__DragonFly__) && !defined(__NetBSD__) #include <malloc.h> #endif -// ANDROID, FreeBSD, and OpenBSD also don't need timeb.h -#if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__ANDROID__) +// ANDROID, FreeBSD, OpenBSD and NetBSD also don't need timeb.h +#if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__ANDROID__) \ + && !defined(__NetBSD__) #include <sys/timeb.h> #else #include <sys/time.h> @@ -473,7 +474,7 @@ OAES_RET oaes_sprintf( #ifdef OAES_HAVE_ISAAC static void oaes_get_seed( char buf[RANDSIZ + 1] ) { - #if !defined(__FreeBSD__) && !defined(__OpenBSD__) + #if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__NetBSD__) struct timeb timer; struct tm *gmTimer; char * _test = NULL; @@ -505,7 +506,7 @@ static void oaes_get_seed( char buf[RANDSIZ + 1] ) #else static uint32_t oaes_get_seed(void) { - #if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__ANDROID__) + #if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__ANDROID__) && !defined(__NetBSD__) struct timeb timer; struct tm *gmTimer; char * _test = NULL; diff --git a/src/crypto/slow-hash.c b/src/crypto/slow-hash.c index 9d4fc0dfa..914ba6dc0 100644 --- a/src/crypto/slow-hash.c +++ b/src/crypto/slow-hash.c @@ -38,6 +38,7 @@ #include "common/int-util.h" #include "hash-ops.h" #include "oaes_lib.h" +#include "variant2_int_sqrt.h" #define MEMORY (1 << 21) // 2MB scratchpad #define ITER (1 << 20) @@ -50,7 +51,7 @@ extern int aesb_single_round(const uint8_t *in, uint8_t*out, const uint8_t *expa extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *expandedKey); #define VARIANT1_1(p) \ - do if (variant > 0) \ + do if (variant == 1) \ { \ const uint8_t tmp = ((const uint8_t*)(p))[11]; \ static const uint32_t table = 0x75310; \ @@ -59,7 +60,7 @@ extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *exp } while(0) #define VARIANT1_2(p) \ - do if (variant > 0) \ + do if (variant == 1) \ { \ xor64(p, tweak1_2); \ } while(0) @@ -67,7 +68,7 @@ extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *exp #define VARIANT1_CHECK() \ do if (length < 43) \ { \ - fprintf(stderr, "Cryptonight variants need at least 43 bytes of data"); \ + fprintf(stderr, "Cryptonight variant 1 needs at least 43 bytes of data"); \ _exit(1); \ } while(0) @@ -75,7 +76,7 @@ extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *exp #define VARIANT1_PORTABLE_INIT() \ uint8_t tweak1_2[8]; \ - do if (variant > 0) \ + do if (variant == 1) \ { \ VARIANT1_CHECK(); \ memcpy(&tweak1_2, &state.hs.b[192], sizeof(tweak1_2)); \ @@ -83,11 +84,119 @@ extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *exp } while(0) #define VARIANT1_INIT64() \ - if (variant > 0) \ + if (variant == 1) \ { \ VARIANT1_CHECK(); \ } \ - const uint64_t tweak1_2 = variant > 0 ? (state.hs.w[24] ^ (*((const uint64_t*)NONCE_POINTER))) : 0 + const uint64_t tweak1_2 = (variant == 1) ? (state.hs.w[24] ^ (*((const uint64_t*)NONCE_POINTER))) : 0 + +#define VARIANT2_INIT64() \ + uint64_t division_result = 0; \ + uint64_t sqrt_result = 0; \ + do if (variant >= 2) \ + { \ + U64(b)[2] = state.hs.w[8] ^ state.hs.w[10]; \ + U64(b)[3] = state.hs.w[9] ^ state.hs.w[11]; \ + division_result = state.hs.w[12]; \ + sqrt_result = state.hs.w[13]; \ + } while (0) + +#define VARIANT2_PORTABLE_INIT() \ + uint64_t division_result = 0; \ + uint64_t sqrt_result = 0; \ + do if (variant >= 2) \ + { \ + memcpy(b + AES_BLOCK_SIZE, state.hs.b + 64, AES_BLOCK_SIZE); \ + xor64(b + AES_BLOCK_SIZE, state.hs.b + 80); \ + xor64(b + AES_BLOCK_SIZE + 8, state.hs.b + 88); \ + division_result = state.hs.w[12]; \ + sqrt_result = state.hs.w[13]; \ + } while (0) + +#define VARIANT2_SHUFFLE_ADD_SSE2(base_ptr, offset) \ + do if (variant >= 2) \ + { \ + const __m128i chunk1 = _mm_load_si128((__m128i *)((base_ptr) + ((offset) ^ 0x10))); \ + const __m128i chunk2 = _mm_load_si128((__m128i *)((base_ptr) + ((offset) ^ 0x20))); \ + const __m128i chunk3 = _mm_load_si128((__m128i *)((base_ptr) + ((offset) ^ 0x30))); \ + _mm_store_si128((__m128i *)((base_ptr) + ((offset) ^ 0x10)), _mm_add_epi64(chunk3, _b1)); \ + _mm_store_si128((__m128i *)((base_ptr) + ((offset) ^ 0x20)), _mm_add_epi64(chunk1, _b)); \ + _mm_store_si128((__m128i *)((base_ptr) + ((offset) ^ 0x30)), _mm_add_epi64(chunk2, _a)); \ + } while (0) + +#define VARIANT2_SHUFFLE_ADD_NEON(base_ptr, offset) \ + do if (variant >= 2) \ + { \ + const uint64x2_t chunk1 = vld1q_u64(U64((base_ptr) + ((offset) ^ 0x10))); \ + const uint64x2_t chunk2 = vld1q_u64(U64((base_ptr) + ((offset) ^ 0x20))); \ + const uint64x2_t chunk3 = vld1q_u64(U64((base_ptr) + ((offset) ^ 0x30))); \ + vst1q_u64(U64((base_ptr) + ((offset) ^ 0x10)), vaddq_u64(chunk3, vreinterpretq_u64_u8(_b1))); \ + vst1q_u64(U64((base_ptr) + ((offset) ^ 0x20)), vaddq_u64(chunk1, vreinterpretq_u64_u8(_b))); \ + vst1q_u64(U64((base_ptr) + ((offset) ^ 0x30)), vaddq_u64(chunk2, vreinterpretq_u64_u8(_a))); \ + } while (0) + +#define VARIANT2_PORTABLE_SHUFFLE_ADD(base_ptr, offset) \ + do if (variant >= 2) \ + { \ + uint64_t* chunk1 = U64((base_ptr) + ((offset) ^ 0x10)); \ + uint64_t* chunk2 = U64((base_ptr) + ((offset) ^ 0x20)); \ + uint64_t* chunk3 = U64((base_ptr) + ((offset) ^ 0x30)); \ + \ + const uint64_t chunk1_old[2] = { chunk1[0], chunk1[1] }; \ + \ + uint64_t b1[2]; \ + memcpy(b1, b + 16, 16); \ + chunk1[0] = chunk3[0] + b1[0]; \ + chunk1[1] = chunk3[1] + b1[1]; \ + \ + uint64_t a0[2]; \ + memcpy(a0, a, 16); \ + chunk3[0] = chunk2[0] + a0[0]; \ + chunk3[1] = chunk2[1] + a0[1]; \ + \ + uint64_t b0[2]; \ + memcpy(b0, b, 16); \ + chunk2[0] = chunk1_old[0] + b0[0]; \ + chunk2[1] = chunk1_old[1] + b0[1]; \ + } while (0) + +#define VARIANT2_INTEGER_MATH_DIVISION_STEP(b, ptr) \ + ((uint64_t*)(b))[0] ^= division_result ^ (sqrt_result << 32); \ + { \ + const uint64_t dividend = ((uint64_t*)(ptr))[1]; \ + const uint32_t divisor = (((uint64_t*)(ptr))[0] + (uint32_t)(sqrt_result << 1)) | 0x80000001UL; \ + division_result = ((uint32_t)(dividend / divisor)) + \ + (((uint64_t)(dividend % divisor)) << 32); \ + } \ + const uint64_t sqrt_input = ((uint64_t*)(ptr))[0] + division_result + +#define VARIANT2_INTEGER_MATH_SSE2(b, ptr) \ + do if (variant >= 2) \ + { \ + VARIANT2_INTEGER_MATH_DIVISION_STEP(b, ptr); \ + VARIANT2_INTEGER_MATH_SQRT_STEP_SSE2(); \ + VARIANT2_INTEGER_MATH_SQRT_FIXUP(sqrt_result); \ + } while(0) + +#if defined DBL_MANT_DIG && (DBL_MANT_DIG >= 50) + // double precision floating point type has enough bits of precision on current platform + #define VARIANT2_PORTABLE_INTEGER_MATH(b, ptr) \ + do if (variant >= 2) \ + { \ + VARIANT2_INTEGER_MATH_DIVISION_STEP(b, ptr); \ + VARIANT2_INTEGER_MATH_SQRT_STEP_FP64(); \ + VARIANT2_INTEGER_MATH_SQRT_FIXUP(sqrt_result); \ + } while (0) +#else + // double precision floating point type is not good enough on current platform + // fall back to the reference code (integer only) + #define VARIANT2_PORTABLE_INTEGER_MATH(b, ptr) \ + do if (variant >= 2) \ + { \ + VARIANT2_INTEGER_MATH_DIVISION_STEP(b, ptr); \ + VARIANT2_INTEGER_MATH_SQRT_STEP_REF(); \ + } while (0) +#endif #if !defined NO_AES && (defined(__x86_64__) || (defined(_MSC_VER) && defined(_WIN64))) // Optimised code below, uses x86-specific intrinsics, SSE2, AES-NI @@ -164,19 +273,22 @@ extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *exp * This code is based upon an optimized implementation by dga. */ #define post_aes() \ + VARIANT2_SHUFFLE_ADD_SSE2(hp_state, j); \ _mm_store_si128(R128(c), _c); \ - _b = _mm_xor_si128(_b, _c); \ - _mm_store_si128(R128(&hp_state[j]), _b); \ + _mm_store_si128(R128(&hp_state[j]), _mm_xor_si128(_b, _c)); \ VARIANT1_1(&hp_state[j]); \ j = state_index(c); \ p = U64(&hp_state[j]); \ b[0] = p[0]; b[1] = p[1]; \ + VARIANT2_INTEGER_MATH_SSE2(b, c); \ __mul(); \ + VARIANT2_SHUFFLE_ADD_SSE2(hp_state, j); \ a[0] += hi; a[1] += lo; \ p = U64(&hp_state[j]); \ p[0] = a[0]; p[1] = a[1]; \ a[0] ^= b[0]; a[1] ^= b[1]; \ VARIANT1_2(p + 1); \ + _b1 = _b; \ _b = _c; \ #if defined(_MSC_VER) @@ -492,7 +604,7 @@ void slow_hash_allocate_state(void) MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); #else #if defined(__APPLE__) || defined(__FreeBSD__) || defined(__OpenBSD__) || \ - defined(__DragonFly__) + defined(__DragonFly__) || defined(__NetBSD__) hp_state = mmap(0, MEMORY, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, 0, 0); #else @@ -570,10 +682,10 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int uint8_t text[INIT_SIZE_BYTE]; RDATA_ALIGN16 uint64_t a[2]; - RDATA_ALIGN16 uint64_t b[2]; + RDATA_ALIGN16 uint64_t b[4]; RDATA_ALIGN16 uint64_t c[2]; union cn_slow_hash_state state; - __m128i _a, _b, _c; + __m128i _a, _b, _b1, _c; uint64_t hi, lo; size_t i, j; @@ -599,6 +711,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int memcpy(text, state.init, INIT_SIZE_BYTE); VARIANT1_INIT64(); + VARIANT2_INIT64(); /* CryptoNight Step 2: Iteratively encrypt the results from Keccak to fill * the 2MB large random access buffer. @@ -637,6 +750,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int */ _b = _mm_load_si128(R128(b)); + _b1 = _mm_load_si128(R128(b) + 1); // Two independent versions, one with AES, one without, to ensure that // the useAes test is only performed once, not every iteration. if(useAes) @@ -761,19 +875,22 @@ union cn_slow_hash_state _a = vld1q_u8((const uint8_t *)a); \ #define post_aes() \ + VARIANT2_SHUFFLE_ADD_NEON(hp_state, j); \ vst1q_u8((uint8_t *)c, _c); \ - _b = veorq_u8(_b, _c); \ - vst1q_u8(&hp_state[j], _b); \ + vst1q_u8(&hp_state[j], veorq_u8(_b, _c)); \ VARIANT1_1(&hp_state[j]); \ j = state_index(c); \ p = U64(&hp_state[j]); \ b[0] = p[0]; b[1] = p[1]; \ + VARIANT2_PORTABLE_INTEGER_MATH(b, c); \ __mul(); \ + VARIANT2_SHUFFLE_ADD_NEON(hp_state, j); \ a[0] += hi; a[1] += lo; \ p = U64(&hp_state[j]); \ p[0] = a[0]; p[1] = a[1]; \ a[0] ^= b[0]; a[1] ^= b[1]; \ VARIANT1_2(p + 1); \ + _b1 = _b; \ _b = _c; \ @@ -912,10 +1029,10 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int uint8_t text[INIT_SIZE_BYTE]; RDATA_ALIGN16 uint64_t a[2]; - RDATA_ALIGN16 uint64_t b[2]; + RDATA_ALIGN16 uint64_t b[4]; RDATA_ALIGN16 uint64_t c[2]; union cn_slow_hash_state state; - uint8x16_t _a, _b, _c, zero = {0}; + uint8x16_t _a, _b, _b1, _c, zero = {0}; uint64_t hi, lo; size_t i, j; @@ -936,6 +1053,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int memcpy(text, state.init, INIT_SIZE_BYTE); VARIANT1_INIT64(); + VARIANT2_INIT64(); /* CryptoNight Step 2: Iteratively encrypt the results from Keccak to fill * the 2MB large random access buffer. @@ -959,7 +1077,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int */ _b = vld1q_u8((const uint8_t *)b); - + _b1 = vld1q_u8(((const uint8_t *)b) + AES_BLOCK_SIZE); for(i = 0; i < ITER / 2; i++) { @@ -1075,6 +1193,11 @@ __asm__ __volatile__( #endif /* !aarch64 */ #endif // NO_OPTIMIZED_MULTIPLY_ON_ARM +STATIC INLINE void copy_block(uint8_t* dst, const uint8_t* src) +{ + memcpy(dst, src, AES_BLOCK_SIZE); +} + STATIC INLINE void sum_half_blocks(uint8_t* a, const uint8_t* b) { uint64_t a0, a1, b0, b1; @@ -1109,7 +1232,9 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int { uint8_t text[INIT_SIZE_BYTE]; uint8_t a[AES_BLOCK_SIZE]; - uint8_t b[AES_BLOCK_SIZE]; + uint8_t b[AES_BLOCK_SIZE * 2]; + uint8_t c[AES_BLOCK_SIZE]; + uint8_t c1[AES_BLOCK_SIZE]; uint8_t d[AES_BLOCK_SIZE]; uint8_t aes_key[AES_KEY_SIZE]; RDATA_ALIGN16 uint8_t expandedKey[256]; @@ -1138,11 +1263,12 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int } memcpy(text, state.init, INIT_SIZE_BYTE); - VARIANT1_INIT64(); - aes_ctx = (oaes_ctx *) oaes_alloc(); oaes_key_import_data(aes_ctx, state.hs.b, AES_KEY_SIZE); + VARIANT1_INIT64(); + VARIANT2_INIT64(); + // use aligned data memcpy(expandedKey, aes_ctx->key->exp_data, aes_ctx->key->exp_data_len); for(i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) @@ -1163,23 +1289,33 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int #define state_index(x) ((*(uint32_t *) x) & MASK) // Iteration 1 - p = &long_state[state_index(a)]; + j = state_index(a); + p = &long_state[j]; aesb_single_round(p, p, a); + copy_block(c1, p); - xor_blocks(b, p); - swap_blocks(b, p); - swap_blocks(a, b); + VARIANT2_PORTABLE_SHUFFLE_ADD(long_state, j); + xor_blocks(p, b); VARIANT1_1(p); // Iteration 2 - p = &long_state[state_index(a)]; - - mul(a, p, d); - sum_half_blocks(b, d); - swap_blocks(b, p); - xor_blocks(b, p); - swap_blocks(a, b); - VARIANT1_2(U64(p) + 1); + j = state_index(c1); + p = &long_state[j]; + copy_block(c, p); + + VARIANT2_PORTABLE_INTEGER_MATH(c, c1); + mul(c1, c, d); + VARIANT2_PORTABLE_SHUFFLE_ADD(long_state, j); + sum_half_blocks(a, d); + swap_blocks(a, c); + xor_blocks(a, c); + VARIANT1_2(U64(c) + 1); + copy_block(p, c); + + if (variant >= 2) { + copy_block(b + AES_BLOCK_SIZE, b); + } + copy_block(b, c1); } memcpy(text, state.init, INIT_SIZE_BYTE); @@ -1298,8 +1434,9 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int union cn_slow_hash_state state; uint8_t text[INIT_SIZE_BYTE]; uint8_t a[AES_BLOCK_SIZE]; - uint8_t b[AES_BLOCK_SIZE]; - uint8_t c[AES_BLOCK_SIZE]; + uint8_t b[AES_BLOCK_SIZE * 2]; + uint8_t c1[AES_BLOCK_SIZE]; + uint8_t c2[AES_BLOCK_SIZE]; uint8_t d[AES_BLOCK_SIZE]; size_t i, j; uint8_t aes_key[AES_KEY_SIZE]; @@ -1315,6 +1452,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int aes_ctx = (oaes_ctx *) oaes_alloc(); VARIANT1_PORTABLE_INIT(); + VARIANT2_PORTABLE_INIT(); oaes_key_import_data(aes_ctx, aes_key, AES_KEY_SIZE); for (i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) { @@ -1324,9 +1462,9 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int memcpy(&long_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE); } - for (i = 0; i < 16; i++) { - a[i] = state.k[ i] ^ state.k[32 + i]; - b[i] = state.k[16 + i] ^ state.k[48 + i]; + for (i = 0; i < AES_BLOCK_SIZE; i++) { + a[i] = state.k[ i] ^ state.k[AES_BLOCK_SIZE * 2 + i]; + b[i] = state.k[AES_BLOCK_SIZE + i] ^ state.k[AES_BLOCK_SIZE * 3 + i]; } for (i = 0; i < ITER / 2; i++) { @@ -1335,26 +1473,32 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int * next address <-+ */ /* Iteration 1 */ - j = e2i(a, MEMORY / AES_BLOCK_SIZE); - copy_block(c, &long_state[j * AES_BLOCK_SIZE]); - aesb_single_round(c, c, a); - xor_blocks(b, c); - swap_blocks(b, c); - copy_block(&long_state[j * AES_BLOCK_SIZE], c); - assert(j == e2i(a, MEMORY / AES_BLOCK_SIZE)); - swap_blocks(a, b); - VARIANT1_1(&long_state[j * AES_BLOCK_SIZE]); + j = e2i(a, MEMORY / AES_BLOCK_SIZE) * AES_BLOCK_SIZE; + copy_block(c1, &long_state[j]); + aesb_single_round(c1, c1, a); + VARIANT2_PORTABLE_SHUFFLE_ADD(long_state, j); + copy_block(&long_state[j], c1); + xor_blocks(&long_state[j], b); + assert(j == e2i(a, MEMORY / AES_BLOCK_SIZE) * AES_BLOCK_SIZE); + VARIANT1_1(&long_state[j]); /* Iteration 2 */ - j = e2i(a, MEMORY / AES_BLOCK_SIZE); - copy_block(c, &long_state[j * AES_BLOCK_SIZE]); - mul(a, c, d); - sum_half_blocks(b, d); - swap_blocks(b, c); - xor_blocks(b, c); - VARIANT1_2(c + 8); - copy_block(&long_state[j * AES_BLOCK_SIZE], c); - assert(j == e2i(a, MEMORY / AES_BLOCK_SIZE)); - swap_blocks(a, b); + j = e2i(c1, MEMORY / AES_BLOCK_SIZE) * AES_BLOCK_SIZE; + copy_block(c2, &long_state[j]); + VARIANT2_PORTABLE_INTEGER_MATH(c2, c1); + mul(c1, c2, d); + VARIANT2_PORTABLE_SHUFFLE_ADD(long_state, j); + swap_blocks(a, c1); + sum_half_blocks(c1, d); + swap_blocks(c1, c2); + xor_blocks(c1, c2); + VARIANT1_2(c2 + 8); + copy_block(&long_state[j], c2); + assert(j == e2i(a, MEMORY / AES_BLOCK_SIZE) * AES_BLOCK_SIZE); + if (variant >= 2) { + copy_block(b + AES_BLOCK_SIZE, b); + } + copy_block(b, a); + copy_block(a, c1); } memcpy(text, state.init, INIT_SIZE_BYTE); diff --git a/src/crypto/tree-hash.c b/src/crypto/tree-hash.c index 57c38b86b..b2dc3ffb2 100644 --- a/src/crypto/tree-hash.c +++ b/src/crypto/tree-hash.c @@ -36,7 +36,8 @@ #ifdef _MSC_VER #include <malloc.h> -#elif !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__DragonFly__) +#elif !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__DragonFly__) \ + && !defined(__NetBSD__) #include <alloca.h> #else #include <stdlib.h> diff --git a/src/crypto/variant2_int_sqrt.h b/src/crypto/variant2_int_sqrt.h new file mode 100644 index 000000000..b405bb798 --- /dev/null +++ b/src/crypto/variant2_int_sqrt.h @@ -0,0 +1,163 @@ +#ifndef VARIANT2_INT_SQRT_H +#define VARIANT2_INT_SQRT_H + +#include <math.h> +#include <float.h> + +#define VARIANT2_INTEGER_MATH_SQRT_STEP_SSE2() \ + do { \ + const __m128i exp_double_bias = _mm_set_epi64x(0, 1023ULL << 52); \ + __m128d x = _mm_castsi128_pd(_mm_add_epi64(_mm_cvtsi64_si128(sqrt_input >> 12), exp_double_bias)); \ + x = _mm_sqrt_sd(_mm_setzero_pd(), x); \ + sqrt_result = (uint64_t)(_mm_cvtsi128_si64(_mm_sub_epi64(_mm_castpd_si128(x), exp_double_bias))) >> 19; \ + } while(0) + +#define VARIANT2_INTEGER_MATH_SQRT_STEP_FP64() \ + do { \ + sqrt_result = sqrt(sqrt_input + 18446744073709551616.0) * 2.0 - 8589934592.0; \ + } while(0) + +#define VARIANT2_INTEGER_MATH_SQRT_STEP_REF() \ + sqrt_result = integer_square_root_v2(sqrt_input) + +// Reference implementation of the integer square root for Cryptonight variant 2 +// Computes integer part of "sqrt(2^64 + n) * 2 - 2^33" +// +// In other words, given 64-bit unsigned integer n: +// 1) Write it as x = 1.NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN000... in binary (1 <= x < 2, all 64 bits of n are used) +// 2) Calculate sqrt(x) = 1.0RRRRRRRRRRRRRRRRRRRRRRRRRRRRRRRR... (1 <= sqrt(x) < sqrt(2), so it will always start with "1.0" in binary) +// 3) Take 32 bits that come after "1.0" and return them as a 32-bit unsigned integer, discard all remaining bits +// +// Some sample inputs and outputs: +// +// Input | Output | Exact value of "sqrt(2^64 + n) * 2 - 2^33" +// -----------------|------------|------------------------------------------- +// 0 | 0 | 0 +// 2^32 | 0 | 0.99999999994179233909330885695244... +// 2^32 + 1 | 1 | 1.0000000001746229827200734316305... +// 2^50 | 262140 | 262140.00012206565608606978175873... +// 2^55 + 20963331 | 8384515 | 8384515.9999999997673963974959744... +// 2^55 + 20963332 | 8384516 | 8384516 +// 2^62 + 26599786 | 1013904242 | 1013904242.9999999999479374853545... +// 2^62 + 26599787 | 1013904243 | 1013904243.0000000001561875439364... +// 2^64 - 1 | 3558067407 | 3558067407.9041987696409179931096... + +// The reference implementation as it is now uses only unsigned int64 arithmetic, so it can't have undefined behavior +// It was tested once for all edge cases and confirmed correct +static inline uint32_t integer_square_root_v2(uint64_t n) +{ + uint64_t r = 1ULL << 63; + + for (uint64_t bit = 1ULL << 60; bit; bit >>= 2) + { + const bool b = (n < r + bit); + const uint64_t n_next = n - (r + bit); + const uint64_t r_next = r + bit * 2; + n = b ? n : n_next; + r = b ? r : r_next; + r >>= 1; + } + + return r * 2 + ((n > r) ? 1 : 0); +} + +/* +VARIANT2_INTEGER_MATH_SQRT_FIXUP checks that "r" is an integer part of "sqrt(2^64 + sqrt_input) * 2 - 2^33" and adds or subtracts 1 if needed +It's hard to understand how it works, so here is a full calculation of formulas used in VARIANT2_INTEGER_MATH_SQRT_FIXUP + +The following inequalities must hold for r if it's an integer part of "sqrt(2^64 + sqrt_input) * 2 - 2^33": +1) r <= sqrt(2^64 + sqrt_input) * 2 - 2^33 +2) r + 1 > sqrt(2^64 + sqrt_input) * 2 - 2^33 + +We need to check them using only unsigned integer arithmetic to avoid rounding errors and undefined behavior + +First inequality: r <= sqrt(2^64 + sqrt_input) * 2 - 2^33 +----------------------------------------------------------------------------------- +r <= sqrt(2^64 + sqrt_input) * 2 - 2^33 +r + 2^33 <= sqrt(2^64 + sqrt_input) * 2 +r/2 + 2^32 <= sqrt(2^64 + sqrt_input) +(r/2 + 2^32)^2 <= 2^64 + sqrt_input + +Rewrite r as r = s * 2 + b (s = trunc(r/2), b is 0 or 1) + +((s*2+b)/2 + 2^32)^2 <= 2^64 + sqrt_input +(s*2+b)^2/4 + 2*2^32*(s*2+b)/2 + 2^64 <= 2^64 + sqrt_input +(s*2+b)^2/4 + 2*2^32*(s*2+b)/2 <= sqrt_input +(s*2+b)^2/4 + 2^32*r <= sqrt_input +(s^2*4+2*s*2*b+b^2)/4 + 2^32*r <= sqrt_input +s^2+s*b+b^2/4 + 2^32*r <= sqrt_input +s*(s+b) + b^2/4 + 2^32*r <= sqrt_input + +Let r2 = s*(s+b) + r*2^32 +r2 + b^2/4 <= sqrt_input + +If this inequality doesn't hold, then we must decrement r: IF "r2 + b^2/4 > sqrt_input" THEN r = r - 1 + +b can be 0 or 1 +If b is 0 then we need to compare "r2 > sqrt_input" +If b is 1 then b^2/4 = 0.25, so we need to compare "r2 + 0.25 > sqrt_input" +Since both r2 and sqrt_input are integers, we can safely replace it with "r2 + 1 > sqrt_input" +----------------------------------------------------------------------------------- +Both cases can be merged to a single expression "r2 + b > sqrt_input" +----------------------------------------------------------------------------------- +There will be no overflow when calculating "r2 + b", so it's safe to compare with sqrt_input: +r2 + b = s*(s+b) + r*2^32 + b +The largest value s, b and r can have is s = 1779033703, b = 1, r = 3558067407 when sqrt_input = 2^64 - 1 +r2 + b <= 1779033703*1779033704 + 3558067407*2^32 + 1 = 18446744068217447385 < 2^64 + +Second inequality: r + 1 > sqrt(2^64 + sqrt_input) * 2 - 2^33 +----------------------------------------------------------------------------------- +r + 1 > sqrt(2^64 + sqrt_input) * 2 - 2^33 +r + 1 + 2^33 > sqrt(2^64 + sqrt_input) * 2 +((r+1)/2 + 2^32)^2 > 2^64 + sqrt_input + +Rewrite r as r = s * 2 + b (s = trunc(r/2), b is 0 or 1) + +((s*2+b+1)/2 + 2^32)^2 > 2^64 + sqrt_input +(s*2+b+1)^2/4 + 2*(s*2+b+1)/2*2^32 + 2^64 > 2^64 + sqrt_input +(s*2+b+1)^2/4 + (s*2+b+1)*2^32 > sqrt_input +(s*2+b+1)^2/4 + (r+1)*2^32 > sqrt_input +(s*2+(b+1))^2/4 + r*2^32 + 2^32 > sqrt_input +(s^2*4+2*s*2*(b+1)+(b+1)^2)/4 + r*2^32 + 2^32 > sqrt_input +s^2+s*(b+1)+(b+1)^2/4 + r*2^32 + 2^32 > sqrt_input +s*(s+b) + s + (b+1)^2/4 + r*2^32 + 2^32 > sqrt_input + +Let r2 = s*(s+b) + r*2^32 + +r2 + s + (b+1)^2/4 + 2^32 > sqrt_input +r2 + 2^32 + (b+1)^2/4 > sqrt_input - s + +If this inequality doesn't hold, then we must decrement r: IF "r2 + 2^32 + (b+1)^2/4 <= sqrt_input - s" THEN r = r - 1 +b can be 0 or 1 +If b is 0 then we need to compare "r2 + 2^32 + 1/4 <= sqrt_input - s" which is equal to "r2 + 2^32 < sqrt_input - s" because all numbers here are integers +If b is 1 then (b+1)^2/4 = 1, so we need to compare "r2 + 2^32 + 1 <= sqrt_input - s" which is also equal to "r2 + 2^32 < sqrt_input - s" +----------------------------------------------------------------------------------- +Both cases can be merged to a single expression "r2 + 2^32 < sqrt_input - s" +----------------------------------------------------------------------------------- +There will be no overflow when calculating "r2 + 2^32": +r2 + 2^32 = s*(s+b) + r*2^32 + 2^32 = s*(s+b) + (r+1)*2^32 +The largest value s, b and r can have is s = 1779033703, b = 1, r = 3558067407 when sqrt_input = 2^64 - 1 +r2 + b <= 1779033703*1779033704 + 3558067408*2^32 = 18446744072512414680 < 2^64 + +There will be no integer overflow when calculating "sqrt_input - s", i.e. "sqrt_input >= s" at all times: +s = trunc(r/2) = trunc(sqrt(2^64 + sqrt_input) - 2^32) < sqrt(2^64 + sqrt_input) - 2^32 + 1 +sqrt_input > sqrt(2^64 + sqrt_input) - 2^32 + 1 +sqrt_input + 2^32 - 1 > sqrt(2^64 + sqrt_input) +(sqrt_input + 2^32 - 1)^2 > sqrt_input + 2^64 +sqrt_input^2 + 2*sqrt_input*(2^32 - 1) + (2^32-1)^2 > sqrt_input + 2^64 +sqrt_input^2 + sqrt_input*(2^33 - 2) + (2^32-1)^2 > sqrt_input + 2^64 +sqrt_input^2 + sqrt_input*(2^33 - 3) + (2^32-1)^2 > 2^64 +sqrt_input^2 + sqrt_input*(2^33 - 3) + 2^64-2^33+1 > 2^64 +sqrt_input^2 + sqrt_input*(2^33 - 3) - 2^33 + 1 > 0 +This inequality is true if sqrt_input > 1 and it's easy to check that s = 0 if sqrt_input is 0 or 1, so there will be no integer overflow +*/ + +#define VARIANT2_INTEGER_MATH_SQRT_FIXUP(r) \ + do { \ + const uint64_t s = r >> 1; \ + const uint64_t b = r & 1; \ + const uint64_t r2 = (uint64_t)(s) * (s + b) + (r << 32); \ + r += ((r2 + b > sqrt_input) ? -1 : 0) + ((r2 + (1ULL << 32) < sqrt_input - s) ? 1 : 0); \ + } while(0) + +#endif diff --git a/src/cryptonote_basic/cryptonote_basic_impl.cpp b/src/cryptonote_basic/cryptonote_basic_impl.cpp index cff23695f..b18ef1c5c 100644 --- a/src/cryptonote_basic/cryptonote_basic_impl.cpp +++ b/src/cryptonote_basic/cryptonote_basic_impl.cpp @@ -67,7 +67,7 @@ namespace cryptonote { /* Cryptonote helper functions */ /************************************************************************/ //----------------------------------------------------------------------------------------------- - size_t get_min_block_size(uint8_t version) + size_t get_min_block_weight(uint8_t version) { if (version < 2) return CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V1; @@ -86,7 +86,7 @@ namespace cryptonote { return CRYPTONOTE_MAX_TX_SIZE; } //----------------------------------------------------------------------------------------------- - bool get_block_reward(size_t median_size, size_t current_block_size, uint64_t already_generated_coins, uint64_t &reward, uint8_t version) { + bool get_block_reward(size_t median_weight, size_t current_block_weight, uint64_t already_generated_coins, uint64_t &reward, uint8_t version) { static_assert(DIFFICULTY_TARGET_V2%60==0&&DIFFICULTY_TARGET_V1%60==0,"difficulty targets must be a multiple of 60"); const int target = version < 2 ? DIFFICULTY_TARGET_V1 : DIFFICULTY_TARGET_V2; const int target_minutes = target / 60; @@ -98,37 +98,37 @@ namespace cryptonote { base_reward = FINAL_SUBSIDY_PER_MINUTE*target_minutes; } - uint64_t full_reward_zone = get_min_block_size(version); + uint64_t full_reward_zone = get_min_block_weight(version); //make it soft - if (median_size < full_reward_zone) { - median_size = full_reward_zone; + if (median_weight < full_reward_zone) { + median_weight = full_reward_zone; } - if (current_block_size <= median_size) { + if (current_block_weight <= median_weight) { reward = base_reward; return true; } - if(current_block_size > 2 * median_size) { - MERROR("Block cumulative size is too big: " << current_block_size << ", expected less than " << 2 * median_size); + if(current_block_weight > 2 * median_weight) { + MERROR("Block cumulative weight is too big: " << current_block_weight << ", expected less than " << 2 * median_weight); return false; } - assert(median_size < std::numeric_limits<uint32_t>::max()); - assert(current_block_size < std::numeric_limits<uint32_t>::max()); + assert(median_weight < std::numeric_limits<uint32_t>::max()); + assert(current_block_weight < std::numeric_limits<uint32_t>::max()); uint64_t product_hi; // BUGFIX: 32-bit saturation bug (e.g. ARM7), the result was being // treated as 32-bit by default. - uint64_t multiplicand = 2 * median_size - current_block_size; - multiplicand *= current_block_size; + uint64_t multiplicand = 2 * median_weight - current_block_weight; + multiplicand *= current_block_weight; uint64_t product_lo = mul128(base_reward, multiplicand, &product_hi); uint64_t reward_hi; uint64_t reward_lo; - div128_32(product_hi, product_lo, static_cast<uint32_t>(median_size), &reward_hi, &reward_lo); - div128_32(reward_hi, reward_lo, static_cast<uint32_t>(median_size), &reward_hi, &reward_lo); + div128_32(product_hi, product_lo, static_cast<uint32_t>(median_weight), &reward_hi, &reward_lo); + div128_32(reward_hi, reward_lo, static_cast<uint32_t>(median_weight), &reward_hi, &reward_lo); assert(0 == reward_hi); assert(reward_lo < base_reward); diff --git a/src/cryptonote_basic/cryptonote_basic_impl.h b/src/cryptonote_basic/cryptonote_basic_impl.h index f59785021..c804a88fa 100644 --- a/src/cryptonote_basic/cryptonote_basic_impl.h +++ b/src/cryptonote_basic/cryptonote_basic_impl.h @@ -86,10 +86,10 @@ namespace cryptonote { /************************************************************************/ /* Cryptonote helper functions */ /************************************************************************/ - size_t get_min_block_size(uint8_t version); + size_t get_min_block_weight(uint8_t version); size_t get_max_block_size(); size_t get_max_tx_size(); - bool get_block_reward(size_t median_size, size_t current_block_size, uint64_t already_generated_coins, uint64_t &reward, uint8_t version); + bool get_block_reward(size_t median_weight, size_t current_block_weight, uint64_t already_generated_coins, uint64_t &reward, uint8_t version); uint8_t get_account_address_checksum(const public_address_outer_blob& bl); uint8_t get_account_integrated_address_checksum(const public_integrated_address_outer_blob& bl); diff --git a/src/cryptonote_basic/cryptonote_boost_serialization.h b/src/cryptonote_basic/cryptonote_boost_serialization.h index 143133163..0725a2bb8 100644 --- a/src/cryptonote_basic/cryptonote_boost_serialization.h +++ b/src/cryptonote_basic/cryptonote_boost_serialization.h @@ -295,7 +295,7 @@ namespace boost a & x.type; if (x.type == rct::RCTTypeNull) return; - if (x.type != rct::RCTTypeFull && x.type != rct::RCTTypeFullBulletproof && x.type != rct::RCTTypeSimple && x.type != rct::RCTTypeSimpleBulletproof) + if (x.type != rct::RCTTypeFull && x.type != rct::RCTTypeSimple && x.type != rct::RCTTypeBulletproof) throw boost::archive::archive_exception(boost::archive::archive_exception::other_exception, "Unsupported rct type"); // a & x.message; message is not serialized, as it can be reconstructed from the tx data // a & x.mixRing; mixRing is not serialized, as it can be reconstructed from the offsets @@ -323,7 +323,7 @@ namespace boost a & x.type; if (x.type == rct::RCTTypeNull) return; - if (x.type != rct::RCTTypeFull && x.type != rct::RCTTypeFullBulletproof && x.type != rct::RCTTypeSimple && x.type != rct::RCTTypeSimpleBulletproof) + if (x.type != rct::RCTTypeFull && x.type != rct::RCTTypeSimple && x.type != rct::RCTTypeBulletproof) throw boost::archive::archive_exception(boost::archive::archive_exception::other_exception, "Unsupported rct type"); // a & x.message; message is not serialized, as it can be reconstructed from the tx data // a & x.mixRing; mixRing is not serialized, as it can be reconstructed from the offsets @@ -337,7 +337,7 @@ namespace boost if (x.p.rangeSigs.empty()) a & x.p.bulletproofs; a & x.p.MGs; - if (x.type == rct::RCTTypeSimpleBulletproof) + if (x.type == rct::RCTTypeBulletproof) a & x.p.pseudoOuts; } } diff --git a/src/cryptonote_basic/cryptonote_format_utils.cpp b/src/cryptonote_basic/cryptonote_format_utils.cpp index e9a9fba03..5fcfa33f6 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.cpp +++ b/src/cryptonote_basic/cryptonote_format_utils.cpp @@ -135,23 +135,41 @@ namespace cryptonote return false; } for (size_t n = 0; n < tx.rct_signatures.outPk.size(); ++n) + { + if (tx.vout[n].target.type() != typeid(txout_to_key)) + { + LOG_PRINT_L1("Unsupported output type in tx " << get_transaction_hash(tx)); + return false; + } rv.outPk[n].dest = rct::pk2rct(boost::get<txout_to_key>(tx.vout[n].target).key); + } if (!base_only) { - const bool bulletproof = rv.type == rct::RCTTypeFullBulletproof || rv.type == rct::RCTTypeSimpleBulletproof; + const bool bulletproof = rct::is_rct_bulletproof(rv.type); if (bulletproof) { - if (rv.p.bulletproofs.size() != tx.vout.size()) + if (rv.p.bulletproofs.size() != 1) { LOG_PRINT_L1("Failed to parse transaction from blob, bad bulletproofs size in tx " << get_transaction_hash(tx)); return false; } - for (size_t n = 0; n < rv.outPk.size(); ++n) + if (rv.p.bulletproofs[0].L.size() < 6) { - rv.p.bulletproofs[n].V.resize(1); - rv.p.bulletproofs[n].V[0] = rv.outPk[n].mask; + LOG_PRINT_L1("Failed to parse transaction from blob, bad bulletproofs L size in tx " << get_transaction_hash(tx)); + return false; + } + const size_t max_outputs = 1 << (rv.p.bulletproofs[0].L.size() - 6); + if (max_outputs < tx.vout.size()) + { + LOG_PRINT_L1("Failed to parse transaction from blob, bad bulletproofs max outputs in tx " << get_transaction_hash(tx)); + return false; } + const size_t n_amounts = tx.vout.size(); + CHECK_AND_ASSERT_MES(n_amounts == rv.outPk.size(), false, "Internal error filling out V"); + rv.p.bulletproofs[0].V.resize(n_amounts); + for (size_t i = 0; i < n_amounts; ++i) + rv.p.bulletproofs[0].V[i] = rct::scalarmultKey(rv.outPk[i].mask, rct::INV_EIGHT); } } } @@ -328,6 +346,37 @@ namespace cryptonote return string_tools::get_xtype_from_string(amount, str_amount); } //--------------------------------------------------------------- + uint64_t get_transaction_weight(const transaction &tx, size_t blob_size) + { + if (tx.version < 2) + return blob_size; + const rct::rctSig &rv = tx.rct_signatures; + if (!rct::is_rct_bulletproof(rv.type)) + return blob_size; + const size_t n_outputs = tx.vout.size(); + if (n_outputs <= 2) + return blob_size; + const uint64_t bp_base = 368; + const size_t n_padded_outputs = rct::n_bulletproof_max_amounts(rv.p.bulletproofs); + size_t nlr = 0; + for (const auto &bp: rv.p.bulletproofs) + nlr += bp.L.size() * 2; + const size_t bp_size = 32 * (9 + nlr); + CHECK_AND_ASSERT_THROW_MES_L1(bp_base * n_padded_outputs >= bp_size, "Invalid bulletproof clawback"); + const uint64_t bp_clawback = (bp_base * n_padded_outputs - bp_size) * 4 / 5; + CHECK_AND_ASSERT_THROW_MES_L1(bp_clawback <= std::numeric_limits<uint64_t>::max() - blob_size, "Weight overflow"); + return blob_size + bp_clawback; + } + //--------------------------------------------------------------- + uint64_t get_transaction_weight(const transaction &tx) + { + std::ostringstream s; + binary_archive<true> a(s); + ::serialization::serialize(a, const_cast<transaction&>(tx)); + const cryptonote::blobdata blob = s.str(); + return get_transaction_weight(tx, blob.size()); + } + //--------------------------------------------------------------- bool get_tx_fee(const transaction& tx, uint64_t & fee) { if (tx.version > 1) diff --git a/src/cryptonote_basic/cryptonote_format_utils.h b/src/cryptonote_basic/cryptonote_format_utils.h index 8a5296d5b..bf71eb591 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.h +++ b/src/cryptonote_basic/cryptonote_format_utils.h @@ -117,6 +117,8 @@ namespace cryptonote bool check_inputs_types_supported(const transaction& tx); bool check_outs_valid(const transaction& tx); bool parse_amount(uint64_t& amount, const std::string& str_amount); + uint64_t get_transaction_weight(const transaction &tx); + uint64_t get_transaction_weight(const transaction &tx, size_t blob_size); bool check_money_overflow(const transaction& tx); bool check_outs_overflow(const transaction& tx); diff --git a/src/cryptonote_config.h b/src/cryptonote_config.h index a0dcf2df1..a6858ce7c 100644 --- a/src/cryptonote_config.h +++ b/src/cryptonote_config.h @@ -65,9 +65,11 @@ #define FEE_PER_KB_OLD ((uint64_t)10000000000) // pow(10, 10) #define FEE_PER_KB ((uint64_t)2000000000) // 2 * pow(10, 9) +#define FEE_PER_BYTE ((uint64_t)300000) #define DYNAMIC_FEE_PER_KB_BASE_FEE ((uint64_t)2000000000) // 2 * pow(10,9) #define DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD ((uint64_t)10000000000000) // 10 * pow(10,12) #define DYNAMIC_FEE_PER_KB_BASE_FEE_V5 ((uint64_t)2000000000 * (uint64_t)CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V2 / CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5) +#define DYNAMIC_FEE_REFERENCE_TRANSACTION_WEIGHT ((uint64_t)3000) #define ORPHANED_BLOCKS_MAX_COUNT 100 @@ -133,13 +135,17 @@ #define HF_VERSION_DYNAMIC_FEE 4 #define HF_VERSION_MIN_MIXIN_4 6 #define HF_VERSION_MIN_MIXIN_6 7 +#define HF_VERSION_MIN_MIXIN_10 8 #define HF_VERSION_ENFORCE_RCT 6 +#define HF_VERSION_PER_BYTE_FEE 8 #define PER_KB_FEE_QUANTIZATION_DECIMALS 8 #define HASH_OF_HASHES_STEP 256 -#define DEFAULT_TXPOOL_MAX_SIZE 648000000ull // 3 days at 300000, in bytes +#define DEFAULT_TXPOOL_MAX_WEIGHT 648000000ull // 3 days at 300000, in bytes + +#define BULLETPROOF_MAX_OUTPUTS 16 // New constants are intended to go here namespace config diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index e96dc6bb6..c7e2f5a65 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -132,6 +132,7 @@ static const struct { { 6, 971400, 0, 1501709789 }, { 7, 1057027, 0, 1512211236 }, { 8, 1057058, 0, 1515967497 }, + { 9, 1057778, 0, 1515967498 }, }; static const uint64_t testnet_hard_fork_version_1_till = 624633; @@ -155,7 +156,7 @@ static const struct { //------------------------------------------------------------------ Blockchain::Blockchain(tx_memory_pool& tx_pool) : - m_db(), m_tx_pool(tx_pool), m_hardfork(NULL), m_timestamps_and_difficulties_height(0), m_current_block_cumul_sz_limit(0), m_current_block_cumul_sz_median(0), + m_db(), m_tx_pool(tx_pool), m_hardfork(NULL), m_timestamps_and_difficulties_height(0), m_current_block_cumul_weight_limit(0), m_current_block_cumul_weight_median(0), m_enforce_dns_checkpoints(false), m_max_prepare_blocks_threads(4), m_db_sync_on_blocks(true), m_db_sync_threshold(1), m_db_sync_mode(db_async), m_db_default_sync(false), m_fast_sync(true), m_show_time_stats(false), m_sync_counter(0), m_bytes_to_sync(0), m_cancel(false), m_difficulty_for_next_block_top_hash(crypto::null_hash), m_difficulty_for_next_block(1), @@ -482,7 +483,7 @@ bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline m_tx_pool.on_blockchain_dec(m_db->height()-1, get_tail_id()); } - update_next_cumulative_size_limit(); + update_next_cumulative_weight_limit(); return true; } //------------------------------------------------------------------ @@ -631,7 +632,7 @@ block Blockchain::pop_block_from_blockchain() m_blocks_txs_check.clear(); m_check_txin_table.clear(); - update_next_cumulative_size_limit(); + update_next_cumulative_weight_limit(); m_tx_pool.on_blockchain_dec(m_db->height()-1, get_tail_id()); invalidate_block_template_cache(); @@ -650,7 +651,7 @@ bool Blockchain::reset_and_set_genesis_block(const block& b) block_verification_context bvc = boost::value_initialized<block_verification_context>(); add_new_block(b, bvc); - update_next_cumulative_size_limit(); + update_next_cumulative_weight_limit(); return bvc.m_added_to_main_chain && !bvc.m_verifivation_failed; } //------------------------------------------------------------------ @@ -1113,7 +1114,7 @@ bool Blockchain::prevalidate_miner_transaction(const block& b, uint64_t height) } //------------------------------------------------------------------ // This function validates the miner transaction reward -bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins, bool &partial_block_reward, uint8_t version) +bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_block_weight, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins, bool &partial_block_reward, uint8_t version) { LOG_PRINT_L3("Blockchain::" << __func__); //validate reward @@ -1131,11 +1132,11 @@ bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_bl } } - std::vector<size_t> last_blocks_sizes; - get_last_n_blocks_sizes(last_blocks_sizes, CRYPTONOTE_REWARD_BLOCKS_WINDOW); - if (!get_block_reward(epee::misc_utils::median(last_blocks_sizes), cumulative_block_size, already_generated_coins, base_reward, version)) + std::vector<size_t> last_blocks_weights; + get_last_n_blocks_weights(last_blocks_weights, CRYPTONOTE_REWARD_BLOCKS_WINDOW); + if (!get_block_reward(epee::misc_utils::median(last_blocks_weights), cumulative_block_weight, already_generated_coins, base_reward, version)) { - MERROR_VER("block size " << cumulative_block_size << " is bigger than allowed for this blockchain"); + MERROR_VER("block weight " << cumulative_block_weight << " is bigger than allowed for this blockchain"); return false; } if(base_reward + fee < money_in_use) @@ -1165,8 +1166,8 @@ bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_bl return true; } //------------------------------------------------------------------ -// get the block sizes of the last <count> blocks, and return by reference <sz>. -void Blockchain::get_last_n_blocks_sizes(std::vector<size_t>& sz, size_t count) const +// get the block weights of the last <count> blocks, and return by reference <sz>. +void Blockchain::get_last_n_blocks_weights(std::vector<size_t>& weights, size_t count) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -1177,26 +1178,26 @@ void Blockchain::get_last_n_blocks_sizes(std::vector<size_t>& sz, size_t count) return; m_db->block_txn_start(true); - // add size of last <count> blocks to vector <sz> (or less, if blockchain size < count) + // add weight of last <count> blocks to vector <weights> (or less, if blockchain size < count) size_t start_offset = h - std::min<size_t>(h, count); - sz.reserve(sz.size() + h - start_offset); + weights.reserve(weights.size() + h - start_offset); for(size_t i = start_offset; i < h; i++) { - sz.push_back(m_db->get_block_size(i)); + weights.push_back(m_db->get_block_weight(i)); } m_db->block_txn_stop(); } //------------------------------------------------------------------ -uint64_t Blockchain::get_current_cumulative_blocksize_limit() const +uint64_t Blockchain::get_current_cumulative_block_weight_limit() const { LOG_PRINT_L3("Blockchain::" << __func__); - return m_current_block_cumul_sz_limit; + return m_current_block_cumul_weight_limit; } //------------------------------------------------------------------ -uint64_t Blockchain::get_current_cumulative_blocksize_median() const +uint64_t Blockchain::get_current_cumulative_block_weight_median() const { LOG_PRINT_L3("Blockchain::" << __func__); - return m_current_block_cumul_sz_median; + return m_current_block_cumul_weight_median; } //------------------------------------------------------------------ //TODO: This function only needed minor modification to work with BlockchainDB, @@ -1213,7 +1214,7 @@ uint64_t Blockchain::get_current_cumulative_blocksize_median() const bool Blockchain::create_block_template(block& b, const account_public_address& miner_address, difficulty_type& diffic, uint64_t& height, uint64_t& expected_reward, const blobdata& ex_nonce) { LOG_PRINT_L3("Blockchain::" << __func__); - size_t median_size; + size_t median_weight; uint64_t already_generated_coins; uint64_t pool_cookie; @@ -1250,20 +1251,20 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m diffic = get_difficulty_for_next_block(); CHECK_AND_ASSERT_MES(diffic, false, "difficulty overhead."); - median_size = m_current_block_cumul_sz_limit / 2; + median_weight = m_current_block_cumul_weight_limit / 2; already_generated_coins = m_db->get_block_already_generated_coins(height - 1); CRITICAL_REGION_END(); - size_t txs_size; + size_t txs_weight; uint64_t fee; - if (!m_tx_pool.fill_block_template(b, median_size, already_generated_coins, txs_size, fee, expected_reward, m_hardfork->get_current_version())) + if (!m_tx_pool.fill_block_template(b, median_weight, already_generated_coins, txs_weight, fee, expected_reward, m_hardfork->get_current_version())) { return false; } pool_cookie = m_tx_pool.cookie(); #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) - size_t real_txs_size = 0; + size_t real_txs_weight = 0; uint64_t real_fee = 0; CRITICAL_REGION_BEGIN(m_tx_pool.m_transactions_lock); for(crypto::hash &cur_hash: b.tx_hashes) @@ -1275,11 +1276,11 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m continue; } tx_memory_pool::tx_details &cur_tx = cur_res->second; - real_txs_size += cur_tx.blob_size; + real_txs_weight += cur_tx.weight; real_fee += cur_tx.fee; - if (cur_tx.blob_size != get_object_blobsize(cur_tx.tx)) + if (cur_tx.weight != get_transaction_weight(cur_tx.tx)) { - LOG_ERROR("Creating block template: error: invalid transaction size"); + LOG_ERROR("Creating block template: error: invalid transaction weight"); } if (cur_tx.tx.version == 1) { @@ -1301,9 +1302,9 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m } } } - if (txs_size != real_txs_size) + if (txs_weight != real_txs_weight) { - LOG_ERROR("Creating block template: error: wrongly calculated transaction size"); + LOG_ERROR("Creating block template: error: wrongly calculated transaction weight"); } if (fee != real_fee) { @@ -1311,70 +1312,70 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m } CRITICAL_REGION_END(); MDEBUG("Creating block template: height " << height << - ", median size " << median_size << + ", median weight " << median_weight << ", already generated coins " << already_generated_coins << - ", transaction size " << txs_size << + ", transaction weight " << txs_weight << ", fee " << fee); #endif /* - two-phase miner transaction generation: we don't know exact block size until we prepare block, but we don't know reward until we know - block size, so first miner transaction generated with fake amount of money, and with phase we know think we know expected block size + two-phase miner transaction generation: we don't know exact block weight until we prepare block, but we don't know reward until we know + block weight, so first miner transaction generated with fake amount of money, and with phase we know think we know expected block weight */ - //make blocks coin-base tx looks close to real coinbase tx to get truthful blob size + //make blocks coin-base tx looks close to real coinbase tx to get truthful blob weight uint8_t hf_version = m_hardfork->get_current_version(); size_t max_outs = hf_version >= 4 ? 1 : 11; - bool r = construct_miner_tx(height, median_size, already_generated_coins, txs_size, fee, miner_address, b.miner_tx, ex_nonce, max_outs, hf_version); + bool r = construct_miner_tx(height, median_weight, already_generated_coins, txs_weight, fee, miner_address, b.miner_tx, ex_nonce, max_outs, hf_version); CHECK_AND_ASSERT_MES(r, false, "Failed to construct miner tx, first chance"); - size_t cumulative_size = txs_size + get_object_blobsize(b.miner_tx); + size_t cumulative_weight = txs_weight + get_transaction_weight(b.miner_tx); #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) - MDEBUG("Creating block template: miner tx size " << get_object_blobsize(b.miner_tx) << - ", cumulative size " << cumulative_size); + MDEBUG("Creating block template: miner tx weight " << get_transaction_weight(b.miner_tx) << + ", cumulative weight " << cumulative_weight); #endif for (size_t try_count = 0; try_count != 10; ++try_count) { - r = construct_miner_tx(height, median_size, already_generated_coins, cumulative_size, fee, miner_address, b.miner_tx, ex_nonce, max_outs, hf_version); + r = construct_miner_tx(height, median_weight, already_generated_coins, cumulative_weight, fee, miner_address, b.miner_tx, ex_nonce, max_outs, hf_version); CHECK_AND_ASSERT_MES(r, false, "Failed to construct miner tx, second chance"); - size_t coinbase_blob_size = get_object_blobsize(b.miner_tx); - if (coinbase_blob_size > cumulative_size - txs_size) + size_t coinbase_weight = get_transaction_weight(b.miner_tx); + if (coinbase_weight > cumulative_weight - txs_weight) { - cumulative_size = txs_size + coinbase_blob_size; + cumulative_weight = txs_weight + coinbase_weight; #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) - MDEBUG("Creating block template: miner tx size " << coinbase_blob_size << - ", cumulative size " << cumulative_size << " is greater than before"); + MDEBUG("Creating block template: miner tx weight " << coinbase_weight << + ", cumulative weight " << cumulative_weight << " is greater than before"); #endif continue; } - if (coinbase_blob_size < cumulative_size - txs_size) + if (coinbase_weight < cumulative_weight - txs_weight) { - size_t delta = cumulative_size - txs_size - coinbase_blob_size; + size_t delta = cumulative_weight - txs_weight - coinbase_weight; #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) - MDEBUG("Creating block template: miner tx size " << coinbase_blob_size << - ", cumulative size " << txs_size + coinbase_blob_size << + MDEBUG("Creating block template: miner tx weight " << coinbase_weight << + ", cumulative weight " << txs_weight + coinbase_weight << " is less than before, adding " << delta << " zero bytes"); #endif b.miner_tx.extra.insert(b.miner_tx.extra.end(), delta, 0); //here could be 1 byte difference, because of extra field counter is varint, and it can become from 1-byte len to 2-bytes len. - if (cumulative_size != txs_size + get_object_blobsize(b.miner_tx)) + if (cumulative_weight != txs_weight + get_transaction_weight(b.miner_tx)) { - CHECK_AND_ASSERT_MES(cumulative_size + 1 == txs_size + get_object_blobsize(b.miner_tx), false, "unexpected case: cumulative_size=" << cumulative_size << " + 1 is not equal txs_cumulative_size=" << txs_size << " + get_object_blobsize(b.miner_tx)=" << get_object_blobsize(b.miner_tx)); + CHECK_AND_ASSERT_MES(cumulative_weight + 1 == txs_weight + get_transaction_weight(b.miner_tx), false, "unexpected case: cumulative_weight=" << cumulative_weight << " + 1 is not equal txs_cumulative_weight=" << txs_weight << " + get_transaction_weight(b.miner_tx)=" << get_transaction_weight(b.miner_tx)); b.miner_tx.extra.resize(b.miner_tx.extra.size() - 1); - if (cumulative_size != txs_size + get_object_blobsize(b.miner_tx)) + if (cumulative_weight != txs_weight + get_transaction_weight(b.miner_tx)) { - //fuck, not lucky, -1 makes varint-counter size smaller, in that case we continue to grow with cumulative_size + //fuck, not lucky, -1 makes varint-counter size smaller, in that case we continue to grow with cumulative_weight MDEBUG("Miner tx creation has no luck with delta_extra size = " << delta << " and " << delta - 1); - cumulative_size += delta - 1; + cumulative_weight += delta - 1; continue; } MDEBUG("Setting extra for block: " << b.miner_tx.extra.size() << ", try_count=" << try_count); } } - CHECK_AND_ASSERT_MES(cumulative_size == txs_size + get_object_blobsize(b.miner_tx), false, "unexpected case: cumulative_size=" << cumulative_size << " is not equal txs_cumulative_size=" << txs_size << " + get_object_blobsize(b.miner_tx)=" << get_object_blobsize(b.miner_tx)); + CHECK_AND_ASSERT_MES(cumulative_weight == txs_weight + get_transaction_weight(b.miner_tx), false, "unexpected case: cumulative_weight=" << cumulative_weight << " is not equal txs_cumulative_weight=" << txs_weight << " + get_transaction_weight(b.miner_tx)=" << get_transaction_weight(b.miner_tx)); #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) - MDEBUG("Creating block template: miner tx size " << coinbase_blob_size << - ", cumulative size " << cumulative_size << " is now good"); + MDEBUG("Creating block template: miner tx weight " << coinbase_weight << + ", cumulative weight " << cumulative_weight << " is now good"); #endif cache_block_template(b, miner_address, ex_nonce, diffic, expected_reward, pool_cookie); @@ -1716,17 +1717,6 @@ size_t Blockchain::get_alternative_blocks_count() const //------------------------------------------------------------------ // This function adds the output specified by <amount, i> to the result_outs container // unlocked and other such checks should be done by here. -void Blockchain::add_out_to_get_random_outs(COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i) const -{ - LOG_PRINT_L3("Blockchain::" << __func__); - CRITICAL_REGION_LOCAL(m_blockchain_lock); - - COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry& oen = *result_outs.outs.insert(result_outs.outs.end(), COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry()); - oen.global_amount_index = i; - output_data_t data = m_db->get_output_key(amount, i); - oen.out_key = data.pubkey; -} - uint64_t Blockchain::get_num_mature_outputs(uint64_t amount) const { uint64_t num_outs = m_db->get_num_outputs(amount); @@ -1744,74 +1734,6 @@ uint64_t Blockchain::get_num_mature_outputs(uint64_t amount) const return num_outs; } -std::vector<uint64_t> Blockchain::get_random_outputs(uint64_t amount, uint64_t count) const -{ - uint64_t num_outs = get_num_mature_outputs(amount); - - std::vector<uint64_t> indices; - - std::unordered_set<uint64_t> seen_indices; - - // if there aren't enough outputs to mix with (or just enough), - // use all of them. Eventually this should become impossible. - if (num_outs <= count) - { - for (uint64_t i = 0; i < num_outs; i++) - { - // get tx_hash, tx_out_index from DB - tx_out_index toi = m_db->get_output_tx_and_index(amount, i); - - // if tx is unlocked, add output to indices - if (is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first))) - { - indices.push_back(i); - } - } - } - else - { - // while we still need more mixins - while (indices.size() < count) - { - // if we've gone through every possible output, we've gotten all we can - if (seen_indices.size() == num_outs) - { - break; - } - - // get a random output index from the DB. If we've already seen it, - // return to the top of the loop and try again, otherwise add it to the - // list of output indices we've seen. - - // triangular distribution over [a,b) with a=0, mode c=b=up_index_limit - uint64_t r = crypto::rand<uint64_t>() % ((uint64_t)1 << 53); - double frac = std::sqrt((double)r / ((uint64_t)1 << 53)); - uint64_t i = (uint64_t)(frac*num_outs); - // just in case rounding up to 1 occurs after sqrt - if (i == num_outs) - --i; - - if (seen_indices.count(i)) - { - continue; - } - seen_indices.emplace(i); - - // get tx_hash, tx_out_index from DB - tx_out_index toi = m_db->get_output_tx_and_index(amount, i); - - // if the output's transaction is unlocked, add the output's index to - // our list. - if (is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first))) - { - indices.push_back(i); - } - } - } - - return indices; -} - crypto::public_key Blockchain::get_output_key(uint64_t amount, uint64_t global_index) const { output_data_t data = m_db->get_output_key(amount, global_index); @@ -1819,169 +1741,6 @@ crypto::public_key Blockchain::get_output_key(uint64_t amount, uint64_t global_i } //------------------------------------------------------------------ -// This function takes an RPC request for mixins and creates an RPC response -// with the requested mixins. -// TODO: figure out why this returns boolean / if we should be returning false -// in some cases -bool Blockchain::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) const -{ - LOG_PRINT_L3("Blockchain::" << __func__); - CRITICAL_REGION_LOCAL(m_blockchain_lock); - - // for each amount that we need to get mixins for, get <n> random outputs - // from BlockchainDB where <n> is req.outs_count (number of mixins). - for (uint64_t amount : req.amounts) - { - // create outs_for_amount struct and populate amount field - COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs = *res.outs.insert(res.outs.end(), COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount()); - result_outs.amount = amount; - - std::vector<uint64_t> indices = get_random_outputs(amount, req.outs_count); - - for (auto i : indices) - { - COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry& oe = *result_outs.outs.insert(result_outs.outs.end(), COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry()); - - oe.global_amount_index = i; - oe.out_key = get_output_key(amount, i); - } - } - return true; -} -//------------------------------------------------------------------ -// This function adds the ringct output at index i to the list -// unlocked and other such checks should be done by here. -void Blockchain::add_out_to_get_rct_random_outs(std::list<COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::out_entry>& outs, uint64_t amount, size_t i) const -{ - LOG_PRINT_L3("Blockchain::" << __func__); - CRITICAL_REGION_LOCAL(m_blockchain_lock); - - COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::out_entry& oen = *outs.insert(outs.end(), COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::out_entry()); - oen.amount = amount; - oen.global_amount_index = i; - output_data_t data = m_db->get_output_key(amount, i); - oen.out_key = data.pubkey; - oen.commitment = data.commitment; -} -//------------------------------------------------------------------ -// This function takes an RPC request for mixins and creates an RPC response -// with the requested mixins. -// TODO: figure out why this returns boolean / if we should be returning false -// in some cases -bool Blockchain::get_random_rct_outs(const COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::request& req, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::response& res) const -{ - LOG_PRINT_L3("Blockchain::" << __func__); - CRITICAL_REGION_LOCAL(m_blockchain_lock); - - // for each amount that we need to get mixins for, get <n> random outputs - // from BlockchainDB where <n> is req.outs_count (number of mixins). - auto num_outs = m_db->get_num_outputs(0); - // ensure we don't include outputs that aren't yet eligible to be used - // outpouts are sorted by height - while (num_outs > 0) - { - const tx_out_index toi = m_db->get_output_tx_and_index(0, num_outs - 1); - const uint64_t height = m_db->get_tx_block_height(toi.first); - if (height + CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE <= m_db->height()) - break; - --num_outs; - } - - std::unordered_set<uint64_t> seen_indices; - - // if there aren't enough outputs to mix with (or just enough), - // use all of them. Eventually this should become impossible. - if (num_outs <= req.outs_count) - { - for (uint64_t i = 0; i < num_outs; i++) - { - // get tx_hash, tx_out_index from DB - tx_out_index toi = m_db->get_output_tx_and_index(0, i); - - // if tx is unlocked, add output to result_outs - if (is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first))) - { - add_out_to_get_rct_random_outs(res.outs, 0, i); - } - } - } - else - { - // while we still need more mixins - while (res.outs.size() < req.outs_count) - { - // if we've gone through every possible output, we've gotten all we can - if (seen_indices.size() == num_outs) - { - break; - } - - // get a random output index from the DB. If we've already seen it, - // return to the top of the loop and try again, otherwise add it to the - // list of output indices we've seen. - - // triangular distribution over [a,b) with a=0, mode c=b=up_index_limit - uint64_t r = crypto::rand<uint64_t>() % ((uint64_t)1 << 53); - double frac = std::sqrt((double)r / ((uint64_t)1 << 53)); - uint64_t i = (uint64_t)(frac*num_outs); - // just in case rounding up to 1 occurs after sqrt - if (i == num_outs) - --i; - - if (seen_indices.count(i)) - { - continue; - } - seen_indices.emplace(i); - - // get tx_hash, tx_out_index from DB - tx_out_index toi = m_db->get_output_tx_and_index(0, i); - - // if the output's transaction is unlocked, add the output's index to - // our list. - if (is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first))) - { - add_out_to_get_rct_random_outs(res.outs, 0, i); - } - } - } - - if (res.outs.size() < req.outs_count) - return false; -#if 0 - // if we do not have enough RCT inputs, we can pick from the non RCT ones - // which will have a zero mask - if (res.outs.size() < req.outs_count) - { - LOG_PRINT_L0("Out of RCT inputs (" << res.outs.size() << "/" << req.outs_count << "), using regular ones"); - - // TODO: arbitrary selection, needs better - COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request req2 = AUTO_VAL_INIT(req2); - COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response res2 = AUTO_VAL_INIT(res2); - req2.outs_count = req.outs_count - res.outs.size(); - static const uint64_t amounts[] = {1, 10, 20, 50, 100, 200, 500, 1000, 10000}; - for (uint64_t a: amounts) - req2.amounts.push_back(a); - if (!get_random_outs_for_amounts(req2, res2)) - return false; - - // pick random ones from there - while (res.outs.size() < req.outs_count) - { - int list_idx = rand() % (sizeof(amounts)/sizeof(amounts[0])); - if (!res2.outs[list_idx].outs.empty()) - { - const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry oe = res2.outs[list_idx].outs.back(); - res2.outs[list_idx].outs.pop_back(); - add_out_to_get_rct_random_outs(res.outs, res2.outs[list_idx].amount, oe.global_amount_index); - } - } - } -#endif - - return true; -} -//------------------------------------------------------------------ bool Blockchain::get_outs(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res) const { LOG_PRINT_L3("Blockchain::" << __func__); @@ -2540,7 +2299,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, uint64_t& max_used_block_heigh if(m_show_time_stats) { size_t ring_size = !tx.vin.empty() && tx.vin[0].type() == typeid(txin_to_key) ? boost::get<txin_to_key>(tx.vin[0]).key_offsets.size() : 0; - MINFO("HASH: " << get_transaction_hash(tx) << " I/M/O: " << tx.vin.size() << "/" << ring_size << "/" << tx.vout.size() << " H: " << max_used_block_height << " ms: " << a + m_fake_scan_time << " B: " << get_object_blobsize(tx)); + MINFO("HASH: " << get_transaction_hash(tx) << " I/M/O: " << tx.vin.size() << "/" << ring_size << "/" << tx.vout.size() << " H: " << max_used_block_height << " ms: " << a + m_fake_scan_time << " B: " << get_object_blobsize(tx) << " W: " << get_transaction_weight(tx)); } if (!res) return false; @@ -2597,12 +2356,27 @@ bool Blockchain::check_tx_outputs(const transaction& tx, tx_verification_context // from v8, allow bulletproofs if (hf_version < 8) { - const bool bulletproof = tx.rct_signatures.type == rct::RCTTypeFullBulletproof || tx.rct_signatures.type == rct::RCTTypeSimpleBulletproof; - if (bulletproof || !tx.rct_signatures.p.bulletproofs.empty()) - { - MERROR("Bulletproofs are not allowed before v8"); - tvc.m_invalid_output = true; - return false; + if (tx.version >= 2) { + const bool bulletproof = rct::is_rct_bulletproof(tx.rct_signatures.type); + if (bulletproof || !tx.rct_signatures.p.bulletproofs.empty()) + { + MERROR("Bulletproofs are not allowed before v8"); + tvc.m_invalid_output = true; + return false; + } + } + } + + // from v9, forbid borromean range proofs + if (hf_version > 8) { + if (tx.version >= 2) { + const bool borromean = rct::is_rct_borromean(tx.rct_signatures.type); + if (borromean) + { + MERROR("Borromean range proofs are not allowed after v8"); + tvc.m_invalid_output = true; + return false; + } } } @@ -2631,7 +2405,7 @@ bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_pr rv.message = rct::hash2rct(tx_prefix_hash); // mixRing - full and simple store it in opposite ways - if (rv.type == rct::RCTTypeFull || rv.type == rct::RCTTypeFullBulletproof) + if (rv.type == rct::RCTTypeFull) { CHECK_AND_ASSERT_MES(!pubkeys.empty() && !pubkeys[0].empty(), false, "empty pubkeys"); rv.mixRing.resize(pubkeys[0].size()); @@ -2646,7 +2420,7 @@ bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_pr } } } - else if (rv.type == rct::RCTTypeSimple || rv.type == rct::RCTTypeSimpleBulletproof) + else if (rv.type == rct::RCTTypeSimple || rv.type == rct::RCTTypeBulletproof) { CHECK_AND_ASSERT_MES(!pubkeys.empty() && !pubkeys[0].empty(), false, "empty pubkeys"); rv.mixRing.resize(pubkeys.size()); @@ -2665,14 +2439,14 @@ bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_pr } // II - if (rv.type == rct::RCTTypeFull || rv.type == rct::RCTTypeFullBulletproof) + if (rv.type == rct::RCTTypeFull) { rv.p.MGs.resize(1); rv.p.MGs[0].II.resize(tx.vin.size()); for (size_t n = 0; n < tx.vin.size(); ++n) rv.p.MGs[0].II[n] = rct::ki2rct(boost::get<txin_to_key>(tx.vin[n]).k_image); } - else if (rv.type == rct::RCTTypeSimple || rv.type == rct::RCTTypeSimpleBulletproof) + else if (rv.type == rct::RCTTypeSimple || rv.type == rct::RCTTypeBulletproof) { CHECK_AND_ASSERT_MES(rv.p.MGs.size() == tx.vin.size(), false, "Bad MGs size"); for (size_t n = 0; n < tx.vin.size(); ++n) @@ -2714,7 +2488,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, { size_t n_unmixable = 0, n_mixable = 0; size_t mixin = std::numeric_limits<size_t>::max(); - const size_t min_mixin = hf_version >= HF_VERSION_MIN_MIXIN_6 ? 6 : hf_version >= HF_VERSION_MIN_MIXIN_4 ? 4 : 2; + const size_t min_mixin = hf_version >= HF_VERSION_MIN_MIXIN_10 ? 10 : hf_version >= HF_VERSION_MIN_MIXIN_6 ? 6 : hf_version >= HF_VERSION_MIN_MIXIN_4 ? 4 : 2; for (const auto& txin : tx.vin) { // non txin_to_key inputs will be rejected below @@ -2743,6 +2517,13 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, } } + if (hf_version >= HF_VERSION_MIN_MIXIN_10 && mixin != 10) + { + MERROR_VER("Tx " << get_transaction_hash(tx) << " has invalid ring size (" << (mixin + 1) << "), it should be 11"); + tvc.m_low_mixin = true; + return false; + } + if (mixin < min_mixin) { if (n_unmixable == 0) @@ -2938,7 +2719,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, return false; } case rct::RCTTypeSimple: - case rct::RCTTypeSimpleBulletproof: + case rct::RCTTypeBulletproof: { // check all this, either reconstructed (so should really pass), or not { @@ -2988,7 +2769,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, } } - if (!rct::verRctSimple(rv, false)) + if (!rct::verRctNonSemanticsSimple(rv)) { MERROR_VER("Failed to check ringct signatures!"); return false; @@ -2996,7 +2777,6 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, break; } case rct::RCTTypeFull: - case rct::RCTTypeFullBulletproof: { // check all this, either reconstructed (so should really pass), or not { @@ -3059,6 +2839,22 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, MERROR_VER("Unsupported rct type: " << rv.type); return false; } + + // for bulletproofs, check they're only multi-output after v8 + if (rct::is_rct_bulletproof(rv.type)) + { + if (hf_version < 8) + { + for (const rct::Bulletproof &proof: rv.p.bulletproofs) + { + if (proof.V.size() > 1) + { + MERROR_VER("Multi output bulletproofs are invalid before v8"); + return false; + } + } + } + } } return true; } @@ -3078,7 +2874,7 @@ void Blockchain::check_ring_signature(const crypto::hash &tx_prefix_hash, const } //------------------------------------------------------------------ -static uint64_t get_fee_quantization_mask() +uint64_t Blockchain::get_fee_quantization_mask() { static uint64_t mask = 0; if (mask == 0) @@ -3091,16 +2887,27 @@ static uint64_t get_fee_quantization_mask() } //------------------------------------------------------------------ -uint64_t Blockchain::get_dynamic_per_kb_fee(uint64_t block_reward, size_t median_block_size, uint8_t version) +uint64_t Blockchain::get_dynamic_base_fee(uint64_t block_reward, size_t median_block_weight, uint8_t version) { - const uint64_t min_block_size = get_min_block_size(version); - const uint64_t fee_per_kb_base = version >= 5 ? DYNAMIC_FEE_PER_KB_BASE_FEE_V5 : DYNAMIC_FEE_PER_KB_BASE_FEE; + const uint64_t min_block_weight = get_min_block_weight(version); + if (median_block_weight < min_block_weight) + median_block_weight = min_block_weight; + uint64_t hi, lo; - if (median_block_size < min_block_size) - median_block_size = min_block_size; + if (version >= HF_VERSION_PER_BYTE_FEE) + { + lo = mul128(block_reward, DYNAMIC_FEE_REFERENCE_TRANSACTION_WEIGHT, &hi); + div128_32(hi, lo, min_block_weight, &hi, &lo); + div128_32(hi, lo, median_block_weight, &hi, &lo); + assert(hi == 0); + lo /= 5; + return lo; + } - uint64_t unscaled_fee_per_kb = (fee_per_kb_base * min_block_size / median_block_size); - uint64_t hi, lo = mul128(unscaled_fee_per_kb, block_reward, &hi); + const uint64_t fee_base = version >= 5 ? DYNAMIC_FEE_PER_KB_BASE_FEE_V5 : DYNAMIC_FEE_PER_KB_BASE_FEE; + + uint64_t unscaled_fee_base = (fee_base * min_block_weight / median_block_weight); + lo = mul128(unscaled_fee_base, block_reward, &hi); static_assert(DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD % 1000000 == 0, "DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD must be divisible by 1000000"); static_assert(DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD / 1000000 <= std::numeric_limits<uint32_t>::max(), "DYNAMIC_FEE_PER_KB_BASE_BLOCK_REWARD is too large"); @@ -3118,29 +2925,48 @@ uint64_t Blockchain::get_dynamic_per_kb_fee(uint64_t block_reward, size_t median } //------------------------------------------------------------------ -bool Blockchain::check_fee(size_t blob_size, uint64_t fee) const +bool Blockchain::check_fee(size_t tx_weight, uint64_t fee) const { const uint8_t version = get_current_hard_fork_version(); - uint64_t fee_per_kb; - if (version < HF_VERSION_DYNAMIC_FEE) - { - fee_per_kb = FEE_PER_KB; - } - else + uint64_t median = 0; + uint64_t already_generated_coins = 0; + uint64_t base_reward = 0; + if (version >= HF_VERSION_DYNAMIC_FEE) { - uint64_t median = m_current_block_cumul_sz_limit / 2; - uint64_t already_generated_coins = m_db->height() ? m_db->get_block_already_generated_coins(m_db->height() - 1) : 0; - uint64_t base_reward; + median = m_current_block_cumul_weight_limit / 2; + already_generated_coins = m_db->height() ? m_db->get_block_already_generated_coins(m_db->height() - 1) : 0; if (!get_block_reward(median, 1, already_generated_coins, base_reward, version)) return false; - fee_per_kb = get_dynamic_per_kb_fee(base_reward, median, version); } - MDEBUG("Using " << print_money(fee_per_kb) << "/kB fee"); - uint64_t needed_fee = blob_size / 1024; - needed_fee += (blob_size % 1024) ? 1 : 0; - needed_fee *= fee_per_kb; + uint64_t needed_fee; + if (version >= HF_VERSION_PER_BYTE_FEE) + { + uint64_t fee_per_byte = get_dynamic_base_fee(base_reward, median, version); + MDEBUG("Using " << print_money(fee_per_byte) << "/byte fee"); + needed_fee = tx_weight * fee_per_byte; + // quantize fee up to 8 decimals + const uint64_t mask = get_fee_quantization_mask(); + needed_fee = (needed_fee + mask - 1) / mask * mask; + } + else + { + uint64_t fee_per_kb; + if (version < HF_VERSION_DYNAMIC_FEE) + { + fee_per_kb = FEE_PER_KB; + } + else + { + fee_per_kb = get_dynamic_base_fee(base_reward, median, version); + } + MDEBUG("Using " << print_money(fee_per_kb) << "/kB fee"); + + needed_fee = tx_weight / 1024; + needed_fee += (tx_weight % 1024) ? 1 : 0; + needed_fee *= fee_per_kb; + } if (fee < needed_fee - needed_fee / 50) // keep a little 2% buffer on acceptance - no integer overflow { @@ -3151,7 +2977,7 @@ bool Blockchain::check_fee(size_t blob_size, uint64_t fee) const } //------------------------------------------------------------------ -uint64_t Blockchain::get_dynamic_per_kb_fee_estimate(uint64_t grace_blocks) const +uint64_t Blockchain::get_dynamic_base_fee_estimate(uint64_t grace_blocks) const { const uint8_t version = get_current_hard_fork_version(); @@ -3161,16 +2987,16 @@ uint64_t Blockchain::get_dynamic_per_kb_fee_estimate(uint64_t grace_blocks) cons if (grace_blocks >= CRYPTONOTE_REWARD_BLOCKS_WINDOW) grace_blocks = CRYPTONOTE_REWARD_BLOCKS_WINDOW - 1; - const uint64_t min_block_size = get_min_block_size(version); - std::vector<size_t> sz; - get_last_n_blocks_sizes(sz, CRYPTONOTE_REWARD_BLOCKS_WINDOW - grace_blocks); - sz.reserve(grace_blocks); + const uint64_t min_block_weight = get_min_block_weight(version); + std::vector<size_t> weights; + get_last_n_blocks_weights(weights, CRYPTONOTE_REWARD_BLOCKS_WINDOW - grace_blocks); + weights.reserve(grace_blocks); for (size_t i = 0; i < grace_blocks; ++i) - sz.push_back(min_block_size); + weights.push_back(min_block_weight); - uint64_t median = epee::misc_utils::median(sz); - if(median <= min_block_size) - median = min_block_size; + uint64_t median = epee::misc_utils::median(weights); + if(median <= min_block_weight) + median = min_block_weight; uint64_t already_generated_coins = m_db->height() ? m_db->get_block_already_generated_coins(m_db->height() - 1) : 0; uint64_t base_reward; @@ -3180,8 +3006,9 @@ uint64_t Blockchain::get_dynamic_per_kb_fee_estimate(uint64_t grace_blocks) cons base_reward = BLOCK_REWARD_OVERESTIMATE; } - uint64_t fee = get_dynamic_per_kb_fee(base_reward, median, version); - MDEBUG("Estimating " << grace_blocks << "-block fee at " << print_money(fee) << "/kB"); + uint64_t fee = get_dynamic_base_fee(base_reward, median, version); + const bool per_byte = version < HF_VERSION_PER_BYTE_FEE; + MDEBUG("Estimating " << grace_blocks << "-block fee at " << print_money(fee) << "/" << (per_byte ? "byte" : "kB")); return fee; } @@ -3357,11 +3184,11 @@ bool Blockchain::flush_txes_from_pool(const std::vector<crypto::hash> &txids) for (const auto &txid: txids) { cryptonote::transaction tx; - size_t blob_size; + size_t tx_weight; uint64_t fee; bool relayed, do_not_relay, double_spend_seen; MINFO("Removing txid " << txid << " from the pool"); - if(m_tx_pool.have_tx(txid) && !m_tx_pool.take_tx(txid, tx, blob_size, fee, relayed, do_not_relay, double_spend_seen)) + if(m_tx_pool.have_tx(txid) && !m_tx_pool.take_tx(txid, tx, tx_weight, fee, relayed, do_not_relay, double_spend_seen)) { MERROR("Failed to remove txid " << txid << " from the pool"); res = false; @@ -3521,8 +3348,8 @@ leave: goto leave; } - size_t coinbase_blob_size = get_object_blobsize(bl.miner_tx); - size_t cumulative_block_size = coinbase_blob_size; + size_t coinbase_weight = get_transaction_weight(bl.miner_tx); + size_t cumulative_block_weight = coinbase_weight; std::vector<transaction> txs; key_images_container keys; @@ -3544,7 +3371,7 @@ leave: for (const crypto::hash& tx_id : bl.tx_hashes) { transaction tx; - size_t blob_size = 0; + size_t tx_weight = 0; uint64_t fee = 0; bool relayed = false, do_not_relay = false, double_spend_seen = false; TIME_MEASURE_START(aa); @@ -3563,7 +3390,7 @@ leave: TIME_MEASURE_START(bb); // get transaction with hash <tx_id> from tx_pool - if(!m_tx_pool.take_tx(tx_id, tx, blob_size, fee, relayed, do_not_relay, double_spend_seen)) + if(!m_tx_pool.take_tx(tx_id, tx, tx_weight, fee, relayed, do_not_relay, double_spend_seen)) { MERROR_VER("Block with id: " << id << " has at least one unknown transaction with id: " << tx_id); bvc.m_verifivation_failed = true; @@ -3634,7 +3461,7 @@ leave: TIME_MEASURE_FINISH(cc); t_checktx += cc; fee_summary += fee; - cumulative_block_size += blob_size; + cumulative_block_weight += tx_weight; } m_blocks_txs_check.clear(); @@ -3642,7 +3469,7 @@ leave: TIME_MEASURE_START(vmt); uint64_t base_reward = 0; uint64_t already_generated_coins = m_db->height() ? m_db->get_block_already_generated_coins(m_db->height() - 1) : 0; - if(!validate_miner_transaction(bl, cumulative_block_size, fee_summary, base_reward, already_generated_coins, bvc.m_partial_block_reward, m_hardfork->get_current_version())) + if(!validate_miner_transaction(bl, cumulative_block_weight, fee_summary, base_reward, already_generated_coins, bvc.m_partial_block_reward, m_hardfork->get_current_version())) { MERROR_VER("Block with id: " << id << " has incorrect miner transaction"); bvc.m_verifivation_failed = true; @@ -3651,11 +3478,11 @@ leave: } TIME_MEASURE_FINISH(vmt); - size_t block_size; + size_t block_weight; difficulty_type cumulative_difficulty; // populate various metadata about the block to be stored alongside it. - block_size = cumulative_block_size; + block_weight = cumulative_block_weight; cumulative_difficulty = current_diffic; // In the "tail" state when the minimum subsidy (implemented in get_block_reward) is in effect, the number of // coins will eventually exceed MONEY_SUPPLY and overflow a uint64. To prevent overflow, cap already_generated_coins @@ -3676,7 +3503,7 @@ leave: { try { - new_height = m_db->add_block(bl, block_size, cumulative_difficulty, already_generated_coins, txs); + new_height = m_db->add_block(bl, block_weight, cumulative_difficulty, already_generated_coins, txs); } catch (const KEY_IMAGE_EXISTS& e) { @@ -3701,14 +3528,14 @@ leave: TIME_MEASURE_FINISH(addblock); - // do this after updating the hard fork state since the size limit may change due to fork - update_next_cumulative_size_limit(); + // do this after updating the hard fork state since the weight limit may change due to fork + update_next_cumulative_weight_limit(); - MINFO("+++++ BLOCK SUCCESSFULLY ADDED" << std::endl << "id:\t" << id << std::endl << "PoW:\t" << proof_of_work << std::endl << "HEIGHT " << new_height-1 << ", difficulty:\t" << current_diffic << std::endl << "block reward: " << print_money(fee_summary + base_reward) << "(" << print_money(base_reward) << " + " << print_money(fee_summary) << "), coinbase_blob_size: " << coinbase_blob_size << ", cumulative size: " << cumulative_block_size << ", " << block_processing_time << "(" << target_calculating_time << "/" << longhash_calculating_time << ")ms"); + MINFO("+++++ BLOCK SUCCESSFULLY ADDED" << std::endl << "id:\t" << id << std::endl << "PoW:\t" << proof_of_work << std::endl << "HEIGHT " << new_height-1 << ", difficulty:\t" << current_diffic << std::endl << "block reward: " << print_money(fee_summary + base_reward) << "(" << print_money(base_reward) << " + " << print_money(fee_summary) << "), coinbase_weight: " << coinbase_weight << ", cumulative weight: " << cumulative_block_weight << ", " << block_processing_time << "(" << target_calculating_time << "/" << longhash_calculating_time << ")ms"); if(m_show_time_stats) { - MINFO("Height: " << new_height << " blob: " << coinbase_blob_size << " cumm: " - << cumulative_block_size << " p/t: " << block_processing_time << " (" + MINFO("Height: " << new_height << " coinbase weight: " << coinbase_weight << " cumm: " + << cumulative_block_weight << " p/t: " << block_processing_time << " (" << target_calculating_time << "/" << longhash_calculating_time << "/" << t1 << "/" << t2 << "/" << t3 << "/" << t_exists << "/" << t_pool << "/" << t_checktx << "/" << t_dblspnd << "/" << vmt << "/" << addblock << ")ms"); @@ -3725,20 +3552,20 @@ leave: return true; } //------------------------------------------------------------------ -bool Blockchain::update_next_cumulative_size_limit() +bool Blockchain::update_next_cumulative_weight_limit() { - uint64_t full_reward_zone = get_min_block_size(get_current_hard_fork_version()); + uint64_t full_reward_zone = get_min_block_weight(get_current_hard_fork_version()); LOG_PRINT_L3("Blockchain::" << __func__); - std::vector<size_t> sz; - get_last_n_blocks_sizes(sz, CRYPTONOTE_REWARD_BLOCKS_WINDOW); + std::vector<size_t> weights; + get_last_n_blocks_weights(weights, CRYPTONOTE_REWARD_BLOCKS_WINDOW); - uint64_t median = epee::misc_utils::median(sz); - m_current_block_cumul_sz_median = median; + uint64_t median = epee::misc_utils::median(weights); + m_current_block_cumul_weight_median = median; if(median <= full_reward_zone) median = full_reward_zone; - m_current_block_cumul_sz_limit = median*2; + m_current_block_cumul_weight_limit = median*2; return true; } //------------------------------------------------------------------ @@ -4635,14 +4462,14 @@ void Blockchain::load_compiled_in_block_hashes() std::vector<transaction> txs; m_tx_pool.get_transactions(txs); - size_t blob_size; + size_t tx_weight; uint64_t fee; bool relayed, do_not_relay, double_spend_seen; transaction pool_tx; for(const transaction &tx : txs) { crypto::hash tx_hash = get_transaction_hash(tx); - m_tx_pool.take_tx(tx_hash, pool_tx, blob_size, fee, relayed, do_not_relay, double_spend_seen); + m_tx_pool.take_tx(tx_hash, pool_tx, tx_weight, fee, relayed, do_not_relay, double_spend_seen); } } } diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index 2292ffbf3..50ceccd0f 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -95,7 +95,7 @@ namespace cryptonote { block bl; //!< the block uint64_t height; //!< the height of the block in the blockchain - size_t block_cumulative_size; //!< the size (in bytes) of the block + size_t block_cumulative_weight; //!< the weight of the block difficulty_type cumulative_difficulty; //!< the accumulated difficulty after that block uint64_t already_generated_coins; //!< the total coins minted after that block }; @@ -447,16 +447,6 @@ namespace cryptonote uint64_t get_num_mature_outputs(uint64_t amount) const; /** - * @brief get random outputs (indices) for an amount - * - * @param amount the amount - * @param count the number of random outputs to choose - * - * @return the outputs' amount-global indices - */ - std::vector<uint64_t> get_random_outputs(uint64_t amount, uint64_t count) const; - - /** * @brief get the public key for an output * * @param amount the output amount @@ -467,22 +457,6 @@ namespace cryptonote crypto::public_key get_output_key(uint64_t amount, uint64_t global_index) const; /** - * @brief gets random outputs to mix with - * - * This function takes an RPC request for outputs to mix with - * and creates an RPC response with the resultant output indices. - * - * Outputs to mix with are randomly selected from the utxo set - * for each output amount in the request. - * - * @param req the output amounts and number of mixins to select - * @param res return-by-reference the resultant output indices - * - * @return true - */ - bool get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) const; - - /** * @brief gets specific outputs to mix with * * This function takes an RPC request for outputs to mix with @@ -509,23 +483,6 @@ namespace cryptonote void get_output_key_mask_unlocked(const uint64_t& amount, const uint64_t& index, crypto::public_key& key, rct::key& mask, bool& unlocked) const; /** - * @brief gets random ringct outputs to mix with - * - * This function takes an RPC request for outputs to mix with - * and creates an RPC response with the resultant output indices - * and the matching keys. - * - * Outputs to mix with are randomly selected from the utxo set - * for each output amount in the request. - * - * @param req the output amounts and number of mixins to select - * @param res return-by-reference the resultant output indices - * - * @return true - */ - bool get_random_rct_outs(const COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::request& req, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::response& res) const; - - /** * @brief gets per block distribution of outputs of a given amount * * @param amount the amount to get a ditribution for @@ -579,46 +536,57 @@ namespace cryptonote bool check_tx_inputs(transaction& tx, uint64_t& pmax_used_block_height, crypto::hash& max_used_block_id, tx_verification_context &tvc, bool kept_by_block = false); /** - * @brief get dynamic per kB fee for a given block size + * @brief get fee quantization mask + * + * The dynamic fee may be quantized, to mask out the last decimal places + * + * @return the fee quantized mask + */ + static uint64_t get_fee_quantization_mask(); + + /** + * @brief get dynamic per kB or byte fee for a given block weight * - * The dynamic fee is based on the block size in a past window, and - * the current block reward. It is expressed by kB. + * The dynamic fee is based on the block weight in a past window, and + * the current block reward. It is expressed by kB before v8, and + * per byte from v8. * * @param block_reward the current block reward - * @param median_block_size the median blob's size in the past window + * @param median_block_weight the median block weight in the past window * @param version hard fork version for rules and constants to use * - * @return the per kB fee + * @return the fee */ - static uint64_t get_dynamic_per_kb_fee(uint64_t block_reward, size_t median_block_size, uint8_t version); + static uint64_t get_dynamic_base_fee(uint64_t block_reward, size_t median_block_weight, uint8_t version); /** - * @brief get dynamic per kB fee estimate for the next few blocks + * @brief get dynamic per kB or byte fee estimate for the next few blocks * - * The dynamic fee is based on the block size in a past window, and - * the current block reward. It is expressed by kB. This function - * calculates an estimate for a dynamic fee which will be valid for - * the next grace_blocks + * The dynamic fee is based on the block weight in a past window, and + * the current block reward. It is expressed by kB before v8, and + * per byte from v8. + * This function calculates an estimate for a dynamic fee which will be + * valid for the next grace_blocks * * @param grace_blocks number of blocks we want the fee to be valid for * - * @return the per kB fee estimate + * @return the fee estimate */ - uint64_t get_dynamic_per_kb_fee_estimate(uint64_t grace_blocks) const; + uint64_t get_dynamic_base_fee_estimate(uint64_t grace_blocks) const; /** * @brief validate a transaction's fee * * This function validates the fee is enough for the transaction. - * This is based on the size of the transaction blob, and, after a - * height threshold, on the average size of transaction in a past window + * This is based on the weight of the transaction, and, after a + * height threshold, on the average weight of transaction in a past window * - * @param blob_size the transaction blob's size + * @param tx_weight the transaction weight * @param fee the fee * * @return true if the fee is enough, false otherwise */ - bool check_fee(size_t blob_size, uint64_t fee) const; + bool check_fee(size_t tx_weight, uint64_t fee) const; /** * @brief check that a transaction's outputs conform to current standards @@ -635,18 +603,18 @@ namespace cryptonote bool check_tx_outputs(const transaction& tx, tx_verification_context &tvc); /** - * @brief gets the blocksize limit based on recent blocks + * @brief gets the block weight limit based on recent blocks * * @return the limit */ - uint64_t get_current_cumulative_blocksize_limit() const; + uint64_t get_current_cumulative_block_weight_limit() const; /** - * @brief gets the blocksize median based on recent blocks (same window as for the limit) + * @brief gets the block weight median based on recent blocks (same window as for the limit) * * @return the median */ - uint64_t get_current_cumulative_blocksize_median() const; + uint64_t get_current_cumulative_block_weight_median() const; /** * @brief gets the difficulty of the block with a given height @@ -1001,8 +969,8 @@ namespace cryptonote // main chain transactions_container m_transactions; - size_t m_current_block_cumul_sz_limit; - size_t m_current_block_cumul_sz_median; + size_t m_current_block_cumul_weight_limit; + size_t m_current_block_cumul_weight_median; // metadata containers std::unordered_map<crypto::hash, std::unordered_map<crypto::key_image, std::vector<output_data_t>>> m_scan_table; @@ -1225,7 +1193,7 @@ namespace cryptonote * and that his miner transaction totals reward + fee. * * @param b the block containing the miner transaction to be validated - * @param cumulative_block_size the block's size + * @param cumulative_block_weight the block's weight * @param fee the total fees collected in the block * @param base_reward return-by-reference the new block's generated coins * @param already_generated_coins the amount of currency generated prior to this block @@ -1234,7 +1202,7 @@ namespace cryptonote * * @return false if anything is found wrong with the miner transaction, otherwise true */ - bool validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins, bool &partial_block_reward, uint8_t version); + bool validate_miner_transaction(const block& b, size_t cumulative_block_weight, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins, bool &partial_block_reward, uint8_t version); /** * @brief reverts the blockchain to its previous state following a failed switch @@ -1251,32 +1219,14 @@ namespace cryptonote bool rollback_blockchain_switching(std::list<block>& original_chain, uint64_t rollback_height); /** - * @brief gets recent block sizes for median calculation + * @brief gets recent block weights for median calculation * - * get the block sizes of the last <count> blocks, and return by reference <sz>. - * - * @param sz return-by-reference the list of sizes - * @param count the number of blocks to get sizes for - */ - void get_last_n_blocks_sizes(std::vector<size_t>& sz, size_t count) const; - - /** - * @brief adds the given output to the requested set of random outputs - * - * @param result_outs return-by-reference the set the output is to be added to - * @param amount the output amount - * @param i the output index (indexed to amount) - */ - void add_out_to_get_random_outs(COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i) const; - - /** - * @brief adds the given output to the requested set of random ringct outputs + * get the block weights of the last <count> blocks, and return by reference <sz>. * - * @param outs return-by-reference the set the output is to be added to - * @param amount the output amount (0 for rct inputs) - * @param i the rct output index + * @param sz return-by-reference the list of weights + * @param count the number of blocks to get weights for */ - void add_out_to_get_rct_random_outs(std::list<COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::out_entry>& outs, uint64_t amount, size_t i) const; + void get_last_n_blocks_weights(std::vector<size_t>& weights, size_t count) const; /** * @brief checks if a transaction is unlocked (its outputs spendable) @@ -1373,11 +1323,11 @@ namespace cryptonote bool complete_timestamps_vector(uint64_t start_height, std::vector<uint64_t>& timestamps); /** - * @brief calculate the block size limit for the next block to be added + * @brief calculate the block weight limit for the next block to be added * * @return true */ - bool update_next_cumulative_size_limit(); + bool update_next_cumulative_weight_limit(); void return_tx_to_pool(std::vector<transaction> &txs); /** diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index d0db38799..7cbf414b7 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -162,10 +162,10 @@ namespace cryptonote , "Relay blocks as normal blocks" , false }; - static const command_line::arg_descriptor<size_t> arg_max_txpool_size = { - "max-txpool-size" - , "Set maximum txpool size in bytes." - , DEFAULT_TXPOOL_MAX_SIZE + static const command_line::arg_descriptor<size_t> arg_max_txpool_weight = { + "max-txpool-weight" + , "Set maximum txpool weight in bytes." + , DEFAULT_TXPOOL_MAX_WEIGHT }; //----------------------------------------------------------------------------------------------- @@ -274,7 +274,7 @@ namespace cryptonote command_line::add_arg(desc, arg_test_dbg_lock_sleep); command_line::add_arg(desc, arg_offline); command_line::add_arg(desc, arg_disable_dns_checkpoints); - command_line::add_arg(desc, arg_max_txpool_size); + command_line::add_arg(desc, arg_max_txpool_weight); miner::init_options(desc); BlockchainDB::init_options(desc); @@ -402,7 +402,7 @@ namespace cryptonote bool fast_sync = command_line::get_arg(vm, arg_fast_block_sync) != 0; uint64_t blocks_threads = command_line::get_arg(vm, arg_prep_blocks_threads); std::string check_updates_string = command_line::get_arg(vm, arg_check_updates); - size_t max_txpool_size = command_line::get_arg(vm, arg_max_txpool_size); + size_t max_txpool_weight = command_line::get_arg(vm, arg_max_txpool_weight); boost::filesystem::path folder(m_config_folder); if (m_nettype == FAKECHAIN) @@ -551,7 +551,7 @@ namespace cryptonote const difficulty_type fixed_difficulty = command_line::get_arg(vm, arg_fixed_difficulty); r = m_blockchain_storage.init(db.release(), m_nettype, m_offline, regtest ? ®test_test_options : test_options, fixed_difficulty); - r = m_mempool.init(max_txpool_size); + r = m_mempool.init(max_txpool_weight); CHECK_AND_ASSERT_MES(r, false, "Failed to initialize memory pool"); // now that we have a valid m_blockchain_storage, we can clean out any @@ -692,26 +692,123 @@ namespace cryptonote return false; } + return true; + } + //----------------------------------------------------------------------------------------------- + void core::set_semantics_failed(const crypto::hash &tx_hash) + { + LOG_PRINT_L1("WRONG TRANSACTION BLOB, Failed to check tx " << tx_hash << " semantic, rejected"); + bad_semantics_txes_lock.lock(); + bad_semantics_txes[0].insert(tx_hash); + if (bad_semantics_txes[0].size() >= BAD_SEMANTICS_TXES_MAX_SIZE) + { + std::swap(bad_semantics_txes[0], bad_semantics_txes[1]); + bad_semantics_txes[0].clear(); + } + bad_semantics_txes_lock.unlock(); + } + //----------------------------------------------------------------------------------------------- + static bool is_canonical_bulletproof_layout(const std::vector<rct::Bulletproof> &proofs) + { + if (proofs.size() != 1) + return false; + const size_t sz = proofs[0].V.size(); + if (sz == 0 || sz > BULLETPROOF_MAX_OUTPUTS) + return false; + return true; + } + //----------------------------------------------------------------------------------------------- + bool core::handle_incoming_tx_accumulated_batch(std::vector<tx_verification_batch_info> &tx_info, bool keeped_by_block) + { + bool ret = true; if (keeped_by_block && get_blockchain_storage().is_within_compiled_block_hash_area()) { MTRACE("Skipping semantics check for tx kept by block in embedded hash area"); + return true; } - else if(!check_tx_semantic(tx, keeped_by_block)) + + std::vector<const rct::rctSig*> rvv; + for (size_t n = 0; n < tx_info.size(); ++n) { - LOG_PRINT_L1("WRONG TRANSACTION BLOB, Failed to check tx " << tx_hash << " semantic, rejected"); - tvc.m_verifivation_failed = true; - bad_semantics_txes_lock.lock(); - bad_semantics_txes[0].insert(tx_hash); - if (bad_semantics_txes[0].size() >= BAD_SEMANTICS_TXES_MAX_SIZE) + if (!check_tx_semantic(*tx_info[n].tx, keeped_by_block)) { - std::swap(bad_semantics_txes[0], bad_semantics_txes[1]); - bad_semantics_txes[0].clear(); + set_semantics_failed(tx_info[n].tx_hash); + tx_info[n].tvc.m_verifivation_failed = true; + tx_info[n].result = false; + continue; + } + + if (tx_info[n].tx->version < 2) + continue; + const rct::rctSig &rv = tx_info[n].tx->rct_signatures; + switch (rv.type) { + case rct::RCTTypeNull: + // coinbase should not come here, so we reject for all other types + MERROR_VER("Unexpected Null rctSig type"); + set_semantics_failed(tx_info[n].tx_hash); + tx_info[n].tvc.m_verifivation_failed = true; + tx_info[n].result = false; + break; + case rct::RCTTypeSimple: + if (!rct::verRctSemanticsSimple(rv)) + { + MERROR_VER("rct signature semantics check failed"); + set_semantics_failed(tx_info[n].tx_hash); + tx_info[n].tvc.m_verifivation_failed = true; + tx_info[n].result = false; + break; + } + break; + case rct::RCTTypeFull: + if (!rct::verRct(rv, true)) + { + MERROR_VER("rct signature semantics check failed"); + set_semantics_failed(tx_info[n].tx_hash); + tx_info[n].tvc.m_verifivation_failed = true; + tx_info[n].result = false; + break; + } + break; + case rct::RCTTypeBulletproof: + if (!is_canonical_bulletproof_layout(rv.p.bulletproofs)) + { + MERROR_VER("Bulletproof does not have canonical form"); + set_semantics_failed(tx_info[n].tx_hash); + tx_info[n].tvc.m_verifivation_failed = true; + tx_info[n].result = false; + break; + } + rvv.push_back(&rv); // delayed batch verification + break; + default: + MERROR_VER("Unknown rct type: " << rv.type); + set_semantics_failed(tx_info[n].tx_hash); + tx_info[n].tvc.m_verifivation_failed = true; + tx_info[n].result = false; + break; + } + } + if (!rvv.empty() && !rct::verRctSemanticsSimple(rvv)) + { + LOG_PRINT_L1("One transaction among this group has bad semantics, verifying one at a time"); + ret = false; + const bool assumed_bad = rvv.size() == 1; // if there's only one tx, it must be the bad one + for (size_t n = 0; n < tx_info.size(); ++n) + { + if (!tx_info[n].result) + continue; + if (tx_info[n].tx->rct_signatures.type != rct::RCTTypeBulletproof) + continue; + if (assumed_bad || !rct::verRctSemanticsSimple(tx_info[n].tx->rct_signatures)) + { + set_semantics_failed(tx_info[n].tx_hash); + tx_info[n].tvc.m_verifivation_failed = true; + tx_info[n].result = false; + } } - bad_semantics_txes_lock.unlock(); - return false; } - return true; + return ret; } //----------------------------------------------------------------------------------------------- bool core::handle_incoming_txs(const std::vector<blobdata>& tx_blobs, std::vector<tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) @@ -769,6 +866,16 @@ namespace cryptonote } waiter.wait(&tpool); + std::vector<tx_verification_batch_info> tx_info; + tx_info.reserve(tx_blobs.size()); + for (size_t i = 0; i < tx_blobs.size(); i++) { + if (!results[i].res) + continue; + tx_info.push_back({&results[i].tx, results[i].hash, tvc[i], results[i].res}); + } + if (!tx_info.empty()) + handle_incoming_tx_accumulated_batch(tx_info, keeped_by_block); + bool ok = true; it = tx_blobs.begin(); for (size_t i = 0; i < tx_blobs.size(); i++, ++it) { @@ -778,7 +885,8 @@ namespace cryptonote continue; } - ok &= add_new_tx(results[i].tx, results[i].hash, results[i].prefix_hash, it->size(), tvc[i], keeped_by_block, relayed, do_not_relay); + const size_t weight = get_transaction_weight(results[i].tx, it->size()); + ok &= add_new_tx(results[i].tx, results[i].hash, results[i].prefix_hash, weight, tvc[i], keeped_by_block, relayed, do_not_relay); if(tvc[i].m_verifivation_failed) {MERROR_VER("Transaction verification failed: " << results[i].hash);} else if(tvc[i].m_verifivation_impossible) @@ -861,9 +969,9 @@ namespace cryptonote } // for version > 1, ringct signatures check verifies amounts match - if(!keeped_by_block && get_object_blobsize(tx) >= m_blockchain_storage.get_current_cumulative_blocksize_limit() - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE) + if(!keeped_by_block && get_transaction_weight(tx) >= m_blockchain_storage.get_current_cumulative_block_weight_limit() - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE) { - MERROR_VER("tx is too large " << get_object_blobsize(tx) << ", expected not bigger than " << m_blockchain_storage.get_current_cumulative_blocksize_limit() - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE); + MERROR_VER("tx is too large " << get_transaction_weight(tx) << ", expected not bigger than " << m_blockchain_storage.get_current_cumulative_block_weight_limit() - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE); return false; } @@ -886,36 +994,6 @@ namespace cryptonote return false; } - if (tx.version >= 2) - { - const rct::rctSig &rv = tx.rct_signatures; - switch (rv.type) { - case rct::RCTTypeNull: - // coinbase should not come here, so we reject for all other types - MERROR_VER("Unexpected Null rctSig type"); - return false; - case rct::RCTTypeSimple: - case rct::RCTTypeSimpleBulletproof: - if (!rct::verRctSimple(rv, true)) - { - MERROR_VER("rct signature semantics check failed"); - return false; - } - break; - case rct::RCTTypeFull: - case rct::RCTTypeFullBulletproof: - if (!rct::verRct(rv, true)) - { - MERROR_VER("rct signature semantics check failed"); - return false; - } - break; - default: - MERROR_VER("Unknown rct type: " << rv.type); - return false; - } - } - return true; } //----------------------------------------------------------------------------------------------- @@ -1025,7 +1103,8 @@ namespace cryptonote crypto::hash tx_prefix_hash = get_transaction_prefix_hash(tx); blobdata bl; t_serializable_object_to_blob(tx, bl); - return add_new_tx(tx, tx_hash, tx_prefix_hash, bl.size(), tvc, keeped_by_block, relayed, do_not_relay); + size_t tx_weight = get_transaction_weight(tx, bl.size()); + return add_new_tx(tx, tx_hash, tx_prefix_hash, tx_weight, tvc, keeped_by_block, relayed, do_not_relay); } //----------------------------------------------------------------------------------------------- size_t core::get_blockchain_total_transactions() const @@ -1033,7 +1112,7 @@ namespace cryptonote return m_blockchain_storage.get_total_transactions(); } //----------------------------------------------------------------------------------------------- - bool core::add_new_tx(transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t blob_size, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) + bool core::add_new_tx(transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t tx_weight, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { if (keeped_by_block) get_blockchain_storage().on_new_tx_from_block(tx); @@ -1051,7 +1130,7 @@ namespace cryptonote } uint8_t version = m_blockchain_storage.get_current_hard_fork_version(); - return m_mempool.add_tx(tx, tx_hash, blob_size, tvc, keeped_by_block, relayed, do_not_relay, version); + return m_mempool.add_tx(tx, tx_hash, tx_weight, tvc, keeped_by_block, relayed, do_not_relay, version); } //----------------------------------------------------------------------------------------------- bool core::relay_txpool_transactions() @@ -1102,21 +1181,11 @@ namespace cryptonote return m_blockchain_storage.find_blockchain_supplement(req_start_block, qblock_ids, blocks, total_height, start_height, pruned, get_miner_tx_hash, max_count); } //----------------------------------------------------------------------------------------------- - bool core::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) const - { - return m_blockchain_storage.get_random_outs_for_amounts(req, res); - } - //----------------------------------------------------------------------------------------------- bool core::get_outs(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res) const { return m_blockchain_storage.get_outs(req, res); } //----------------------------------------------------------------------------------------------- - bool core::get_random_rct_outs(const COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::request& req, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::response& res) const - { - return m_blockchain_storage.get_random_rct_outs(req, res); - } - //----------------------------------------------------------------------------------------------- bool core::get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t to_height, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) const { return m_blockchain_storage.get_output_distribution(amount, from_height, to_height, start_height, distribution, base); diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 84e1bb918..b40575ae9 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -551,13 +551,6 @@ namespace cryptonote difficulty_type get_block_cumulative_difficulty(uint64_t height) const; /** - * @copydoc Blockchain::get_random_outs_for_amounts - * - * @note see Blockchain::get_random_outs_for_amounts - */ - bool get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) const; - - /** * @copydoc Blockchain::get_outs * * @note see Blockchain::get_outs @@ -565,14 +558,6 @@ namespace cryptonote bool get_outs(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res) const; /** - * - * @copydoc Blockchain::get_random_rct_outs - * - * @note see Blockchain::get_random_rct_outs - */ - bool get_random_rct_outs(const COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::request& req, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::response& res) const; - - /** * @copydoc Blockchain::get_output_distribution * * @brief get per block distribution of outputs of a given amount @@ -788,12 +773,12 @@ namespace cryptonote * * @param tx_hash the transaction's hash * @param tx_prefix_hash the transaction prefix' hash - * @param blob_size the size of the transaction + * @param tx_weight the weight of the transaction * @param relayed whether or not the transaction was relayed to us * @param do_not_relay whether to prevent the transaction from being relayed * */ - bool add_new_tx(transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t blob_size, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); + bool add_new_tx(transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t tx_weight, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); /** * @brief add a new transaction to the transaction pool @@ -864,9 +849,12 @@ namespace cryptonote * @return true if all the checks pass, otherwise false */ bool check_tx_semantic(const transaction& tx, bool keeped_by_block) const; + void set_semantics_failed(const crypto::hash &tx_hash); bool handle_incoming_tx_pre(const blobdata& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, crypto::hash &tx_prefixt_hash, bool keeped_by_block, bool relayed, bool do_not_relay); bool handle_incoming_tx_post(const blobdata& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, crypto::hash &tx_prefixt_hash, bool keeped_by_block, bool relayed, bool do_not_relay); + struct tx_verification_batch_info { const cryptonote::transaction *tx; crypto::hash tx_hash; tx_verification_context &tvc; bool &result; }; + bool handle_incoming_tx_accumulated_batch(std::vector<tx_verification_batch_info> &tx_info, bool keeped_by_block); /** * @copydoc miner::on_block_chain_update diff --git a/src/cryptonote_core/cryptonote_tx_utils.cpp b/src/cryptonote_core/cryptonote_tx_utils.cpp index 071ce591e..fb2af9ceb 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.cpp +++ b/src/cryptonote_core/cryptonote_tx_utils.cpp @@ -74,7 +74,7 @@ namespace cryptonote LOG_PRINT_L2("destinations include " << num_stdaddresses << " standard addresses and " << num_subaddresses << " subaddresses"); } //--------------------------------------------------------------- - bool construct_miner_tx(size_t height, size_t median_size, uint64_t already_generated_coins, size_t current_block_size, uint64_t fee, const account_public_address &miner_address, transaction& tx, const blobdata& extra_nonce, size_t max_outs, uint8_t hard_fork_version) { + bool construct_miner_tx(size_t height, size_t median_weight, uint64_t already_generated_coins, size_t current_block_weight, uint64_t fee, const account_public_address &miner_address, transaction& tx, const blobdata& extra_nonce, size_t max_outs, uint8_t hard_fork_version) { tx.vin.clear(); tx.vout.clear(); tx.extra.clear(); @@ -89,7 +89,7 @@ namespace cryptonote in.height = height; uint64_t block_reward; - if(!get_block_reward(median_size, current_block_size, already_generated_coins, block_reward, hard_fork_version)) + if(!get_block_reward(median_weight, current_block_weight, already_generated_coins, block_reward, hard_fork_version)) { LOG_PRINT_L0("Block is too big"); return false; @@ -195,7 +195,7 @@ namespace cryptonote return addr.m_view_public_key; } //--------------------------------------------------------------- - bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct, bool bulletproof, rct::multisig_out *msout, bool shuffle_outs) + bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct, rct::RangeProofType range_proof_type, rct::multisig_out *msout, bool shuffle_outs) { hw::device &hwdev = sender_account_keys.get_device(); @@ -491,7 +491,7 @@ namespace cryptonote // the non-simple version is slightly smaller, but assumes all real inputs // are on the same index, so can only be used if there just one ring. - bool use_simple_rct = sources.size() > 1; + bool use_simple_rct = sources.size() > 1 || range_proof_type != rct::RangeProofBorromean; if (!use_simple_rct) { @@ -516,6 +516,7 @@ namespace cryptonote uint64_t amount_in = 0, amount_out = 0; rct::ctkeyV inSk; + inSk.reserve(sources.size()); // mixRing indexing is done the other way round for simple rct::ctkeyM mixRing(use_simple_rct ? sources.size() : n_total_outs); rct::keyV destinations; @@ -532,6 +533,7 @@ namespace cryptonote ctkey.dest = rct::sk2rct(in_contexts[i].in_ephemeral.sec); ctkey.mask = sources[i].mask; inSk.push_back(ctkey); + memwipe(&ctkey, sizeof(rct::ctkey)); // inPk: (public key, commitment) // will be done when filling in mixRing if (msout) @@ -587,9 +589,10 @@ namespace cryptonote get_transaction_prefix_hash(tx, tx_prefix_hash); rct::ctkeyV outSk; if (use_simple_rct) - tx.rct_signatures = rct::genRctSimple(rct::hash2rct(tx_prefix_hash), inSk, destinations, inamounts, outamounts, amount_in - amount_out, mixRing, amount_keys, msout ? &kLRki : NULL, msout, index, outSk, bulletproof, hwdev); + tx.rct_signatures = rct::genRctSimple(rct::hash2rct(tx_prefix_hash), inSk, destinations, inamounts, outamounts, amount_in - amount_out, mixRing, amount_keys, msout ? &kLRki : NULL, msout, index, outSk, range_proof_type, hwdev); else - tx.rct_signatures = rct::genRct(rct::hash2rct(tx_prefix_hash), inSk, destinations, outamounts, mixRing, amount_keys, msout ? &kLRki[0] : NULL, msout, sources[0].real_output, outSk, bulletproof, hwdev); // same index assumption + tx.rct_signatures = rct::genRct(rct::hash2rct(tx_prefix_hash), inSk, destinations, outamounts, mixRing, amount_keys, msout ? &kLRki[0] : NULL, msout, sources[0].real_output, outSk, hwdev); // same index assumption + memwipe(inSk.data(), inSk.size() * sizeof(rct::ctkey)); CHECK_AND_ASSERT_MES(tx.vout.size() == outSk.size(), false, "outSk size does not match vout"); @@ -601,7 +604,7 @@ namespace cryptonote return true; } //--------------------------------------------------------------- - bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys, bool rct, bool bulletproof, rct::multisig_out *msout) + bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys, bool rct, rct::RangeProofType range_proof_type, rct::multisig_out *msout) { hw::device &hwdev = sender_account_keys.get_device(); hwdev.open_tx(tx_key); @@ -619,7 +622,7 @@ namespace cryptonote additional_tx_keys.push_back(keypair::generate(sender_account_keys.get_device()).sec); } - bool r = construct_tx_with_tx_key(sender_account_keys, subaddresses, sources, destinations, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, rct, bulletproof, msout); + bool r = construct_tx_with_tx_key(sender_account_keys, subaddresses, sources, destinations, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, rct, range_proof_type, msout); hwdev.close_tx(); return r; } @@ -631,7 +634,7 @@ namespace cryptonote crypto::secret_key tx_key; std::vector<crypto::secret_key> additional_tx_keys; std::vector<tx_destination_entry> destinations_copy = destinations; - return construct_tx_and_get_tx_key(sender_account_keys, subaddresses, sources, destinations_copy, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, false, NULL); + return construct_tx_and_get_tx_key(sender_account_keys, subaddresses, sources, destinations_copy, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, rct::RangeProofBorromean, NULL); } //--------------------------------------------------------------- bool generate_genesis_block( diff --git a/src/cryptonote_core/cryptonote_tx_utils.h b/src/cryptonote_core/cryptonote_tx_utils.h index a5d149fca..f2cf7b6ca 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.h +++ b/src/cryptonote_core/cryptonote_tx_utils.h @@ -37,7 +37,7 @@ namespace cryptonote { //--------------------------------------------------------------- - bool construct_miner_tx(size_t height, size_t median_size, uint64_t already_generated_coins, size_t current_block_size, uint64_t fee, const account_public_address &miner_address, transaction& tx, const blobdata& extra_nonce = blobdata(), size_t max_outs = 999, uint8_t hard_fork_version = 1); + bool construct_miner_tx(size_t height, size_t median_weight, uint64_t already_generated_coins, size_t current_block_weight, uint64_t fee, const account_public_address &miner_address, transaction& tx, const blobdata& extra_nonce = blobdata(), size_t max_outs = 999, uint8_t hard_fork_version = 1); struct tx_source_entry { @@ -90,8 +90,8 @@ namespace cryptonote //--------------------------------------------------------------- crypto::public_key get_destination_view_key_pub(const std::vector<tx_destination_entry> &destinations, const boost::optional<cryptonote::account_public_address>& change_addr); bool construct_tx(const account_keys& sender_account_keys, std::vector<tx_source_entry> &sources, const std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time); - bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, bool bulletproof = false, rct::multisig_out *msout = NULL, bool shuffle_outs = true); - bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, bool bulletproof = false, rct::multisig_out *msout = NULL); + bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, rct::RangeProofType range_proof_type = rct::RangeProofBorromean, rct::multisig_out *msout = NULL, bool shuffle_outs = true); + bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, rct::RangeProofType range_proof_type = rct::RangeProofBorromean, rct::multisig_out *msout = NULL); bool generate_genesis_block( block& bl diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index 5807867d9..a725eac6e 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -80,9 +80,13 @@ namespace cryptonote return amount * ACCEPT_THRESHOLD; } - uint64_t get_transaction_size_limit(uint8_t version) + uint64_t get_transaction_weight_limit(uint8_t version) { - return get_min_block_size(version) - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; + // from v8, limit a tx to 50% of the minimum block weight + if (version >= 8) + return get_min_block_weight(version) / 2 - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; + else + return get_min_block_weight(version) - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; } // This class is meant to create a batch when none currently exists. @@ -102,12 +106,12 @@ namespace cryptonote } //--------------------------------------------------------------------------------- //--------------------------------------------------------------------------------- - tx_memory_pool::tx_memory_pool(Blockchain& bchs): m_blockchain(bchs), m_txpool_max_size(DEFAULT_TXPOOL_MAX_SIZE), m_txpool_size(0), m_cookie(0) + tx_memory_pool::tx_memory_pool(Blockchain& bchs): m_blockchain(bchs), m_txpool_max_weight(DEFAULT_TXPOOL_MAX_WEIGHT), m_txpool_weight(0), m_cookie(0) { } //--------------------------------------------------------------------------------- - bool tx_memory_pool::add_tx(transaction &tx, /*const crypto::hash& tx_prefix_hash,*/ const crypto::hash &id, size_t blob_size, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version) + bool tx_memory_pool::add_tx(transaction &tx, /*const crypto::hash& tx_prefix_hash,*/ const crypto::hash &id, size_t tx_weight, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version) { // this should already be called with that lock, but let's make it explicit for clarity CRITICAL_REGION_LOCAL(m_transactions_lock); @@ -173,17 +177,17 @@ namespace cryptonote fee = tx.rct_signatures.txnFee; } - if (!kept_by_block && !m_blockchain.check_fee(blob_size, fee)) + if (!kept_by_block && !m_blockchain.check_fee(tx_weight, fee)) { tvc.m_verifivation_failed = true; tvc.m_fee_too_low = true; return false; } - size_t tx_size_limit = get_transaction_size_limit(version); - if (!kept_by_block && blob_size > tx_size_limit) + size_t tx_weight_limit = get_transaction_weight_limit(version); + if ((!kept_by_block || version >= HF_VERSION_PER_BYTE_FEE) && tx_weight > tx_weight_limit) { - LOG_PRINT_L1("transaction is too big: " << blob_size << " bytes, maximum size: " << tx_size_limit); + LOG_PRINT_L1("transaction is too heavy: " << tx_weight << " bytes, maximum weight: " << tx_weight_limit); tvc.m_verifivation_failed = true; tvc.m_too_big = true; return false; @@ -227,7 +231,7 @@ namespace cryptonote // may become valid again, so ignore the failed inputs check. if(kept_by_block) { - meta.blob_size = blob_size; + meta.weight = tx_weight; meta.fee = fee; meta.max_used_block_id = null_hash; meta.max_used_block_height = 0; @@ -248,7 +252,7 @@ namespace cryptonote m_blockchain.add_txpool_tx(tx, meta); if (!insert_key_images(tx, kept_by_block)) return false; - m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)blob_size, receive_time), id); + m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)tx_weight, receive_time), id); } catch (const std::exception &e) { @@ -267,7 +271,7 @@ namespace cryptonote }else { //update transactions container - meta.blob_size = blob_size; + meta.weight = tx_weight; meta.kept_by_block = kept_by_block; meta.fee = fee; meta.max_used_block_id = max_used_block_id; @@ -290,7 +294,7 @@ namespace cryptonote m_blockchain.add_txpool_tx(tx, meta); if (!insert_key_images(tx, kept_by_block)) return false; - m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)blob_size, receive_time), id); + m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)tx_weight, receive_time), id); } catch (const std::exception &e) { @@ -304,13 +308,13 @@ namespace cryptonote } tvc.m_verifivation_failed = false; - m_txpool_size += blob_size; + m_txpool_weight += tx_weight; ++m_cookie; - MINFO("Transaction added to pool: txid " << id << " bytes: " << blob_size << " fee/byte: " << (fee / (double)blob_size)); + MINFO("Transaction added to pool: txid " << id << " weight: " << tx_weight << " fee/byte: " << (fee / (double)tx_weight)); - prune(m_txpool_max_size); + prune(m_txpool_max_weight); return true; } @@ -321,26 +325,26 @@ namespace cryptonote size_t blob_size = 0; if (!get_transaction_hash(tx, h, blob_size) || blob_size == 0) return false; - return add_tx(tx, h, blob_size, tvc, keeped_by_block, relayed, do_not_relay, version); + return add_tx(tx, h, get_transaction_weight(tx, blob_size), tvc, keeped_by_block, relayed, do_not_relay, version); } //--------------------------------------------------------------------------------- - size_t tx_memory_pool::get_txpool_size() const + size_t tx_memory_pool::get_txpool_weight() const { CRITICAL_REGION_LOCAL(m_transactions_lock); - return m_txpool_size; + return m_txpool_weight; } //--------------------------------------------------------------------------------- - void tx_memory_pool::set_txpool_max_size(size_t bytes) + void tx_memory_pool::set_txpool_max_weight(size_t bytes) { CRITICAL_REGION_LOCAL(m_transactions_lock); - m_txpool_max_size = bytes; + m_txpool_max_weight = bytes; } //--------------------------------------------------------------------------------- void tx_memory_pool::prune(size_t bytes) { CRITICAL_REGION_LOCAL(m_transactions_lock); if (bytes == 0) - bytes = m_txpool_max_size; + bytes = m_txpool_max_weight; CRITICAL_REGION_LOCAL1(m_blockchain); LockedTXN lock(m_blockchain); bool changed = false; @@ -349,7 +353,7 @@ namespace cryptonote auto it = --m_txs_by_fee_and_receive_time.end(); while (it != m_txs_by_fee_and_receive_time.begin()) { - if (m_txpool_size <= bytes) + if (m_txpool_weight <= bytes) break; try { @@ -374,11 +378,11 @@ namespace cryptonote return; } // remove first, in case this throws, so key images aren't removed - MINFO("Pruning tx " << txid << " from txpool: size: " << it->first.second << ", fee/byte: " << it->first.first); + MINFO("Pruning tx " << txid << " from txpool: weight: " << it->first.second << ", fee/byte: " << it->first.first); m_blockchain.remove_txpool_tx(txid); - m_txpool_size -= txblob.size(); + m_txpool_weight -= it->first.second; remove_transaction_keyimages(tx); - MINFO("Pruned tx " << txid << " from txpool: size: " << it->first.second << ", fee/byte: " << it->first.first); + MINFO("Pruned tx " << txid << " from txpool: weight: " << it->first.second << ", fee/byte: " << it->first.first); m_txs_by_fee_and_receive_time.erase(it--); changed = true; } @@ -390,8 +394,8 @@ namespace cryptonote } if (changed) ++m_cookie; - if (m_txpool_size > bytes) - MINFO("Pool size after pruning is larger than limit: " << m_txpool_size << "/" << bytes); + if (m_txpool_weight > bytes) + MINFO("Pool weight after pruning is larger than limit: " << m_txpool_weight << "/" << bytes); } //--------------------------------------------------------------------------------- bool tx_memory_pool::insert_key_images(const transaction &tx, bool kept_by_block) @@ -446,7 +450,7 @@ namespace cryptonote return true; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::take_tx(const crypto::hash &id, transaction &tx, size_t& blob_size, uint64_t& fee, bool &relayed, bool &do_not_relay, bool &double_spend_seen) + bool tx_memory_pool::take_tx(const crypto::hash &id, transaction &tx, size_t& tx_weight, uint64_t& fee, bool &relayed, bool &do_not_relay, bool &double_spend_seen) { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); @@ -470,7 +474,7 @@ namespace cryptonote MERROR("Failed to parse tx from txpool"); return false; } - blob_size = meta.blob_size; + tx_weight = meta.weight; fee = meta.fee; relayed = meta.relayed; do_not_relay = meta.do_not_relay; @@ -478,7 +482,7 @@ namespace cryptonote // remove first, in case this throws, so key images aren't removed m_blockchain.remove_txpool_tx(id); - m_txpool_size -= blob_size; + m_txpool_weight -= tx_weight; remove_transaction_keyimages(tx); } catch (const std::exception &e) @@ -552,7 +556,7 @@ namespace cryptonote { // remove first, so we only remove key images if the tx removal succeeds m_blockchain.remove_txpool_tx(txid); - m_txpool_size -= bd.size(); + m_txpool_weight -= get_transaction_weight(tx, bd.size()); remove_transaction_keyimages(tx); } } @@ -670,7 +674,7 @@ namespace cryptonote const uint64_t now = time(NULL); backlog.reserve(m_blockchain.get_txpool_tx_count(include_unrelayed_txes)); m_blockchain.for_all_txpool_txes([&backlog, now](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ - backlog.push_back({meta.blob_size, meta.fee, meta.receive_time - now}); + backlog.push_back({meta.weight, meta.fee, meta.receive_time - now}); return true; }, false, include_unrelayed_txes); } @@ -682,15 +686,15 @@ namespace cryptonote const uint64_t now = time(NULL); std::map<uint64_t, txpool_histo> agebytes; stats.txs_total = m_blockchain.get_txpool_tx_count(include_unrelayed_txes); - std::vector<uint32_t> sizes; - sizes.reserve(stats.txs_total); - m_blockchain.for_all_txpool_txes([&stats, &sizes, now, &agebytes](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ - sizes.push_back(meta.blob_size); - stats.bytes_total += meta.blob_size; - if (!stats.bytes_min || meta.blob_size < stats.bytes_min) - stats.bytes_min = meta.blob_size; - if (meta.blob_size > stats.bytes_max) - stats.bytes_max = meta.blob_size; + std::vector<uint32_t> weights; + weights.reserve(stats.txs_total); + m_blockchain.for_all_txpool_txes([&stats, &weights, now, &agebytes](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ + weights.push_back(meta.weight); + stats.bytes_total += meta.weight; + if (!stats.bytes_min || meta.weight < stats.bytes_min) + stats.bytes_min = meta.weight; + if (meta.weight > stats.bytes_max) + stats.bytes_max = meta.weight; if (!meta.relayed) stats.num_not_relayed++; stats.fee_total += meta.fee; @@ -702,12 +706,12 @@ namespace cryptonote stats.num_failing++; uint64_t age = now - meta.receive_time + (now == meta.receive_time); agebytes[age].txs++; - agebytes[age].bytes += meta.blob_size; + agebytes[age].bytes += meta.weight; if (meta.double_spend_seen) ++stats.num_double_spends; return true; }, false, include_unrelayed_txes); - stats.bytes_med = epee::misc_utils::median(sizes); + stats.bytes_med = epee::misc_utils::median(weights); if (stats.txs_total > 1) { /* looking for 98th percentile */ @@ -771,7 +775,8 @@ namespace cryptonote return true; } txi.tx_json = obj_to_json_str(tx); - txi.blob_size = meta.blob_size; + txi.blob_size = bd->size(); + txi.weight = meta.weight; txi.fee = meta.fee; txi.kept_by_block = meta.kept_by_block; txi.max_used_block_height = meta.max_used_block_height; @@ -842,7 +847,8 @@ namespace cryptonote return true; } txi.tx = tx; - txi.blob_size = meta.blob_size; + txi.blob_size = bd->size(); + txi.weight = meta.weight; txi.fee = meta.fee; txi.kept_by_block = meta.kept_by_block; txi.max_used_block_height = meta.max_used_block_height; @@ -1116,7 +1122,8 @@ namespace cryptonote } ss << obj_to_json_str(tx) << std::endl; } - ss << "blob_size: " << meta.blob_size << std::endl + ss << "blob_size: " << (short_format ? "-" : std::to_string(txblob->size())) << std::endl + << "weight: " << meta.weight << std::endl << "fee: " << print_money(meta.fee) << std::endl << "kept_by_block: " << (meta.kept_by_block ? 'T' : 'F') << std::endl << "double_spend_seen: " << (meta.double_spend_seen ? 'T' : 'F') << std::endl @@ -1131,30 +1138,30 @@ namespace cryptonote } //--------------------------------------------------------------------------------- //TODO: investigate whether boolean return is appropriate - bool tx_memory_pool::fill_block_template(block &bl, size_t median_size, uint64_t already_generated_coins, size_t &total_size, uint64_t &fee, uint64_t &expected_reward, uint8_t version) + bool tx_memory_pool::fill_block_template(block &bl, size_t median_weight, uint64_t already_generated_coins, size_t &total_weight, uint64_t &fee, uint64_t &expected_reward, uint8_t version) { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); uint64_t best_coinbase = 0, coinbase = 0; - total_size = 0; + total_weight = 0; fee = 0; //baseline empty block - get_block_reward(median_size, total_size, already_generated_coins, best_coinbase, version); + get_block_reward(median_weight, total_weight, already_generated_coins, best_coinbase, version); - size_t max_total_size_pre_v5 = (130 * median_size) / 100 - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; - size_t max_total_size_v5 = 2 * median_size - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; - size_t max_total_size = version >= 5 ? max_total_size_v5 : max_total_size_pre_v5; + size_t max_total_weight_pre_v5 = (130 * median_weight) / 100 - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; + size_t max_total_weight_v5 = 2 * median_weight - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; + size_t max_total_weight = version >= 5 ? max_total_weight_v5 : max_total_weight_pre_v5; std::unordered_set<crypto::key_image> k_images; - LOG_PRINT_L2("Filling block template, median size " << median_size << ", " << m_txs_by_fee_and_receive_time.size() << " txes in the pool"); + LOG_PRINT_L2("Filling block template, median weight " << median_weight << ", " << m_txs_by_fee_and_receive_time.size() << " txes in the pool"); LockedTXN lock(m_blockchain); auto sorted_it = m_txs_by_fee_and_receive_time.begin(); - while (sorted_it != m_txs_by_fee_and_receive_time.end()) + for (; sorted_it != m_txs_by_fee_and_receive_time.end(); ++sorted_it) { txpool_tx_meta_t meta; if (!m_blockchain.get_txpool_tx_meta(sorted_it->second, meta)) @@ -1162,13 +1169,12 @@ namespace cryptonote MERROR(" failed to find tx meta"); continue; } - LOG_PRINT_L2("Considering " << sorted_it->second << ", size " << meta.blob_size << ", current block size " << total_size << "/" << max_total_size << ", current coinbase " << print_money(best_coinbase)); + LOG_PRINT_L2("Considering " << sorted_it->second << ", weight " << meta.weight << ", current block weight " << total_weight << "/" << max_total_weight << ", current coinbase " << print_money(best_coinbase)); - // Can not exceed maximum block size - if (max_total_size < total_size + meta.blob_size) + // Can not exceed maximum block weight + if (max_total_weight < total_weight + meta.weight) { - LOG_PRINT_L2(" would exceed maximum block size"); - sorted_it++; + LOG_PRINT_L2(" would exceed maximum block weight"); continue; } @@ -1178,27 +1184,25 @@ namespace cryptonote // If we're getting lower coinbase tx, // stop including more tx uint64_t block_reward; - if(!get_block_reward(median_size, total_size + meta.blob_size, already_generated_coins, block_reward, version)) + if(!get_block_reward(median_weight, total_weight + meta.weight, already_generated_coins, block_reward, version)) { - LOG_PRINT_L2(" would exceed maximum block size"); - sorted_it++; + LOG_PRINT_L2(" would exceed maximum block weight"); continue; } coinbase = block_reward + fee + meta.fee; if (coinbase < template_accept_threshold(best_coinbase)) { LOG_PRINT_L2(" would decrease coinbase to " << print_money(coinbase)); - sorted_it++; continue; } } else { - // If we've exceeded the penalty free size, + // If we've exceeded the penalty free weight, // stop including more tx - if (total_size > median_size) + if (total_weight > median_weight) { - LOG_PRINT_L2(" would exceed median block size"); + LOG_PRINT_L2(" would exceed median block weight"); break; } } @@ -1235,28 +1239,25 @@ namespace cryptonote if (!ready) { LOG_PRINT_L2(" not ready to go"); - sorted_it++; continue; } if (have_key_images(k_images, tx)) { LOG_PRINT_L2(" key images already seen"); - sorted_it++; continue; } bl.tx_hashes.push_back(sorted_it->second); - total_size += meta.blob_size; + total_weight += meta.weight; fee += meta.fee; best_coinbase = coinbase; append_key_images(k_images, tx); - sorted_it++; - LOG_PRINT_L2(" added, new block size " << total_size << "/" << max_total_size << ", coinbase " << print_money(best_coinbase)); + LOG_PRINT_L2(" added, new block weight " << total_weight << "/" << max_total_weight << ", coinbase " << print_money(best_coinbase)); } expected_reward = best_coinbase; - LOG_PRINT_L2("Block template filled with " << bl.tx_hashes.size() << " txes, size " - << total_size << "/" << max_total_size << ", coinbase " << print_money(best_coinbase) + LOG_PRINT_L2("Block template filled with " << bl.tx_hashes.size() << " txes, weight " + << total_weight << "/" << max_total_weight << ", coinbase " << print_money(best_coinbase) << " (including " << print_money(fee) << " in fees)"); return true; } @@ -1265,14 +1266,14 @@ namespace cryptonote { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); - size_t tx_size_limit = get_transaction_size_limit(version); + size_t tx_weight_limit = get_transaction_weight_limit(version); std::unordered_set<crypto::hash> remove; - m_txpool_size = 0; - m_blockchain.for_all_txpool_txes([this, &remove, tx_size_limit](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata*) { - m_txpool_size += meta.blob_size; - if (meta.blob_size > tx_size_limit) { - LOG_PRINT_L1("Transaction " << txid << " is too big (" << meta.blob_size << " bytes), removing it from pool"); + m_txpool_weight = 0; + m_blockchain.for_all_txpool_txes([this, &remove, tx_weight_limit](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata*) { + m_txpool_weight += meta.weight; + if (meta.weight > tx_weight_limit) { + LOG_PRINT_L1("Transaction " << txid << " is too big (" << meta.weight << " bytes), removing it from pool"); remove.insert(txid); } else if (m_blockchain.have_tx(txid)) { @@ -1299,7 +1300,7 @@ namespace cryptonote } // remove tx from db first m_blockchain.remove_txpool_tx(txid); - m_txpool_size -= txblob.size(); + m_txpool_weight -= get_transaction_weight(tx, txblob.size()); remove_transaction_keyimages(tx); auto sorted_it = find_tx_in_sorted_container(txid); if (sorted_it == m_txs_by_fee_and_receive_time.end()) @@ -1324,15 +1325,15 @@ namespace cryptonote return n_removed; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::init(size_t max_txpool_size) + bool tx_memory_pool::init(size_t max_txpool_weight) { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); - m_txpool_max_size = max_txpool_size ? max_txpool_size : DEFAULT_TXPOOL_MAX_SIZE; + m_txpool_max_weight = max_txpool_weight ? max_txpool_weight : DEFAULT_TXPOOL_MAX_WEIGHT; m_txs_by_fee_and_receive_time.clear(); m_spent_key_images.clear(); - m_txpool_size = 0; + m_txpool_weight = 0; std::vector<crypto::hash> remove; // first add the not kept by block, then the kept by block, @@ -1354,8 +1355,8 @@ namespace cryptonote MFATAL("Failed to insert key images from txpool tx"); return false; } - m_txs_by_fee_and_receive_time.emplace(std::pair<double, time_t>(meta.fee / (double)meta.blob_size, meta.receive_time), txid); - m_txpool_size += meta.blob_size; + m_txs_by_fee_and_receive_time.emplace(std::pair<double, time_t>(meta.fee / (double)meta.weight, meta.receive_time), txid); + m_txpool_weight += meta.weight; return true; }, true); if (!r) diff --git a/src/cryptonote_core/tx_pool.h b/src/cryptonote_core/tx_pool.h index 4abfef85c..892cadc69 100644 --- a/src/cryptonote_core/tx_pool.h +++ b/src/cryptonote_core/tx_pool.h @@ -84,7 +84,7 @@ namespace cryptonote * * This handling includes: * storing the transactions - * organizing the transactions by fee per size + * organizing the transactions by fee per weight unit * taking/giving transactions to and from various other components * saving the transactions to disk on shutdown * helping create a new block template by choosing transactions for it @@ -105,9 +105,9 @@ namespace cryptonote * @copydoc add_tx(transaction&, tx_verification_context&, bool, bool, uint8_t) * * @param id the transaction's hash - * @param blob_size the transaction's size + * @param tx_weight the transaction's weight */ - bool add_tx(transaction &tx, const crypto::hash &id, size_t blob_size, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version); + bool add_tx(transaction &tx, const crypto::hash &id, size_t tx_weight, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version); /** * @brief add a transaction to the transaction pool @@ -133,7 +133,7 @@ namespace cryptonote * * @param id the hash of the transaction * @param tx return-by-reference the transaction taken - * @param blob_size return-by-reference the transaction's size + * @param tx_weight return-by-reference the transaction's weight * @param fee the transaction fee * @param relayed return-by-reference was transaction relayed to us by the network? * @param do_not_relay return-by-reference is transaction not to be relayed to the network? @@ -141,7 +141,7 @@ namespace cryptonote * * @return true unless the transaction cannot be found in the pool */ - bool take_tx(const crypto::hash &id, transaction &tx, size_t& blob_size, uint64_t& fee, bool &relayed, bool &do_not_relay, bool &double_spend_seen); + bool take_tx(const crypto::hash &id, transaction &tx, size_t& tx_weight, uint64_t& fee, bool &relayed, bool &do_not_relay, bool &double_spend_seen); /** * @brief checks if the pool has a transaction with the given hash @@ -198,11 +198,11 @@ namespace cryptonote /** * @brief loads pool state (if any) from disk, and initializes pool * - * @param max_txpool_size the max size in bytes + * @param max_txpool_weight the max weight in bytes * * @return true */ - bool init(size_t max_txpool_size = 0); + bool init(size_t max_txpool_weight = 0); /** * @brief attempts to save the transaction pool state to disk @@ -219,16 +219,16 @@ namespace cryptonote * @brief Chooses transactions for a block to include * * @param bl return-by-reference the block to fill in with transactions - * @param median_size the current median block size + * @param median_weight the current median block weight * @param already_generated_coins the current total number of coins "minted" - * @param total_size return-by-reference the total size of the new block + * @param total_weight return-by-reference the total weight of the new block * @param fee return-by-reference the total of fees from the included transactions * @param expected_reward return-by-reference the total reward awarded to the miner finding this block, including transaction fees * @param version hard fork version to use for consensus rules * * @return true */ - bool fill_block_template(block &bl, size_t median_size, uint64_t already_generated_coins, size_t &total_size, uint64_t &fee, uint64_t &expected_reward, uint8_t version); + bool fill_block_template(block &bl, size_t median_weight, uint64_t already_generated_coins, size_t &total_weight, uint64_t &fee, uint64_t &expected_reward, uint8_t version); /** * @brief get a list of all transactions in the pool @@ -249,7 +249,7 @@ namespace cryptonote void get_transaction_hashes(std::vector<crypto::hash>& txs, bool include_unrelayed_txes = true) const; /** - * @brief get (size, fee, receive time) for all transaction in the pool + * @brief get (weight, fee, receive time) for all transaction in the pool * * @param txs return-by-reference that data * @param include_unrelayed_txes include unrelayed txes in the result @@ -370,21 +370,21 @@ namespace cryptonote uint64_t cookie() const { return m_cookie; } /** - * @brief get the cumulative txpool size in bytes + * @brief get the cumulative txpool weight in bytes * - * @return the cumulative txpool size in bytes + * @return the cumulative txpool weight in bytes */ - size_t get_txpool_size() const; + size_t get_txpool_weight() const; /** - * @brief set the max cumulative txpool size in bytes + * @brief set the max cumulative txpool weight in bytes * - * @param bytes the max cumulative txpool size in bytes + * @param bytes the max cumulative txpool weight in bytes */ - void set_txpool_max_size(size_t bytes); + void set_txpool_max_weight(size_t bytes); #define CURRENT_MEMPOOL_ARCHIVE_VER 11 -#define CURRENT_MEMPOOL_TX_DETAILS_ARCHIVE_VER 12 +#define CURRENT_MEMPOOL_TX_DETAILS_ARCHIVE_VER 13 /** * @brief information about a single transaction @@ -393,6 +393,7 @@ namespace cryptonote { transaction tx; //!< the transaction size_t blob_size; //!< the transaction's size + size_t weight; //!< the transaction's weight uint64_t fee; //!< the transaction's fee amount crypto::hash max_used_block_id; //!< the hash of the highest block referenced by an input uint64_t max_used_block_height; //!< the height of the highest block referenced by an input @@ -522,7 +523,7 @@ namespace cryptonote /** * @brief prune lowest fee/byte txes till we're not above bytes * - * if bytes is 0, use m_txpool_max_size + * if bytes is 0, use m_txpool_max_weight */ void prune(size_t bytes = 0); @@ -578,8 +579,8 @@ private: Blockchain& m_blockchain; //!< reference to the Blockchain object - size_t m_txpool_max_size; - size_t m_txpool_size; + size_t m_txpool_max_weight; + size_t m_txpool_weight; mutable std::unordered_map<crypto::hash, std::tuple<bool, tx_verification_context, uint64_t, crypto::hash>> m_input_cache; }; @@ -608,6 +609,9 @@ namespace boost if (version < 12) return; ar & td.do_not_relay; + if (version < 13) + return; + ar & td.weight; } } } diff --git a/src/cryptonote_protocol/block_queue.cpp b/src/cryptonote_protocol/block_queue.cpp index c39d67ceb..05f4189fb 100644 --- a/src/cryptonote_protocol/block_queue.cpp +++ b/src/cryptonote_protocol/block_queue.cpp @@ -57,7 +57,11 @@ void block_queue::add_blocks(uint64_t height, std::vector<cryptonote::block_comp bool has_hashes = remove_span(height, &hashes); blocks.insert(span(height, std::move(bcel), connection_id, rate, size)); if (has_hashes) + { + for (const crypto::hash &h: hashes) + requested_hashes.insert(h); set_span_hashes(height, connection_id, hashes); + } } void block_queue::add_blocks(uint64_t height, uint64_t nblocks, const boost::uuids::uuid &connection_id, boost::posix_time::ptime time) @@ -76,11 +80,19 @@ void block_queue::flush_spans(const boost::uuids::uuid &connection_id, bool all) block_map::iterator j = i++; if (j->connection_id == connection_id && (all || j->blocks.size() == 0)) { - blocks.erase(j); + erase_block(j); } } } +void block_queue::erase_block(block_map::iterator j) +{ + CHECK_AND_ASSERT_THROW_MES(j != blocks.end(), "Invalid iterator"); + for (const crypto::hash &h: j->hashes) + requested_hashes.erase(h); + blocks.erase(j); +} + void block_queue::flush_stale_spans(const std::set<boost::uuids::uuid> &live_connections) { boost::unique_lock<boost::recursive_mutex> lock(mutex); @@ -92,7 +104,7 @@ void block_queue::flush_stale_spans(const std::set<boost::uuids::uuid> &live_con block_map::iterator j = i++; if (live_connections.find(j->connection_id) == live_connections.end() && j->blocks.size() == 0) { - blocks.erase(j); + erase_block(j); } } } @@ -106,7 +118,7 @@ bool block_queue::remove_span(uint64_t start_block_height, std::vector<crypto::h { if (hashes) *hashes = std::move(i->hashes); - blocks.erase(i); + erase_block(i); return true; } } @@ -121,7 +133,7 @@ void block_queue::remove_spans(const boost::uuids::uuid &connection_id, uint64_t block_map::iterator j = i++; if (j->connection_id == connection_id && j->start_block_height <= start_block_height) { - blocks.erase(j); + erase_block(j); } } } @@ -160,16 +172,15 @@ std::string block_queue::get_overview() const return s; } +inline bool block_queue::requested_internal(const crypto::hash &hash) const +{ + return requested_hashes.find(hash) != requested_hashes.end(); +} + bool block_queue::requested(const crypto::hash &hash) const { boost::unique_lock<boost::recursive_mutex> lock(mutex); - for (const auto &span: blocks) - { - for (const auto &h: span.hashes) - if (h == hash) - return true; - } - return false; + return requested_internal(hash); } std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, const std::vector<crypto::hash> &block_hashes, boost::posix_time::ptime time) @@ -184,7 +195,7 @@ std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_hei uint64_t span_start_height = last_block_height - block_hashes.size() + 1; std::vector<crypto::hash>::const_iterator i = block_hashes.begin(); - while (i != block_hashes.end() && requested(*i)) + while (i != block_hashes.end() && requested_internal(*i)) { ++i; ++span_start_height; @@ -256,8 +267,10 @@ void block_queue::set_span_hashes(uint64_t start_height, const boost::uuids::uui if (i->start_block_height == start_height && i->connection_id == connection_id) { span s = *i; - blocks.erase(i); + erase_block(i); s.hashes = std::move(hashes); + for (const crypto::hash &h: s.hashes) + requested_hashes.insert(h); blocks.insert(s); return; } diff --git a/src/cryptonote_protocol/block_queue.h b/src/cryptonote_protocol/block_queue.h index 9059e89ac..9cce95075 100644 --- a/src/cryptonote_protocol/block_queue.h +++ b/src/cryptonote_protocol/block_queue.h @@ -33,6 +33,7 @@ #include <string> #include <vector> #include <set> +#include <unordered_set> #include <boost/thread/recursive_mutex.hpp> #include <boost/uuid/uuid.hpp> @@ -93,7 +94,12 @@ namespace cryptonote bool requested(const crypto::hash &hash) const; private: + void erase_block(block_map::iterator j); + inline bool requested_internal(const crypto::hash &hash) const; + + private: block_map blocks; mutable boost::recursive_mutex mutex; + std::unordered_set<crypto::hash> requested_hashes; }; } diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp index 45ba81e16..9ab1be246 100644 --- a/src/daemon/rpc_command_executor.cpp +++ b/src/daemon/rpc_command_executor.cpp @@ -76,6 +76,7 @@ namespace { << "difficulty: " << boost::lexical_cast<std::string>(header.difficulty) << std::endl << "POW hash: " << header.pow_hash << std::endl << "block size: " << header.block_size << std::endl + << "block weight: " << header.block_weight << std::endl << "num txes: " << header.num_txes << std::endl << "reward: " << cryptonote::print_money(header.reward); } @@ -558,7 +559,7 @@ bool t_rpc_command_executor::print_blockchain_info(uint64_t start_block_index, u std::cout << std::endl; std::cout << "height: " << header.height << ", timestamp: " << header.timestamp - << ", size: " << header.block_size << ", transactions: " << header.num_txes << std::endl + << ", size: " << header.block_size << ", weight: " << header.block_weight << ", transactions: " << header.num_txes << std::endl << "major version: " << (unsigned)header.major_version << ", minor version: " << (unsigned)header.minor_version << std::endl << "block id: " << header.hash << ", previous block id: " << header.prev_hash << std::endl << "difficulty: " << header.difficulty << ", nonce " << header.nonce << ", reward " << cryptonote::print_money(header.reward) << std::endl; @@ -857,8 +858,9 @@ bool t_rpc_command_executor::print_transaction_pool_long() { tools::msg_writer() << "id: " << tx_info.id_hash << std::endl << tx_info.tx_json << std::endl << "blob_size: " << tx_info.blob_size << std::endl + << "weight: " << tx_info.weight << std::endl << "fee: " << cryptonote::print_money(tx_info.fee) << std::endl - << "fee/byte: " << cryptonote::print_money(tx_info.fee / (double)tx_info.blob_size) << std::endl + << "fee/byte: " << cryptonote::print_money(tx_info.fee / (double)tx_info.weight) << std::endl << "receive_time: " << tx_info.receive_time << " (" << get_human_time_ago(tx_info.receive_time, now) << ")" << std::endl << "relayed: " << [&](const cryptonote::tx_info &tx_info)->std::string { if (!tx_info.relayed) return "no"; return boost::lexical_cast<std::string>(tx_info.last_relayed_time) + " (" + get_human_time_ago(tx_info.last_relayed_time, now) + ")"; } (tx_info) << std::endl << "do_not_relay: " << (tx_info.do_not_relay ? 'T' : 'F') << std::endl @@ -940,8 +942,9 @@ bool t_rpc_command_executor::print_transaction_pool_short() { { tools::msg_writer() << "id: " << tx_info.id_hash << std::endl << "blob_size: " << tx_info.blob_size << std::endl + << "weight: " << tx_info.weight << std::endl << "fee: " << cryptonote::print_money(tx_info.fee) << std::endl - << "fee/byte: " << cryptonote::print_money(tx_info.fee / (double)tx_info.blob_size) << std::endl + << "fee/byte: " << cryptonote::print_money(tx_info.fee / (double)tx_info.weight) << std::endl << "receive_time: " << tx_info.receive_time << " (" << get_human_time_ago(tx_info.receive_time, now) << ")" << std::endl << "relayed: " << [&](const cryptonote::tx_info &tx_info)->std::string { if (!tx_info.relayed) return "no"; return boost::lexical_cast<std::string>(tx_info.last_relayed_time) + " (" + get_human_time_ago(tx_info.last_relayed_time, now) + ")"; } (tx_info) << std::endl << "do_not_relay: " << (tx_info.do_not_relay ? 'T' : 'F') << std::endl @@ -996,7 +999,7 @@ bool t_rpc_command_executor::print_transaction_pool_stats() { size_t avg_bytes = n_transactions ? res.pool_stats.bytes_total / n_transactions : 0; std::string backlog_message; - const uint64_t full_reward_zone = ires.block_size_limit / 2; + const uint64_t full_reward_zone = ires.block_weight_limit / 2; if (res.pool_stats.bytes_total <= full_reward_zone) { backlog_message = "no backlog"; @@ -1701,8 +1704,8 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks) cryptonote::COMMAND_RPC_GET_INFO::response ires; cryptonote::COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::request bhreq; cryptonote::COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::response bhres; - cryptonote::COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::request fereq; - cryptonote::COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::response feres; + cryptonote::COMMAND_RPC_GET_BASE_FEE_ESTIMATE::request fereq; + cryptonote::COMMAND_RPC_GET_BASE_FEE_ESTIMATE::response feres; epee::json_rpc::error error_resp; std::string fail_message = "Problem fetching info"; @@ -1726,7 +1729,7 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks) tools::fail_msg_writer() << make_error(fail_message, ires.status); return true; } - if (!m_rpc_server->on_get_per_kb_fee_estimate(fereq, feres, error_resp) || feres.status != CORE_RPC_STATUS_OK) + if (!m_rpc_server->on_get_base_fee_estimate(fereq, feres, error_resp) || feres.status != CORE_RPC_STATUS_OK) { tools::fail_msg_writer() << make_error(fail_message, feres.status); return true; @@ -1762,8 +1765,8 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks) double avgdiff = 0; double avgnumtxes = 0; double avgreward = 0; - std::vector<uint64_t> sizes; - sizes.reserve(nblocks); + std::vector<uint64_t> weights; + weights.reserve(nblocks); uint64_t earliest = std::numeric_limits<uint64_t>::max(), latest = 0; std::vector<unsigned> major_versions(256, 0), minor_versions(256, 0); for (const auto &bhr: bhres.headers) @@ -1771,7 +1774,7 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks) avgdiff += bhr.difficulty; avgnumtxes += bhr.num_txes; avgreward += bhr.reward; - sizes.push_back(bhr.block_size); + weights.push_back(bhr.block_weight); static_assert(sizeof(bhr.major_version) == 1, "major_version expected to be uint8_t"); static_assert(sizeof(bhr.minor_version) == 1, "major_version expected to be uint8_t"); major_versions[(unsigned)bhr.major_version]++; @@ -1782,9 +1785,9 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks) avgdiff /= nblocks; avgnumtxes /= nblocks; avgreward /= nblocks; - uint64_t median_block_size = epee::misc_utils::median(sizes); + uint64_t median_block_weight = epee::misc_utils::median(weights); tools::msg_writer() << "Last " << nblocks << ": avg. diff " << (uint64_t)avgdiff << ", " << (latest - earliest) / nblocks << " avg sec/block, avg num txes " << avgnumtxes - << ", avg. reward " << cryptonote::print_money(avgreward) << ", median block size " << median_block_size; + << ", avg. reward " << cryptonote::print_money(avgreward) << ", median block weight " << median_block_weight; unsigned int max_major = 256, max_minor = 256; while (max_major > 0 && !major_versions[--max_major]); diff --git a/src/debug_utilities/cn_deserialize.cpp b/src/debug_utilities/cn_deserialize.cpp index 6c09b0f18..3e2552230 100644 --- a/src/debug_utilities/cn_deserialize.cpp +++ b/src/debug_utilities/cn_deserialize.cpp @@ -43,6 +43,15 @@ using namespace epee; using namespace cryptonote; +static std::string extra_nonce_to_string(const cryptonote::tx_extra_nonce &extra_nonce) +{ + if (extra_nonce.nonce.size() == 9 && extra_nonce.nonce[0] == TX_EXTRA_NONCE_ENCRYPTED_PAYMENT_ID) + return "encrypted payment ID: " + epee::string_tools::buff_to_hex_nodelimer(extra_nonce.nonce.substr(1)); + if (extra_nonce.nonce.size() == 33 && extra_nonce.nonce[0] == TX_EXTRA_NONCE_PAYMENT_ID) + return "plaintext payment ID: " + epee::string_tools::buff_to_hex_nodelimer(extra_nonce.nonce.substr(1)); + return epee::string_tools::buff_to_hex_nodelimer(extra_nonce.nonce); +} + static void print_extra_fields(const std::vector<cryptonote::tx_extra_field> &fields) { std::cout << "tx_extra has " << fields.size() << " field(s)" << std::endl; @@ -51,7 +60,7 @@ static void print_extra_fields(const std::vector<cryptonote::tx_extra_field> &fi std::cout << "field " << n << ": "; if (typeid(cryptonote::tx_extra_padding) == fields[n].type()) std::cout << "extra padding: " << boost::get<cryptonote::tx_extra_padding>(fields[n]).size << " bytes"; else if (typeid(cryptonote::tx_extra_pub_key) == fields[n].type()) std::cout << "extra pub key: " << boost::get<cryptonote::tx_extra_pub_key>(fields[n]).pub_key; - else if (typeid(cryptonote::tx_extra_nonce) == fields[n].type()) std::cout << "extra nonce: " << epee::string_tools::buff_to_hex_nodelimer(boost::get<cryptonote::tx_extra_nonce>(fields[n]).nonce); + else if (typeid(cryptonote::tx_extra_nonce) == fields[n].type()) std::cout << "extra nonce: " << extra_nonce_to_string(boost::get<cryptonote::tx_extra_nonce>(fields[n])); else if (typeid(cryptonote::tx_extra_merge_mining_tag) == fields[n].type()) std::cout << "extra merge mining tag: depth " << boost::get<cryptonote::tx_extra_merge_mining_tag>(fields[n]).depth << ", merkle root " << boost::get<cryptonote::tx_extra_merge_mining_tag>(fields[n]).merkle_root; else if (typeid(cryptonote::tx_extra_additional_pub_keys) == fields[n].type()) std::cout << "additional tx pubkeys: " << boost::join(boost::get<cryptonote::tx_extra_additional_pub_keys>(fields[n]).data | boost::adaptors::transformed([](const crypto::public_key &key){ return epee::string_tools::pod_to_hex(key); }), ", " ); else if (typeid(cryptonote::tx_extra_mysterious_minergate) == fields[n].type()) std::cout << "extra minergate custom: " << epee::string_tools::buff_to_hex_nodelimer(boost::get<cryptonote::tx_extra_mysterious_minergate>(fields[n]).data); @@ -168,9 +177,14 @@ int main(int argc, char* argv[]) std::cout << "Parsed block:" << std::endl; std::cout << cryptonote::obj_to_json_str(block) << std::endl; } - else if (cryptonote::parse_and_validate_tx_from_blob(blob, tx)) + else if (cryptonote::parse_and_validate_tx_from_blob(blob, tx) || cryptonote::parse_and_validate_tx_base_from_blob(blob, tx)) { - std::cout << "Parsed transaction:" << std::endl; +/* + if (tx.pruned) + std::cout << "Parsed pruned transaction:" << std::endl; + else +*/ + std::cout << "Parsed transaction:" << std::endl; std::cout << cryptonote::obj_to_json_str(tx) << std::endl; bool parsed = cryptonote::parse_tx_extra(tx.extra, fields); diff --git a/src/device/device_ledger.cpp b/src/device/device_ledger.cpp index c4e9e40b7..9b5ea0fd7 100644 --- a/src/device/device_ledger.cpp +++ b/src/device/device_ledger.cpp @@ -1354,7 +1354,7 @@ namespace hw { this->exchange(); //pseudoOuts - if ((type == rct::RCTTypeSimple) || (type == rct::RCTTypeSimpleBulletproof)) { + if (type == rct::RCTTypeSimple) { for ( i = 0; i < inputs_size; i++) { offset = set_command_header(INS_VALIDATE, 0x01, i+2); //options diff --git a/src/multisig/multisig.cpp b/src/multisig/multisig.cpp index d85c47772..a0a788b7d 100644 --- a/src/multisig/multisig.cpp +++ b/src/multisig/multisig.cpp @@ -47,9 +47,12 @@ namespace cryptonote crypto::secret_key get_multisig_blinded_secret_key(const crypto::secret_key &key) { rct::keyV data; + data.reserve(2); data.push_back(rct::sk2rct(key)); data.push_back(multisig_salt); - return rct::rct2sk(rct::hash_to_scalar(data)); + crypto::secret_key result = rct::rct2sk(rct::hash_to_scalar(data)); + memwipe(&data[0], sizeof(rct::key)); + return result; } //----------------------------------------------------------------- void generate_multisig_N_N(const account_keys &keys, const std::vector<crypto::public_key> &spend_keys, std::vector<crypto::secret_key> &multisig_keys, rct::key &spend_skey, rct::key &spend_pkey) diff --git a/src/ringct/CMakeLists.txt b/src/ringct/CMakeLists.txt index c8dcdca26..29f166a3b 100644 --- a/src/ringct/CMakeLists.txt +++ b/src/ringct/CMakeLists.txt @@ -30,11 +30,13 @@ set(ringct_basic_sources rctOps.cpp rctTypes.cpp rctCryptoOps.c + multiexp.cc bulletproofs.cc) set(ringct_basic_private_headers rctOps.h rctTypes.h + multiexp.h bulletproofs.h) monero_private_headers(ringct_basic diff --git a/src/ringct/bulletproofs.cc b/src/ringct/bulletproofs.cc index fd15ffbc4..abe4ef18d 100644 --- a/src/ringct/bulletproofs.cc +++ b/src/ringct/bulletproofs.cc @@ -30,14 +30,17 @@ #include <stdlib.h> #include <openssl/ssl.h> +#include <openssl/bn.h> #include <boost/thread/mutex.hpp> #include "misc_log_ex.h" #include "common/perf_timer.h" +#include "cryptonote_config.h" extern "C" { #include "crypto/crypto-ops.h" } #include "rctOps.h" +#include "multiexp.h" #include "bulletproofs.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -47,27 +50,99 @@ extern "C" #define PERF_TIMER_START_BP(x) PERF_TIMER_START_UNIT(x, 1000000) +#define STRAUS_SIZE_LIMIT 128 +#define PIPPENGER_SIZE_LIMIT 0 + namespace rct { static rct::key vector_exponent(const rct::keyV &a, const rct::keyV &b); -static rct::keyV vector_powers(rct::key x, size_t n); +static rct::keyV vector_powers(const rct::key &x, size_t n); +static rct::keyV vector_dup(const rct::key &x, size_t n); static rct::key inner_product(const rct::keyV &a, const rct::keyV &b); static constexpr size_t maxN = 64; -static rct::key Hi[maxN], Gi[maxN]; -static ge_dsmp Gprecomp[64], Hprecomp[64]; +static constexpr size_t maxM = BULLETPROOF_MAX_OUTPUTS; +static rct::key Hi[maxN*maxM], Gi[maxN*maxM]; +static ge_p3 Hi_p3[maxN*maxM], Gi_p3[maxN*maxM]; +static std::shared_ptr<straus_cached_data> straus_HiGi_cache; +static std::shared_ptr<pippenger_cached_data> pippenger_HiGi_cache; static const rct::key TWO = { {0x02, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 } }; -static const rct::keyV oneN = vector_powers(rct::identity(), maxN); +static const rct::key MINUS_ONE = { { 0xec, 0xd3, 0xf5, 0x5c, 0x1a, 0x63, 0x12, 0x58, 0xd6, 0x9c, 0xf7, 0xa2, 0xde, 0xf9, 0xde, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10 } }; +static const rct::key MINUS_INV_EIGHT = { { 0x74, 0xa4, 0x19, 0x7a, 0xf0, 0x7d, 0x0b, 0xf7, 0x05, 0xc2, 0xda, 0x25, 0x2b, 0x5c, 0x0b, 0x0d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a } }; +static const rct::keyV oneN = vector_dup(rct::identity(), maxN); static const rct::keyV twoN = vector_powers(TWO, maxN); static const rct::key ip12 = inner_product(oneN, twoN); static boost::mutex init_mutex; +static inline rct::key multiexp(const std::vector<MultiexpData> &data, bool HiGi) +{ + if (HiGi) + { + static_assert(128 <= STRAUS_SIZE_LIMIT, "Straus in precalc mode can only be calculated till STRAUS_SIZE_LIMIT"); + return data.size() <= 128 ? straus(data, straus_HiGi_cache, 0) : pippenger(data, pippenger_HiGi_cache, get_pippenger_c(data.size())); + } + else + return data.size() <= 64 ? straus(data, NULL, 0) : pippenger(data, NULL, get_pippenger_c(data.size())); +} + +static bool is_reduced(const rct::key &scalar) +{ + rct::key reduced = scalar; + sc_reduce32(reduced.bytes); + return scalar == reduced; +} + +static void addKeys_acc_p3(ge_p3 *acc_p3, const rct::key &a, const rct::key &point) +{ + ge_p3 p3; + CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&p3, point.bytes) == 0, "ge_frombytes_vartime failed"); + ge_scalarmult_p3(&p3, a.bytes, &p3); + ge_cached cached; + ge_p3_to_cached(&cached, acc_p3); + ge_p1p1 p1; + ge_add(&p1, &p3, &cached); + ge_p1p1_to_p3(acc_p3, &p1); +} + +static void add_acc_p3(ge_p3 *acc_p3, const rct::key &point) +{ + ge_p3 p3; + CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&p3, point.bytes) == 0, "ge_frombytes_vartime failed"); + ge_cached cached; + ge_p3_to_cached(&cached, &p3); + ge_p1p1 p1; + ge_add(&p1, acc_p3, &cached); + ge_p1p1_to_p3(acc_p3, &p1); +} + +static void sub_acc_p3(ge_p3 *acc_p3, const rct::key &point) +{ + ge_p3 p3; + CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&p3, point.bytes) == 0, "ge_frombytes_vartime failed"); + ge_cached cached; + ge_p3_to_cached(&cached, &p3); + ge_p1p1 p1; + ge_sub(&p1, acc_p3, &cached); + ge_p1p1_to_p3(acc_p3, &p1); +} + +static rct::key scalarmultKey(const ge_p3 &P, const rct::key &a) +{ + ge_p2 R; + ge_scalarmult(&R, a.bytes, &P); + rct::key aP; + ge_tobytes(aP.bytes, &R); + return aP; +} + static rct::key get_exponent(const rct::key &base, size_t idx) { static const std::string salt("bulletproof"); std::string hashed = std::string((const char*)base.bytes, sizeof(base)) + salt + tools::get_varint_data(idx); - return rct::hashToPoint(rct::hash2rct(crypto::cn_fast_hash(hashed.data(), hashed.size()))); + const rct::key e = rct::hashToPoint(rct::hash2rct(crypto::cn_fast_hash(hashed.data(), hashed.size()))); + CHECK_AND_ASSERT_THROW_MES(!(e == rct::identity()), "Exponent is point at infinity"); + return e; } static void init_exponents() @@ -77,13 +152,27 @@ static void init_exponents() static bool init_done = false; if (init_done) return; - for (size_t i = 0; i < maxN; ++i) + std::vector<MultiexpData> data; + for (size_t i = 0; i < maxN*maxM; ++i) { Hi[i] = get_exponent(rct::H, i * 2); - rct::precomp(Hprecomp[i], Hi[i]); + CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&Hi_p3[i], Hi[i].bytes) == 0, "ge_frombytes_vartime failed"); Gi[i] = get_exponent(rct::H, i * 2 + 1); - rct::precomp(Gprecomp[i], Gi[i]); + CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&Gi_p3[i], Gi[i].bytes) == 0, "ge_frombytes_vartime failed"); + + data.push_back({rct::zero(), Gi[i]}); + data.push_back({rct::zero(), Hi[i]}); } + + straus_HiGi_cache = straus_init_cache(data, STRAUS_SIZE_LIMIT); + pippenger_HiGi_cache = pippenger_init_cache(data, PIPPENGER_SIZE_LIMIT); + + MINFO("Hi/Gi cache size: " << (sizeof(Hi)+sizeof(Gi))/1024 << " kB"); + MINFO("Hi_p3/Gi_p3 cache size: " << (sizeof(Hi_p3)+sizeof(Gi_p3))/1024 << " kB"); + MINFO("Straus cache size: " << straus_get_cache_size(straus_HiGi_cache)/1024 << " kB"); + MINFO("Pippenger cache size: " << pippenger_get_cache_size(pippenger_HiGi_cache)/1024 << " kB"); + size_t cache_size = (sizeof(Hi)+sizeof(Hi_p3))*2 + straus_get_cache_size(straus_HiGi_cache) + pippenger_get_cache_size(pippenger_HiGi_cache); + MINFO("Total cache size: " << cache_size/1024 << "kB"); init_done = true; } @@ -91,15 +180,16 @@ static void init_exponents() static rct::key vector_exponent(const rct::keyV &a, const rct::keyV &b) { CHECK_AND_ASSERT_THROW_MES(a.size() == b.size(), "Incompatible sizes of a and b"); - CHECK_AND_ASSERT_THROW_MES(a.size() <= maxN, "Incompatible sizes of a and maxN"); - rct::key res = rct::identity(); + CHECK_AND_ASSERT_THROW_MES(a.size() <= maxN*maxM, "Incompatible sizes of a and maxN"); + + std::vector<MultiexpData> multiexp_data; + multiexp_data.reserve(a.size()*2); for (size_t i = 0; i < a.size(); ++i) { - rct::key term; - rct::addKeys3(term, a[i], Gprecomp[i], b[i], Hprecomp[i]); - rct::addKeys(res, res, term); + multiexp_data.emplace_back(a[i], Gi_p3[i]); + multiexp_data.emplace_back(b[i], Hi_p3[i]); } - return res; + return multiexp(multiexp_data, true); } /* Compute a custom vector-scalar commitment */ @@ -108,44 +198,24 @@ static rct::key vector_exponent_custom(const rct::keyV &A, const rct::keyV &B, c CHECK_AND_ASSERT_THROW_MES(A.size() == B.size(), "Incompatible sizes of A and B"); CHECK_AND_ASSERT_THROW_MES(a.size() == b.size(), "Incompatible sizes of a and b"); CHECK_AND_ASSERT_THROW_MES(a.size() == A.size(), "Incompatible sizes of a and A"); - CHECK_AND_ASSERT_THROW_MES(a.size() <= maxN, "Incompatible sizes of a and maxN"); - rct::key res = rct::identity(); + CHECK_AND_ASSERT_THROW_MES(a.size() <= maxN*maxM, "Incompatible sizes of a and maxN"); + + std::vector<MultiexpData> multiexp_data; + multiexp_data.reserve(a.size()*2); for (size_t i = 0; i < a.size(); ++i) { - rct::key term; -#if 0 - // we happen to know where A and B might fall, so don't bother checking the rest - ge_dsmp *Acache = NULL, *Bcache = NULL; - ge_dsmp Acache_custom[1], Bcache_custom[1]; - if (Gi[i] == A[i]) - Acache = Gprecomp + i; - else if (i<32 && Gi[i+32] == A[i]) - Acache = Gprecomp + i + 32; - else - { - rct::precomp(Acache_custom[0], A[i]); - Acache = Acache_custom; - } - if (i == 0 && B[i] == Hi[0]) - Bcache = Hprecomp; - else - { - rct::precomp(Bcache_custom[0], B[i]); - Bcache = Bcache_custom; - } - rct::addKeys3(term, a[i], *Acache, b[i], *Bcache); -#else - ge_dsmp Acache, Bcache; - rct::precomp(Bcache, B[i]); - rct::addKeys3(term, a[i], A[i], b[i], Bcache); -#endif - rct::addKeys(res, res, term); + multiexp_data.resize(multiexp_data.size() + 1); + multiexp_data.back().scalar = a[i]; + CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&multiexp_data.back().point, A[i].bytes) == 0, "ge_frombytes_vartime failed"); + multiexp_data.resize(multiexp_data.size() + 1); + multiexp_data.back().scalar = b[i]; + CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&multiexp_data.back().point, B[i].bytes) == 0, "ge_frombytes_vartime failed"); } - return res; + return multiexp(multiexp_data, false); } /* Given a scalar, construct a vector of powers */ -static rct::keyV vector_powers(rct::key x, size_t n) +static rct::keyV vector_powers(const rct::key &x, size_t n) { rct::keyV res(n); if (n == 0) @@ -161,6 +231,24 @@ static rct::keyV vector_powers(rct::key x, size_t n) return res; } +/* Given a scalar, return the sum of its powers from 0 to n-1 */ +static rct::key vector_power_sum(const rct::key &x, size_t n) +{ + if (n == 0) + return rct::zero(); + rct::key res = rct::identity(); + if (n == 1) + return res; + rct::key prev = x; + for (size_t i = 1; i < n; ++i) + { + if (i > 1) + sc_mul(prev.bytes, prev.bytes, x.bytes); + sc_add(res.bytes, res.bytes, prev.bytes); + } + return res; +} + /* Given two scalar arrays, construct the inner product */ static rct::key inner_product(const rct::keyV &a, const rct::keyV &b) { @@ -232,6 +320,12 @@ static rct::keyV vector_scalar(const rct::keyV &a, const rct::key &x) return res; } +/* Create a vector from copies of a single value */ +static rct::keyV vector_dup(const rct::key &x, size_t N) +{ + return rct::keyV(N, x); +} + /* Exponentiate a curve vector by a scalar */ static rct::keyV vector_scalar2(const rct::keyV &a, const rct::key &x) { @@ -243,6 +337,17 @@ static rct::keyV vector_scalar2(const rct::keyV &a, const rct::key &x) return res; } +/* Get the sum of a vector's elements */ +static rct::key vector_sum(const rct::keyV &a) +{ + rct::key res = rct::zero(); + for (size_t i = 0; i < a.size(); ++i) + { + sc_add(res.bytes, res.bytes, a[i].bytes); + } + return res; +} + static rct::key switch_endianness(rct::key k) { std::reverse(k.bytes, k.bytes + sizeof(k)); @@ -345,6 +450,7 @@ Bulletproof bulletproof_PROVE(const rct::key &sv, const rct::key &gamma) PERF_TIMER_START_BP(PROVE_v); rct::addKeys2(V, gamma, sv, rct::H); + V = rct::scalarmultKey(V, INV_EIGHT); PERF_TIMER_STOP(PROVE_v); PERF_TIMER_START_BP(PROVE_aLaR); @@ -380,12 +486,14 @@ Bulletproof bulletproof_PROVE(const rct::key &sv, const rct::key &gamma) CHECK_AND_ASSERT_THROW_MES(test_aR == v_test, "test_aR failed"); #endif +try_again: PERF_TIMER_START_BP(PROVE_step1); // PAPER LINES 38-39 rct::key alpha = rct::skGen(); rct::key ve = vector_exponent(aL, aR); rct::key A; rct::addKeys(A, ve, rct::scalarmultBase(alpha)); + A = rct::scalarmultKey(A, INV_EIGHT); // PAPER LINES 40-42 rct::keyV sL = rct::skvGen(N), sR = rct::skvGen(N); @@ -393,10 +501,23 @@ Bulletproof bulletproof_PROVE(const rct::key &sv, const rct::key &gamma) ve = vector_exponent(sL, sR); rct::key S; rct::addKeys(S, ve, rct::scalarmultBase(rho)); + S = rct::scalarmultKey(S, INV_EIGHT); // PAPER LINES 43-45 rct::key y = hash_cache_mash(hash_cache, A, S); + if (y == rct::zero()) + { + PERF_TIMER_STOP(PROVE_step1); + MINFO("y is 0, trying again"); + goto try_again; + } rct::key z = hash_cache = rct::hash_to_scalar(y); + if (z == rct::zero()) + { + PERF_TIMER_STOP(PROVE_step1); + MINFO("z is 0, trying again"); + goto try_again; + } // Polynomial construction before PAPER LINE 46 rct::key t0 = rct::zero(); @@ -405,7 +526,7 @@ Bulletproof bulletproof_PROVE(const rct::key &sv, const rct::key &gamma) const auto yN = vector_powers(y, N); - rct::key ip1y = inner_product(oneN, yN); + rct::key ip1y = vector_sum(yN); rct::key tmp; sc_muladd(t0.bytes, z.bytes, ip1y.bytes, t0.bytes); @@ -437,7 +558,7 @@ Bulletproof bulletproof_PROVE(const rct::key &sv, const rct::key &gamma) PERF_TIMER_START_BP(PROVE_step2); const auto HyNsR = hadamard(yN, sR); - const auto vpIz = vector_scalar(oneN, z); + const auto vpIz = vector_dup(z, N); const auto vp2zsq = vector_scalar(twoN, zsq); const auto aL_vpIz = vector_subtract(aL, vpIz); const auto aR_vpIz = vector_add(aR, vpIz); @@ -454,11 +575,19 @@ Bulletproof bulletproof_PROVE(const rct::key &sv, const rct::key &gamma) // PAPER LINES 47-48 rct::key tau1 = rct::skGen(), tau2 = rct::skGen(); - rct::key T1 = rct::addKeys(rct::scalarmultKey(rct::H, t1), rct::scalarmultBase(tau1)); - rct::key T2 = rct::addKeys(rct::scalarmultKey(rct::H, t2), rct::scalarmultBase(tau2)); + rct::key T1 = rct::addKeys(rct::scalarmultH(t1), rct::scalarmultBase(tau1)); + T1 = rct::scalarmultKey(T1, INV_EIGHT); + rct::key T2 = rct::addKeys(rct::scalarmultH(t2), rct::scalarmultBase(tau2)); + T2 = rct::scalarmultKey(T2, INV_EIGHT); // PAPER LINES 49-51 rct::key x = hash_cache_mash(hash_cache, z, T1, T2); + if (x == rct::zero()) + { + PERF_TIMER_STOP(PROVE_step2); + MINFO("x is 0, trying again"); + goto try_again; + } // PAPER LINES 52-53 rct::key taux = rct::zero(); @@ -500,7 +629,7 @@ Bulletproof bulletproof_PROVE(const rct::key &sv, const rct::key &gamma) for (size_t i = 0; i < N; ++i) { Gprime[i] = Gi[i]; - Hprime[i] = scalarmultKey(Hi[i], yinvpow); + Hprime[i] = scalarmultKey(Hi_p3[i], yinvpow); sc_mul(yinvpow.bytes, yinvpow.bytes, yinv.bytes); aprime[i] = l[i]; bprime[i] = r[i]; @@ -525,13 +654,21 @@ Bulletproof bulletproof_PROVE(const rct::key &sv, const rct::key &gamma) // PAPER LINES 18-19 L[round] = vector_exponent_custom(slice(Gprime, nprime, Gprime.size()), slice(Hprime, 0, nprime), slice(aprime, 0, nprime), slice(bprime, nprime, bprime.size())); sc_mul(tmp.bytes, cL.bytes, x_ip.bytes); - rct::addKeys(L[round], L[round], rct::scalarmultKey(rct::H, tmp)); + rct::addKeys(L[round], L[round], rct::scalarmultH(tmp)); + L[round] = rct::scalarmultKey(L[round], INV_EIGHT); R[round] = vector_exponent_custom(slice(Gprime, 0, nprime), slice(Hprime, nprime, Hprime.size()), slice(aprime, nprime, aprime.size()), slice(bprime, 0, nprime)); sc_mul(tmp.bytes, cR.bytes, x_ip.bytes); - rct::addKeys(R[round], R[round], rct::scalarmultKey(rct::H, tmp)); + rct::addKeys(R[round], R[round], rct::scalarmultH(tmp)); + R[round] = rct::scalarmultKey(R[round], INV_EIGHT); // PAPER LINES 21-22 w[round] = hash_cache_mash(hash_cache, L[round], R[round]); + if (w[round] == rct::zero()) + { + PERF_TIMER_STOP(PROVE_step4); + MINFO("w[round] is 0, trying again"); + goto try_again; + } // PAPER LINES 24-25 const rct::key winv = invert(w[round]); @@ -567,179 +704,540 @@ Bulletproof bulletproof_PROVE(uint64_t v, const rct::key &gamma) return bulletproof_PROVE(sv, gamma); } -/* Given a range proof, determine if it is valid */ -bool bulletproof_VERIFY(const Bulletproof &proof) +/* Given a set of values v (0..2^N-1) and masks gamma, construct a range proof */ +Bulletproof bulletproof_PROVE(const rct::keyV &sv, const rct::keyV &gamma) { + CHECK_AND_ASSERT_THROW_MES(sv.size() == gamma.size(), "Incompatible sizes of sv and gamma"); + CHECK_AND_ASSERT_THROW_MES(!sv.empty(), "sv is empty"); + for (const rct::key &sve: sv) + CHECK_AND_ASSERT_THROW_MES(is_reduced(sve), "Invalid sv input"); + for (const rct::key &g: gamma) + CHECK_AND_ASSERT_THROW_MES(is_reduced(g), "Invalid gamma input"); + init_exponents(); - CHECK_AND_ASSERT_MES(proof.V.size() == 1, false, "V does not have exactly one element"); - CHECK_AND_ASSERT_MES(proof.L.size() == proof.R.size(), false, "Mismatched L and R sizes"); - CHECK_AND_ASSERT_MES(proof.L.size() > 0, false, "Empty proof"); - CHECK_AND_ASSERT_MES(proof.L.size() == 6, false, "Proof is not for 64 bits"); + PERF_TIMER_UNIT(PROVE, 1000000); - const size_t logN = proof.L.size(); - const size_t N = 1 << logN; + constexpr size_t logN = 6; // log2(64) + constexpr size_t N = 1<<logN; + size_t M, logM; + for (logM = 0; (M = 1<<logM) <= maxM && M < sv.size(); ++logM); + CHECK_AND_ASSERT_THROW_MES(M <= maxM, "sv/gamma are too large"); + const size_t logMN = logM + logN; + const size_t MN = M * N; + + rct::keyV V(sv.size()); + rct::keyV aL(MN), aR(MN); + rct::key tmp; - // Reconstruct the challenges - PERF_TIMER_START_BP(VERIFY); - PERF_TIMER_START_BP(VERIFY_start); - rct::key hash_cache = rct::hash_to_scalar(proof.V[0]); - rct::key y = hash_cache_mash(hash_cache, proof.A, proof.S); + PERF_TIMER_START_BP(PROVE_v); + for (size_t i = 0; i < sv.size(); ++i) + { + rct::addKeys2(V[i], gamma[i], sv[i], rct::H); + V[i] = rct::scalarmultKey(V[i], INV_EIGHT); + } + PERF_TIMER_STOP(PROVE_v); + + PERF_TIMER_START_BP(PROVE_aLaR); + for (size_t j = 0; j < M; ++j) + { + for (size_t i = N; i-- > 0; ) + { + if (j >= sv.size()) + { + aL[j*N+i] = rct::zero(); + } + else if (sv[j][i/8] & (((uint64_t)1)<<(i%8))) + { + aL[j*N+i] = rct::identity(); + } + else + { + aL[j*N+i] = rct::zero(); + } + sc_sub(aR[j*N+i].bytes, aL[j*N+i].bytes, rct::identity().bytes); + } + } + PERF_TIMER_STOP(PROVE_aLaR); + + // DEBUG: Test to ensure this recovers the value +#ifdef DEBUG_BP + for (size_t j = 0; j < M; ++j) + { + uint64_t test_aL = 0, test_aR = 0; + for (size_t i = 0; i < N; ++i) + { + if (aL[j*N+i] == rct::identity()) + test_aL += ((uint64_t)1)<<i; + if (aR[j*N+i] == rct::zero()) + test_aR += ((uint64_t)1)<<i; + } + uint64_t v_test = 0; + if (j < sv.size()) + for (int n = 0; n < 8; ++n) v_test |= (((uint64_t)sv[j][n]) << (8*n)); + CHECK_AND_ASSERT_THROW_MES(test_aL == v_test, "test_aL failed"); + CHECK_AND_ASSERT_THROW_MES(test_aR == v_test, "test_aR failed"); + } +#endif + +try_again: + rct::key hash_cache = rct::hash_to_scalar(V); + + PERF_TIMER_START_BP(PROVE_step1); + // PAPER LINES 38-39 + rct::key alpha = rct::skGen(); + rct::key ve = vector_exponent(aL, aR); + rct::key A; + rct::addKeys(A, ve, rct::scalarmultBase(alpha)); + A = rct::scalarmultKey(A, INV_EIGHT); + + // PAPER LINES 40-42 + rct::keyV sL = rct::skvGen(MN), sR = rct::skvGen(MN); + rct::key rho = rct::skGen(); + ve = vector_exponent(sL, sR); + rct::key S; + rct::addKeys(S, ve, rct::scalarmultBase(rho)); + S = rct::scalarmultKey(S, INV_EIGHT); + + // PAPER LINES 43-45 + rct::key y = hash_cache_mash(hash_cache, A, S); + if (y == rct::zero()) + { + PERF_TIMER_STOP(PROVE_step1); + MINFO("y is 0, trying again"); + goto try_again; + } rct::key z = hash_cache = rct::hash_to_scalar(y); - rct::key x = hash_cache_mash(hash_cache, z, proof.T1, proof.T2); - PERF_TIMER_STOP(VERIFY_start); + if (z == rct::zero()) + { + PERF_TIMER_STOP(PROVE_step1); + MINFO("z is 0, trying again"); + goto try_again; + } - PERF_TIMER_START_BP(VERIFY_line_60); - // Reconstruct the challenges - rct::key x_ip = hash_cache_mash(hash_cache, x, proof.taux, proof.mu, proof.t); - PERF_TIMER_STOP(VERIFY_line_60); + // Polynomial construction by coefficients + const auto zMN = vector_dup(z, MN); + rct::keyV l0 = vector_subtract(aL, zMN); + const rct::keyV &l1 = sL; - PERF_TIMER_START_BP(VERIFY_line_61); - // PAPER LINE 61 - rct::key L61Left = rct::addKeys(rct::scalarmultBase(proof.taux), rct::scalarmultKey(rct::H, proof.t)); + // This computes the ugly sum/concatenation from PAPER LINE 65 + rct::keyV zero_twos(MN); + const rct::keyV zpow = vector_powers(z, M+2); + for (size_t i = 0; i < MN; ++i) + { + zero_twos[i] = rct::zero(); + for (size_t j = 1; j <= M; ++j) + { + if (i >= (j-1)*N && i < j*N) + { + CHECK_AND_ASSERT_THROW_MES(1+j < zpow.size(), "invalid zpow index"); + CHECK_AND_ASSERT_THROW_MES(i-(j-1)*N < twoN.size(), "invalid twoN index"); + sc_muladd(zero_twos[i].bytes, zpow[1+j].bytes, twoN[i-(j-1)*N].bytes, zero_twos[i].bytes); + } + } + } - rct::key k = rct::zero(); - const auto yN = vector_powers(y, N); - rct::key ip1y = inner_product(oneN, yN); - rct::key zsq; - sc_mul(zsq.bytes, z.bytes, z.bytes); - rct::key tmp, tmp2; - sc_mulsub(k.bytes, zsq.bytes, ip1y.bytes, k.bytes); - rct::key zcu; - sc_mul(zcu.bytes, zsq.bytes, z.bytes); - sc_mulsub(k.bytes, zcu.bytes, ip12.bytes, k.bytes); - PERF_TIMER_STOP(VERIFY_line_61); + rct::keyV r0 = vector_add(aR, zMN); + const auto yMN = vector_powers(y, MN); + r0 = hadamard(r0, yMN); + r0 = vector_add(r0, zero_twos); + rct::keyV r1 = hadamard(yMN, sR); - PERF_TIMER_START_BP(VERIFY_line_61rl); - sc_muladd(tmp.bytes, z.bytes, ip1y.bytes, k.bytes); - rct::key L61Right = rct::scalarmultKey(rct::H, tmp); + // Polynomial construction before PAPER LINE 46 + rct::key t1_1 = inner_product(l0, r1); + rct::key t1_2 = inner_product(l1, r0); + rct::key t1; + sc_add(t1.bytes, t1_1.bytes, t1_2.bytes); + rct::key t2 = inner_product(l1, r1); - CHECK_AND_ASSERT_MES(proof.V.size() == 1, false, "proof.V does not have exactly one element"); - tmp = rct::scalarmultKey(proof.V[0], zsq); - rct::addKeys(L61Right, L61Right, tmp); + PERF_TIMER_STOP(PROVE_step1); - tmp = rct::scalarmultKey(proof.T1, x); - rct::addKeys(L61Right, L61Right, tmp); + PERF_TIMER_START_BP(PROVE_step2); + // PAPER LINES 47-48 + rct::key tau1 = rct::skGen(), tau2 = rct::skGen(); + + rct::key T1 = rct::addKeys(rct::scalarmultH(t1), rct::scalarmultBase(tau1)); + T1 = rct::scalarmultKey(T1, INV_EIGHT); + rct::key T2 = rct::addKeys(rct::scalarmultH(t2), rct::scalarmultBase(tau2)); + T2 = rct::scalarmultKey(T2, INV_EIGHT); + // PAPER LINES 49-51 + rct::key x = hash_cache_mash(hash_cache, z, T1, T2); + if (x == rct::zero()) + { + PERF_TIMER_STOP(PROVE_step2); + MINFO("x is 0, trying again"); + goto try_again; + } + + // PAPER LINES 52-53 + rct::key taux; + sc_mul(taux.bytes, tau1.bytes, x.bytes); rct::key xsq; sc_mul(xsq.bytes, x.bytes, x.bytes); - tmp = rct::scalarmultKey(proof.T2, xsq); - rct::addKeys(L61Right, L61Right, tmp); - PERF_TIMER_STOP(VERIFY_line_61rl); - - if (!(L61Right == L61Left)) + sc_muladd(taux.bytes, tau2.bytes, xsq.bytes, taux.bytes); + for (size_t j = 1; j <= sv.size(); ++j) { - MERROR("Verification failure at step 1"); - return false; + CHECK_AND_ASSERT_THROW_MES(j+1 < zpow.size(), "invalid zpow index"); + sc_muladd(taux.bytes, zpow[j+1].bytes, gamma[j-1].bytes, taux.bytes); } + rct::key mu; + sc_muladd(mu.bytes, x.bytes, rho.bytes, alpha.bytes); - PERF_TIMER_START_BP(VERIFY_line_62); - // PAPER LINE 62 - rct::key P = rct::addKeys(proof.A, rct::scalarmultKey(proof.S, x)); - PERF_TIMER_STOP(VERIFY_line_62); + // PAPER LINES 54-57 + rct::keyV l = l0; + l = vector_add(l, vector_scalar(l1, x)); + rct::keyV r = r0; + r = vector_add(r, vector_scalar(r1, x)); + PERF_TIMER_STOP(PROVE_step2); - // Compute the number of rounds for the inner product - const size_t rounds = proof.L.size(); - CHECK_AND_ASSERT_MES(rounds > 0, false, "Zero rounds"); + PERF_TIMER_START_BP(PROVE_step3); + rct::key t = inner_product(l, r); - PERF_TIMER_START_BP(VERIFY_line_21_22); - // PAPER LINES 21-22 - // The inner product challenges are computed per round - rct::keyV w(rounds); - for (size_t i = 0; i < rounds; ++i) + // DEBUG: Test if the l and r vectors match the polynomial forms +#ifdef DEBUG_BP + rct::key test_t; + const rct::key t0 = inner_product(l0, r0); + sc_muladd(test_t.bytes, t1.bytes, x.bytes, t0.bytes); + sc_muladd(test_t.bytes, t2.bytes, xsq.bytes, test_t.bytes); + CHECK_AND_ASSERT_THROW_MES(test_t == t, "test_t check failed"); +#endif + + // PAPER LINES 32-33 + rct::key x_ip = hash_cache_mash(hash_cache, x, taux, mu, t); + if (x_ip == rct::zero()) { - w[i] = hash_cache_mash(hash_cache, proof.L[i], proof.R[i]); + PERF_TIMER_STOP(PROVE_step3); + MINFO("x_ip is 0, trying again"); + goto try_again; } - PERF_TIMER_STOP(VERIFY_line_21_22); - PERF_TIMER_START_BP(VERIFY_line_24_25); - // Basically PAPER LINES 24-25 - // Compute the curvepoints from G[i] and H[i] - rct::key inner_prod = rct::identity(); + // These are used in the inner product rounds + size_t nprime = MN; + rct::keyV Gprime(MN); + rct::keyV Hprime(MN); + rct::keyV aprime(MN); + rct::keyV bprime(MN); + const rct::key yinv = invert(y); rct::key yinvpow = rct::identity(); - rct::key ypow = rct::identity(); + for (size_t i = 0; i < MN; ++i) + { + Gprime[i] = Gi[i]; + Hprime[i] = scalarmultKey(Hi_p3[i], yinvpow); + sc_mul(yinvpow.bytes, yinvpow.bytes, yinv.bytes); + aprime[i] = l[i]; + bprime[i] = r[i]; + } + rct::keyV L(logMN); + rct::keyV R(logMN); + int round = 0; + rct::keyV w(logMN); // this is the challenge x in the inner product protocol + PERF_TIMER_STOP(PROVE_step3); - PERF_TIMER_START_BP(VERIFY_line_24_25_invert); - const rct::key yinv = invert(y); - rct::keyV winv(rounds); - for (size_t i = 0; i < rounds; ++i) - winv[i] = invert(w[i]); - PERF_TIMER_STOP(VERIFY_line_24_25_invert); + PERF_TIMER_START_BP(PROVE_step4); + // PAPER LINE 13 + while (nprime > 1) + { + // PAPER LINE 15 + nprime /= 2; - for (size_t i = 0; i < N; ++i) + // PAPER LINES 16-17 + rct::key cL = inner_product(slice(aprime, 0, nprime), slice(bprime, nprime, bprime.size())); + rct::key cR = inner_product(slice(aprime, nprime, aprime.size()), slice(bprime, 0, nprime)); + + // PAPER LINES 18-19 + L[round] = vector_exponent_custom(slice(Gprime, nprime, Gprime.size()), slice(Hprime, 0, nprime), slice(aprime, 0, nprime), slice(bprime, nprime, bprime.size())); + sc_mul(tmp.bytes, cL.bytes, x_ip.bytes); + rct::addKeys(L[round], L[round], rct::scalarmultH(tmp)); + L[round] = rct::scalarmultKey(L[round], INV_EIGHT); + R[round] = vector_exponent_custom(slice(Gprime, 0, nprime), slice(Hprime, nprime, Hprime.size()), slice(aprime, nprime, aprime.size()), slice(bprime, 0, nprime)); + sc_mul(tmp.bytes, cR.bytes, x_ip.bytes); + rct::addKeys(R[round], R[round], rct::scalarmultH(tmp)); + R[round] = rct::scalarmultKey(R[round], INV_EIGHT); + + // PAPER LINES 21-22 + w[round] = hash_cache_mash(hash_cache, L[round], R[round]); + if (w[round] == rct::zero()) + { + PERF_TIMER_STOP(PROVE_step4); + MINFO("w[round] is 0, trying again"); + goto try_again; + } + + // PAPER LINES 24-25 + const rct::key winv = invert(w[round]); + Gprime = hadamard2(vector_scalar2(slice(Gprime, 0, nprime), winv), vector_scalar2(slice(Gprime, nprime, Gprime.size()), w[round])); + Hprime = hadamard2(vector_scalar2(slice(Hprime, 0, nprime), w[round]), vector_scalar2(slice(Hprime, nprime, Hprime.size()), winv)); + + // PAPER LINES 28-29 + aprime = vector_add(vector_scalar(slice(aprime, 0, nprime), w[round]), vector_scalar(slice(aprime, nprime, aprime.size()), winv)); + bprime = vector_add(vector_scalar(slice(bprime, 0, nprime), winv), vector_scalar(slice(bprime, nprime, bprime.size()), w[round])); + + ++round; + } + PERF_TIMER_STOP(PROVE_step4); + + // PAPER LINE 58 (with inclusions from PAPER LINE 8 and PAPER LINE 20) + return Bulletproof(V, A, S, T1, T2, taux, mu, L, R, aprime[0], bprime[0], t); +} + +Bulletproof bulletproof_PROVE(const std::vector<uint64_t> &v, const rct::keyV &gamma) +{ + CHECK_AND_ASSERT_THROW_MES(v.size() == gamma.size(), "Incompatible sizes of v and gamma"); + + // vG + gammaH + PERF_TIMER_START_BP(PROVE_v); + rct::keyV sv(v.size()); + for (size_t i = 0; i < v.size(); ++i) { - // Convert the index to binary IN REVERSE and construct the scalar exponent - rct::key g_scalar = proof.a; - rct::key h_scalar; - sc_mul(h_scalar.bytes, proof.b.bytes, yinvpow.bytes); + sv[i] = rct::zero(); + sv[i].bytes[0] = v[i] & 255; + sv[i].bytes[1] = (v[i] >> 8) & 255; + sv[i].bytes[2] = (v[i] >> 16) & 255; + sv[i].bytes[3] = (v[i] >> 24) & 255; + sv[i].bytes[4] = (v[i] >> 32) & 255; + sv[i].bytes[5] = (v[i] >> 40) & 255; + sv[i].bytes[6] = (v[i] >> 48) & 255; + sv[i].bytes[7] = (v[i] >> 56) & 255; + } + PERF_TIMER_STOP(PROVE_v); + return bulletproof_PROVE(sv, gamma); +} + +/* Given a range proof, determine if it is valid */ +bool bulletproof_VERIFY(const std::vector<const Bulletproof*> &proofs) +{ + init_exponents(); + + PERF_TIMER_START_BP(VERIFY); + + // sanity and figure out which proof is longest + size_t max_length = 0; + for (const Bulletproof *p: proofs) + { + const Bulletproof &proof = *p; + + // check scalar range + CHECK_AND_ASSERT_MES(is_reduced(proof.taux), false, "Input scalar not in range"); + CHECK_AND_ASSERT_MES(is_reduced(proof.mu), false, "Input scalar not in range"); + CHECK_AND_ASSERT_MES(is_reduced(proof.a), false, "Input scalar not in range"); + CHECK_AND_ASSERT_MES(is_reduced(proof.b), false, "Input scalar not in range"); + CHECK_AND_ASSERT_MES(is_reduced(proof.t), false, "Input scalar not in range"); + + CHECK_AND_ASSERT_MES(proof.V.size() >= 1, false, "V does not have at least one element"); + CHECK_AND_ASSERT_MES(proof.L.size() == proof.R.size(), false, "Mismatched L and R sizes"); + CHECK_AND_ASSERT_MES(proof.L.size() > 0, false, "Empty proof"); - for (size_t j = rounds; j-- > 0; ) + max_length = std::max(max_length, proof.L.size()); + } + CHECK_AND_ASSERT_MES(max_length < 32, false, "At least one proof is too large"); + size_t maxMN = 1u << max_length; + + const size_t logN = 6; + const size_t N = 1 << logN; + rct::key tmp; + + // setup weighted aggregates + rct::key Z0 = rct::identity(); + rct::key z1 = rct::zero(); + rct::key Z2 = rct::identity(); + rct::key z3 = rct::zero(); + rct::keyV z4(maxMN, rct::zero()), z5(maxMN, rct::zero()); + rct::key Y2 = rct::identity(), Y3 = rct::identity(), Y4 = rct::identity(); + rct::key y0 = rct::zero(), y1 = rct::zero(); + for (const Bulletproof *p: proofs) + { + const Bulletproof &proof = *p; + + size_t M, logM; + for (logM = 0; (M = 1<<logM) <= maxM && M < proof.V.size(); ++logM); + CHECK_AND_ASSERT_MES(proof.L.size() == 6+logM, false, "Proof is not the expected size"); + const size_t MN = M*N; + rct::key weight = rct::skGen(); + + // Reconstruct the challenges + PERF_TIMER_START_BP(VERIFY_start); + rct::key hash_cache = rct::hash_to_scalar(proof.V); + rct::key y = hash_cache_mash(hash_cache, proof.A, proof.S); + CHECK_AND_ASSERT_MES(!(y == rct::zero()), false, "y == 0"); + rct::key z = hash_cache = rct::hash_to_scalar(y); + CHECK_AND_ASSERT_MES(!(z == rct::zero()), false, "z == 0"); + rct::key x = hash_cache_mash(hash_cache, z, proof.T1, proof.T2); + CHECK_AND_ASSERT_MES(!(x == rct::zero()), false, "x == 0"); + rct::key x_ip = hash_cache_mash(hash_cache, x, proof.taux, proof.mu, proof.t); + CHECK_AND_ASSERT_MES(!(x_ip == rct::zero()), false, "x_ip == 0"); + PERF_TIMER_STOP(VERIFY_start); + + PERF_TIMER_START_BP(VERIFY_line_61); + // PAPER LINE 61 + sc_muladd(y0.bytes, proof.taux.bytes, weight.bytes, y0.bytes); + + const rct::keyV zpow = vector_powers(z, M+3); + + rct::key k; + const rct::key ip1y = vector_power_sum(y, MN); + sc_mulsub(k.bytes, zpow[2].bytes, ip1y.bytes, rct::zero().bytes); + for (size_t j = 1; j <= M; ++j) + { + CHECK_AND_ASSERT_MES(j+2 < zpow.size(), false, "invalid zpow index"); + sc_mulsub(k.bytes, zpow[j+2].bytes, ip12.bytes, k.bytes); + } + PERF_TIMER_STOP(VERIFY_line_61); + + PERF_TIMER_START_BP(VERIFY_line_61rl_new); + sc_muladd(tmp.bytes, z.bytes, ip1y.bytes, k.bytes); + std::vector<MultiexpData> multiexp_data; + multiexp_data.reserve(proof.V.size()); + sc_sub(tmp.bytes, proof.t.bytes, tmp.bytes); + sc_muladd(y1.bytes, tmp.bytes, weight.bytes, y1.bytes); + for (size_t j = 0; j < proof.V.size(); j++) { - size_t J = w.size() - j - 1; + sc_mul(tmp.bytes, zpow[j+2].bytes, EIGHT.bytes); + multiexp_data.emplace_back(tmp, proof.V[j]); + } + rct::addKeys(Y2, Y2, rct::scalarmultKey(multiexp(multiexp_data, false), weight)); + rct::key weight8; + sc_mul(weight8.bytes, weight.bytes, EIGHT.bytes); + sc_mul(tmp.bytes, x.bytes, weight8.bytes); + rct::addKeys(Y3, Y3, rct::scalarmultKey(proof.T1, tmp)); + rct::key xsq; + sc_mul(xsq.bytes, x.bytes, x.bytes); + sc_mul(tmp.bytes, xsq.bytes, weight8.bytes); + rct::addKeys(Y4, Y4, rct::scalarmultKey(proof.T2, tmp)); + PERF_TIMER_STOP(VERIFY_line_61rl_new); + + PERF_TIMER_START_BP(VERIFY_line_62); + // PAPER LINE 62 + sc_mul(tmp.bytes, x.bytes, EIGHT.bytes); + rct::addKeys(Z0, Z0, rct::scalarmultKey(rct::addKeys(rct::scalarmult8(proof.A), rct::scalarmultKey(proof.S, tmp)), weight)); + PERF_TIMER_STOP(VERIFY_line_62); + + // Compute the number of rounds for the inner product + const size_t rounds = logM+logN; + CHECK_AND_ASSERT_MES(rounds > 0, false, "Zero rounds"); + + PERF_TIMER_START_BP(VERIFY_line_21_22); + // PAPER LINES 21-22 + // The inner product challenges are computed per round + rct::keyV w(rounds); + for (size_t i = 0; i < rounds; ++i) + { + w[i] = hash_cache_mash(hash_cache, proof.L[i], proof.R[i]); + CHECK_AND_ASSERT_MES(!(w[i] == rct::zero()), false, "w[i] == 0"); + } + PERF_TIMER_STOP(VERIFY_line_21_22); + + PERF_TIMER_START_BP(VERIFY_line_24_25); + // Basically PAPER LINES 24-25 + // Compute the curvepoints from G[i] and H[i] + rct::key yinvpow = rct::identity(); + rct::key ypow = rct::identity(); + + PERF_TIMER_START_BP(VERIFY_line_24_25_invert); + const rct::key yinv = invert(y); + rct::keyV winv(rounds); + for (size_t i = 0; i < rounds; ++i) + winv[i] = invert(w[i]); + PERF_TIMER_STOP(VERIFY_line_24_25_invert); + + for (size_t i = 0; i < MN; ++i) + { + // Convert the index to binary IN REVERSE and construct the scalar exponent + rct::key g_scalar = proof.a; + rct::key h_scalar; + sc_mul(h_scalar.bytes, proof.b.bytes, yinvpow.bytes); - if ((i & (((size_t)1)<<j)) == 0) + for (size_t j = rounds; j-- > 0; ) { - sc_mul(g_scalar.bytes, g_scalar.bytes, winv[J].bytes); - sc_mul(h_scalar.bytes, h_scalar.bytes, w[J].bytes); + size_t J = w.size() - j - 1; + + if ((i & (((size_t)1)<<j)) == 0) + { + sc_mul(g_scalar.bytes, g_scalar.bytes, winv[J].bytes); + sc_mul(h_scalar.bytes, h_scalar.bytes, w[J].bytes); + } + else + { + sc_mul(g_scalar.bytes, g_scalar.bytes, w[J].bytes); + sc_mul(h_scalar.bytes, h_scalar.bytes, winv[J].bytes); + } } - else + + // Adjust the scalars using the exponents from PAPER LINE 62 + sc_add(g_scalar.bytes, g_scalar.bytes, z.bytes); + CHECK_AND_ASSERT_MES(2+i/N < zpow.size(), false, "invalid zpow index"); + CHECK_AND_ASSERT_MES(i%N < twoN.size(), false, "invalid twoN index"); + sc_mul(tmp.bytes, zpow[2+i/N].bytes, twoN[i%N].bytes); + sc_muladd(tmp.bytes, z.bytes, ypow.bytes, tmp.bytes); + sc_mulsub(h_scalar.bytes, tmp.bytes, yinvpow.bytes, h_scalar.bytes); + + sc_muladd(z4[i].bytes, g_scalar.bytes, weight.bytes, z4[i].bytes); + sc_muladd(z5[i].bytes, h_scalar.bytes, weight.bytes, z5[i].bytes); + + if (i != MN-1) { - sc_mul(g_scalar.bytes, g_scalar.bytes, w[J].bytes); - sc_mul(h_scalar.bytes, h_scalar.bytes, winv[J].bytes); + sc_mul(yinvpow.bytes, yinvpow.bytes, yinv.bytes); + sc_mul(ypow.bytes, ypow.bytes, y.bytes); } } - // Adjust the scalars using the exponents from PAPER LINE 62 - sc_add(g_scalar.bytes, g_scalar.bytes, z.bytes); - sc_mul(tmp.bytes, zsq.bytes, twoN[i].bytes); - sc_muladd(tmp.bytes, z.bytes, ypow.bytes, tmp.bytes); - sc_mulsub(h_scalar.bytes, tmp.bytes, yinvpow.bytes, h_scalar.bytes); + PERF_TIMER_STOP(VERIFY_line_24_25); - // Now compute the basepoint's scalar multiplication - // Each of these could be written as a multiexp operation instead - rct::addKeys3(tmp, g_scalar, Gprecomp[i], h_scalar, Hprecomp[i]); - rct::addKeys(inner_prod, inner_prod, tmp); + // PAPER LINE 26 + PERF_TIMER_START_BP(VERIFY_line_26_new); + multiexp_data.clear(); + multiexp_data.reserve(2*rounds); - if (i != N-1) + sc_muladd(z1.bytes, proof.mu.bytes, weight.bytes, z1.bytes); + for (size_t i = 0; i < rounds; ++i) { - sc_mul(yinvpow.bytes, yinvpow.bytes, yinv.bytes); - sc_mul(ypow.bytes, ypow.bytes, y.bytes); + sc_mul(tmp.bytes, w[i].bytes, w[i].bytes); + sc_mul(tmp.bytes, tmp.bytes, EIGHT.bytes); + multiexp_data.emplace_back(tmp, proof.L[i]); + sc_mul(tmp.bytes, winv[i].bytes, winv[i].bytes); + sc_mul(tmp.bytes, tmp.bytes, EIGHT.bytes); + multiexp_data.emplace_back(tmp, proof.R[i]); } + rct::key acc = multiexp(multiexp_data, false); + rct::addKeys(Z2, Z2, rct::scalarmultKey(acc, weight)); + sc_mulsub(tmp.bytes, proof.a.bytes, proof.b.bytes, proof.t.bytes); + sc_mul(tmp.bytes, tmp.bytes, x_ip.bytes); + sc_muladd(z3.bytes, tmp.bytes, weight.bytes, z3.bytes); + PERF_TIMER_STOP(VERIFY_line_26_new); } - PERF_TIMER_STOP(VERIFY_line_24_25); - - PERF_TIMER_START_BP(VERIFY_line_26); - // PAPER LINE 26 - rct::key pprime; - sc_sub(tmp.bytes, rct::zero().bytes, proof.mu.bytes); - rct::addKeys(pprime, P, rct::scalarmultBase(tmp)); - - for (size_t i = 0; i < rounds; ++i) - { - sc_mul(tmp.bytes, w[i].bytes, w[i].bytes); - sc_mul(tmp2.bytes, winv[i].bytes, winv[i].bytes); -#if 1 - ge_dsmp cacheL, cacheR; - rct::precomp(cacheL, proof.L[i]); - rct::precomp(cacheR, proof.R[i]); - rct::addKeys3(tmp, tmp, cacheL, tmp2, cacheR); - rct::addKeys(pprime, pprime, tmp); -#else - rct::addKeys(pprime, pprime, rct::scalarmultKey(proof.L[i], tmp)); - rct::addKeys(pprime, pprime, rct::scalarmultKey(proof.R[i], tmp2)); -#endif - } - sc_mul(tmp.bytes, proof.t.bytes, x_ip.bytes); - rct::addKeys(pprime, pprime, rct::scalarmultKey(rct::H, tmp)); - PERF_TIMER_STOP(VERIFY_line_26); + // now check all proofs at once PERF_TIMER_START_BP(VERIFY_step2_check); - sc_mul(tmp.bytes, proof.a.bytes, proof.b.bytes); - sc_mul(tmp.bytes, tmp.bytes, x_ip.bytes); - tmp = rct::scalarmultKey(rct::H, tmp); - rct::addKeys(tmp, tmp, inner_prod); + ge_p3 check1; + ge_scalarmult_base(&check1, y0.bytes); + addKeys_acc_p3(&check1, y1, rct::H); + sub_acc_p3(&check1, Y2); + sub_acc_p3(&check1, Y3); + sub_acc_p3(&check1, Y4); + if (!ge_p3_is_point_at_infinity(&check1)) + { + MERROR("Verification failure at step 1"); + return false; + } + ge_p3 check2; + sc_sub(tmp.bytes, rct::zero().bytes, z1.bytes); + ge_double_scalarmult_base_vartime_p3(&check2, z3.bytes, &ge_p3_H, tmp.bytes); + add_acc_p3(&check2, Z0); + add_acc_p3(&check2, Z2); + + std::vector<MultiexpData> multiexp_data; + multiexp_data.reserve(2 * maxMN); + for (size_t i = 0; i < maxMN; ++i) + { + sc_sub(tmp.bytes, rct::zero().bytes, z4[i].bytes); + multiexp_data.emplace_back(tmp, Gi_p3[i]); + sc_sub(tmp.bytes, rct::zero().bytes, z5[i].bytes); + multiexp_data.emplace_back(tmp, Hi_p3[i]); + } + add_acc_p3(&check2, multiexp(multiexp_data, true)); PERF_TIMER_STOP(VERIFY_step2_check); - if (!(pprime == tmp)) + + if (!ge_p3_is_point_at_infinity(&check2)) { MERROR("Verification failure at step 2"); return false; @@ -749,4 +1247,19 @@ bool bulletproof_VERIFY(const Bulletproof &proof) return true; } +bool bulletproof_VERIFY(const std::vector<Bulletproof> &proofs) +{ + std::vector<const Bulletproof*> proof_pointers; + for (const Bulletproof &proof: proofs) + proof_pointers.push_back(&proof); + return bulletproof_VERIFY(proof_pointers); +} + +bool bulletproof_VERIFY(const Bulletproof &proof) +{ + std::vector<const Bulletproof*> proofs; + proofs.push_back(&proof); + return bulletproof_VERIFY(proofs); +} + } diff --git a/src/ringct/bulletproofs.h b/src/ringct/bulletproofs.h index 3061d272e..b86202ccc 100644 --- a/src/ringct/bulletproofs.h +++ b/src/ringct/bulletproofs.h @@ -40,7 +40,11 @@ namespace rct Bulletproof bulletproof_PROVE(const rct::key &v, const rct::key &gamma); Bulletproof bulletproof_PROVE(uint64_t v, const rct::key &gamma); +Bulletproof bulletproof_PROVE(const rct::keyV &v, const rct::keyV &gamma); +Bulletproof bulletproof_PROVE(const std::vector<uint64_t> &v, const rct::keyV &gamma); bool bulletproof_VERIFY(const Bulletproof &proof); +bool bulletproof_VERIFY(const std::vector<const Bulletproof*> &proofs); +bool bulletproof_VERIFY(const std::vector<Bulletproof> &proofs); } diff --git a/src/ringct/multiexp.cc b/src/ringct/multiexp.cc new file mode 100644 index 000000000..21957b94c --- /dev/null +++ b/src/ringct/multiexp.cc @@ -0,0 +1,665 @@ +// Copyright (c) 2017, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Adapted from Python code by Sarang Noether + +#include "misc_log_ex.h" +#include "common/perf_timer.h" +extern "C" +{ +#include "crypto/crypto-ops.h" +} +#include "common/aligned.h" +#include "rctOps.h" +#include "multiexp.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "multiexp" + +//#define MULTIEXP_PERF(x) x +#define MULTIEXP_PERF(x) + +#define RAW_MEMORY_BLOCK +//#define ALTERNATE_LAYOUT +//#define TRACK_STRAUS_ZERO_IDENTITY + +// per points us for N/B points (B point bands) +// raw alt 128/192 4096/192 4096/4096 +// 0 0 52.6 71 71.2 +// 0 1 53.2 72.2 72.4 +// 1 0 52.7 67 67.1 +// 1 1 52.8 70.4 70.2 + +// Pippenger: +// 1 2 3 4 5 6 7 8 9 bestN +// 2 555 598 621 804 1038 1733 2486 5020 8304 1 +// 4 783 747 800 1006 1428 2132 3285 5185 9806 2 +// 8 1174 1071 1095 1286 1640 2398 3869 6378 12080 2 +// 16 2279 1874 1745 1739 2144 2831 4209 6964 12007 4 +// 32 3910 3706 2588 2477 2782 3467 4856 7489 12618 4 +// 64 7184 5429 4710 4368 4010 4672 6027 8559 13684 5 +// 128 14097 10574 8452 7297 6841 6718 8615 10580 15641 6 +// 256 27715 20800 16000 13550 11875 11400 11505 14090 18460 6 +// 512 55100 41250 31740 26570 22030 19830 20760 21380 25215 6 +// 1024 111520 79000 61080 49720 43080 38320 37600 35040 36750 8 +// 2048 219480 162680 122120 102080 83760 70360 66600 63920 66160 8 +// 4096 453320 323080 247240 210200 180040 150240 132440 114920 110560 9 + +// 2 4 8 16 32 64 128 256 512 1024 2048 4096 +// Bos Coster 858 994 1316 1949 3183 5512 9865 17830 33485 63160 124280 246320 +// Straus 226 341 548 980 1870 3538 7039 14490 29020 57200 118640 233640 +// Straus/cached 226 315 485 785 1514 2858 5753 11065 22970 45120 98880 194840 +// Pippenger 555 747 1071 1739 2477 4010 6718 11400 19830 35040 63920 110560 + +// Best/cached Straus Straus Straus Straus Straus Straus Straus Straus Pip Pip Pip Pip +// Best/uncached Straus Straus Straus Straus Straus Straus Pip Pip Pip Pip Pip Pip + +namespace rct +{ + +static inline bool operator<(const rct::key &k0, const rct::key&k1) +{ + for (int n = 31; n >= 0; --n) + { + if (k0.bytes[n] < k1.bytes[n]) + return true; + if (k0.bytes[n] > k1.bytes[n]) + return false; + } + return false; +} + +static inline rct::key div2(const rct::key &k) +{ + rct::key res; + int carry = 0; + for (int n = 31; n >= 0; --n) + { + int new_carry = (k.bytes[n] & 1) << 7; + res.bytes[n] = k.bytes[n] / 2 + carry; + carry = new_carry; + } + return res; +} + +static inline rct::key pow2(size_t n) +{ + CHECK_AND_ASSERT_THROW_MES(n < 256, "Invalid pow2 argument"); + rct::key res = rct::zero(); + res[n >> 3] |= 1<<(n&7); + return res; +} + +static inline int test(const rct::key &k, size_t n) +{ + if (n >= 256) return 0; + return k[n >> 3] & (1 << (n & 7)); +} + +static inline void add(ge_p3 &p3, const ge_cached &other) +{ + ge_p1p1 p1; + ge_add(&p1, &p3, &other); + ge_p1p1_to_p3(&p3, &p1); +} + +static inline void add(ge_p3 &p3, const ge_p3 &other) +{ + ge_cached cached; + ge_p3_to_cached(&cached, &other); + add(p3, cached); +} + +rct::key bos_coster_heap_conv(std::vector<MultiexpData> data) +{ + MULTIEXP_PERF(PERF_TIMER_START_UNIT(bos_coster, 1000000)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(setup, 1000000)); + size_t points = data.size(); + CHECK_AND_ASSERT_THROW_MES(points > 1, "Not enough points"); + std::vector<size_t> heap(points); + for (size_t n = 0; n < points; ++n) + heap[n] = n; + + auto Comp = [&](size_t e0, size_t e1) { return data[e0].scalar < data[e1].scalar; }; + std::make_heap(heap.begin(), heap.end(), Comp); + MULTIEXP_PERF(PERF_TIMER_STOP(setup)); + + MULTIEXP_PERF(PERF_TIMER_START_UNIT(loop, 1000000)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(pop, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(pop)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(add, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(add)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(sub, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(sub)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(push, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(push)); + while (heap.size() > 1) + { + MULTIEXP_PERF(PERF_TIMER_RESUME(pop)); + std::pop_heap(heap.begin(), heap.end(), Comp); + size_t index1 = heap.back(); + heap.pop_back(); + std::pop_heap(heap.begin(), heap.end(), Comp); + size_t index2 = heap.back(); + heap.pop_back(); + MULTIEXP_PERF(PERF_TIMER_PAUSE(pop)); + + MULTIEXP_PERF(PERF_TIMER_RESUME(add)); + ge_cached cached; + ge_p3_to_cached(&cached, &data[index1].point); + ge_p1p1 p1; + ge_add(&p1, &data[index2].point, &cached); + ge_p1p1_to_p3(&data[index2].point, &p1); + MULTIEXP_PERF(PERF_TIMER_PAUSE(add)); + + MULTIEXP_PERF(PERF_TIMER_RESUME(sub)); + sc_sub(data[index1].scalar.bytes, data[index1].scalar.bytes, data[index2].scalar.bytes); + MULTIEXP_PERF(PERF_TIMER_PAUSE(sub)); + + MULTIEXP_PERF(PERF_TIMER_RESUME(push)); + if (!(data[index1].scalar == rct::zero())) + { + heap.push_back(index1); + std::push_heap(heap.begin(), heap.end(), Comp); + } + + heap.push_back(index2); + std::push_heap(heap.begin(), heap.end(), Comp); + MULTIEXP_PERF(PERF_TIMER_PAUSE(push)); + } + MULTIEXP_PERF(PERF_TIMER_STOP(push)); + MULTIEXP_PERF(PERF_TIMER_STOP(sub)); + MULTIEXP_PERF(PERF_TIMER_STOP(add)); + MULTIEXP_PERF(PERF_TIMER_STOP(pop)); + MULTIEXP_PERF(PERF_TIMER_STOP(loop)); + + MULTIEXP_PERF(PERF_TIMER_START_UNIT(end, 1000000)); + //return rct::scalarmultKey(data[index1].point, data[index1].scalar); + std::pop_heap(heap.begin(), heap.end(), Comp); + size_t index1 = heap.back(); + heap.pop_back(); + ge_p2 p2; + ge_scalarmult(&p2, data[index1].scalar.bytes, &data[index1].point); + rct::key res; + ge_tobytes(res.bytes, &p2); + return res; +} + +rct::key bos_coster_heap_conv_robust(std::vector<MultiexpData> data) +{ + MULTIEXP_PERF(PERF_TIMER_START_UNIT(bos_coster, 1000000)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(setup, 1000000)); + size_t points = data.size(); + CHECK_AND_ASSERT_THROW_MES(points > 0, "Not enough points"); + std::vector<size_t> heap; + heap.reserve(points); + for (size_t n = 0; n < points; ++n) + { + if (!(data[n].scalar == rct::zero()) && !ge_p3_is_point_at_infinity(&data[n].point)) + heap.push_back(n); + } + points = heap.size(); + if (points == 0) + return rct::identity(); + + auto Comp = [&](size_t e0, size_t e1) { return data[e0].scalar < data[e1].scalar; }; + std::make_heap(heap.begin(), heap.end(), Comp); + + if (points < 2) + { + std::pop_heap(heap.begin(), heap.end(), Comp); + size_t index1 = heap.back(); + ge_p2 p2; + ge_scalarmult(&p2, data[index1].scalar.bytes, &data[index1].point); + rct::key res; + ge_tobytes(res.bytes, &p2); + return res; + } + + MULTIEXP_PERF(PERF_TIMER_STOP(setup)); + + MULTIEXP_PERF(PERF_TIMER_START_UNIT(loop, 1000000)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(pop, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(pop)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(div, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(div)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(add, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(add)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(sub, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(sub)); + MULTIEXP_PERF(PERF_TIMER_START_UNIT(push, 1000000)); MULTIEXP_PERF(PERF_TIMER_PAUSE(push)); + while (heap.size() > 1) + { + MULTIEXP_PERF(PERF_TIMER_RESUME(pop)); + std::pop_heap(heap.begin(), heap.end(), Comp); + size_t index1 = heap.back(); + heap.pop_back(); + std::pop_heap(heap.begin(), heap.end(), Comp); + size_t index2 = heap.back(); + heap.pop_back(); + MULTIEXP_PERF(PERF_TIMER_PAUSE(pop)); + + ge_cached cached; + ge_p1p1 p1; + ge_p2 p2; + + MULTIEXP_PERF(PERF_TIMER_RESUME(div)); + while (1) + { + rct::key s1_2 = div2(data[index1].scalar); + if (!(data[index2].scalar < s1_2)) + break; + if (data[index1].scalar.bytes[0] & 1) + { + data.resize(data.size()+1); + data.back().scalar = rct::identity(); + data.back().point = data[index1].point; + heap.push_back(data.size() - 1); + std::push_heap(heap.begin(), heap.end(), Comp); + } + data[index1].scalar = div2(data[index1].scalar); + ge_p3_to_p2(&p2, &data[index1].point); + ge_p2_dbl(&p1, &p2); + ge_p1p1_to_p3(&data[index1].point, &p1); + } + MULTIEXP_PERF(PERF_TIMER_PAUSE(div)); + + MULTIEXP_PERF(PERF_TIMER_RESUME(add)); + ge_p3_to_cached(&cached, &data[index1].point); + ge_add(&p1, &data[index2].point, &cached); + ge_p1p1_to_p3(&data[index2].point, &p1); + MULTIEXP_PERF(PERF_TIMER_PAUSE(add)); + + MULTIEXP_PERF(PERF_TIMER_RESUME(sub)); + sc_sub(data[index1].scalar.bytes, data[index1].scalar.bytes, data[index2].scalar.bytes); + MULTIEXP_PERF(PERF_TIMER_PAUSE(sub)); + + MULTIEXP_PERF(PERF_TIMER_RESUME(push)); + if (!(data[index1].scalar == rct::zero())) + { + heap.push_back(index1); + std::push_heap(heap.begin(), heap.end(), Comp); + } + + heap.push_back(index2); + std::push_heap(heap.begin(), heap.end(), Comp); + MULTIEXP_PERF(PERF_TIMER_PAUSE(push)); + } + MULTIEXP_PERF(PERF_TIMER_STOP(push)); + MULTIEXP_PERF(PERF_TIMER_STOP(sub)); + MULTIEXP_PERF(PERF_TIMER_STOP(add)); + MULTIEXP_PERF(PERF_TIMER_STOP(pop)); + MULTIEXP_PERF(PERF_TIMER_STOP(loop)); + + MULTIEXP_PERF(PERF_TIMER_START_UNIT(end, 1000000)); + //return rct::scalarmultKey(data[index1].point, data[index1].scalar); + std::pop_heap(heap.begin(), heap.end(), Comp); + size_t index1 = heap.back(); + heap.pop_back(); + ge_p2 p2; + ge_scalarmult(&p2, data[index1].scalar.bytes, &data[index1].point); + rct::key res; + ge_tobytes(res.bytes, &p2); + return res; +} + +static constexpr unsigned int STRAUS_C = 4; + +struct straus_cached_data +{ +#ifdef RAW_MEMORY_BLOCK + size_t size; + ge_cached *multiples; + straus_cached_data(): size(0), multiples(NULL) {} + ~straus_cached_data() { aligned_free(multiples); } +#else + std::vector<std::vector<ge_cached>> multiples; +#endif +}; +#ifdef RAW_MEMORY_BLOCK +#ifdef ALTERNATE_LAYOUT +#define CACHE_OFFSET(cache,point,digit) cache->multiples[(point)*((1<<STRAUS_C)-1)+((digit)-1)] +#else +#define CACHE_OFFSET(cache,point,digit) cache->multiples[(point)+cache->size*((digit)-1)] +#endif +#else +#ifdef ALTERNATE_LAYOUT +#define CACHE_OFFSET(cache,point,digit) local_cache->multiples[j][digit-1] +#else +#define CACHE_OFFSET(cache,point,digit) local_cache->multiples[digit][j] +#endif +#endif + +std::shared_ptr<straus_cached_data> straus_init_cache(const std::vector<MultiexpData> &data, size_t N) +{ + MULTIEXP_PERF(PERF_TIMER_START_UNIT(multiples, 1000000)); + if (N == 0) + N = data.size(); + CHECK_AND_ASSERT_THROW_MES(N <= data.size(), "Bad cache base data"); + ge_cached cached; + ge_p1p1 p1; + ge_p3 p3; + std::shared_ptr<straus_cached_data> cache(new straus_cached_data()); + +#ifdef RAW_MEMORY_BLOCK + const size_t offset = cache->size; + cache->multiples = (ge_cached*)aligned_realloc(cache->multiples, sizeof(ge_cached) * ((1<<STRAUS_C)-1) * std::max(offset, N), 4096); + CHECK_AND_ASSERT_THROW_MES(cache->multiples, "Out of memory"); + cache->size = N; + for (size_t j=offset;j<N;++j) + { + ge_p3_to_cached(&CACHE_OFFSET(cache, j, 1), &data[j].point); + for (size_t i=2;i<1<<STRAUS_C;++i) + { + ge_add(&p1, &data[j].point, &CACHE_OFFSET(cache, j, i-1)); + ge_p1p1_to_p3(&p3, &p1); + ge_p3_to_cached(&CACHE_OFFSET(cache, j, i), &p3); + } + } +#else +#ifdef ALTERNATE_LAYOUT + const size_t offset = cache->multiples.size(); + cache->multiples.resize(std::max(offset, N)); + for (size_t i = offset; i < N; ++i) + { + cache->multiples[i].resize((1<<STRAUS_C)-1); + ge_p3_to_cached(&cache->multiples[i][0], &data[i].point); + for (size_t j=2;j<1<<STRAUS_C;++j) + { + ge_add(&p1, &data[i].point, &cache->multiples[i][j-2]); + ge_p1p1_to_p3(&p3, &p1); + ge_p3_to_cached(&cache->multiples[i][j-1], &p3); + } + } +#else + cache->multiples.resize(1<<STRAUS_C); + size_t offset = cache->multiples[1].size(); + cache->multiples[1].resize(std::max(offset, N)); + for (size_t i = offset; i < N; ++i) + ge_p3_to_cached(&cache->multiples[1][i], &data[i].point); + for (size_t i=2;i<1<<STRAUS_C;++i) + cache->multiples[i].resize(std::max(offset, N)); + for (size_t j=offset;j<N;++j) + { + for (size_t i=2;i<1<<STRAUS_C;++i) + { + ge_add(&p1, &data[j].point, &cache->multiples[i-1][j]); + ge_p1p1_to_p3(&p3, &p1); + ge_p3_to_cached(&cache->multiples[i][j], &p3); + } + } +#endif +#endif + MULTIEXP_PERF(PERF_TIMER_STOP(multiples)); + + return cache; +} + +size_t straus_get_cache_size(const std::shared_ptr<straus_cached_data> &cache) +{ + size_t sz = 0; +#ifdef RAW_MEMORY_BLOCK + sz += cache->size * sizeof(ge_cached) * ((1<<STRAUS_C)-1); +#else + for (const auto &e0: cache->multiples) + sz += e0.size() * sizeof(ge_cached); +#endif + return sz; +} + +rct::key straus(const std::vector<MultiexpData> &data, const std::shared_ptr<straus_cached_data> &cache, size_t STEP) +{ + CHECK_AND_ASSERT_THROW_MES(cache == NULL || cache->size >= data.size(), "Cache is too small"); + MULTIEXP_PERF(PERF_TIMER_UNIT(straus, 1000000)); + bool HiGi = cache != NULL; + STEP = STEP ? STEP : 192; + + MULTIEXP_PERF(PERF_TIMER_START_UNIT(setup, 1000000)); + static constexpr unsigned int mask = (1<<STRAUS_C)-1; + std::shared_ptr<straus_cached_data> local_cache = cache == NULL ? straus_init_cache(data) : cache; + ge_cached cached; + ge_p1p1 p1; + ge_p3 p3; + +#ifdef TRACK_STRAUS_ZERO_IDENTITY + MULTIEXP_PERF(PERF_TIMER_START_UNIT(skip, 1000000)); + std::vector<uint8_t> skip(data.size()); + for (size_t i = 0; i < data.size(); ++i) + skip[i] = data[i].scalar == rct::zero() || ge_p3_is_point_at_infinity(&data[i].point); + MULTIEXP_PERF(PERF_TIMER_STOP(skip)); +#endif + + MULTIEXP_PERF(PERF_TIMER_START_UNIT(digits, 1000000)); + std::unique_ptr<uint8_t[]> digits{new uint8_t[256 * data.size()]}; + for (size_t j = 0; j < data.size(); ++j) + { + unsigned char bytes33[33]; + memcpy(bytes33, data[j].scalar.bytes, 32); + bytes33[32] = 0; + const unsigned char *bytes = bytes33; +#if 1 + static_assert(STRAUS_C == 4, "optimized version needs STRAUS_C == 4"); + unsigned int i; + for (i = 0; i < 256; i += 8, bytes++) + { + digits[j*256+i] = bytes[0] & 0xf; + digits[j*256+i+1] = (bytes[0] >> 1) & 0xf; + digits[j*256+i+2] = (bytes[0] >> 2) & 0xf; + digits[j*256+i+3] = (bytes[0] >> 3) & 0xf; + digits[j*256+i+4] = ((bytes[0] >> 4) | (bytes[1]<<4)) & 0xf; + digits[j*256+i+5] = ((bytes[0] >> 5) | (bytes[1]<<3)) & 0xf; + digits[j*256+i+6] = ((bytes[0] >> 6) | (bytes[1]<<2)) & 0xf; + digits[j*256+i+7] = ((bytes[0] >> 7) | (bytes[1]<<1)) & 0xf; + } +#elif 1 + for (size_t i = 0; i < 256; ++i) + digits[j*256+i] = ((bytes[i>>3] | (bytes[(i>>3)+1]<<8)) >> (i&7)) & mask; +#else + rct::key shifted = data[j].scalar; + for (size_t i = 0; i < 256; ++i) + { + digits[j*256+i] = shifted.bytes[0] & 0xf; + shifted = div2(shifted, (256-i)>>3); + } +#endif + } + MULTIEXP_PERF(PERF_TIMER_STOP(digits)); + + rct::key maxscalar = rct::zero(); + for (size_t i = 0; i < data.size(); ++i) + if (maxscalar < data[i].scalar) + maxscalar = data[i].scalar; + size_t start_i = 0; + while (start_i < 256 && !(maxscalar < pow2(start_i))) + start_i += STRAUS_C; + MULTIEXP_PERF(PERF_TIMER_STOP(setup)); + + ge_p3 res_p3 = ge_p3_identity; + + for (size_t start_offset = 0; start_offset < data.size(); start_offset += STEP) + { + const size_t num_points = std::min(data.size() - start_offset, STEP); + + ge_p3 band_p3 = ge_p3_identity; + size_t i = start_i; + if (!(i < STRAUS_C)) + goto skipfirst; + while (!(i < STRAUS_C)) + { + ge_p2 p2; + ge_p3_to_p2(&p2, &band_p3); + for (size_t j = 0; j < STRAUS_C; ++j) + { + ge_p2_dbl(&p1, &p2); + if (j == STRAUS_C - 1) + ge_p1p1_to_p3(&band_p3, &p1); + else + ge_p1p1_to_p2(&p2, &p1); + } +skipfirst: + i -= STRAUS_C; + for (size_t j = start_offset; j < start_offset + num_points; ++j) + { +#ifdef TRACK_STRAUS_ZERO_IDENTITY + if (skip[j]) + continue; +#endif + const uint8_t digit = digits[j*256+i]; + if (digit) + { + ge_add(&p1, &band_p3, &CACHE_OFFSET(local_cache, j, digit)); + ge_p1p1_to_p3(&band_p3, &p1); + } + } + } + + ge_p3_to_cached(&cached, &band_p3); + ge_add(&p1, &res_p3, &cached); + ge_p1p1_to_p3(&res_p3, &p1); + } + + rct::key res; + ge_p3_tobytes(res.bytes, &res_p3); + return res; +} + +size_t get_pippenger_c(size_t N) +{ +// uncached: 2:1, 4:2, 8:2, 16:3, 32:4, 64:4, 128:5, 256:6, 512:7, 1024:7, 2048:8, 4096:9 +// cached: 2:1, 4:2, 8:2, 16:3, 32:4, 64:4, 128:5, 256:6, 512:7, 1024:7, 2048:8, 4096:9 + if (N <= 2) return 1; + if (N <= 8) return 2; + if (N <= 16) return 3; + if (N <= 64) return 4; + if (N <= 128) return 5; + if (N <= 256) return 6; + if (N <= 1024) return 7; + if (N <= 2048) return 8; + return 9; +} + +struct pippenger_cached_data +{ + size_t size; + ge_cached *cached; + pippenger_cached_data(): size(0), cached(NULL) {} + ~pippenger_cached_data() { aligned_free(cached); } +}; + +std::shared_ptr<pippenger_cached_data> pippenger_init_cache(const std::vector<MultiexpData> &data, size_t N) +{ + MULTIEXP_PERF(PERF_TIMER_START_UNIT(pippenger_init_cache, 1000000)); + if (N == 0) + N = data.size(); + CHECK_AND_ASSERT_THROW_MES(N <= data.size(), "Bad cache base data"); + ge_cached cached; + std::shared_ptr<pippenger_cached_data> cache(new pippenger_cached_data()); + + cache->size = N; + cache->cached = (ge_cached*)aligned_realloc(cache->cached, N * sizeof(ge_cached), 4096); + CHECK_AND_ASSERT_THROW_MES(cache->cached, "Out of memory"); + for (size_t i = 0; i < N; ++i) + ge_p3_to_cached(&cache->cached[i], &data[i].point); + + MULTIEXP_PERF(PERF_TIMER_STOP(pippenger_init_cache)); + return cache; +} + +size_t pippenger_get_cache_size(const std::shared_ptr<pippenger_cached_data> &cache) +{ + return cache->size * sizeof(*cache->cached); +} + +rct::key pippenger(const std::vector<MultiexpData> &data, const std::shared_ptr<pippenger_cached_data> &cache, size_t c) +{ + CHECK_AND_ASSERT_THROW_MES(cache == NULL || cache->size >= data.size(), "Cache is too small"); + if (c == 0) + c = get_pippenger_c(data.size()); + CHECK_AND_ASSERT_THROW_MES(c <= 9, "c is too large"); + + ge_p3 result = ge_p3_identity; + std::unique_ptr<ge_p3[]> buckets{new ge_p3[1<<c]}; + std::shared_ptr<pippenger_cached_data> local_cache = cache == NULL ? pippenger_init_cache(data) : cache; + + rct::key maxscalar = rct::zero(); + for (size_t i = 0; i < data.size(); ++i) + { + if (maxscalar < data[i].scalar) + maxscalar = data[i].scalar; + } + size_t groups = 0; + while (groups < 256 && !(maxscalar < pow2(groups))) + ++groups; + groups = (groups + c - 1) / c; + + for (size_t k = groups; k-- > 0; ) + { + if (!ge_p3_is_point_at_infinity(&result)) + { + ge_p2 p2; + ge_p3_to_p2(&p2, &result); + for (size_t i = 0; i < c; ++i) + { + ge_p1p1 p1; + ge_p2_dbl(&p1, &p2); + if (i == c - 1) + ge_p1p1_to_p3(&result, &p1); + else + ge_p1p1_to_p2(&p2, &p1); + } + } + for (size_t i = 0; i < (1u<<c); ++i) + buckets[i] = ge_p3_identity; + + // partition scalars into buckets + for (size_t i = 0; i < data.size(); ++i) + { + unsigned int bucket = 0; + for (size_t j = 0; j < c; ++j) + if (test(data[i].scalar, k*c+j)) + bucket |= 1<<j; + if (bucket == 0) + continue; + CHECK_AND_ASSERT_THROW_MES(bucket < (1u<<c), "bucket overflow"); + if (!ge_p3_is_point_at_infinity(&buckets[bucket])) + { + add(buckets[bucket], local_cache->cached[i]); + } + else + buckets[bucket] = data[i].point; + } + + // sum the buckets + ge_p3 pail = ge_p3_identity; + for (size_t i = (1<<c)-1; i > 0; --i) + { + if (!ge_p3_is_point_at_infinity(&buckets[i])) + add(pail, buckets[i]); + if (!ge_p3_is_point_at_infinity(&pail)) + add(result, pail); + } + } + + rct::key res; + ge_p3_tobytes(res.bytes, &result); + return res; +} + +} diff --git a/src/ringct/multiexp.h b/src/ringct/multiexp.h new file mode 100644 index 000000000..559ab664a --- /dev/null +++ b/src/ringct/multiexp.h @@ -0,0 +1,71 @@ +// Copyright (c) 2017, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Adapted from Python code by Sarang Noether + +#pragma once + +#ifndef MULTIEXP_H +#define MULTIEXP_H + +#include <vector> +#include "crypto/crypto.h" +#include "rctTypes.h" +#include "misc_log_ex.h" + +namespace rct +{ + +struct MultiexpData { + rct::key scalar; + ge_p3 point; + + MultiexpData() {} + MultiexpData(const rct::key &s, const ge_p3 &p): scalar(s), point(p) {} + MultiexpData(const rct::key &s, const rct::key &p): scalar(s) + { + CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&point, p.bytes) == 0, "ge_frombytes_vartime failed"); + } +}; + +struct straus_cached_data; +struct pippenger_cached_data; + +rct::key bos_coster_heap_conv(std::vector<MultiexpData> data); +rct::key bos_coster_heap_conv_robust(std::vector<MultiexpData> data); +std::shared_ptr<straus_cached_data> straus_init_cache(const std::vector<MultiexpData> &data, size_t N =0); +size_t straus_get_cache_size(const std::shared_ptr<straus_cached_data> &cache); +rct::key straus(const std::vector<MultiexpData> &data, const std::shared_ptr<straus_cached_data> &cache = NULL, size_t STEP = 0); +std::shared_ptr<pippenger_cached_data> pippenger_init_cache(const std::vector<MultiexpData> &data, size_t N =0); +size_t pippenger_get_cache_size(const std::shared_ptr<pippenger_cached_data> &cache); +size_t get_pippenger_c(size_t N); +rct::key pippenger(const std::vector<MultiexpData> &data, const std::shared_ptr<pippenger_cached_data> &cache = NULL, size_t c = 0); + +} + +#endif diff --git a/src/ringct/rctOps.cpp b/src/ringct/rctOps.cpp index 50693bad7..6c3c4500e 100644 --- a/src/ringct/rctOps.cpp +++ b/src/ringct/rctOps.cpp @@ -60,6 +60,17 @@ namespace rct { //Various key generation functions + bool toPointCheckOrder(ge_p3 *P, const unsigned char *data) + { + if (ge_frombytes_vartime(P, data)) + return false; + ge_p2 R; + ge_scalarmult(&R, curveOrder().bytes, P); + key tmp; + ge_tobytes(tmp.bytes, &R); + return tmp == identity(); + } + //generates a random scalar which can be used as a secret key or mask void skGen(key &sk) { random32_unbiased(sk.bytes); @@ -193,15 +204,33 @@ namespace rct { //Computes aH where H= toPoint(cn_fast_hash(G)), G the basepoint key scalarmultH(const key & a) { - ge_p3 A; ge_p2 R; - CHECK_AND_ASSERT_THROW_MES_L1(ge_frombytes_vartime(&A, H.bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast<std::string>(__LINE__)); - ge_scalarmult(&R, a.bytes, &A); + ge_scalarmult(&R, a.bytes, &ge_p3_H); key aP; ge_tobytes(aP.bytes, &R); return aP; } + //Computes 8P + key scalarmult8(const key & P) { + ge_p3 p3; + CHECK_AND_ASSERT_THROW_MES_L1(ge_frombytes_vartime(&p3, P.bytes) == 0, "ge_frombytes_vartime failed at "+boost::lexical_cast<std::string>(__LINE__)); + ge_p2 p2; + ge_p3_to_p2(&p2, &p3); + ge_p1p1 p1; + ge_mul8(&p1, &p2); + ge_p1p1_to_p2(&p2, &p1); + rct::key res; + ge_tobytes(res.bytes, &p2); + return res; + } + + //Computes aL where L is the curve order + bool isInMainSubgroup(const key & a) { + ge_p3 p3; + return toPointCheckOrder(&p3, a.bytes); + } + //Curve addition / subtractions //for curve points: AB = A + B diff --git a/src/ringct/rctOps.h b/src/ringct/rctOps.h index f8889af5c..50645821c 100644 --- a/src/ringct/rctOps.h +++ b/src/ringct/rctOps.h @@ -63,6 +63,8 @@ namespace rct { static const key I = { {0x01, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 } }; static const key L = { {0xed, 0xd3, 0xf5, 0x5c, 0x1a, 0x63, 0x12, 0x58, 0xd6, 0x9c, 0xf7, 0xa2, 0xde, 0xf9, 0xde, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10 } }; static const key G = { {0x58, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66 } }; + static const key EIGHT = { {0x08, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 } }; + static const key INV_EIGHT = { { 0x79, 0x2f, 0xdc, 0xe2, 0x29, 0xe5, 0x06, 0x61, 0xd0, 0xda, 0x1c, 0x7d, 0xb3, 0x9d, 0xd3, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06 } }; //Creates a zero scalar inline key zero() { return Z; } @@ -83,6 +85,7 @@ namespace rct { keyM keyMInit(size_t rows, size_t cols); //Various key generation functions + bool toPointCheckOrder(ge_p3 *P, const unsigned char *data); //generates a random scalar which can be used as a secret key or mask key skGen(); @@ -119,6 +122,10 @@ namespace rct { key scalarmultKey(const key &P, const key &a); //Computes aH where H= toPoint(cn_fast_hash(G)), G the basepoint key scalarmultH(const key & a); + // multiplies a point by 8 + key scalarmult8(const key & P); + // checks a is in the main subgroup (ie, not a small one) + bool isInMainSubgroup(const key & a); //Curve addition / subtractions diff --git a/src/ringct/rctSigs.cpp b/src/ringct/rctSigs.cpp index f74216ed4..fe0cd9c57 100644 --- a/src/ringct/rctSigs.cpp +++ b/src/ringct/rctSigs.cpp @@ -45,30 +45,6 @@ using namespace std; #define CHECK_AND_ASSERT_MES_L1(expr, ret, message) {if(!(expr)) {MCERROR("verify", message); return ret;}} namespace rct { - bool is_simple(int type) - { - switch (type) - { - case RCTTypeSimple: - case RCTTypeSimpleBulletproof: - return true; - default: - return false; - } - } - - bool is_bulletproof(int type) - { - switch (type) - { - case RCTTypeSimpleBulletproof: - case RCTTypeFullBulletproof: - return true; - default: - return false; - } - } - Bulletproof proveRangeBulletproof(key &C, key &mask, uint64_t amount) { mask = rct::skGen(); @@ -78,6 +54,15 @@ namespace rct { return proof; } + Bulletproof proveRangeBulletproof(keyV &C, keyV &masks, const std::vector<uint64_t> &amounts) + { + masks = rct::skvGen(amounts.size()); + Bulletproof proof = bulletproof_PROVE(amounts, masks); + CHECK_AND_ASSERT_THROW_MES(proof.V.size() == amounts.size(), "V does not have the expected size"); + C = proof.V; + return proof; + } + bool verBulletproof(const Bulletproof &proof) { try { return bulletproof_VERIFY(proof); } @@ -85,6 +70,13 @@ namespace rct { catch (...) { return false; } } + bool verBulletproof(const std::vector<const Bulletproof*> &proofs) + { + try { return bulletproof_VERIFY(proofs); } + // we can get deep throws from ge_frombytes_vartime if input isn't valid + catch (...) { return false; } + } + //Borromean (c.f. gmax/andytoshi's paper) boroSig genBorromean(const key64 x, const key64 P1, const key64 P2, const bits indices) { key64 L[2], alpha; @@ -285,6 +277,7 @@ namespace rct { for (j = 0; j < dsRows; j++) { addKeys2(L, rv.ss[i][j], c_old, pk[i][j]); hashToPoint(Hi, pk[i][j]); + CHECK_AND_ASSERT_MES(!(Hi == rct::identity()), false, "Data hashed to point at infinity"); addKeys3(R, rv.ss[i][j], Hi, c_old, Ip[j].k); toHash[3 * j + 1] = pk[i][j]; toHash[3 * j + 2] = L; @@ -389,7 +382,7 @@ namespace rct { std::stringstream ss; binary_archive<true> ba(ss); CHECK_AND_ASSERT_THROW_MES(!rv.mixRing.empty(), "Empty mixRing"); - const size_t inputs = is_simple(rv.type) ? rv.mixRing.size() : rv.mixRing[0].size(); + const size_t inputs = is_rct_simple(rv.type) ? rv.mixRing.size() : rv.mixRing[0].size(); const size_t outputs = rv.ecdhInfo.size(); key prehash; CHECK_AND_ASSERT_THROW_MES(const_cast<rctSig&>(rv).serialize_rctsig_base(ba, inputs, outputs), @@ -398,7 +391,7 @@ namespace rct { hashes.push_back(hash2rct(h)); keyV kv; - if (rv.type == RCTTypeSimpleBulletproof || rv.type == RCTTypeFullBulletproof) + if (rv.type == RCTTypeBulletproof) { kv.reserve((6*2+9) * rv.p.bulletproofs.size()); for (const auto &p: rv.p.bulletproofs) @@ -492,7 +485,9 @@ namespace rct { for (size_t j = 0; j < outPk.size(); j++) { sc_sub(sk[rows].bytes, sk[rows].bytes, outSk[j].mask.bytes); //subtract output masks in last row.. } - return MLSAG_Gen(message, M, sk, kLRki, mscout, index, rows, hwdev); + mgSig result = MLSAG_Gen(message, M, sk, kLRki, mscout, index, rows, hwdev); + memwipe(sk.data(), sk.size() * sizeof(key)); + return result; } @@ -521,7 +516,9 @@ namespace rct { M[i][0] = pubs[i].dest; subKeys(M[i][1], pubs[i].mask, Cout); } - return MLSAG_Gen(message, M, sk, kLRki, mscout, index, rows, hwdev); + mgSig result = MLSAG_Gen(message, M, sk, kLRki, mscout, index, rows, hwdev); + memwipe(&sk[0], sizeof(key)); + return result; } @@ -655,7 +652,7 @@ namespace rct { // must know the destination private key to find the correct amount, else will return a random number // Note: For txn fees, the last index in the amounts vector should contain that // Thus the amounts vector will be "one" longer than the destinations vectort - rctSig genRct(const key &message, const ctkeyV & inSk, const keyV & destinations, const vector<xmr_amount> & amounts, const ctkeyM &mixRing, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, unsigned int index, ctkeyV &outSk, bool bulletproof, hw::device &hwdev) { + rctSig genRct(const key &message, const ctkeyV & inSk, const keyV & destinations, const vector<xmr_amount> & amounts, const ctkeyM &mixRing, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, unsigned int index, ctkeyV &outSk, hw::device &hwdev) { CHECK_AND_ASSERT_THROW_MES(amounts.size() == destinations.size() || amounts.size() == destinations.size() + 1, "Different number of amounts/destinations"); CHECK_AND_ASSERT_THROW_MES(amount_keys.size() == destinations.size(), "Different number of amount_keys/destinations"); CHECK_AND_ASSERT_THROW_MES(index < mixRing.size(), "Bad index into mixRing"); @@ -665,13 +662,10 @@ namespace rct { CHECK_AND_ASSERT_THROW_MES((kLRki && msout) || (!kLRki && !msout), "Only one of kLRki/msout is present"); rctSig rv; - rv.type = bulletproof ? RCTTypeFullBulletproof : RCTTypeFull; + rv.type = RCTTypeFull; rv.message = message; rv.outPk.resize(destinations.size()); - if (bulletproof) - rv.p.bulletproofs.resize(destinations.size()); - else - rv.p.rangeSigs.resize(destinations.size()); + rv.p.rangeSigs.resize(destinations.size()); rv.ecdhInfo.resize(destinations.size()); size_t i = 0; @@ -681,17 +675,10 @@ namespace rct { //add destination to sig rv.outPk[i].dest = copy(destinations[i]); //compute range proof - if (bulletproof) - rv.p.bulletproofs[i] = proveRangeBulletproof(rv.outPk[i].mask, outSk[i].mask, amounts[i]); - else - rv.p.rangeSigs[i] = proveRange(rv.outPk[i].mask, outSk[i].mask, amounts[i]); + rv.p.rangeSigs[i] = proveRange(rv.outPk[i].mask, outSk[i].mask, amounts[i]); #ifdef DBG - if (bulletproof) - CHECK_AND_ASSERT_THROW_MES(verBulletproof(rv.p.bulletproofs[i]), "verBulletproof failed on newly created proof"); - else - CHECK_AND_ASSERT_THROW_MES(verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]), "verRange failed on newly created proof"); + CHECK_AND_ASSERT_THROW_MES(verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]), "verRange failed on newly created proof"); #endif - //mask amount and mask rv.ecdhInfo[i].mask = copy(outSk[i].mask); rv.ecdhInfo[i].amount = d2h(amounts[i]); @@ -721,12 +708,13 @@ namespace rct { ctkeyM mixRing; ctkeyV outSk; tie(mixRing, index) = populateFromBlockchain(inPk, mixin); - return genRct(message, inSk, destinations, amounts, mixRing, amount_keys, kLRki, msout, index, outSk, false, hwdev); + return genRct(message, inSk, destinations, amounts, mixRing, amount_keys, kLRki, msout, index, outSk, hwdev); } //RCT simple //for post-rct only - rctSig genRctSimple(const key &message, const ctkeyV & inSk, const keyV & destinations, const vector<xmr_amount> &inamounts, const vector<xmr_amount> &outamounts, xmr_amount txnFee, const ctkeyM & mixRing, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, const std::vector<unsigned int> & index, ctkeyV &outSk, bool bulletproof, hw::device &hwdev) { + rctSig genRctSimple(const key &message, const ctkeyV & inSk, const keyV & destinations, const vector<xmr_amount> &inamounts, const vector<xmr_amount> &outamounts, xmr_amount txnFee, const ctkeyM & mixRing, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, const std::vector<unsigned int> & index, ctkeyV &outSk, RangeProofType range_proof_type, hw::device &hwdev) { + const bool bulletproof = range_proof_type != RangeProofBorromean; CHECK_AND_ASSERT_THROW_MES(inamounts.size() > 0, "Empty inamounts"); CHECK_AND_ASSERT_THROW_MES(inamounts.size() == inSk.size(), "Different number of inamounts/inSk"); CHECK_AND_ASSERT_THROW_MES(outamounts.size() == destinations.size(), "Different number of amounts/destinations"); @@ -742,35 +730,74 @@ namespace rct { } rctSig rv; - rv.type = bulletproof ? RCTTypeSimpleBulletproof : RCTTypeSimple; + rv.type = bulletproof ? RCTTypeBulletproof : RCTTypeSimple; rv.message = message; rv.outPk.resize(destinations.size()); - if (bulletproof) - rv.p.bulletproofs.resize(destinations.size()); - else + if (!bulletproof) rv.p.rangeSigs.resize(destinations.size()); rv.ecdhInfo.resize(destinations.size()); size_t i; keyV masks(destinations.size()); //sk mask.. outSk.resize(destinations.size()); - key sumout = zero(); for (i = 0; i < destinations.size(); i++) { //add destination to sig rv.outPk[i].dest = copy(destinations[i]); //compute range proof - if (bulletproof) - rv.p.bulletproofs[i] = proveRangeBulletproof(rv.outPk[i].mask, outSk[i].mask, outamounts[i]); - else + if (!bulletproof) rv.p.rangeSigs[i] = proveRange(rv.outPk[i].mask, outSk[i].mask, outamounts[i]); #ifdef DBG - if (bulletproof) - CHECK_AND_ASSERT_THROW_MES(verBulletproof(rv.p.bulletproofs[i]), "verBulletproof failed on newly created proof"); - else + if (!bulletproof) CHECK_AND_ASSERT_THROW_MES(verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]), "verRange failed on newly created proof"); #endif - + } + + rv.p.bulletproofs.clear(); + if (bulletproof) + { + std::vector<uint64_t> proof_amounts; + size_t n_amounts = outamounts.size(); + size_t amounts_proved = 0; + if (range_proof_type == RangeProofPaddedBulletproof) + { + rct::keyV C, masks; + rv.p.bulletproofs.push_back(proveRangeBulletproof(C, masks, outamounts)); + #ifdef DBG + CHECK_AND_ASSERT_THROW_MES(verBulletproof(rv.p.bulletproofs.back()), "verBulletproof failed on newly created proof"); + #endif + for (i = 0; i < outamounts.size(); ++i) + { + rv.outPk[i].mask = rct::scalarmult8(C[i]); + outSk[i].mask = masks[i]; + } + } + else while (amounts_proved < n_amounts) + { + size_t batch_size = 1; + if (range_proof_type == RangeProofMultiOutputBulletproof) + while (batch_size * 2 + amounts_proved <= n_amounts && batch_size * 2 <= BULLETPROOF_MAX_OUTPUTS) + batch_size *= 2; + rct::keyV C, masks; + std::vector<uint64_t> batch_amounts(batch_size); + for (i = 0; i < batch_size; ++i) + batch_amounts[i] = outamounts[i + amounts_proved]; + rv.p.bulletproofs.push_back(proveRangeBulletproof(C, masks, batch_amounts)); + #ifdef DBG + CHECK_AND_ASSERT_THROW_MES(verBulletproof(rv.p.bulletproofs.back()), "verBulletproof failed on newly created proof"); + #endif + for (i = 0; i < batch_size; ++i) + { + rv.outPk[i + amounts_proved].mask = rct::scalarmult8(C[i]); + outSk[i + amounts_proved].mask = masks[i]; + } + amounts_proved += batch_size; + } + } + + key sumout = zero(); + for (i = 0; i < outSk.size(); ++i) + { sc_add(sumout.bytes, outSk[i].mask.bytes, sumout.bytes); //mask amount and mask @@ -818,7 +845,7 @@ namespace rct { mixRing[i].resize(mixin+1); index[i] = populateFromBlockchainSimple(mixRing[i], inPk[i], mixin); } - return genRctSimple(message, inSk, destinations, inamounts, outamounts, txnFee, mixRing, amount_keys, kLRki, msout, index, outSk, false, hwdev); + return genRctSimple(message, inSk, destinations, inamounts, outamounts, txnFee, mixRing, amount_keys, kLRki, msout, index, outSk, RangeProofBorromean, hwdev); } //RingCT protocol @@ -833,13 +860,10 @@ namespace rct { // must know the destination private key to find the correct amount, else will return a random number bool verRct(const rctSig & rv, bool semantics) { PERF_TIMER(verRct); - CHECK_AND_ASSERT_MES(rv.type == RCTTypeFull || rv.type == RCTTypeFullBulletproof, false, "verRct called on non-full rctSig"); + CHECK_AND_ASSERT_MES(rv.type == RCTTypeFull, false, "verRct called on non-full rctSig"); if (semantics) { - if (rv.type == RCTTypeFullBulletproof) - CHECK_AND_ASSERT_MES(rv.outPk.size() == rv.p.bulletproofs.size(), false, "Mismatched sizes of outPk and rv.p.bulletproofs"); - else - CHECK_AND_ASSERT_MES(rv.outPk.size() == rv.p.rangeSigs.size(), false, "Mismatched sizes of outPk and rv.p.rangeSigs"); + CHECK_AND_ASSERT_MES(rv.outPk.size() == rv.p.rangeSigs.size(), false, "Mismatched sizes of outPk and rv.p.rangeSigs"); CHECK_AND_ASSERT_MES(rv.outPk.size() == rv.ecdhInfo.size(), false, "Mismatched sizes of outPk and rv.ecdhInfo"); CHECK_AND_ASSERT_MES(rv.p.MGs.size() == 1, false, "full rctSig has not one MG"); } @@ -856,19 +880,13 @@ namespace rct { tools::threadpool::waiter waiter; std::deque<bool> results(rv.outPk.size(), false); DP("range proofs verified?"); - for (size_t i = 0; i < rv.outPk.size(); i++) { - tpool.submit(&waiter, [&, i] { - if (rv.p.rangeSigs.empty()) - results[i] = verBulletproof(rv.p.bulletproofs[i]); - else - results[i] = verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]); - }, true); - } + for (size_t i = 0; i < rv.outPk.size(); i++) + tpool.submit(&waiter, [&, i] { results[i] = verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]); }); waiter.wait(&tpool); - for (size_t i = 0; i < rv.outPk.size(); ++i) { + for (size_t i = 0; i < results.size(); ++i) { if (!results[i]) { - LOG_PRINT_L1("Range proof verified failed for output " << i); + LOG_PRINT_L1("Range proof verified failed for proof " << i); return false; } } @@ -902,17 +920,26 @@ namespace rct { //ver RingCT simple //assumes only post-rct style inputs (at least for max anonymity) - bool verRctSimple(const rctSig & rv, bool semantics) { + bool verRctSemanticsSimple(const std::vector<const rctSig*> & rvv) { try { - PERF_TIMER(verRctSimple); + PERF_TIMER(verRctSemanticsSimple); - CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeSimpleBulletproof, false, "verRctSimple called on non simple rctSig"); - if (semantics) + tools::threadpool& tpool = tools::threadpool::getInstance(); + tools::threadpool::waiter waiter; + std::deque<bool> results; + std::vector<const Bulletproof*> proofs; + size_t max_non_bp_proofs = 0, offset = 0; + + for (const rctSig *rvp: rvv) { - if (rv.type == RCTTypeSimpleBulletproof) + CHECK_AND_ASSERT_MES(rvp, false, "rctSig pointer is NULL"); + const rctSig &rv = *rvp; + CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof, false, "verRctSemanticsSimple called on non simple rctSig"); + const bool bulletproof = is_rct_bulletproof(rv.type); + if (bulletproof) { - CHECK_AND_ASSERT_MES(rv.outPk.size() == rv.p.bulletproofs.size(), false, "Mismatched sizes of outPk and rv.p.bulletproofs"); + CHECK_AND_ASSERT_MES(rv.outPk.size() == n_bulletproof_amounts(rv.p.bulletproofs), false, "Mismatched sizes of outPk and bulletproofs"); CHECK_AND_ASSERT_MES(rv.p.pseudoOuts.size() == rv.p.MGs.size(), false, "Mismatched sizes of rv.p.pseudoOuts and rv.p.MGs"); CHECK_AND_ASSERT_MES(rv.pseudoOuts.empty(), false, "rv.pseudoOuts is not empty"); } @@ -923,28 +950,22 @@ namespace rct { CHECK_AND_ASSERT_MES(rv.p.pseudoOuts.empty(), false, "rv.p.pseudoOuts is not empty"); } CHECK_AND_ASSERT_MES(rv.outPk.size() == rv.ecdhInfo.size(), false, "Mismatched sizes of outPk and rv.ecdhInfo"); - } - else - { - // semantics check is early, and mixRing/MGs aren't resolved yet - if (rv.type == RCTTypeSimpleBulletproof) - CHECK_AND_ASSERT_MES(rv.p.pseudoOuts.size() == rv.mixRing.size(), false, "Mismatched sizes of rv.p.pseudoOuts and mixRing"); - else - CHECK_AND_ASSERT_MES(rv.pseudoOuts.size() == rv.mixRing.size(), false, "Mismatched sizes of rv.pseudoOuts and mixRing"); - } - const size_t threads = std::max(rv.outPk.size(), rv.mixRing.size()); + if (!bulletproof) + max_non_bp_proofs += rv.p.rangeSigs.size(); + } - std::deque<bool> results(threads); - tools::threadpool& tpool = tools::threadpool::getInstance(); - tools::threadpool::waiter waiter; + results.resize(max_non_bp_proofs); + for (const rctSig *rvp: rvv) + { + const rctSig &rv = *rvp; - const keyV &pseudoOuts = is_bulletproof(rv.type) ? rv.p.pseudoOuts : rv.pseudoOuts; + const bool bulletproof = is_rct_bulletproof(rv.type); + const keyV &pseudoOuts = bulletproof ? rv.p.pseudoOuts : rv.pseudoOuts; - if (semantics) { key sumOutpks = identity(); for (size_t i = 0; i < rv.outPk.size(); i++) { - addKeys(sumOutpks, sumOutpks, rv.outPk[i].mask); + addKeys(sumOutpks, sumOutpks, rv.outPk[i].mask); } DP(sumOutpks); key txnFeeKey = scalarmultH(d2h(rv.txnFee)); @@ -952,52 +973,100 @@ namespace rct { key sumPseudoOuts = identity(); for (size_t i = 0 ; i < pseudoOuts.size() ; i++) { - addKeys(sumPseudoOuts, sumPseudoOuts, pseudoOuts[i]); + addKeys(sumPseudoOuts, sumPseudoOuts, pseudoOuts[i]); } DP(sumPseudoOuts); //check pseudoOuts vs Outs.. if (!equalKeys(sumPseudoOuts, sumOutpks)) { - LOG_PRINT_L1("Sum check failed"); - return false; + LOG_PRINT_L1("Sum check failed"); + return false; } - results.clear(); - results.resize(rv.outPk.size()); - for (size_t i = 0; i < rv.outPk.size(); i++) { - tpool.submit(&waiter, [&, i] { - if (rv.p.rangeSigs.empty()) - results[i] = verBulletproof(rv.p.bulletproofs[i]); - else - results[i] = verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]); - }, true); + if (bulletproof) + { + for (size_t i = 0; i < rv.p.bulletproofs.size(); i++) + proofs.push_back(&rv.p.bulletproofs[i]); } - waiter.wait(&tpool); - - for (size_t i = 0; i < results.size(); ++i) { - if (!results[i]) { - LOG_PRINT_L1("Range proof verified failed for output " << i); - return false; - } + else + { + for (size_t i = 0; i < rv.p.rangeSigs.size(); i++) + tpool.submit(&waiter, [&, i, offset] { results[i+offset] = verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]); }); + offset += rv.p.rangeSigs.size(); } } - else { - const key message = get_pre_mlsag_hash(rv, hw::get_device("default")); - - results.clear(); - results.resize(rv.mixRing.size()); - for (size_t i = 0 ; i < rv.mixRing.size() ; i++) { - tpool.submit(&waiter, [&, i] { - results[i] = verRctMGSimple(message, rv.p.MGs[i], rv.mixRing[i], pseudoOuts[i]); - }, true); + if (!proofs.empty() && !verBulletproof(proofs)) + { + LOG_PRINT_L1("Aggregate range proof verified failed"); + return false; + } + + waiter.wait(&tpool); + for (size_t i = 0; i < results.size(); ++i) { + if (!results[i]) { + LOG_PRINT_L1("Range proof verified failed for proof " << i); + return false; } - waiter.wait(&tpool); + } - for (size_t i = 0; i < results.size(); ++i) { - if (!results[i]) { - LOG_PRINT_L1("verRctMGSimple failed for input " << i); - return false; - } + return true; + } + // we can get deep throws from ge_frombytes_vartime if input isn't valid + catch (const std::exception &e) + { + LOG_PRINT_L1("Error in verRctSemanticsSimple: " << e.what()); + return false; + } + catch (...) + { + LOG_PRINT_L1("Error in verRctSemanticsSimple, but not an actual exception"); + return false; + } + } + + bool verRctSemanticsSimple(const rctSig & rv) + { + return verRctSemanticsSimple(std::vector<const rctSig*>(1, &rv)); + } + + //ver RingCT simple + //assumes only post-rct style inputs (at least for max anonymity) + bool verRctNonSemanticsSimple(const rctSig & rv) { + try + { + PERF_TIMER(verRctNonSemanticsSimple); + + CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof, false, "verRctNonSemanticsSimple called on non simple rctSig"); + const bool bulletproof = is_rct_bulletproof(rv.type); + // semantics check is early, and mixRing/MGs aren't resolved yet + if (bulletproof) + CHECK_AND_ASSERT_MES(rv.p.pseudoOuts.size() == rv.mixRing.size(), false, "Mismatched sizes of rv.p.pseudoOuts and mixRing"); + else + CHECK_AND_ASSERT_MES(rv.pseudoOuts.size() == rv.mixRing.size(), false, "Mismatched sizes of rv.pseudoOuts and mixRing"); + + const size_t threads = std::max(rv.outPk.size(), rv.mixRing.size()); + + std::deque<bool> results(threads); + tools::threadpool& tpool = tools::threadpool::getInstance(); + tools::threadpool::waiter waiter; + + const keyV &pseudoOuts = bulletproof ? rv.p.pseudoOuts : rv.pseudoOuts; + + const key message = get_pre_mlsag_hash(rv, hw::get_device("default")); + + results.clear(); + results.resize(rv.mixRing.size()); + for (size_t i = 0 ; i < rv.mixRing.size() ; i++) { + tpool.submit(&waiter, [&, i] { + results[i] = verRctMGSimple(message, rv.p.MGs[i], rv.mixRing[i], pseudoOuts[i]); + }); + } + waiter.wait(&tpool); + + for (size_t i = 0; i < results.size(); ++i) { + if (!results[i]) { + LOG_PRINT_L1("verRctMGSimple failed for input " << i); + return false; } } @@ -1006,12 +1075,12 @@ namespace rct { // we can get deep throws from ge_frombytes_vartime if input isn't valid catch (const std::exception &e) { - LOG_PRINT_L1("Error in verRct: " << e.what()); + LOG_PRINT_L1("Error in verRctNonSemanticsSimple: " << e.what()); return false; } catch (...) { - LOG_PRINT_L1("Error in verRct, but not an actual exception"); + LOG_PRINT_L1("Error in verRctNonSemanticsSimple, but not an actual exception"); return false; } } @@ -1027,7 +1096,7 @@ namespace rct { // uses the attached ecdh info to find the amounts represented by each output commitment // must know the destination private key to find the correct amount, else will return a random number xmr_amount decodeRct(const rctSig & rv, const key & sk, unsigned int i, key & mask, hw::device &hwdev) { - CHECK_AND_ASSERT_MES(rv.type == RCTTypeFull || rv.type == RCTTypeFullBulletproof, false, "decodeRct called on non-full rctSig"); + CHECK_AND_ASSERT_MES(rv.type == RCTTypeFull, false, "decodeRct called on non-full rctSig"); CHECK_AND_ASSERT_THROW_MES(i < rv.ecdhInfo.size(), "Bad index"); CHECK_AND_ASSERT_THROW_MES(rv.outPk.size() == rv.ecdhInfo.size(), "Mismatched sizes of rv.outPk and rv.ecdhInfo"); @@ -1040,6 +1109,8 @@ namespace rct { DP("C"); DP(C); key Ctmp; + CHECK_AND_ASSERT_THROW_MES(sc_check(mask.bytes) == 0, "warning, bad ECDH mask"); + CHECK_AND_ASSERT_THROW_MES(sc_check(amount.bytes) == 0, "warning, bad ECDH amount"); addKeys2(Ctmp, mask, amount, H); DP("Ctmp"); DP(Ctmp); @@ -1055,7 +1126,7 @@ namespace rct { } xmr_amount decodeRctSimple(const rctSig & rv, const key & sk, unsigned int i, key &mask, hw::device &hwdev) { - CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeSimpleBulletproof, false, "decodeRct called on non simple rctSig"); + CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof, false, "decodeRct called on non simple rctSig"); CHECK_AND_ASSERT_THROW_MES(i < rv.ecdhInfo.size(), "Bad index"); CHECK_AND_ASSERT_THROW_MES(rv.outPk.size() == rv.ecdhInfo.size(), "Mismatched sizes of rv.outPk and rv.ecdhInfo"); @@ -1068,6 +1139,8 @@ namespace rct { DP("C"); DP(C); key Ctmp; + CHECK_AND_ASSERT_THROW_MES(sc_check(mask.bytes) == 0, "warning, bad ECDH mask"); + CHECK_AND_ASSERT_THROW_MES(sc_check(amount.bytes) == 0, "warning, bad ECDH amount"); addKeys2(Ctmp, mask, amount, H); DP("Ctmp"); DP(Ctmp); @@ -1083,12 +1156,12 @@ namespace rct { } bool signMultisig(rctSig &rv, const std::vector<unsigned int> &indices, const keyV &k, const multisig_out &msout, const key &secret_key) { - CHECK_AND_ASSERT_MES(rv.type == RCTTypeFull || rv.type == RCTTypeSimple || rv.type == RCTTypeFullBulletproof || rv.type == RCTTypeSimpleBulletproof, + CHECK_AND_ASSERT_MES(rv.type == RCTTypeFull || rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof, false, "unsupported rct type"); CHECK_AND_ASSERT_MES(indices.size() == k.size(), false, "Mismatched k/indices sizes"); CHECK_AND_ASSERT_MES(k.size() == rv.p.MGs.size(), false, "Mismatched k/MGs size"); CHECK_AND_ASSERT_MES(k.size() == msout.c.size(), false, "Mismatched k/msout.c size"); - if (rv.type == RCTTypeFull || rv.type == RCTTypeFullBulletproof) + if (rv.type == RCTTypeFull) { CHECK_AND_ASSERT_MES(rv.p.MGs.size() == 1, false, "MGs not a single element"); } diff --git a/src/ringct/rctSigs.h b/src/ringct/rctSigs.h index 5a9b2dd44..ae8bb91d7 100644 --- a/src/ringct/rctSigs.h +++ b/src/ringct/rctSigs.h @@ -119,14 +119,16 @@ namespace rct { //decodeRct: (c.f. https://eprint.iacr.org/2015/1098 section 5.1.1) // uses the attached ecdh info to find the amounts represented by each output commitment // must know the destination private key to find the correct amount, else will return a random number - rctSig genRct(const key &message, const ctkeyV & inSk, const keyV & destinations, const std::vector<xmr_amount> & amounts, const ctkeyM &mixRing, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, unsigned int index, ctkeyV &outSk, bool bulletproof, hw::device &hwdev); + rctSig genRct(const key &message, const ctkeyV & inSk, const keyV & destinations, const std::vector<xmr_amount> & amounts, const ctkeyM &mixRing, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, unsigned int index, ctkeyV &outSk, hw::device &hwdev); rctSig genRct(const key &message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const std::vector<xmr_amount> & amounts, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, const int mixin, hw::device &hwdev); rctSig genRctSimple(const key & message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const std::vector<xmr_amount> & inamounts, const std::vector<xmr_amount> & outamounts, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, xmr_amount txnFee, unsigned int mixin, hw::device &hwdev); - rctSig genRctSimple(const key & message, const ctkeyV & inSk, const keyV & destinations, const std::vector<xmr_amount> & inamounts, const std::vector<xmr_amount> & outamounts, xmr_amount txnFee, const ctkeyM & mixRing, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, const std::vector<unsigned int> & index, ctkeyV &outSk, bool bulletproof, hw::device &hwdev); + rctSig genRctSimple(const key & message, const ctkeyV & inSk, const keyV & destinations, const std::vector<xmr_amount> & inamounts, const std::vector<xmr_amount> & outamounts, xmr_amount txnFee, const ctkeyM & mixRing, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, const std::vector<unsigned int> & index, ctkeyV &outSk, RangeProofType range_proof_type, hw::device &hwdev); bool verRct(const rctSig & rv, bool semantics); static inline bool verRct(const rctSig & rv) { return verRct(rv, true) && verRct(rv, false); } - bool verRctSimple(const rctSig & rv, bool semantics); - static inline bool verRctSimple(const rctSig & rv) { return verRctSimple(rv, true) && verRctSimple(rv, false); } + bool verRctSemanticsSimple(const rctSig & rv); + bool verRctSemanticsSimple(const std::vector<const rctSig*> & rv); + bool verRctNonSemanticsSimple(const rctSig & rv); + static inline bool verRctSimple(const rctSig & rv) { return verRctSemanticsSimple(rv) && verRctNonSemanticsSimple(rv); } xmr_amount decodeRct(const rctSig & rv, const key & sk, unsigned int i, key & mask, hw::device &hwdev); xmr_amount decodeRct(const rctSig & rv, const key & sk, unsigned int i, hw::device &hwdev); xmr_amount decodeRctSimple(const rctSig & rv, const key & sk, unsigned int i, key & mask, hw::device &hwdev); diff --git a/src/ringct/rctTypes.cpp b/src/ringct/rctTypes.cpp index 5650b3ba1..90ed65df0 100644 --- a/src/ringct/rctTypes.cpp +++ b/src/ringct/rctTypes.cpp @@ -28,6 +28,8 @@ // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include "misc_log_ex.h" +#include "cryptonote_config.h" #include "rctTypes.h" using namespace crypto; using namespace std; @@ -209,4 +211,90 @@ namespace rct { return vali; } + bool is_rct_simple(int type) + { + switch (type) + { + case RCTTypeSimple: + case RCTTypeBulletproof: + return true; + default: + return false; + } + } + + bool is_rct_bulletproof(int type) + { + switch (type) + { + case RCTTypeBulletproof: + return true; + default: + return false; + } + } + + bool is_rct_borromean(int type) + { + switch (type) + { + case RCTTypeSimple: + case RCTTypeFull: + return true; + default: + return false; + } + } + + size_t n_bulletproof_amounts(const Bulletproof &proof) + { + CHECK_AND_ASSERT_MES(proof.L.size() >= 6, 0, "Invalid bulletproof L size"); + CHECK_AND_ASSERT_MES(proof.L.size() == proof.R.size(), 0, "Mismatched bulletproof L/R size"); + static const size_t extra_bits = 4; + static_assert((1 << extra_bits) == BULLETPROOF_MAX_OUTPUTS, "log2(BULLETPROOF_MAX_OUTPUTS) is out of date"); + CHECK_AND_ASSERT_MES(proof.L.size() <= 6 + extra_bits, 0, "Invalid bulletproof L size"); + CHECK_AND_ASSERT_MES(proof.V.size() <= (1u<<(proof.L.size()-6)), 0, "Invalid bulletproof V/L"); + CHECK_AND_ASSERT_MES(proof.V.size() * 2 > (1u<<(proof.L.size()-6)), 0, "Invalid bulletproof V/L"); + CHECK_AND_ASSERT_MES(proof.V.size() > 0, 0, "Empty bulletproof"); + return proof.V.size(); + } + + size_t n_bulletproof_amounts(const std::vector<Bulletproof> &proofs) + { + size_t n = 0; + for (const Bulletproof &proof: proofs) + { + size_t n2 = n_bulletproof_amounts(proof); + CHECK_AND_ASSERT_MES(n2 < std::numeric_limits<uint32_t>::max() - n, 0, "Invalid number of bulletproofs"); + if (n2 == 0) + return 0; + n += n2; + } + return n; + } + + size_t n_bulletproof_max_amounts(const Bulletproof &proof) + { + CHECK_AND_ASSERT_MES(proof.L.size() >= 6, 0, "Invalid bulletproof L size"); + CHECK_AND_ASSERT_MES(proof.L.size() == proof.R.size(), 0, "Mismatched bulletproof L/R size"); + static const size_t extra_bits = 4; + static_assert((1 << extra_bits) == BULLETPROOF_MAX_OUTPUTS, "log2(BULLETPROOF_MAX_OUTPUTS) is out of date"); + CHECK_AND_ASSERT_MES(proof.L.size() <= 6 + extra_bits, 0, "Invalid bulletproof L size"); + return 1 << (proof.L.size() - 6); + } + + size_t n_bulletproof_max_amounts(const std::vector<Bulletproof> &proofs) + { + size_t n = 0; + for (const Bulletproof &proof: proofs) + { + size_t n2 = n_bulletproof_max_amounts(proof); + CHECK_AND_ASSERT_MES(n2 < std::numeric_limits<uint32_t>::max() - n, 0, "Invalid number of bulletproofs"); + if (n2 == 0) + return 0; + n += n2; + } + return n; + } + } diff --git a/src/ringct/rctTypes.h b/src/ringct/rctTypes.h index 452a68eb2..ffc4df3ed 100644 --- a/src/ringct/rctTypes.h +++ b/src/ringct/rctTypes.h @@ -190,6 +190,8 @@ namespace rct { Bulletproof() {} Bulletproof(const rct::key &V, const rct::key &A, const rct::key &S, const rct::key &T1, const rct::key &T2, const rct::key &taux, const rct::key &mu, const rct::keyV &L, const rct::keyV &R, const rct::key &a, const rct::key &b, const rct::key &t): V({V}), A(A), S(S), T1(T1), T2(T2), taux(taux), mu(mu), L(L), R(R), a(a), b(b), t(t) {} + Bulletproof(const rct::keyV &V, const rct::key &A, const rct::key &S, const rct::key &T1, const rct::key &T2, const rct::key &taux, const rct::key &mu, const rct::keyV &L, const rct::keyV &R, const rct::key &a, const rct::key &b, const rct::key &t): + V(V), A(A), S(S), T1(T1), T2(T2), taux(taux), mu(mu), L(L), R(R), a(a), b(b), t(t) {} BEGIN_SERIALIZE_OBJECT() // Commitments aren't saved, they're restored via outPk @@ -211,6 +213,11 @@ namespace rct { END_SERIALIZE() }; + size_t n_bulletproof_amounts(const Bulletproof &proof); + size_t n_bulletproof_max_amounts(const Bulletproof &proof); + size_t n_bulletproof_amounts(const std::vector<Bulletproof> &proofs); + size_t n_bulletproof_max_amounts(const std::vector<Bulletproof> &proofs); + //A container to hold all signatures necessary for RingCT // rangeSigs holds all the rangeproof data of a transaction // MG holds the MLSAG signature of a transaction @@ -222,9 +229,9 @@ namespace rct { RCTTypeNull = 0, RCTTypeFull = 1, RCTTypeSimple = 2, - RCTTypeFullBulletproof = 3, - RCTTypeSimpleBulletproof = 4, + RCTTypeBulletproof = 3, }; + enum RangeProofType { RangeProofBorromean, RangeProofBulletproof, RangeProofMultiOutputBulletproof, RangeProofPaddedBulletproof }; struct rctSigBase { uint8_t type; key message; @@ -241,7 +248,7 @@ namespace rct { FIELD(type) if (type == RCTTypeNull) return true; - if (type != RCTTypeFull && type != RCTTypeFullBulletproof && type != RCTTypeSimple && type != RCTTypeSimpleBulletproof) + if (type != RCTTypeFull && type != RCTTypeSimple && type != RCTTypeBulletproof) return false; VARINT_FIELD(txnFee) // inputs/outputs not saved, only here for serialization help @@ -302,21 +309,25 @@ namespace rct { { if (type == RCTTypeNull) return true; - if (type != RCTTypeFull && type != RCTTypeFullBulletproof && type != RCTTypeSimple && type != RCTTypeSimpleBulletproof) + if (type != RCTTypeFull && type != RCTTypeSimple && type != RCTTypeBulletproof) return false; - if (type == RCTTypeSimpleBulletproof || type == RCTTypeFullBulletproof) + if (type == RCTTypeBulletproof) { ar.tag("bp"); ar.begin_array(); - PREPARE_CUSTOM_VECTOR_SERIALIZATION(outputs, bulletproofs); - if (bulletproofs.size() != outputs) + uint32_t nbp = bulletproofs.size(); + FIELD(nbp) + if (nbp > outputs) return false; - for (size_t i = 0; i < outputs; ++i) + PREPARE_CUSTOM_VECTOR_SERIALIZATION(nbp, bulletproofs); + for (size_t i = 0; i < nbp; ++i) { FIELDS(bulletproofs[i]) - if (outputs - i > 1) + if (nbp - i > 1) ar.delimit_array(); } + if (n_bulletproof_max_amounts(bulletproofs) < outputs) + return false; ar.end_array(); } else @@ -339,7 +350,7 @@ namespace rct { ar.begin_array(); // we keep a byte for size of MGs, because we don't know whether this is // a simple or full rct signature, and it's starting to annoy the hell out of me - size_t mg_elements = (type == RCTTypeSimple || type == RCTTypeSimpleBulletproof) ? inputs : 1; + size_t mg_elements = (type == RCTTypeSimple || type == RCTTypeBulletproof) ? inputs : 1; PREPARE_CUSTOM_VECTOR_SERIALIZATION(mg_elements, MGs); if (MGs.size() != mg_elements) return false; @@ -357,7 +368,7 @@ namespace rct { for (size_t j = 0; j < mixin + 1; ++j) { ar.begin_array(); - size_t mg_ss2_elements = ((type == RCTTypeSimple || type == RCTTypeSimpleBulletproof) ? 1 : inputs) + 1; + size_t mg_ss2_elements = ((type == RCTTypeSimple || type == RCTTypeBulletproof) ? 1 : inputs) + 1; PREPARE_CUSTOM_VECTOR_SERIALIZATION(mg_ss2_elements, MGs[i].ss[j]); if (MGs[i].ss[j].size() != mg_ss2_elements) return false; @@ -383,7 +394,7 @@ namespace rct { ar.delimit_array(); } ar.end_array(); - if (type == RCTTypeSimpleBulletproof) + if (type == RCTTypeBulletproof) { ar.tag("pseudoOuts"); ar.begin_array(); @@ -407,12 +418,12 @@ namespace rct { keyV& get_pseudo_outs() { - return type == RCTTypeSimpleBulletproof ? p.pseudoOuts : pseudoOuts; + return type == RCTTypeBulletproof ? p.pseudoOuts : pseudoOuts; } keyV const& get_pseudo_outs() const { - return type == RCTTypeSimpleBulletproof ? p.pseudoOuts : pseudoOuts; + return type == RCTTypeBulletproof ? p.pseudoOuts : pseudoOuts; } }; @@ -517,14 +528,18 @@ namespace rct { //int[64] to uint long long xmr_amount b2d(bits amountb); - static inline const rct::key pk2rct(const crypto::public_key &pk) { return (const rct::key&)pk; } - static inline const rct::key sk2rct(const crypto::secret_key &sk) { return (const rct::key&)sk; } - static inline const rct::key ki2rct(const crypto::key_image &ki) { return (const rct::key&)ki; } - static inline const rct::key hash2rct(const crypto::hash &h) { return (const rct::key&)h; } - static inline const crypto::public_key rct2pk(const rct::key &k) { return (const crypto::public_key&)k; } - static inline const crypto::secret_key rct2sk(const rct::key &k) { return (const crypto::secret_key&)k; } - static inline const crypto::key_image rct2ki(const rct::key &k) { return (const crypto::key_image&)k; } - static inline const crypto::hash rct2hash(const rct::key &k) { return (const crypto::hash&)k; } + bool is_rct_simple(int type); + bool is_rct_bulletproof(int type); + bool is_rct_borromean(int type); + + static inline const rct::key &pk2rct(const crypto::public_key &pk) { return (const rct::key&)pk; } + static inline const rct::key &sk2rct(const crypto::secret_key &sk) { return (const rct::key&)sk; } + static inline const rct::key &ki2rct(const crypto::key_image &ki) { return (const rct::key&)ki; } + static inline const rct::key &hash2rct(const crypto::hash &h) { return (const rct::key&)h; } + static inline const crypto::public_key &rct2pk(const rct::key &k) { return (const crypto::public_key&)k; } + static inline const crypto::secret_key &rct2sk(const rct::key &k) { return (const crypto::secret_key&)k; } + static inline const crypto::key_image &rct2ki(const rct::key &k) { return (const crypto::key_image&)k; } + static inline const crypto::hash &rct2hash(const rct::key &k) { return (const crypto::hash&)k; } static inline bool operator==(const rct::key &k0, const crypto::public_key &k1) { return !crypto_verify_32(k0.bytes, (const unsigned char*)&k1); } static inline bool operator!=(const rct::key &k0, const crypto::public_key &k1) { return crypto_verify_32(k0.bytes, (const unsigned char*)&k1); } } diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 0a6daf8f0..db7f2dbaa 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -195,8 +195,8 @@ namespace cryptonote res.stagenet = m_nettype == STAGENET; res.nettype = m_nettype == MAINNET ? "mainnet" : m_nettype == TESTNET ? "testnet" : m_nettype == STAGENET ? "stagenet" : "fakechain"; res.cumulative_difficulty = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.height - 1); - res.block_size_limit = m_core.get_blockchain_storage().get_current_cumulative_blocksize_limit(); - res.block_size_median = m_core.get_blockchain_storage().get_current_cumulative_blocksize_median(); + res.block_size_limit = res.block_weight_limit = m_core.get_blockchain_storage().get_current_cumulative_block_weight_limit(); + res.block_size_median = res.block_weight_median = m_core.get_blockchain_storage().get_current_cumulative_block_weight_median(); res.status = CORE_RPC_STATUS_OK; res.start_time = (uint64_t)m_core.get_start_time(); res.free_space = m_restricted ? std::numeric_limits<uint64_t>::max() : m_core.get_free_space(); @@ -364,49 +364,6 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_random_outs(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) - { - PERF_TIMER(on_get_random_outs); - bool r; - if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS>(invoke_http_mode::BIN, "/getrandom_outs.bin", req, res, r)) - return r; - - res.status = "Failed"; - - if (m_restricted) - { - if (req.amounts.size() > 100 || req.outs_count > MAX_RESTRICTED_FAKE_OUTS_COUNT) - { - res.status = "Too many outs requested"; - return true; - } - } - - if(!m_core.get_random_outs_for_amounts(req, res)) - { - return true; - } - - res.status = CORE_RPC_STATUS_OK; - std::stringstream ss; - typedef COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount outs_for_amount; - typedef COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry out_entry; - std::for_each(res.outs.begin(), res.outs.end(), [&](outs_for_amount& ofa) - { - ss << "[" << ofa.amount << "]:"; - CHECK_AND_ASSERT_MES(ofa.outs.size(), ;, "internal error: ofa.outs.size() is empty for amount " << ofa.amount); - std::for_each(ofa.outs.begin(), ofa.outs.end(), [&](out_entry& oe) - { - ss << oe.global_amount_index << " "; - }); - ss << ENDL; - }); - std::string s = ss.str(); - LOG_PRINT_L2("COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS: " << ENDL << s); - res.status = CORE_RPC_STATUS_OK; - return true; - } - //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_get_outs_bin(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res) { PERF_TIMER(on_get_outs_bin); @@ -476,34 +433,6 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_random_rct_outs(const COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::request& req, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::response& res) - { - PERF_TIMER(on_get_random_rct_outs); - bool r; - if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS>(invoke_http_mode::BIN, "/getrandom_rctouts.bin", req, res, r)) - return r; - - res.status = "Failed"; - if(!m_core.get_random_rct_outs(req, res)) - { - return true; - } - - res.status = CORE_RPC_STATUS_OK; - std::stringstream ss; - typedef COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::out_entry out_entry; - CHECK_AND_ASSERT_MES(res.outs.size(), true, "internal error: res.outs.size() is empty"); - std::for_each(res.outs.begin(), res.outs.end(), [&](out_entry& oe) - { - ss << oe.global_amount_index << " "; - }); - ss << ENDL; - std::string s = ss.str(); - LOG_PRINT_L2("COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS: " << ENDL << s); - res.status = CORE_RPC_STATUS_OK; - return true; - } - //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_get_indexes(const COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::request& req, COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::response& res) { PERF_TIMER(on_get_indexes); @@ -1313,7 +1242,7 @@ namespace cryptonote response.hash = string_tools::pod_to_hex(hash); response.difficulty = m_core.get_blockchain_storage().block_difficulty(height); response.reward = get_block_reward(blk); - response.block_size = m_core.get_blockchain_storage().get_db().get_block_size(height); + response.block_size = response.block_weight = m_core.get_blockchain_storage().get_db().get_block_weight(height); response.num_txes = blk.tx_hashes.size(); response.pow_hash = fill_pow_hash ? string_tools::pod_to_hex(get_block_longhash(blk, height)) : ""; return true; @@ -1646,8 +1575,8 @@ namespace cryptonote res.stagenet = m_nettype == STAGENET; res.nettype = m_nettype == MAINNET ? "mainnet" : m_nettype == TESTNET ? "testnet" : m_nettype == STAGENET ? "stagenet" : "fakechain"; res.cumulative_difficulty = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.height - 1); - res.block_size_limit = m_core.get_blockchain_storage().get_current_cumulative_blocksize_limit(); - res.block_size_median = m_core.get_blockchain_storage().get_current_cumulative_blocksize_median(); + res.block_size_limit = res.block_weight_limit = m_core.get_blockchain_storage().get_current_cumulative_block_weight_limit(); + res.block_size_median = res.block_weight_median = m_core.get_blockchain_storage().get_current_cumulative_block_weight_median(); res.status = CORE_RPC_STATUS_OK; res.start_time = (uint64_t)m_core.get_start_time(); res.free_space = m_restricted ? std::numeric_limits<uint64_t>::max() : m_core.get_free_space(); @@ -1839,14 +1768,15 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_per_kb_fee_estimate(const COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::request& req, COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::response& res, epee::json_rpc::error& error_resp) + bool core_rpc_server::on_get_base_fee_estimate(const COMMAND_RPC_GET_BASE_FEE_ESTIMATE::request& req, COMMAND_RPC_GET_BASE_FEE_ESTIMATE::response& res, epee::json_rpc::error& error_resp) { - PERF_TIMER(on_get_per_kb_fee_estimate); + PERF_TIMER(on_get_base_fee_estimate); bool r; - if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE>(invoke_http_mode::JON_RPC, "get_fee_estimate", req, res, r)) + if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_BASE_FEE_ESTIMATE>(invoke_http_mode::JON_RPC, "get_fee_estimate", req, res, r)) return r; - res.fee = m_core.get_blockchain_storage().get_dynamic_per_kb_fee_estimate(req.grace_blocks); + res.fee = m_core.get_blockchain_storage().get_dynamic_base_fee_estimate(req.grace_blocks); + res.quantization_mask = Blockchain::get_fee_quantization_mask(); res.status = CORE_RPC_STATUS_OK; return true; } diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index 166020d01..3ba882b23 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -87,11 +87,7 @@ namespace cryptonote MAP_URI_AUTO_BIN2("/get_hashes.bin", on_get_hashes, COMMAND_RPC_GET_HASHES_FAST) MAP_URI_AUTO_BIN2("/gethashes.bin", on_get_hashes, COMMAND_RPC_GET_HASHES_FAST) MAP_URI_AUTO_BIN2("/get_o_indexes.bin", on_get_indexes, COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES) - MAP_URI_AUTO_BIN2("/get_random_outs.bin", on_get_random_outs, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS) - MAP_URI_AUTO_BIN2("/getrandom_outs.bin", on_get_random_outs, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS) MAP_URI_AUTO_BIN2("/get_outs.bin", on_get_outs_bin, COMMAND_RPC_GET_OUTPUTS_BIN) - MAP_URI_AUTO_BIN2("/get_random_rctouts.bin", on_get_random_rct_outs, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS) - MAP_URI_AUTO_BIN2("/getrandom_rctouts.bin", on_get_random_rct_outs, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS) MAP_URI_AUTO_JON2("/get_transactions", on_get_transactions, COMMAND_RPC_GET_TRANSACTIONS) MAP_URI_AUTO_JON2("/gettransactions", on_get_transactions, COMMAND_RPC_GET_TRANSACTIONS) MAP_URI_AUTO_JON2("/get_alt_blocks_hashes", on_get_alt_blocks_hashes, COMMAND_RPC_GET_ALT_BLOCKS_HASHES) @@ -150,7 +146,7 @@ namespace cryptonote MAP_JON_RPC_WE("get_output_histogram", on_get_output_histogram, COMMAND_RPC_GET_OUTPUT_HISTOGRAM) MAP_JON_RPC_WE("get_version", on_get_version, COMMAND_RPC_GET_VERSION) MAP_JON_RPC_WE_IF("get_coinbase_tx_sum", on_get_coinbase_tx_sum, COMMAND_RPC_GET_COINBASE_TX_SUM, !m_restricted) - MAP_JON_RPC_WE("get_fee_estimate", on_get_per_kb_fee_estimate, COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE) + MAP_JON_RPC_WE("get_fee_estimate", on_get_base_fee_estimate, COMMAND_RPC_GET_BASE_FEE_ESTIMATE) MAP_JON_RPC_WE_IF("get_alternate_chains",on_get_alternate_chains, COMMAND_RPC_GET_ALTERNATE_CHAINS, !m_restricted) MAP_JON_RPC_WE_IF("relay_tx", on_relay_tx, COMMAND_RPC_RELAY_TX, !m_restricted) MAP_JON_RPC_WE_IF("sync_info", on_sync_info, COMMAND_RPC_SYNC_INFO, !m_restricted) @@ -171,10 +167,8 @@ namespace cryptonote bool on_start_mining(const COMMAND_RPC_START_MINING::request& req, COMMAND_RPC_START_MINING::response& res); bool on_stop_mining(const COMMAND_RPC_STOP_MINING::request& req, COMMAND_RPC_STOP_MINING::response& res); bool on_mining_status(const COMMAND_RPC_MINING_STATUS::request& req, COMMAND_RPC_MINING_STATUS::response& res); - bool on_get_random_outs(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res); bool on_get_outs_bin(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res); bool on_get_outs(const COMMAND_RPC_GET_OUTPUTS::request& req, COMMAND_RPC_GET_OUTPUTS::response& res); - bool on_get_random_rct_outs(const COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::request& req, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::response& res); bool on_get_info(const COMMAND_RPC_GET_INFO::request& req, COMMAND_RPC_GET_INFO::response& res); bool on_save_bc(const COMMAND_RPC_SAVE_BC::request& req, COMMAND_RPC_SAVE_BC::response& res); bool on_get_peer_list(const COMMAND_RPC_GET_PEER_LIST::request& req, COMMAND_RPC_GET_PEER_LIST::response& res); @@ -214,7 +208,7 @@ namespace cryptonote bool on_get_output_histogram(const COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request& req, COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response& res, epee::json_rpc::error& error_resp); bool on_get_version(const COMMAND_RPC_GET_VERSION::request& req, COMMAND_RPC_GET_VERSION::response& res, epee::json_rpc::error& error_resp); bool on_get_coinbase_tx_sum(const COMMAND_RPC_GET_COINBASE_TX_SUM::request& req, COMMAND_RPC_GET_COINBASE_TX_SUM::response& res, epee::json_rpc::error& error_resp); - bool on_get_per_kb_fee_estimate(const COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::request& req, COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::response& res, epee::json_rpc::error& error_resp); + bool on_get_base_fee_estimate(const COMMAND_RPC_GET_BASE_FEE_ESTIMATE::request& req, COMMAND_RPC_GET_BASE_FEE_ESTIMATE::response& res, epee::json_rpc::error& error_resp); bool on_get_alternate_chains(const COMMAND_RPC_GET_ALTERNATE_CHAINS::request& req, COMMAND_RPC_GET_ALTERNATE_CHAINS::response& res, epee::json_rpc::error& error_resp); bool on_relay_tx(const COMMAND_RPC_RELAY_TX::request& req, COMMAND_RPC_RELAY_TX::response& res, epee::json_rpc::error& error_resp); bool on_sync_info(const COMMAND_RPC_SYNC_INFO::request& req, COMMAND_RPC_SYNC_INFO::response& res, epee::json_rpc::error& error_resp); diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index e2d120cad..1a84ee614 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -48,8 +48,8 @@ namespace cryptonote // whether they can talk to a given daemon without having to know in // advance which version they will stop working with // Don't go over 32767 for any of these -#define CORE_RPC_VERSION_MAJOR 1 -#define CORE_RPC_VERSION_MINOR 21 +#define CORE_RPC_VERSION_MAJOR 2 +#define CORE_RPC_VERSION_MINOR 0 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR) @@ -680,50 +680,6 @@ namespace cryptonote }; }; //----------------------------------------------- - struct COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS - { - struct request - { - std::vector<uint64_t> amounts; - uint64_t outs_count; - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(amounts) - KV_SERIALIZE(outs_count) - END_KV_SERIALIZE_MAP() - }; - -#pragma pack (push, 1) - struct out_entry - { - uint64_t global_amount_index; - crypto::public_key out_key; - }; -#pragma pack(pop) - - struct outs_for_amount - { - uint64_t amount; - std::list<out_entry> outs; - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(amount) - KV_SERIALIZE_CONTAINER_POD_AS_BLOB(outs) - END_KV_SERIALIZE_MAP() - }; - - struct response - { - std::vector<outs_for_amount> outs; - std::string status; - bool untrusted; - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(outs) - KV_SERIALIZE(status) - KV_SERIALIZE(untrusted) - END_KV_SERIALIZE_MAP() - }; - }; - //----------------------------------------------- struct get_outputs_out { uint64_t amount; @@ -818,39 +774,6 @@ namespace cryptonote END_KV_SERIALIZE_MAP() }; }; - - struct COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS - { - struct request - { - uint64_t outs_count; - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(outs_count) - END_KV_SERIALIZE_MAP() - }; - -#pragma pack (push, 1) - struct out_entry - { - uint64_t amount; - uint64_t global_amount_index; - crypto::public_key out_key; - rct::key commitment; - }; -#pragma pack(pop) - - struct response - { - std::list<out_entry> outs; - std::string status; - bool untrusted; - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE_CONTAINER_POD_AS_BLOB(outs) - KV_SERIALIZE(status) - KV_SERIALIZE(untrusted) - END_KV_SERIALIZE_MAP() - }; - }; //----------------------------------------------- struct COMMAND_RPC_SEND_RAW_TX { @@ -959,7 +882,9 @@ namespace cryptonote std::string top_block_hash; uint64_t cumulative_difficulty; uint64_t block_size_limit; + uint64_t block_weight_limit; uint64_t block_size_median; + uint64_t block_weight_median; uint64_t start_time; uint64_t free_space; bool offline; @@ -990,7 +915,9 @@ namespace cryptonote KV_SERIALIZE(top_block_hash) KV_SERIALIZE(cumulative_difficulty) KV_SERIALIZE(block_size_limit) + KV_SERIALIZE_OPT(block_weight_limit, (uint64_t)0) KV_SERIALIZE(block_size_median) + KV_SERIALIZE_OPT(block_weight_median, (uint64_t)0) KV_SERIALIZE(start_time) KV_SERIALIZE(free_space) KV_SERIALIZE(offline) @@ -1195,6 +1122,7 @@ namespace cryptonote difficulty_type difficulty; uint64_t reward; uint64_t block_size; + uint64_t block_weight; uint64_t num_txes; std::string pow_hash; @@ -1211,6 +1139,7 @@ namespace cryptonote KV_SERIALIZE(difficulty) KV_SERIALIZE(reward) KV_SERIALIZE(block_size) + KV_SERIALIZE_OPT(block_weight, (uint64_t)0) KV_SERIALIZE(num_txes) KV_SERIALIZE(pow_hash) END_KV_SERIALIZE_MAP() @@ -1451,6 +1380,7 @@ namespace cryptonote std::string id_hash; std::string tx_json; // TODO - expose this data directly uint64_t blob_size; + uint64_t weight; uint64_t fee; std::string max_used_block_id_hash; uint64_t max_used_block_height; @@ -1468,6 +1398,7 @@ namespace cryptonote KV_SERIALIZE(id_hash) KV_SERIALIZE(tx_json) KV_SERIALIZE(blob_size) + KV_SERIALIZE_OPT(weight, (uint64_t)0) KV_SERIALIZE(fee) KV_SERIALIZE(max_used_block_id_hash) KV_SERIALIZE(max_used_block_height) @@ -1564,7 +1495,7 @@ namespace cryptonote struct tx_backlog_entry { - uint64_t blob_size; + uint64_t weight; uint64_t fee; uint64_t time_in_pool; }; @@ -2102,7 +2033,7 @@ namespace cryptonote }; }; - struct COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE + struct COMMAND_RPC_GET_BASE_FEE_ESTIMATE { struct request { @@ -2117,11 +2048,13 @@ namespace cryptonote { std::string status; uint64_t fee; + uint64_t quantization_mask; bool untrusted; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(status) KV_SERIALIZE(fee) + KV_SERIALIZE_OPT(quantization_mask, (uint64_t)1) KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; diff --git a/src/rpc/daemon_handler.cpp b/src/rpc/daemon_handler.cpp index 25abe4825..9d3b09b68 100644 --- a/src/rpc/daemon_handler.cpp +++ b/src/rpc/daemon_handler.cpp @@ -260,44 +260,6 @@ namespace rpc } - //TODO: handle "restricted" RPC - void DaemonHandler::handle(const GetRandomOutputsForAmounts::Request& req, GetRandomOutputsForAmounts::Response& res) - { - auto& chain = m_core.get_blockchain_storage(); - - try - { - for (const uint64_t& amount : req.amounts) - { - std::vector<uint64_t> indices = chain.get_random_outputs(amount, req.count); - - outputs_for_amount ofa; - - ofa.resize(indices.size()); - - for (size_t i = 0; i < indices.size(); i++) - { - crypto::public_key key = chain.get_output_key(amount, indices[i]); - ofa[i].amount_index = indices[i]; - ofa[i].key = key; - } - - amount_with_random_outputs amt; - amt.amount = amount; - amt.outputs = ofa; - - res.amounts_with_outputs.push_back(amt); - } - - res.status = Message::STATUS_OK; - } - catch (const std::exception& e) - { - res.status = Message::STATUS_FAILED; - res.error_details = e.what(); - } - } - void DaemonHandler::handle(const SendRawTx::Request& req, SendRawTx::Response& res) { auto tx_blob = cryptonote::tx_to_blob(req.tx); @@ -472,7 +434,8 @@ namespace rpc res.info.testnet = m_core.get_nettype() == TESTNET; res.info.stagenet = m_core.get_nettype() == STAGENET; res.info.cumulative_difficulty = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.info.height - 1); - res.info.block_size_limit = m_core.get_blockchain_storage().get_current_cumulative_blocksize_limit(); + res.info.block_size_limit = res.info.block_weight_limit = m_core.get_blockchain_storage().get_current_cumulative_block_weight_limit(); + res.info.block_size_median = res.info.block_weight_median = m_core.get_blockchain_storage().get_current_cumulative_block_weight_median(); res.info.start_time = (uint64_t)m_core.get_start_time(); res.status = Message::STATUS_OK; @@ -763,7 +726,7 @@ namespace rpc void DaemonHandler::handle(const GetPerKBFeeEstimate::Request& req, GetPerKBFeeEstimate::Response& res) { - res.estimated_fee_per_kb = m_core.get_blockchain_storage().get_dynamic_per_kb_fee_estimate(req.num_grace_blocks); + res.estimated_fee_per_kb = m_core.get_blockchain_storage().get_dynamic_base_fee_estimate(req.num_grace_blocks); res.status = Message::STATUS_OK; } @@ -823,7 +786,6 @@ namespace rpc REQ_RESP_TYPES_MACRO(request_type, GetTransactions, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, KeyImagesSpent, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, GetTxGlobalOutputIndices, req_json, resp_message, handle); - REQ_RESP_TYPES_MACRO(request_type, GetRandomOutputsForAmounts, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, SendRawTx, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, GetInfo, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, StartMining, req_json, resp_message, handle); diff --git a/src/rpc/daemon_handler.h b/src/rpc/daemon_handler.h index f43711640..5f9687511 100644 --- a/src/rpc/daemon_handler.h +++ b/src/rpc/daemon_handler.h @@ -66,8 +66,6 @@ class DaemonHandler : public RpcHandler void handle(const GetTxGlobalOutputIndices::Request& req, GetTxGlobalOutputIndices::Response& res); - void handle(const GetRandomOutputsForAmounts::Request& req, GetRandomOutputsForAmounts::Response& res); - void handle(const SendRawTx::Request& req, SendRawTx::Response& res); void handle(const StartMining::Request& req, StartMining::Response& res); diff --git a/src/rpc/daemon_messages.cpp b/src/rpc/daemon_messages.cpp index e5fb9781c..56f6f6a8c 100644 --- a/src/rpc/daemon_messages.cpp +++ b/src/rpc/daemon_messages.cpp @@ -41,7 +41,6 @@ const char* const GetHashesFast::name = "get_hashes_fast"; const char* const GetTransactions::name = "get_transactions"; const char* const KeyImagesSpent::name = "key_images_spent"; const char* const GetTxGlobalOutputIndices::name = "get_tx_global_output_indices"; -const char* const GetRandomOutputsForAmounts::name = "get_random_outputs_for_amounts"; const char* const SendRawTx::name = "send_raw_tx"; const char* const StartMining::name = "start_mining"; const char* const StopMining::name = "stop_mining"; @@ -273,42 +272,6 @@ void GetTxGlobalOutputIndices::Response::fromJson(rapidjson::Value& val) GET_FROM_JSON_OBJECT(val, output_indices, output_indices); } - -rapidjson::Value GetRandomOutputsForAmounts::Request::toJson(rapidjson::Document& doc) const -{ - auto val = Message::toJson(doc); - - auto& al = doc.GetAllocator(); - - INSERT_INTO_JSON_OBJECT(val, doc, amounts, amounts); - INSERT_INTO_JSON_OBJECT(val, doc, count, count); - - return val; -} - -void GetRandomOutputsForAmounts::Request::fromJson(rapidjson::Value& val) -{ - GET_FROM_JSON_OBJECT(val, amounts, amounts); - GET_FROM_JSON_OBJECT(val, count, count); -} - -rapidjson::Value GetRandomOutputsForAmounts::Response::toJson(rapidjson::Document& doc) const -{ - auto val = Message::toJson(doc); - - auto& al = doc.GetAllocator(); - - INSERT_INTO_JSON_OBJECT(val, doc, amounts_with_outputs, amounts_with_outputs); - - return val; -} - -void GetRandomOutputsForAmounts::Response::fromJson(rapidjson::Value& val) -{ - GET_FROM_JSON_OBJECT(val, amounts_with_outputs, amounts_with_outputs); -} - - rapidjson::Value SendRawTx::Request::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); diff --git a/src/rpc/message_data_structs.h b/src/rpc/message_data_structs.h index 20390aee8..cf15ade1c 100644 --- a/src/rpc/message_data_structs.h +++ b/src/rpc/message_data_structs.h @@ -85,6 +85,7 @@ namespace rpc cryptonote::transaction tx; crypto::hash tx_hash; uint64_t blob_size; + uint64_t weight; uint64_t fee; crypto::hash max_used_block_hash; uint64_t max_used_block_height; @@ -185,6 +186,9 @@ namespace rpc crypto::hash top_block_hash; uint64_t cumulative_difficulty; uint64_t block_size_limit; + uint64_t block_weight_limit; + uint64_t block_size_median; + uint64_t block_weight_median; uint64_t start_time; }; diff --git a/src/serialization/json_object.cpp b/src/serialization/json_object.cpp index 67fe709dc..89a1dbd23 100644 --- a/src/serialization/json_object.cpp +++ b/src/serialization/json_object.cpp @@ -757,6 +757,7 @@ void toJsonValue(rapidjson::Document& doc, const cryptonote::rpc::tx_in_pool& tx INSERT_INTO_JSON_OBJECT(val, doc, tx, tx.tx); INSERT_INTO_JSON_OBJECT(val, doc, tx_hash, tx.tx_hash); INSERT_INTO_JSON_OBJECT(val, doc, blob_size, tx.blob_size); + INSERT_INTO_JSON_OBJECT(val, doc, weight, tx.weight); INSERT_INTO_JSON_OBJECT(val, doc, fee, tx.fee); INSERT_INTO_JSON_OBJECT(val, doc, max_used_block_hash, tx.max_used_block_hash); INSERT_INTO_JSON_OBJECT(val, doc, max_used_block_height, tx.max_used_block_height); @@ -780,6 +781,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::tx_in_pool& tx) GET_FROM_JSON_OBJECT(val, tx.tx, tx); GET_FROM_JSON_OBJECT(val, tx.blob_size, blob_size); + GET_FROM_JSON_OBJECT(val, tx.weight, weight); GET_FROM_JSON_OBJECT(val, tx.fee, fee); GET_FROM_JSON_OBJECT(val, tx.max_used_block_hash, max_used_block_hash); GET_FROM_JSON_OBJECT(val, tx.max_used_block_height, max_used_block_height); @@ -1195,6 +1197,9 @@ void toJsonValue(rapidjson::Document& doc, const cryptonote::rpc::DaemonInfo& in INSERT_INTO_JSON_OBJECT(val, doc, top_block_hash, info.top_block_hash); INSERT_INTO_JSON_OBJECT(val, doc, cumulative_difficulty, info.cumulative_difficulty); INSERT_INTO_JSON_OBJECT(val, doc, block_size_limit, info.block_size_limit); + INSERT_INTO_JSON_OBJECT(val, doc, block_weight_limit, info.block_weight_limit); + INSERT_INTO_JSON_OBJECT(val, doc, block_size_median, info.block_size_median); + INSERT_INTO_JSON_OBJECT(val, doc, block_weight_median, info.block_weight_median); INSERT_INTO_JSON_OBJECT(val, doc, start_time, info.start_time); } @@ -1221,6 +1226,9 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::DaemonInfo& inf GET_FROM_JSON_OBJECT(val, info.top_block_hash, top_block_hash); GET_FROM_JSON_OBJECT(val, info.cumulative_difficulty, cumulative_difficulty); GET_FROM_JSON_OBJECT(val, info.block_size_limit, block_size_limit); + GET_FROM_JSON_OBJECT(val, info.block_weight_limit, block_weight_limit); + GET_FROM_JSON_OBJECT(val, info.block_size_median, block_size_median); + GET_FROM_JSON_OBJECT(val, info.block_weight_median, block_weight_median); GET_FROM_JSON_OBJECT(val, info.start_time, start_time); } diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index bdf4212ce..602cdfe44 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -86,9 +86,9 @@ typedef cryptonote::simple_wallet sw; #define EXTENDED_LOGS_FILE "wallet_details.log" -#define DEFAULT_MIX 6 +#define DEFAULT_MIX 10 -#define MIN_RING_SIZE 7 // Used to inform user about min ring size -- does not track actual protocol +#define MIN_RING_SIZE 11 // Used to inform user about min ring size -- does not track actual protocol #define LOCK_IDLE_SCOPE() \ bool auto_refresh_enabled = m_auto_refresh_enabled.load(std::memory_order_relaxed); \ @@ -129,8 +129,6 @@ namespace const command_line::arg_descriptor<bool> arg_restore_deterministic_wallet = {"restore-deterministic-wallet", sw::tr("Recover wallet using Electrum-style mnemonic seed"), false}; const command_line::arg_descriptor<bool> arg_restore_multisig_wallet = {"restore-multisig-wallet", sw::tr("Recover multisig wallet using Electrum-style mnemonic seed"), false}; const command_line::arg_descriptor<bool> arg_non_deterministic = {"non-deterministic", sw::tr("Generate non-deterministic view and spend keys"), false}; - const command_line::arg_descriptor<bool> arg_trusted_daemon = {"trusted-daemon", sw::tr("Enable commands which rely on a trusted daemon"), false}; - const command_line::arg_descriptor<bool> arg_untrusted_daemon = {"untrusted-daemon", sw::tr("Disable commands which rely on a trusted daemon"), false}; const command_line::arg_descriptor<bool> arg_allow_mismatched_daemon_version = {"allow-mismatched-daemon-version", sw::tr("Allow communicating with a daemon that uses a different RPC version"), false}; const command_line::arg_descriptor<uint64_t> arg_restore_height = {"restore-height", sw::tr("Restore from specific blockchain height"), 0}; const command_line::arg_descriptor<bool> arg_do_not_relay = {"do-not-relay", sw::tr("The newly created transaction will not be relayed to the monero network"), false}; @@ -447,7 +445,7 @@ namespace LOG_ERROR("RPC error: " << e.to_string()); fail_msg_writer() << tr("RPC error: ") << e.what(); } - catch (const tools::error::get_random_outs_error &e) + catch (const tools::error::get_outs_error &e) { fail_msg_writer() << tr("failed to get random outputs to mix: ") << e.what(); } @@ -581,6 +579,18 @@ bool parse_priority(const std::string& arg, uint32_t& priority) return false; } +std::string join_priority_strings(const char *delimiter) +{ + std::string s; + for (size_t n = 0; n < allowed_priority_strings.size(); ++n) + { + if (!s.empty()) + s += delimiter; + s += allowed_priority_strings[n]; + } + return s; +} + std::string simple_wallet::get_commands_str() { std::stringstream ss; @@ -819,21 +829,24 @@ bool simple_wallet::print_fee_info(const std::vector<std::string> &args/* = std: fail_msg_writer() << tr("Cannot connect to daemon"); return true; } - const uint64_t per_kb_fee = m_wallet->get_per_kb_fee(); - const uint64_t typical_size_kb = 13; - message_writer() << (boost::format(tr("Current fee is %s %s per kB")) % print_money(per_kb_fee) % cryptonote::get_unit(cryptonote::get_default_decimal_point())).str(); + const bool per_byte = m_wallet->use_fork_rules(HF_VERSION_PER_BYTE_FEE); + const uint64_t base_fee = m_wallet->get_base_fee(); + const char *base = per_byte ? "byte" : "kB"; + const uint64_t typical_size = per_byte ? 2500 : 13; + const uint64_t size_granularity = per_byte ? 1 : 1024; + message_writer() << (boost::format(tr("Current fee is %s %s per %s")) % print_money(base_fee) % cryptonote::get_unit(cryptonote::get_default_decimal_point()) % base).str(); std::vector<uint64_t> fees; for (uint32_t priority = 1; priority <= 4; ++priority) { uint64_t mult = m_wallet->get_fee_multiplier(priority); - fees.push_back(per_kb_fee * typical_size_kb * mult); + fees.push_back(base_fee * typical_size * mult); } std::vector<std::pair<uint64_t, uint64_t>> blocks; try { - uint64_t base_size = typical_size_kb * 1024; - blocks = m_wallet->estimate_backlog(base_size, base_size + 1023, fees); + uint64_t base_size = typical_size * size_granularity; + blocks = m_wallet->estimate_backlog(base_size, base_size + size_granularity - 1, fees); } catch (const std::exception &e) { @@ -1137,7 +1150,7 @@ bool simple_wallet::import_multisig(const std::vector<std::string> &args) fail_msg_writer() << tr("Failed to import multisig info: ") << e.what(); return true; } - if (is_daemon_trusted()) + if (m_wallet->is_trusted_daemon()) { try { @@ -1291,7 +1304,7 @@ bool simple_wallet::submit_multisig(const std::vector<std::string> &args) } catch (const std::exception &e) { - handle_transfer_exception(std::current_exception(), is_daemon_trusted()); + handle_transfer_exception(std::current_exception(), m_wallet->is_trusted_daemon()); } catch (...) { @@ -1614,23 +1627,23 @@ bool simple_wallet::set_ring(const std::vector<std::string> &args) bool simple_wallet::blackball(const std::vector<std::string> &args) { - crypto::public_key output; + uint64_t amount = std::numeric_limits<uint64_t>::max(), offset, num_offsets; if (args.size() == 0) { - fail_msg_writer() << tr("usage: blackball <output_public_key> | <filename> [add]"); + fail_msg_writer() << tr("usage: blackball <amount>/<offset> | <filename> [add]"); return true; } try { - if (epee::string_tools::hex_to_pod(args[0], output)) + if (sscanf(args[0].c_str(), "%" PRIu64 "/%" PRIu64, &amount, &offset) == 2) { - m_wallet->blackball_output(output); + m_wallet->blackball_output(std::make_pair(amount, offset)); } else if (epee::file_io_utils::is_file_exist(args[0])) { - std::vector<crypto::public_key> outputs; - char str[65]; + std::vector<std::pair<uint64_t, uint64_t>> outputs; + char str[256]; std::unique_ptr<FILE, tools::close_file> f(fopen(args[0].c_str(), "r")); if (f) @@ -1644,10 +1657,27 @@ bool simple_wallet::blackball(const std::vector<std::string> &args) str[len - 1] = 0; if (!str[0]) continue; - outputs.push_back(crypto::public_key()); - if (!epee::string_tools::hex_to_pod(str, outputs.back())) + if (sscanf(str, "@%" PRIu64, &amount) == 1) + { + continue; + } + if (amount == std::numeric_limits<uint64_t>::max()) { - fail_msg_writer() << tr("Invalid public key: ") << str; + fail_msg_writer() << tr("First line is not an amount"); + return true; + } + if (sscanf(str, "%" PRIu64 "*%" PRIu64, &offset, &num_offsets) == 2 && num_offsets <= std::numeric_limits<uint64_t>::max() - offset) + { + while (num_offsets--) + outputs.push_back(std::make_pair(amount, offset++)); + } + else if (sscanf(str, "%" PRIu64, &offset) == 1) + { + outputs.push_back(std::make_pair(amount, offset)); + } + else + { + fail_msg_writer() << tr("Invalid output: ") << str; return true; } } @@ -1672,7 +1702,7 @@ bool simple_wallet::blackball(const std::vector<std::string> &args) } else { - fail_msg_writer() << tr("Invalid public key, and file doesn't exist"); + fail_msg_writer() << tr("Invalid output key, and file doesn't exist"); return true; } } @@ -1686,16 +1716,16 @@ bool simple_wallet::blackball(const std::vector<std::string> &args) bool simple_wallet::unblackball(const std::vector<std::string> &args) { - crypto::public_key output; + std::pair<uint64_t, uint64_t> output; if (args.size() != 1) { - fail_msg_writer() << tr("usage: unblackball <output_public_key>"); + fail_msg_writer() << tr("usage: unblackball <amount>/<offset>"); return true; } - if (!epee::string_tools::hex_to_pod(args[0], output)) + if (sscanf(args[0].c_str(), "%" PRIu64 "/%" PRIu64, &output.first, &output.second) != 2) { - fail_msg_writer() << tr("Invalid public key"); + fail_msg_writer() << tr("Invalid output"); return true; } @@ -1713,25 +1743,25 @@ bool simple_wallet::unblackball(const std::vector<std::string> &args) bool simple_wallet::blackballed(const std::vector<std::string> &args) { - crypto::public_key output; + std::pair<uint64_t, uint64_t> output; if (args.size() != 1) { - fail_msg_writer() << tr("usage: blackballed <output_public_key>"); + fail_msg_writer() << tr("usage: blackballed <amount>/<offset>"); return true; } - if (!epee::string_tools::hex_to_pod(args[0], output)) + if (sscanf(args[0].c_str(), "%" PRIu64 "/%" PRIu64, &output.first, &output.second) != 2) { - fail_msg_writer() << tr("Invalid public key"); + fail_msg_writer() << tr("Invalid output"); return true; } try { if (m_wallet->is_output_blackballed(output)) - message_writer() << tr("Blackballed: ") << output; + message_writer() << tr("Blackballed: ") << output.first << "/" << output.second; else - message_writer() << tr("not blackballed: ") << output; + message_writer() << tr("not blackballed: ") << output.first << "/" << output.second; } catch (const std::exception &e) { @@ -1829,6 +1859,8 @@ bool simple_wallet::set_default_ring_size(const std::vector<std::string> &args/* if (ring_size != 0 && ring_size != DEFAULT_MIX+1) message_writer() << tr("WARNING: this is a non default ring size, which may harm your privacy. Default is recommended."); + else if (ring_size == DEFAULT_MIX) + message_writer() << tr("WARNING: from v8, ring size will be fixed and this setting will be ignored."); const auto pwd_container = get_and_verify_password(); if (pwd_container) @@ -1852,12 +1884,12 @@ bool simple_wallet::set_default_ring_size(const std::vector<std::string> &args/* bool simple_wallet::set_default_priority(const std::vector<std::string> &args/* = std::vector<std::string>()*/) { - int priority = 0; + uint32_t priority = 0; try { if (strchr(args[1].c_str(), '-')) { - fail_msg_writer() << tr("priority must be 0, 1, 2, 3, or 4 "); + fail_msg_writer() << tr("priority must be either 0, 1, 2, 3, or 4, or one of: ") << join_priority_strings(", "); return true; } if (args[1] == "0") @@ -1866,11 +1898,23 @@ bool simple_wallet::set_default_priority(const std::vector<std::string> &args/* } else { - priority = boost::lexical_cast<int>(args[1]); - if (priority < 1 || priority > 4) + bool found = false; + for (size_t n = 0; n < allowed_priority_strings.size(); ++n) { - fail_msg_writer() << tr("priority must be 0, 1, 2, 3, or 4"); - return true; + if (allowed_priority_strings[n] == args[1]) + { + found = true; + priority = n; + } + } + if (!found) + { + priority = boost::lexical_cast<int>(args[1]); + if (priority < 1 || priority > 4) + { + fail_msg_writer() << tr("priority must be either 0, 1, 2, 3, or 4, or one of: ") << join_priority_strings(", "); + return true; + } } } @@ -1884,7 +1928,7 @@ bool simple_wallet::set_default_priority(const std::vector<std::string> &args/* } catch(const boost::bad_lexical_cast &) { - fail_msg_writer() << tr("priority must be 0, 1, 2, 3, or 4"); + fail_msg_writer() << tr("priority must be either 0, 1, 2, 3, or 4, or one of: ") << join_priority_strings(", "); return true; } catch(...) @@ -2244,15 +2288,15 @@ simple_wallet::simple_wallet() tr("Show the blockchain height.")); m_cmd_binder.set_handler("transfer_original", boost::bind(&simple_wallet::transfer, this, _1), - tr("transfer_original [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> <amount> [<payment_id>]"), - tr("Transfer <amount> to <address> using an older transaction building algorithm. If the parameter \"index=<N1>[,<N2>,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. <priority> is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability. Multiple payments can be made at once by adding <address_2> <amount_2> etcetera (before the payment ID, if it's included)")); + tr("transfer_original [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] (<URI> | <address> <amount>) [<payment_id>]"), + tr("Transfer <amount> to <address> using an older transaction building algorithm. If the parameter \"index=<N1>[,<N2>,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. <priority> is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability. Multiple payments can be made at once by adding URI_2 or <address_2> <amount_2> etcetera (before the payment ID, if it's included)")); m_cmd_binder.set_handler("transfer", boost::bind(&simple_wallet::transfer_new, this, _1), - tr("transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> <amount> [<payment_id>]"), - tr("Transfer <amount> to <address>. If the parameter \"index=<N1>[,<N2>,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. <priority> is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability. Multiple payments can be made at once by adding <address_2> <amount_2> etcetera (before the payment ID, if it's included)")); + tr("transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] (<URI> | <address> <amount>) [<payment_id>]"), + tr("Transfer <amount> to <address>. If the parameter \"index=<N1>[,<N2>,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. <priority> is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability. Multiple payments can be made at once by adding URI_2 or <address_2> <amount_2> etcetera (before the payment ID, if it's included)")); m_cmd_binder.set_handler("locked_transfer", boost::bind(&simple_wallet::locked_transfer, this, _1), - tr("locked_transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <addr> <amount> <lockblocks> [<payment_id>]"), - tr("Transfer <amount> to <address> and lock it for <lockblocks> (max. 1000000). If the parameter \"index=<N1>[,<N2>,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. <priority> is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability. Multiple payments can be made at once by adding <address_2> <amount_2> etcetera (before the payment ID, if it's included)")); + tr("locked_transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] (<URI> | <addr> <amount>) <lockblocks> [<payment_id>]"), + tr("Transfer <amount> to <address> and lock it for <lockblocks> (max. 1000000). If the parameter \"index=<N1>[,<N2>,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. <priority> is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability. Multiple payments can be made at once by adding URI_2 or <address_2> <amount_2> etcetera (before the payment ID, if it's included)")); m_cmd_binder.set_handler("locked_sweep_all", boost::bind(&simple_wallet::locked_sweep_all, this, _1), tr("locked_sweep_all [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> <lockblocks> [<payment_id>]"), @@ -2532,15 +2576,15 @@ simple_wallet::simple_wallet() tr("Save known rings to the shared rings database")); m_cmd_binder.set_handler("blackball", boost::bind(&simple_wallet::blackball, this, _1), - tr("blackball <output public key> | <filename> [add]"), + tr("blackball <amount>/<offset> | <filename> [add]"), tr("Blackball output(s) so they never get selected as fake outputs in a ring")); m_cmd_binder.set_handler("unblackball", boost::bind(&simple_wallet::unblackball, this, _1), - tr("unblackball <output public key>"), + tr("unblackball <amount>/<offset>"), tr("Unblackballs an output so it may get selected as a fake output in a ring")); m_cmd_binder.set_handler("blackballed", boost::bind(&simple_wallet::blackballed, this, _1), - tr("blackballed <output public key>"), + tr("blackballed <amount>/<offset>"), tr("Checks whether an output is blackballed")); m_cmd_binder.set_handler("version", boost::bind(&simple_wallet::version, this, _1), @@ -2559,6 +2603,10 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args) std::string seed_language = m_wallet->get_seed_language(); if (m_use_english_language_names) seed_language = crypto::ElectrumWords::get_english_name_for(seed_language); + std::string priority_string = "invalid"; + uint32_t priority = m_wallet->get_default_priority(); + if (priority < allowed_priority_strings.size()) + priority_string = allowed_priority_strings[priority]; success_msg_writer() << "seed = " << seed_language; success_msg_writer() << "always-confirm-transfers = " << m_wallet->always_confirm_transfers(); success_msg_writer() << "print-ring-members = " << m_wallet->print_ring_members(); @@ -2566,7 +2614,7 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args) success_msg_writer() << "default-ring-size = " << (m_wallet->default_mixin() ? m_wallet->default_mixin() + 1 : 0); success_msg_writer() << "auto-refresh = " << m_wallet->auto_refresh(); success_msg_writer() << "refresh-type = " << get_refresh_type_name(m_wallet->get_refresh_type()); - success_msg_writer() << "priority = " << m_wallet->get_default_priority(); + success_msg_writer() << "priority = " << priority<< " (" << priority_string << ")"; success_msg_writer() << "confirm-missing-payment-id = " << m_wallet->confirm_missing_payment_id(); success_msg_writer() << "ask-password = " << m_wallet->ask_password(); success_msg_writer() << "unit = " << cryptonote::get_unit(cryptonote::get_default_decimal_point()); @@ -2622,7 +2670,7 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args) CHECK_SIMPLE_VARIABLE("default-ring-size", set_default_ring_size, tr("integer >= ") << MIN_RING_SIZE); CHECK_SIMPLE_VARIABLE("auto-refresh", set_auto_refresh, tr("0 or 1")); CHECK_SIMPLE_VARIABLE("refresh-type", set_refresh_type, tr("full (slowest, no assumptions); optimize-coinbase (fast, assumes the whole coinbase is paid to a single address); no-coinbase (fastest, assumes we receive no coinbase transaction), default (same as optimize-coinbase)")); - CHECK_SIMPLE_VARIABLE("priority", set_default_priority, tr("0, 1, 2, 3, or 4")); + CHECK_SIMPLE_VARIABLE("priority", set_default_priority, tr("0, 1, 2, 3, or 4, or one of ") << join_priority_strings(", ")); CHECK_SIMPLE_VARIABLE("confirm-missing-payment-id", set_confirm_missing_payment_id, tr("0 or 1")); CHECK_SIMPLE_VARIABLE("ask-password", set_ask_password, tr("0 or 1")); CHECK_SIMPLE_VARIABLE("unit", set_unit, tr("monero, millinero, micronero, nanonero, piconero")); @@ -3363,22 +3411,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) return false; } - // set --trusted-daemon if local and not overridden - if (!m_trusted_daemon) - { - try - { - m_trusted_daemon = false; - if (tools::is_local_address(m_wallet->get_daemon_address())) - { - MINFO(tr("Daemon is local, assuming trusted")); - m_trusted_daemon = true; - } - } - catch (const std::exception &e) { } - } - - if (!is_daemon_trusted()) + if (!m_wallet->is_trusted_daemon()) message_writer() << (boost::format(tr("Warning: using an untrusted daemon at %s, privacy will be lessened")) % m_wallet->get_daemon_address()).str(); if (m_wallet->get_ring_database().empty()) @@ -3412,10 +3445,6 @@ bool simple_wallet::handle_command_line(const boost::program_options::variables_ m_restore_deterministic_wallet = command_line::get_arg(vm, arg_restore_deterministic_wallet); m_restore_multisig_wallet = command_line::get_arg(vm, arg_restore_multisig_wallet); m_non_deterministic = command_line::get_arg(vm, arg_non_deterministic); - if (!command_line::is_arg_defaulted(vm, arg_trusted_daemon) || !command_line::is_arg_defaulted(vm, arg_untrusted_daemon)) - m_trusted_daemon = command_line::get_arg(vm, arg_trusted_daemon) && !command_line::get_arg(vm, arg_untrusted_daemon); - if (!command_line::is_arg_defaulted(vm, arg_trusted_daemon) && !command_line::is_arg_defaulted(vm, arg_untrusted_daemon)) - message_writer() << tr("--trusted-daemon and --untrusted-daemon are both seen, assuming untrusted"); m_allow_mismatched_daemon_version = command_line::get_arg(vm, arg_allow_mismatched_daemon_version); m_restore_height = command_line::get_arg(vm, arg_restore_height); m_do_not_relay = command_line::get_arg(vm, arg_do_not_relay); @@ -3927,7 +3956,7 @@ bool simple_wallet::save_watch_only(const std::vector<std::string> &args/* = std //---------------------------------------------------------------------------------------------------- bool simple_wallet::start_mining(const std::vector<std::string>& args) { - if (!is_daemon_trusted()) + if (!m_wallet->is_trusted_daemon()) { fail_msg_writer() << tr("this command requires a trusted daemon. Enable with --trusted-daemon"); return true; @@ -4040,29 +4069,29 @@ bool simple_wallet::set_daemon(const std::vector<std::string>& args) if (args.size() == 2) { if (args[1] == "trusted") - m_trusted_daemon = true; + m_wallet->set_trusted_daemon(true); else if (args[1] == "untrusted") - m_trusted_daemon = false; + m_wallet->set_trusted_daemon(false); else { fail_msg_writer() << tr("Expected trusted or untrusted, got ") << args[1] << ": assuming untrusted"; - m_trusted_daemon = false; + m_wallet->set_trusted_daemon(false); } } else { - m_trusted_daemon = false; + m_wallet->set_trusted_daemon(false); try { if (tools::is_local_address(m_wallet->get_daemon_address())) { MINFO(tr("Daemon is local, assuming trusted")); - m_trusted_daemon = true; + m_wallet->set_trusted_daemon(true); } } catch (const std::exception &e) { } } - success_msg_writer() << boost::format("Daemon set to %s, %s") % daemon_url % (*m_trusted_daemon ? tr("trusted") : tr("untrusted")); + success_msg_writer() << boost::format("Daemon set to %s, %s") % daemon_url % (m_wallet->is_trusted_daemon() ? tr("trusted") : tr("untrusted")); } else { fail_msg_writer() << tr("This does not seem to be a valid daemon URL."); } @@ -4198,7 +4227,7 @@ bool simple_wallet::refresh_main(uint64_t start_height, bool reset, bool is_init { m_in_manual_refresh.store(true, std::memory_order_relaxed); epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){m_in_manual_refresh.store(false, std::memory_order_relaxed);}); - m_wallet->refresh(is_daemon_trusted(), start_height, fetched_blocks); + m_wallet->refresh(m_wallet->is_trusted_daemon(), start_height, fetched_blocks); ok = true; // Clear line "Height xxx of xxx" std::cout << "\r \r"; @@ -4489,7 +4518,7 @@ bool simple_wallet::show_blockchain_height(const std::vector<std::string>& args) //---------------------------------------------------------------------------------------------------- bool simple_wallet::rescan_spent(const std::vector<std::string> &args) { - if (!is_daemon_trusted()) + if (!m_wallet->is_trusted_daemon()) { fail_msg_writer() << tr("this command requires a trusted daemon. Enable with --trusted-daemon"); return true; @@ -4697,8 +4726,13 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri fail_msg_writer() << (boost::format(tr("ring size %u is too small, minimum is %u")) % (fake_outs_count+1) % (adjusted_fake_outs_count+1)).str(); return true; } + if (adjusted_fake_outs_count < fake_outs_count) + { + fail_msg_writer() << (boost::format(tr("ring size %u is too large, maximum is %u")) % (fake_outs_count+1) % (adjusted_fake_outs_count+1)).str(); + return true; + } - const size_t min_args = (transfer_type == TransferLocked) ? 3 : 2; + const size_t min_args = (transfer_type == TransferLocked) ? 2 : 1; if(local_args.size() < min_args) { fail_msg_writer() << tr("wrong number of arguments"); @@ -4707,39 +4741,38 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri std::vector<uint8_t> extra; bool payment_id_seen = false; - bool expect_even = (transfer_type == TransferLocked); - if ((expect_even ? 0 : 1) == local_args.size() % 2) + if (!local_args.empty()) { std::string payment_id_str = local_args.back(); - local_args.pop_back(); - crypto::hash payment_id; - bool r = tools::wallet2::parse_long_payment_id(payment_id_str, payment_id); - if(r) + bool r = true; + if (tools::wallet2::parse_long_payment_id(payment_id_str, payment_id)) { std::string extra_nonce; set_payment_id_to_tx_extra_nonce(extra_nonce, payment_id); r = add_extra_nonce_to_tx_extra(extra, extra_nonce); + local_args.pop_back(); + payment_id_seen = true; + message_writer() << tr("Unencrypted payment IDs are bad for privacy: ask the recipient to use subaddresses instead"); } else { crypto::hash8 payment_id8; - r = tools::wallet2::parse_short_payment_id(payment_id_str, payment_id8); - if(r) + if (tools::wallet2::parse_short_payment_id(payment_id_str, payment_id8)) { std::string extra_nonce; set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, payment_id8); r = add_extra_nonce_to_tx_extra(extra, extra_nonce); + local_args.pop_back(); + payment_id_seen = true; } } if(!r) { - fail_msg_writer() << tr("payment id has invalid format, expected 16 or 64 character hex string: ") << payment_id_str; + fail_msg_writer() << tr("payment id failed to encode"); return true; } - payment_id_seen = true; - message_writer() << tr("Unencrypted payment IDs are bad for privacy: ask the recipient to use subaddresses instead"); } uint64_t locked_blocks = 0; @@ -4764,11 +4797,54 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri vector<cryptonote::tx_destination_entry> dsts; size_t num_subaddresses = 0; - for (size_t i = 0; i < local_args.size(); i += 2) + for (size_t i = 0; i < local_args.size(); ) { - cryptonote::address_parse_info info; cryptonote::tx_destination_entry de; - if (!cryptonote::get_account_address_from_str_or_url(info, m_wallet->nettype(), local_args[i], oa_prompter)) + cryptonote::address_parse_info info; + bool r = true; + + // check for a URI + std::string address_uri, payment_id_uri, tx_description, recipient_name, error; + std::vector<std::string> unknown_parameters; + uint64_t amount = 0; + bool has_uri = m_wallet->parse_uri(local_args[i], address_uri, payment_id_uri, amount, tx_description, recipient_name, unknown_parameters, error); + if (has_uri) + { + r = cryptonote::get_account_address_from_str_or_url(info, m_wallet->nettype(), address_uri, oa_prompter); + if (payment_id_uri.size() == 16) + { + if (!tools::wallet2::parse_short_payment_id(payment_id_uri, info.payment_id)) + { + fail_msg_writer() << tr("failed to parse short payment ID from URI"); + return true; + } + info.has_payment_id = true; + } + de.amount = amount; + ++i; + } + else if (i + 1 < local_args.size()) + { + r = cryptonote::get_account_address_from_str_or_url(info, m_wallet->nettype(), local_args[i], oa_prompter); + bool ok = cryptonote::parse_amount(de.amount, local_args[i + 1]); + if(!ok || 0 == de.amount) + { + fail_msg_writer() << tr("amount is wrong: ") << local_args[i] << ' ' << local_args[i + 1] << + ", " << tr("expected number from 0 to ") << print_money(std::numeric_limits<uint64_t>::max()); + return true; + } + i += 2; + } + else + { + if (boost::starts_with(local_args[i], "monero:")) + fail_msg_writer() << tr("Invalid last argument: ") << local_args.back() << ": " << error; + else + fail_msg_writer() << tr("Invalid last argument: ") << local_args.back(); + return true; + } + + if (!r) { fail_msg_writer() << tr("failed to parse address"); return true; @@ -4777,16 +4853,30 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri de.is_subaddress = info.is_subaddress; num_subaddresses += info.is_subaddress; - if (info.has_payment_id) + if (info.has_payment_id || !payment_id_uri.empty()) { if (payment_id_seen) { - fail_msg_writer() << tr("a single transaction cannot use more than one payment id: ") << local_args[i]; + fail_msg_writer() << tr("a single transaction cannot use more than one payment id"); return true; } + crypto::hash payment_id; std::string extra_nonce; - set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, info.payment_id); + if (info.has_payment_id) + { + set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, info.payment_id); + } + else if (tools::wallet2::parse_payment_id(payment_id_uri, payment_id)) + { + set_payment_id_to_tx_extra_nonce(extra_nonce, payment_id); + message_writer() << tr("Unencrypted payment IDs are bad for privacy: ask the recipient to use subaddresses instead"); + } + else + { + fail_msg_writer() << tr("failed to parse payment id, though it was detected"); + return true; + } bool r = add_extra_nonce_to_tx_extra(extra, extra_nonce); if(!r) { @@ -4796,14 +4886,6 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri payment_id_seen = true; } - bool ok = cryptonote::parse_amount(de.amount, local_args[i + 1]); - if(!ok || 0 == de.amount) - { - fail_msg_writer() << tr("amount is wrong: ") << local_args[i] << ' ' << local_args[i + 1] << - ", " << tr("expected number from 0 to ") << print_money(std::numeric_limits<uint64_t>::max()); - return true; - } - dsts.push_back(de); } @@ -4837,17 +4919,14 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri return true; } unlock_block = bc_height + locked_blocks; - ptx_vector = m_wallet->create_transactions_2(dsts, fake_outs_count, unlock_block /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices, is_daemon_trusted()); - break; - case TransferNew: - ptx_vector = m_wallet->create_transactions_2(dsts, fake_outs_count, 0 /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices, is_daemon_trusted()); + ptx_vector = m_wallet->create_transactions_2(dsts, fake_outs_count, unlock_block /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices); break; default: - LOG_ERROR("Unknown transfer method, using original"); + LOG_ERROR("Unknown transfer method, using default"); /* FALLTHRU */ - case TransferOriginal: - ptx_vector = m_wallet->create_transactions(dsts, fake_outs_count, 0 /* unlock_time */, priority, extra, is_daemon_trusted()); - break; + case TransferNew: + ptx_vector = m_wallet->create_transactions_2(dsts, fake_outs_count, 0 /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices); + break; } if (ptx_vector.empty()) @@ -5022,7 +5101,7 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri } catch (const std::exception &e) { - handle_transfer_exception(std::current_exception(), is_daemon_trusted()); + handle_transfer_exception(std::current_exception(), m_wallet->is_trusted_daemon()); } catch (...) { @@ -5064,7 +5143,7 @@ bool simple_wallet::sweep_unmixable(const std::vector<std::string> &args_) try { // figure out what tx will be necessary - auto ptx_vector = m_wallet->create_unmixable_sweep_transactions(is_daemon_trusted()); + auto ptx_vector = m_wallet->create_unmixable_sweep_transactions(); if (ptx_vector.empty()) { @@ -5143,13 +5222,13 @@ bool simple_wallet::sweep_unmixable(const std::vector<std::string> &args_) { try { - m_wallet->discard_unmixable_outputs(is_daemon_trusted()); + m_wallet->discard_unmixable_outputs(); } catch (...) {} } } catch (const std::exception &e) { - handle_transfer_exception(std::current_exception(), is_daemon_trusted()); + handle_transfer_exception(std::current_exception(), m_wallet->is_trusted_daemon()); } catch (...) { @@ -5221,6 +5300,11 @@ bool simple_wallet::sweep_main(uint64_t below, bool locked, const std::vector<st fail_msg_writer() << (boost::format(tr("ring size %u is too small, minimum is %u")) % (fake_outs_count+1) % (adjusted_fake_outs_count+1)).str(); return true; } + if (adjusted_fake_outs_count < fake_outs_count) + { + fail_msg_writer() << (boost::format(tr("ring size %u is too large, maximum is %u")) % (fake_outs_count+1) % (adjusted_fake_outs_count+1)).str(); + return true; + } uint64_t unlock_block = 0; if (locked) { @@ -5341,7 +5425,7 @@ bool simple_wallet::sweep_main(uint64_t below, bool locked, const std::vector<st try { // figure out what tx will be necessary - auto ptx_vector = m_wallet->create_transactions_all(below, info.address, info.is_subaddress, fake_outs_count, unlock_block /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices, is_daemon_trusted()); + auto ptx_vector = m_wallet->create_transactions_all(below, info.address, info.is_subaddress, fake_outs_count, unlock_block /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices); if (ptx_vector.empty()) { @@ -5425,7 +5509,7 @@ bool simple_wallet::sweep_main(uint64_t below, bool locked, const std::vector<st } catch (const std::exception& e) { - handle_transfer_exception(std::current_exception(), is_daemon_trusted()); + handle_transfer_exception(std::current_exception(), m_wallet->is_trusted_daemon()); } catch (...) { @@ -5459,12 +5543,28 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) if (fake_outs_count == 0) fake_outs_count = DEFAULT_MIX; } + else if (ring_size == 0) + { + fail_msg_writer() << tr("Ring size must not be 0"); + return true; + } else { fake_outs_count = ring_size - 1; local_args.erase(local_args.begin()); } } + uint64_t adjusted_fake_outs_count = m_wallet->adjust_mixin(fake_outs_count); + if (adjusted_fake_outs_count > fake_outs_count) + { + fail_msg_writer() << (boost::format(tr("ring size %u is too small, minimum is %u")) % (fake_outs_count+1) % (adjusted_fake_outs_count+1)).str(); + return true; + } + if (adjusted_fake_outs_count < fake_outs_count) + { + fail_msg_writer() << (boost::format(tr("ring size %u is too large, maximum is %u")) % (fake_outs_count+1) % (adjusted_fake_outs_count+1)).str(); + return true; + } std::vector<uint8_t> extra; bool payment_id_seen = false; @@ -5554,7 +5654,7 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) try { // figure out what tx will be necessary - auto ptx_vector = m_wallet->create_transactions_single(ki, info.address, info.is_subaddress, fake_outs_count, 0 /* unlock_time */, priority, extra, is_daemon_trusted()); + auto ptx_vector = m_wallet->create_transactions_single(ki, info.address, info.is_subaddress, fake_outs_count, 0 /* unlock_time */, priority, extra); if (ptx_vector.empty()) { @@ -5624,7 +5724,7 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) } catch (const std::exception& e) { - handle_transfer_exception(std::current_exception(), is_daemon_trusted()); + handle_transfer_exception(std::current_exception(), m_wallet->is_trusted_daemon()); } catch (...) { @@ -5930,7 +6030,7 @@ bool simple_wallet::submit_transfer(const std::vector<std::string> &args_) } catch (const std::exception& e) { - handle_transfer_exception(std::current_exception(), is_daemon_trusted()); + handle_transfer_exception(std::current_exception(), m_wallet->is_trusted_daemon()); } catch (...) { @@ -6843,7 +6943,7 @@ void simple_wallet::wallet_idle_thread() { uint64_t fetched_blocks; if (try_connect_to_daemon(true)) - m_wallet->refresh(is_daemon_trusted(), 0, fetched_blocks); + m_wallet->refresh(m_wallet->is_trusted_daemon(), 0, fetched_blocks); } catch(...) {} m_auto_refresh_refreshing = false; @@ -7591,7 +7691,7 @@ bool simple_wallet::import_key_images(const std::vector<std::string> &args) fail_msg_writer() << tr("command not supported by HW wallet"); return true; } - if (!is_daemon_trusted()) + if (!m_wallet->is_trusted_daemon()) { fail_msg_writer() << tr("this command requires a trusted daemon. Enable with --trusted-daemon"); return true; @@ -7922,8 +8022,6 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_params, arg_restore_multisig_wallet ); command_line::add_arg(desc_params, arg_non_deterministic ); command_line::add_arg(desc_params, arg_electrum_seed ); - command_line::add_arg(desc_params, arg_trusted_daemon); - command_line::add_arg(desc_params, arg_untrusted_daemon); command_line::add_arg(desc_params, arg_allow_mismatched_daemon_version); command_line::add_arg(desc_params, arg_restore_height); command_line::add_arg(desc_params, arg_do_not_relay); diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h index 99fc19c00..bfbe633ac 100644 --- a/src/simplewallet/simplewallet.h +++ b/src/simplewallet/simplewallet.h @@ -233,7 +233,6 @@ namespace cryptonote bool print_ring_members(const std::vector<tools::wallet2::pending_tx>& ptx_vector, std::ostream& ostr); std::string get_prompt() const; bool print_seed(bool encrypted); - bool is_daemon_trusted() const { return *m_trusted_daemon; } /*! * \brief Prints the seed with a nice message @@ -337,7 +336,6 @@ namespace cryptonote bool m_restore_deterministic_wallet; // recover flag bool m_restore_multisig_wallet; // recover flag bool m_non_deterministic; // old 2-random generation - boost::optional<bool> m_trusted_daemon; bool m_allow_mismatched_daemon_version; bool m_restoring; // are we restoring, by whatever method? uint64_t m_restore_height; // optional diff --git a/src/wallet/api/wallet.cpp b/src/wallet/api/wallet.cpp index bfd0e4aff..e9e9a4e67 100644 --- a/src/wallet/api/wallet.cpp +++ b/src/wallet/api/wallet.cpp @@ -369,7 +369,6 @@ void Wallet::error(const std::string &category, const std::string &str) { WalletImpl::WalletImpl(NetworkType nettype, uint64_t kdf_rounds) :m_wallet(nullptr) , m_status(Wallet::Status_Ok) - , m_trustedDaemon(false) , m_wallet2Callback(nullptr) , m_recoveringFromSeed(false) , m_recoveringFromDevice(false) @@ -1298,6 +1297,7 @@ PendingTransaction *WalletImpl::createTransaction(const string &dst_addr, const size_t fake_outs_count = mixin_count > 0 ? mixin_count : m_wallet->default_mixin(); if (fake_outs_count == 0) fake_outs_count = DEFAULT_MIXIN; + fake_outs_count = m_wallet->adjust_mixin(fake_outs_count); uint32_t adjusted_priority = m_wallet->adjust_priority(static_cast<uint32_t>(priority)); @@ -1358,7 +1358,7 @@ PendingTransaction *WalletImpl::createTransaction(const string &dst_addr, const dsts.push_back(de); transaction->m_pending_tx = m_wallet->create_transactions_2(dsts, fake_outs_count, 0 /* unlock_time */, adjusted_priority, - extra, subaddr_account, subaddr_indices, m_trustedDaemon); + extra, subaddr_account, subaddr_indices); } else { // for the GUI, sweep_all (i.e. amount set as "(all)") will always sweep all the funds in all the addresses if (subaddr_indices.empty()) @@ -1368,7 +1368,7 @@ PendingTransaction *WalletImpl::createTransaction(const string &dst_addr, const } transaction->m_pending_tx = m_wallet->create_transactions_all(0, info.address, info.is_subaddress, fake_outs_count, 0 /* unlock_time */, adjusted_priority, - extra, subaddr_account, subaddr_indices, m_trustedDaemon); + extra, subaddr_account, subaddr_indices); } if (multisig().isMultisig) { @@ -1381,8 +1381,8 @@ PendingTransaction *WalletImpl::createTransaction(const string &dst_addr, const setStatusError(tr("no connection to daemon. Please make sure daemon is running.")); } catch (const tools::error::wallet_rpc_error& e) { setStatusError(tr("RPC error: ") + e.to_string()); - } catch (const tools::error::get_random_outs_error &e) { - setStatusError((boost::format(tr("failed to get random outputs to mix: %s")) % e.what()).str()); + } catch (const tools::error::get_outs_error &e) { + setStatusError((boost::format(tr("failed to get outputs to mix: %s")) % e.what()).str()); } catch (const tools::error::not_enough_unlocked_money& e) { std::ostringstream writer; @@ -1454,7 +1454,7 @@ PendingTransaction *WalletImpl::createSweepUnmixableTransaction() do { try { - transaction->m_pending_tx = m_wallet->create_unmixable_sweep_transactions(m_trustedDaemon); + transaction->m_pending_tx = m_wallet->create_unmixable_sweep_transactions(); } catch (const tools::error::daemon_busy&) { // TODO: make it translatable with "tr"? @@ -1463,8 +1463,8 @@ PendingTransaction *WalletImpl::createSweepUnmixableTransaction() setStatusError(tr("no connection to daemon. Please make sure daemon is running.")); } catch (const tools::error::wallet_rpc_error& e) { setStatusError(tr("RPC error: ") + e.to_string()); - } catch (const tools::error::get_random_outs_error&) { - setStatusError(tr("failed to get random outputs to mix")); + } catch (const tools::error::get_outs_error&) { + setStatusError(tr("failed to get outputs to mix")); } catch (const tools::error::not_enough_unlocked_money& e) { setStatusError(""); std::ostringstream writer; @@ -1891,12 +1891,12 @@ Wallet::ConnectionStatus WalletImpl::connected() const void WalletImpl::setTrustedDaemon(bool arg) { - m_trustedDaemon = arg; + m_wallet->set_trusted_daemon(arg); } bool WalletImpl::trustedDaemon() const { - return m_trustedDaemon; + return m_wallet->is_trusted_daemon(); } bool WalletImpl::watchOnly() const diff --git a/src/wallet/api/wallet.h b/src/wallet/api/wallet.h index 9218d3ad5..0f3b1ce04 100644 --- a/src/wallet/api/wallet.h +++ b/src/wallet/api/wallet.h @@ -219,7 +219,6 @@ private: mutable std::string m_errorString; std::string m_password; TransactionHistoryImpl * m_history; - bool m_trustedDaemon; Wallet2CallbackImpl * m_wallet2Callback; AddressBookImpl * m_addressBook; SubaddressImpl * m_subaddress; diff --git a/src/wallet/node_rpc_proxy.cpp b/src/wallet/node_rpc_proxy.cpp index 2072495cd..21f75371b 100644 --- a/src/wallet/node_rpc_proxy.cpp +++ b/src/wallet/node_rpc_proxy.cpp @@ -50,12 +50,13 @@ void NodeRPCProxy::invalidate() m_height = 0; for (size_t n = 0; n < 256; ++n) m_earliest_height[n] = 0; - m_dynamic_per_kb_fee_estimate = 0; - m_dynamic_per_kb_fee_estimate_cached_height = 0; - m_dynamic_per_kb_fee_estimate_grace_blocks = 0; + m_dynamic_base_fee_estimate = 0; + m_dynamic_base_fee_estimate_cached_height = 0; + m_dynamic_base_fee_estimate_grace_blocks = 0; + m_fee_quantization_mask = 1; m_rpc_version = 0; m_target_height = 0; - m_block_size_limit = 0; + m_block_weight_limit = 0; m_get_info_time = 0; } @@ -99,7 +100,7 @@ boost::optional<std::string> NodeRPCProxy::get_info() const CHECK_AND_ASSERT_MES(resp_t.status == CORE_RPC_STATUS_OK, resp_t.status, "Failed to get target blockchain height"); m_height = resp_t.height; m_target_height = resp_t.target_height; - m_block_size_limit = resp_t.block_size_limit; + m_block_weight_limit = resp_t.block_weight_limit ? resp_t.block_weight_limit : resp_t.block_size_limit; m_get_info_time = now; } return boost::optional<std::string>(); @@ -123,12 +124,12 @@ boost::optional<std::string> NodeRPCProxy::get_target_height(uint64_t &height) c return boost::optional<std::string>(); } -boost::optional<std::string> NodeRPCProxy::get_block_size_limit(uint64_t &block_size_limit) const +boost::optional<std::string> NodeRPCProxy::get_block_weight_limit(uint64_t &block_weight_limit) const { auto res = get_info(); if (res) return res; - block_size_limit = m_block_size_limit; + block_weight_limit = m_block_weight_limit; return boost::optional<std::string>(); } @@ -153,7 +154,7 @@ boost::optional<std::string> NodeRPCProxy::get_earliest_height(uint8_t version, return boost::optional<std::string>(); } -boost::optional<std::string> NodeRPCProxy::get_dynamic_per_kb_fee_estimate(uint64_t grace_blocks, uint64_t &fee) const +boost::optional<std::string> NodeRPCProxy::get_dynamic_base_fee_estimate(uint64_t grace_blocks, uint64_t &fee) const { uint64_t height; @@ -161,10 +162,10 @@ boost::optional<std::string> NodeRPCProxy::get_dynamic_per_kb_fee_estimate(uint6 if (result) return result; - if (m_dynamic_per_kb_fee_estimate_cached_height != height || m_dynamic_per_kb_fee_estimate_grace_blocks != grace_blocks) + if (m_dynamic_base_fee_estimate_cached_height != height || m_dynamic_base_fee_estimate_grace_blocks != grace_blocks) { - cryptonote::COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::request req_t = AUTO_VAL_INIT(req_t); - cryptonote::COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::response resp_t = AUTO_VAL_INIT(resp_t); + cryptonote::COMMAND_RPC_GET_BASE_FEE_ESTIMATE::request req_t = AUTO_VAL_INIT(req_t); + cryptonote::COMMAND_RPC_GET_BASE_FEE_ESTIMATE::response resp_t = AUTO_VAL_INIT(resp_t); m_daemon_rpc_mutex.lock(); req_t.grace_blocks = grace_blocks; @@ -173,12 +174,47 @@ boost::optional<std::string> NodeRPCProxy::get_dynamic_per_kb_fee_estimate(uint6 CHECK_AND_ASSERT_MES(r, std::string(), "Failed to connect to daemon"); CHECK_AND_ASSERT_MES(resp_t.status != CORE_RPC_STATUS_BUSY, resp_t.status, "Failed to connect to daemon"); CHECK_AND_ASSERT_MES(resp_t.status == CORE_RPC_STATUS_OK, resp_t.status, "Failed to get fee estimate"); - m_dynamic_per_kb_fee_estimate = resp_t.fee; - m_dynamic_per_kb_fee_estimate_cached_height = height; - m_dynamic_per_kb_fee_estimate_grace_blocks = grace_blocks; + m_dynamic_base_fee_estimate = resp_t.fee; + m_dynamic_base_fee_estimate_cached_height = height; + m_dynamic_base_fee_estimate_grace_blocks = grace_blocks; + m_fee_quantization_mask = resp_t.quantization_mask; } - fee = m_dynamic_per_kb_fee_estimate; + fee = m_dynamic_base_fee_estimate; + return boost::optional<std::string>(); +} + +boost::optional<std::string> NodeRPCProxy::get_fee_quantization_mask(uint64_t &fee_quantization_mask) const +{ + uint64_t height; + + boost::optional<std::string> result = get_height(height); + if (result) + return result; + + if (m_dynamic_base_fee_estimate_cached_height != height) + { + cryptonote::COMMAND_RPC_GET_BASE_FEE_ESTIMATE::request req_t = AUTO_VAL_INIT(req_t); + cryptonote::COMMAND_RPC_GET_BASE_FEE_ESTIMATE::response resp_t = AUTO_VAL_INIT(resp_t); + + m_daemon_rpc_mutex.lock(); + req_t.grace_blocks = m_dynamic_base_fee_estimate_grace_blocks; + bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_fee_estimate", req_t, resp_t, m_http_client, rpc_timeout); + m_daemon_rpc_mutex.unlock(); + CHECK_AND_ASSERT_MES(r, std::string(), "Failed to connect to daemon"); + CHECK_AND_ASSERT_MES(resp_t.status != CORE_RPC_STATUS_BUSY, resp_t.status, "Failed to connect to daemon"); + CHECK_AND_ASSERT_MES(resp_t.status == CORE_RPC_STATUS_OK, resp_t.status, "Failed to get fee estimate"); + m_dynamic_base_fee_estimate = resp_t.fee; + m_dynamic_base_fee_estimate_cached_height = height; + m_fee_quantization_mask = resp_t.quantization_mask; + } + + fee_quantization_mask = m_fee_quantization_mask; + if (fee_quantization_mask == 0) + { + MERROR("Fee quantization mask is 0, forcing to 1"); + fee_quantization_mask = 1; + } return boost::optional<std::string>(); } diff --git a/src/wallet/node_rpc_proxy.h b/src/wallet/node_rpc_proxy.h index 8a65884f7..65f13eaaa 100644 --- a/src/wallet/node_rpc_proxy.h +++ b/src/wallet/node_rpc_proxy.h @@ -47,9 +47,10 @@ public: boost::optional<std::string> get_height(uint64_t &height) const; void set_height(uint64_t h); boost::optional<std::string> get_target_height(uint64_t &height) const; - boost::optional<std::string> get_block_size_limit(uint64_t &block_size_limit) const; + boost::optional<std::string> get_block_weight_limit(uint64_t &block_weight_limit) const; boost::optional<std::string> get_earliest_height(uint8_t version, uint64_t &earliest_height) const; - boost::optional<std::string> get_dynamic_per_kb_fee_estimate(uint64_t grace_blocks, uint64_t &fee) const; + boost::optional<std::string> get_dynamic_base_fee_estimate(uint64_t grace_blocks, uint64_t &fee) const; + boost::optional<std::string> get_fee_quantization_mask(uint64_t &fee_quantization_mask) const; private: boost::optional<std::string> get_info() const; @@ -59,12 +60,13 @@ private: mutable uint64_t m_height; mutable uint64_t m_earliest_height[256]; - mutable uint64_t m_dynamic_per_kb_fee_estimate; - mutable uint64_t m_dynamic_per_kb_fee_estimate_cached_height; - mutable uint64_t m_dynamic_per_kb_fee_estimate_grace_blocks; + mutable uint64_t m_dynamic_base_fee_estimate; + mutable uint64_t m_dynamic_base_fee_estimate_cached_height; + mutable uint64_t m_dynamic_base_fee_estimate_grace_blocks; + mutable uint64_t m_fee_quantization_mask; mutable uint32_t m_rpc_version; mutable uint64_t m_target_height; - mutable uint64_t m_block_size_limit; + mutable uint64_t m_block_weight_limit; mutable time_t m_get_info_time; }; diff --git a/src/wallet/ringdb.cpp b/src/wallet/ringdb.cpp index 3f2634c8b..e9fc6866d 100644 --- a/src/wallet/ringdb.cpp +++ b/src/wallet/ringdb.cpp @@ -55,6 +55,13 @@ static int compare_hash32(const MDB_val *a, const MDB_val *b) return 0; } +static int compare_uint64(const MDB_val *a, const MDB_val *b) +{ + const uint64_t va = *(const uint64_t*) a->mv_data; + const uint64_t vb = *(const uint64_t*) b->mv_data; + return va < vb ? -1 : va > vb; +} + static std::string compress_ring(const std::vector<uint64_t> &ring) { std::string s; @@ -146,7 +153,7 @@ static int resize_env(MDB_env *env, const char *db_path, size_t needed) MDB_stat mst; int ret; - needed = std::max(needed, (size_t)(2ul * 1024 * 1024)); // at least 2 MB + needed = std::max(needed, (size_t)(100ul * 1024 * 1024)); // at least 100 MB ret = mdb_env_info(env, &mei); if (ret) @@ -217,9 +224,9 @@ ringdb::ringdb(std::string filename, const std::string &genesis): THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); mdb_set_compare(txn, dbi_rings, compare_hash32); - dbr = mdb_dbi_open(txn, ("blackballs-" + genesis).c_str(), MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi_blackballs); + dbr = mdb_dbi_open(txn, ("blackballs2-" + genesis).c_str(), MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi_blackballs); THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); - mdb_set_dupsort(txn, dbi_blackballs, compare_hash32); + mdb_set_dupsort(txn, dbi_blackballs, compare_uint64); dbr = mdb_txn_commit(txn); THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); @@ -374,7 +381,7 @@ bool ringdb::set_ring(const crypto::chacha_key &chacha_key, const crypto::key_im return true; } -bool ringdb::blackball_worker(const crypto::public_key &output, int op) +bool ringdb::blackball_worker(const std::vector<std::pair<uint64_t, uint64_t>> &outputs, int op) { MDB_txn *txn; MDB_cursor *cursor; @@ -382,49 +389,61 @@ bool ringdb::blackball_worker(const crypto::public_key &output, int op) bool tx_active = false; bool ret = true; - dbr = resize_env(env, filename.c_str(), 32 * 2); // a pubkey, and some slack + THROW_WALLET_EXCEPTION_IF(outputs.size() > 1 && op == BLACKBALL_QUERY, tools::error::wallet_internal_error, "Blackball query only makes sense for a single output"); + + dbr = resize_env(env, filename.c_str(), 32 * 2 * outputs.size()); // a pubkey, and some slack THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set env map size: " + std::string(mdb_strerror(dbr))); dbr = mdb_txn_begin(env, NULL, 0, &txn); THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); tx_active = true; - MDB_val key = zerokeyval; - MDB_val data; - data.mv_data = (void*)&output; - data.mv_size = sizeof(output); - switch (op) + MDB_val key, data; + for (const std::pair<uint64_t, uint64_t> &output: outputs) + { + key.mv_data = (void*)&output.first; + key.mv_size = sizeof(output.first); + data.mv_data = (void*)&output.second; + data.mv_size = sizeof(output.second); + + switch (op) + { + case BLACKBALL_BLACKBALL: + MDEBUG("Blackballing output " << output.first << "/" << output.second); + dbr = mdb_put(txn, dbi_blackballs, &key, &data, MDB_APPENDDUP); + if (dbr == MDB_KEYEXIST) + dbr = 0; + break; + case BLACKBALL_UNBLACKBALL: + MDEBUG("Unblackballing output " << output.first << "/" << output.second); + dbr = mdb_del(txn, dbi_blackballs, &key, &data); + if (dbr == MDB_NOTFOUND) + dbr = 0; + break; + case BLACKBALL_QUERY: + dbr = mdb_cursor_open(txn, dbi_blackballs, &cursor); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create cursor for blackballs table: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_get(cursor, &key, &data, MDB_GET_BOTH); + THROW_WALLET_EXCEPTION_IF(dbr && dbr != MDB_NOTFOUND, tools::error::wallet_internal_error, "Failed to lookup in blackballs table: " + std::string(mdb_strerror(dbr))); + ret = dbr != MDB_NOTFOUND; + if (dbr == MDB_NOTFOUND) + dbr = 0; + mdb_cursor_close(cursor); + break; + case BLACKBALL_CLEAR: + break; + default: + THROW_WALLET_EXCEPTION(tools::error::wallet_internal_error, "Invalid blackball op"); + } + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to query blackballs table: " + std::string(mdb_strerror(dbr))); + } + + if (op == BLACKBALL_CLEAR) { - case BLACKBALL_BLACKBALL: - MDEBUG("Blackballing output " << output); - dbr = mdb_put(txn, dbi_blackballs, &key, &data, MDB_NODUPDATA); - if (dbr == MDB_KEYEXIST) - dbr = 0; - break; - case BLACKBALL_UNBLACKBALL: - MDEBUG("Unblackballing output " << output); - dbr = mdb_del(txn, dbi_blackballs, &key, &data); - if (dbr == MDB_NOTFOUND) - dbr = 0; - break; - case BLACKBALL_QUERY: - dbr = mdb_cursor_open(txn, dbi_blackballs, &cursor); - THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create cursor for blackballs table: " + std::string(mdb_strerror(dbr))); - dbr = mdb_cursor_get(cursor, &key, &data, MDB_GET_BOTH); - THROW_WALLET_EXCEPTION_IF(dbr && dbr != MDB_NOTFOUND, tools::error::wallet_internal_error, "Failed to lookup in blackballs table: " + std::string(mdb_strerror(dbr))); - ret = dbr != MDB_NOTFOUND; - if (dbr == MDB_NOTFOUND) - dbr = 0; - mdb_cursor_close(cursor); - break; - case BLACKBALL_CLEAR: - dbr = mdb_drop(txn, dbi_blackballs, 0); - break; - default: - THROW_WALLET_EXCEPTION(tools::error::wallet_internal_error, "Invalid blackball op"); + dbr = mdb_drop(txn, dbi_blackballs, 0); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to clear blackballs table: " + std::string(mdb_strerror(dbr))); } - THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to query blackballs table: " + std::string(mdb_strerror(dbr))); dbr = mdb_txn_commit(txn); THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn blackballing output to database: " + std::string(mdb_strerror(dbr))); @@ -432,24 +451,32 @@ bool ringdb::blackball_worker(const crypto::public_key &output, int op) return ret; } -bool ringdb::blackball(const crypto::public_key &output) +bool ringdb::blackball(const std::vector<std::pair<uint64_t, uint64_t>> &outputs) +{ + return blackball_worker(outputs, BLACKBALL_BLACKBALL); +} + +bool ringdb::blackball(const std::pair<uint64_t, uint64_t> &output) { - return blackball_worker(output, BLACKBALL_BLACKBALL); + std::vector<std::pair<uint64_t, uint64_t>> outputs(1, output); + return blackball_worker(outputs, BLACKBALL_BLACKBALL); } -bool ringdb::unblackball(const crypto::public_key &output) +bool ringdb::unblackball(const std::pair<uint64_t, uint64_t> &output) { - return blackball_worker(output, BLACKBALL_UNBLACKBALL); + std::vector<std::pair<uint64_t, uint64_t>> outputs(1, output); + return blackball_worker(outputs, BLACKBALL_UNBLACKBALL); } -bool ringdb::blackballed(const crypto::public_key &output) +bool ringdb::blackballed(const std::pair<uint64_t, uint64_t> &output) { - return blackball_worker(output, BLACKBALL_QUERY); + std::vector<std::pair<uint64_t, uint64_t>> outputs(1, output); + return blackball_worker(outputs, BLACKBALL_QUERY); } bool ringdb::clear_blackballs() { - return blackball_worker(crypto::public_key(), BLACKBALL_CLEAR); + return blackball_worker(std::vector<std::pair<uint64_t, uint64_t>>(), BLACKBALL_CLEAR); } } diff --git a/src/wallet/ringdb.h b/src/wallet/ringdb.h index 6b4bce124..7b448b0d7 100644 --- a/src/wallet/ringdb.h +++ b/src/wallet/ringdb.h @@ -49,13 +49,14 @@ namespace tools bool get_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, std::vector<uint64_t> &outs); bool set_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, const std::vector<uint64_t> &outs, bool relative); - bool blackball(const crypto::public_key &output); - bool unblackball(const crypto::public_key &output); - bool blackballed(const crypto::public_key &output); + bool blackball(const std::pair<uint64_t, uint64_t> &output); + bool blackball(const std::vector<std::pair<uint64_t, uint64_t>> &outputs); + bool unblackball(const std::pair<uint64_t, uint64_t> &output); + bool blackballed(const std::pair<uint64_t, uint64_t> &output); bool clear_blackballs(); private: - bool blackball_worker(const crypto::public_key &output, int op); + bool blackball_worker(const std::vector<std::pair<uint64_t, uint64_t>> &outputs, int op); private: std::string filename; diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 96b77a7a6..f670febbf 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -84,8 +84,8 @@ using namespace cryptonote; // used to choose when to stop adding outputs to a tx #define APPROXIMATE_INPUT_BYTES 80 -// used to target a given block size (additional outputs may be added on top to build fee) -#define TX_SIZE_TARGET(bytes) (bytes*2/3) +// used to target a given block weight (additional outputs may be added on top to build fee) +#define TX_WEIGHT_TARGET(bytes) (bytes*2/3) // arbitrary, used to generate different hashes from the same input #define CHACHA8_KEY_TAIL 0x8c @@ -140,6 +140,8 @@ namespace struct options { const command_line::arg_descriptor<std::string> daemon_address = {"daemon-address", tools::wallet2::tr("Use daemon instance at <host>:<port>"), ""}; const command_line::arg_descriptor<std::string> daemon_host = {"daemon-host", tools::wallet2::tr("Use daemon instance at host <arg> instead of localhost"), ""}; + const command_line::arg_descriptor<bool> trusted_daemon = {"trusted-daemon", tools::wallet2::tr("Enable commands which rely on a trusted daemon"), false}; + const command_line::arg_descriptor<bool> untrusted_daemon = {"untrusted-daemon", tools::wallet2::tr("Disable commands which rely on a trusted daemon"), false}; const command_line::arg_descriptor<std::string> password = {"password", tools::wallet2::tr("Wallet password (escape/quote as needed)"), "", true}; const command_line::arg_descriptor<std::string> password_file = {"password-file", tools::wallet2::tr("Wallet password file"), "", true}; const command_line::arg_descriptor<int> daemon_port = {"daemon-port", tools::wallet2::tr("Use daemon instance at port <arg> instead of 18081"), 0}; @@ -181,19 +183,21 @@ uint64_t calculate_fee(uint64_t fee_per_kb, size_t bytes, uint64_t fee_multiplie return kB * fee_per_kb * fee_multiplier; } -uint64_t calculate_fee(uint64_t fee_per_kb, const cryptonote::blobdata &blob, uint64_t fee_multiplier) +uint64_t calculate_fee_from_weight(uint64_t base_fee, uint64_t weight, uint64_t fee_multiplier, uint64_t fee_quantization_mask) { - return calculate_fee(fee_per_kb, blob.size(), fee_multiplier); + uint64_t fee = weight * base_fee * fee_multiplier; + fee = (fee + fee_quantization_mask - 1) / fee_quantization_mask * fee_quantization_mask; + return fee; } -std::string get_size_string(size_t sz) +std::string get_weight_string(size_t weight) { - return std::to_string(sz) + " bytes (" + std::to_string((sz + 1023) / 1024) + " kB)"; + return std::to_string(weight) + " weight"; } -std::string get_size_string(const cryptonote::blobdata &tx) +std::string get_weight_string(const cryptonote::transaction &tx, size_t blob_size) { - return get_size_string(tx.size()); + return get_weight_string(get_transaction_weight(tx, blob_size)); } std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variables_map& vm, bool rpc, const options& opts, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter) @@ -236,8 +240,29 @@ std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variabl if (daemon_address.empty()) daemon_address = std::string("http://") + daemon_host + ":" + std::to_string(daemon_port); + boost::optional<bool> trusted_daemon; + if (!command_line::is_arg_defaulted(vm, opts.trusted_daemon) || !command_line::is_arg_defaulted(vm, opts.untrusted_daemon)) + trusted_daemon = command_line::get_arg(vm, opts.trusted_daemon) && !command_line::get_arg(vm, opts.untrusted_daemon); + THROW_WALLET_EXCEPTION_IF(!command_line::is_arg_defaulted(vm, opts.trusted_daemon) && !command_line::is_arg_defaulted(vm, opts.untrusted_daemon), + tools::error::wallet_internal_error, tools::wallet2::tr("--trusted-daemon and --untrusted-daemon are both seen, assuming untrusted")); + + // set --trusted-daemon if local and not overridden + if (!trusted_daemon) + { + try + { + trusted_daemon = false; + if (tools::is_local_address(daemon_address)) + { + MINFO(tr("Daemon is local, assuming trusted")); + trusted_daemon = true; + } + } + catch (const std::exception &e) { } + } + std::unique_ptr<tools::wallet2> wallet(new tools::wallet2(nettype, kdf_rounds)); - wallet->init(rpc, std::move(daemon_address), std::move(login)); + wallet->init(rpc, std::move(daemon_address), std::move(login), 0, false, *trusted_daemon); boost::filesystem::path ringdb_path = command_line::get_arg(vm, opts.shared_ringdb_dir); wallet->set_ring_database(ringdb_path.string()); return wallet; @@ -553,7 +578,12 @@ size_t estimate_rct_tx_size(int n_inputs, int mixin, int n_outputs, size_t extra // rangeSigs if (bulletproof) - size += ((2*6 + 4 + 5)*32 + 3) * n_outputs; + { + size_t log_padded_outputs = 0; + while ((1<<log_padded_outputs) < n_outputs) + ++log_padded_outputs; + size += (2 * (6 + log_padded_outputs) + 4 + 5) * 32 + 3; + } else size += (2*64*32+32+64*32) * n_outputs; @@ -572,23 +602,63 @@ size_t estimate_rct_tx_size(int n_inputs, int mixin, int n_outputs, size_t extra // txnFee size += 4; - LOG_PRINT_L2("estimated rct tx size for " << n_inputs << " with ring size " << (mixin+1) << " and " << n_outputs << ": " << size << " (" << ((32 * n_inputs/*+1*/) + 2 * 32 * (mixin+1) * n_inputs + 32 * n_outputs) << " saved)"); + LOG_PRINT_L2("estimated " << (bulletproof ? "bulletproof" : "borromean") << " rct tx size for " << n_inputs << " inputs with ring size " << (mixin+1) << " and " << n_outputs << " outputs: " << size << " (" << ((32 * n_inputs/*+1*/) + 2 * 32 * (mixin+1) * n_inputs + 32 * n_outputs) << " saved)"); return size; } size_t estimate_tx_size(bool use_rct, int n_inputs, int mixin, int n_outputs, size_t extra_size, bool bulletproof) { if (use_rct) - return estimate_rct_tx_size(n_inputs, mixin, n_outputs + 1, extra_size, bulletproof); + return estimate_rct_tx_size(n_inputs, mixin, n_outputs, extra_size, bulletproof); else return n_inputs * (mixin+1) * APPROXIMATE_INPUT_BYTES + extra_size; } +uint64_t estimate_tx_weight(bool use_rct, int n_inputs, int mixin, int n_outputs, size_t extra_size, bool bulletproof) +{ + size_t size = estimate_tx_size(use_rct, n_inputs, mixin, n_outputs, extra_size, bulletproof); + if (use_rct && bulletproof && n_outputs > 2) + { + const uint64_t bp_base = 368; + size_t log_padded_outputs = 2; + while ((1<<log_padded_outputs) < n_outputs) + ++log_padded_outputs; + uint64_t nlr = 2 * (6 + log_padded_outputs); + const uint64_t bp_size = 32 * (9 + nlr); + const uint64_t bp_clawback = (bp_base * (1<<log_padded_outputs) - bp_size) * 4 / 5; + MDEBUG("clawback on size " << size << ": " << bp_clawback); + size += bp_clawback; + } + return size; +} + uint8_t get_bulletproof_fork() { return 8; } +uint64_t estimate_fee(bool use_per_byte_fee, bool use_rct, int n_inputs, int mixin, int n_outputs, size_t extra_size, bool bulletproof, uint64_t base_fee, uint64_t fee_multiplier, uint64_t fee_quantization_mask) +{ + if (use_per_byte_fee) + { + const size_t estimated_tx_weight = estimate_tx_weight(use_rct, n_inputs, mixin, n_outputs, extra_size, bulletproof); + return calculate_fee_from_weight(base_fee, estimated_tx_weight, fee_multiplier, fee_quantization_mask); + } + else + { + const size_t estimated_tx_size = estimate_tx_size(use_rct, n_inputs, mixin, n_outputs, extra_size, bulletproof); + return calculate_fee(base_fee, estimated_tx_size, fee_multiplier); + } +} + +uint64_t calculate_fee(bool use_per_byte_fee, const cryptonote::transaction &tx, size_t blob_size, uint64_t base_fee, uint64_t fee_multiplier, uint64_t fee_quantization_mask) +{ + if (use_per_byte_fee) + return calculate_fee_from_weight(base_fee, cryptonote::get_transaction_weight(tx, blob_size), fee_multiplier, fee_quantization_mask); + else + return calculate_fee(base_fee, blob_size, fee_multiplier); +} + crypto::hash8 get_short_payment_id(const tools::wallet2::pending_tx &ptx, hw::device &hwdev) { crypto::hash8 payment_id8 = null_hash8; @@ -680,6 +750,7 @@ wallet2::wallet2(network_type nettype, uint64_t kdf_rounds): m_multisig_rescan_k(NULL), m_run(true), m_callback(0), + m_trusted_daemon(false), m_nettype(nettype), m_always_confirm_transfers(true), m_print_ring_members(false), @@ -745,6 +816,8 @@ void wallet2::init_options(boost::program_options::options_description& desc_par const options opts{}; command_line::add_arg(desc_params, opts.daemon_address); command_line::add_arg(desc_params, opts.daemon_host); + command_line::add_arg(desc_params, opts.trusted_daemon); + command_line::add_arg(desc_params, opts.untrusted_daemon); command_line::add_arg(desc_params, opts.password); command_line::add_arg(desc_params, opts.password_file); command_line::add_arg(desc_params, opts.daemon_port); @@ -796,16 +869,17 @@ std::unique_ptr<wallet2> wallet2::make_dummy(const boost::program_options::varia } //---------------------------------------------------------------------------------------------------- -bool wallet2::init(bool rpc, std::string daemon_address, boost::optional<epee::net_utils::http::login> daemon_login, uint64_t upper_transaction_size_limit, bool ssl) +bool wallet2::init(bool rpc, std::string daemon_address, boost::optional<epee::net_utils::http::login> daemon_login, uint64_t upper_transaction_weight_limit, bool ssl, bool trusted_daemon) { m_rpc = rpc; m_checkpoints.init_default_checkpoints(m_nettype); if(m_http_client.is_connected()) m_http_client.disconnect(); m_is_initialized = true; - m_upper_transaction_size_limit = upper_transaction_size_limit; + m_upper_transaction_weight_limit = upper_transaction_weight_limit; m_daemon_address = std::move(daemon_address); m_daemon_login = std::move(daemon_login); + m_trusted_daemon = trusted_daemon; // When switching from light wallet to full wallet, we need to reset the height we got from lw node. return m_http_client.set_server(get_daemon_address(), get_daemon_login(), ssl); } @@ -1115,10 +1189,9 @@ static uint64_t decodeRct(const rct::rctSig & rv, const crypto::key_derivation & switch (rv.type) { case rct::RCTTypeSimple: - case rct::RCTTypeSimpleBulletproof: + case rct::RCTTypeBulletproof: return rct::decodeRctSimple(rv, rct::sk2rct(scalar1), i, mask, hwdev); case rct::RCTTypeFull: - case rct::RCTTypeFullBulletproof: return rct::decodeRct(rv, rct::sk2rct(scalar1), i, mask, hwdev); default: LOG_ERROR("Unsupported rct type: " << rv.type); @@ -1808,34 +1881,7 @@ void wallet2::pull_blocks(uint64_t start_height, uint64_t &blocks_start_height, cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::response res = AUTO_VAL_INIT(res); req.block_ids = short_chain_history; - uint32_t rpc_version; - boost::optional<std::string> result = m_node_rpc_proxy.get_rpc_version(rpc_version); - // no error - if (!!result) - { - // empty string -> not connection - THROW_WALLET_EXCEPTION_IF(result->empty(), tools::error::no_connection_to_daemon, "getversion"); - THROW_WALLET_EXCEPTION_IF(*result == CORE_RPC_STATUS_BUSY, tools::error::daemon_busy, "getversion"); - if (*result != CORE_RPC_STATUS_OK) - { - MDEBUG("Cannot determine daemon RPC version, not asking for pruned blocks"); - req.prune = false; // old daemon - } - } - else - { - if (rpc_version >= MAKE_CORE_RPC_VERSION(1, 7)) - { - MDEBUG("Daemon is recent enough, asking for pruned blocks"); - req.prune = true; - } - else - { - MDEBUG("Daemon is too old, not asking for pruned blocks"); - req.prune = false; - } - } - + req.prune = true; req.start_height = start_height; req.no_miner_tx = m_refresh_type == RefreshNoCoinbase; m_daemon_rpc_mutex.lock(); @@ -2021,9 +2067,11 @@ void wallet2::pull_and_parse_next_blocks(uint64_t start_height, uint64_t &blocks { drop_from_short_history(short_chain_history, 3); + THROW_WALLET_EXCEPTION_IF(prev_blocks.size() != prev_parsed_blocks.size(), error::wallet_internal_error, "size mismatch"); + // prepend the last 3 blocks, should be enough to guard against a block or two's reorg std::vector<parsed_block>::const_reverse_iterator i = prev_parsed_blocks.rbegin(); - for (size_t n = 0; n < std::min((size_t)3, prev_blocks.size()); ++n) + for (size_t n = 0; n < std::min((size_t)3, prev_parsed_blocks.size()); ++n) { short_chain_history.push_front(i->hash); ++i; @@ -2343,6 +2391,7 @@ void wallet2::fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, while (missing_blocks-- > 0) m_blockchain.push_back(crypto::null_hash); // maybe a bit suboptimal, but deque won't do huge reallocs like vector m_blockchain.push_back(m_checkpoints.get_points().at(checkpoint_height)); + m_blockchain.trim(checkpoint_height); short_chain_history.clear(); get_short_chain_history(short_chain_history); } @@ -2554,10 +2603,6 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo break; } - // switch to the new blocks from the daemon - blocks_start_height = next_blocks_start_height; - blocks = std::move(next_blocks); - parsed_blocks = std::move(next_parsed_blocks); first = false; // handle error from async fetching thread @@ -2565,6 +2610,11 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo { throw std::runtime_error("proxy exception in refresh thread"); } + + // switch to the new blocks from the daemon + blocks_start_height = next_blocks_start_height; + blocks = std::move(next_blocks); + parsed_blocks = std::move(next_parsed_blocks); } catch (const tools::error::password_needed&) { @@ -2656,6 +2706,7 @@ bool wallet2::get_rct_distribution(uint64_t &start_height, std::vector<uint64_t> req.amounts.push_back(0); req.from_height = 0; req.cumulative = true; + req.binary = true; m_daemon_rpc_mutex.lock(); bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_distribution", req, res, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); @@ -3388,6 +3439,7 @@ void wallet2::generate(const std::string& wallet_, const epee::wipeable_string& for (const auto &msk: multisig_keys) sc_add(skey.bytes, skey.bytes, rct::sk2rct(msk).bytes); THROW_WALLET_EXCEPTION_IF(!(rct::rct2sk(skey) == spend_secret_key), error::invalid_multisig_seed); + memwipe(&skey, sizeof(rct::key)); m_account.make_multisig(view_secret_key, spend_secret_key, spend_public_key, multisig_keys); m_account.finalize_multisig(spend_public_key); @@ -3750,6 +3802,7 @@ std::string wallet2::make_multisig(const epee::wipeable_string &password, MINFO("Creating multisig address..."); CHECK_AND_ASSERT_THROW_MES(m_account.make_multisig(view_skey, rct::rct2sk(spend_skey), rct::rct2pk(spend_pkey), multisig_keys), "Failed to create multisig wallet due to bad keys"); + memwipe(&spend_skey, sizeof(rct::key)); m_account_public_address = m_account.get_keys().m_account_address; m_watch_only = false; @@ -4901,7 +4954,7 @@ size_t wallet2::pop_best_value(std::vector<size_t> &unused_indices, const std::v // returns: // direct return: amount of money found // modified reference: selected_transfers, a list of iterators/indices of input sources -uint64_t wallet2::select_transfers(uint64_t needed_money, std::vector<size_t> unused_transfers_indices, std::vector<size_t>& selected_transfers, bool trusted_daemon) const +uint64_t wallet2::select_transfers(uint64_t needed_money, std::vector<size_t> unused_transfers_indices, std::vector<size_t>& selected_transfers) const { uint64_t found_money = 0; selected_transfers.reserve(unused_transfers_indices.size()); @@ -4944,69 +4997,6 @@ void wallet2::add_unconfirmed_tx(const cryptonote::transaction& tx, uint64_t amo } //---------------------------------------------------------------------------------------------------- -void wallet2::transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outs_count, const std::vector<size_t> &unused_transfers_indices, - uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx& ptx, bool trusted_daemon) -{ - transfer(dsts, fake_outs_count, unused_transfers_indices, unlock_time, fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), tx, ptx, trusted_daemon); -} -//---------------------------------------------------------------------------------------------------- -void wallet2::transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outs_count, const std::vector<size_t> &unused_transfers_indices, - uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, bool trusted_daemon) -{ - cryptonote::transaction tx; - pending_tx ptx; - transfer(dsts, fake_outs_count, unused_transfers_indices, unlock_time, fee, extra, tx, ptx, trusted_daemon); -} - -namespace { -// split_amounts(vector<cryptonote::tx_destination_entry> dsts, size_t num_splits) -// -// split amount for each dst in dsts into num_splits parts -// and make num_splits new vector<crypt...> instances to hold these new amounts -std::vector<std::vector<cryptonote::tx_destination_entry>> split_amounts( - std::vector<cryptonote::tx_destination_entry> dsts, size_t num_splits) -{ - std::vector<std::vector<cryptonote::tx_destination_entry>> retVal; - - if (num_splits <= 1) - { - retVal.push_back(dsts); - return retVal; - } - - // for each split required - for (size_t i=0; i < num_splits; i++) - { - std::vector<cryptonote::tx_destination_entry> new_dsts; - - // for each destination - for (size_t j=0; j < dsts.size(); j++) - { - cryptonote::tx_destination_entry de; - uint64_t amount; - - amount = dsts[j].amount; - amount = amount / num_splits; - - // if last split, add remainder - if (i + 1 == num_splits) - { - amount += dsts[j].amount % num_splits; - } - - de.addr = dsts[j].addr; - de.amount = amount; - - new_dsts.push_back(de); - } - - retVal.push_back(new_dsts); - } - - return retVal; -} -} // anonymous namespace -//---------------------------------------------------------------------------------------------------- crypto::hash wallet2::get_payment_id(const pending_tx &ptx) const { std::vector<tx_extra_field> tx_extra_fields; @@ -5265,10 +5255,18 @@ bool wallet2::sign_tx(unsigned_tx_set &exported_txs, std::vector<wallet2::pendin LOG_PRINT_L1(" " << (n+1) << ": " << sd.sources.size() << " inputs, ring size " << sd.sources[0].outputs.size()); signed_txes.ptx.push_back(pending_tx()); tools::wallet2::pending_tx &ptx = signed_txes.ptx.back(); + rct::RangeProofType range_proof_type = rct::RangeProofBorromean; + if (sd.use_bulletproofs) + { + range_proof_type = rct::RangeProofBulletproof; + for (const rct::Bulletproof &proof: ptx.tx.rct_signatures.p.bulletproofs) + if (proof.V.size() > 1) + range_proof_type = rct::RangeProofPaddedBulletproof; + } crypto::secret_key tx_key; std::vector<crypto::secret_key> additional_tx_keys; rct::multisig_out msout; - bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sd.sources, sd.splitted_dsts, sd.change_dts.addr, sd.extra, ptx.tx, sd.unlock_time, tx_key, additional_tx_keys, sd.use_rct, sd.use_bulletproofs, m_multisig ? &msout : NULL); + bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sd.sources, sd.splitted_dsts, sd.change_dts.addr, sd.extra, ptx.tx, sd.unlock_time, tx_key, additional_tx_keys, sd.use_rct, range_proof_type, m_multisig ? &msout : NULL); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sd.sources, sd.splitted_dsts, sd.unlock_time, m_nettype); // we don't test tx size, because we don't know the current limit, due to not having a blockchain, // and it's a bit pointless to fail there anyway, since it'd be a (good) guess only. We sign anyway, @@ -5681,7 +5679,15 @@ bool wallet2::sign_multisig_tx(multisig_tx_set &exported_txs, std::vector<crypto cryptonote::transaction tx; rct::multisig_out msout = ptx.multisig_sigs.front().msout; auto sources = sd.sources; - bool r = cryptonote::construct_tx_with_tx_key(m_account.get_keys(), m_subaddresses, sources, sd.splitted_dsts, ptx.change_dts.addr, sd.extra, tx, sd.unlock_time, ptx.tx_key, ptx.additional_tx_keys, sd.use_rct, sd.use_bulletproofs, &msout, false); + rct::RangeProofType range_proof_type = rct::RangeProofBorromean; + if (sd.use_bulletproofs) + { + range_proof_type = rct::RangeProofBulletproof; + for (const rct::Bulletproof &proof: ptx.tx.rct_signatures.p.bulletproofs) + if (proof.V.size() > 1) + range_proof_type = rct::RangeProofPaddedBulletproof; + } + bool r = cryptonote::construct_tx_with_tx_key(m_account.get_keys(), m_subaddresses, sources, sd.splitted_dsts, ptx.change_dts.addr, sd.extra, tx, sd.unlock_time, ptx.tx_key, ptx.additional_tx_keys, sd.use_rct, range_proof_type, &msout, false); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sd.sources, sd.splitted_dsts, sd.unlock_time, m_nettype); THROW_WALLET_EXCEPTION_IF(get_transaction_prefix_hash (tx) != get_transaction_prefix_hash(ptx.tx), @@ -5781,9 +5787,18 @@ bool wallet2::sign_multisig_tx_from_file(const std::string &filename, std::vecto //---------------------------------------------------------------------------------------------------- uint64_t wallet2::get_fee_multiplier(uint32_t priority, int fee_algorithm) const { - static const uint64_t old_multipliers[3] = {1, 2, 3}; - static const uint64_t new_multipliers[3] = {1, 20, 166}; - static const uint64_t newer_multipliers[4] = {1, 4, 20, 166}; + static const struct + { + size_t count; + uint64_t multipliers[4]; + } + multipliers[] = + { + { 3, {1, 2, 3} }, + { 3, {1, 20, 166} }, + { 4, {1, 4, 20, 166} }, + { 4, {1, 5, 25, 1000} }, + }; if (fee_algorithm == -1) fee_algorithm = get_fee_algorithm(); @@ -5799,47 +5814,68 @@ uint64_t wallet2::get_fee_multiplier(uint32_t priority, int fee_algorithm) const priority = 1; } + THROW_WALLET_EXCEPTION_IF(fee_algorithm < 0 || fee_algorithm > 3, error::invalid_priority); + // 1 to 3/4 are allowed as priorities - uint32_t max_priority = (fee_algorithm >= 2) ? 4 : 3; + const uint32_t max_priority = multipliers[fee_algorithm].count; if (priority >= 1 && priority <= max_priority) { - switch (fee_algorithm) - { - case 0: return old_multipliers[priority-1]; - case 1: return new_multipliers[priority-1]; - case 2: return newer_multipliers[priority-1]; - default: THROW_WALLET_EXCEPTION_IF (true, error::invalid_priority); - } + return multipliers[fee_algorithm].multipliers[priority-1]; } THROW_WALLET_EXCEPTION_IF (false, error::invalid_priority); return 1; } //---------------------------------------------------------------------------------------------------- -uint64_t wallet2::get_dynamic_per_kb_fee_estimate() const +uint64_t wallet2::get_dynamic_base_fee_estimate() const { uint64_t fee; - boost::optional<std::string> result = m_node_rpc_proxy.get_dynamic_per_kb_fee_estimate(FEE_ESTIMATE_GRACE_BLOCKS, fee); + boost::optional<std::string> result = m_node_rpc_proxy.get_dynamic_base_fee_estimate(FEE_ESTIMATE_GRACE_BLOCKS, fee); if (!result) return fee; - LOG_PRINT_L1("Failed to query per kB fee, using " << print_money(FEE_PER_KB)); - return FEE_PER_KB; + const uint64_t base_fee = use_fork_rules(HF_VERSION_PER_BYTE_FEE) ? FEE_PER_BYTE : FEE_PER_KB; + LOG_PRINT_L1("Failed to query base fee, using " << print_money(base_fee)); + return base_fee; } //---------------------------------------------------------------------------------------------------- -uint64_t wallet2::get_per_kb_fee() const +uint64_t wallet2::get_base_fee() const { if(m_light_wallet) - return m_light_wallet_per_kb_fee; + { + if (use_fork_rules(HF_VERSION_PER_BYTE_FEE)) + return m_light_wallet_per_kb_fee / 1024; + else + return m_light_wallet_per_kb_fee; + } bool use_dyn_fee = use_fork_rules(HF_VERSION_DYNAMIC_FEE, -720 * 1); if (!use_dyn_fee) return FEE_PER_KB; - return get_dynamic_per_kb_fee_estimate(); + return get_dynamic_base_fee_estimate(); +} +//---------------------------------------------------------------------------------------------------- +uint64_t wallet2::get_fee_quantization_mask() const +{ + if(m_light_wallet) + { + return 1; // TODO + } + bool use_per_byte_fee = use_fork_rules(HF_VERSION_PER_BYTE_FEE, 0); + if (!use_per_byte_fee) + return 1; + + uint64_t fee_quantization_mask; + boost::optional<std::string> result = m_node_rpc_proxy.get_fee_quantization_mask(fee_quantization_mask); + if (result) + return 1; + return fee_quantization_mask; } //---------------------------------------------------------------------------------------------------- int wallet2::get_fee_algorithm() const { - // changes at v3 and v5 + // changes at v3, v5, v8 + if (use_fork_rules(HF_VERSION_PER_BYTE_FEE, 0)) + return 3; if (use_fork_rules(5, 0)) return 2; if (use_fork_rules(3, -720 * 14)) @@ -5847,19 +5883,39 @@ int wallet2::get_fee_algorithm() const return 0; } //------------------------------------------------------------------------------------------------------------------------------ +uint64_t wallet2::get_min_ring_size() const +{ + if (use_fork_rules(8, 10)) + return 11; + if (use_fork_rules(7, 10)) + return 7; + if (use_fork_rules(6, 10)) + return 5; + if (use_fork_rules(2, 10)) + return 3; + return 0; +} +//------------------------------------------------------------------------------------------------------------------------------ +uint64_t wallet2::get_max_ring_size() const +{ + if (use_fork_rules(8, 10)) + return 11; + return 0; +} +//------------------------------------------------------------------------------------------------------------------------------ uint64_t wallet2::adjust_mixin(uint64_t mixin) const { - if (mixin < 6 && use_fork_rules(7, 10)) { - MWARNING("Requested ring size " << (mixin + 1) << " too low for hard fork 7, using 7"); - mixin = 6; - } - else if (mixin < 4 && use_fork_rules(6, 10)) { - MWARNING("Requested ring size " << (mixin + 1) << " too low for hard fork 6, using 5"); - mixin = 4; + const uint64_t min_ring_size = get_min_ring_size(); + if (mixin + 1 < min_ring_size) + { + MWARNING("Requested ring size " << (mixin + 1) << " too low, using " << min_ring_size); + mixin = min_ring_size-1; } - else if (mixin < 2 && use_fork_rules(2, 10)) { - MWARNING("Requested ring size " << (mixin + 1) << " too low for hard fork 2, using 3"); - mixin = 2; + const uint64_t max_ring_size = get_max_ring_size(); + if (max_ring_size && mixin + 1 > max_ring_size) + { + MWARNING("Requested ring size " << (mixin + 1) << " too high, using " << max_ring_size); + mixin = max_ring_size-1; } return mixin; } @@ -5871,7 +5927,10 @@ uint32_t wallet2::adjust_priority(uint32_t priority) try { // check if there's a backlog in the tx pool - const double fee_level = get_fee_multiplier(1) * get_per_kb_fee() * (12/(double)13) / (double)1024; + const bool use_per_byte_fee = use_fork_rules(HF_VERSION_PER_BYTE_FEE, 0); + const uint64_t base_fee = get_base_fee(); + const uint64_t fee_multiplier = get_fee_multiplier(1); + const double fee_level = fee_multiplier * base_fee * (use_per_byte_fee ? 1 : (12/(double)13 / (double)1024)); const std::vector<std::pair<uint64_t, uint64_t>> blocks = estimate_backlog({std::make_pair(fee_level, fee_level)}); if (blocks.size() != 1) { @@ -5885,10 +5944,10 @@ uint32_t wallet2::adjust_priority(uint32_t priority) } // get the current full reward zone - uint64_t block_size_limit = 0; - const auto result = m_node_rpc_proxy.get_block_size_limit(block_size_limit); + uint64_t block_weight_limit = 0; + const auto result = m_node_rpc_proxy.get_block_weight_limit(block_weight_limit); throw_on_rpc_response_error(result, "get_info"); - const uint64_t full_reward_zone = block_size_limit / 2; + const uint64_t full_reward_zone = block_weight_limit / 2; // get the last N block headers and sum the block sizes const size_t N = 10; @@ -5912,14 +5971,14 @@ uint32_t wallet2::adjust_priority(uint32_t priority) MERROR("Bad blockheaders size"); return priority; } - size_t block_size_sum = 0; + size_t block_weight_sum = 0; for (const cryptonote::block_header_response &i : getbh_res.headers) { - block_size_sum += i.block_size; + block_weight_sum += i.block_weight; } // estimate how 'full' the last N blocks are - const size_t P = 100 * block_size_sum / (N * full_reward_zone); + const size_t P = 100 * block_weight_sum / (N * full_reward_zone); MINFO((boost::format("The last %d blocks fill roughly %d%% of the full reward zone.") % N % P).str()); if (P > 80) { @@ -5937,115 +5996,6 @@ uint32_t wallet2::adjust_priority(uint32_t priority) return priority; } //---------------------------------------------------------------------------------------------------- -// separated the call(s) to wallet2::transfer into their own function -// -// this function will make multiple calls to wallet2::transfer if multiple -// transactions will be required -std::vector<wallet2::pending_tx> wallet2::create_transactions(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, bool trusted_daemon) -{ - const std::vector<size_t> unused_transfers_indices = select_available_outputs_from_histogram(fake_outs_count + 1, true, true, true, trusted_daemon); - - const uint64_t fee_per_kb = get_per_kb_fee(); - const uint64_t fee_multiplier = get_fee_multiplier(priority, get_fee_algorithm()); - - // failsafe split attempt counter - size_t attempt_count = 0; - - for(attempt_count = 1; ;attempt_count++) - { - size_t num_tx = 0.5 + pow(1.7,attempt_count-1); - - auto split_values = split_amounts(dsts, num_tx); - - // Throw if split_amounts comes back with a vector of size different than it should - if (split_values.size() != num_tx) - { - throw std::runtime_error("Splitting transactions returned a number of potential tx not equal to what was requested"); - } - - std::vector<pending_tx> ptx_vector; - try - { - // for each new destination vector (i.e. for each new tx) - for (auto & dst_vector : split_values) - { - cryptonote::transaction tx; - pending_tx ptx; - - // loop until fee is met without increasing tx size to next KB boundary. - const size_t estimated_tx_size = estimate_tx_size(false, unused_transfers_indices.size(), fake_outs_count, dst_vector.size(), extra.size(), false); - uint64_t needed_fee = calculate_fee(fee_per_kb, estimated_tx_size, fee_multiplier); - do - { - transfer(dst_vector, fake_outs_count, unused_transfers_indices, unlock_time, needed_fee, extra, tx, ptx, trusted_daemon); - auto txBlob = t_serializable_object_to_blob(ptx.tx); - needed_fee = calculate_fee(fee_per_kb, txBlob, fee_multiplier); - } while (ptx.fee < needed_fee); - - ptx_vector.push_back(ptx); - - // mark transfers to be used as "spent" - for(size_t idx: ptx.selected_transfers) - { - set_spent(idx, 0); - } - } - - // if we made it this far, we've selected our transactions. committing them will mark them spent, - // so this is a failsafe in case they don't go through - // unmark pending tx transfers as spent - for (auto & ptx : ptx_vector) - { - // mark transfers to be used as not spent - for(size_t idx2: ptx.selected_transfers) - { - set_unspent(idx2); - } - - } - - // if we made it this far, we're OK to actually send the transactions - return ptx_vector; - - } - // only catch this here, other exceptions need to pass through to the calling function - catch (const tools::error::tx_too_big& e) - { - - // unmark pending tx transfers as spent - for (auto & ptx : ptx_vector) - { - // mark transfers to be used as not spent - for(size_t idx2: ptx.selected_transfers) - { - set_unspent(idx2); - } - } - - if (attempt_count >= MAX_SPLIT_ATTEMPTS) - { - throw; - } - } - catch (...) - { - // in case of some other exception, make sure any tx in queue are marked unspent again - - // unmark pending tx transfers as spent - for (auto & ptx : ptx_vector) - { - // mark transfers to be used as not spent - for(size_t idx2: ptx.selected_transfers) - { - set_unspent(idx2); - } - } - - throw; - } - } -} - bool wallet2::set_ring_database(const std::string &filename) { m_ring_database = filename; @@ -6218,7 +6168,7 @@ bool wallet2::find_and_save_rings(bool force) return true; } -bool wallet2::blackball_output(const crypto::public_key &output) +bool wallet2::blackball_output(const std::pair<uint64_t, uint64_t> &output) { if (!m_ringdb) return false; @@ -6226,7 +6176,7 @@ bool wallet2::blackball_output(const crypto::public_key &output) catch (const std::exception &e) { return false; } } -bool wallet2::set_blackballed_outputs(const std::vector<crypto::public_key> &outputs, bool add) +bool wallet2::set_blackballed_outputs(const std::vector<std::pair<uint64_t, uint64_t>> &outputs, bool add) { if (!m_ringdb) return false; @@ -6235,14 +6185,13 @@ bool wallet2::set_blackballed_outputs(const std::vector<crypto::public_key> &out bool ret = true; if (!add) ret &= m_ringdb->clear_blackballs(); - for (const auto &output: outputs) - ret &= m_ringdb->blackball(output); + ret &= m_ringdb->blackball(outputs); return ret; } catch (const std::exception &e) { return false; } } -bool wallet2::unblackball_output(const crypto::public_key &output) +bool wallet2::unblackball_output(const std::pair<uint64_t, uint64_t> &output) { if (!m_ringdb) return false; @@ -6250,7 +6199,7 @@ bool wallet2::unblackball_output(const crypto::public_key &output) catch (const std::exception &e) { return false; } } -bool wallet2::is_output_blackballed(const crypto::public_key &output) const +bool wallet2::is_output_blackballed(const std::pair<uint64_t, uint64_t> &output) const { if (!m_ringdb) return false; @@ -6295,8 +6244,8 @@ bool wallet2::tx_add_fake_output(std::vector<std::vector<tools::wallet2::get_out CHECK_AND_ASSERT_MES(!outs.empty(), false, "internal error: outs is empty"); if (std::find(outs.back().begin(), outs.back().end(), item) != outs.back().end()) // don't add duplicates return false; - if (is_output_blackballed(output_public_key)) // don't add blackballed outputs - return false; +// if (is_output_blackballed(output_public_key)) // don't add blackballed outputs +// return false; outs.back().push_back(item); return true; } @@ -6476,6 +6425,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> req_t.from_height = std::max<uint64_t>(segregation_fork_height, RECENT_OUTPUT_BLOCKS) - RECENT_OUTPUT_BLOCKS; req_t.to_height = segregation_fork_height + 1; req_t.cumulative = true; + req_t.binary = true; m_daemon_rpc_mutex.lock(); bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_distribution", req_t, resp_t, m_http_client, rpc_timeout * 1000); m_daemon_rpc_mutex.unlock(); @@ -6792,6 +6742,8 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> if (seen_indices.count(i)) continue; + if (is_output_blackballed(std::make_pair(amount, i))) // don't add blackballed outputs + continue; seen_indices.emplace(i); LOG_PRINT_L2("picking " << i << " as " << type); @@ -6814,7 +6766,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "get_outs.bin"); THROW_WALLET_EXCEPTION_IF(daemon_resp.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_outs.bin"); - THROW_WALLET_EXCEPTION_IF(daemon_resp.status != CORE_RPC_STATUS_OK, error::get_random_outs_error, daemon_resp.status); + THROW_WALLET_EXCEPTION_IF(daemon_resp.status != CORE_RPC_STATUS_OK, error::get_outs_error, daemon_resp.status); THROW_WALLET_EXCEPTION_IF(daemon_resp.outs.size() != req.outputs.size(), error::wallet_internal_error, "daemon returned wrong response for get_outs.bin, wrong amounts count = " + std::to_string(daemon_resp.outs.size()) + ", expected " + std::to_string(req.outputs.size())); @@ -6948,7 +6900,7 @@ void wallet2::transfer_selected(const std::vector<cryptonote::tx_destination_ent THROW_WALLET_EXCEPTION_IF(m_multisig, error::wallet_internal_error, "Multisig wallets cannot spend non rct outputs"); - uint64_t upper_transaction_size_limit = get_upper_transaction_size_limit(); + uint64_t upper_transaction_weight_limit = get_upper_transaction_weight_limit(); uint64_t needed_money = fee; LOG_PRINT_L2("transfer: starting with fee " << print_money (needed_money)); @@ -7050,10 +7002,10 @@ void wallet2::transfer_selected(const std::vector<cryptonote::tx_destination_ent std::vector<crypto::secret_key> additional_tx_keys; rct::multisig_out msout; LOG_PRINT_L2("constructing tx"); - bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sources, splitted_dsts, change_dts.addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, false, m_multisig ? &msout : NULL); + bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sources, splitted_dsts, change_dts.addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, rct::RangeProofBulletproof, m_multisig ? &msout : NULL); LOG_PRINT_L2("constructed tx, r="<<r); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sources, splitted_dsts, unlock_time, m_nettype); - THROW_WALLET_EXCEPTION_IF(upper_transaction_size_limit <= get_object_blobsize(tx), error::tx_too_big, tx, upper_transaction_size_limit); + THROW_WALLET_EXCEPTION_IF(upper_transaction_weight_limit <= get_transaction_weight(tx), error::tx_too_big, tx, upper_transaction_weight_limit); std::string key_images; bool all_are_txin_to_key = std::all_of(tx.vin.begin(), tx.vin.end(), [&](const txin_v& s_e) -> bool @@ -7099,13 +7051,13 @@ void wallet2::transfer_selected(const std::vector<cryptonote::tx_destination_ent void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry> dsts, const std::vector<size_t>& selected_transfers, size_t fake_outputs_count, std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, - uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx &ptx, bool bulletproof) + uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx &ptx, rct::RangeProofType range_proof_type) { using namespace cryptonote; // throw if attempting a transaction with no destinations THROW_WALLET_EXCEPTION_IF(dsts.empty(), error::zero_destination); - uint64_t upper_transaction_size_limit = get_upper_transaction_size_limit(); + uint64_t upper_transaction_weight_limit = get_upper_transaction_weight_limit(); uint64_t needed_money = fee; LOG_PRINT_L2("transfer_selected_rct: starting with fee " << print_money (needed_money)); LOG_PRINT_L2("selected transfers: " << strjoin(selected_transfers, " ")); @@ -7255,10 +7207,10 @@ void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry rct::multisig_out msout; LOG_PRINT_L2("constructing tx"); auto sources_copy = sources; - bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sources, splitted_dsts, change_dts.addr, extra, tx, unlock_time, tx_key, additional_tx_keys, true, bulletproof, m_multisig ? &msout : NULL); + bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sources, splitted_dsts, change_dts.addr, extra, tx, unlock_time, tx_key, additional_tx_keys, true, range_proof_type, m_multisig ? &msout : NULL); LOG_PRINT_L2("constructed tx, r="<<r); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sources, dsts, unlock_time, m_nettype); - THROW_WALLET_EXCEPTION_IF(upper_transaction_size_limit <= get_object_blobsize(tx), error::tx_too_big, tx, upper_transaction_size_limit); + THROW_WALLET_EXCEPTION_IF(upper_transaction_weight_limit <= get_transaction_weight(tx), error::tx_too_big, tx, upper_transaction_weight_limit); // work out the permutation done on sources std::vector<size_t> ins_order; @@ -7300,10 +7252,10 @@ void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry LOG_PRINT_L2("Creating supplementary multisig transaction"); cryptonote::transaction ms_tx; auto sources_copy_copy = sources_copy; - bool r = cryptonote::construct_tx_with_tx_key(m_account.get_keys(), m_subaddresses, sources_copy_copy, splitted_dsts, change_dts.addr, extra, ms_tx, unlock_time,tx_key, additional_tx_keys, true, bulletproof, &msout, false); + bool r = cryptonote::construct_tx_with_tx_key(m_account.get_keys(), m_subaddresses, sources_copy_copy, splitted_dsts, change_dts.addr, extra, ms_tx, unlock_time,tx_key, additional_tx_keys, true, range_proof_type, &msout, false); LOG_PRINT_L2("constructed tx, r="<<r); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sources, splitted_dsts, unlock_time, m_nettype); - THROW_WALLET_EXCEPTION_IF(upper_transaction_size_limit <= get_object_blobsize(tx), error::tx_too_big, tx, upper_transaction_size_limit); + THROW_WALLET_EXCEPTION_IF(upper_transaction_weight_limit <= get_transaction_weight(tx), error::tx_too_big, tx, upper_transaction_weight_limit); THROW_WALLET_EXCEPTION_IF(cryptonote::get_transaction_prefix_hash(ms_tx) != prefix_hash, error::wallet_internal_error, "Multisig txes do not share prefix"); multisig_sigs.push_back({ms_tx.rct_signatures, multisig_signers[signer_index], new_used_L, std::unordered_set<crypto::public_key>(), msout}); @@ -7951,7 +7903,7 @@ bool wallet2::light_wallet_key_image_is_ours(const crypto::key_image& key_image, // This system allows for sending (almost) the entire balance, since it does // not generate spurious change in all txes, thus decreasing the instantaneous // usable balance. -std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices, bool trusted_daemon) +std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices) { //ensure device is let in NONE mode in any case hw::device &hwdev = m_account.get_device(); @@ -7971,11 +7923,11 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp std::vector<cryptonote::tx_destination_entry> dsts; cryptonote::transaction tx; pending_tx ptx; - size_t bytes; + size_t weight; uint64_t needed_fee; std::vector<std::vector<tools::wallet2::get_outs_entry>> outs; - TX() : bytes(0), needed_fee(0) {} + TX() : weight(0), needed_fee(0) {} void add(const account_public_address &addr, bool is_subaddress, uint64_t amount, unsigned int original_output_index, bool merge_destinations) { if (merge_destinations) @@ -8003,12 +7955,15 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp std::vector<TX> txes; bool adding_fee; // true if new outputs go towards fee, rather than destinations uint64_t needed_fee, available_for_fee = 0; - uint64_t upper_transaction_size_limit = get_upper_transaction_size_limit(); + uint64_t upper_transaction_weight_limit = get_upper_transaction_weight_limit(); + const bool use_per_byte_fee = use_fork_rules(HF_VERSION_PER_BYTE_FEE, 0); const bool use_rct = use_fork_rules(4, 0); const bool bulletproof = use_fork_rules(get_bulletproof_fork(), 0); + const rct::RangeProofType range_proof_type = bulletproof ? rct::RangeProofPaddedBulletproof : rct::RangeProofBorromean; - const uint64_t fee_per_kb = get_per_kb_fee(); + const uint64_t base_fee = get_base_fee(); const uint64_t fee_multiplier = get_fee_multiplier(priority, get_fee_algorithm()); + const uint64_t fee_quantization_mask = get_fee_quantization_mask(); // throw if attempting a transaction with no destinations THROW_WALLET_EXCEPTION_IF(dsts.empty(), error::zero_destination); @@ -8039,7 +7994,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp // early out if we know we can't make it anyway // we could also check for being within FEE_PER_KB, but if the fee calculation // ever changes, this might be missed, so let this go through - const uint64_t min_fee = (fee_multiplier * fee_per_kb * estimate_tx_size(use_rct, 1, fake_outs_count, 2, extra.size(), bulletproof)) / 1024; + const uint64_t min_fee = (fee_multiplier * base_fee * estimate_tx_size(use_rct, 1, fake_outs_count, 2, extra.size(), bulletproof)) / 1024; uint64_t balance_subtotal = 0; uint64_t unlocked_balance_subtotal = 0; for (uint32_t index_minor : subaddr_indices) @@ -8057,11 +8012,11 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp LOG_PRINT_L2("Candidate subaddress index for spending: " << i); // determine threshold for fractional amount - const size_t tx_size_one_ring = estimate_tx_size(use_rct, 1, fake_outs_count, 2, 0, bulletproof); - const size_t tx_size_two_rings = estimate_tx_size(use_rct, 2, fake_outs_count, 2, 0, bulletproof); - THROW_WALLET_EXCEPTION_IF(tx_size_one_ring > tx_size_two_rings, error::wallet_internal_error, "Estimated tx size with 1 input is larger than with 2 inputs!"); - const size_t tx_size_per_ring = tx_size_two_rings - tx_size_one_ring; - const uint64_t fractional_threshold = (fee_multiplier * fee_per_kb * tx_size_per_ring) / 1024; + const size_t tx_weight_one_ring = estimate_tx_weight(use_rct, 1, fake_outs_count, 2, 0, bulletproof); + const size_t tx_weight_two_rings = estimate_tx_weight(use_rct, 2, fake_outs_count, 2, 0, bulletproof); + THROW_WALLET_EXCEPTION_IF(tx_weight_one_ring > tx_weight_two_rings, error::wallet_internal_error, "Estimated tx weight with 1 input is larger than with 2 inputs!"); + const size_t tx_weight_per_ring = tx_weight_two_rings - tx_weight_one_ring; + const uint64_t fractional_threshold = (fee_multiplier * base_fee * tx_weight_per_ring) / (use_per_byte_fee ? 1 : 1024); // gather all dust and non-dust outputs belonging to specified subaddresses size_t num_nondust_outputs = 0; @@ -8154,7 +8109,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp { // this is used to build a tx that's 1 or 2 inputs, and 2 outputs, which // will get us a known fee. - uint64_t estimated_fee = calculate_fee(fee_per_kb, estimate_rct_tx_size(2, fake_outs_count, 2, extra.size(), bulletproof), fee_multiplier); + uint64_t estimated_fee = estimate_fee(use_per_byte_fee, use_rct, 2, fake_outs_count, 2, extra.size(), bulletproof, base_fee, fee_multiplier, fee_quantization_mask); preferred_inputs = pick_preferred_rct_inputs(needed_money + estimated_fee, subaddr_account, subaddr_indices); if (!preferred_inputs.empty()) { @@ -8196,7 +8151,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp while ((!dsts.empty() && dsts[0].amount > 0) || adding_fee || !preferred_inputs.empty() || should_pick_a_second_output(use_rct, txes.back().selected_transfers.size(), *unused_transfers_indices, *unused_dust_indices)) { TX &tx = txes.back(); - LOG_PRINT_L2("Start of loop with " << unused_transfers_indices->size() << " " << unused_dust_indices->size()); + LOG_PRINT_L2("Start of loop with " << unused_transfers_indices->size() << " " << unused_dust_indices->size() << ", tx.dsts.size() " << tx.dsts.size()); LOG_PRINT_L2("unused_transfers_indices: " << strjoin(*unused_transfers_indices, " ")); LOG_PRINT_L2("unused_dust_indices: " << strjoin(*unused_dust_indices, " ")); LOG_PRINT_L2("dsts size " << dsts.size() << ", first " << (dsts.empty() ? "-" : cryptonote::print_money(dsts[0].amount))); @@ -8259,7 +8214,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp } else { - while (!dsts.empty() && dsts[0].amount <= available_amount && estimate_tx_size(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size(), extra.size(), bulletproof) < TX_SIZE_TARGET(upper_transaction_size_limit)) + while (!dsts.empty() && dsts[0].amount <= available_amount && estimate_tx_weight(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size()+1, extra.size(), bulletproof) < TX_WEIGHT_TARGET(upper_transaction_weight_limit)) { // we can fully pay that destination LOG_PRINT_L2("We can fully pay " << get_account_address_as_str(m_nettype, dsts[0].is_subaddress, dsts[0].addr) << @@ -8271,7 +8226,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp ++original_output_index; } - if (available_amount > 0 && !dsts.empty() && estimate_tx_size(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size(), extra.size(), bulletproof) < TX_SIZE_TARGET(upper_transaction_size_limit)) { + if (available_amount > 0 && !dsts.empty() && estimate_tx_weight(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size()+1, extra.size(), bulletproof) < TX_WEIGHT_TARGET(upper_transaction_weight_limit)) { // we can partially fill that destination LOG_PRINT_L2("We can partially pay " << get_account_address_as_str(m_nettype, dsts[0].is_subaddress, dsts[0].addr) << " for " << print_money(available_amount) << "/" << print_money(dsts[0].amount)); @@ -8283,7 +8238,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp // here, check if we need to sent tx and start a new one LOG_PRINT_L2("Considering whether to create a tx now, " << tx.selected_transfers.size() << " inputs, tx limit " - << upper_transaction_size_limit); + << upper_transaction_weight_limit); bool try_tx = false; // if we have preferred picks, but haven't yet used all of them, continue if (preferred_inputs.empty()) @@ -8295,8 +8250,8 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp } else { - const size_t estimated_rct_tx_size = estimate_tx_size(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size(), extra.size(), bulletproof); - try_tx = dsts.empty() || (estimated_rct_tx_size >= TX_SIZE_TARGET(upper_transaction_size_limit)); + const size_t estimated_rct_tx_weight = estimate_tx_weight(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size()+1, extra.size(), bulletproof); + try_tx = dsts.empty() || (estimated_rct_tx_weight >= TX_WEIGHT_TARGET(upper_transaction_weight_limit)); } } @@ -8304,8 +8259,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp cryptonote::transaction test_tx; pending_tx test_ptx; - const size_t estimated_tx_size = estimate_tx_size(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size(), extra.size(), bulletproof); - needed_fee = calculate_fee(fee_per_kb, estimated_tx_size, fee_multiplier); + needed_fee = estimate_fee(use_per_byte_fee, use_rct ,tx.selected_transfers.size(), fake_outs_count, tx.dsts.size()+1, extra.size(), bulletproof, base_fee, fee_multiplier, fee_multiplier); uint64_t inputs = 0, outputs = needed_fee; for (size_t idx: tx.selected_transfers) inputs += m_transfers[idx].amount(); @@ -8322,14 +8276,14 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp tx.selected_transfers.size() << " inputs"); if (use_rct) transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, - test_tx, test_ptx, bulletproof); + test_tx, test_ptx, range_proof_type); else transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); auto txBlob = t_serializable_object_to_blob(test_ptx.tx); - needed_fee = calculate_fee(fee_per_kb, txBlob, fee_multiplier); + needed_fee = calculate_fee(use_per_byte_fee, test_ptx.tx, txBlob.size(), base_fee, fee_multiplier, fee_quantization_mask); available_for_fee = test_ptx.fee + test_ptx.change_dts.amount + (!test_ptx.dust_added_to_fee ? test_ptx.dust : 0); - LOG_PRINT_L2("Made a " << get_size_string(txBlob) << " tx, with " << print_money(available_for_fee) << " available for fee (" << + LOG_PRINT_L2("Made a " << get_weight_string(test_ptx.tx, txBlob.size()) << " tx, with " << print_money(available_for_fee) << " available for fee (" << print_money(needed_fee) << " needed)"); if (needed_fee > available_for_fee && !dsts.empty() && dsts[0].amount > 0) @@ -8365,22 +8319,22 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp while (needed_fee > test_ptx.fee) { if (use_rct) transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, - test_tx, test_ptx, bulletproof); + test_tx, test_ptx, range_proof_type); else transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); txBlob = t_serializable_object_to_blob(test_ptx.tx); - needed_fee = calculate_fee(fee_per_kb, txBlob, fee_multiplier); - LOG_PRINT_L2("Made an attempt at a final " << get_size_string(txBlob) << " tx, with " << print_money(test_ptx.fee) << + needed_fee = calculate_fee(use_per_byte_fee, test_ptx.tx, txBlob.size(), base_fee, fee_multiplier, fee_quantization_mask); + LOG_PRINT_L2("Made an attempt at a final " << get_weight_string(test_ptx.tx, txBlob.size()) << " tx, with " << print_money(test_ptx.fee) << " fee and " << print_money(test_ptx.change_dts.amount) << " change"); } - LOG_PRINT_L2("Made a final " << get_size_string(txBlob) << " tx, with " << print_money(test_ptx.fee) << + LOG_PRINT_L2("Made a final " << get_weight_string(test_ptx.tx, txBlob.size()) << " tx, with " << print_money(test_ptx.fee) << " fee and " << print_money(test_ptx.change_dts.amount) << " change"); tx.tx = test_tx; tx.ptx = test_ptx; - tx.bytes = txBlob.size(); + tx.weight = get_transaction_weight(test_tx, txBlob.size()); tx.outs = outs; tx.needed_fee = needed_fee; accumulated_fee += test_ptx.fee; @@ -8438,7 +8392,7 @@ skip_tx: extra, /* const std::vector<uint8_t>& extra, */ test_tx, /* OUT cryptonote::transaction& tx, */ test_ptx, /* OUT cryptonote::transaction& tx, */ - bulletproof); + range_proof_type); } else { transfer_selected(tx.dsts, tx.selected_transfers, @@ -8455,7 +8409,7 @@ skip_tx: auto txBlob = t_serializable_object_to_blob(test_ptx.tx); tx.tx = test_tx; tx.ptx = test_ptx; - tx.bytes = txBlob.size(); + tx.weight = get_transaction_weight(test_tx, txBlob.size()); } std::vector<wallet2::pending_tx> ptx_vector; @@ -8466,7 +8420,7 @@ skip_tx: for (size_t idx: tx.selected_transfers) tx_money += m_transfers[idx].amount(); LOG_PRINT_L1(" Transaction " << (1+std::distance(txes.begin(), i)) << "/" << txes.size() << - " " << get_transaction_hash(tx.ptx.tx) << ": " << get_size_string(tx.bytes) << ", sending " << print_money(tx_money) << " in " << tx.selected_transfers.size() << + " " << get_transaction_hash(tx.ptx.tx) << ": " << get_weight_string(tx.weight) << ", sending " << print_money(tx_money) << " in " << tx.selected_transfers.size() << " outputs to " << tx.dsts.size() << " destination(s), including " << print_money(tx.ptx.fee) << " fee, " << print_money(tx.ptx.change_dts.amount) << " change"); ptx_vector.push_back(tx.ptx); @@ -8476,7 +8430,7 @@ skip_tx: return ptx_vector; } -std::vector<wallet2::pending_tx> wallet2::create_transactions_all(uint64_t below, const cryptonote::account_public_address &address, bool is_subaddress, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices, bool trusted_daemon) +std::vector<wallet2::pending_tx> wallet2::create_transactions_all(uint64_t below, const cryptonote::account_public_address &address, bool is_subaddress, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices) { std::vector<size_t> unused_transfers_indices; std::vector<size_t> unused_dust_indices; @@ -8527,10 +8481,10 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_all(uint64_t below } } - return create_transactions_from(address, is_subaddress, unused_transfers_indices, unused_dust_indices, fake_outs_count, unlock_time, priority, extra, trusted_daemon); + return create_transactions_from(address, is_subaddress, unused_transfers_indices, unused_dust_indices, fake_outs_count, unlock_time, priority, extra); } -std::vector<wallet2::pending_tx> wallet2::create_transactions_single(const crypto::key_image &ki, const cryptonote::account_public_address &address, bool is_subaddress, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, bool trusted_daemon) +std::vector<wallet2::pending_tx> wallet2::create_transactions_single(const crypto::key_image &ki, const cryptonote::account_public_address &address, bool is_subaddress, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra) { std::vector<size_t> unused_transfers_indices; std::vector<size_t> unused_dust_indices; @@ -8548,10 +8502,10 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_single(const crypt break; } } - return create_transactions_from(address, is_subaddress, unused_transfers_indices, unused_dust_indices, fake_outs_count, unlock_time, priority, extra, trusted_daemon); + return create_transactions_from(address, is_subaddress, unused_transfers_indices, unused_dust_indices, fake_outs_count, unlock_time, priority, extra); } -std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const cryptonote::account_public_address &address, bool is_subaddress, std::vector<size_t> unused_transfers_indices, std::vector<size_t> unused_dust_indices, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, bool trusted_daemon) +std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const cryptonote::account_public_address &address, bool is_subaddress, std::vector<size_t> unused_transfers_indices, std::vector<size_t> unused_dust_indices, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra) { //ensure device is let in NONE mode in any case hw::device &hwdev = m_account.get_device(); @@ -8564,21 +8518,24 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton std::vector<cryptonote::tx_destination_entry> dsts; cryptonote::transaction tx; pending_tx ptx; - size_t bytes; + size_t weight; uint64_t needed_fee; std::vector<std::vector<get_outs_entry>> outs; - TX() : bytes(0), needed_fee(0) {} + TX() : weight(0), needed_fee(0) {} }; std::vector<TX> txes; uint64_t needed_fee, available_for_fee = 0; - uint64_t upper_transaction_size_limit = get_upper_transaction_size_limit(); + uint64_t upper_transaction_weight_limit = get_upper_transaction_weight_limit(); std::vector<std::vector<get_outs_entry>> outs; + const bool use_per_byte_fee = use_fork_rules(HF_VERSION_PER_BYTE_FEE); const bool use_rct = fake_outs_count > 0 && use_fork_rules(4, 0); const bool bulletproof = use_fork_rules(get_bulletproof_fork(), 0); - const uint64_t fee_per_kb = get_per_kb_fee(); + const rct::RangeProofType range_proof_type = bulletproof ? rct::RangeProofPaddedBulletproof : rct::RangeProofBorromean; + const uint64_t base_fee = get_base_fee(); const uint64_t fee_multiplier = get_fee_multiplier(priority, get_fee_algorithm()); + const uint64_t fee_quantization_mask = get_fee_quantization_mask(); LOG_PRINT_L2("Starting with " << unused_transfers_indices.size() << " non-dust outputs and " << unused_dust_indices.size() << " dust outputs"); @@ -8600,7 +8557,25 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton // get a random unspent output and use it to pay next chunk. We try to alternate // dust and non dust to ensure we never get with only dust, from which we might // get a tx that can't pay for itself - size_t idx = unused_transfers_indices.empty() ? pop_best_value(unused_dust_indices, tx.selected_transfers) : unused_dust_indices.empty() ? pop_best_value(unused_transfers_indices, tx.selected_transfers) : ((tx.selected_transfers.size() & 1) || accumulated_outputs > fee_per_kb * fee_multiplier * (upper_transaction_size_limit + 1023) / 1024) ? pop_best_value(unused_dust_indices, tx.selected_transfers) : pop_best_value(unused_transfers_indices, tx.selected_transfers); + uint64_t fee_dust_threshold; + if (use_fork_rules(HF_VERSION_PER_BYTE_FEE)) + { + const uint64_t estimated_tx_weight_with_one_extra_output = estimate_tx_weight(use_rct, tx.selected_transfers.size() + 1, fake_outs_count, tx.dsts.size()+1, extra.size(), bulletproof); + fee_dust_threshold = calculate_fee_from_weight(base_fee, estimated_tx_weight_with_one_extra_output, fee_multiplier, fee_quantization_mask); + } + else + { + fee_dust_threshold = base_fee * fee_multiplier * (upper_transaction_weight_limit + 1023) / 1024; + } + + size_t idx = + unused_transfers_indices.empty() + ? pop_best_value(unused_dust_indices, tx.selected_transfers) + : unused_dust_indices.empty() + ? pop_best_value(unused_transfers_indices, tx.selected_transfers) + : ((tx.selected_transfers.size() & 1) || accumulated_outputs > fee_dust_threshold) + ? pop_best_value(unused_dust_indices, tx.selected_transfers) + : pop_best_value(unused_transfers_indices, tx.selected_transfers); const transfer_details &td = m_transfers[idx]; LOG_PRINT_L2("Picking output " << idx << ", amount " << print_money(td.amount())); @@ -8615,16 +8590,15 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton // here, check if we need to sent tx and start a new one LOG_PRINT_L2("Considering whether to create a tx now, " << tx.selected_transfers.size() << " inputs, tx limit " - << upper_transaction_size_limit); - const size_t estimated_rct_tx_size = estimate_tx_size(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size() + 1, extra.size(), bulletproof); - bool try_tx = (unused_dust_indices.empty() && unused_transfers_indices.empty()) || ( estimated_rct_tx_size >= TX_SIZE_TARGET(upper_transaction_size_limit)); + << upper_transaction_weight_limit); + const size_t estimated_rct_tx_weight = estimate_tx_weight(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size() + 2, extra.size(), bulletproof); + bool try_tx = (unused_dust_indices.empty() && unused_transfers_indices.empty()) || ( estimated_rct_tx_weight >= TX_WEIGHT_TARGET(upper_transaction_weight_limit)); if (try_tx) { cryptonote::transaction test_tx; pending_tx test_ptx; - const size_t estimated_tx_size = estimate_tx_size(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size(), extra.size(), bulletproof); - needed_fee = calculate_fee(fee_per_kb, estimated_tx_size, fee_multiplier); + needed_fee = estimate_fee(use_per_byte_fee, use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size()+1, extra.size(), bulletproof, base_fee, fee_multiplier, fee_quantization_mask); tx.dsts.push_back(tx_destination_entry(1, address, is_subaddress)); @@ -8632,14 +8606,14 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton tx.selected_transfers.size() << " outputs"); if (use_rct) transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, - test_tx, test_ptx, bulletproof); + test_tx, test_ptx, range_proof_type); else transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); auto txBlob = t_serializable_object_to_blob(test_ptx.tx); - needed_fee = calculate_fee(fee_per_kb, txBlob, fee_multiplier); + needed_fee = calculate_fee(use_per_byte_fee, test_ptx.tx, txBlob.size(), base_fee, fee_multiplier, fee_quantization_mask); available_for_fee = test_ptx.fee + test_ptx.dests[0].amount + test_ptx.change_dts.amount; - LOG_PRINT_L2("Made a " << get_size_string(txBlob) << " tx, with " << print_money(available_for_fee) << " available for fee (" << + LOG_PRINT_L2("Made a " << get_weight_string(test_ptx.tx, txBlob.size()) << " tx, with " << print_money(available_for_fee) << " available for fee (" << print_money(needed_fee) << " needed)"); THROW_WALLET_EXCEPTION_IF(needed_fee > available_for_fee, error::wallet_internal_error, "Transaction cannot pay for itself"); @@ -8649,22 +8623,22 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton tx.dsts[0].amount = available_for_fee - needed_fee; if (use_rct) transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, - test_tx, test_ptx, bulletproof); + test_tx, test_ptx, range_proof_type); else transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); txBlob = t_serializable_object_to_blob(test_ptx.tx); - needed_fee = calculate_fee(fee_per_kb, txBlob, fee_multiplier); - LOG_PRINT_L2("Made an attempt at a final " << get_size_string(txBlob) << " tx, with " << print_money(test_ptx.fee) << + needed_fee = calculate_fee(use_per_byte_fee, test_ptx.tx, txBlob.size(), base_fee, fee_multiplier, fee_quantization_mask); + LOG_PRINT_L2("Made an attempt at a final " << get_weight_string(test_ptx.tx, txBlob.size()) << " tx, with " << print_money(test_ptx.fee) << " fee and " << print_money(test_ptx.change_dts.amount) << " change"); } while (needed_fee > test_ptx.fee); - LOG_PRINT_L2("Made a final " << get_size_string(txBlob) << " tx, with " << print_money(test_ptx.fee) << + LOG_PRINT_L2("Made a final " << get_weight_string(test_ptx.tx, txBlob.size()) << " tx, with " << print_money(test_ptx.fee) << " fee and " << print_money(test_ptx.change_dts.amount) << " change"); tx.tx = test_tx; tx.ptx = test_ptx; - tx.bytes = txBlob.size(); + tx.weight = get_transaction_weight(test_tx, txBlob.size()); tx.outs = outs; tx.needed_fee = needed_fee; accumulated_fee += test_ptx.fee; @@ -8688,7 +8662,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton pending_tx test_ptx; if (use_rct) { transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, tx.outs, unlock_time, tx.needed_fee, extra, - test_tx, test_ptx, bulletproof); + test_tx, test_ptx, range_proof_type); } else { transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, tx.outs, unlock_time, tx.needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); @@ -8696,7 +8670,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton auto txBlob = t_serializable_object_to_blob(test_ptx.tx); tx.tx = test_tx; tx.ptx = test_ptx; - tx.bytes = txBlob.size(); + tx.weight = get_transaction_weight(test_tx, txBlob.size()); } std::vector<wallet2::pending_tx> ptx_vector; @@ -8707,7 +8681,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton for (size_t idx: tx.selected_transfers) tx_money += m_transfers[idx].amount(); LOG_PRINT_L1(" Transaction " << (1+std::distance(txes.begin(), i)) << "/" << txes.size() << - " " << get_transaction_hash(tx.ptx.tx) << ": " << get_size_string(tx.bytes) << ", sending " << print_money(tx_money) << " in " << tx.selected_transfers.size() << + " " << get_transaction_hash(tx.ptx.tx) << ": " << get_weight_string(tx.weight) << ", sending " << print_money(tx_money) << " in " << tx.selected_transfers.size() << " outputs to " << tx.dsts.size() << " destination(s), including " << print_money(tx.ptx.fee) << " fee, " << print_money(tx.ptx.change_dts.amount) << " change"); ptx_vector.push_back(tx.ptx); @@ -8742,12 +8716,15 @@ bool wallet2::use_fork_rules(uint8_t version, int64_t early_blocks) const return close_enough; } //---------------------------------------------------------------------------------------------------- -uint64_t wallet2::get_upper_transaction_size_limit() const +uint64_t wallet2::get_upper_transaction_weight_limit() const { - if (m_upper_transaction_size_limit > 0) - return m_upper_transaction_size_limit; + if (m_upper_transaction_weight_limit > 0) + return m_upper_transaction_weight_limit; uint64_t full_reward_zone = use_fork_rules(5, 10) ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 : use_fork_rules(2, 10) ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V2 : CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V1; - return full_reward_zone - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; + if (use_fork_rules(8, 10)) + return full_reward_zone / 2 - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; + else + return full_reward_zone - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE; } //---------------------------------------------------------------------------------------------------- std::vector<size_t> wallet2::select_available_outputs(const std::function<bool(const transfer_details &td)> &f) const @@ -8785,12 +8762,12 @@ std::vector<uint64_t> wallet2::get_unspent_amounts_vector() const return vector; } //---------------------------------------------------------------------------------------------------- -std::vector<size_t> wallet2::select_available_outputs_from_histogram(uint64_t count, bool atleast, bool unlocked, bool allow_rct, bool trusted_daemon) +std::vector<size_t> wallet2::select_available_outputs_from_histogram(uint64_t count, bool atleast, bool unlocked, bool allow_rct) { cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request req_t = AUTO_VAL_INIT(req_t); cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response resp_t = AUTO_VAL_INIT(resp_t); m_daemon_rpc_mutex.lock(); - if (trusted_daemon) + if (is_trusted_daemon()) req_t.amounts = get_unspent_amounts_vector(); req_t.min_count = count; req_t.max_count = 0; @@ -8851,30 +8828,28 @@ const wallet2::transfer_details &wallet2::get_transfer_details(size_t idx) const return m_transfers[idx]; } //---------------------------------------------------------------------------------------------------- -std::vector<size_t> wallet2::select_available_unmixable_outputs(bool trusted_daemon) +std::vector<size_t> wallet2::select_available_unmixable_outputs() { - // request all outputs with less than 3 instances - const size_t min_mixin = use_fork_rules(7, 10) ? 6 : use_fork_rules(6, 10) ? 4 : 2; // v6 increases min mixin from 2 to 4, v7 to 6 - return select_available_outputs_from_histogram(min_mixin + 1, false, true, false, trusted_daemon); + // request all outputs with less instances than the min ring size + return select_available_outputs_from_histogram(get_min_ring_size(), false, true, false); } //---------------------------------------------------------------------------------------------------- -std::vector<size_t> wallet2::select_available_mixable_outputs(bool trusted_daemon) +std::vector<size_t> wallet2::select_available_mixable_outputs() { - // request all outputs with at least 3 instances, so we can use mixin 2 with - const size_t min_mixin = use_fork_rules(7, 10) ? 6 : use_fork_rules(6, 10) ? 4 : 2; // v6 increases min mixin from 2 to 4, v7 to 6 - return select_available_outputs_from_histogram(min_mixin + 1, true, true, true, trusted_daemon); + // request all outputs with at least as many instances as the min ring size + return select_available_outputs_from_histogram(get_min_ring_size(), true, true, true); } //---------------------------------------------------------------------------------------------------- -std::vector<wallet2::pending_tx> wallet2::create_unmixable_sweep_transactions(bool trusted_daemon) +std::vector<wallet2::pending_tx> wallet2::create_unmixable_sweep_transactions() { // From hard fork 1, we don't consider small amounts to be dust anymore const bool hf1_rules = use_fork_rules(2, 10); // first hard fork has version 2 tx_dust_policy dust_policy(hf1_rules ? 0 : ::config::DEFAULT_DUST_THRESHOLD); - const uint64_t fee_per_kb = get_per_kb_fee(); + const uint64_t base_fee = get_base_fee(); // may throw - std::vector<size_t> unmixable_outputs = select_available_unmixable_outputs(trusted_daemon); + std::vector<size_t> unmixable_outputs = select_available_unmixable_outputs(); size_t num_dust_outputs = unmixable_outputs.size(); if (num_dust_outputs == 0) @@ -8886,19 +8861,19 @@ std::vector<wallet2::pending_tx> wallet2::create_unmixable_sweep_transactions(bo std::vector<size_t> unmixable_transfer_outputs, unmixable_dust_outputs; for (auto n: unmixable_outputs) { - if (m_transfers[n].amount() < fee_per_kb) + if (m_transfers[n].amount() < base_fee) unmixable_dust_outputs.push_back(n); else unmixable_transfer_outputs.push_back(n); } - return create_transactions_from(m_account_public_address, false, unmixable_transfer_outputs, unmixable_dust_outputs, 0 /*fake_outs_count */, 0 /* unlock_time */, 1 /*priority */, std::vector<uint8_t>(), trusted_daemon); + return create_transactions_from(m_account_public_address, false, unmixable_transfer_outputs, unmixable_dust_outputs, 0 /*fake_outs_count */, 0 /* unlock_time */, 1 /*priority */, std::vector<uint8_t>()); } //---------------------------------------------------------------------------------------------------- -void wallet2::discard_unmixable_outputs(bool trusted_daemon) +void wallet2::discard_unmixable_outputs() { // may throw - std::vector<size_t> unmixable_outputs = select_available_unmixable_outputs(trusted_daemon); + std::vector<size_t> unmixable_outputs = select_available_unmixable_outputs(); for (size_t idx : unmixable_outputs) { m_transfers[idx].m_spent = true; @@ -9279,6 +9254,8 @@ void wallet2::check_tx_key_helper(const crypto::hash &txid, const crypto::key_de hwdev.ecdhDecode(ecdh_info, rct::sk2rct(scalar1)); const rct::key C = tx.rct_signatures.outPk[n].mask; rct::key Ctmp; + THROW_WALLET_EXCEPTION_IF(sc_check(ecdh_info.mask.bytes) != 0, error::wallet_internal_error, "Bad ECDH input mask"); + THROW_WALLET_EXCEPTION_IF(sc_check(ecdh_info.amount.bytes) != 0, error::wallet_internal_error, "Bad ECDH input amount"); rct::addKeys2(Ctmp, ecdh_info.mask, ecdh_info.amount, rct::H); if (rct::equalKeys(C, Ctmp)) amount = rct::h2d(ecdh_info.amount); @@ -9849,7 +9826,7 @@ uint64_t wallet2::get_approximate_blockchain_height() const // Calculated blockchain height uint64_t approx_blockchain_height = fork_block + (time(NULL) - fork_time)/seconds_per_block; // testnet got some huge rollbacks, so the estimation is way off - static const uint64_t approximate_testnet_rolled_back_blocks = 148540; + static const uint64_t approximate_testnet_rolled_back_blocks = 303967; if (m_nettype == TESTNET && approx_blockchain_height > approximate_testnet_rolled_back_blocks) approx_blockchain_height -= approximate_testnet_rolled_back_blocks; LOG_PRINT_L2("Calculated blockchain height: " << approx_blockchain_height); @@ -11207,46 +11184,46 @@ std::vector<std::pair<uint64_t, uint64_t>> wallet2::estimate_backlog(const std:: THROW_WALLET_EXCEPTION_IF(res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_txpool_backlog"); THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::get_tx_pool_error); - uint64_t block_size_limit = 0; - const auto result = m_node_rpc_proxy.get_block_size_limit(block_size_limit); + uint64_t block_weight_limit = 0; + const auto result = m_node_rpc_proxy.get_block_weight_limit(block_weight_limit); throw_on_rpc_response_error(result, "get_info"); - uint64_t full_reward_zone = block_size_limit / 2; - THROW_WALLET_EXCEPTION_IF(full_reward_zone == 0, error::wallet_internal_error, "Invalid block size limit from daemon"); + uint64_t full_reward_zone = block_weight_limit / 2; + THROW_WALLET_EXCEPTION_IF(full_reward_zone == 0, error::wallet_internal_error, "Invalid block weight limit from daemon"); std::vector<std::pair<uint64_t, uint64_t>> blocks; for (const auto &fee_level: fee_levels) { const double our_fee_byte_min = fee_level.first; const double our_fee_byte_max = fee_level.second; - uint64_t priority_size_min = 0, priority_size_max = 0; + uint64_t priority_weight_min = 0, priority_weight_max = 0; for (const auto &i: res.backlog) { - if (i.blob_size == 0) + if (i.weight == 0) { - MWARNING("Got 0 sized blob from txpool, ignored"); + MWARNING("Got 0 weight tx from txpool, ignored"); continue; } - double this_fee_byte = i.fee / (double)i.blob_size; + double this_fee_byte = i.fee / (double)i.weight; if (this_fee_byte >= our_fee_byte_min) - priority_size_min += i.blob_size; + priority_weight_min += i.weight; if (this_fee_byte >= our_fee_byte_max) - priority_size_max += i.blob_size; + priority_weight_max += i.weight; } - uint64_t nblocks_min = priority_size_min / full_reward_zone; - uint64_t nblocks_max = priority_size_max / full_reward_zone; - MDEBUG("estimate_backlog: priority_size " << priority_size_min << " - " << priority_size_max << " for " + uint64_t nblocks_min = priority_weight_min / full_reward_zone; + uint64_t nblocks_max = priority_weight_max / full_reward_zone; + MDEBUG("estimate_backlog: priority_weight " << priority_weight_min << " - " << priority_weight_max << " for " << our_fee_byte_min << " - " << our_fee_byte_max << " piconero byte fee, " - << nblocks_min << " - " << nblocks_max << " blocks at block size " << full_reward_zone); + << nblocks_min << " - " << nblocks_max << " blocks at block weight " << full_reward_zone); blocks.push_back(std::make_pair(nblocks_min, nblocks_max)); } return blocks; } //---------------------------------------------------------------------------------------------------- -std::vector<std::pair<uint64_t, uint64_t>> wallet2::estimate_backlog(uint64_t min_blob_size, uint64_t max_blob_size, const std::vector<uint64_t> &fees) +std::vector<std::pair<uint64_t, uint64_t>> wallet2::estimate_backlog(uint64_t min_tx_weight, uint64_t max_tx_weight, const std::vector<uint64_t> &fees) { - THROW_WALLET_EXCEPTION_IF(min_blob_size == 0, error::wallet_internal_error, "Invalid 0 fee"); - THROW_WALLET_EXCEPTION_IF(max_blob_size == 0, error::wallet_internal_error, "Invalid 0 fee"); + THROW_WALLET_EXCEPTION_IF(min_tx_weight == 0, error::wallet_internal_error, "Invalid 0 fee"); + THROW_WALLET_EXCEPTION_IF(max_tx_weight == 0, error::wallet_internal_error, "Invalid 0 fee"); for (uint64_t fee: fees) { THROW_WALLET_EXCEPTION_IF(fee == 0, error::wallet_internal_error, "Invalid 0 fee"); @@ -11254,7 +11231,7 @@ std::vector<std::pair<uint64_t, uint64_t>> wallet2::estimate_backlog(uint64_t mi std::vector<std::pair<double, double>> fee_levels; for (uint64_t fee: fees) { - double our_fee_byte_min = fee / (double)min_blob_size, our_fee_byte_max = fee / (double)max_blob_size; + double our_fee_byte_min = fee / (double)min_tx_weight, our_fee_byte_max = fee / (double)max_tx_weight; fee_levels.emplace_back(our_fee_byte_min, our_fee_byte_max); } return estimate_backlog(fee_levels); diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 2d45f4e3e..0347092c0 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -630,20 +630,18 @@ namespace tools void explicit_refresh_from_block_height(bool expl) {m_explicit_refresh_from_block_height = expl;} bool explicit_refresh_from_block_height() const {return m_explicit_refresh_from_block_height;} - // upper_transaction_size_limit as defined below is set to - // approximately 125% of the fixed minimum allowable penalty - // free block size. TODO: fix this so that it actually takes - // into account the current median block size rather than - // the minimum block size. bool deinit(); bool init(bool rpc, std::string daemon_address = "http://localhost:8080", - boost::optional<epee::net_utils::http::login> daemon_login = boost::none, uint64_t upper_transaction_size_limit = 0, bool ssl = false); + boost::optional<epee::net_utils::http::login> daemon_login = boost::none, uint64_t upper_transaction_weight_limit = 0, bool ssl = false, bool trusted_daemon = false); void stop() { m_run.store(false, std::memory_order_relaxed); } i_wallet2_callback* callback() const { return m_callback; } void callback(i_wallet2_callback* callback) { m_callback = callback; } + bool is_trusted_daemon() const { return m_trusted_daemon; } + void set_trusted_daemon(bool trusted) { m_trusted_daemon = trusted; } + /*! * \brief Checks if deterministic wallet */ @@ -715,18 +713,12 @@ namespace tools uint64_t balance_all() const; uint64_t unlocked_balance_all() const; template<typename T> - void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, bool trusted_daemon); - template<typename T> - void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, cryptonote::transaction& tx, pending_tx& ptx, bool trusted_daemon); - void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, bool trusted_daemon); - void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx& ptx, bool trusted_daemon); - template<typename T> void transfer_selected(const std::vector<cryptonote::tx_destination_entry>& dsts, const std::vector<size_t>& selected_transfers, size_t fake_outputs_count, std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, cryptonote::transaction& tx, pending_tx &ptx); void transfer_selected_rct(std::vector<cryptonote::tx_destination_entry> dsts, const std::vector<size_t>& selected_transfers, size_t fake_outputs_count, std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, - uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx &ptx, bool bulletproof); + uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx &ptx, rct::RangeProofType range_proof_type); void commit_tx(pending_tx& ptx_vector); void commit_tx(std::vector<pending_tx>& ptx_vector); @@ -748,18 +740,17 @@ namespace tools bool parse_unsigned_tx_from_str(const std::string &unsigned_tx_st, unsigned_tx_set &exported_txs) const; bool load_tx(const std::string &signed_filename, std::vector<tools::wallet2::pending_tx> &ptx, std::function<bool(const signed_tx_set&)> accept_func = NULL); bool parse_tx_from_str(const std::string &signed_tx_st, std::vector<tools::wallet2::pending_tx> &ptx, std::function<bool(const signed_tx_set &)> accept_func); - std::vector<pending_tx> create_transactions(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, bool trusted_daemon); - std::vector<wallet2::pending_tx> create_transactions_2(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices, bool trusted_daemon); // pass subaddr_indices by value on purpose - std::vector<wallet2::pending_tx> create_transactions_all(uint64_t below, const cryptonote::account_public_address &address, bool is_subaddress, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices, bool trusted_daemon); - std::vector<wallet2::pending_tx> create_transactions_single(const crypto::key_image &ki, const cryptonote::account_public_address &address, bool is_subaddress, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, bool trusted_daemon); - std::vector<wallet2::pending_tx> create_transactions_from(const cryptonote::account_public_address &address, bool is_subaddress, std::vector<size_t> unused_transfers_indices, std::vector<size_t> unused_dust_indices, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, bool trusted_daemon); + std::vector<wallet2::pending_tx> create_transactions_2(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices); // pass subaddr_indices by value on purpose + std::vector<wallet2::pending_tx> create_transactions_all(uint64_t below, const cryptonote::account_public_address &address, bool is_subaddress, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices); + std::vector<wallet2::pending_tx> create_transactions_single(const crypto::key_image &ki, const cryptonote::account_public_address &address, bool is_subaddress, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra); + std::vector<wallet2::pending_tx> create_transactions_from(const cryptonote::account_public_address &address, bool is_subaddress, std::vector<size_t> unused_transfers_indices, std::vector<size_t> unused_dust_indices, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra); bool load_multisig_tx(cryptonote::blobdata blob, multisig_tx_set &exported_txs, std::function<bool(const multisig_tx_set&)> accept_func = NULL); bool load_multisig_tx_from_file(const std::string &filename, multisig_tx_set &exported_txs, std::function<bool(const multisig_tx_set&)> accept_func = NULL); bool sign_multisig_tx_from_file(const std::string &filename, std::vector<crypto::hash> &txids, std::function<bool(const multisig_tx_set&)> accept_func); bool sign_multisig_tx(multisig_tx_set &exported_txs, std::vector<crypto::hash> &txids); bool sign_multisig_tx_to_file(multisig_tx_set &exported_txs, const std::string &filename, std::vector<crypto::hash> &txids); - std::vector<pending_tx> create_unmixable_sweep_transactions(bool trusted_daemon); - void discard_unmixable_outputs(bool trusted_daemon); + std::vector<pending_tx> create_unmixable_sweep_transactions(); + void discard_unmixable_outputs(); bool check_connection(uint32_t *version = NULL, uint32_t timeout = 200000); void get_transfers(wallet2::transfer_container& incoming_transfers) const; void get_payments(const crypto::hash& payment_id, std::list<wallet2::payment_details>& payments, uint64_t min_height = 0, const boost::optional<uint32_t>& subaddr_account = boost::none, const std::set<uint32_t>& subaddr_indices = {}) const; @@ -996,10 +987,10 @@ namespace tools */ uint64_t get_approximate_blockchain_height() const; uint64_t estimate_blockchain_height(); - std::vector<size_t> select_available_outputs_from_histogram(uint64_t count, bool atleast, bool unlocked, bool allow_rct, bool trusted_daemon); + std::vector<size_t> select_available_outputs_from_histogram(uint64_t count, bool atleast, bool unlocked, bool allow_rct); std::vector<size_t> select_available_outputs(const std::function<bool(const transfer_details &td)> &f) const; - std::vector<size_t> select_available_unmixable_outputs(bool trusted_daemon); - std::vector<size_t> select_available_mixable_outputs(bool trusted_daemon); + std::vector<size_t> select_available_unmixable_outputs(); + std::vector<size_t> select_available_mixable_outputs(); size_t pop_best_value_from(const transfer_container &transfers, std::vector<size_t> &unused_dust_indices, const std::vector<size_t>& selected_transfers, bool smallest = false) const; size_t pop_best_value(std::vector<size_t> &unused_dust_indices, const std::vector<size_t>& selected_transfers, bool smallest = false) const; @@ -1081,10 +1072,13 @@ namespace tools bool is_synced() const; std::vector<std::pair<uint64_t, uint64_t>> estimate_backlog(const std::vector<std::pair<double, double>> &fee_levels); - std::vector<std::pair<uint64_t, uint64_t>> estimate_backlog(uint64_t min_blob_size, uint64_t max_blob_size, const std::vector<uint64_t> &fees); + std::vector<std::pair<uint64_t, uint64_t>> estimate_backlog(uint64_t min_tx_weight, uint64_t max_tx_weight, const std::vector<uint64_t> &fees); uint64_t get_fee_multiplier(uint32_t priority, int fee_algorithm = -1) const; - uint64_t get_per_kb_fee() const; + uint64_t get_base_fee() const; + uint64_t get_fee_quantization_mask() const; + uint64_t get_min_ring_size() const; + uint64_t get_max_ring_size() const; uint64_t adjust_mixin(uint64_t mixin) const; uint32_t adjust_priority(uint32_t priority); @@ -1158,10 +1152,10 @@ namespace tools bool set_ring(const crypto::key_image &key_image, const std::vector<uint64_t> &outs, bool relative); bool find_and_save_rings(bool force = true); - bool blackball_output(const crypto::public_key &output); - bool set_blackballed_outputs(const std::vector<crypto::public_key> &outputs, bool add = false); - bool unblackball_output(const crypto::public_key &output); - bool is_output_blackballed(const crypto::public_key &output) const; + bool blackball_output(const std::pair<uint64_t, uint64_t> &output); + bool set_blackballed_outputs(const std::vector<std::pair<uint64_t, uint64_t>> &outputs, bool add = false); + bool unblackball_output(const std::pair<uint64_t, uint64_t> &output); + bool is_output_blackballed(const std::pair<uint64_t, uint64_t> &output) const; bool lock_keys_file(); bool unlock_keys_file(); @@ -1195,7 +1189,7 @@ namespace tools void fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, bool force = false); void pull_and_parse_next_blocks(uint64_t start_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, const std::vector<cryptonote::block_complete_entry> &prev_blocks, const std::vector<parsed_block> &prev_parsed_blocks, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<parsed_block> &parsed_blocks, bool &error); void process_parsed_blocks(uint64_t start_height, const std::vector<cryptonote::block_complete_entry> &blocks, const std::vector<parsed_block> &parsed_blocks, uint64_t& blocks_added); - uint64_t select_transfers(uint64_t needed_money, std::vector<size_t> unused_transfers_indices, std::vector<size_t>& selected_transfers, bool trusted_daemon) const; + uint64_t select_transfers(uint64_t needed_money, std::vector<size_t> unused_transfers_indices, std::vector<size_t>& selected_transfers) const; bool prepare_file_names(const std::string& file_path); void process_unconfirmed(const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t height); void process_outgoing(const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t height, uint64_t ts, uint64_t spent, uint64_t received, uint32_t subaddr_account, const std::set<uint32_t>& subaddr_indices); @@ -1209,9 +1203,9 @@ namespace tools void check_acc_out_precomp(const cryptonote::tx_out &o, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, size_t i, const is_out_data *is_out_data, tx_scan_info_t &tx_scan_info) const; void check_acc_out_precomp_once(const cryptonote::tx_out &o, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, size_t i, const is_out_data *is_out_data, tx_scan_info_t &tx_scan_info, bool &already_seen) const; void parse_block_round(const cryptonote::blobdata &blob, cryptonote::block &bl, crypto::hash &bl_id, bool &error) const; - uint64_t get_upper_transaction_size_limit() const; + uint64_t get_upper_transaction_weight_limit() const; std::vector<uint64_t> get_unspent_amounts_vector() const; - uint64_t get_dynamic_per_kb_fee_estimate() const; + uint64_t get_dynamic_base_fee_estimate() const; float get_output_relatedness(const transfer_details &td0, const transfer_details &td1) const; std::vector<size_t> pick_preferred_rct_inputs(uint64_t needed_money, uint32_t subaddr_account, const std::set<uint32_t> &subaddr_indices) const; void set_spent(size_t idx, uint64_t height); @@ -1266,7 +1260,7 @@ namespace tools std::unordered_map<std::string, std::string> m_attributes; std::vector<tools::wallet2::address_book_row> m_address_book; std::pair<std::map<std::string, std::string>, std::vector<std::string>> m_account_tags; - uint64_t m_upper_transaction_size_limit; //TODO: auto-calc this value or request from daemon, now use some fixed value + uint64_t m_upper_transaction_weight_limit; //TODO: auto-calc this value or request from daemon, now use some fixed value const std::vector<std::vector<tools::wallet2::multisig_info>> *m_multisig_rescan_info; const std::vector<std::vector<rct::key>> *m_multisig_rescan_k; @@ -1274,6 +1268,7 @@ namespace tools boost::mutex m_daemon_rpc_mutex; + bool m_trusted_daemon; i_wallet2_callback* m_callback; bool m_key_on_device; cryptonote::network_type m_nettype; @@ -1815,198 +1810,4 @@ namespace tools //---------------------------------------------------------------------------------------------------- } //---------------------------------------------------------------------------------------------------- - template<typename T> - void wallet2::transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outs_count, const std::vector<size_t> &unused_transfers_indices, - uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, bool trusted_daemon) - { - pending_tx ptx; - cryptonote::transaction tx; - transfer(dsts, fake_outs_count, unused_transfers_indices, unlock_time, fee, extra, destination_split_strategy, dust_policy, tx, ptx, trusted_daemon); - } - - template<typename T> - void wallet2::transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices, - uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, cryptonote::transaction& tx, pending_tx &ptx, bool trusted_daemon) - { - using namespace cryptonote; - // throw if attempting a transaction with no destinations - THROW_WALLET_EXCEPTION_IF(dsts.empty(), error::zero_destination); - - THROW_WALLET_EXCEPTION_IF(m_multisig, error::wallet_internal_error, "Multisig wallets cannot spend non rct outputs"); - - uint64_t upper_transaction_size_limit = get_upper_transaction_size_limit(); - uint64_t needed_money = fee; - - // calculate total amount being sent to all destinations - // throw if total amount overflows uint64_t - for(auto& dt: dsts) - { - THROW_WALLET_EXCEPTION_IF(0 == dt.amount, error::zero_destination); - needed_money += dt.amount; - THROW_WALLET_EXCEPTION_IF(needed_money < dt.amount, error::tx_sum_overflow, dsts, fee, m_nettype); - } - - // randomly select inputs for transaction - // throw if requested send amount is greater than (unlocked) amount available to send - std::vector<size_t> selected_transfers; - uint64_t found_money = select_transfers(needed_money, unused_transfers_indices, selected_transfers, trusted_daemon); - THROW_WALLET_EXCEPTION_IF(found_money < needed_money, error::not_enough_unlocked_money, found_money, needed_money - fee, fee); - - uint32_t subaddr_account = m_transfers[*selected_transfers.begin()].m_subaddr_index.major; - for (auto i = ++selected_transfers.begin(); i != selected_transfers.end(); ++i) - THROW_WALLET_EXCEPTION_IF(subaddr_account != *i, error::wallet_internal_error, "the tx uses funds from multiple accounts"); - - typedef COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry out_entry; - typedef cryptonote::tx_source_entry::output_entry tx_output_entry; - - COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response daemon_resp = AUTO_VAL_INIT(daemon_resp); - if(fake_outputs_count) - { - COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request req = AUTO_VAL_INIT(req); - req.outs_count = fake_outputs_count + 1;// add one to make possible (if need) to skip real output key - for(size_t idx: selected_transfers) - { - const transfer_container::const_iterator it = m_transfers.begin() + idx; - THROW_WALLET_EXCEPTION_IF(it->m_tx.vout.size() <= it->m_internal_output_index, error::wallet_internal_error, - "m_internal_output_index = " + std::to_string(it->m_internal_output_index) + - " is greater or equal to outputs count = " + std::to_string(it->m_tx.vout.size())); - req.amounts.push_back(it->amount()); - } - - m_daemon_rpc_mutex.lock(); - bool r = epee::net_utils::invoke_http_bin("/getrandom_outs.bin", req, daemon_resp, m_http_client, rpc_timeout); - m_daemon_rpc_mutex.unlock(); - THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "getrandom_outs.bin"); - THROW_WALLET_EXCEPTION_IF(daemon_resp.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "getrandom_outs.bin"); - THROW_WALLET_EXCEPTION_IF(daemon_resp.status != CORE_RPC_STATUS_OK, error::get_random_outs_error, daemon_resp.status); - THROW_WALLET_EXCEPTION_IF(daemon_resp.outs.size() != selected_transfers.size(), error::wallet_internal_error, - "daemon returned wrong response for getrandom_outs.bin, wrong amounts count = " + - std::to_string(daemon_resp.outs.size()) + ", expected " + std::to_string(selected_transfers.size())); - - std::unordered_map<uint64_t, uint64_t> scanty_outs; - for(COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& amount_outs: daemon_resp.outs) - { - if (amount_outs.outs.size() < fake_outputs_count) - { - scanty_outs[amount_outs.amount] = amount_outs.outs.size(); - } - } - THROW_WALLET_EXCEPTION_IF(!scanty_outs.empty(), error::not_enough_outs_to_mix, scanty_outs, fake_outputs_count); - } - - //prepare inputs - size_t i = 0; - std::vector<cryptonote::tx_source_entry> sources; - for(size_t idx: selected_transfers) - { - sources.resize(sources.size()+1); - cryptonote::tx_source_entry& src = sources.back(); - const transfer_details& td = m_transfers[idx]; - src.amount = td.amount(); - src.rct = false; - //paste mixin transaction - if(daemon_resp.outs.size()) - { - daemon_resp.outs[i].outs.sort([](const out_entry& a, const out_entry& b){return a.global_amount_index < b.global_amount_index;}); - for(out_entry& daemon_oe: daemon_resp.outs[i].outs) - { - if(td.m_global_output_index == daemon_oe.global_amount_index) - continue; - tx_output_entry oe; - oe.first = daemon_oe.global_amount_index; - oe.second.dest = rct::pk2rct(daemon_oe.out_key); - oe.second.mask = rct::identity(); - src.outputs.push_back(oe); - if(src.outputs.size() >= fake_outputs_count) - break; - } - } - - //paste real transaction to the random index - auto it_to_insert = std::find_if(src.outputs.begin(), src.outputs.end(), [&](const tx_output_entry& a) - { - return a.first >= td.m_global_output_index; - }); - //size_t real_index = src.outputs.size() ? (rand() % src.outputs.size() ):0; - tx_output_entry real_oe; - real_oe.first = td.m_global_output_index; - real_oe.second.dest = rct::pk2rct(boost::get<txout_to_key>(td.m_tx.vout[td.m_internal_output_index].target).key); - real_oe.second.mask = rct::identity(); - auto interted_it = src.outputs.insert(it_to_insert, real_oe); - src.real_out_tx_key = get_tx_pub_key_from_extra(td.m_tx); - src.real_output = interted_it - src.outputs.begin(); - src.real_output_in_tx_index = td.m_internal_output_index; - src.multisig_kLRki = rct::multisig_kLRki({rct::zero(), rct::zero(), rct::zero(), rct::zero()}); - detail::print_source_entry(src); - ++i; - } - - cryptonote::tx_destination_entry change_dts = AUTO_VAL_INIT(change_dts); - if (needed_money < found_money) - { - change_dts.addr = get_subaddress({subaddr_account, 0}); - change_dts.amount = found_money - needed_money; - } - - std::vector<cryptonote::tx_destination_entry> splitted_dsts, dust_dsts; - uint64_t dust = 0; - destination_split_strategy(dsts, change_dts, dust_policy.dust_threshold, splitted_dsts, dust_dsts); - for(auto& d: dust_dsts) { - THROW_WALLET_EXCEPTION_IF(dust_policy.dust_threshold < d.amount, error::wallet_internal_error, "invalid dust value: dust = " + - std::to_string(d.amount) + ", dust_threshold = " + std::to_string(dust_policy.dust_threshold)); - } - for(auto& d: dust_dsts) { - if (!dust_policy.add_to_fee) - splitted_dsts.push_back(cryptonote::tx_destination_entry(d.amount, dust_policy.addr_for_dust, d.is_subaddress)); - dust += d.amount; - } - - crypto::secret_key tx_key; - std::vector<crypto::secret_key> additional_tx_keys; - rct::multisig_out msout; - bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sources, splitted_dsts, change_dts.addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, false, m_multisig ? &msout : NULL); - THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sources, splitted_dsts, unlock_time, m_nettype); - THROW_WALLET_EXCEPTION_IF(upper_transaction_size_limit <= get_object_blobsize(tx), error::tx_too_big, tx, upper_transaction_size_limit); - - std::string key_images; - bool all_are_txin_to_key = std::all_of(tx.vin.begin(), tx.vin.end(), [&](const txin_v& s_e) -> bool - { - CHECKED_GET_SPECIFIC_VARIANT(s_e, const txin_to_key, in, false); - key_images += boost::to_string(in.k_image) + " "; - return true; - }); - THROW_WALLET_EXCEPTION_IF(!all_are_txin_to_key, error::unexpected_txin_type, tx); - - bool dust_sent_elsewhere = (dust_policy.addr_for_dust.m_view_public_key != change_dts.addr.m_view_public_key - || dust_policy.addr_for_dust.m_spend_public_key != change_dts.addr.m_spend_public_key); - - if (dust_policy.add_to_fee || dust_sent_elsewhere) change_dts.amount -= dust; - - ptx.key_images = key_images; - ptx.fee = (dust_policy.add_to_fee ? fee+dust : fee); - ptx.dust = ((dust_policy.add_to_fee || dust_sent_elsewhere) ? dust : 0); - ptx.dust_added_to_fee = dust_policy.add_to_fee; - ptx.tx = tx; - ptx.change_dts = change_dts; - ptx.selected_transfers = selected_transfers; - ptx.tx_key = tx_key; - ptx.additional_tx_keys = additional_tx_keys; - ptx.dests = dsts; - ptx.construction_data.sources = sources; - ptx.construction_data.change_dts = change_dts; - ptx.construction_data.splitted_dsts = splitted_dsts; - ptx.construction_data.selected_transfers = selected_transfers; - ptx.construction_data.extra = tx.extra; - ptx.construction_data.unlock_time = unlock_time; - ptx.construction_data.use_rct = false; - ptx.construction_data.use_bulletproofs = false; - ptx.construction_data.dests = dsts; - // record which subaddress indices are being used as inputs - ptx.construction_data.subaddr_account = subaddr_account; - ptx.construction_data.subaddr_indices.clear(); - for (size_t idx: selected_transfers) - ptx.construction_data.subaddr_indices.insert(m_transfers[idx].m_subaddr_index.minor); - } - - } diff --git a/src/wallet/wallet_errors.h b/src/wallet/wallet_errors.h index 243953280..bc518d04a 100644 --- a/src/wallet/wallet_errors.h +++ b/src/wallet/wallet_errors.h @@ -73,7 +73,7 @@ namespace tools // get_tx_pool_error // out_of_hashchain_bounds_error // transfer_error * - // get_random_outs_general_error + // get_outs_general_error // not_enough_unlocked_money // not_enough_money // tx_not_possible @@ -128,7 +128,7 @@ namespace tools get_blocks_error_message_index, get_hashes_error_message_index, get_out_indices_error_message_index, - get_random_outs_error_message_index + get_outs_error_message_index }; template<typename Base, int msg_index> @@ -427,7 +427,7 @@ namespace tools } }; //---------------------------------------------------------------------------------------------------- - typedef failed_rpc_request<transfer_error, get_random_outs_error_message_index> get_random_outs_error; + typedef failed_rpc_request<transfer_error, get_outs_error_message_index> get_outs_error; //---------------------------------------------------------------------------------------------------- struct not_enough_unlocked_money : public transfer_error { @@ -679,30 +679,30 @@ namespace tools //---------------------------------------------------------------------------------------------------- struct tx_too_big : public transfer_error { - explicit tx_too_big(std::string&& loc, const cryptonote::transaction& tx, uint64_t tx_size_limit) + explicit tx_too_big(std::string&& loc, const cryptonote::transaction& tx, uint64_t tx_weight_limit) : transfer_error(std::move(loc), "transaction is too big") , m_tx(tx) - , m_tx_size_limit(tx_size_limit) + , m_tx_weight_limit(tx_weight_limit) { } const cryptonote::transaction& tx() const { return m_tx; } - uint64_t tx_size_limit() const { return m_tx_size_limit; } + uint64_t tx_weight_limit() const { return m_tx_weight_limit; } std::string to_string() const { std::ostringstream ss; cryptonote::transaction tx = m_tx; ss << transfer_error::to_string() << - ", tx_size_limit = " << m_tx_size_limit << - ", tx size = " << get_object_blobsize(m_tx) << + ", tx_weight_limit = " << m_tx_weight_limit << + ", tx weight = " << get_transaction_weight(m_tx) << ", tx:\n" << cryptonote::obj_to_json_str(tx); return ss.str(); } private: cryptonote::transaction m_tx; - uint64_t m_tx_size_limit; + uint64_t m_tx_weight_limit; }; //---------------------------------------------------------------------------------------------------- struct zero_destination : public transfer_error diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index 2cddea25d..0e0b2e4eb 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -59,7 +59,6 @@ namespace { const command_line::arg_descriptor<std::string, true> arg_rpc_bind_port = {"rpc-bind-port", "Sets bind port for server"}; const command_line::arg_descriptor<bool> arg_disable_rpc_login = {"disable-rpc-login", "Disable HTTP authentication for RPC connections served by this process"}; - const command_line::arg_descriptor<bool> arg_trusted_daemon = {"trusted-daemon", "Enable commands which rely on a trusted daemon", false}; const command_line::arg_descriptor<bool> arg_restricted = {"restricted-rpc", "Restricts to view-only commands", false}; const command_line::arg_descriptor<std::string> arg_wallet_dir = {"wallet-dir", "Directory for newly created wallets"}; const command_line::arg_descriptor<bool> arg_prompt_for_password = {"prompt-for-password", "Prompts for password when not provided", false}; @@ -100,7 +99,7 @@ namespace tools } //------------------------------------------------------------------------------------------------------------------------------ - wallet_rpc_server::wallet_rpc_server():m_wallet(NULL), rpc_login_file(), m_stop(false), m_trusted_daemon(false), m_restricted(false), m_vm(NULL) + wallet_rpc_server::wallet_rpc_server():m_wallet(NULL), rpc_login_file(), m_stop(false), m_restricted(false), m_vm(NULL) { } //------------------------------------------------------------------------------------------------------------------------------ @@ -120,7 +119,7 @@ namespace tools m_stop = false; m_net_server.add_idle_handler([this](){ try { - if (m_wallet) m_wallet->refresh(m_trusted_daemon); + if (m_wallet) m_wallet->refresh(m_wallet->is_trusted_daemon()); } catch (const std::exception& ex) { LOG_ERROR("Exception at while refreshing, what=" << ex.what()); } @@ -156,28 +155,10 @@ namespace tools return false; m_vm = vm; - tools::wallet2 *walvars; - std::unique_ptr<tools::wallet2> tmpwal; - if (m_wallet) - walvars = m_wallet; - else - { - tmpwal = tools::wallet2::make_dummy(*m_vm, true, password_prompter); - walvars = tmpwal.get(); - } boost::optional<epee::net_utils::http::login> http_login{}; std::string bind_port = command_line::get_arg(*m_vm, arg_rpc_bind_port); const bool disable_auth = command_line::get_arg(*m_vm, arg_disable_rpc_login); - m_trusted_daemon = command_line::get_arg(*m_vm, arg_trusted_daemon); - if (!command_line::has_arg(*m_vm, arg_trusted_daemon)) - { - if (tools::is_local_address(walvars->get_daemon_address())) - { - MINFO(tr("Daemon is local, assuming trusted")); - m_trusted_daemon = true; - } - } m_restricted = command_line::get_arg(*m_vm, arg_restricted); if (command_line::has_arg(*m_vm, arg_wallet_dir)) { @@ -857,7 +838,7 @@ namespace tools mixin = m_wallet->adjust_mixin(req.mixin); } uint32_t priority = m_wallet->adjust_priority(req.priority); - std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_2(dsts, mixin, req.unlock_time, priority, extra, req.account_index, req.subaddr_indices, m_trusted_daemon); + std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_2(dsts, mixin, req.unlock_time, priority, extra, req.account_index, req.subaddr_indices); if (ptx_vector.empty()) { @@ -918,7 +899,7 @@ namespace tools } uint32_t priority = m_wallet->adjust_priority(req.priority); LOG_PRINT_L2("on_transfer_split calling create_transactions_2"); - std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_2(dsts, mixin, req.unlock_time, priority, extra, req.account_index, req.subaddr_indices, m_trusted_daemon); + std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_2(dsts, mixin, req.unlock_time, priority, extra, req.account_index, req.subaddr_indices); LOG_PRINT_L2("on_transfer_split called create_transactions_2"); return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay, @@ -1079,7 +1060,7 @@ namespace tools try { - std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_unmixable_sweep_transactions(m_trusted_daemon); + std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_unmixable_sweep_transactions(); return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay, res.tx_hash_list, req.get_tx_hex, res.tx_blob_list, req.get_tx_metadata, res.tx_metadata_list, er); @@ -1127,7 +1108,7 @@ namespace tools mixin = m_wallet->adjust_mixin(req.mixin); } uint32_t priority = m_wallet->adjust_priority(req.priority); - std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_all(req.below_amount, dsts[0].addr, dsts[0].is_subaddress, mixin, req.unlock_time, priority, extra, req.account_index, req.subaddr_indices, m_trusted_daemon); + std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_all(req.below_amount, dsts[0].addr, dsts[0].is_subaddress, mixin, req.unlock_time, priority, extra, req.account_index, req.subaddr_indices); return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay, res.tx_hash_list, req.get_tx_hex, res.tx_blob_list, req.get_tx_metadata, res.tx_metadata_list, er); @@ -1183,7 +1164,7 @@ namespace tools mixin = m_wallet->adjust_mixin(req.mixin); } uint32_t priority = m_wallet->adjust_priority(req.priority); - std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_single(ki, dsts[0].addr, dsts[0].is_subaddress, mixin, req.unlock_time, priority, extra, m_trusted_daemon); + std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_single(ki, dsts[0].addr, dsts[0].is_subaddress, mixin, req.unlock_time, priority, extra); if (ptx_vector.empty()) { @@ -1552,7 +1533,6 @@ namespace tools rpc_transfers.spent = td.m_spent; rpc_transfers.global_index = td.m_global_output_index; rpc_transfers.tx_hash = epee::string_tools::pod_to_hex(td.m_txid); - rpc_transfers.tx_size = txBlob.size(); rpc_transfers.subaddr_index = td.m_subaddr_index.minor; rpc_transfers.key_image = req.verbose && td.m_key_image_known ? epee::string_tools::pod_to_hex(td.m_key_image) : ""; res.transfers.push_back(rpc_transfers); @@ -2304,7 +2284,7 @@ namespace tools er.message = "Command unavailable in restricted mode."; return false; } - if (!m_trusted_daemon) + if (!m_wallet->is_trusted_daemon()) { er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; er.message = "This command requires a trusted daemon."; @@ -2518,7 +2498,7 @@ namespace tools } try { - m_wallet->refresh(m_trusted_daemon, req.start_height, res.blocks_fetched, res.received_money); + m_wallet->refresh(m_wallet->is_trusted_daemon(), req.start_height, res.blocks_fetched, res.received_money); return true; } catch (const std::exception& e) @@ -2554,7 +2534,7 @@ namespace tools bool wallet_rpc_server::on_start_mining(const wallet_rpc::COMMAND_RPC_START_MINING::request& req, wallet_rpc::COMMAND_RPC_START_MINING::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); - if (!m_trusted_daemon) + if (!m_wallet->is_trusted_daemon()) { er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; er.message = "This command requires a trusted daemon."; @@ -2688,8 +2668,20 @@ namespace tools er.message = "Failed to generate wallet"; return false; } + if (m_wallet) + { + try + { + m_wallet->store(); + } + catch (const std::exception& e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } delete m_wallet; + } m_wallet = wal.release(); return true; } @@ -2746,12 +2738,74 @@ namespace tools er.message = "Failed to open wallet"; return false; } + if (m_wallet) + { + try + { + m_wallet->store(); + } + catch (const std::exception& e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } delete m_wallet; + } m_wallet = wal.release(); return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_close_wallet(const wallet_rpc::COMMAND_RPC_CLOSE_WALLET::request& req, wallet_rpc::COMMAND_RPC_CLOSE_WALLET::response& res, epee::json_rpc::error& er) + { + if (!m_wallet) return not_open(er); + + try + { + m_wallet->store(); + } + catch (const std::exception& e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + delete m_wallet; + m_wallet = NULL; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_change_wallet_password(const wallet_rpc::COMMAND_RPC_CHANGE_WALLET_PASSWORD::request& req, wallet_rpc::COMMAND_RPC_CHANGE_WALLET_PASSWORD::response& res, epee::json_rpc::error& er) + { + if (!m_wallet) return not_open(er); + if (m_restricted) + { + er.code = WALLET_RPC_ERROR_CODE_DENIED; + er.message = "Command unavailable in restricted mode."; + return false; + } + if (m_wallet->verify_password(req.old_password)) + { + try + { + m_wallet->rewrite(m_wallet->get_wallet_file(), req.new_password); + m_wallet->store(); + LOG_PRINT_L0("Wallet password changed."); + } + catch (const std::exception& e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + } + else + { + er.code = WALLET_RPC_ERROR_CODE_INVALID_PASSWORD; + er.message = "Invalid original password."; + return false; + } + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ void wallet_rpc_server::handle_rpc_exception(const std::exception_ptr& e, epee::json_rpc::error& er, int default_error_code) { try { @@ -2993,7 +3047,7 @@ namespace tools return false; } - if (m_trusted_daemon) + if (m_wallet->is_trusted_daemon()) { try { @@ -3216,7 +3270,6 @@ int main(int argc, char** argv) { tools::wallet2::init_options(desc_params); command_line::add_arg(desc_params, arg_rpc_bind_port); command_line::add_arg(desc_params, arg_disable_rpc_login); - command_line::add_arg(desc_params, arg_trusted_daemon); command_line::add_arg(desc_params, arg_restricted); cryptonote::rpc_args::init_options(desc_params); command_line::add_arg(desc_params, arg_wallet_file); @@ -3309,7 +3362,7 @@ int main(int argc, char** argv) { wal->stop(); }); - wal->refresh(command_line::get_arg(*vm, arg_trusted_daemon)); + wal->refresh(wal->is_trusted_daemon()); // if we ^C during potentially length load/refresh, there's no server loop yet if (quit) { diff --git a/src/wallet/wallet_rpc_server.h b/src/wallet/wallet_rpc_server.h index bfe49a459..ab7917a78 100644 --- a/src/wallet/wallet_rpc_server.h +++ b/src/wallet/wallet_rpc_server.h @@ -133,6 +133,8 @@ namespace tools MAP_JON_RPC_WE("get_languages", on_get_languages, wallet_rpc::COMMAND_RPC_GET_LANGUAGES) MAP_JON_RPC_WE("create_wallet", on_create_wallet, wallet_rpc::COMMAND_RPC_CREATE_WALLET) MAP_JON_RPC_WE("open_wallet", on_open_wallet, wallet_rpc::COMMAND_RPC_OPEN_WALLET) + MAP_JON_RPC_WE("close_wallet", on_close_wallet, wallet_rpc::COMMAND_RPC_CLOSE_WALLET) + MAP_JON_RPC_WE("change_wallet_password", on_change_wallet_password, wallet_rpc::COMMAND_RPC_CHANGE_WALLET_PASSWORD) MAP_JON_RPC_WE("is_multisig", on_is_multisig, wallet_rpc::COMMAND_RPC_IS_MULTISIG) MAP_JON_RPC_WE("prepare_multisig", on_prepare_multisig, wallet_rpc::COMMAND_RPC_PREPARE_MULTISIG) MAP_JON_RPC_WE("make_multisig", on_make_multisig, wallet_rpc::COMMAND_RPC_MAKE_MULTISIG) @@ -208,6 +210,8 @@ namespace tools bool on_get_languages(const wallet_rpc::COMMAND_RPC_GET_LANGUAGES::request& req, wallet_rpc::COMMAND_RPC_GET_LANGUAGES::response& res, epee::json_rpc::error& er); bool on_create_wallet(const wallet_rpc::COMMAND_RPC_CREATE_WALLET::request& req, wallet_rpc::COMMAND_RPC_CREATE_WALLET::response& res, epee::json_rpc::error& er); bool on_open_wallet(const wallet_rpc::COMMAND_RPC_OPEN_WALLET::request& req, wallet_rpc::COMMAND_RPC_OPEN_WALLET::response& res, epee::json_rpc::error& er); + bool on_close_wallet(const wallet_rpc::COMMAND_RPC_CLOSE_WALLET::request& req, wallet_rpc::COMMAND_RPC_CLOSE_WALLET::response& res, epee::json_rpc::error& er); + bool on_change_wallet_password(const wallet_rpc::COMMAND_RPC_CHANGE_WALLET_PASSWORD::request& req, wallet_rpc::COMMAND_RPC_CHANGE_WALLET_PASSWORD::response& res, epee::json_rpc::error& er); bool on_is_multisig(const wallet_rpc::COMMAND_RPC_IS_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_IS_MULTISIG::response& res, epee::json_rpc::error& er); bool on_prepare_multisig(const wallet_rpc::COMMAND_RPC_PREPARE_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_PREPARE_MULTISIG::response& res, epee::json_rpc::error& er); bool on_make_multisig(const wallet_rpc::COMMAND_RPC_MAKE_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_MAKE_MULTISIG::response& res, epee::json_rpc::error& er); @@ -238,7 +242,6 @@ namespace tools std::string m_wallet_dir; tools::private_file rpc_login_file; std::atomic<bool> m_stop; - bool m_trusted_daemon; bool m_restricted; const boost::program_options::variables_map *m_vm; }; diff --git a/src/wallet/wallet_rpc_server_commands_defs.h b/src/wallet/wallet_rpc_server_commands_defs.h index 3f946eaed..4501cf575 100644 --- a/src/wallet/wallet_rpc_server_commands_defs.h +++ b/src/wallet/wallet_rpc_server_commands_defs.h @@ -47,7 +47,7 @@ // advance which version they will stop working with // Don't go over 32767 for any of these #define WALLET_RPC_VERSION_MAJOR 1 -#define WALLET_RPC_VERSION_MINOR 2 +#define WALLET_RPC_VERSION_MINOR 3 #define MAKE_WALLET_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define WALLET_RPC_VERSION MAKE_WALLET_RPC_VERSION(WALLET_RPC_VERSION_MAJOR, WALLET_RPC_VERSION_MINOR) namespace tools @@ -445,7 +445,6 @@ namespace wallet_rpc { std::string tx_hash; std::string tx_key; - std::list<std::string> amount_keys; uint64_t amount; uint64_t fee; std::string tx_blob; @@ -456,7 +455,6 @@ namespace wallet_rpc BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tx_hash) KV_SERIALIZE(tx_key) - KV_SERIALIZE(amount_keys) KV_SERIALIZE(amount) KV_SERIALIZE(fee) KV_SERIALIZE(tx_blob) @@ -768,15 +766,9 @@ namespace wallet_rpc struct response { std::string tx_hash; - std::string tx_key; - uint64_t fee; - std::string tx_blob; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tx_hash) - KV_SERIALIZE(tx_key) - KV_SERIALIZE(fee) - KV_SERIALIZE(tx_blob) END_KV_SERIALIZE_MAP() }; }; @@ -867,7 +859,6 @@ namespace wallet_rpc bool spent; uint64_t global_index; std::string tx_hash; - uint64_t tx_size; uint32_t subaddr_index; std::string key_image; @@ -876,7 +867,6 @@ namespace wallet_rpc KV_SERIALIZE(spent) KV_SERIALIZE(global_index) KV_SERIALIZE(tx_hash) - KV_SERIALIZE(tx_size) KV_SERIALIZE(subaddr_index) KV_SERIALIZE(key_image) END_KV_SERIALIZE_MAP() @@ -1831,6 +1821,40 @@ namespace wallet_rpc }; }; + struct COMMAND_RPC_CLOSE_WALLET + { + struct request + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + + struct response + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + }; + + struct COMMAND_RPC_CHANGE_WALLET_PASSWORD + { + struct request + { + std::string old_password; + std::string new_password; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(old_password) + KV_SERIALIZE(new_password) + END_KV_SERIALIZE_MAP() + }; + struct response + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + }; + struct COMMAND_RPC_IS_MULTISIG { struct request |