diff options
Diffstat (limited to 'src')
33 files changed, 2769 insertions, 248 deletions
diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index cce288793..d7230f644 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -1448,7 +1448,9 @@ public: * * @return false if the function returns false for any output, otherwise true */ - virtual bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, size_t tx_idx)> f) const = 0; + virtual bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, uint64_t height, size_t tx_idx)> f) const = 0; + virtual bool for_all_outputs(uint64_t amount, const std::function<bool(uint64_t height)> &f) const = 0; + // diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index f9c829013..9a755fb0c 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -2564,7 +2564,7 @@ bool BlockchainLMDB::for_all_transactions(std::function<bool(const crypto::hash& return fret; } -bool BlockchainLMDB::for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, size_t tx_idx)> f) const +bool BlockchainLMDB::for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, uint64_t height, size_t tx_idx)> f) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -2588,7 +2588,47 @@ bool BlockchainLMDB::for_all_outputs(std::function<bool(uint64_t amount, const c uint64_t amount = *(const uint64_t*)k.mv_data; outkey *ok = (outkey *)v.mv_data; tx_out_index toi = get_output_tx_and_index_from_global(ok->output_id); - if (!f(amount, toi.first, toi.second)) { + if (!f(amount, toi.first, ok->data.height, toi.second)) { + fret = false; + break; + } + } + + TXN_POSTFIX_RDONLY(); + + return fret; +} + +bool BlockchainLMDB::for_all_outputs(uint64_t amount, const std::function<bool(uint64_t height)> &f) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + RCURSOR(output_amounts); + + MDB_val_set(k, amount); + MDB_val v; + bool fret = true; + + MDB_cursor_op op = MDB_SET; + while (1) + { + int ret = mdb_cursor_get(m_cur_output_amounts, &k, &v, op); + op = MDB_NEXT_DUP; + if (ret == MDB_NOTFOUND) + break; + if (ret) + throw0(DB_ERROR("Failed to enumerate outputs")); + uint64_t out_amount = *(const uint64_t*)k.mv_data; + if (amount != out_amount) + { + MERROR("Amount is not the expected amount"); + fret = false; + break; + } + const outkey *ok = (const outkey *)v.mv_data; + if (!f(ok->data.height)) { fret = false; break; } diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 1b76abf35..0aa4bb86e 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -255,7 +255,8 @@ public: virtual bool for_all_key_images(std::function<bool(const crypto::key_image&)>) const; virtual bool for_blocks_range(const uint64_t& h1, const uint64_t& h2, std::function<bool(uint64_t, const crypto::hash&, const cryptonote::block&)>) const; virtual bool for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)>) const; - virtual bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, size_t tx_idx)> f) const; + virtual bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, uint64_t height, size_t tx_idx)> f) const; + virtual bool for_all_outputs(uint64_t amount, const std::function<bool(uint64_t height)> &f) const; virtual uint64_t add_block( const block& blk , const size_t& block_size diff --git a/src/blockchain_utilities/CMakeLists.txt b/src/blockchain_utilities/CMakeLists.txt index a701bc605..a5dd69556 100644 --- a/src/blockchain_utilities/CMakeLists.txt +++ b/src/blockchain_utilities/CMakeLists.txt @@ -67,6 +67,30 @@ monero_private_headers(blockchain_export ${blockchain_export_private_headers}) +set(blockchain_blackball_sources + blockchain_blackball.cpp + ) + +set(blockchain_blackball_private_headers + bootstrap_file.h + blocksdat_file.h + bootstrap_serialization.h + ) + +monero_private_headers(blockchain_blackball + ${blockchain_blackball_private_headers}) + +set(blockchain_usage_sources + blockchain_usage.cpp + ) + +set(blockchain_usage_private_headers) + +monero_private_headers(blockchain_usage + ${blockchain_usage_private_headers}) + + + monero_add_executable(blockchain_import ${blockchain_import_sources} ${blockchain_import_private_headers} @@ -117,3 +141,49 @@ set_property(TARGET blockchain_export OUTPUT_NAME "monero-blockchain-export") install(TARGETS blockchain_export DESTINATION bin) +monero_add_executable(blockchain_blackball + ${blockchain_blackball_sources} + ${blockchain_blackball_private_headers}) + +target_link_libraries(blockchain_blackball + PRIVATE + wallet + cryptonote_core + blockchain_db + p2p + version + epee + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${Boost_THREAD_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES}) + +set_property(TARGET blockchain_blackball + PROPERTY + OUTPUT_NAME "monero-blockchain-blackball") +install(TARGETS blockchain_blackball DESTINATION bin) + + +monero_add_executable(blockchain_usage + ${blockchain_usage_sources} + ${blockchain_usage_private_headers}) + +target_link_libraries(blockchain_usage + PRIVATE + cryptonote_core + blockchain_db + p2p + version + epee + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${Boost_THREAD_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES}) + +set_property(TARGET blockchain_usage + PROPERTY + OUTPUT_NAME "monero-blockchain-usage") +install(TARGETS blockchain_usage DESTINATION bin) + diff --git a/src/blockchain_utilities/blockchain_blackball.cpp b/src/blockchain_utilities/blockchain_blackball.cpp new file mode 100644 index 000000000..40ce898d9 --- /dev/null +++ b/src/blockchain_utilities/blockchain_blackball.cpp @@ -0,0 +1,427 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <boost/range/adaptor/transformed.hpp> +#include <boost/algorithm/string.hpp> +#include "common/command_line.h" +#include "common/varint.h" +#include "cryptonote_core/tx_pool.h" +#include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_core/blockchain.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/db_types.h" +#include "wallet/ringdb.h" +#include "version.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "bcutil" + +namespace po = boost::program_options; +using namespace epee; +using namespace cryptonote; + +struct output_data +{ + uint64_t amount; + uint64_t index; + output_data(uint64_t a, uint64_t i): amount(a), index(i) {} + bool operator==(const output_data &other) const { return other.amount == amount && other.index == index; } +}; +namespace std +{ + template<> struct hash<output_data> + { + size_t operator()(const output_data &od) const + { + const uint64_t data[2] = {od.amount, od.index}; + crypto::hash h; + crypto::cn_fast_hash(data, 2 * sizeof(uint64_t), h); + return reinterpret_cast<const std::size_t &>(h); + } + }; +} + +static std::string get_default_db_path() +{ + boost::filesystem::path dir = tools::get_default_data_dir(); + // remove .bitmonero, replace with .shared-ringdb + dir = dir.remove_filename(); + dir /= ".shared-ringdb"; + return dir.string(); +} + +static bool for_all_transactions(const std::string &filename, const std::function<bool(const cryptonote::transaction_prefix&)> &f) +{ + MDB_env *env; + MDB_dbi dbi; + MDB_txn *txn; + MDB_cursor *cur; + int dbr; + bool tx_active = false; + + dbr = mdb_env_create(&env); + if (dbr) throw std::runtime_error("Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); + dbr = mdb_env_set_maxdbs(env, 2); + if (dbr) throw std::runtime_error("Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); + const std::string actual_filename = filename; + dbr = mdb_env_open(env, actual_filename.c_str(), 0, 0664); + if (dbr) throw std::runtime_error("Failed to open rings database file '" + + actual_filename + "': " + std::string(mdb_strerror(dbr))); + + dbr = mdb_txn_begin(env, NULL, 0, &txn); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + dbr = mdb_dbi_open(txn, "txs_pruned", MDB_INTEGERKEY, &dbi); + if (dbr) + dbr = mdb_dbi_open(txn, "txs", MDB_INTEGERKEY, &dbi); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_open(txn, dbi, &cur); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + + MDB_val k; + MDB_val v; + bool fret = true; + + MDB_cursor_op op = MDB_FIRST; + while (1) + { + int ret = mdb_cursor_get(cur, &k, &v, op); + op = MDB_NEXT; + if (ret == MDB_NOTFOUND) + break; + if (ret) + throw std::runtime_error("Failed to enumerate transactions: " + std::string(mdb_strerror(ret))); + + cryptonote::transaction_prefix tx; + blobdata bd; + bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size); + std::stringstream ss; + ss << bd; + binary_archive<false> ba(ss); + bool r = do_serialize(ba, tx); + CHECK_AND_ASSERT_MES(r, false, "Failed to parse transaction from blob"); + + if (!f(tx)) { + fret = false; + break; + } + } + + mdb_cursor_close(cur); + mdb_txn_commit(txn); + tx_active = false; + mdb_dbi_close(env, dbi); + mdb_env_close(env); + return fret; +} + +int main(int argc, char* argv[]) +{ + TRY_ENTRY(); + + epee::string_tools::set_module_name_and_folder(argv[0]); + + std::string default_db_type = "lmdb"; + + std::string available_dbs = cryptonote::blockchain_db_types(", "); + available_dbs = "available: " + available_dbs; + + uint32_t log_level = 0; + + tools::on_startup(); + + boost::filesystem::path output_file_path; + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor<std::string, false, true, 2> arg_blackball_db_dir = { + "blackball-db-dir", "Specify blackball database directory", + get_default_db_path(), + {{ &arg_testnet_on, &arg_stagenet_on }}, + [](std::array<bool, 2> testnet_stagenet, bool defaulted, std::string val) { + if (testnet_stagenet[0]) + return (boost::filesystem::path(val) / "testnet").string(); + else if (testnet_stagenet[1]) + return (boost::filesystem::path(val) / "stagenet").string(); + return val; + } + }; + const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""}; + const command_line::arg_descriptor<std::string> arg_database = { + "database", available_dbs.c_str(), default_db_type + }; + const command_line::arg_descriptor<bool> arg_rct_only = {"rct-only", "Only work on ringCT outputs", false}; + const command_line::arg_descriptor<std::vector<std::string> > arg_inputs = {"inputs", "Path to Monero DB, and path to any fork DBs"}; + + command_line::add_arg(desc_cmd_sett, arg_blackball_db_dir); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_database); + command_line::add_arg(desc_cmd_sett, arg_rct_only); + command_line::add_arg(desc_cmd_sett, arg_inputs); + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::positional_options_description positional_options; + positional_options.add(arg_inputs.name, -1); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + auto parser = po::command_line_parser(argc, argv).options(desc_options).positional(positional_options); + po::store(parser.run(), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + mlog_configure(mlog_get_default_log_path("monero-blockchain-blackball.log"), true); + if (!command_line::is_arg_defaulted(vm, arg_log_level)) + mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); + else + mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str()); + + LOG_PRINT_L0("Starting..."); + + bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); + bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); + network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; + output_file_path = command_line::get_arg(vm, arg_blackball_db_dir); + bool opt_rct_only = command_line::get_arg(vm, arg_rct_only); + + std::string db_type = command_line::get_arg(vm, arg_database); + if (!cryptonote::blockchain_valid_db_type(db_type)) + { + std::cerr << "Invalid database type: " << db_type << std::endl; + return 1; + } + + // If we wanted to use the memory pool, we would set up a fake_core. + + // Use Blockchain instead of lower-level BlockchainDB for two reasons: + // 1. Blockchain has the init() method for easy setup + // 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash() + // + // cannot match blockchain_storage setup above with just one line, + // e.g. + // Blockchain* core_storage = new Blockchain(NULL); + // because unlike blockchain_storage constructor, which takes a pointer to + // tx_memory_pool, Blockchain's constructor takes tx_memory_pool object. + LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); + const std::vector<std::string> inputs = command_line::get_arg(vm, arg_inputs); + if (inputs.empty()) + { + LOG_PRINT_L0("No inputs given"); + return 1; + } + std::vector<std::unique_ptr<Blockchain>> core_storage(inputs.size()); + tx_memory_pool m_mempool(*(Blockchain*)NULL); + for (size_t n = 0; n < inputs.size(); ++n) + { + core_storage[n].reset(new Blockchain(m_mempool)); + + BlockchainDB* db = new_db(db_type); + if (db == NULL) + { + LOG_ERROR("Attempted to use non-existent database type: " << db_type); + throw std::runtime_error("Attempting to use non-existent database type"); + } + LOG_PRINT_L0("database: " << db_type); + + std::string filename = inputs[n]; + while (boost::ends_with(filename, "/") || boost::ends_with(filename, "\\")) + filename.pop_back(); + LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); + + try + { + db->open(filename, DBF_RDONLY); + } + catch (const std::exception& e) + { + LOG_PRINT_L0("Error opening database: " << e.what()); + return 1; + } + r = core_storage[n]->init(db, net_type); + + CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage"); + LOG_PRINT_L0("Source blockchain storage initialized OK"); + } + + boost::filesystem::path direc(output_file_path.string()); + if (boost::filesystem::exists(direc)) + { + if (!boost::filesystem::is_directory(direc)) + { + MERROR("LMDB needs a directory path, but a file was passed: " << output_file_path.string()); + return 1; + } + } + else + { + if (!boost::filesystem::create_directories(direc)) + { + MERROR("Failed to create directory: " << output_file_path.string()); + return 1; + } + } + + LOG_PRINT_L0("Scanning for blackballable outputs..."); + + size_t done = 0; + std::unordered_map<crypto::key_image, std::vector<uint64_t>> relative_rings; + std::unordered_map<output_data, std::unordered_set<crypto::key_image>> outputs; + std::unordered_set<output_data> spent, newly_spent; + + cryptonote::block b = core_storage[0]->get_db().get_block_from_height(0); + tools::ringdb ringdb(output_file_path.string(), epee::string_tools::pod_to_hex(get_block_hash(b))); + + for (size_t n = 0; n < inputs.size(); ++n) + { + LOG_PRINT_L0("Reading blockchain from " << inputs[n]); + for_all_transactions(inputs[n], [&](const cryptonote::transaction_prefix &tx)->bool + { + for (const auto &in: tx.vin) + { + if (in.type() != typeid(txin_to_key)) + continue; + const auto &txin = boost::get<txin_to_key>(in); + if (opt_rct_only && txin.amount != 0) + continue; + + const std::vector<uint64_t> absolute = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets); + if (n == 0) + for (uint64_t out: absolute) + outputs[output_data(txin.amount, out)].insert(txin.k_image); + + std::vector<uint64_t> new_ring = txin.key_offsets; + const uint32_t ring_size = txin.key_offsets.size(); + if (ring_size == 1) + { + const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, txin.key_offsets[0]); + MINFO("Blackballing output " << pkey << ", due to being used in a 1-ring"); + ringdb.blackball(pkey); + newly_spent.insert(output_data(txin.amount, txin.key_offsets[0])); + spent.insert(output_data(txin.amount, txin.key_offsets[0])); + } + else if (relative_rings.find(txin.k_image) != relative_rings.end()) + { + MINFO("Key image " << txin.k_image << " already seen: rings " << + boost::join(relative_rings[txin.k_image] | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " ") << + ", " << boost::join(txin.key_offsets | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " ")); + if (relative_rings[txin.k_image] != txin.key_offsets) + { + MINFO("Rings are different"); + const std::vector<uint64_t> r0 = cryptonote::relative_output_offsets_to_absolute(relative_rings[txin.k_image]); + const std::vector<uint64_t> r1 = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets); + std::vector<uint64_t> common; + for (uint64_t out: r0) + { + if (std::find(r1.begin(), r1.end(), out) != r1.end()) + common.push_back(out); + } + if (common.empty()) + { + MERROR("Rings for the same key image are disjoint"); + } + else if (common.size() == 1) + { + const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, common[0]); + MINFO("Blackballing output " << pkey << ", due to being used in rings with a single common element"); + ringdb.blackball(pkey); + newly_spent.insert(output_data(txin.amount, common[0])); + spent.insert(output_data(txin.amount, common[0])); + } + else + { + MINFO("The intersection has more than one element, it's still ok"); + for (const auto &out: r0) + if (std::find(common.begin(), common.end(), out) != common.end()) + new_ring.push_back(out); + new_ring = cryptonote::absolute_output_offsets_to_relative(new_ring); + } + } + } + relative_rings[txin.k_image] = new_ring; + } + return true; + }); + } + + while (!newly_spent.empty()) + { + LOG_PRINT_L0("Secondary pass due to " << newly_spent.size() << " newly found spent outputs"); + std::unordered_set<output_data> work_spent = std::move(newly_spent); + newly_spent.clear(); + + for (const output_data &od: work_spent) + { + for (const crypto::key_image &ki: outputs[od]) + { + std::vector<uint64_t> absolute = cryptonote::relative_output_offsets_to_absolute(relative_rings[ki]); + size_t known = 0; + uint64_t last_unknown = 0; + for (uint64_t out: absolute) + { + output_data new_od(od.amount, out); + if (spent.find(new_od) != spent.end()) + ++known; + else + last_unknown = out; + } + if (known == absolute.size() - 1) + { + const crypto::public_key pkey = core_storage[0]->get_output_key(od.amount, last_unknown); + MINFO("Blackballing output " << pkey << ", due to being used in a " << + absolute.size() << "-ring where all other outputs are known to be spent"); + ringdb.blackball(pkey); + newly_spent.insert(output_data(od.amount, last_unknown)); + spent.insert(output_data(od.amount, last_unknown)); + } + } + } + } + + LOG_PRINT_L0("Blockchain blackball data exported OK"); + return 0; + + CATCH_ENTRY("Export error", 1); +} diff --git a/src/blockchain_utilities/blockchain_usage.cpp b/src/blockchain_utilities/blockchain_usage.cpp new file mode 100644 index 000000000..7c3c83167 --- /dev/null +++ b/src/blockchain_utilities/blockchain_usage.cpp @@ -0,0 +1,256 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <boost/range/adaptor/transformed.hpp> +#include <boost/algorithm/string.hpp> +#include "common/command_line.h" +#include "common/varint.h" +#include "cryptonote_core/tx_pool.h" +#include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_core/blockchain.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/db_types.h" +#include "version.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "bcutil" + +namespace po = boost::program_options; +using namespace epee; +using namespace cryptonote; + +struct output_data +{ + uint64_t amount; + uint64_t index; + mutable bool coinbase; + mutable uint64_t height; + output_data(uint64_t a, uint64_t i, bool cb, uint64_t h): amount(a), index(i), coinbase(cb), height(h) {} + bool operator==(const output_data &other) const { return other.amount == amount && other.index == index; } + void info(bool c, uint64_t h) const { coinbase = c; height =h; } +}; +namespace std +{ + template<> struct hash<output_data> + { + size_t operator()(const output_data &od) const + { + const uint64_t data[2] = {od.amount, od.index}; + crypto::hash h; + crypto::cn_fast_hash(data, 2 * sizeof(uint64_t), h); + return reinterpret_cast<const std::size_t &>(h); + } + }; +} + +struct reference +{ + uint64_t height; + uint64_t ring_size; + uint64_t position; + reference(uint64_t h, uint64_t rs, uint64_t p): height(h), ring_size(rs), position(p) {} +}; + +int main(int argc, char* argv[]) +{ + TRY_ENTRY(); + + epee::string_tools::set_module_name_and_folder(argv[0]); + + std::string default_db_type = "lmdb"; + + std::string available_dbs = cryptonote::blockchain_db_types(", "); + available_dbs = "available: " + available_dbs; + + uint32_t log_level = 0; + + tools::on_startup(); + + boost::filesystem::path output_file_path; + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""}; + const command_line::arg_descriptor<std::string> arg_database = { + "database", available_dbs.c_str(), default_db_type + }; + const command_line::arg_descriptor<bool> arg_rct_only = {"rct-only", "Only work on ringCT outputs", false}; + const command_line::arg_descriptor<std::string> arg_input = {"input", ""}; + + command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_database); + command_line::add_arg(desc_cmd_sett, arg_rct_only); + command_line::add_arg(desc_cmd_sett, arg_input); + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::positional_options_description positional_options; + positional_options.add(arg_input.name, -1); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + auto parser = po::command_line_parser(argc, argv).options(desc_options).positional(positional_options); + po::store(parser.run(), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + mlog_configure(mlog_get_default_log_path("monero-blockchain-usage.log"), true); + if (!command_line::is_arg_defaulted(vm, arg_log_level)) + mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); + else + mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str()); + + LOG_PRINT_L0("Starting..."); + + bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); + bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); + network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; + bool opt_rct_only = command_line::get_arg(vm, arg_rct_only); + + std::string db_type = command_line::get_arg(vm, arg_database); + if (!cryptonote::blockchain_valid_db_type(db_type)) + { + std::cerr << "Invalid database type: " << db_type << std::endl; + return 1; + } + + // If we wanted to use the memory pool, we would set up a fake_core. + + // Use Blockchain instead of lower-level BlockchainDB for two reasons: + // 1. Blockchain has the init() method for easy setup + // 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash() + // + // cannot match blockchain_storage setup above with just one line, + // e.g. + // Blockchain* core_storage = new Blockchain(NULL); + // because unlike blockchain_storage constructor, which takes a pointer to + // tx_memory_pool, Blockchain's constructor takes tx_memory_pool object. + LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); + const std::string input = command_line::get_arg(vm, arg_input); + std::unique_ptr<Blockchain> core_storage; + tx_memory_pool m_mempool(*core_storage); + core_storage.reset(new Blockchain(m_mempool)); + BlockchainDB* db = new_db(db_type); + if (db == NULL) + { + LOG_ERROR("Attempted to use non-existent database type: " << db_type); + throw std::runtime_error("Attempting to use non-existent database type"); + } + LOG_PRINT_L0("database: " << db_type); + + const std::string filename = input; + LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); + + try + { + db->open(filename, DBF_RDONLY); + } + catch (const std::exception& e) + { + LOG_PRINT_L0("Error opening database: " << e.what()); + return 1; + } + r = core_storage->init(db, net_type); + + CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage"); + LOG_PRINT_L0("Source blockchain storage initialized OK"); + + LOG_PRINT_L0("Building usage patterns..."); + + size_t done = 0; + std::unordered_map<output_data, std::list<reference>> outputs; + std::unordered_map<uint64_t,uint64_t> indices; + + LOG_PRINT_L0("Reading blockchain from " << input); + core_storage->for_all_transactions([&](const crypto::hash &hash, const cryptonote::transaction &tx)->bool + { + const bool coinbase = tx.vin.size() == 1 && tx.vin[0].type() == typeid(txin_gen); + const uint64_t height = core_storage->get_db().get_tx_block_height(hash); + + // create new outputs + for (const auto &out: tx.vout) + { + if (opt_rct_only && out.amount) + continue; + uint64_t index = indices[out.amount]++; + output_data od(out.amount, indices[out.amount], coinbase, height); + auto itb = outputs.emplace(od, std::list<reference>()); + itb.first->first.info(coinbase, height); + } + + for (const auto &in: tx.vin) + { + if (in.type() != typeid(txin_to_key)) + continue; + const auto &txin = boost::get<txin_to_key>(in); + if (opt_rct_only && txin.amount != 0) + continue; + + const std::vector<uint64_t> absolute = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets); + for (size_t n = 0; n < txin.key_offsets.size(); ++n) + { + output_data od(txin.amount, absolute[n], coinbase, height); + outputs[od].push_back(reference(height, txin.key_offsets.size(), n)); + } + } + return true; + }); + + std::unordered_map<uint64_t, uint64_t> counts; + size_t total = 0; + for (const auto &out: outputs) + { + counts[out.second.size()]++; + total++; + } + for (const auto &c: counts) + { + float percent = 100.f * c.second / total; + MINFO(std::to_string(c.second) << " outputs used " << c.first << " times (" << percent << "%)"); + } + + LOG_PRINT_L0("Blockchain usage exported OK"); + return 0; + + CATCH_ENTRY("Export error", 1); +} diff --git a/src/cryptonote_basic/miner.cpp b/src/cryptonote_basic/miner.cpp index f949bbd2b..2c777f5a2 100644 --- a/src/cryptonote_basic/miner.cpp +++ b/src/cryptonote_basic/miner.cpp @@ -91,7 +91,7 @@ namespace cryptonote const command_line::arg_descriptor<bool> arg_bg_mining_ignore_battery = {"bg-mining-ignore-battery", "if true, assumes plugged in when unable to query system power status", false, true}; const command_line::arg_descriptor<uint64_t> arg_bg_mining_min_idle_interval_seconds = {"bg-mining-min-idle-interval", "Specify min lookback interval in seconds for determining idle state", miner::BACKGROUND_MINING_DEFAULT_MIN_IDLE_INTERVAL_IN_SECONDS, true}; const command_line::arg_descriptor<uint16_t> arg_bg_mining_idle_threshold_percentage = {"bg-mining-idle-threshold", "Specify minimum avg idle percentage over lookback interval", miner::BACKGROUND_MINING_DEFAULT_IDLE_THRESHOLD_PERCENTAGE, true}; - const command_line::arg_descriptor<uint16_t> arg_bg_mining_miner_target_percentage = {"bg-mining-miner-target", "Specificy maximum percentage cpu use by miner(s)", miner::BACKGROUND_MINING_DEFAULT_MINING_TARGET_PERCENTAGE, true}; + const command_line::arg_descriptor<uint16_t> arg_bg_mining_miner_target_percentage = {"bg-mining-miner-target", "Specify maximum percentage cpu use by miner(s)", miner::BACKGROUND_MINING_DEFAULT_MINING_TARGET_PERCENTAGE, true}; } diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 507b5f943..bc3575aa1 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -744,25 +744,6 @@ bool Blockchain::get_block_by_hash(const crypto::hash &h, block &blk, bool *orph return false; } //------------------------------------------------------------------ -//FIXME: this function does not seem to be called from anywhere, but -// if it ever is, should probably change std::list for std::vector -void Blockchain::get_all_known_block_ids(std::list<crypto::hash> &main, std::list<crypto::hash> &alt, std::list<crypto::hash> &invalid) const -{ - LOG_PRINT_L3("Blockchain::" << __func__); - CRITICAL_REGION_LOCAL(m_blockchain_lock); - - for (auto& a : m_db->get_hashes_range(0, m_db->height() - 1)) - { - main.push_back(a); - } - - for (const blocks_ext_by_hash::value_type &v: m_alternative_chains) - alt.push_back(v.first); - - for (const blocks_ext_by_hash::value_type &v: m_invalid_blocks) - invalid.push_back(v.first); -} -//------------------------------------------------------------------ // This function aggregates the cumulative difficulties and timestamps of the // last DIFFICULTY_BLOCKS_COUNT blocks and passes them to next_difficulty, // returning the result of that call. Ignores the genesis block, and can use @@ -1255,7 +1236,7 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m cumulative_size = txs_size + coinbase_blob_size; #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) MDEBUG("Creating block template: miner tx size " << coinbase_blob_size << - ", cumulative size " << cumulative_size << " is greater then before"); + ", cumulative size " << cumulative_size << " is greater than before"); #endif continue; } @@ -1266,7 +1247,7 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) MDEBUG("Creating block template: miner tx size " << coinbase_blob_size << ", cumulative size " << txs_size + coinbase_blob_size << - " is less then before, adding " << delta << " zero bytes"); + " is less than before, adding " << delta << " zero bytes"); #endif b.miner_tx.extra.insert(b.miner_tx.extra.end(), delta, 0); //here could be 1 byte difference, because of extra field counter is varint, and it can become from 1-byte len to 2-bytes len. @@ -1922,6 +1903,45 @@ void Blockchain::get_output_key_mask_unlocked(const uint64_t& amount, const uint unlocked = is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first)); } //------------------------------------------------------------------ +bool Blockchain::get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) const +{ + // rct outputs don't exist before v3 + if (amount == 0) + { + switch (m_nettype) + { + case STAGENET: start_height = stagenet_hard_forks[2].height; break; + case TESTNET: start_height = testnet_hard_forks[2].height; break; + case MAINNET: start_height = mainnet_hard_forks[2].height; break; + default: return false; + } + } + else + start_height = 0; + base = 0; + + const uint64_t real_start_height = start_height; + if (from_height > start_height) + start_height = from_height; + + distribution.clear(); + uint64_t db_height = m_db->height(); + if (start_height >= db_height) + return false; + distribution.resize(db_height - start_height, 0); + bool r = for_all_outputs(amount, [&](uint64_t height) { + CHECK_AND_ASSERT_MES(height >= real_start_height && height <= db_height, false, "Height not in expected range"); + if (height >= start_height) + distribution[height - start_height]++; + else + base++; + return true; + }); + if (!r) + return false; + return true; +} +//------------------------------------------------------------------ // This function takes a list of block hashes from another node // on the network to find where the split point is between us and them. // This is used to see what to send another node that needs to sync. @@ -4460,11 +4480,16 @@ bool Blockchain::for_all_transactions(std::function<bool(const crypto::hash&, co return m_db->for_all_transactions(f); } -bool Blockchain::for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, size_t tx_idx)> f) const +bool Blockchain::for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, uint64_t height, size_t tx_idx)> f) const { return m_db->for_all_outputs(f);; } +bool Blockchain::for_all_outputs(uint64_t amount, std::function<bool(uint64_t height)> f) const +{ + return m_db->for_all_outputs(amount, f);; +} + namespace cryptonote { template bool Blockchain::get_transactions(const std::vector<crypto::hash>&, std::list<transaction>&, std::list<crypto::hash>&) const; } diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index dd490cdd5..4423199de 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -207,15 +207,6 @@ namespace cryptonote bool get_block_by_hash(const crypto::hash &h, block &blk, bool *orphan = NULL) const; /** - * @brief get all block hashes (main chain, alt chains, and invalid blocks) - * - * @param main return-by-reference container to put result main chain blocks' hashes in - * @param alt return-by-reference container to put result alt chain blocks' hashes in - * @param invalid return-by-reference container to put result invalid blocks' hashes in - */ - void get_all_known_block_ids(std::list<crypto::hash> &main, std::list<crypto::hash> &alt, std::list<crypto::hash> &invalid) const; - - /** * @brief performs some preprocessing on a group of incoming blocks to speed up verification * * @param blocks a list of incoming blocks @@ -533,6 +524,17 @@ namespace cryptonote bool get_random_rct_outs(const COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::request& req, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::response& res) const; /** + * @brief gets per block distribution of outputs of a given amount + * + * @param amount the amount to get a ditribution for + * @param return-by-reference from_height the height before which we do not care about the data + * @param return-by-reference start_height the height of the first rct output + * @param return-by-reference distribution the start offset of the first rct output in this block (same as previous if none) + * @param return-by-reference base how many outputs of that amount are before the stated distribution + */ + bool get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) const; + + /** * @brief gets the global indices for outputs from a given transaction * * This function gets the global indices for all outputs belonging @@ -866,7 +868,17 @@ namespace cryptonote * * @return false if any output fails the check, otherwise true */ - bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, size_t tx_idx)>) const; + bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, uint64_t height, size_t tx_idx)>) const; + + /** + * @brief perform a check on all outputs of a given amount in the blockchain + * + * @param amount the amount to iterate through + * @param std::function the check to perform, pass/fail + * + * @return false if any output fails the check, otherwise true + */ + bool for_all_outputs(uint64_t amount, std::function<bool(uint64_t height)>) const; /** * @brief get a reference to the BlockchainDB in use by Blockchain diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 8c0118803..adccd0469 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -104,7 +104,7 @@ namespace cryptonote }; static const command_line::arg_descriptor<uint64_t> arg_test_drop_download_height = { "test-drop-download-height" - , "Like test-drop-download but disards only after around certain height" + , "Like test-drop-download but discards only after around certain height" , 0 }; static const command_line::arg_descriptor<int> arg_test_dbg_lock_sleep = { @@ -1105,6 +1105,11 @@ namespace cryptonote return m_blockchain_storage.get_random_rct_outs(req, res); } //----------------------------------------------------------------------------------------------- + bool core::get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) const + { + return m_blockchain_storage.get_output_distribution(amount, from_height, start_height, distribution, base); + } + //----------------------------------------------------------------------------------------------- bool core::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<uint64_t>& indexs) const { return m_blockchain_storage.get_tx_outputs_gindexs(tx_id, indexs); @@ -1169,7 +1174,7 @@ namespace cryptonote LOG_PRINT_L1("Block found but, seems that reorganize just happened after that, do not relay this block"); return true; } - CHECK_AND_ASSERT_MES(txs.size() == b.tx_hashes.size() && !missed_txs.size(), false, "cant find some transactions in found block:" << get_block_hash(b) << " txs.size()=" << txs.size() + CHECK_AND_ASSERT_MES(txs.size() == b.tx_hashes.size() && !missed_txs.size(), false, "can't find some transactions in found block:" << get_block_hash(b) << " txs.size()=" << txs.size() << ", b.tx_hashes.size()=" << b.tx_hashes.size() << ", missed_txs.size()" << missed_txs.size()); block_to_blob(b, arg.b.block); diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index e1e430516..abf79be1d 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -571,6 +571,12 @@ namespace cryptonote */ bool get_random_rct_outs(const COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::request& req, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::response& res) const; + /** + * @copydoc Blockchain::get_output_distribution + * + * @brief get per block distribution of outputs of a given amount + */ + bool get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) const; /** * @copydoc miner::pause diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index 762feb5ee..0af9737a7 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -153,7 +153,7 @@ namespace cryptonote uint64_t outputs_amount = get_outs_money_amount(tx); if(outputs_amount > inputs_amount) { - LOG_PRINT_L1("transaction use more money then it has: use " << print_money(outputs_amount) << ", have " << print_money(inputs_amount)); + LOG_PRINT_L1("transaction use more money than it has: use " << print_money(outputs_amount) << ", have " << print_money(inputs_amount)); tvc.m_verifivation_failed = true; tvc.m_overspend = true; return false; @@ -292,7 +292,7 @@ namespace cryptonote } catch (const std::exception &e) { - MERROR("internal error: transaction already exists at inserting in memorypool: " << e.what()); + MERROR("internal error: transaction already exists at inserting in memory pool: " << e.what()); return false; } tvc.m_added_to_pool = true; diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index 7c4f16c77..5d91ad569 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -1395,7 +1395,7 @@ skip: const uint64_t first_block_height_needed = span.first; const uint64_t last_block_height_needed = span.first + std::min(span.second, (uint64_t)count_limit) - 1; MDEBUG(context << " gap found, span: " << span.first << " - " << span.first + span.second - 1 << " (" << last_block_height_needed << ")"); - MDEBUG(context << " current known hashes from from " << first_block_height_known << " to " << last_block_height_known); + MDEBUG(context << " current known hashes from " << first_block_height_known << " to " << last_block_height_known); if (first_block_height_needed < first_block_height_known || last_block_height_needed > last_block_height_known) { MDEBUG(context << " we are missing some of the necessary hashes for this gap, requesting chain again"); diff --git a/src/daemon/p2p.h b/src/daemon/p2p.h index 7fcb03751..0f01c746d 100644 --- a/src/daemon/p2p.h +++ b/src/daemon/p2p.h @@ -65,7 +65,7 @@ public: { throw std::runtime_error("Failed to initialize p2p server."); } - MGINFO("P2p server initialized OK"); + MGINFO("p2p server initialized OK"); } t_node_server & get() diff --git a/src/gen_multisig/gen_multisig.cpp b/src/gen_multisig/gen_multisig.cpp index e165b8053..943589b4a 100644 --- a/src/gen_multisig/gen_multisig.cpp +++ b/src/gen_multisig/gen_multisig.cpp @@ -69,7 +69,7 @@ namespace { const command_line::arg_descriptor<std::string> arg_filename_base = {"filename-base", genms::tr("Base filename (-1, -2, etc suffixes will be appended as needed)"), ""}; const command_line::arg_descriptor<std::string> arg_scheme = {"scheme", genms::tr("Give threshold and participants at once as M/N"), ""}; - const command_line::arg_descriptor<uint32_t> arg_participants = {"participants", genms::tr("How many participants wil share parts of the multisig wallet"), 0}; + const command_line::arg_descriptor<uint32_t> arg_participants = {"participants", genms::tr("How many participants will share parts of the multisig wallet"), 0}; const command_line::arg_descriptor<uint32_t> arg_threshold = {"threshold", genms::tr("How many signers are required to sign a valid transaction"), 0}; const command_line::arg_descriptor<bool, false> arg_testnet = {"testnet", genms::tr("Create testnet multisig wallets"), false}; const command_line::arg_descriptor<bool, false> arg_stagenet = {"stagenet", genms::tr("Create stagenet multisig wallets"), false}; diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 0e3e6757b..c3d1a9d11 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -905,7 +905,7 @@ namespace cryptonote PERF_TIMER(on_save_bc); if( !m_core.get_blockchain_storage().store_blockchain() ) { - res.status = "Error while storing blockhain"; + res.status = "Error while storing blockchain"; return true; } res.status = CORE_RPC_STATUS_OK; @@ -1102,7 +1102,7 @@ namespace cryptonote if(req.reserve_size > 255) { error_resp.code = CORE_RPC_ERROR_CODE_TOO_BIG_RESERVE_SIZE; - error_resp.message = "To big reserved size, maximum 255"; + error_resp.message = "Too big reserved size, maximum 255"; return false; } @@ -2076,6 +2076,41 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_get_output_distribution(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res, epee::json_rpc::error& error_resp) + { + try + { + for (uint64_t amount: req.amounts) + { + std::vector<uint64_t> distribution; + uint64_t start_height, base; + if (!m_core.get_output_distribution(amount, req.from_height, start_height, distribution, base)) + { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = "Failed to get rct distribution"; + return false; + } + if (req.cumulative) + { + distribution[0] += base; + for (size_t n = 1; n < distribution.size(); ++n) + distribution[n] += distribution[n-1]; + } + res.distributions.push_back({amount, start_height, std::move(distribution), base}); + } + } + catch (const std::exception &e) + { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = "Failed to get output distribution"; + return false; + } + + res.status = CORE_RPC_STATUS_OK; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + const command_line::arg_descriptor<std::string, false, true, 2> core_rpc_server::arg_rpc_bind_port = { "rpc-bind-port" diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index 4754a5d5f..a5755e062 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -153,6 +153,7 @@ namespace cryptonote MAP_JON_RPC_WE_IF("relay_tx", on_relay_tx, COMMAND_RPC_RELAY_TX, !m_restricted) MAP_JON_RPC_WE_IF("sync_info", on_sync_info, COMMAND_RPC_SYNC_INFO, !m_restricted) MAP_JON_RPC_WE("get_txpool_backlog", on_get_txpool_backlog, COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG) + MAP_JON_RPC_WE_IF("get_output_distribution", on_get_output_distribution, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION, !m_restricted) END_JSON_RPC_MAP() END_URI_MAP2() @@ -214,6 +215,7 @@ namespace cryptonote bool on_relay_tx(const COMMAND_RPC_RELAY_TX::request& req, COMMAND_RPC_RELAY_TX::response& res, epee::json_rpc::error& error_resp); bool on_sync_info(const COMMAND_RPC_SYNC_INFO::request& req, COMMAND_RPC_SYNC_INFO::response& res, epee::json_rpc::error& error_resp); bool on_get_txpool_backlog(const COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::response& res, epee::json_rpc::error& error_resp); + bool on_get_output_distribution(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res, epee::json_rpc::error& error_resp); //----------------------- private: diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index a2c780376..660fb7889 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -49,7 +49,7 @@ namespace cryptonote // advance which version they will stop working with // Don't go over 32767 for any of these #define CORE_RPC_VERSION_MAJOR 1 -#define CORE_RPC_VERSION_MINOR 18 +#define CORE_RPC_VERSION_MINOR 19 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR) @@ -2203,4 +2203,47 @@ namespace cryptonote END_KV_SERIALIZE_MAP() }; }; + + struct COMMAND_RPC_GET_OUTPUT_DISTRIBUTION + { + struct request + { + std::vector<uint64_t> amounts; + uint64_t from_height; + bool cumulative; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(amounts) + KV_SERIALIZE_OPT(from_height, (uint64_t)0) + KV_SERIALIZE_OPT(cumulative, false) + END_KV_SERIALIZE_MAP() + }; + + struct distribution + { + uint64_t amount; + uint64_t start_height; + std::vector<uint64_t> distribution; + uint64_t base; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(amount) + KV_SERIALIZE(start_height) + KV_SERIALIZE_CONTAINER_POD_AS_BLOB(distribution) + KV_SERIALIZE(base) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string status; + std::vector<distribution> distributions; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + KV_SERIALIZE(distributions) + END_KV_SERIALIZE_MAP() + }; + }; + } diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index 3668df7b9..6ac8fee9d 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -375,8 +375,9 @@ namespace return true; } - void handle_transfer_exception(const std::exception_ptr &e) + void handle_transfer_exception(const std::exception_ptr &e, bool trusted_daemon) { + bool warn_of_possible_attack = !trusted_daemon; try { std::rethrow_exception(e); @@ -404,6 +405,7 @@ namespace print_money(e.available()) % print_money(e.tx_amount())); fail_msg_writer() << tr("Not enough money in unlocked balance"); + warn_of_possible_attack = false; } catch (const tools::error::not_enough_money& e) { @@ -411,6 +413,7 @@ namespace print_money(e.available()) % print_money(e.tx_amount())); fail_msg_writer() << tr("Not enough money in unlocked balance"); + warn_of_possible_attack = false; } catch (const tools::error::tx_not_possible& e) { @@ -420,6 +423,7 @@ namespace print_money(e.tx_amount()) % print_money(e.fee())); fail_msg_writer() << tr("Failed to find a way to create transactions. This is usually due to dust which is so small it cannot pay for itself in fees, or trying to send more money than the unlocked balance, or not leaving enough for fees"); + warn_of_possible_attack = false; } catch (const tools::error::not_enough_outs_to_mix& e) { @@ -434,6 +438,7 @@ namespace catch (const tools::error::tx_not_constructed&) { fail_msg_writer() << tr("transaction was not constructed"); + warn_of_possible_attack = false; } catch (const tools::error::tx_rejected& e) { @@ -445,14 +450,17 @@ namespace catch (const tools::error::tx_sum_overflow& e) { fail_msg_writer() << e.what(); + warn_of_possible_attack = false; } catch (const tools::error::zero_destination&) { fail_msg_writer() << tr("one of destinations is zero"); + warn_of_possible_attack = false; } catch (const tools::error::tx_too_big& e) { fail_msg_writer() << tr("failed to find a suitable way to split transactions"); + warn_of_possible_attack = false; } catch (const tools::error::transfer_error& e) { @@ -463,6 +471,7 @@ namespace { LOG_ERROR("Multisig error: " << e.to_string()); fail_msg_writer() << tr("Multisig error: ") << e.what(); + warn_of_possible_attack = false; } catch (const tools::error::wallet_internal_error& e) { @@ -474,6 +483,9 @@ namespace LOG_ERROR("unexpected error: " << e.what()); fail_msg_writer() << tr("unexpected error: ") << e.what(); } + + if (warn_of_possible_attack) + fail_msg_writer() << tr("There was an error, which could mean the node may be trying to get you to retry creating a transaction, and zero in on which outputs you own. Or it could be a bona fide error. It may be prudent to disconnect from this node, and not try to send a tranasction immediately. Alternatively, connect to another node so the original node cannot correlate information."); } bool check_file_overwrite(const std::string &filename) @@ -1195,7 +1207,7 @@ bool simple_wallet::submit_multisig(const std::vector<std::string> &args) } catch (const std::exception &e) { - handle_transfer_exception(std::current_exception()); + handle_transfer_exception(std::current_exception(), m_trusted_daemon); } catch (...) { @@ -1282,6 +1294,275 @@ bool simple_wallet::export_raw_multisig(const std::vector<std::string> &args) return true; } +bool simple_wallet::print_ring(const std::vector<std::string> &args) +{ + crypto::key_image key_image; + crypto::hash txid; + if (args.size() != 1) + { + fail_msg_writer() << tr("usage: print_ring <key_image|txid>"); + return true; + } + + if (!epee::string_tools::hex_to_pod(args[0], key_image)) + { + fail_msg_writer() << tr("Invalid key image"); + return true; + } + // this one will always work, they're all 32 byte hex + if (!epee::string_tools::hex_to_pod(args[0], txid)) + { + fail_msg_writer() << tr("Invalid txid"); + return true; + } + + std::vector<uint64_t> ring; + std::vector<std::pair<crypto::key_image, std::vector<uint64_t>>> rings; + try + { + if (m_wallet->get_ring(key_image, ring)) + rings.push_back({key_image, ring}); + else if (!m_wallet->get_rings(txid, rings)) + { + fail_msg_writer() << tr("Key image either not spent, or spent with mixin 0"); + return true; + } + + for (const auto &ring: rings) + { + std::stringstream str; + for (const auto &x: ring.second) + str << x<< " "; + // do NOT translate this "absolute" below, the lin can be used as input to set_ring + success_msg_writer() << epee::string_tools::pod_to_hex(ring.first) << " absolute " << str.str(); + } + } + catch (const std::exception &e) + { + fail_msg_writer() << tr("Failed to get key image ring: ") << e.what(); + } + + return true; +} + +bool simple_wallet::set_ring(const std::vector<std::string> &args) +{ + crypto::key_image key_image; + if (args.size() < 3) + { + fail_msg_writer() << tr("usage: set_ring <key_image> absolute|relative <index> [<index>...]"); + return true; + } + + if (!epee::string_tools::hex_to_pod(args[0], key_image)) + { + fail_msg_writer() << tr("Invalid key image"); + return true; + } + + bool relative; + if (args[1] == "absolute") + { + relative = false; + } + else if (args[1] == "relative") + { + relative = true; + } + else + { + fail_msg_writer() << tr("Missing absolute or relative keyword"); + return true; + } + + std::vector<uint64_t> ring; + for (size_t n = 2; n < args.size(); ++n) + { + ring.resize(ring.size() + 1); + if (!string_tools::get_xtype_from_string(ring.back(), args[n])) + { + fail_msg_writer() << tr("invalid index: must be a strictly positive unsigned integer"); + return true; + } + if (relative) + { + if (ring.size() > 1 && !ring.back()) + { + fail_msg_writer() << tr("invalid index: must be a strictly positive unsigned integer"); + return true; + } + uint64_t sum = 0; + for (uint64_t out: ring) + { + if (out > std::numeric_limits<uint64_t>::max() - sum) + { + fail_msg_writer() << tr("invalid index: indices wrap"); + return true; + } + sum += out; + } + } + else + { + if (ring.size() > 1 && ring[ring.size() - 2] >= ring[ring.size() - 1]) + { + fail_msg_writer() << tr("invalid index: indices should be in strictly ascending order"); + return true; + } + } + } + if (!m_wallet->set_ring(key_image, ring, relative)) + { + fail_msg_writer() << tr("failed to set ring"); + return true; + } + + return true; +} + +bool simple_wallet::blackball(const std::vector<std::string> &args) +{ + crypto::public_key output; + if (args.size() == 0) + { + fail_msg_writer() << tr("usage: blackball <output_public_key> | <filename> [add]"); + return true; + } + + try + { + if (epee::string_tools::hex_to_pod(args[0], output)) + { + m_wallet->blackball_output(output); + } + else if (epee::file_io_utils::is_file_exist(args[0])) + { + std::vector<crypto::public_key> outputs; + char str[65]; + + std::unique_ptr<FILE, tools::close_file> f(fopen(args[0].c_str(), "r")); + if (f) + { + while (!feof(f.get())) + { + if (!fgets(str, sizeof(str), f.get())) + break; + const size_t len = strlen(str); + if (len > 0 && str[len - 1] == '\n') + str[len - 1] = 0; + if (!str[0]) + continue; + outputs.push_back(crypto::public_key()); + if (!epee::string_tools::hex_to_pod(str, outputs.back())) + { + fail_msg_writer() << tr("Invalid public key: ") << str; + return true; + } + } + f.reset(); + bool add = false; + if (args.size() > 1) + { + if (args[1] != "add") + { + fail_msg_writer() << tr("Bad argument: ") + args[1] + ": " + tr("should be \"add\""); + return true; + } + add = true; + } + m_wallet->set_blackballed_outputs(outputs, add); + } + else + { + fail_msg_writer() << tr("Failed to open file"); + return true; + } + } + else + { + fail_msg_writer() << tr("Invalid public key, and file doesn't exist"); + return true; + } + } + catch (const std::exception &e) + { + fail_msg_writer() << tr("Failed to blackball output: ") << e.what(); + } + + return true; +} + +bool simple_wallet::unblackball(const std::vector<std::string> &args) +{ + crypto::public_key output; + if (args.size() != 1) + { + fail_msg_writer() << tr("usage: unblackball <output_public_key>"); + return true; + } + + if (!epee::string_tools::hex_to_pod(args[0], output)) + { + fail_msg_writer() << tr("Invalid public key"); + return true; + } + + try + { + m_wallet->unblackball_output(output); + } + catch (const std::exception &e) + { + fail_msg_writer() << tr("Failed to unblackball output: ") << e.what(); + } + + return true; +} + +bool simple_wallet::blackballed(const std::vector<std::string> &args) +{ + crypto::public_key output; + if (args.size() != 1) + { + fail_msg_writer() << tr("usage: blackballed <output_public_key>"); + return true; + } + + if (!epee::string_tools::hex_to_pod(args[0], output)) + { + fail_msg_writer() << tr("Invalid public key"); + return true; + } + + try + { + if (m_wallet->is_output_blackballed(output)) + message_writer() << tr("Blackballed: ") << output; + else + message_writer() << tr("not blackballed: ") << output; + } + catch (const std::exception &e) + { + fail_msg_writer() << tr("Failed to unblackball output: ") << e.what(); + } + + return true; +} + +bool simple_wallet::save_known_rings(const std::vector<std::string> &args) +{ + try + { + LOCK_IDLE_SCOPE(); + m_wallet->find_and_save_rings(); + } + catch (const std::exception &e) + { + fail_msg_writer() << tr("Failed to save known rings: ") << e.what(); + } + return true; +} + bool simple_wallet::set_always_confirm_transfers(const std::vector<std::string> &args/* = std::vector<std::string>()*/) { const auto pwd_container = get_and_verify_password(); @@ -1626,6 +1907,32 @@ bool simple_wallet::set_auto_low_priority(const std::vector<std::string> &args/* return true; } +bool simple_wallet::set_segregate_pre_fork_outputs(const std::vector<std::string> &args/* = std::vector<std::string>()*/) +{ + const auto pwd_container = get_and_verify_password(); + if (pwd_container) + { + parse_bool_and_use(args[1], [&](bool r) { + m_wallet->segregate_pre_fork_outputs(r); + m_wallet->rewrite(m_wallet_file, pwd_container->password()); + }); + } + return true; +} + +bool simple_wallet::set_key_reuse_mitigation2(const std::vector<std::string> &args/* = std::vector<std::string>()*/) +{ + const auto pwd_container = get_and_verify_password(); + if (pwd_container) + { + parse_bool_and_use(args[1], [&](bool r) { + m_wallet->key_reuse_mitigation2(r); + m_wallet->rewrite(m_wallet_file, pwd_container->password()); + }); + } + return true; +} + bool simple_wallet::help(const std::vector<std::string> &args/* = std::vector<std::string>()*/) { if(args.empty()) @@ -1739,7 +2046,7 @@ simple_wallet::simple_wallet() m_cmd_binder.set_handler("address", boost::bind(&simple_wallet::print_address, this, _1), tr("address [ new <label text with white spaces allowed> | all | <index_min> [<index_max>] | label <index> <label text with white spaces allowed>]"), - tr("If no arguments are specified or <index> is specified, the wallet shows the default or specified address. If \"all\" is specified, the walllet shows all the existing addresses in the currently selected account. If \"new \" is specified, the wallet creates a new address with the provided label text (which can be empty). If \"label\" is specified, the wallet sets the label of the address specified by <index> to the provided label text.")); + tr("If no arguments are specified or <index> is specified, the wallet shows the default or specified address. If \"all\" is specified, the wallet shows all the existing addresses in the currently selected account. If \"new \" is specified, the wallet creates a new address with the provided label text (which can be empty). If \"label\" is specified, the wallet sets the label of the address specified by <index> to the provided label text.")); m_cmd_binder.set_handler("integrated_address", boost::bind(&simple_wallet::print_integrated_address, this, _1), tr("integrated_address [<payment_id> | <address>]"), @@ -1800,7 +2107,11 @@ simple_wallet::simple_wallet() "refresh-from-block-height [n]\n " " Set the height before which to ignore blocks.\n " "auto-low-priority <1|0>\n " - " Whether to automatically use the low priority fee level when it's safe to do so.")); + " Whether to automatically use the low priority fee level when it's safe to do so.\n " + "segregate-pre-fork-outputs <1|0>\n " + " Set this if you intend to spend outputs on both Monero AND a key reusing fork.\n " + "key-reuse-mitigation2 <1|0>\n " + " Set this if you are not sure whether you will spend on a key reusing Monero fork later.")); m_cmd_binder.set_handler("encrypted_seed", boost::bind(&simple_wallet::encrypted_seed, this, _1), tr("Display the encrypted Electrum-style mnemonic seed.")); @@ -1939,6 +2250,30 @@ simple_wallet::simple_wallet() boost::bind(&simple_wallet::export_raw_multisig, this, _1), tr("export_raw_multisig_tx <filename>"), tr("Export a signed multisig transaction to a file")); + m_cmd_binder.set_handler("print_ring", + boost::bind(&simple_wallet::print_ring, this, _1), + tr("print_ring <key_image> | <txid>"), + tr("Print the ring(s) used to spend a given key image or transaction (if the ring size is > 1)")); + m_cmd_binder.set_handler("set_ring", + boost::bind(&simple_wallet::set_ring, this, _1), + tr("set_ring <key_image> absolute|relative <index> [<index>...]"), + tr("Set the ring used for a given key image, so it can be reused in a fork")); + m_cmd_binder.set_handler("save_known_rings", + boost::bind(&simple_wallet::save_known_rings, this, _1), + tr("save_known_rings"), + tr("Save known rings to the shared rings database")); + m_cmd_binder.set_handler("blackball", + boost::bind(&simple_wallet::blackball, this, _1), + tr("blackball <output public key> | <filename> [add]"), + tr("Blackball output(s) so they never get selected as fake outputs in a ring")); + m_cmd_binder.set_handler("unblackball", + boost::bind(&simple_wallet::unblackball, this, _1), + tr("unblackball <output public key>"), + tr("Unblackballs an output so it may get selected as a fake output in a ring")); + m_cmd_binder.set_handler("blackballed", + boost::bind(&simple_wallet::blackballed, this, _1), + tr("blackballed <output public key>"), + tr("Checks whether an output is blackballed")); m_cmd_binder.set_handler("help", boost::bind(&simple_wallet::help, this, _1), tr("help [<command>]"), @@ -1968,6 +2303,8 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args) success_msg_writer() << "confirm-export-overwrite = " << m_wallet->confirm_export_overwrite(); success_msg_writer() << "refresh-from-block-height = " << m_wallet->get_refresh_from_block_height(); success_msg_writer() << "auto-low-priority = " << m_wallet->auto_low_priority(); + success_msg_writer() << "segregate-pre-fork-outputs = " << m_wallet->segregate_pre_fork_outputs(); + success_msg_writer() << "key-reuse-mitigation2 = " << m_wallet->key_reuse_mitigation2(); return true; } else @@ -2018,6 +2355,8 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args) CHECK_SIMPLE_VARIABLE("confirm-export-overwrite", set_confirm_export_overwrite, tr("0 or 1")); CHECK_SIMPLE_VARIABLE("refresh-from-block-height", set_refresh_from_block_height, tr("block height")); CHECK_SIMPLE_VARIABLE("auto-low-priority", set_auto_low_priority, tr("0 or 1")); + CHECK_SIMPLE_VARIABLE("segregate-pre-fork-outputs", set_segregate_pre_fork_outputs, tr("0 or 1")); + CHECK_SIMPLE_VARIABLE("key-reuse-mitigation2", set_key_reuse_mitigation2, tr("0 or 1")); } fail_msg_writer() << tr("set: unrecognized argument(s)"); return true; @@ -2683,7 +3022,6 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) if (!m_trusted_daemon) message_writer() << (boost::format(tr("Warning: using an untrusted daemon at %s, privacy will be lessened")) % m_wallet->get_daemon_address()).str(); - m_http_client.set_server(m_wallet->get_daemon_address(), m_wallet->get_daemon_login()); m_wallet->callback(this); return true; @@ -3216,7 +3554,7 @@ bool simple_wallet::start_mining(const std::vector<std::string>& args) } COMMAND_RPC_START_MINING::response res; - bool r = net_utils::invoke_http_json("/start_mining", req, res, m_http_client); + bool r = m_wallet->invoke_http_json("/start_mining", req, res); std::string err = interpret_rpc_response(r, res.status); if (err.empty()) success_msg_writer() << tr("Mining started in daemon"); @@ -3238,7 +3576,7 @@ bool simple_wallet::stop_mining(const std::vector<std::string>& args) COMMAND_RPC_STOP_MINING::request req; COMMAND_RPC_STOP_MINING::response res; - bool r = net_utils::invoke_http_json("/stop_mining", req, res, m_http_client); + bool r = m_wallet->invoke_http_json("/stop_mining", req, res); std::string err = interpret_rpc_response(r, res.status); if (err.empty()) success_msg_writer() << tr("Mining stopped in daemon"); @@ -3295,7 +3633,7 @@ bool simple_wallet::save_bc(const std::vector<std::string>& args) } COMMAND_RPC_SAVE_BC::request req; COMMAND_RPC_SAVE_BC::response res; - bool r = net_utils::invoke_http_json("/save_bc", req, res, m_http_client); + bool r = m_wallet->invoke_http_json("/save_bc", req, res); std::string err = interpret_rpc_response(r, res.status); if (err.empty()) success_msg_writer() << tr("Blockchain saved"); @@ -3641,7 +3979,7 @@ uint64_t simple_wallet::get_daemon_blockchain_height(std::string& err) COMMAND_RPC_GET_HEIGHT::request req; COMMAND_RPC_GET_HEIGHT::response res = boost::value_initialized<COMMAND_RPC_GET_HEIGHT::response>(); - bool r = net_utils::invoke_http_json("/getheight", req, res, m_http_client); + bool r = m_wallet->invoke_http_json("/getheight", req, res); err = interpret_rpc_response(r, res.status); return res.height; } @@ -3763,7 +4101,7 @@ bool simple_wallet::print_ring_members(const std::vector<tools::wallet2::pending req.outputs[j].index = absolute_offsets[j]; } COMMAND_RPC_GET_OUTPUTS_BIN::response res = AUTO_VAL_INIT(res); - bool r = net_utils::invoke_http_bin("/get_outs.bin", req, res, m_http_client); + bool r = m_wallet->invoke_http_bin("/get_outs.bin", req, res); err = interpret_rpc_response(r, res.status); if (!err.empty()) { @@ -4176,7 +4514,7 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri } catch (const std::exception &e) { - handle_transfer_exception(std::current_exception()); + handle_transfer_exception(std::current_exception(), m_trusted_daemon); } catch (...) { @@ -4284,7 +4622,7 @@ bool simple_wallet::sweep_unmixable(const std::vector<std::string> &args_) } catch (const std::exception &e) { - handle_transfer_exception(std::current_exception()); + handle_transfer_exception(std::current_exception(), m_trusted_daemon); } catch (...) { @@ -4517,7 +4855,7 @@ bool simple_wallet::sweep_main(uint64_t below, const std::vector<std::string> &a } catch (const std::exception& e) { - handle_transfer_exception(std::current_exception()); + handle_transfer_exception(std::current_exception(), m_trusted_daemon); } catch (...) { @@ -4716,7 +5054,7 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) } catch (const std::exception& e) { - handle_transfer_exception(std::current_exception()); + handle_transfer_exception(std::current_exception(), m_trusted_daemon); } catch (...) { @@ -5021,7 +5359,7 @@ bool simple_wallet::submit_transfer(const std::vector<std::string> &args_) } catch (const std::exception& e) { - handle_transfer_exception(std::current_exception()); + handle_transfer_exception(std::current_exception(), m_trusted_daemon); } catch (...) { @@ -6464,9 +6802,13 @@ bool simple_wallet::wallet_info(const std::vector<std::string> &args) { bool ready; uint32_t threshold, total; - + std::string description = m_wallet->get_description(); + if (description.empty()) + { + description = "<Not set>"; + } message_writer() << tr("Filename: ") << m_wallet->get_wallet_file(); - message_writer() << tr("Description: ") << m_wallet->get_description(); + message_writer() << tr("Description: ") << description; message_writer() << tr("Address: ") << m_wallet->get_account().get_public_address_str(m_wallet->nettype()); std::string type; if (m_wallet->watch_only()) diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h index 4c7818bf1..c69df817d 100644 --- a/src/simplewallet/simplewallet.h +++ b/src/simplewallet/simplewallet.h @@ -133,6 +133,8 @@ namespace cryptonote bool set_confirm_export_overwrite(const std::vector<std::string> &args = std::vector<std::string>()); bool set_refresh_from_block_height(const std::vector<std::string> &args = std::vector<std::string>()); bool set_auto_low_priority(const std::vector<std::string> &args = std::vector<std::string>()); + bool set_segregate_pre_fork_outputs(const std::vector<std::string> &args = std::vector<std::string>()); + bool set_key_reuse_mitigation2(const std::vector<std::string> &args = std::vector<std::string>()); bool help(const std::vector<std::string> &args = std::vector<std::string>()); bool start_mining(const std::vector<std::string> &args); bool stop_mining(const std::vector<std::string> &args); @@ -208,6 +210,12 @@ namespace cryptonote bool sign_multisig(const std::vector<std::string>& args); bool submit_multisig(const std::vector<std::string>& args); bool export_raw_multisig(const std::vector<std::string>& args); + bool print_ring(const std::vector<std::string>& args); + bool set_ring(const std::vector<std::string>& args); + bool save_known_rings(const std::vector<std::string>& args); + bool blackball(const std::vector<std::string>& args); + bool unblackball(const std::vector<std::string>& args); + bool blackballed(const std::vector<std::string>& args); uint64_t get_daemon_blockchain_height(std::string& err); bool try_connect_to_daemon(bool silent = false, uint32_t* version = nullptr); @@ -328,7 +336,6 @@ namespace cryptonote epee::console_handlers_binder m_cmd_binder; std::unique_ptr<tools::wallet2> m_wallet; - epee::net_utils::http::http_simple_client m_http_client; refresh_progress_reporter_t m_refresh_progress_reporter; std::atomic<bool> m_idle_run; diff --git a/src/wallet/CMakeLists.txt b/src/wallet/CMakeLists.txt index d82e1dace..a5a4c7f56 100644 --- a/src/wallet/CMakeLists.txt +++ b/src/wallet/CMakeLists.txt @@ -33,6 +33,7 @@ set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) set(wallet_sources wallet2.cpp wallet_args.cpp + ringdb.cpp node_rpc_proxy.cpp) set(wallet_private_headers @@ -42,6 +43,7 @@ set(wallet_private_headers wallet_rpc_server.h wallet_rpc_server_commands_defs.h wallet_rpc_server_error_codes.h + ringdb.h node_rpc_proxy.h) monero_private_headers(wallet @@ -55,6 +57,7 @@ target_link_libraries(wallet common cryptonote_core mnemonics + ${LMDB_LIBRARY} ${Boost_CHRONO_LIBRARY} ${Boost_SERIALIZATION_LIBRARY} ${Boost_FILESYSTEM_LIBRARY} diff --git a/src/wallet/api/CMakeLists.txt b/src/wallet/api/CMakeLists.txt index 1e67495f1..d6f2bf6b7 100644 --- a/src/wallet/api/CMakeLists.txt +++ b/src/wallet/api/CMakeLists.txt @@ -69,6 +69,7 @@ target_link_libraries(wallet_api common cryptonote_core mnemonics + ${LMDB_LIBRARY} ${Boost_CHRONO_LIBRARY} ${Boost_SERIALIZATION_LIBRARY} ${Boost_FILESYSTEM_LIBRARY} diff --git a/src/wallet/api/wallet.cpp b/src/wallet/api/wallet.cpp index fb9e8b28b..b02884f67 100644 --- a/src/wallet/api/wallet.cpp +++ b/src/wallet/api/wallet.cpp @@ -68,6 +68,15 @@ namespace { static const int DEFAULT_REMOTE_NODE_REFRESH_INTERVAL_MILLIS = 1000 * 10; // Connection timeout 30 sec static const int DEFAULT_CONNECTION_TIMEOUT_MILLIS = 1000 * 30; + + std::string get_default_ringdb_path() + { + boost::filesystem::path dir = tools::get_default_data_dir(); + // remove .bitmonero, replace with .shared-ringdb + dir = dir.remove_filename(); + dir /= ".shared-ringdb"; + return dir.string(); + } } struct Wallet2CallbackImpl : public tools::i_wallet2_callback @@ -590,6 +599,7 @@ bool WalletImpl::open(const std::string &path, const std::string &password) // Rebuilding wallet cache, using refresh height from .keys file m_rebuildWalletCache = true; } + m_wallet->set_ring_database(get_default_ringdb_path()); m_wallet->load(path, password); m_password = password; @@ -1777,6 +1787,7 @@ void WalletImpl::doRefresh() if (m_history->count() == 0) { m_history->refresh(); } + m_wallet->find_and_save_rings(false); } else { LOG_PRINT_L3(__FUNCTION__ << ": skipping refresh - daemon is not synced"); } @@ -1897,6 +1908,127 @@ bool WalletImpl::useForkRules(uint8_t version, int64_t early_blocks) const return m_wallet->use_fork_rules(version,early_blocks); } +bool WalletImpl::blackballOutputs(const std::vector<std::string> &pubkeys, bool add) +{ + std::vector<crypto::public_key> raw_pubkeys; + raw_pubkeys.reserve(pubkeys.size()); + for (const std::string &str: pubkeys) + { + crypto::public_key pkey; + if (!epee::string_tools::hex_to_pod(str, pkey)) + { + m_status = Status_Error; + m_errorString = tr("Failed to parse output public key"); + return false; + } + raw_pubkeys.push_back(pkey); + } + bool ret = m_wallet->set_blackballed_outputs(raw_pubkeys, add); + if (!ret) + { + m_status = Status_Error; + m_errorString = tr("Failed to set blackballed outputs"); + return false; + } + return true; +} + +bool WalletImpl::unblackballOutput(const std::string &pubkey) +{ + crypto::public_key raw_pubkey; + if (!epee::string_tools::hex_to_pod(pubkey, raw_pubkey)) + { + m_status = Status_Error; + m_errorString = tr("Failed to parse output public key"); + return false; + } + bool ret = m_wallet->unblackball_output(raw_pubkey); + if (!ret) + { + m_status = Status_Error; + m_errorString = tr("Failed to unblackball output"); + return false; + } + return true; +} + +bool WalletImpl::getRing(const std::string &key_image, std::vector<uint64_t> &ring) const +{ + crypto::key_image raw_key_image; + if (!epee::string_tools::hex_to_pod(key_image, raw_key_image)) + { + m_status = Status_Error; + m_errorString = tr("Failed to parse key image"); + return false; + } + bool ret = m_wallet->get_ring(raw_key_image, ring); + if (!ret) + { + m_status = Status_Error; + m_errorString = tr("Failed to get ring"); + return false; + } + return true; +} + +bool WalletImpl::getRings(const std::string &txid, std::vector<std::pair<std::string, std::vector<uint64_t>>> &rings) const +{ + crypto::hash raw_txid; + if (!epee::string_tools::hex_to_pod(txid, raw_txid)) + { + m_status = Status_Error; + m_errorString = tr("Failed to parse txid"); + return false; + } + std::vector<std::pair<crypto::key_image, std::vector<uint64_t>>> raw_rings; + bool ret = m_wallet->get_rings(raw_txid, raw_rings); + if (!ret) + { + m_status = Status_Error; + m_errorString = tr("Failed to get rings"); + return false; + } + for (const auto &r: raw_rings) + { + rings.push_back(std::make_pair(epee::string_tools::pod_to_hex(r.first), r.second)); + } + return true; +} + +bool WalletImpl::setRing(const std::string &key_image, const std::vector<uint64_t> &ring, bool relative) +{ + crypto::key_image raw_key_image; + if (!epee::string_tools::hex_to_pod(key_image, raw_key_image)) + { + m_status = Status_Error; + m_errorString = tr("Failed to parse key image"); + return false; + } + bool ret = m_wallet->set_ring(raw_key_image, ring, relative); + if (!ret) + { + m_status = Status_Error; + m_errorString = tr("Failed to set ring"); + return false; + } + return true; +} + +void WalletImpl::segregatePreForkOutputs(bool segregate) +{ + m_wallet->segregate_pre_fork_outputs(segregate); +} + +void WalletImpl::segregationHeight(uint64_t height) +{ + m_wallet->segregation_height(height); +} + +void WalletImpl::keyReuseMitigation2(bool mitigation) +{ + m_wallet->key_reuse_mitigation2(mitigation); +} + } // namespace namespace Bitmonero = Monero; diff --git a/src/wallet/api/wallet.h b/src/wallet/api/wallet.h index 9b4a0cc12..4929c9673 100644 --- a/src/wallet/api/wallet.h +++ b/src/wallet/api/wallet.h @@ -163,6 +163,14 @@ public: virtual std::string getDefaultDataDir() const; virtual bool lightWalletLogin(bool &isNewWallet) const; virtual bool lightWalletImportWalletRequest(std::string &payment_id, uint64_t &fee, bool &new_request, bool &request_fulfilled, std::string &payment_address, std::string &status); + virtual bool blackballOutputs(const std::vector<std::string> &pubkeys, bool add); + virtual bool unblackballOutput(const std::string &pubkey); + virtual bool getRing(const std::string &key_image, std::vector<uint64_t> &ring) const; + virtual bool getRings(const std::string &txid, std::vector<std::pair<std::string, std::vector<uint64_t>>> &rings) const; + virtual bool setRing(const std::string &key_image, const std::vector<uint64_t> &ring, bool relative); + virtual void segregatePreForkOutputs(bool segregate); + virtual void segregationHeight(uint64_t height); + virtual void keyReuseMitigation2(bool mitigation); private: void clearStatus() const; diff --git a/src/wallet/api/wallet2_api.h b/src/wallet/api/wallet2_api.h index 87c1cccfa..d4e41c5aa 100644 --- a/src/wallet/api/wallet2_api.h +++ b/src/wallet/api/wallet2_api.h @@ -33,6 +33,7 @@ #include <string> #include <vector> +#include <list> #include <set> #include <ctime> #include <iostream> @@ -756,6 +757,30 @@ struct Wallet */ virtual bool rescanSpent() = 0; + //! blackballs a set of outputs + virtual bool blackballOutputs(const std::vector<std::string> &pubkeys, bool add) = 0; + + //! unblackballs an output + virtual bool unblackballOutput(const std::string &pubkey) = 0; + + //! gets the ring used for a key image, if any + virtual bool getRing(const std::string &key_image, std::vector<uint64_t> &ring) const = 0; + + //! gets the rings used for a txid, if any + virtual bool getRings(const std::string &txid, std::vector<std::pair<std::string, std::vector<uint64_t>>> &rings) const = 0; + + //! sets the ring used for a key image + virtual bool setRing(const std::string &key_image, const std::vector<uint64_t> &ring, bool relative) = 0; + + //! sets whether pre-fork outs are to be segregated + virtual void segregatePreForkOutputs(bool segregate) = 0; + + //! sets the height where segregation should occur + virtual void segregationHeight(uint64_t height) = 0; + + //! secondary key reuse mitigation + virtual void keyReuseMitigation2(bool mitigation) = 0; + //! Light wallet authenticate and login virtual bool lightWalletLogin(bool &isNewWallet) const = 0; diff --git a/src/wallet/node_rpc_proxy.cpp b/src/wallet/node_rpc_proxy.cpp index f11ff9295..c5d869354 100644 --- a/src/wallet/node_rpc_proxy.cpp +++ b/src/wallet/node_rpc_proxy.cpp @@ -70,18 +70,15 @@ boost::optional<std::string> NodeRPCProxy::get_rpc_version(uint32_t &rpc_version { if (m_rpc_version == 0) { - epee::json_rpc::request<cryptonote::COMMAND_RPC_GET_VERSION::request> req_t = AUTO_VAL_INIT(req_t); - epee::json_rpc::response<cryptonote::COMMAND_RPC_GET_VERSION::response, std::string> resp_t = AUTO_VAL_INIT(resp_t); - req_t.jsonrpc = "2.0"; - req_t.id = epee::serialization::storage_entry(0); - req_t.method = "get_version"; + cryptonote::COMMAND_RPC_GET_VERSION::request req_t = AUTO_VAL_INIT(req_t); + cryptonote::COMMAND_RPC_GET_VERSION::response resp_t = AUTO_VAL_INIT(resp_t); m_daemon_rpc_mutex.lock(); - bool r = net_utils::invoke_http_json("/json_rpc", req_t, resp_t, m_http_client, rpc_timeout); + bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_version", req_t, resp_t, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); CHECK_AND_ASSERT_MES(r, std::string(), "Failed to connect to daemon"); - CHECK_AND_ASSERT_MES(resp_t.result.status != CORE_RPC_STATUS_BUSY, resp_t.result.status, "Failed to connect to daemon"); - CHECK_AND_ASSERT_MES(resp_t.result.status == CORE_RPC_STATUS_OK, resp_t.result.status, "Failed to get daemon RPC version"); - m_rpc_version = resp_t.result.version; + CHECK_AND_ASSERT_MES(resp_t.status != CORE_RPC_STATUS_BUSY, resp_t.status, "Failed to connect to daemon"); + CHECK_AND_ASSERT_MES(resp_t.status == CORE_RPC_STATUS_OK, resp_t.status, "Failed to get daemon RPC version"); + m_rpc_version = resp_t.version; } rpc_version = m_rpc_version; return boost::optional<std::string>(); @@ -118,20 +115,17 @@ boost::optional<std::string> NodeRPCProxy::get_target_height(uint64_t &height) c const time_t now = time(NULL); if (m_target_height == 0 || now >= m_target_height_time + 30) // re-cache every 30 seconds { - epee::json_rpc::request<cryptonote::COMMAND_RPC_GET_INFO::request> req_t = AUTO_VAL_INIT(req_t); - epee::json_rpc::response<cryptonote::COMMAND_RPC_GET_INFO::response, std::string> resp_t = AUTO_VAL_INIT(resp_t); + cryptonote::COMMAND_RPC_GET_INFO::request req_t = AUTO_VAL_INIT(req_t); + cryptonote::COMMAND_RPC_GET_INFO::response resp_t = AUTO_VAL_INIT(resp_t); - req_t.jsonrpc = "2.0"; - req_t.id = epee::serialization::storage_entry(0); - req_t.method = "get_info"; m_daemon_rpc_mutex.lock(); - bool r = net_utils::invoke_http_json("/json_rpc", req_t, resp_t, m_http_client, rpc_timeout); + bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_info", req_t, resp_t, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); CHECK_AND_ASSERT_MES(r, std::string(), "Failed to connect to daemon"); - CHECK_AND_ASSERT_MES(resp_t.result.status != CORE_RPC_STATUS_BUSY, resp_t.result.status, "Failed to connect to daemon"); - CHECK_AND_ASSERT_MES(resp_t.result.status == CORE_RPC_STATUS_OK, resp_t.result.status, "Failed to get target blockchain height"); - m_target_height = resp_t.result.target_height; + CHECK_AND_ASSERT_MES(resp_t.status != CORE_RPC_STATUS_BUSY, resp_t.status, "Failed to connect to daemon"); + CHECK_AND_ASSERT_MES(resp_t.status == CORE_RPC_STATUS_OK, resp_t.status, "Failed to get target blockchain height"); + m_target_height = resp_t.target_height; m_target_height_time = now; } height = m_target_height; @@ -142,20 +136,17 @@ boost::optional<std::string> NodeRPCProxy::get_earliest_height(uint8_t version, { if (m_earliest_height[version] == 0) { - epee::json_rpc::request<cryptonote::COMMAND_RPC_HARD_FORK_INFO::request> req_t = AUTO_VAL_INIT(req_t); - epee::json_rpc::response<cryptonote::COMMAND_RPC_HARD_FORK_INFO::response, std::string> resp_t = AUTO_VAL_INIT(resp_t); + cryptonote::COMMAND_RPC_HARD_FORK_INFO::request req_t = AUTO_VAL_INIT(req_t); + cryptonote::COMMAND_RPC_HARD_FORK_INFO::response resp_t = AUTO_VAL_INIT(resp_t); m_daemon_rpc_mutex.lock(); - req_t.jsonrpc = "2.0"; - req_t.id = epee::serialization::storage_entry(0); - req_t.method = "hard_fork_info"; - req_t.params.version = version; - bool r = net_utils::invoke_http_json("/json_rpc", req_t, resp_t, m_http_client, rpc_timeout); + req_t.version = version; + bool r = net_utils::invoke_http_json_rpc("/json_rpc", "hard_fork_info", req_t, resp_t, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); CHECK_AND_ASSERT_MES(r, std::string(), "Failed to connect to daemon"); - CHECK_AND_ASSERT_MES(resp_t.result.status != CORE_RPC_STATUS_BUSY, resp_t.result.status, "Failed to connect to daemon"); - CHECK_AND_ASSERT_MES(resp_t.result.status == CORE_RPC_STATUS_OK, resp_t.result.status, "Failed to get hard fork status"); - m_earliest_height[version] = resp_t.result.enabled ? resp_t.result.earliest_height : std::numeric_limits<uint64_t>::max(); + CHECK_AND_ASSERT_MES(resp_t.status != CORE_RPC_STATUS_BUSY, resp_t.status, "Failed to connect to daemon"); + CHECK_AND_ASSERT_MES(resp_t.status == CORE_RPC_STATUS_OK, resp_t.status, "Failed to get hard fork status"); + m_earliest_height[version] = resp_t.enabled ? resp_t.earliest_height : std::numeric_limits<uint64_t>::max(); } earliest_height = m_earliest_height[version]; @@ -172,20 +163,17 @@ boost::optional<std::string> NodeRPCProxy::get_dynamic_per_kb_fee_estimate(uint6 if (m_dynamic_per_kb_fee_estimate_cached_height != height || m_dynamic_per_kb_fee_estimate_grace_blocks != grace_blocks) { - epee::json_rpc::request<cryptonote::COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::request> req_t = AUTO_VAL_INIT(req_t); - epee::json_rpc::response<cryptonote::COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::response, std::string> resp_t = AUTO_VAL_INIT(resp_t); + cryptonote::COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::request req_t = AUTO_VAL_INIT(req_t); + cryptonote::COMMAND_RPC_GET_PER_KB_FEE_ESTIMATE::response resp_t = AUTO_VAL_INIT(resp_t); m_daemon_rpc_mutex.lock(); - req_t.jsonrpc = "2.0"; - req_t.id = epee::serialization::storage_entry(0); - req_t.method = "get_fee_estimate"; - req_t.params.grace_blocks = grace_blocks; - bool r = net_utils::invoke_http_json("/json_rpc", req_t, resp_t, m_http_client, rpc_timeout); + req_t.grace_blocks = grace_blocks; + bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_fee_estimate", req_t, resp_t, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); CHECK_AND_ASSERT_MES(r, std::string(), "Failed to connect to daemon"); - CHECK_AND_ASSERT_MES(resp_t.result.status != CORE_RPC_STATUS_BUSY, resp_t.result.status, "Failed to connect to daemon"); - CHECK_AND_ASSERT_MES(resp_t.result.status == CORE_RPC_STATUS_OK, resp_t.result.status, "Failed to get fee estimate"); - m_dynamic_per_kb_fee_estimate = resp_t.result.fee; + CHECK_AND_ASSERT_MES(resp_t.status != CORE_RPC_STATUS_BUSY, resp_t.status, "Failed to connect to daemon"); + CHECK_AND_ASSERT_MES(resp_t.status == CORE_RPC_STATUS_OK, resp_t.status, "Failed to get fee estimate"); + m_dynamic_per_kb_fee_estimate = resp_t.fee; m_dynamic_per_kb_fee_estimate_cached_height = height; m_dynamic_per_kb_fee_estimate_grace_blocks = grace_blocks; } diff --git a/src/wallet/ringdb.cpp b/src/wallet/ringdb.cpp new file mode 100644 index 000000000..44992520f --- /dev/null +++ b/src/wallet/ringdb.cpp @@ -0,0 +1,445 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <lmdb.h> +#include <boost/algorithm/string.hpp> +#include <boost/range/adaptor/transformed.hpp> +#include <boost/filesystem.hpp> +#include "misc_log_ex.h" +#include "misc_language.h" +#include "wallet_errors.h" +#include "ringdb.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "wallet.ringdb" + +static const char zerokey[8] = {0}; +static const MDB_val zerokeyval = { sizeof(zerokey), (void *)zerokey }; + +static int compare_hash32(const MDB_val *a, const MDB_val *b) +{ + uint32_t *va = (uint32_t*) a->mv_data; + uint32_t *vb = (uint32_t*) b->mv_data; + for (int n = 7; n >= 0; n--) + { + if (va[n] == vb[n]) + continue; + return va[n] < vb[n] ? -1 : 1; + } + + return 0; +} + +static std::string compress_ring(const std::vector<uint64_t> &ring) +{ + std::string s; + for (uint64_t out: ring) + s += tools::get_varint_data(out); + return s; +} + +static std::vector<uint64_t> decompress_ring(const std::string &s) +{ + std::vector<uint64_t> ring; + int read = 0; + for (std::string::const_iterator i = s.begin(); i != s.cend(); std::advance(i, read)) + { + uint64_t out; + std::string tmp(i, s.cend()); + read = tools::read_varint(tmp.begin(), tmp.end(), out); + THROW_WALLET_EXCEPTION_IF(read <= 0 || read > 256, tools::error::wallet_internal_error, "Internal error decompressing ring"); + ring.push_back(out); + } + return ring; +} + +std::string get_rings_filename(boost::filesystem::path filename) +{ + if (!boost::filesystem::is_directory(filename)) + filename.remove_filename(); + return filename.string(); +} + +static crypto::chacha_iv make_iv(const crypto::key_image &key_image, const crypto::chacha_key &key) +{ + static const char salt[] = "ringdsb"; + + uint8_t buffer[sizeof(key_image) + sizeof(key) + sizeof(salt)]; + memcpy(buffer, &key_image, sizeof(key_image)); + memcpy(buffer + sizeof(key_image), &key, sizeof(key)); + memcpy(buffer + sizeof(key_image) + sizeof(key), salt, sizeof(salt)); + crypto::hash hash; + crypto::cn_fast_hash(buffer, sizeof(buffer), hash.data); + static_assert(sizeof(hash) >= CHACHA_IV_SIZE, "Incompatible hash and chacha IV sizes"); + crypto::chacha_iv iv; + memcpy(&iv, &hash, CHACHA_IV_SIZE); + return iv; +} + +static std::string encrypt(const std::string &plaintext, const crypto::key_image &key_image, const crypto::chacha_key &key) +{ + const crypto::chacha_iv iv = make_iv(key_image, key); + std::string ciphertext; + ciphertext.resize(plaintext.size() + sizeof(iv)); + crypto::chacha20(plaintext.data(), plaintext.size(), key, iv, &ciphertext[sizeof(iv)]); + memcpy(&ciphertext[0], &iv, sizeof(iv)); + return ciphertext; +} + +static std::string encrypt(const crypto::key_image &key_image, const crypto::chacha_key &key) +{ + return encrypt(std::string((const char*)&key_image, sizeof(key_image)), key_image, key); +} + +static std::string decrypt(const std::string &ciphertext, const crypto::key_image &key_image, const crypto::chacha_key &key) +{ + const crypto::chacha_iv iv = make_iv(key_image, key); + std::string plaintext; + THROW_WALLET_EXCEPTION_IF(ciphertext.size() < sizeof(iv), tools::error::wallet_internal_error, "Bad ciphertext text"); + plaintext.resize(ciphertext.size() - sizeof(iv)); + crypto::chacha20(ciphertext.data() + sizeof(iv), ciphertext.size() - sizeof(iv), key, iv, &plaintext[0]); + return plaintext; +} + +static void store_relative_ring(MDB_txn *txn, MDB_dbi &dbi, const crypto::key_image &key_image, const std::vector<uint64_t> &relative_ring, const crypto::chacha_key &chacha_key) +{ + MDB_val key, data; + std::string key_ciphertext = encrypt(key_image, chacha_key); + key.mv_data = (void*)key_ciphertext.data(); + key.mv_size = key_ciphertext.size(); + std::string compressed_ring = compress_ring(relative_ring); + std::string data_ciphertext = encrypt(compressed_ring, key_image, chacha_key); + data.mv_size = data_ciphertext.size(); + data.mv_data = (void*)data_ciphertext.c_str(); + int dbr = mdb_put(txn, dbi, &key, &data, 0); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set ring for key image in LMDB table: " + std::string(mdb_strerror(dbr))); +} + +static int resize_env(MDB_env *env, const char *db_path, size_t needed) +{ + MDB_envinfo mei; + MDB_stat mst; + int ret; + + needed = std::max(needed, (size_t)(2ul * 1024 * 1024)); // at least 2 MB + + ret = mdb_env_info(env, &mei); + if (ret) + return ret; + ret = mdb_env_stat(env, &mst); + if (ret) + return ret; + uint64_t size_used = mst.ms_psize * mei.me_last_pgno; + uint64_t mapsize = mei.me_mapsize; + if (size_used + needed > mei.me_mapsize) + { + try + { + boost::filesystem::path path(db_path); + boost::filesystem::space_info si = boost::filesystem::space(path); + if(si.available < needed) + { + MERROR("!! WARNING: Insufficient free space to extend database !!: " << (si.available >> 20L) << " MB available"); + return ENOSPC; + } + } + catch(...) + { + // print something but proceed. + MWARNING("Unable to query free disk space."); + } + + mapsize += needed; + } + return mdb_env_set_mapsize(env, mapsize); +} + +static size_t get_ring_data_size(size_t n_entries) +{ + return n_entries * (32 + 1024); // highball 1kB for the ring data to make sure +} + +enum { BLACKBALL_BLACKBALL, BLACKBALL_UNBLACKBALL, BLACKBALL_QUERY, BLACKBALL_CLEAR}; + +namespace tools +{ + +ringdb::ringdb(std::string filename, const std::string &genesis): + filename(filename) +{ + MDB_txn *txn; + bool tx_active = false; + int dbr; + + tools::create_directories_if_necessary(filename); + + dbr = mdb_env_create(&env); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); + dbr = mdb_env_set_maxdbs(env, 2); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); + const std::string actual_filename = get_rings_filename(filename); + dbr = mdb_env_open(env, actual_filename.c_str(), 0, 0664); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to open rings database file '" + + actual_filename + "': " + std::string(mdb_strerror(dbr))); + + dbr = mdb_txn_begin(env, NULL, 0, &txn); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + dbr = mdb_dbi_open(txn, ("rings-" + genesis).c_str(), MDB_CREATE, &dbi_rings); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_compare(txn, dbi_rings, compare_hash32); + + dbr = mdb_dbi_open(txn, ("blackballs-" + genesis).c_str(), MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi_blackballs); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_dupsort(txn, dbi_blackballs, compare_hash32); + + dbr = mdb_txn_commit(txn); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); + tx_active = false; +} + +ringdb::~ringdb() +{ + mdb_dbi_close(env, dbi_rings); + mdb_dbi_close(env, dbi_blackballs); + mdb_env_close(env); +} + +bool ringdb::add_rings(const crypto::chacha_key &chacha_key, const cryptonote::transaction_prefix &tx) +{ + MDB_txn *txn; + int dbr; + bool tx_active = false; + + dbr = resize_env(env, filename.c_str(), get_ring_data_size(tx.vin.size())); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set env map size"); + dbr = mdb_txn_begin(env, NULL, 0, &txn); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + for (const auto &in: tx.vin) + { + if (in.type() != typeid(cryptonote::txin_to_key)) + continue; + const auto &txin = boost::get<cryptonote::txin_to_key>(in); + const uint32_t ring_size = txin.key_offsets.size(); + if (ring_size == 1) + continue; + + store_relative_ring(txn, dbi_rings, txin.k_image, txin.key_offsets, chacha_key); + } + + dbr = mdb_txn_commit(txn); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn adding ring to database: " + std::string(mdb_strerror(dbr))); + tx_active = false; + return true; +} + +bool ringdb::remove_rings(const crypto::chacha_key &chacha_key, const cryptonote::transaction_prefix &tx) +{ + MDB_txn *txn; + int dbr; + bool tx_active = false; + + dbr = resize_env(env, filename.c_str(), 0); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set env map size"); + dbr = mdb_txn_begin(env, NULL, 0, &txn); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + for (const auto &in: tx.vin) + { + if (in.type() != typeid(cryptonote::txin_to_key)) + continue; + const auto &txin = boost::get<cryptonote::txin_to_key>(in); + const uint32_t ring_size = txin.key_offsets.size(); + if (ring_size == 1) + continue; + + MDB_val key, data; + std::string key_ciphertext = encrypt(txin.k_image, chacha_key); + key.mv_data = (void*)key_ciphertext.data(); + key.mv_size = key_ciphertext.size(); + + dbr = mdb_get(txn, dbi_rings, &key, &data); + THROW_WALLET_EXCEPTION_IF(dbr && dbr != MDB_NOTFOUND, tools::error::wallet_internal_error, "Failed to look for key image in LMDB table: " + std::string(mdb_strerror(dbr))); + if (dbr == MDB_NOTFOUND) + continue; + THROW_WALLET_EXCEPTION_IF(data.mv_size <= 0, tools::error::wallet_internal_error, "Invalid ring data size"); + + MDEBUG("Removing ring data for key image " << txin.k_image); + dbr = mdb_del(txn, dbi_rings, &key, NULL); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to remove ring to database: " + std::string(mdb_strerror(dbr))); + } + + dbr = mdb_txn_commit(txn); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn removing ring to database: " + std::string(mdb_strerror(dbr))); + tx_active = false; + return true; +} + +bool ringdb::get_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, std::vector<uint64_t> &outs) +{ + MDB_txn *txn; + int dbr; + bool tx_active = false; + + dbr = resize_env(env, filename.c_str(), 0); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set env map size: " + std::string(mdb_strerror(dbr))); + dbr = mdb_txn_begin(env, NULL, 0, &txn); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + MDB_val key, data; + std::string key_ciphertext = encrypt(key_image, chacha_key); + key.mv_data = (void*)key_ciphertext.data(); + key.mv_size = key_ciphertext.size(); + dbr = mdb_get(txn, dbi_rings, &key, &data); + THROW_WALLET_EXCEPTION_IF(dbr && dbr != MDB_NOTFOUND, tools::error::wallet_internal_error, "Failed to look for key image in LMDB table: " + std::string(mdb_strerror(dbr))); + if (dbr == MDB_NOTFOUND) + return false; + THROW_WALLET_EXCEPTION_IF(data.mv_size <= 0, tools::error::wallet_internal_error, "Invalid ring data size"); + + std::string data_plaintext = decrypt(std::string((const char*)data.mv_data, data.mv_size), key_image, chacha_key); + outs = decompress_ring(data_plaintext); + MDEBUG("Found ring for key image " << key_image << ":"); + MDEBUG("Relative: " << boost::join(outs | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " ")); + outs = cryptonote::relative_output_offsets_to_absolute(outs); + MDEBUG("Absolute: " << boost::join(outs | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " ")); + + dbr = mdb_txn_commit(txn); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn getting ring from database: " + std::string(mdb_strerror(dbr))); + tx_active = false; + return true; +} + +bool ringdb::set_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, const std::vector<uint64_t> &outs, bool relative) +{ + MDB_txn *txn; + int dbr; + bool tx_active = false; + + dbr = resize_env(env, filename.c_str(), outs.size() * 64); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set env map size: " + std::string(mdb_strerror(dbr))); + dbr = mdb_txn_begin(env, NULL, 0, &txn); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + store_relative_ring(txn, dbi_rings, key_image, relative ? outs : cryptonote::absolute_output_offsets_to_relative(outs), chacha_key); + + dbr = mdb_txn_commit(txn); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn setting ring to database: " + std::string(mdb_strerror(dbr))); + tx_active = false; + return true; +} + +bool ringdb::blackball_worker(const crypto::public_key &output, int op) +{ + MDB_txn *txn; + MDB_cursor *cursor; + int dbr; + bool tx_active = false; + bool ret = true; + + dbr = resize_env(env, filename.c_str(), 32 * 2); // a pubkey, and some slack + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set env map size: " + std::string(mdb_strerror(dbr))); + dbr = mdb_txn_begin(env, NULL, 0, &txn); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + MDB_val key = zerokeyval; + MDB_val data; + data.mv_data = (void*)&output; + data.mv_size = sizeof(output); + + switch (op) + { + case BLACKBALL_BLACKBALL: + MDEBUG("Blackballing output " << output); + dbr = mdb_put(txn, dbi_blackballs, &key, &data, MDB_NODUPDATA); + if (dbr == MDB_KEYEXIST) + dbr = 0; + break; + case BLACKBALL_UNBLACKBALL: + MDEBUG("Unblackballing output " << output); + dbr = mdb_del(txn, dbi_blackballs, &key, &data); + if (dbr == MDB_NOTFOUND) + dbr = 0; + break; + case BLACKBALL_QUERY: + dbr = mdb_cursor_open(txn, dbi_blackballs, &cursor); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create cursor for blackballs table: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_get(cursor, &key, &data, MDB_GET_BOTH); + THROW_WALLET_EXCEPTION_IF(dbr && dbr != MDB_NOTFOUND, tools::error::wallet_internal_error, "Failed to lookup in blackballs table: " + std::string(mdb_strerror(dbr))); + ret = dbr != MDB_NOTFOUND; + if (dbr == MDB_NOTFOUND) + dbr = 0; + mdb_cursor_close(cursor); + break; + case BLACKBALL_CLEAR: + dbr = mdb_drop(txn, dbi_blackballs, 0); + break; + default: + THROW_WALLET_EXCEPTION(tools::error::wallet_internal_error, "Invalid blackball op"); + } + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to query blackballs table: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_txn_commit(txn); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn blackballing output to database: " + std::string(mdb_strerror(dbr))); + tx_active = false; + return ret; +} + +bool ringdb::blackball(const crypto::public_key &output) +{ + return blackball_worker(output, BLACKBALL_BLACKBALL); +} + +bool ringdb::unblackball(const crypto::public_key &output) +{ + return blackball_worker(output, BLACKBALL_UNBLACKBALL); +} + +bool ringdb::blackballed(const crypto::public_key &output) +{ + return blackball_worker(output, BLACKBALL_QUERY); +} + +bool ringdb::clear_blackballs() +{ + return blackball_worker(crypto::public_key(), BLACKBALL_CLEAR); +} + +} diff --git a/src/wallet/ringdb.h b/src/wallet/ringdb.h new file mode 100644 index 000000000..2bd1ac149 --- /dev/null +++ b/src/wallet/ringdb.h @@ -0,0 +1,65 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <string> +#include <vector> +#include <lmdb.h> +#include "wipeable_string.h" +#include "crypto/crypto.h" +#include "cryptonote_basic/cryptonote_basic.h" + +namespace tools +{ + class ringdb + { + public: + ringdb(std::string filename, const std::string &genesis); + ~ringdb(); + + bool add_rings(const crypto::chacha_key &chacha_key, const cryptonote::transaction_prefix &tx); + bool remove_rings(const crypto::chacha_key &chacha_key, const cryptonote::transaction_prefix &tx); + bool get_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, std::vector<uint64_t> &outs); + bool set_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, const std::vector<uint64_t> &outs, bool relative); + + bool blackball(const crypto::public_key &output); + bool unblackball(const crypto::public_key &output); + bool blackballed(const crypto::public_key &output); + bool clear_blackballs(); + + private: + bool blackball_worker(const crypto::public_key &output, int op); + + private: + std::string filename; + MDB_env *env; + MDB_dbi dbi_rings; + MDB_dbi dbi_blackballs; + }; +} diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 04d8c799a..d6fb1edb8 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -66,6 +66,7 @@ using namespace epee; #include "memwipe.h" #include "common/base58.h" #include "ringct/rctSigs.h" +#include "ringdb.h" extern "C" { @@ -93,7 +94,9 @@ using namespace cryptonote; #define MULTISIG_UNSIGNED_TX_PREFIX "Monero multisig unsigned tx set\001" #define RECENT_OUTPUT_RATIO (0.5) // 50% of outputs are from the recent zone -#define RECENT_OUTPUT_ZONE ((time_t)(1.8 * 86400)) // last 1.8 day makes up the recent zone (taken from monerolink.pdf, Miller et al) +#define RECENT_OUTPUT_DAYS (1.8) // last 1.8 day makes up the recent zone (taken from monerolink.pdf, Miller et al) +#define RECENT_OUTPUT_ZONE ((time_t)(RECENT_OUTPUT_DAYS * 86400)) +#define RECENT_OUTPUT_BLOCKS (RECENT_OUTPUT_DAYS * 720) #define FEE_ESTIMATE_GRACE_BLOCKS 10 // estimate fee valid for that many blocks @@ -106,6 +109,24 @@ using namespace cryptonote; #define MULTISIG_EXPORT_FILE_MAGIC "Monero multisig export\001" +#define SEGREGATION_FORK_HEIGHT 1564965 +#define TESTNET_SEGREGATION_FORK_HEIGHT 1000000 +#define STAGENET_SEGREGATION_FORK_HEIGHT 1000000 +#define SEGREGATION_FORK_VICINITY 1500 /* blocks */ + + +namespace +{ + std::string get_default_ringdb_path() + { + boost::filesystem::path dir = tools::get_default_data_dir(); + // remove .bitmonero, replace with .shared-ringdb + dir = dir.remove_filename(); + dir /= ".shared-ringdb"; + return dir.string(); + } +} + namespace { // Create on-demand to prevent static initialization order fiasco issues. @@ -119,6 +140,16 @@ struct options { const command_line::arg_descriptor<bool> testnet = {"testnet", tools::wallet2::tr("For testnet. Daemon must also be launched with --testnet flag"), false}; const command_line::arg_descriptor<bool> stagenet = {"stagenet", tools::wallet2::tr("For stagenet. Daemon must also be launched with --stagenet flag"), false}; const command_line::arg_descriptor<bool> restricted = {"restricted-rpc", tools::wallet2::tr("Restricts to view-only commands"), false}; + const command_line::arg_descriptor<std::string, false, true> shared_ringdb_dir = { + "shared-ringdb-dir", tools::wallet2::tr("Set shared ring database path"), + get_default_ringdb_path(), + testnet, + [](bool testnet, bool defaulted, std::string val) { + if (testnet) + return (boost::filesystem::path(val) / "testnet").string(); + return val; + } + }; }; void do_prepare_file_names(const std::string& file_path, std::string& keys_file, std::string& wallet_file) @@ -196,6 +227,8 @@ std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variabl std::unique_ptr<tools::wallet2> wallet(new tools::wallet2(testnet ? TESTNET : stagenet ? STAGENET : MAINNET, restricted)); wallet->init(std::move(daemon_address), std::move(login)); + boost::filesystem::path ringdb_path = command_line::get_arg(vm, opts.shared_ringdb_dir); + wallet->set_ring_database(ringdb_path.string()); return wallet; } @@ -627,6 +660,8 @@ wallet2::wallet2(network_type nettype, bool restricted): m_confirm_backlog_threshold(0), m_confirm_export_overwrite(true), m_auto_low_priority(true), + m_segregate_pre_fork_outputs(true), + m_key_reuse_mitigation2(true), m_is_initialized(false), m_restricted(restricted), is_old_file_format(false), @@ -639,7 +674,13 @@ wallet2::wallet2(network_type nettype, bool restricted): m_light_wallet_connected(false), m_light_wallet_balance(0), m_light_wallet_unlocked_balance(0), - m_key_on_device(false) + m_key_on_device(false), + m_ring_history_saved(false), + m_ringdb() +{ +} + +wallet2::~wallet2() { } @@ -665,6 +706,7 @@ void wallet2::init_options(boost::program_options::options_description& desc_par command_line::add_arg(desc_params, opts.testnet); command_line::add_arg(desc_params, opts.stagenet); command_line::add_arg(desc_params, opts.restricted); + command_line::add_arg(desc_params, opts.shared_ringdb_dir); } std::unique_ptr<wallet2> wallet2::make_from_json(const boost::program_options::variables_map& vm, const std::string& json_file, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter) @@ -1479,9 +1521,19 @@ void wallet2::process_outgoing(const crypto::hash &txid, const cryptonote::trans entry.first->second.m_subaddr_account = subaddr_account; entry.first->second.m_subaddr_indices = subaddr_indices; } + + for (const auto &in: tx.vin) + { + if (in.type() != typeid(cryptonote::txin_to_key)) + continue; + const auto &txin = boost::get<cryptonote::txin_to_key>(in); + entry.first->second.m_rings.push_back(std::make_pair(txin.k_image, txin.key_offsets)); + } entry.first->second.m_block_height = height; entry.first->second.m_timestamp = ts; entry.first->second.m_unlock_time = tx.unlock_time; + + add_rings(tx); } //---------------------------------------------------------------------------------------------------- void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const crypto::hash& bl_id, uint64_t height, const cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices &o_indices) @@ -1580,7 +1632,7 @@ void wallet2::pull_blocks(uint64_t start_height, uint64_t &blocks_start_height, THROW_WALLET_EXCEPTION_IF(*result == CORE_RPC_STATUS_BUSY, tools::error::daemon_busy, "getversion"); if (*result != CORE_RPC_STATUS_OK) { - MDEBUG("Cannot determined daemon RPC version, not asking for pruned blocks"); + MDEBUG("Cannot determine daemon RPC version, not asking for pruned blocks"); req.prune = false; // old daemon } } @@ -1852,6 +1904,7 @@ void wallet2::update_pool_state(bool refreshed) pit->second.m_state = wallet2::unconfirmed_transfer_details::failed; // the inputs aren't spent anymore, since the tx failed + remove_rings(pit->second.m_tx); for (size_t vini = 0; vini < pit->second.m_tx.vin.size(); ++vini) { if (pit->second.m_tx.vin[vini].type() == typeid(txin_to_key)) @@ -2266,6 +2319,73 @@ bool wallet2::refresh(uint64_t & blocks_fetched, bool& received_money, bool& ok) return ok; } //---------------------------------------------------------------------------------------------------- +bool wallet2::get_output_distribution(uint64_t &start_height, std::vector<uint64_t> &distribution) +{ + uint32_t rpc_version; + boost::optional<std::string> result = m_node_rpc_proxy.get_rpc_version(rpc_version); + // no error + if (!!result) + { + // empty string -> not connection + THROW_WALLET_EXCEPTION_IF(result->empty(), tools::error::no_connection_to_daemon, "getversion"); + THROW_WALLET_EXCEPTION_IF(*result == CORE_RPC_STATUS_BUSY, tools::error::daemon_busy, "getversion"); + if (*result != CORE_RPC_STATUS_OK) + { + MDEBUG("Cannot determine daemon RPC version, not requesting rct distribution"); + return false; + } + } + else + { + if (rpc_version >= MAKE_CORE_RPC_VERSION(1, 19)) + { + MDEBUG("Daemon is recent enough, requesting rct distribution"); + } + else + { + MDEBUG("Daemon is too old, not requesting rct distribution"); + return false; + } + } + + cryptonote::COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request req = AUTO_VAL_INIT(req); + cryptonote::COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response res = AUTO_VAL_INIT(res); + req.amounts.push_back(0); + req.from_height = 0; + req.cumulative = true; + m_daemon_rpc_mutex.lock(); + bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_distribution", req, res, m_http_client, rpc_timeout); + m_daemon_rpc_mutex.unlock(); + if (!r) + { + MWARNING("Failed to request output distribution: no connection to daemon"); + return false; + } + if (res.status == CORE_RPC_STATUS_BUSY) + { + MWARNING("Failed to request output distribution: daemon is busy"); + return false; + } + if (res.status != CORE_RPC_STATUS_OK) + { + MWARNING("Failed to request output distribution: " << res.status); + return false; + } + if (res.distributions.size() != 1) + { + MWARNING("Failed to request output distribution: not the expected single result"); + return false; + } + if (res.distributions[0].amount != 0) + { + MWARNING("Failed to request output distribution: results are not for amount 0"); + return false; + } + start_height = res.distributions[0].start_height; + distribution = std::move(res.distributions[0].distribution); + return true; +} +//---------------------------------------------------------------------------------------------------- void wallet2::detach_blockchain(uint64_t height) { LOG_PRINT_L0("Detaching blockchain on height " << height); @@ -2468,6 +2588,12 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable value2.SetUint(m_nettype); json.AddMember("nettype", value2, json.GetAllocator()); + value2.SetInt(m_segregate_pre_fork_outputs ? 1 : 0); + json.AddMember("segregate_pre_fork_outputs", value2, json.GetAllocator()); + + value2.SetInt(m_key_reuse_mitigation2 ? 1 : 0); + json.AddMember("key_reuse_mitigation2", value2, json.GetAllocator()); + // Serialize the JSON object rapidjson::StringBuffer buffer; rapidjson::Writer<rapidjson::StringBuffer> writer(buffer); @@ -2651,9 +2777,9 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, nettype, uint8_t, Uint, false, static_cast<uint8_t>(m_nettype)); // The network type given in the program argument is inconsistent with the network type saved in the wallet THROW_WALLET_EXCEPTION_IF(static_cast<uint8_t>(m_nettype) != field_nettype, error::wallet_internal_error, - (boost::format("%s wallet can not be opened as %s wallet") - % (field_nettype == 0 ? "Mainnet" : field_nettype == 1 ? "Testnet" : "Stagenet") - % (m_nettype == MAINNET ? "mainnet" : m_nettype == TESTNET ? "testnet" : "stagenet")).str()); + (boost::format("%s wallet cannot be opened as %s wallet") + % (field_nettype == 0 ? "Mainnet" : field_nettype == 1 ? "Testnet" : "Stagenet") + % (m_nettype == MAINNET ? "mainnet" : m_nettype == TESTNET ? "testnet" : "stagenet")).str()); } else { @@ -3540,20 +3666,17 @@ bool wallet2::check_connection(uint32_t *version, uint32_t timeout) if (version) { - epee::json_rpc::request<cryptonote::COMMAND_RPC_GET_VERSION::request> req_t = AUTO_VAL_INIT(req_t); - epee::json_rpc::response<cryptonote::COMMAND_RPC_GET_VERSION::response, std::string> resp_t = AUTO_VAL_INIT(resp_t); - req_t.jsonrpc = "2.0"; - req_t.id = epee::serialization::storage_entry(0); - req_t.method = "get_version"; - bool r = net_utils::invoke_http_json("/json_rpc", req_t, resp_t, m_http_client); + cryptonote::COMMAND_RPC_GET_VERSION::request req_t = AUTO_VAL_INIT(req_t); + cryptonote::COMMAND_RPC_GET_VERSION::response resp_t = AUTO_VAL_INIT(resp_t); + bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_version", req_t, resp_t, m_http_client); if(!r) { *version = 0; return false; } - if (resp_t.result.status != CORE_RPC_STATUS_OK) + if (resp_t.status != CORE_RPC_STATUS_OK) *version = 0; else - *version = resp_t.result.version; + *version = resp_t.version; } return true; @@ -3680,6 +3803,15 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass add_subaddress_account(tr("Primary account")); m_local_bc_height = m_blockchain.size(); + + try + { + find_and_save_rings(false); + } + catch (const std::exception &e) + { + MERROR("Failed to save rings, will try again next time"); + } } //---------------------------------------------------------------------------------------------------- void wallet2::trim_hashchain() @@ -3688,19 +3820,16 @@ void wallet2::trim_hashchain() if (!m_blockchain.empty() && m_blockchain.size() == m_blockchain.offset()) { MINFO("Fixing empty hashchain"); - epee::json_rpc::request<cryptonote::COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::request> req = AUTO_VAL_INIT(req); - epee::json_rpc::response<cryptonote::COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::response, std::string> res = AUTO_VAL_INIT(res); + cryptonote::COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::request req = AUTO_VAL_INIT(req); + cryptonote::COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::response res = AUTO_VAL_INIT(res); m_daemon_rpc_mutex.lock(); - req.jsonrpc = "2.0"; - req.id = epee::serialization::storage_entry(0); - req.method = "getblockheaderbyheight"; - req.params.height = m_blockchain.size() - 1; - bool r = net_utils::invoke_http_json("/json_rpc", req, res, m_http_client, rpc_timeout); + req.height = m_blockchain.size() - 1; + bool r = net_utils::invoke_http_json_rpc("/json_rpc", "getblockheaderbyheight", req, res, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); - if (r && res.result.status == CORE_RPC_STATUS_OK) + if (r && res.status == CORE_RPC_STATUS_OK) { crypto::hash hash; - epee::string_tools::hex_to_pod(res.result.block_header.hash, hash); + epee::string_tools::hex_to_pod(res.block_header.hash, hash); m_blockchain.refill(hash); } else @@ -4269,6 +4398,13 @@ void wallet2::add_unconfirmed_tx(const cryptonote::transaction& tx, uint64_t amo utd.m_timestamp = time(NULL); utd.m_subaddr_account = subaddr_account; utd.m_subaddr_indices = subaddr_indices; + for (const auto &in: tx.vin) + { + if (in.type() != typeid(cryptonote::txin_to_key)) + continue; + const auto &txin = boost::get<cryptonote::txin_to_key>(in); + utd.m_rings.push_back(std::make_pair(txin.k_image, txin.key_offsets)); + } } //---------------------------------------------------------------------------------------------------- @@ -5160,18 +5296,15 @@ uint32_t wallet2::adjust_priority(uint32_t priority) } // get the current full reward zone - epee::json_rpc::request<cryptonote::COMMAND_RPC_GET_INFO::request> getinfo_req = AUTO_VAL_INIT(getinfo_req); - epee::json_rpc::response<cryptonote::COMMAND_RPC_GET_INFO::response, std::string> getinfo_res = AUTO_VAL_INIT(getinfo_res); + cryptonote::COMMAND_RPC_GET_INFO::request getinfo_req = AUTO_VAL_INIT(getinfo_req); + cryptonote::COMMAND_RPC_GET_INFO::response getinfo_res = AUTO_VAL_INIT(getinfo_res); m_daemon_rpc_mutex.lock(); - getinfo_req.jsonrpc = "2.0"; - getinfo_req.id = epee::serialization::storage_entry(0); - getinfo_req.method = "get_info"; - bool r = net_utils::invoke_http_json("/json_rpc", getinfo_req, getinfo_res, m_http_client); + bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_info", getinfo_req, getinfo_res, m_http_client); m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "get_info"); - THROW_WALLET_EXCEPTION_IF(getinfo_res.result.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_info"); - THROW_WALLET_EXCEPTION_IF(getinfo_res.result.status != CORE_RPC_STATUS_OK, error::get_tx_pool_error); - const uint64_t full_reward_zone = getinfo_res.result.block_size_limit / 2; + THROW_WALLET_EXCEPTION_IF(getinfo_res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_info"); + THROW_WALLET_EXCEPTION_IF(getinfo_res.status != CORE_RPC_STATUS_OK, error::get_tx_pool_error); + const uint64_t full_reward_zone = getinfo_res.block_size_limit / 2; // get the last N block headers and sum the block sizes const size_t N = 10; @@ -5180,26 +5313,23 @@ uint32_t wallet2::adjust_priority(uint32_t priority) MERROR("The blockchain is too short"); return priority; } - epee::json_rpc::request<cryptonote::COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::request> getbh_req = AUTO_VAL_INIT(getbh_req); - epee::json_rpc::response<cryptonote::COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::response, std::string> getbh_res = AUTO_VAL_INIT(getbh_res); + cryptonote::COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::request getbh_req = AUTO_VAL_INIT(getbh_req); + cryptonote::COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::response getbh_res = AUTO_VAL_INIT(getbh_res); m_daemon_rpc_mutex.lock(); - getbh_req.jsonrpc = "2.0"; - getbh_req.id = epee::serialization::storage_entry(0); - getbh_req.method = "getblockheadersrange"; - getbh_req.params.start_height = m_blockchain.size() - N; - getbh_req.params.end_height = m_blockchain.size() - 1; - r = net_utils::invoke_http_json("/json_rpc", getbh_req, getbh_res, m_http_client, rpc_timeout); + getbh_req.start_height = m_blockchain.size() - N; + getbh_req.end_height = m_blockchain.size() - 1; + r = net_utils::invoke_http_json_rpc("/json_rpc", "getblockheadersrange", getbh_req, getbh_res, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "getblockheadersrange"); - THROW_WALLET_EXCEPTION_IF(getbh_res.result.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "getblockheadersrange"); - THROW_WALLET_EXCEPTION_IF(getbh_res.result.status != CORE_RPC_STATUS_OK, error::get_blocks_error, getbh_res.result.status); - if (getbh_res.result.headers.size() != N) + THROW_WALLET_EXCEPTION_IF(getbh_res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "getblockheadersrange"); + THROW_WALLET_EXCEPTION_IF(getbh_res.status != CORE_RPC_STATUS_OK, error::get_blocks_error, getbh_res.status); + if (getbh_res.headers.size() != N) { MERROR("Bad blockheaders size"); return priority; } size_t block_size_sum = 0; - for (const cryptonote::block_header_response &i : getbh_res.result.headers) + for (const cryptonote::block_header_response &i : getbh_res.headers) { block_size_sum += i.block_size; } @@ -5332,17 +5462,195 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions(std::vector<crypto } } +void wallet2::set_ring_database(const std::string &filename) +{ + m_ring_database = filename; + MINFO("ringdb path set to " << filename); + m_ringdb.reset(); + cryptonote::block b; + generate_genesis(b); + if (!m_ring_database.empty()) + m_ringdb.reset(new tools::ringdb(m_ring_database, epee::string_tools::pod_to_hex(get_block_hash(b)))); +} + +bool wallet2::add_rings(const crypto::chacha_key &key, const cryptonote::transaction_prefix &tx) +{ + if (!m_ringdb) + return true; + return m_ringdb->add_rings(key, tx); +} + +bool wallet2::add_rings(const cryptonote::transaction_prefix &tx) +{ + crypto::chacha_key key; + generate_chacha_key_from_secret_keys(key); + return add_rings(key, tx); +} + +bool wallet2::remove_rings(const cryptonote::transaction_prefix &tx) +{ + if (!m_ringdb) + return true; + crypto::chacha_key key; + generate_chacha_key_from_secret_keys(key); + return m_ringdb->remove_rings(key, tx); +} + +bool wallet2::get_ring(const crypto::chacha_key &key, const crypto::key_image &key_image, std::vector<uint64_t> &outs) +{ + if (!m_ringdb) + return true; + return m_ringdb->get_ring(key, key_image, outs); +} -bool wallet2::tx_add_fake_output(std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, uint64_t global_index, const crypto::public_key& tx_public_key, const rct::key& mask, uint64_t real_index, bool unlocked) const +bool wallet2::get_rings(const crypto::hash &txid, std::vector<std::pair<crypto::key_image, std::vector<uint64_t>>> &outs) +{ + for (auto i: m_confirmed_txs) + { + if (txid == i.first) + { + for (const auto &x: i.second.m_rings) + outs.push_back({x.first, cryptonote::relative_output_offsets_to_absolute(x.second)}); + return true; + } + } + for (auto i: m_unconfirmed_txs) + { + if (txid == i.first) + { + for (const auto &x: i.second.m_rings) + outs.push_back({x.first, cryptonote::relative_output_offsets_to_absolute(x.second)}); + return true; + } + } + return false; +} + +bool wallet2::get_ring(const crypto::key_image &key_image, std::vector<uint64_t> &outs) +{ + crypto::chacha_key key; + generate_chacha_key_from_secret_keys(key); + + return get_ring(key, key_image, outs); +} + +bool wallet2::set_ring(const crypto::key_image &key_image, const std::vector<uint64_t> &outs, bool relative) +{ + if (!m_ringdb) + return true; + + crypto::chacha_key key; + generate_chacha_key_from_secret_keys(key); + + return m_ringdb->set_ring(key, key_image, outs, relative); +} + +bool wallet2::find_and_save_rings(bool force) +{ + if (!force && m_ring_history_saved) + return true; + if (!m_ringdb) + return true; + + COMMAND_RPC_GET_TRANSACTIONS::request req = AUTO_VAL_INIT(req); + COMMAND_RPC_GET_TRANSACTIONS::response res = AUTO_VAL_INIT(res); + + MDEBUG("Finding and saving rings..."); + + // get payments we made + std::list<std::pair<crypto::hash,wallet2::confirmed_transfer_details>> payments; + get_payments_out(payments, 0, std::numeric_limits<uint64_t>::max(), boost::none, std::set<uint32_t>()); + for (const std::pair<crypto::hash,wallet2::confirmed_transfer_details> &entry: payments) + { + const crypto::hash &txid = entry.first; + req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); + } + + MDEBUG("Found " << std::to_string(req.txs_hashes.size()) << " transactions"); + + // get those transactions from the daemon + req.decode_as_json = false; + bool r; + { + const boost::lock_guard<boost::mutex> lock{m_daemon_rpc_mutex}; + r = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client, rpc_timeout); + } + THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "gettransactions"); + THROW_WALLET_EXCEPTION_IF(res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "gettransactions"); + THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::wallet_internal_error, "gettransactions"); + THROW_WALLET_EXCEPTION_IF(res.txs.size() != req.txs_hashes.size(), error::wallet_internal_error, + "daemon returned wrong response for gettransactions, wrong txs count = " + + std::to_string(res.txs.size()) + ", expected " + std::to_string(req.txs_hashes.size())); + + MDEBUG("Scanning " << res.txs.size() << " transactions"); + + crypto::chacha_key key; + generate_chacha_key_from_secret_keys(key); + + auto it = req.txs_hashes.begin(); + for (size_t i = 0; i < res.txs.size(); ++i, ++it) + { + const auto &tx_info = res.txs[i]; + THROW_WALLET_EXCEPTION_IF(tx_info.tx_hash != *it, error::wallet_internal_error, "Wrong txid received"); + cryptonote::blobdata bd; + THROW_WALLET_EXCEPTION_IF(!epee::string_tools::parse_hexstr_to_binbuff(tx_info.as_hex, bd), error::wallet_internal_error, "failed to parse tx from hexstr"); + cryptonote::transaction tx; + crypto::hash tx_hash, tx_prefix_hash; + THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(bd, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, "failed to parse tx from blob"); + THROW_WALLET_EXCEPTION_IF(epee::string_tools::pod_to_hex(tx_hash) != tx_info.tx_hash, error::wallet_internal_error, "txid mismatch"); + THROW_WALLET_EXCEPTION_IF(!add_rings(key, tx), error::wallet_internal_error, "Failed to save ring"); + } + + MINFO("Found and saved rings for " << res.txs.size() << " transactions"); + m_ring_history_saved = true; + return true; +} + +bool wallet2::blackball_output(const crypto::public_key &output) +{ + if (!m_ringdb) + return true; + return m_ringdb->blackball(output); +} + +bool wallet2::set_blackballed_outputs(const std::vector<crypto::public_key> &outputs, bool add) +{ + if (!m_ringdb) + return true; + bool ret = true; + if (!add) + ret &= m_ringdb->clear_blackballs(); + for (const auto &output: outputs) + ret &= m_ringdb->blackball(output); + return ret; +} + +bool wallet2::unblackball_output(const crypto::public_key &output) +{ + if (!m_ringdb) + return true; + return m_ringdb->unblackball(output); +} + +bool wallet2::is_output_blackballed(const crypto::public_key &output) const +{ + if (!m_ringdb) + return true; + return m_ringdb->blackballed(output); +} + +bool wallet2::tx_add_fake_output(std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, uint64_t global_index, const crypto::public_key& output_public_key, const rct::key& mask, uint64_t real_index, bool unlocked) const { if (!unlocked) // don't add locked outs return false; if (global_index == real_index) // don't re-add real one return false; - auto item = std::make_tuple(global_index, tx_public_key, mask); + auto item = std::make_tuple(global_index, output_public_key, mask); CHECK_AND_ASSERT_MES(!outs.empty(), false, "internal error: outs is empty"); if (std::find(outs.back().begin(), outs.back().end(), item) != outs.back().end()) // don't add duplicates return false; + if (is_output_blackballed(output_public_key)) // don't add blackballed outputs + return false; outs.back().push_back(item); return true; } @@ -5461,27 +5769,85 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> return; } + crypto::chacha_key key; + generate_chacha_key_from_secret_keys(key); + if (fake_outputs_count > 0) { + uint64_t segregation_fork_height; + switch (m_nettype) + { + case TESTNET: segregation_fork_height = TESTNET_SEGREGATION_FORK_HEIGHT; break; + case STAGENET: segregation_fork_height = STAGENET_SEGREGATION_FORK_HEIGHT; break; + case MAINNET: segregation_fork_height = SEGREGATION_FORK_HEIGHT; break; + default: THROW_WALLET_EXCEPTION(error::wallet_internal_error, "Invalid network type"); + } + // check whether we're shortly after the fork + uint64_t height; + boost::optional<std::string> result = m_node_rpc_proxy.get_height(height); + throw_on_rpc_response_error(result, "get_info"); + bool is_shortly_after_segregation_fork = height >= segregation_fork_height && height < segregation_fork_height + SEGREGATION_FORK_VICINITY; + // get histogram for the amounts we need - epee::json_rpc::request<cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request> req_t = AUTO_VAL_INIT(req_t); - epee::json_rpc::response<cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response, std::string> resp_t = AUTO_VAL_INIT(resp_t); + cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request req_t = AUTO_VAL_INIT(req_t); + cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response resp_t = AUTO_VAL_INIT(resp_t); m_daemon_rpc_mutex.lock(); - req_t.jsonrpc = "2.0"; - req_t.id = epee::serialization::storage_entry(0); - req_t.method = "get_output_histogram"; for(size_t idx: selected_transfers) - req_t.params.amounts.push_back(m_transfers[idx].is_rct() ? 0 : m_transfers[idx].amount()); - std::sort(req_t.params.amounts.begin(), req_t.params.amounts.end()); - auto end = std::unique(req_t.params.amounts.begin(), req_t.params.amounts.end()); - req_t.params.amounts.resize(std::distance(req_t.params.amounts.begin(), end)); - req_t.params.unlocked = true; - req_t.params.recent_cutoff = time(NULL) - RECENT_OUTPUT_ZONE; - bool r = net_utils::invoke_http_json("/json_rpc", req_t, resp_t, m_http_client, rpc_timeout); + req_t.amounts.push_back(m_transfers[idx].is_rct() ? 0 : m_transfers[idx].amount()); + std::sort(req_t.amounts.begin(), req_t.amounts.end()); + auto end = std::unique(req_t.amounts.begin(), req_t.amounts.end()); + req_t.amounts.resize(std::distance(req_t.amounts.begin(), end)); + req_t.unlocked = true; + req_t.recent_cutoff = time(NULL) - RECENT_OUTPUT_ZONE; + bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_histogram", req_t, resp_t, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "transfer_selected"); - THROW_WALLET_EXCEPTION_IF(resp_t.result.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_output_histogram"); - THROW_WALLET_EXCEPTION_IF(resp_t.result.status != CORE_RPC_STATUS_OK, error::get_histogram_error, resp_t.result.status); + THROW_WALLET_EXCEPTION_IF(resp_t.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_output_histogram"); + THROW_WALLET_EXCEPTION_IF(resp_t.status != CORE_RPC_STATUS_OK, error::get_histogram_error, resp_t.status); + + // if we want to segregate fake outs pre or post fork, get distribution + std::unordered_map<uint64_t, std::pair<uint64_t, uint64_t>> segregation_limit; + if (m_segregate_pre_fork_outputs || m_key_reuse_mitigation2) + { + cryptonote::COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request req_t = AUTO_VAL_INIT(req_t); + cryptonote::COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response resp_t = AUTO_VAL_INIT(resp_t); + for(size_t idx: selected_transfers) + req_t.amounts.push_back(m_transfers[idx].is_rct() ? 0 : m_transfers[idx].amount()); + std::sort(req_t.amounts.begin(), req_t.amounts.end()); + auto end = std::unique(req_t.amounts.begin(), req_t.amounts.end()); + req_t.amounts.resize(std::distance(req_t.amounts.begin(), end)); + req_t.from_height = segregation_fork_height >= RECENT_OUTPUT_ZONE ? height >= (segregation_fork_height ? segregation_fork_height : height) - RECENT_OUTPUT_BLOCKS : 0; + req_t.cumulative = true; + m_daemon_rpc_mutex.lock(); + bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_distribution", req_t, resp_t, m_http_client, rpc_timeout); + m_daemon_rpc_mutex.unlock(); + THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "transfer_selected"); + THROW_WALLET_EXCEPTION_IF(resp_t.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_output_distribution"); + THROW_WALLET_EXCEPTION_IF(resp_t.status != CORE_RPC_STATUS_OK, error::get_output_distribution, resp_t.status); + + // check we got all data + for(size_t idx: selected_transfers) + { + const uint64_t amount = m_transfers[idx].is_rct() ? 0 : m_transfers[idx].amount(); + bool found = false; + for (const auto &d: resp_t.distributions) + { + if (d.amount == amount) + { + THROW_WALLET_EXCEPTION_IF(d.start_height > segregation_fork_height, error::get_output_distribution, "Distribution start_height too high"); + THROW_WALLET_EXCEPTION_IF(segregation_fork_height - d.start_height >= d.distribution.size(), error::get_output_distribution, "Distribution size too small"); + THROW_WALLET_EXCEPTION_IF(segregation_fork_height <= RECENT_OUTPUT_BLOCKS, error::wallet_internal_error, "Fork height too low"); + THROW_WALLET_EXCEPTION_IF(segregation_fork_height - RECENT_OUTPUT_BLOCKS < d.start_height, error::get_output_distribution, "Bad start height"); + uint64_t till_fork = d.distribution[segregation_fork_height - d.start_height]; + uint64_t recent = till_fork - d.distribution[segregation_fork_height - RECENT_OUTPUT_BLOCKS - d.start_height]; + segregation_limit[amount] = std::make_pair(till_fork, recent); + found = true; + break; + } + } + THROW_WALLET_EXCEPTION_IF(!found, error::get_output_distribution, "Requested amount not found in response"); + } + } // we ask for more, to have spares if some outputs are still locked size_t base_requested_outputs_count = (size_t)((fake_outputs_count + 1) * 1.5 + 1); @@ -5502,37 +5868,126 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> size_t requested_outputs_count = base_requested_outputs_count + (td.is_rct() ? CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW - CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE : 0); size_t start = req.outputs.size(); - // if there are just enough outputs to mix with, use all of them. - // Eventually this should become impossible. + const bool output_is_pre_fork = td.m_block_height < segregation_fork_height; uint64_t num_outs = 0, num_recent_outs = 0; - for (const auto &he: resp_t.result.histogram) + uint64_t num_post_fork_outs = 0; + float pre_fork_num_out_ratio = 0.0f; + float post_fork_num_out_ratio = 0.0f; + + if (m_segregate_pre_fork_outputs && output_is_pre_fork) { - if (he.amount == amount) + num_outs = segregation_limit[amount].first; + num_recent_outs = segregation_limit[amount].second; + } + else + { + // if there are just enough outputs to mix with, use all of them. + // Eventually this should become impossible. + for (const auto &he: resp_t.histogram) { - LOG_PRINT_L2("Found " << print_money(amount) << ": " << he.total_instances << " total, " - << he.unlocked_instances << " unlocked, " << he.recent_instances << " recent"); - num_outs = he.unlocked_instances; - num_recent_outs = he.recent_instances; - break; + if (he.amount == amount) + { + LOG_PRINT_L2("Found " << print_money(amount) << ": " << he.total_instances << " total, " + << he.unlocked_instances << " unlocked, " << he.recent_instances << " recent"); + num_outs = he.unlocked_instances; + num_recent_outs = he.recent_instances; + break; + } } + if (m_key_reuse_mitigation2) + { + if (output_is_pre_fork) + { + if (is_shortly_after_segregation_fork) + { + pre_fork_num_out_ratio = 33.4/100.0f * (1.0f - RECENT_OUTPUT_RATIO); + } + else + { + pre_fork_num_out_ratio = 33.4/100.0f * (1.0f - RECENT_OUTPUT_RATIO); + post_fork_num_out_ratio = 33.4/100.0f * (1.0f - RECENT_OUTPUT_RATIO); + } + } + else + { + if (is_shortly_after_segregation_fork) + { + } + else + { + post_fork_num_out_ratio = 67.8/100.0f * (1.0f - RECENT_OUTPUT_RATIO); + } + } + } + num_post_fork_outs = num_outs - segregation_limit[amount].first; } + LOG_PRINT_L1("" << num_outs << " unlocked outputs of size " << print_money(amount)); THROW_WALLET_EXCEPTION_IF(num_outs == 0, error::wallet_internal_error, "histogram reports no unlocked outputs for " + boost::lexical_cast<std::string>(amount) + ", not even ours"); THROW_WALLET_EXCEPTION_IF(num_recent_outs > num_outs, error::wallet_internal_error, "histogram reports more recent outs than outs for " + boost::lexical_cast<std::string>(amount)); + // how many fake outs to draw on a pre-fork triangular distribution + size_t pre_fork_outputs_count = requested_outputs_count * pre_fork_num_out_ratio; + size_t post_fork_outputs_count = requested_outputs_count * post_fork_num_out_ratio; + // how many fake outs to draw otherwise + size_t normal_output_count = requested_outputs_count - pre_fork_outputs_count - post_fork_outputs_count; + // X% of those outs are to be taken from recent outputs - size_t recent_outputs_count = requested_outputs_count * RECENT_OUTPUT_RATIO; + size_t recent_outputs_count = normal_output_count * RECENT_OUTPUT_RATIO; if (recent_outputs_count == 0) recent_outputs_count = 1; // ensure we have at least one, if possible if (recent_outputs_count > num_recent_outs) recent_outputs_count = num_recent_outs; if (td.m_global_output_index >= num_outs - num_recent_outs && recent_outputs_count > 0) --recent_outputs_count; // if the real out is recent, pick one less recent fake out - LOG_PRINT_L1("Using " << recent_outputs_count << " recent outputs"); + LOG_PRINT_L1("Fake output makeup: " << requested_outputs_count << " requested: " << recent_outputs_count << " recent, " << + pre_fork_outputs_count << " pre-fork, " << post_fork_outputs_count << " post-fork, " << + (requested_outputs_count - recent_outputs_count - pre_fork_outputs_count - post_fork_outputs_count) << " full-chain"); + + uint64_t num_found = 0; - if (num_outs <= requested_outputs_count) + // if we have a known ring, use it + bool existing_ring_found = false; + if (td.m_key_image_known && !td.m_key_image_partial) + { + std::vector<uint64_t> ring; + if (get_ring(key, td.m_key_image, ring)) + { + MINFO("This output has a known ring, reusing (size " << ring.size() << ")"); + THROW_WALLET_EXCEPTION_IF(ring.size() > fake_outputs_count + 1, error::wallet_internal_error, + "An output in this transaction was previously spent on another chain with ring size " + + std::to_string(ring.size()) + ", it cannot be spent now with ring size " + + std::to_string(fake_outputs_count + 1) + " as it is smaller: use a higher ring size"); + bool own_found = false; + existing_ring_found = true; + for (const auto &out: ring) + { + MINFO("Ring has output " << out); + if (out < num_outs) + { + MINFO("Using it"); + req.outputs.push_back({amount, out}); + ++num_found; + seen_indices.emplace(out); + if (out == td.m_global_output_index) + { + MINFO("This is the real output"); + own_found = true; + } + } + else + { + MINFO("Ignoring output " << out << ", too recent"); + } + } + THROW_WALLET_EXCEPTION_IF(!own_found, error::wallet_internal_error, + "Known ring does not include the spent output: " + std::to_string(td.m_global_output_index)); + } + } + + if (num_outs <= requested_outputs_count && !existing_ring_found) { for (uint64_t i = 0; i < num_outs; i++) req.outputs.push_back({amount, i}); @@ -5545,10 +6000,13 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> else { // start with real one - uint64_t num_found = 1; - seen_indices.emplace(td.m_global_output_index); - req.outputs.push_back({amount, td.m_global_output_index}); - LOG_PRINT_L1("Selecting real output: " << td.m_global_output_index << " for " << print_money(amount)); + if (num_found == 0) + { + num_found = 1; + seen_indices.emplace(td.m_global_output_index); + req.outputs.push_back({amount, td.m_global_output_index}); + LOG_PRINT_L1("Selecting real output: " << td.m_global_output_index << " for " << print_money(amount)); + } // while we still need more mixins while (num_found < requested_outputs_count) @@ -5562,6 +6020,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> // list of output indices we've seen. uint64_t i; + const char *type = ""; if (num_found - 1 < recent_outputs_count) // -1 to account for the real one we seeded with { // triangular distribution over [a,b) with a=0, mode c=b=up_index_limit @@ -5571,7 +6030,29 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> // just in case rounding up to 1 occurs after calc if (i == num_outs) --i; - LOG_PRINT_L2("picking " << i << " as recent"); + type = "recent"; + } + else if (num_found -1 < recent_outputs_count + pre_fork_outputs_count) + { + // triangular distribution over [a,b) with a=0, mode c=b=up_index_limit + uint64_t r = crypto::rand<uint64_t>() % ((uint64_t)1 << 53); + double frac = std::sqrt((double)r / ((uint64_t)1 << 53)); + i = (uint64_t)(frac*segregation_limit[amount].first); + // just in case rounding up to 1 occurs after calc + if (i == num_outs) + --i; + type = " pre-fork"; + } + else if (num_found -1 < recent_outputs_count + pre_fork_outputs_count + post_fork_outputs_count) + { + // triangular distribution over [a,b) with a=0, mode c=b=up_index_limit + uint64_t r = crypto::rand<uint64_t>() % ((uint64_t)1 << 53); + double frac = std::sqrt((double)r / ((uint64_t)1 << 53)); + i = (uint64_t)(frac*num_post_fork_outs) + segregation_limit[amount].first; + // just in case rounding up to 1 occurs after calc + if (i == num_post_fork_outs+segregation_limit[amount].first) + --i; + type = "post-fork"; } else { @@ -5582,13 +6063,14 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> // just in case rounding up to 1 occurs after calc if (i == num_outs) --i; - LOG_PRINT_L2("picking " << i << " as triangular"); + type = "triangular"; } if (seen_indices.count(i)) continue; seen_indices.emplace(i); + LOG_PRINT_L2("picking " << i << " as " << type); req.outputs.push_back({amount, i}); ++num_found; } @@ -5624,6 +6106,20 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> outs.back().reserve(fake_outputs_count + 1); const rct::key mask = td.is_rct() ? rct::commit(td.amount(), td.m_mask) : rct::zeroCommit(td.amount()); + uint64_t num_outs = 0; + const uint64_t amount = td.is_rct() ? 0 : td.amount(); + const bool output_is_pre_fork = td.m_block_height < segregation_fork_height; + if (m_segregate_pre_fork_outputs && output_is_pre_fork) + num_outs = segregation_limit[amount].first; + else for (const auto &he: resp_t.histogram) + { + if (he.amount == amount) + { + num_outs = he.unlocked_instances; + break; + } + } + // make sure the real outputs we asked for are really included, along // with the correct key and mask: this guards against an active attack // where the node sends dummy data for all outputs, and we then send @@ -5644,6 +6140,38 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> // pick real out first (it will be sorted when done) outs.back().push_back(std::make_tuple(td.m_global_output_index, boost::get<txout_to_key>(td.m_tx.vout[td.m_internal_output_index].target).key, mask)); + // then pick outs from an existing ring, if any + bool existing_ring_found = false; + if (td.m_key_image_known && !td.m_key_image_partial) + { + std::vector<uint64_t> ring; + if (get_ring(key, td.m_key_image, ring)) + { + for (uint64_t out: ring) + { + if (out < num_outs) + { + if (out != td.m_global_output_index) + { + bool found = false; + for (size_t o = 0; o < requested_outputs_count; ++o) + { + size_t i = base + o; + if (req.outputs[i].index == out) + { + LOG_PRINT_L2("Index " << i << "/" << requested_outputs_count << ": idx " << req.outputs[i].index << " (real " << td.m_global_output_index << "), unlocked " << daemon_resp.outs[i].unlocked << ", key " << daemon_resp.outs[i].key << " (from existing ring)"); + tx_add_fake_output(outs, req.outputs[i].index, daemon_resp.outs[i].key, daemon_resp.outs[i].mask, td.m_global_output_index, daemon_resp.outs[i].unlocked); + found = true; + break; + } + } + THROW_WALLET_EXCEPTION_IF(!found, error::wallet_internal_error, "Falied to find existing ring output in daemon out data"); + } + } + } + } + } + // then pick others in random order till we reach the required number // since we use an equiprobable pick here, we don't upset the triangular distribution std::vector<size_t> order; @@ -7502,25 +8030,22 @@ std::vector<uint64_t> wallet2::get_unspent_amounts_vector() const //---------------------------------------------------------------------------------------------------- std::vector<size_t> wallet2::select_available_outputs_from_histogram(uint64_t count, bool atleast, bool unlocked, bool allow_rct, bool trusted_daemon) { - epee::json_rpc::request<cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request> req_t = AUTO_VAL_INIT(req_t); - epee::json_rpc::response<cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response, std::string> resp_t = AUTO_VAL_INIT(resp_t); + cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request req_t = AUTO_VAL_INIT(req_t); + cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response resp_t = AUTO_VAL_INIT(resp_t); m_daemon_rpc_mutex.lock(); - req_t.jsonrpc = "2.0"; - req_t.id = epee::serialization::storage_entry(0); - req_t.method = "get_output_histogram"; if (trusted_daemon) - req_t.params.amounts = get_unspent_amounts_vector(); - req_t.params.min_count = count; - req_t.params.max_count = 0; - req_t.params.unlocked = unlocked; - bool r = net_utils::invoke_http_json("/json_rpc", req_t, resp_t, m_http_client, rpc_timeout); + req_t.amounts = get_unspent_amounts_vector(); + req_t.min_count = count; + req_t.max_count = 0; + req_t.unlocked = unlocked; + bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_histogram", req_t, resp_t, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "select_available_outputs_from_histogram"); - THROW_WALLET_EXCEPTION_IF(resp_t.result.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_output_histogram"); - THROW_WALLET_EXCEPTION_IF(resp_t.result.status != CORE_RPC_STATUS_OK, error::get_histogram_error, resp_t.result.status); + THROW_WALLET_EXCEPTION_IF(resp_t.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_output_histogram"); + THROW_WALLET_EXCEPTION_IF(resp_t.status != CORE_RPC_STATUS_OK, error::get_histogram_error, resp_t.status); std::set<uint64_t> mixable; - for (const auto &i: resp_t.result.histogram) + for (const auto &i: resp_t.histogram) { mixable.insert(i.amount); } @@ -7543,24 +8068,21 @@ std::vector<size_t> wallet2::select_available_outputs_from_histogram(uint64_t co //---------------------------------------------------------------------------------------------------- uint64_t wallet2::get_num_rct_outputs() { - epee::json_rpc::request<cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request> req_t = AUTO_VAL_INIT(req_t); - epee::json_rpc::response<cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response, std::string> resp_t = AUTO_VAL_INIT(resp_t); + cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request req_t = AUTO_VAL_INIT(req_t); + cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response resp_t = AUTO_VAL_INIT(resp_t); m_daemon_rpc_mutex.lock(); - req_t.jsonrpc = "2.0"; - req_t.id = epee::serialization::storage_entry(0); - req_t.method = "get_output_histogram"; - req_t.params.amounts.push_back(0); - req_t.params.min_count = 0; - req_t.params.max_count = 0; - bool r = net_utils::invoke_http_json("/json_rpc", req_t, resp_t, m_http_client, rpc_timeout); + req_t.amounts.push_back(0); + req_t.min_count = 0; + req_t.max_count = 0; + bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_histogram", req_t, resp_t, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "get_num_rct_outputs"); - THROW_WALLET_EXCEPTION_IF(resp_t.result.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_output_histogram"); - THROW_WALLET_EXCEPTION_IF(resp_t.result.status != CORE_RPC_STATUS_OK, error::get_histogram_error, resp_t.result.status); - THROW_WALLET_EXCEPTION_IF(resp_t.result.histogram.size() != 1, error::get_histogram_error, "Expected exactly one response"); - THROW_WALLET_EXCEPTION_IF(resp_t.result.histogram[0].amount != 0, error::get_histogram_error, "Expected 0 amount"); + THROW_WALLET_EXCEPTION_IF(resp_t.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_output_histogram"); + THROW_WALLET_EXCEPTION_IF(resp_t.status != CORE_RPC_STATUS_OK, error::get_histogram_error, resp_t.status); + THROW_WALLET_EXCEPTION_IF(resp_t.histogram.size() != 1, error::get_histogram_error, "Expected exactly one response"); + THROW_WALLET_EXCEPTION_IF(resp_t.histogram[0].amount != 0, error::get_histogram_error, "Expected 0 amount"); - return resp_t.result.histogram[0].total_instances; + return resp_t.histogram[0].total_instances; } //---------------------------------------------------------------------------------------------------- const wallet2::transfer_details &wallet2::get_transfer_details(size_t idx) const @@ -8476,23 +8998,20 @@ uint64_t wallet2::get_daemon_blockchain_height(string &err) const uint64_t wallet2::get_daemon_blockchain_target_height(string &err) { - epee::json_rpc::request<cryptonote::COMMAND_RPC_GET_INFO::request> req_t = AUTO_VAL_INIT(req_t); - epee::json_rpc::response<cryptonote::COMMAND_RPC_GET_INFO::response, std::string> resp_t = AUTO_VAL_INIT(resp_t); + cryptonote::COMMAND_RPC_GET_INFO::request req_t = AUTO_VAL_INIT(req_t); + cryptonote::COMMAND_RPC_GET_INFO::response resp_t = AUTO_VAL_INIT(resp_t); m_daemon_rpc_mutex.lock(); - req_t.jsonrpc = "2.0"; - req_t.id = epee::serialization::storage_entry(0); - req_t.method = "get_info"; - bool ok = net_utils::invoke_http_json("/json_rpc", req_t, resp_t, m_http_client); + bool ok = net_utils::invoke_http_json_rpc("/json_rpc", "get_info", req_t, resp_t, m_http_client); m_daemon_rpc_mutex.unlock(); if (ok) { - if (resp_t.result.status == CORE_RPC_STATUS_BUSY) + if (resp_t.status == CORE_RPC_STATUS_BUSY) { err = "daemon is busy. Please try again later."; } - else if (resp_t.result.status != CORE_RPC_STATUS_OK) + else if (resp_t.status != CORE_RPC_STATUS_OK) { - err = resp_t.result.status; + err = resp_t.status; } else // success, cleaning up error message { @@ -8503,7 +9022,7 @@ uint64_t wallet2::get_daemon_blockchain_target_height(string &err) { err = "possibly lost connection to daemon"; } - return resp_t.result.target_height; + return resp_t.target_height; } uint64_t wallet2::get_approximate_blockchain_height() const @@ -9739,30 +10258,24 @@ std::vector<std::pair<uint64_t, uint64_t>> wallet2::estimate_backlog(const std:: } // get txpool backlog - epee::json_rpc::request<cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::request> req = AUTO_VAL_INIT(req); - epee::json_rpc::response<cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::response, std::string> res = AUTO_VAL_INIT(res); + cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::request req = AUTO_VAL_INIT(req); + cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::response res = AUTO_VAL_INIT(res); m_daemon_rpc_mutex.lock(); - req.jsonrpc = "2.0"; - req.id = epee::serialization::storage_entry(0); - req.method = "get_txpool_backlog"; - bool r = net_utils::invoke_http_json("/json_rpc", req, res, m_http_client, rpc_timeout); + bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_txpool_backlog", req, res, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "Failed to connect to daemon"); - THROW_WALLET_EXCEPTION_IF(res.result.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_txpool_backlog"); - THROW_WALLET_EXCEPTION_IF(res.result.status != CORE_RPC_STATUS_OK, error::get_tx_pool_error); + THROW_WALLET_EXCEPTION_IF(res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_txpool_backlog"); + THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::get_tx_pool_error); - epee::json_rpc::request<cryptonote::COMMAND_RPC_GET_INFO::request> req_t = AUTO_VAL_INIT(req_t); - epee::json_rpc::response<cryptonote::COMMAND_RPC_GET_INFO::response, std::string> resp_t = AUTO_VAL_INIT(resp_t); + cryptonote::COMMAND_RPC_GET_INFO::request req_t = AUTO_VAL_INIT(req_t); + cryptonote::COMMAND_RPC_GET_INFO::response resp_t = AUTO_VAL_INIT(resp_t); m_daemon_rpc_mutex.lock(); - req_t.jsonrpc = "2.0"; - req_t.id = epee::serialization::storage_entry(0); - req_t.method = "get_info"; - r = net_utils::invoke_http_json("/json_rpc", req_t, resp_t, m_http_client); + r = net_utils::invoke_http_json_rpc("/json_rpc", "get_info", req_t, resp_t, m_http_client); m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "get_info"); - THROW_WALLET_EXCEPTION_IF(resp_t.result.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_info"); - THROW_WALLET_EXCEPTION_IF(resp_t.result.status != CORE_RPC_STATUS_OK, error::get_tx_pool_error); - uint64_t full_reward_zone = resp_t.result.block_size_limit / 2; + THROW_WALLET_EXCEPTION_IF(resp_t.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_info"); + THROW_WALLET_EXCEPTION_IF(resp_t.status != CORE_RPC_STATUS_OK, error::get_tx_pool_error); + uint64_t full_reward_zone = resp_t.block_size_limit / 2; std::vector<std::pair<uint64_t, uint64_t>> blocks; for (const auto &fee_level: fee_levels) @@ -9770,7 +10283,7 @@ std::vector<std::pair<uint64_t, uint64_t>> wallet2::estimate_backlog(const std:: const double our_fee_byte_min = fee_level.first; const double our_fee_byte_max = fee_level.second; uint64_t priority_size_min = 0, priority_size_max = 0; - for (const auto &i: res.result.backlog) + for (const auto &i: res.backlog) { if (i.blob_size == 0) { diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 9accc65ca..46f1ddc2d 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -66,6 +66,8 @@ class Serialization_portability_wallet_Test; namespace tools { + class ringdb; + class i_wallet2_callback { public: @@ -166,6 +168,7 @@ namespace tools static bool verify_password(const std::string& keys_file_name, const epee::wipeable_string& password, bool no_spend_key, hw::device &hwdev); wallet2(cryptonote::network_type nettype = cryptonote::MAINNET, bool restricted = false); + ~wallet2(); struct multisig_info { @@ -286,6 +289,7 @@ namespace tools uint64_t m_timestamp; uint32_t m_subaddr_account; // subaddress account of your wallet to be used in this transfer std::set<uint32_t> m_subaddr_indices; // set of address indices used as inputs in this transfer + std::vector<std::pair<crypto::key_image, std::vector<uint64_t>>> m_rings; // relative }; struct confirmed_transfer_details @@ -300,10 +304,11 @@ namespace tools uint64_t m_unlock_time; uint32_t m_subaddr_account; // subaddress account of your wallet to be used in this transfer std::set<uint32_t> m_subaddr_indices; // set of address indices used as inputs in this transfer + std::vector<std::pair<crypto::key_image, std::vector<uint64_t>>> m_rings; // relative confirmed_transfer_details(): m_amount_in(0), m_amount_out(0), m_change((uint64_t)-1), m_block_height(0), m_payment_id(crypto::null_hash), m_timestamp(0), m_unlock_time(0), m_subaddr_account((uint32_t)-1) {} confirmed_transfer_details(const unconfirmed_transfer_details &utd, uint64_t height): - m_amount_in(utd.m_amount_in), m_amount_out(utd.m_amount_out), m_change(utd.m_change), m_block_height(height), m_dests(utd.m_dests), m_payment_id(utd.m_payment_id), m_timestamp(utd.m_timestamp), m_unlock_time(utd.m_tx.unlock_time), m_subaddr_account(utd.m_subaddr_account), m_subaddr_indices(utd.m_subaddr_indices) {} + m_amount_in(utd.m_amount_in), m_amount_out(utd.m_amount_out), m_change(utd.m_change), m_block_height(height), m_dests(utd.m_dests), m_payment_id(utd.m_payment_id), m_timestamp(utd.m_timestamp), m_unlock_time(utd.m_tx.unlock_time), m_subaddr_account(utd.m_subaddr_account), m_subaddr_indices(utd.m_subaddr_indices), m_rings(utd.m_rings) {} }; struct tx_construction_data @@ -810,6 +815,9 @@ namespace tools if(ver < 23) return; a & m_account_tags; + if(ver < 24) + return; + a & m_ring_history_saved; } /*! @@ -859,6 +867,10 @@ namespace tools void confirm_export_overwrite(bool always) { m_confirm_export_overwrite = always; } bool auto_low_priority() const { return m_auto_low_priority; } void auto_low_priority(bool value) { m_auto_low_priority = value; } + bool segregate_pre_fork_outputs() const { return m_segregate_pre_fork_outputs; } + void segregate_pre_fork_outputs(bool value) { m_segregate_pre_fork_outputs = value; } + bool key_reuse_mitigation2() const { return m_key_reuse_mitigation2; } + void key_reuse_mitigation2(bool value) { m_key_reuse_mitigation2 = value; } bool get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys) const; void check_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, const cryptonote::account_public_address &address, uint64_t &received, bool &in_pool, uint64_t &confirmations); @@ -1027,6 +1039,37 @@ namespace tools crypto::public_key get_multisig_signing_public_key(size_t idx) const; crypto::public_key get_multisig_signing_public_key(const crypto::secret_key &skey) const; + template<class t_request, class t_response> + inline bool invoke_http_json(const boost::string_ref uri, const t_request& req, t_response& res, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref http_method = "GET") + { + boost::lock_guard<boost::mutex> lock(m_daemon_rpc_mutex); + return epee::net_utils::invoke_http_json(uri, req, res, m_http_client, timeout, http_method); + } + template<class t_request, class t_response> + inline bool invoke_http_bin(const boost::string_ref uri, const t_request& req, t_response& res, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref http_method = "GET") + { + boost::lock_guard<boost::mutex> lock(m_daemon_rpc_mutex); + return epee::net_utils::invoke_http_bin(uri, req, res, m_http_client, timeout, http_method); + } + template<class t_request, class t_response> + inline bool invoke_http_json_rpc(const boost::string_ref uri, const std::string& method_name, const t_request& req, t_response& res, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref http_method = "GET", const std::string& req_id = "0") + { + boost::lock_guard<boost::mutex> lock(m_daemon_rpc_mutex); + return epee::net_utils::invoke_http_json_rpc(uri, method_name, req, res, m_http_client, timeout, http_method, req_id); + } + + void set_ring_database(const std::string &filename); + const std::string get_ring_database() const { return m_ring_database; } + bool get_ring(const crypto::key_image &key_image, std::vector<uint64_t> &outs); + bool get_rings(const crypto::hash &txid, std::vector<std::pair<crypto::key_image, std::vector<uint64_t>>> &outs); + bool set_ring(const crypto::key_image &key_image, const std::vector<uint64_t> &outs, bool relative); + bool find_and_save_rings(bool force = true); + + bool blackball_output(const crypto::public_key &output); + bool set_blackballed_outputs(const std::vector<crypto::public_key> &outputs, bool add = false); + bool unblackball_output(const crypto::public_key &output); + bool is_output_blackballed(const crypto::public_key &output) const; + private: /*! * \brief Stores wallet information to wallet file. @@ -1083,6 +1126,12 @@ namespace tools rct::multisig_kLRki get_multisig_kLRki(size_t n, const rct::key &k) const; rct::key get_multisig_k(size_t idx, const std::unordered_set<rct::key> &used_L) const; void update_multisig_rescan_info(const std::vector<std::vector<rct::key>> &multisig_k, const std::vector<std::vector<tools::wallet2::multisig_info>> &info, size_t n); + bool add_rings(const crypto::chacha_key &key, const cryptonote::transaction_prefix &tx); + bool add_rings(const cryptonote::transaction_prefix &tx); + bool remove_rings(const cryptonote::transaction_prefix &tx); + bool get_ring(const crypto::chacha_key &key, const crypto::key_image &key_image, std::vector<uint64_t> &outs); + + bool get_output_distribution(uint64_t &start_height, std::vector<uint64_t> &distribution); cryptonote::account_base m_account; boost::optional<epee::net_utils::http::login> m_daemon_login; @@ -1148,6 +1197,8 @@ namespace tools uint32_t m_confirm_backlog_threshold; bool m_confirm_export_overwrite; bool m_auto_low_priority; + bool m_segregate_pre_fork_outputs; + bool m_key_reuse_mitigation2; bool m_is_initialized; NodeRPCProxy m_node_rpc_proxy; std::unordered_set<crypto::hash> m_scanned_pool_txs[2]; @@ -1166,17 +1217,21 @@ namespace tools std::unordered_map<crypto::hash, address_tx> m_light_wallet_address_txs; // store calculated key image for faster lookup std::unordered_map<crypto::public_key, std::map<uint64_t, crypto::key_image> > m_key_image_cache; + + std::string m_ring_database; + bool m_ring_history_saved; + std::unique_ptr<ringdb> m_ringdb; }; } -BOOST_CLASS_VERSION(tools::wallet2, 23) +BOOST_CLASS_VERSION(tools::wallet2, 24) BOOST_CLASS_VERSION(tools::wallet2::transfer_details, 9) BOOST_CLASS_VERSION(tools::wallet2::multisig_info, 1) BOOST_CLASS_VERSION(tools::wallet2::multisig_info::LR, 0) BOOST_CLASS_VERSION(tools::wallet2::multisig_tx_set, 1) BOOST_CLASS_VERSION(tools::wallet2::payment_details, 3) BOOST_CLASS_VERSION(tools::wallet2::pool_payment_details, 1) -BOOST_CLASS_VERSION(tools::wallet2::unconfirmed_transfer_details, 7) -BOOST_CLASS_VERSION(tools::wallet2::confirmed_transfer_details, 5) +BOOST_CLASS_VERSION(tools::wallet2::unconfirmed_transfer_details, 8) +BOOST_CLASS_VERSION(tools::wallet2::confirmed_transfer_details, 6) BOOST_CLASS_VERSION(tools::wallet2::address_book_row, 17) BOOST_CLASS_VERSION(tools::wallet2::reserve_proof_entry, 0) BOOST_CLASS_VERSION(tools::wallet2::unsigned_tx_set, 0) @@ -1376,6 +1431,9 @@ namespace boost } a & x.m_subaddr_account; a & x.m_subaddr_indices; + if (ver < 8) + return; + a & x.m_rings; } template <class Archive> @@ -1420,6 +1478,9 @@ namespace boost } a & x.m_subaddr_account; a & x.m_subaddr_indices; + if (ver < 6) + return; + a & x.m_rings; } template <class Archive> diff --git a/src/wallet/wallet_errors.h b/src/wallet/wallet_errors.h index 32a0231b1..214d51cde 100644 --- a/src/wallet/wallet_errors.h +++ b/src/wallet/wallet_errors.h @@ -36,6 +36,7 @@ #include <vector> #include "cryptonote_basic/cryptonote_format_utils.h" +#include "cryptonote_core/cryptonote_tx_utils.h" #include "rpc/core_rpc_server_commands_defs.h" #include "include_base_utils.h" @@ -85,6 +86,7 @@ namespace tools // no_connection_to_daemon // is_key_image_spent_error // get_histogram_error + // get_output_distribution // wallet_files_doesnt_correspond // // * - class with protected ctor @@ -389,7 +391,7 @@ namespace tools struct get_tx_pool_error : public refresh_error { explicit get_tx_pool_error(std::string&& loc) - : refresh_error(std::move(loc), "error getting tranaction pool") + : refresh_error(std::move(loc), "error getting transaction pool") { } @@ -757,6 +759,14 @@ namespace tools } }; //---------------------------------------------------------------------------------------------------- + struct get_output_distribution : public wallet_rpc_error + { + explicit get_output_distribution(std::string&& loc, const std::string& request) + : wallet_rpc_error(std::move(loc), "failed to get output distribution", request) + { + } + }; + //---------------------------------------------------------------------------------------------------- struct wallet_files_doesnt_correspond : public wallet_logic_error { explicit wallet_files_doesnt_correspond(std::string&& loc, const std::string& keys_file, const std::string& wallet_file) diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index 220dbbc58..a9d211532 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -229,8 +229,6 @@ namespace tools assert(bool(http_login)); } // end auth enabled - m_http_client.set_server(walvars->get_daemon_address(), walvars->get_daemon_login()); - m_net_server.set_threads_prefix("RPC"); auto rng = [](size_t len, uint8_t *ptr) { return crypto::rand(len, ptr); }; return epee::http_server_impl_base<wallet_rpc_server, connection_context>::init( @@ -2257,7 +2255,7 @@ namespace tools daemon_req.ignore_battery = req.ignore_battery; cryptonote::COMMAND_RPC_START_MINING::response daemon_res; - bool r = net_utils::invoke_http_json("/start_mining", daemon_req, daemon_res, m_http_client); + bool r = m_wallet->invoke_http_json("/start_mining", daemon_req, daemon_res); if (!r || daemon_res.status != CORE_RPC_STATUS_OK) { er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; @@ -2271,7 +2269,7 @@ namespace tools { cryptonote::COMMAND_RPC_STOP_MINING::request daemon_req; cryptonote::COMMAND_RPC_STOP_MINING::response daemon_res; - bool r = net_utils::invoke_http_json("/stop_mining", daemon_req, daemon_res, m_http_client); + bool r = m_wallet->invoke_http_json("/stop_mining", daemon_req, daemon_res); if (!r || daemon_res.status != CORE_RPC_STATUS_OK) { er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; @@ -2351,7 +2349,7 @@ namespace tools cryptonote::COMMAND_RPC_GET_HEIGHT::request hreq; cryptonote::COMMAND_RPC_GET_HEIGHT::response hres; hres.height = 0; - bool r = net_utils::invoke_http_json("/getheight", hreq, hres, m_http_client); + bool r = wal->invoke_http_json("/getheight", hreq, hres); wal->set_refresh_from_block_height(hres.height); crypto::secret_key dummy_key; try { @@ -2686,7 +2684,7 @@ namespace tools } else { - er.message = "Success, but cannot update spent status after import multisig info as dameon is untrusted"; + er.message = "Success, but cannot update spent status after import multisig info as daemon is untrusted"; } return true; diff --git a/src/wallet/wallet_rpc_server.h b/src/wallet/wallet_rpc_server.h index bb458aa99..63e886dda 100644 --- a/src/wallet/wallet_rpc_server.h +++ b/src/wallet/wallet_rpc_server.h @@ -224,7 +224,6 @@ namespace tools tools::private_file rpc_login_file; std::atomic<bool> m_stop; bool m_trusted_daemon; - epee::net_utils::http::http_simple_client m_http_client; const boost::program_options::variables_map *m_vm; }; } |