diff options
27 files changed, 2609 insertions, 35 deletions
diff --git a/external/db_drivers/liblmdb/CMakeLists.txt b/external/db_drivers/liblmdb/CMakeLists.txt index cb2501ee5..2e8822f54 100644 --- a/external/db_drivers/liblmdb/CMakeLists.txt +++ b/external/db_drivers/liblmdb/CMakeLists.txt @@ -50,4 +50,16 @@ if(${ARCH_WIDTH} EQUAL 32) target_compile_definitions(lmdb PUBLIC -DMDB_VL32) endif() + +# GUI/libwallet install target +if (BUILD_GUI_DEPS) + if(IOS) + set(lib_folder lib-${ARCH}) + else() + set(lib_folder lib) + endif() + install(TARGETS lmdb + ARCHIVE DESTINATION ${lib_folder} + LIBRARY DESTINATION ${lib_folder}) +endif() set_property(TARGET lmdb APPEND PROPERTY COMPILE_FLAGS "-fPIC") diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index cce288793..d7230f644 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -1448,7 +1448,9 @@ public: * * @return false if the function returns false for any output, otherwise true */ - virtual bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, size_t tx_idx)> f) const = 0; + virtual bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, uint64_t height, size_t tx_idx)> f) const = 0; + virtual bool for_all_outputs(uint64_t amount, const std::function<bool(uint64_t height)> &f) const = 0; + // diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index f9c829013..9a755fb0c 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -2564,7 +2564,7 @@ bool BlockchainLMDB::for_all_transactions(std::function<bool(const crypto::hash& return fret; } -bool BlockchainLMDB::for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, size_t tx_idx)> f) const +bool BlockchainLMDB::for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, uint64_t height, size_t tx_idx)> f) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -2588,7 +2588,47 @@ bool BlockchainLMDB::for_all_outputs(std::function<bool(uint64_t amount, const c uint64_t amount = *(const uint64_t*)k.mv_data; outkey *ok = (outkey *)v.mv_data; tx_out_index toi = get_output_tx_and_index_from_global(ok->output_id); - if (!f(amount, toi.first, toi.second)) { + if (!f(amount, toi.first, ok->data.height, toi.second)) { + fret = false; + break; + } + } + + TXN_POSTFIX_RDONLY(); + + return fret; +} + +bool BlockchainLMDB::for_all_outputs(uint64_t amount, const std::function<bool(uint64_t height)> &f) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + RCURSOR(output_amounts); + + MDB_val_set(k, amount); + MDB_val v; + bool fret = true; + + MDB_cursor_op op = MDB_SET; + while (1) + { + int ret = mdb_cursor_get(m_cur_output_amounts, &k, &v, op); + op = MDB_NEXT_DUP; + if (ret == MDB_NOTFOUND) + break; + if (ret) + throw0(DB_ERROR("Failed to enumerate outputs")); + uint64_t out_amount = *(const uint64_t*)k.mv_data; + if (amount != out_amount) + { + MERROR("Amount is not the expected amount"); + fret = false; + break; + } + const outkey *ok = (const outkey *)v.mv_data; + if (!f(ok->data.height)) { fret = false; break; } diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 1b76abf35..0aa4bb86e 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -255,7 +255,8 @@ public: virtual bool for_all_key_images(std::function<bool(const crypto::key_image&)>) const; virtual bool for_blocks_range(const uint64_t& h1, const uint64_t& h2, std::function<bool(uint64_t, const crypto::hash&, const cryptonote::block&)>) const; virtual bool for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)>) const; - virtual bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, size_t tx_idx)> f) const; + virtual bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, uint64_t height, size_t tx_idx)> f) const; + virtual bool for_all_outputs(uint64_t amount, const std::function<bool(uint64_t height)> &f) const; virtual uint64_t add_block( const block& blk , const size_t& block_size diff --git a/src/blockchain_utilities/CMakeLists.txt b/src/blockchain_utilities/CMakeLists.txt index a701bc605..a5dd69556 100644 --- a/src/blockchain_utilities/CMakeLists.txt +++ b/src/blockchain_utilities/CMakeLists.txt @@ -67,6 +67,30 @@ monero_private_headers(blockchain_export ${blockchain_export_private_headers}) +set(blockchain_blackball_sources + blockchain_blackball.cpp + ) + +set(blockchain_blackball_private_headers + bootstrap_file.h + blocksdat_file.h + bootstrap_serialization.h + ) + +monero_private_headers(blockchain_blackball + ${blockchain_blackball_private_headers}) + +set(blockchain_usage_sources + blockchain_usage.cpp + ) + +set(blockchain_usage_private_headers) + +monero_private_headers(blockchain_usage + ${blockchain_usage_private_headers}) + + + monero_add_executable(blockchain_import ${blockchain_import_sources} ${blockchain_import_private_headers} @@ -117,3 +141,49 @@ set_property(TARGET blockchain_export OUTPUT_NAME "monero-blockchain-export") install(TARGETS blockchain_export DESTINATION bin) +monero_add_executable(blockchain_blackball + ${blockchain_blackball_sources} + ${blockchain_blackball_private_headers}) + +target_link_libraries(blockchain_blackball + PRIVATE + wallet + cryptonote_core + blockchain_db + p2p + version + epee + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${Boost_THREAD_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES}) + +set_property(TARGET blockchain_blackball + PROPERTY + OUTPUT_NAME "monero-blockchain-blackball") +install(TARGETS blockchain_blackball DESTINATION bin) + + +monero_add_executable(blockchain_usage + ${blockchain_usage_sources} + ${blockchain_usage_private_headers}) + +target_link_libraries(blockchain_usage + PRIVATE + cryptonote_core + blockchain_db + p2p + version + epee + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${Boost_THREAD_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES}) + +set_property(TARGET blockchain_usage + PROPERTY + OUTPUT_NAME "monero-blockchain-usage") +install(TARGETS blockchain_usage DESTINATION bin) + diff --git a/src/blockchain_utilities/blockchain_blackball.cpp b/src/blockchain_utilities/blockchain_blackball.cpp new file mode 100644 index 000000000..40ce898d9 --- /dev/null +++ b/src/blockchain_utilities/blockchain_blackball.cpp @@ -0,0 +1,427 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <boost/range/adaptor/transformed.hpp> +#include <boost/algorithm/string.hpp> +#include "common/command_line.h" +#include "common/varint.h" +#include "cryptonote_core/tx_pool.h" +#include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_core/blockchain.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/db_types.h" +#include "wallet/ringdb.h" +#include "version.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "bcutil" + +namespace po = boost::program_options; +using namespace epee; +using namespace cryptonote; + +struct output_data +{ + uint64_t amount; + uint64_t index; + output_data(uint64_t a, uint64_t i): amount(a), index(i) {} + bool operator==(const output_data &other) const { return other.amount == amount && other.index == index; } +}; +namespace std +{ + template<> struct hash<output_data> + { + size_t operator()(const output_data &od) const + { + const uint64_t data[2] = {od.amount, od.index}; + crypto::hash h; + crypto::cn_fast_hash(data, 2 * sizeof(uint64_t), h); + return reinterpret_cast<const std::size_t &>(h); + } + }; +} + +static std::string get_default_db_path() +{ + boost::filesystem::path dir = tools::get_default_data_dir(); + // remove .bitmonero, replace with .shared-ringdb + dir = dir.remove_filename(); + dir /= ".shared-ringdb"; + return dir.string(); +} + +static bool for_all_transactions(const std::string &filename, const std::function<bool(const cryptonote::transaction_prefix&)> &f) +{ + MDB_env *env; + MDB_dbi dbi; + MDB_txn *txn; + MDB_cursor *cur; + int dbr; + bool tx_active = false; + + dbr = mdb_env_create(&env); + if (dbr) throw std::runtime_error("Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); + dbr = mdb_env_set_maxdbs(env, 2); + if (dbr) throw std::runtime_error("Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); + const std::string actual_filename = filename; + dbr = mdb_env_open(env, actual_filename.c_str(), 0, 0664); + if (dbr) throw std::runtime_error("Failed to open rings database file '" + + actual_filename + "': " + std::string(mdb_strerror(dbr))); + + dbr = mdb_txn_begin(env, NULL, 0, &txn); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + dbr = mdb_dbi_open(txn, "txs_pruned", MDB_INTEGERKEY, &dbi); + if (dbr) + dbr = mdb_dbi_open(txn, "txs", MDB_INTEGERKEY, &dbi); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_open(txn, dbi, &cur); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + + MDB_val k; + MDB_val v; + bool fret = true; + + MDB_cursor_op op = MDB_FIRST; + while (1) + { + int ret = mdb_cursor_get(cur, &k, &v, op); + op = MDB_NEXT; + if (ret == MDB_NOTFOUND) + break; + if (ret) + throw std::runtime_error("Failed to enumerate transactions: " + std::string(mdb_strerror(ret))); + + cryptonote::transaction_prefix tx; + blobdata bd; + bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size); + std::stringstream ss; + ss << bd; + binary_archive<false> ba(ss); + bool r = do_serialize(ba, tx); + CHECK_AND_ASSERT_MES(r, false, "Failed to parse transaction from blob"); + + if (!f(tx)) { + fret = false; + break; + } + } + + mdb_cursor_close(cur); + mdb_txn_commit(txn); + tx_active = false; + mdb_dbi_close(env, dbi); + mdb_env_close(env); + return fret; +} + +int main(int argc, char* argv[]) +{ + TRY_ENTRY(); + + epee::string_tools::set_module_name_and_folder(argv[0]); + + std::string default_db_type = "lmdb"; + + std::string available_dbs = cryptonote::blockchain_db_types(", "); + available_dbs = "available: " + available_dbs; + + uint32_t log_level = 0; + + tools::on_startup(); + + boost::filesystem::path output_file_path; + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor<std::string, false, true, 2> arg_blackball_db_dir = { + "blackball-db-dir", "Specify blackball database directory", + get_default_db_path(), + {{ &arg_testnet_on, &arg_stagenet_on }}, + [](std::array<bool, 2> testnet_stagenet, bool defaulted, std::string val) { + if (testnet_stagenet[0]) + return (boost::filesystem::path(val) / "testnet").string(); + else if (testnet_stagenet[1]) + return (boost::filesystem::path(val) / "stagenet").string(); + return val; + } + }; + const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""}; + const command_line::arg_descriptor<std::string> arg_database = { + "database", available_dbs.c_str(), default_db_type + }; + const command_line::arg_descriptor<bool> arg_rct_only = {"rct-only", "Only work on ringCT outputs", false}; + const command_line::arg_descriptor<std::vector<std::string> > arg_inputs = {"inputs", "Path to Monero DB, and path to any fork DBs"}; + + command_line::add_arg(desc_cmd_sett, arg_blackball_db_dir); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_database); + command_line::add_arg(desc_cmd_sett, arg_rct_only); + command_line::add_arg(desc_cmd_sett, arg_inputs); + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::positional_options_description positional_options; + positional_options.add(arg_inputs.name, -1); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + auto parser = po::command_line_parser(argc, argv).options(desc_options).positional(positional_options); + po::store(parser.run(), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + mlog_configure(mlog_get_default_log_path("monero-blockchain-blackball.log"), true); + if (!command_line::is_arg_defaulted(vm, arg_log_level)) + mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); + else + mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str()); + + LOG_PRINT_L0("Starting..."); + + bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); + bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); + network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; + output_file_path = command_line::get_arg(vm, arg_blackball_db_dir); + bool opt_rct_only = command_line::get_arg(vm, arg_rct_only); + + std::string db_type = command_line::get_arg(vm, arg_database); + if (!cryptonote::blockchain_valid_db_type(db_type)) + { + std::cerr << "Invalid database type: " << db_type << std::endl; + return 1; + } + + // If we wanted to use the memory pool, we would set up a fake_core. + + // Use Blockchain instead of lower-level BlockchainDB for two reasons: + // 1. Blockchain has the init() method for easy setup + // 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash() + // + // cannot match blockchain_storage setup above with just one line, + // e.g. + // Blockchain* core_storage = new Blockchain(NULL); + // because unlike blockchain_storage constructor, which takes a pointer to + // tx_memory_pool, Blockchain's constructor takes tx_memory_pool object. + LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); + const std::vector<std::string> inputs = command_line::get_arg(vm, arg_inputs); + if (inputs.empty()) + { + LOG_PRINT_L0("No inputs given"); + return 1; + } + std::vector<std::unique_ptr<Blockchain>> core_storage(inputs.size()); + tx_memory_pool m_mempool(*(Blockchain*)NULL); + for (size_t n = 0; n < inputs.size(); ++n) + { + core_storage[n].reset(new Blockchain(m_mempool)); + + BlockchainDB* db = new_db(db_type); + if (db == NULL) + { + LOG_ERROR("Attempted to use non-existent database type: " << db_type); + throw std::runtime_error("Attempting to use non-existent database type"); + } + LOG_PRINT_L0("database: " << db_type); + + std::string filename = inputs[n]; + while (boost::ends_with(filename, "/") || boost::ends_with(filename, "\\")) + filename.pop_back(); + LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); + + try + { + db->open(filename, DBF_RDONLY); + } + catch (const std::exception& e) + { + LOG_PRINT_L0("Error opening database: " << e.what()); + return 1; + } + r = core_storage[n]->init(db, net_type); + + CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage"); + LOG_PRINT_L0("Source blockchain storage initialized OK"); + } + + boost::filesystem::path direc(output_file_path.string()); + if (boost::filesystem::exists(direc)) + { + if (!boost::filesystem::is_directory(direc)) + { + MERROR("LMDB needs a directory path, but a file was passed: " << output_file_path.string()); + return 1; + } + } + else + { + if (!boost::filesystem::create_directories(direc)) + { + MERROR("Failed to create directory: " << output_file_path.string()); + return 1; + } + } + + LOG_PRINT_L0("Scanning for blackballable outputs..."); + + size_t done = 0; + std::unordered_map<crypto::key_image, std::vector<uint64_t>> relative_rings; + std::unordered_map<output_data, std::unordered_set<crypto::key_image>> outputs; + std::unordered_set<output_data> spent, newly_spent; + + cryptonote::block b = core_storage[0]->get_db().get_block_from_height(0); + tools::ringdb ringdb(output_file_path.string(), epee::string_tools::pod_to_hex(get_block_hash(b))); + + for (size_t n = 0; n < inputs.size(); ++n) + { + LOG_PRINT_L0("Reading blockchain from " << inputs[n]); + for_all_transactions(inputs[n], [&](const cryptonote::transaction_prefix &tx)->bool + { + for (const auto &in: tx.vin) + { + if (in.type() != typeid(txin_to_key)) + continue; + const auto &txin = boost::get<txin_to_key>(in); + if (opt_rct_only && txin.amount != 0) + continue; + + const std::vector<uint64_t> absolute = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets); + if (n == 0) + for (uint64_t out: absolute) + outputs[output_data(txin.amount, out)].insert(txin.k_image); + + std::vector<uint64_t> new_ring = txin.key_offsets; + const uint32_t ring_size = txin.key_offsets.size(); + if (ring_size == 1) + { + const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, txin.key_offsets[0]); + MINFO("Blackballing output " << pkey << ", due to being used in a 1-ring"); + ringdb.blackball(pkey); + newly_spent.insert(output_data(txin.amount, txin.key_offsets[0])); + spent.insert(output_data(txin.amount, txin.key_offsets[0])); + } + else if (relative_rings.find(txin.k_image) != relative_rings.end()) + { + MINFO("Key image " << txin.k_image << " already seen: rings " << + boost::join(relative_rings[txin.k_image] | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " ") << + ", " << boost::join(txin.key_offsets | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " ")); + if (relative_rings[txin.k_image] != txin.key_offsets) + { + MINFO("Rings are different"); + const std::vector<uint64_t> r0 = cryptonote::relative_output_offsets_to_absolute(relative_rings[txin.k_image]); + const std::vector<uint64_t> r1 = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets); + std::vector<uint64_t> common; + for (uint64_t out: r0) + { + if (std::find(r1.begin(), r1.end(), out) != r1.end()) + common.push_back(out); + } + if (common.empty()) + { + MERROR("Rings for the same key image are disjoint"); + } + else if (common.size() == 1) + { + const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, common[0]); + MINFO("Blackballing output " << pkey << ", due to being used in rings with a single common element"); + ringdb.blackball(pkey); + newly_spent.insert(output_data(txin.amount, common[0])); + spent.insert(output_data(txin.amount, common[0])); + } + else + { + MINFO("The intersection has more than one element, it's still ok"); + for (const auto &out: r0) + if (std::find(common.begin(), common.end(), out) != common.end()) + new_ring.push_back(out); + new_ring = cryptonote::absolute_output_offsets_to_relative(new_ring); + } + } + } + relative_rings[txin.k_image] = new_ring; + } + return true; + }); + } + + while (!newly_spent.empty()) + { + LOG_PRINT_L0("Secondary pass due to " << newly_spent.size() << " newly found spent outputs"); + std::unordered_set<output_data> work_spent = std::move(newly_spent); + newly_spent.clear(); + + for (const output_data &od: work_spent) + { + for (const crypto::key_image &ki: outputs[od]) + { + std::vector<uint64_t> absolute = cryptonote::relative_output_offsets_to_absolute(relative_rings[ki]); + size_t known = 0; + uint64_t last_unknown = 0; + for (uint64_t out: absolute) + { + output_data new_od(od.amount, out); + if (spent.find(new_od) != spent.end()) + ++known; + else + last_unknown = out; + } + if (known == absolute.size() - 1) + { + const crypto::public_key pkey = core_storage[0]->get_output_key(od.amount, last_unknown); + MINFO("Blackballing output " << pkey << ", due to being used in a " << + absolute.size() << "-ring where all other outputs are known to be spent"); + ringdb.blackball(pkey); + newly_spent.insert(output_data(od.amount, last_unknown)); + spent.insert(output_data(od.amount, last_unknown)); + } + } + } + } + + LOG_PRINT_L0("Blockchain blackball data exported OK"); + return 0; + + CATCH_ENTRY("Export error", 1); +} diff --git a/src/blockchain_utilities/blockchain_usage.cpp b/src/blockchain_utilities/blockchain_usage.cpp new file mode 100644 index 000000000..7c3c83167 --- /dev/null +++ b/src/blockchain_utilities/blockchain_usage.cpp @@ -0,0 +1,256 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <boost/range/adaptor/transformed.hpp> +#include <boost/algorithm/string.hpp> +#include "common/command_line.h" +#include "common/varint.h" +#include "cryptonote_core/tx_pool.h" +#include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_core/blockchain.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/db_types.h" +#include "version.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "bcutil" + +namespace po = boost::program_options; +using namespace epee; +using namespace cryptonote; + +struct output_data +{ + uint64_t amount; + uint64_t index; + mutable bool coinbase; + mutable uint64_t height; + output_data(uint64_t a, uint64_t i, bool cb, uint64_t h): amount(a), index(i), coinbase(cb), height(h) {} + bool operator==(const output_data &other) const { return other.amount == amount && other.index == index; } + void info(bool c, uint64_t h) const { coinbase = c; height =h; } +}; +namespace std +{ + template<> struct hash<output_data> + { + size_t operator()(const output_data &od) const + { + const uint64_t data[2] = {od.amount, od.index}; + crypto::hash h; + crypto::cn_fast_hash(data, 2 * sizeof(uint64_t), h); + return reinterpret_cast<const std::size_t &>(h); + } + }; +} + +struct reference +{ + uint64_t height; + uint64_t ring_size; + uint64_t position; + reference(uint64_t h, uint64_t rs, uint64_t p): height(h), ring_size(rs), position(p) {} +}; + +int main(int argc, char* argv[]) +{ + TRY_ENTRY(); + + epee::string_tools::set_module_name_and_folder(argv[0]); + + std::string default_db_type = "lmdb"; + + std::string available_dbs = cryptonote::blockchain_db_types(", "); + available_dbs = "available: " + available_dbs; + + uint32_t log_level = 0; + + tools::on_startup(); + + boost::filesystem::path output_file_path; + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""}; + const command_line::arg_descriptor<std::string> arg_database = { + "database", available_dbs.c_str(), default_db_type + }; + const command_line::arg_descriptor<bool> arg_rct_only = {"rct-only", "Only work on ringCT outputs", false}; + const command_line::arg_descriptor<std::string> arg_input = {"input", ""}; + + command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_database); + command_line::add_arg(desc_cmd_sett, arg_rct_only); + command_line::add_arg(desc_cmd_sett, arg_input); + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::positional_options_description positional_options; + positional_options.add(arg_input.name, -1); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + auto parser = po::command_line_parser(argc, argv).options(desc_options).positional(positional_options); + po::store(parser.run(), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + mlog_configure(mlog_get_default_log_path("monero-blockchain-usage.log"), true); + if (!command_line::is_arg_defaulted(vm, arg_log_level)) + mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); + else + mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str()); + + LOG_PRINT_L0("Starting..."); + + bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); + bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); + network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; + bool opt_rct_only = command_line::get_arg(vm, arg_rct_only); + + std::string db_type = command_line::get_arg(vm, arg_database); + if (!cryptonote::blockchain_valid_db_type(db_type)) + { + std::cerr << "Invalid database type: " << db_type << std::endl; + return 1; + } + + // If we wanted to use the memory pool, we would set up a fake_core. + + // Use Blockchain instead of lower-level BlockchainDB for two reasons: + // 1. Blockchain has the init() method for easy setup + // 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash() + // + // cannot match blockchain_storage setup above with just one line, + // e.g. + // Blockchain* core_storage = new Blockchain(NULL); + // because unlike blockchain_storage constructor, which takes a pointer to + // tx_memory_pool, Blockchain's constructor takes tx_memory_pool object. + LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); + const std::string input = command_line::get_arg(vm, arg_input); + std::unique_ptr<Blockchain> core_storage; + tx_memory_pool m_mempool(*core_storage); + core_storage.reset(new Blockchain(m_mempool)); + BlockchainDB* db = new_db(db_type); + if (db == NULL) + { + LOG_ERROR("Attempted to use non-existent database type: " << db_type); + throw std::runtime_error("Attempting to use non-existent database type"); + } + LOG_PRINT_L0("database: " << db_type); + + const std::string filename = input; + LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); + + try + { + db->open(filename, DBF_RDONLY); + } + catch (const std::exception& e) + { + LOG_PRINT_L0("Error opening database: " << e.what()); + return 1; + } + r = core_storage->init(db, net_type); + + CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage"); + LOG_PRINT_L0("Source blockchain storage initialized OK"); + + LOG_PRINT_L0("Building usage patterns..."); + + size_t done = 0; + std::unordered_map<output_data, std::list<reference>> outputs; + std::unordered_map<uint64_t,uint64_t> indices; + + LOG_PRINT_L0("Reading blockchain from " << input); + core_storage->for_all_transactions([&](const crypto::hash &hash, const cryptonote::transaction &tx)->bool + { + const bool coinbase = tx.vin.size() == 1 && tx.vin[0].type() == typeid(txin_gen); + const uint64_t height = core_storage->get_db().get_tx_block_height(hash); + + // create new outputs + for (const auto &out: tx.vout) + { + if (opt_rct_only && out.amount) + continue; + uint64_t index = indices[out.amount]++; + output_data od(out.amount, indices[out.amount], coinbase, height); + auto itb = outputs.emplace(od, std::list<reference>()); + itb.first->first.info(coinbase, height); + } + + for (const auto &in: tx.vin) + { + if (in.type() != typeid(txin_to_key)) + continue; + const auto &txin = boost::get<txin_to_key>(in); + if (opt_rct_only && txin.amount != 0) + continue; + + const std::vector<uint64_t> absolute = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets); + for (size_t n = 0; n < txin.key_offsets.size(); ++n) + { + output_data od(txin.amount, absolute[n], coinbase, height); + outputs[od].push_back(reference(height, txin.key_offsets.size(), n)); + } + } + return true; + }); + + std::unordered_map<uint64_t, uint64_t> counts; + size_t total = 0; + for (const auto &out: outputs) + { + counts[out.second.size()]++; + total++; + } + for (const auto &c: counts) + { + float percent = 100.f * c.second / total; + MINFO(std::to_string(c.second) << " outputs used " << c.first << " times (" << percent << "%)"); + } + + LOG_PRINT_L0("Blockchain usage exported OK"); + return 0; + + CATCH_ENTRY("Export error", 1); +} diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index a0031559b..0a6f1cdcd 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -1903,6 +1903,45 @@ void Blockchain::get_output_key_mask_unlocked(const uint64_t& amount, const uint unlocked = is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first)); } //------------------------------------------------------------------ +bool Blockchain::get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) const +{ + // rct outputs don't exist before v3 + if (amount == 0) + { + switch (m_nettype) + { + case STAGENET: start_height = stagenet_hard_forks[2].height; break; + case TESTNET: start_height = testnet_hard_forks[2].height; break; + case MAINNET: start_height = mainnet_hard_forks[2].height; break; + default: return false; + } + } + else + start_height = 0; + base = 0; + + const uint64_t real_start_height = start_height; + if (from_height > start_height) + start_height = from_height; + + distribution.clear(); + uint64_t db_height = m_db->height(); + if (start_height >= db_height) + return false; + distribution.resize(db_height - start_height, 0); + bool r = for_all_outputs(amount, [&](uint64_t height) { + CHECK_AND_ASSERT_MES(height >= real_start_height && height <= db_height, false, "Height not in expected range"); + if (height >= start_height) + distribution[height - start_height]++; + else + base++; + return true; + }); + if (!r) + return false; + return true; +} +//------------------------------------------------------------------ // This function takes a list of block hashes from another node // on the network to find where the split point is between us and them. // This is used to see what to send another node that needs to sync. @@ -4441,11 +4480,16 @@ bool Blockchain::for_all_transactions(std::function<bool(const crypto::hash&, co return m_db->for_all_transactions(f); } -bool Blockchain::for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, size_t tx_idx)> f) const +bool Blockchain::for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, uint64_t height, size_t tx_idx)> f) const { return m_db->for_all_outputs(f);; } +bool Blockchain::for_all_outputs(uint64_t amount, std::function<bool(uint64_t height)> f) const +{ + return m_db->for_all_outputs(amount, f);; +} + namespace cryptonote { template bool Blockchain::get_transactions(const std::vector<crypto::hash>&, std::list<transaction>&, std::list<crypto::hash>&) const; } diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index 1a52e20bf..4423199de 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -524,6 +524,17 @@ namespace cryptonote bool get_random_rct_outs(const COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::request& req, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::response& res) const; /** + * @brief gets per block distribution of outputs of a given amount + * + * @param amount the amount to get a ditribution for + * @param return-by-reference from_height the height before which we do not care about the data + * @param return-by-reference start_height the height of the first rct output + * @param return-by-reference distribution the start offset of the first rct output in this block (same as previous if none) + * @param return-by-reference base how many outputs of that amount are before the stated distribution + */ + bool get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) const; + + /** * @brief gets the global indices for outputs from a given transaction * * This function gets the global indices for all outputs belonging @@ -857,7 +868,17 @@ namespace cryptonote * * @return false if any output fails the check, otherwise true */ - bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, size_t tx_idx)>) const; + bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, uint64_t height, size_t tx_idx)>) const; + + /** + * @brief perform a check on all outputs of a given amount in the blockchain + * + * @param amount the amount to iterate through + * @param std::function the check to perform, pass/fail + * + * @return false if any output fails the check, otherwise true + */ + bool for_all_outputs(uint64_t amount, std::function<bool(uint64_t height)>) const; /** * @brief get a reference to the BlockchainDB in use by Blockchain diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 8c0118803..a91cc9638 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -1105,6 +1105,11 @@ namespace cryptonote return m_blockchain_storage.get_random_rct_outs(req, res); } //----------------------------------------------------------------------------------------------- + bool core::get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) const + { + return m_blockchain_storage.get_output_distribution(amount, from_height, start_height, distribution, base); + } + //----------------------------------------------------------------------------------------------- bool core::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<uint64_t>& indexs) const { return m_blockchain_storage.get_tx_outputs_gindexs(tx_id, indexs); diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index e1e430516..abf79be1d 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -571,6 +571,12 @@ namespace cryptonote */ bool get_random_rct_outs(const COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::request& req, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::response& res) const; + /** + * @copydoc Blockchain::get_output_distribution + * + * @brief get per block distribution of outputs of a given amount + */ + bool get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) const; /** * @copydoc miner::pause diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 0e3e6757b..f649a0ca5 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -2076,6 +2076,41 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_get_output_distribution(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res, epee::json_rpc::error& error_resp) + { + try + { + for (uint64_t amount: req.amounts) + { + std::vector<uint64_t> distribution; + uint64_t start_height, base; + if (!m_core.get_output_distribution(amount, req.from_height, start_height, distribution, base)) + { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = "Failed to get rct distribution"; + return false; + } + if (req.cumulative) + { + distribution[0] += base; + for (size_t n = 1; n < distribution.size(); ++n) + distribution[n] += distribution[n-1]; + } + res.distributions.push_back({amount, start_height, std::move(distribution), base}); + } + } + catch (const std::exception &e) + { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = "Failed to get output distribution"; + return false; + } + + res.status = CORE_RPC_STATUS_OK; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + const command_line::arg_descriptor<std::string, false, true, 2> core_rpc_server::arg_rpc_bind_port = { "rpc-bind-port" diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index 4754a5d5f..a5755e062 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -153,6 +153,7 @@ namespace cryptonote MAP_JON_RPC_WE_IF("relay_tx", on_relay_tx, COMMAND_RPC_RELAY_TX, !m_restricted) MAP_JON_RPC_WE_IF("sync_info", on_sync_info, COMMAND_RPC_SYNC_INFO, !m_restricted) MAP_JON_RPC_WE("get_txpool_backlog", on_get_txpool_backlog, COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG) + MAP_JON_RPC_WE_IF("get_output_distribution", on_get_output_distribution, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION, !m_restricted) END_JSON_RPC_MAP() END_URI_MAP2() @@ -214,6 +215,7 @@ namespace cryptonote bool on_relay_tx(const COMMAND_RPC_RELAY_TX::request& req, COMMAND_RPC_RELAY_TX::response& res, epee::json_rpc::error& error_resp); bool on_sync_info(const COMMAND_RPC_SYNC_INFO::request& req, COMMAND_RPC_SYNC_INFO::response& res, epee::json_rpc::error& error_resp); bool on_get_txpool_backlog(const COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::response& res, epee::json_rpc::error& error_resp); + bool on_get_output_distribution(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res, epee::json_rpc::error& error_resp); //----------------------- private: diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index a2c780376..660fb7889 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -49,7 +49,7 @@ namespace cryptonote // advance which version they will stop working with // Don't go over 32767 for any of these #define CORE_RPC_VERSION_MAJOR 1 -#define CORE_RPC_VERSION_MINOR 18 +#define CORE_RPC_VERSION_MINOR 19 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR) @@ -2203,4 +2203,47 @@ namespace cryptonote END_KV_SERIALIZE_MAP() }; }; + + struct COMMAND_RPC_GET_OUTPUT_DISTRIBUTION + { + struct request + { + std::vector<uint64_t> amounts; + uint64_t from_height; + bool cumulative; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(amounts) + KV_SERIALIZE_OPT(from_height, (uint64_t)0) + KV_SERIALIZE_OPT(cumulative, false) + END_KV_SERIALIZE_MAP() + }; + + struct distribution + { + uint64_t amount; + uint64_t start_height; + std::vector<uint64_t> distribution; + uint64_t base; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(amount) + KV_SERIALIZE(start_height) + KV_SERIALIZE_CONTAINER_POD_AS_BLOB(distribution) + KV_SERIALIZE(base) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string status; + std::vector<distribution> distributions; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + KV_SERIALIZE(distributions) + END_KV_SERIALIZE_MAP() + }; + }; + } diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index 8bae1e2c9..3c257979e 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -1294,6 +1294,275 @@ bool simple_wallet::export_raw_multisig(const std::vector<std::string> &args) return true; } +bool simple_wallet::print_ring(const std::vector<std::string> &args) +{ + crypto::key_image key_image; + crypto::hash txid; + if (args.size() != 1) + { + fail_msg_writer() << tr("usage: print_ring <key_image|txid>"); + return true; + } + + if (!epee::string_tools::hex_to_pod(args[0], key_image)) + { + fail_msg_writer() << tr("Invalid key image"); + return true; + } + // this one will always work, they're all 32 byte hex + if (!epee::string_tools::hex_to_pod(args[0], txid)) + { + fail_msg_writer() << tr("Invalid txid"); + return true; + } + + std::vector<uint64_t> ring; + std::vector<std::pair<crypto::key_image, std::vector<uint64_t>>> rings; + try + { + if (m_wallet->get_ring(key_image, ring)) + rings.push_back({key_image, ring}); + else if (!m_wallet->get_rings(txid, rings)) + { + fail_msg_writer() << tr("Key image either not spent, or spent with mixin 0"); + return true; + } + + for (const auto &ring: rings) + { + std::stringstream str; + for (const auto &x: ring.second) + str << x<< " "; + // do NOT translate this "absolute" below, the lin can be used as input to set_ring + success_msg_writer() << epee::string_tools::pod_to_hex(ring.first) << " absolute " << str.str(); + } + } + catch (const std::exception &e) + { + fail_msg_writer() << tr("Failed to get key image ring: ") << e.what(); + } + + return true; +} + +bool simple_wallet::set_ring(const std::vector<std::string> &args) +{ + crypto::key_image key_image; + if (args.size() < 3) + { + fail_msg_writer() << tr("usage: set_ring <key_image> absolute|relative <index> [<index>...]"); + return true; + } + + if (!epee::string_tools::hex_to_pod(args[0], key_image)) + { + fail_msg_writer() << tr("Invalid key image"); + return true; + } + + bool relative; + if (args[1] == "absolute") + { + relative = false; + } + else if (args[1] == "relative") + { + relative = true; + } + else + { + fail_msg_writer() << tr("Missing absolute or relative keyword"); + return true; + } + + std::vector<uint64_t> ring; + for (size_t n = 2; n < args.size(); ++n) + { + ring.resize(ring.size() + 1); + if (!string_tools::get_xtype_from_string(ring.back(), args[n])) + { + fail_msg_writer() << tr("invalid index: must be a strictly positive unsigned integer"); + return true; + } + if (relative) + { + if (ring.size() > 1 && !ring.back()) + { + fail_msg_writer() << tr("invalid index: must be a strictly positive unsigned integer"); + return true; + } + uint64_t sum = 0; + for (uint64_t out: ring) + { + if (out > std::numeric_limits<uint64_t>::max() - sum) + { + fail_msg_writer() << tr("invalid index: indices wrap"); + return true; + } + sum += out; + } + } + else + { + if (ring.size() > 1 && ring[ring.size() - 2] >= ring[ring.size() - 1]) + { + fail_msg_writer() << tr("invalid index: indices should be in strictly ascending order"); + return true; + } + } + } + if (!m_wallet->set_ring(key_image, ring, relative)) + { + fail_msg_writer() << tr("failed to set ring"); + return true; + } + + return true; +} + +bool simple_wallet::blackball(const std::vector<std::string> &args) +{ + crypto::public_key output; + if (args.size() == 0) + { + fail_msg_writer() << tr("usage: blackball <output_public_key> | <filename> [add]"); + return true; + } + + try + { + if (epee::string_tools::hex_to_pod(args[0], output)) + { + m_wallet->blackball_output(output); + } + else if (epee::file_io_utils::is_file_exist(args[0])) + { + std::vector<crypto::public_key> outputs; + char str[65]; + + std::unique_ptr<FILE, tools::close_file> f(fopen(args[0].c_str(), "r")); + if (f) + { + while (!feof(f.get())) + { + if (!fgets(str, sizeof(str), f.get())) + break; + const size_t len = strlen(str); + if (len > 0 && str[len - 1] == '\n') + str[len - 1] = 0; + if (!str[0]) + continue; + outputs.push_back(crypto::public_key()); + if (!epee::string_tools::hex_to_pod(str, outputs.back())) + { + fail_msg_writer() << tr("Invalid public key: ") << str; + return true; + } + } + f.reset(); + bool add = false; + if (args.size() > 1) + { + if (args[1] != "add") + { + fail_msg_writer() << tr("Bad argument: ") + args[1] + ": " + tr("should be \"add\""); + return true; + } + add = true; + } + m_wallet->set_blackballed_outputs(outputs, add); + } + else + { + fail_msg_writer() << tr("Failed to open file"); + return true; + } + } + else + { + fail_msg_writer() << tr("Invalid public key, and file doesn't exist"); + return true; + } + } + catch (const std::exception &e) + { + fail_msg_writer() << tr("Failed to blackball output: ") << e.what(); + } + + return true; +} + +bool simple_wallet::unblackball(const std::vector<std::string> &args) +{ + crypto::public_key output; + if (args.size() != 1) + { + fail_msg_writer() << tr("usage: unblackball <output_public_key>"); + return true; + } + + if (!epee::string_tools::hex_to_pod(args[0], output)) + { + fail_msg_writer() << tr("Invalid public key"); + return true; + } + + try + { + m_wallet->unblackball_output(output); + } + catch (const std::exception &e) + { + fail_msg_writer() << tr("Failed to unblackball output: ") << e.what(); + } + + return true; +} + +bool simple_wallet::blackballed(const std::vector<std::string> &args) +{ + crypto::public_key output; + if (args.size() != 1) + { + fail_msg_writer() << tr("usage: blackballed <output_public_key>"); + return true; + } + + if (!epee::string_tools::hex_to_pod(args[0], output)) + { + fail_msg_writer() << tr("Invalid public key"); + return true; + } + + try + { + if (m_wallet->is_output_blackballed(output)) + message_writer() << tr("Blackballed: ") << output; + else + message_writer() << tr("not blackballed: ") << output; + } + catch (const std::exception &e) + { + fail_msg_writer() << tr("Failed to unblackball output: ") << e.what(); + } + + return true; +} + +bool simple_wallet::save_known_rings(const std::vector<std::string> &args) +{ + try + { + LOCK_IDLE_SCOPE(); + m_wallet->find_and_save_rings(); + } + catch (const std::exception &e) + { + fail_msg_writer() << tr("Failed to save known rings: ") << e.what(); + } + return true; +} + bool simple_wallet::set_always_confirm_transfers(const std::vector<std::string> &args/* = std::vector<std::string>()*/) { const auto pwd_container = get_and_verify_password(); @@ -1638,6 +1907,32 @@ bool simple_wallet::set_auto_low_priority(const std::vector<std::string> &args/* return true; } +bool simple_wallet::set_segregate_pre_fork_outputs(const std::vector<std::string> &args/* = std::vector<std::string>()*/) +{ + const auto pwd_container = get_and_verify_password(); + if (pwd_container) + { + parse_bool_and_use(args[1], [&](bool r) { + m_wallet->segregate_pre_fork_outputs(r); + m_wallet->rewrite(m_wallet_file, pwd_container->password()); + }); + } + return true; +} + +bool simple_wallet::set_key_reuse_mitigation2(const std::vector<std::string> &args/* = std::vector<std::string>()*/) +{ + const auto pwd_container = get_and_verify_password(); + if (pwd_container) + { + parse_bool_and_use(args[1], [&](bool r) { + m_wallet->key_reuse_mitigation2(r); + m_wallet->rewrite(m_wallet_file, pwd_container->password()); + }); + } + return true; +} + bool simple_wallet::help(const std::vector<std::string> &args/* = std::vector<std::string>()*/) { if(args.empty()) @@ -1812,7 +2107,11 @@ simple_wallet::simple_wallet() "refresh-from-block-height [n]\n " " Set the height before which to ignore blocks.\n " "auto-low-priority <1|0>\n " - " Whether to automatically use the low priority fee level when it's safe to do so.")); + " Whether to automatically use the low priority fee level when it's safe to do so.\n " + "segregate-pre-fork-outputs <1|0>\n " + " Set this if you intend to spend outputs on both Monero AND a key reusing fork.\n " + "key-reuse-mitigation2 <1|0>\n " + " Set this if you are not sure whether you will spend on a key reusing Monero fork later.")); m_cmd_binder.set_handler("encrypted_seed", boost::bind(&simple_wallet::encrypted_seed, this, _1), tr("Display the encrypted Electrum-style mnemonic seed.")); @@ -1951,6 +2250,30 @@ simple_wallet::simple_wallet() boost::bind(&simple_wallet::export_raw_multisig, this, _1), tr("export_raw_multisig_tx <filename>"), tr("Export a signed multisig transaction to a file")); + m_cmd_binder.set_handler("print_ring", + boost::bind(&simple_wallet::print_ring, this, _1), + tr("print_ring <key_image> | <txid>"), + tr("Print the ring(s) used to spend a given key image or transaction (if the ring size is > 1)")); + m_cmd_binder.set_handler("set_ring", + boost::bind(&simple_wallet::set_ring, this, _1), + tr("set_ring <key_image> absolute|relative <index> [<index>...]"), + tr("Set the ring used for a given key image, so it can be reused in a fork")); + m_cmd_binder.set_handler("save_known_rings", + boost::bind(&simple_wallet::save_known_rings, this, _1), + tr("save_known_rings"), + tr("Save known rings to the shared rings database")); + m_cmd_binder.set_handler("blackball", + boost::bind(&simple_wallet::blackball, this, _1), + tr("blackball <output public key> | <filename> [add]"), + tr("Blackball output(s) so they never get selected as fake outputs in a ring")); + m_cmd_binder.set_handler("unblackball", + boost::bind(&simple_wallet::unblackball, this, _1), + tr("unblackball <output public key>"), + tr("Unblackballs an output so it may get selected as a fake output in a ring")); + m_cmd_binder.set_handler("blackballed", + boost::bind(&simple_wallet::blackballed, this, _1), + tr("blackballed <output public key>"), + tr("Checks whether an output is blackballed")); m_cmd_binder.set_handler("help", boost::bind(&simple_wallet::help, this, _1), tr("help [<command>]"), @@ -1980,6 +2303,8 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args) success_msg_writer() << "confirm-export-overwrite = " << m_wallet->confirm_export_overwrite(); success_msg_writer() << "refresh-from-block-height = " << m_wallet->get_refresh_from_block_height(); success_msg_writer() << "auto-low-priority = " << m_wallet->auto_low_priority(); + success_msg_writer() << "segregate-pre-fork-outputs = " << m_wallet->segregate_pre_fork_outputs(); + success_msg_writer() << "key-reuse-mitigation2 = " << m_wallet->key_reuse_mitigation2(); return true; } else @@ -2030,6 +2355,8 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args) CHECK_SIMPLE_VARIABLE("confirm-export-overwrite", set_confirm_export_overwrite, tr("0 or 1")); CHECK_SIMPLE_VARIABLE("refresh-from-block-height", set_refresh_from_block_height, tr("block height")); CHECK_SIMPLE_VARIABLE("auto-low-priority", set_auto_low_priority, tr("0 or 1")); + CHECK_SIMPLE_VARIABLE("segregate-pre-fork-outputs", set_segregate_pre_fork_outputs, tr("0 or 1")); + CHECK_SIMPLE_VARIABLE("key-reuse-mitigation2", set_key_reuse_mitigation2, tr("0 or 1")); } fail_msg_writer() << tr("set: unrecognized argument(s)"); return true; diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h index 6f344439e..c69df817d 100644 --- a/src/simplewallet/simplewallet.h +++ b/src/simplewallet/simplewallet.h @@ -133,6 +133,8 @@ namespace cryptonote bool set_confirm_export_overwrite(const std::vector<std::string> &args = std::vector<std::string>()); bool set_refresh_from_block_height(const std::vector<std::string> &args = std::vector<std::string>()); bool set_auto_low_priority(const std::vector<std::string> &args = std::vector<std::string>()); + bool set_segregate_pre_fork_outputs(const std::vector<std::string> &args = std::vector<std::string>()); + bool set_key_reuse_mitigation2(const std::vector<std::string> &args = std::vector<std::string>()); bool help(const std::vector<std::string> &args = std::vector<std::string>()); bool start_mining(const std::vector<std::string> &args); bool stop_mining(const std::vector<std::string> &args); @@ -208,6 +210,12 @@ namespace cryptonote bool sign_multisig(const std::vector<std::string>& args); bool submit_multisig(const std::vector<std::string>& args); bool export_raw_multisig(const std::vector<std::string>& args); + bool print_ring(const std::vector<std::string>& args); + bool set_ring(const std::vector<std::string>& args); + bool save_known_rings(const std::vector<std::string>& args); + bool blackball(const std::vector<std::string>& args); + bool unblackball(const std::vector<std::string>& args); + bool blackballed(const std::vector<std::string>& args); uint64_t get_daemon_blockchain_height(std::string& err); bool try_connect_to_daemon(bool silent = false, uint32_t* version = nullptr); diff --git a/src/wallet/CMakeLists.txt b/src/wallet/CMakeLists.txt index d82e1dace..a5a4c7f56 100644 --- a/src/wallet/CMakeLists.txt +++ b/src/wallet/CMakeLists.txt @@ -33,6 +33,7 @@ set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) set(wallet_sources wallet2.cpp wallet_args.cpp + ringdb.cpp node_rpc_proxy.cpp) set(wallet_private_headers @@ -42,6 +43,7 @@ set(wallet_private_headers wallet_rpc_server.h wallet_rpc_server_commands_defs.h wallet_rpc_server_error_codes.h + ringdb.h node_rpc_proxy.h) monero_private_headers(wallet @@ -55,6 +57,7 @@ target_link_libraries(wallet common cryptonote_core mnemonics + ${LMDB_LIBRARY} ${Boost_CHRONO_LIBRARY} ${Boost_SERIALIZATION_LIBRARY} ${Boost_FILESYSTEM_LIBRARY} diff --git a/src/wallet/api/CMakeLists.txt b/src/wallet/api/CMakeLists.txt index 1e67495f1..d6f2bf6b7 100644 --- a/src/wallet/api/CMakeLists.txt +++ b/src/wallet/api/CMakeLists.txt @@ -69,6 +69,7 @@ target_link_libraries(wallet_api common cryptonote_core mnemonics + ${LMDB_LIBRARY} ${Boost_CHRONO_LIBRARY} ${Boost_SERIALIZATION_LIBRARY} ${Boost_FILESYSTEM_LIBRARY} diff --git a/src/wallet/api/wallet.cpp b/src/wallet/api/wallet.cpp index fb9e8b28b..b02884f67 100644 --- a/src/wallet/api/wallet.cpp +++ b/src/wallet/api/wallet.cpp @@ -68,6 +68,15 @@ namespace { static const int DEFAULT_REMOTE_NODE_REFRESH_INTERVAL_MILLIS = 1000 * 10; // Connection timeout 30 sec static const int DEFAULT_CONNECTION_TIMEOUT_MILLIS = 1000 * 30; + + std::string get_default_ringdb_path() + { + boost::filesystem::path dir = tools::get_default_data_dir(); + // remove .bitmonero, replace with .shared-ringdb + dir = dir.remove_filename(); + dir /= ".shared-ringdb"; + return dir.string(); + } } struct Wallet2CallbackImpl : public tools::i_wallet2_callback @@ -590,6 +599,7 @@ bool WalletImpl::open(const std::string &path, const std::string &password) // Rebuilding wallet cache, using refresh height from .keys file m_rebuildWalletCache = true; } + m_wallet->set_ring_database(get_default_ringdb_path()); m_wallet->load(path, password); m_password = password; @@ -1777,6 +1787,7 @@ void WalletImpl::doRefresh() if (m_history->count() == 0) { m_history->refresh(); } + m_wallet->find_and_save_rings(false); } else { LOG_PRINT_L3(__FUNCTION__ << ": skipping refresh - daemon is not synced"); } @@ -1897,6 +1908,127 @@ bool WalletImpl::useForkRules(uint8_t version, int64_t early_blocks) const return m_wallet->use_fork_rules(version,early_blocks); } +bool WalletImpl::blackballOutputs(const std::vector<std::string> &pubkeys, bool add) +{ + std::vector<crypto::public_key> raw_pubkeys; + raw_pubkeys.reserve(pubkeys.size()); + for (const std::string &str: pubkeys) + { + crypto::public_key pkey; + if (!epee::string_tools::hex_to_pod(str, pkey)) + { + m_status = Status_Error; + m_errorString = tr("Failed to parse output public key"); + return false; + } + raw_pubkeys.push_back(pkey); + } + bool ret = m_wallet->set_blackballed_outputs(raw_pubkeys, add); + if (!ret) + { + m_status = Status_Error; + m_errorString = tr("Failed to set blackballed outputs"); + return false; + } + return true; +} + +bool WalletImpl::unblackballOutput(const std::string &pubkey) +{ + crypto::public_key raw_pubkey; + if (!epee::string_tools::hex_to_pod(pubkey, raw_pubkey)) + { + m_status = Status_Error; + m_errorString = tr("Failed to parse output public key"); + return false; + } + bool ret = m_wallet->unblackball_output(raw_pubkey); + if (!ret) + { + m_status = Status_Error; + m_errorString = tr("Failed to unblackball output"); + return false; + } + return true; +} + +bool WalletImpl::getRing(const std::string &key_image, std::vector<uint64_t> &ring) const +{ + crypto::key_image raw_key_image; + if (!epee::string_tools::hex_to_pod(key_image, raw_key_image)) + { + m_status = Status_Error; + m_errorString = tr("Failed to parse key image"); + return false; + } + bool ret = m_wallet->get_ring(raw_key_image, ring); + if (!ret) + { + m_status = Status_Error; + m_errorString = tr("Failed to get ring"); + return false; + } + return true; +} + +bool WalletImpl::getRings(const std::string &txid, std::vector<std::pair<std::string, std::vector<uint64_t>>> &rings) const +{ + crypto::hash raw_txid; + if (!epee::string_tools::hex_to_pod(txid, raw_txid)) + { + m_status = Status_Error; + m_errorString = tr("Failed to parse txid"); + return false; + } + std::vector<std::pair<crypto::key_image, std::vector<uint64_t>>> raw_rings; + bool ret = m_wallet->get_rings(raw_txid, raw_rings); + if (!ret) + { + m_status = Status_Error; + m_errorString = tr("Failed to get rings"); + return false; + } + for (const auto &r: raw_rings) + { + rings.push_back(std::make_pair(epee::string_tools::pod_to_hex(r.first), r.second)); + } + return true; +} + +bool WalletImpl::setRing(const std::string &key_image, const std::vector<uint64_t> &ring, bool relative) +{ + crypto::key_image raw_key_image; + if (!epee::string_tools::hex_to_pod(key_image, raw_key_image)) + { + m_status = Status_Error; + m_errorString = tr("Failed to parse key image"); + return false; + } + bool ret = m_wallet->set_ring(raw_key_image, ring, relative); + if (!ret) + { + m_status = Status_Error; + m_errorString = tr("Failed to set ring"); + return false; + } + return true; +} + +void WalletImpl::segregatePreForkOutputs(bool segregate) +{ + m_wallet->segregate_pre_fork_outputs(segregate); +} + +void WalletImpl::segregationHeight(uint64_t height) +{ + m_wallet->segregation_height(height); +} + +void WalletImpl::keyReuseMitigation2(bool mitigation) +{ + m_wallet->key_reuse_mitigation2(mitigation); +} + } // namespace namespace Bitmonero = Monero; diff --git a/src/wallet/api/wallet.h b/src/wallet/api/wallet.h index 9b4a0cc12..4929c9673 100644 --- a/src/wallet/api/wallet.h +++ b/src/wallet/api/wallet.h @@ -163,6 +163,14 @@ public: virtual std::string getDefaultDataDir() const; virtual bool lightWalletLogin(bool &isNewWallet) const; virtual bool lightWalletImportWalletRequest(std::string &payment_id, uint64_t &fee, bool &new_request, bool &request_fulfilled, std::string &payment_address, std::string &status); + virtual bool blackballOutputs(const std::vector<std::string> &pubkeys, bool add); + virtual bool unblackballOutput(const std::string &pubkey); + virtual bool getRing(const std::string &key_image, std::vector<uint64_t> &ring) const; + virtual bool getRings(const std::string &txid, std::vector<std::pair<std::string, std::vector<uint64_t>>> &rings) const; + virtual bool setRing(const std::string &key_image, const std::vector<uint64_t> &ring, bool relative); + virtual void segregatePreForkOutputs(bool segregate); + virtual void segregationHeight(uint64_t height); + virtual void keyReuseMitigation2(bool mitigation); private: void clearStatus() const; diff --git a/src/wallet/api/wallet2_api.h b/src/wallet/api/wallet2_api.h index 87c1cccfa..d4e41c5aa 100644 --- a/src/wallet/api/wallet2_api.h +++ b/src/wallet/api/wallet2_api.h @@ -33,6 +33,7 @@ #include <string> #include <vector> +#include <list> #include <set> #include <ctime> #include <iostream> @@ -756,6 +757,30 @@ struct Wallet */ virtual bool rescanSpent() = 0; + //! blackballs a set of outputs + virtual bool blackballOutputs(const std::vector<std::string> &pubkeys, bool add) = 0; + + //! unblackballs an output + virtual bool unblackballOutput(const std::string &pubkey) = 0; + + //! gets the ring used for a key image, if any + virtual bool getRing(const std::string &key_image, std::vector<uint64_t> &ring) const = 0; + + //! gets the rings used for a txid, if any + virtual bool getRings(const std::string &txid, std::vector<std::pair<std::string, std::vector<uint64_t>>> &rings) const = 0; + + //! sets the ring used for a key image + virtual bool setRing(const std::string &key_image, const std::vector<uint64_t> &ring, bool relative) = 0; + + //! sets whether pre-fork outs are to be segregated + virtual void segregatePreForkOutputs(bool segregate) = 0; + + //! sets the height where segregation should occur + virtual void segregationHeight(uint64_t height) = 0; + + //! secondary key reuse mitigation + virtual void keyReuseMitigation2(bool mitigation) = 0; + //! Light wallet authenticate and login virtual bool lightWalletLogin(bool &isNewWallet) const = 0; diff --git a/src/wallet/ringdb.cpp b/src/wallet/ringdb.cpp new file mode 100644 index 000000000..44992520f --- /dev/null +++ b/src/wallet/ringdb.cpp @@ -0,0 +1,445 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <lmdb.h> +#include <boost/algorithm/string.hpp> +#include <boost/range/adaptor/transformed.hpp> +#include <boost/filesystem.hpp> +#include "misc_log_ex.h" +#include "misc_language.h" +#include "wallet_errors.h" +#include "ringdb.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "wallet.ringdb" + +static const char zerokey[8] = {0}; +static const MDB_val zerokeyval = { sizeof(zerokey), (void *)zerokey }; + +static int compare_hash32(const MDB_val *a, const MDB_val *b) +{ + uint32_t *va = (uint32_t*) a->mv_data; + uint32_t *vb = (uint32_t*) b->mv_data; + for (int n = 7; n >= 0; n--) + { + if (va[n] == vb[n]) + continue; + return va[n] < vb[n] ? -1 : 1; + } + + return 0; +} + +static std::string compress_ring(const std::vector<uint64_t> &ring) +{ + std::string s; + for (uint64_t out: ring) + s += tools::get_varint_data(out); + return s; +} + +static std::vector<uint64_t> decompress_ring(const std::string &s) +{ + std::vector<uint64_t> ring; + int read = 0; + for (std::string::const_iterator i = s.begin(); i != s.cend(); std::advance(i, read)) + { + uint64_t out; + std::string tmp(i, s.cend()); + read = tools::read_varint(tmp.begin(), tmp.end(), out); + THROW_WALLET_EXCEPTION_IF(read <= 0 || read > 256, tools::error::wallet_internal_error, "Internal error decompressing ring"); + ring.push_back(out); + } + return ring; +} + +std::string get_rings_filename(boost::filesystem::path filename) +{ + if (!boost::filesystem::is_directory(filename)) + filename.remove_filename(); + return filename.string(); +} + +static crypto::chacha_iv make_iv(const crypto::key_image &key_image, const crypto::chacha_key &key) +{ + static const char salt[] = "ringdsb"; + + uint8_t buffer[sizeof(key_image) + sizeof(key) + sizeof(salt)]; + memcpy(buffer, &key_image, sizeof(key_image)); + memcpy(buffer + sizeof(key_image), &key, sizeof(key)); + memcpy(buffer + sizeof(key_image) + sizeof(key), salt, sizeof(salt)); + crypto::hash hash; + crypto::cn_fast_hash(buffer, sizeof(buffer), hash.data); + static_assert(sizeof(hash) >= CHACHA_IV_SIZE, "Incompatible hash and chacha IV sizes"); + crypto::chacha_iv iv; + memcpy(&iv, &hash, CHACHA_IV_SIZE); + return iv; +} + +static std::string encrypt(const std::string &plaintext, const crypto::key_image &key_image, const crypto::chacha_key &key) +{ + const crypto::chacha_iv iv = make_iv(key_image, key); + std::string ciphertext; + ciphertext.resize(plaintext.size() + sizeof(iv)); + crypto::chacha20(plaintext.data(), plaintext.size(), key, iv, &ciphertext[sizeof(iv)]); + memcpy(&ciphertext[0], &iv, sizeof(iv)); + return ciphertext; +} + +static std::string encrypt(const crypto::key_image &key_image, const crypto::chacha_key &key) +{ + return encrypt(std::string((const char*)&key_image, sizeof(key_image)), key_image, key); +} + +static std::string decrypt(const std::string &ciphertext, const crypto::key_image &key_image, const crypto::chacha_key &key) +{ + const crypto::chacha_iv iv = make_iv(key_image, key); + std::string plaintext; + THROW_WALLET_EXCEPTION_IF(ciphertext.size() < sizeof(iv), tools::error::wallet_internal_error, "Bad ciphertext text"); + plaintext.resize(ciphertext.size() - sizeof(iv)); + crypto::chacha20(ciphertext.data() + sizeof(iv), ciphertext.size() - sizeof(iv), key, iv, &plaintext[0]); + return plaintext; +} + +static void store_relative_ring(MDB_txn *txn, MDB_dbi &dbi, const crypto::key_image &key_image, const std::vector<uint64_t> &relative_ring, const crypto::chacha_key &chacha_key) +{ + MDB_val key, data; + std::string key_ciphertext = encrypt(key_image, chacha_key); + key.mv_data = (void*)key_ciphertext.data(); + key.mv_size = key_ciphertext.size(); + std::string compressed_ring = compress_ring(relative_ring); + std::string data_ciphertext = encrypt(compressed_ring, key_image, chacha_key); + data.mv_size = data_ciphertext.size(); + data.mv_data = (void*)data_ciphertext.c_str(); + int dbr = mdb_put(txn, dbi, &key, &data, 0); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set ring for key image in LMDB table: " + std::string(mdb_strerror(dbr))); +} + +static int resize_env(MDB_env *env, const char *db_path, size_t needed) +{ + MDB_envinfo mei; + MDB_stat mst; + int ret; + + needed = std::max(needed, (size_t)(2ul * 1024 * 1024)); // at least 2 MB + + ret = mdb_env_info(env, &mei); + if (ret) + return ret; + ret = mdb_env_stat(env, &mst); + if (ret) + return ret; + uint64_t size_used = mst.ms_psize * mei.me_last_pgno; + uint64_t mapsize = mei.me_mapsize; + if (size_used + needed > mei.me_mapsize) + { + try + { + boost::filesystem::path path(db_path); + boost::filesystem::space_info si = boost::filesystem::space(path); + if(si.available < needed) + { + MERROR("!! WARNING: Insufficient free space to extend database !!: " << (si.available >> 20L) << " MB available"); + return ENOSPC; + } + } + catch(...) + { + // print something but proceed. + MWARNING("Unable to query free disk space."); + } + + mapsize += needed; + } + return mdb_env_set_mapsize(env, mapsize); +} + +static size_t get_ring_data_size(size_t n_entries) +{ + return n_entries * (32 + 1024); // highball 1kB for the ring data to make sure +} + +enum { BLACKBALL_BLACKBALL, BLACKBALL_UNBLACKBALL, BLACKBALL_QUERY, BLACKBALL_CLEAR}; + +namespace tools +{ + +ringdb::ringdb(std::string filename, const std::string &genesis): + filename(filename) +{ + MDB_txn *txn; + bool tx_active = false; + int dbr; + + tools::create_directories_if_necessary(filename); + + dbr = mdb_env_create(&env); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); + dbr = mdb_env_set_maxdbs(env, 2); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); + const std::string actual_filename = get_rings_filename(filename); + dbr = mdb_env_open(env, actual_filename.c_str(), 0, 0664); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to open rings database file '" + + actual_filename + "': " + std::string(mdb_strerror(dbr))); + + dbr = mdb_txn_begin(env, NULL, 0, &txn); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + dbr = mdb_dbi_open(txn, ("rings-" + genesis).c_str(), MDB_CREATE, &dbi_rings); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_compare(txn, dbi_rings, compare_hash32); + + dbr = mdb_dbi_open(txn, ("blackballs-" + genesis).c_str(), MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi_blackballs); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_dupsort(txn, dbi_blackballs, compare_hash32); + + dbr = mdb_txn_commit(txn); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); + tx_active = false; +} + +ringdb::~ringdb() +{ + mdb_dbi_close(env, dbi_rings); + mdb_dbi_close(env, dbi_blackballs); + mdb_env_close(env); +} + +bool ringdb::add_rings(const crypto::chacha_key &chacha_key, const cryptonote::transaction_prefix &tx) +{ + MDB_txn *txn; + int dbr; + bool tx_active = false; + + dbr = resize_env(env, filename.c_str(), get_ring_data_size(tx.vin.size())); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set env map size"); + dbr = mdb_txn_begin(env, NULL, 0, &txn); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + for (const auto &in: tx.vin) + { + if (in.type() != typeid(cryptonote::txin_to_key)) + continue; + const auto &txin = boost::get<cryptonote::txin_to_key>(in); + const uint32_t ring_size = txin.key_offsets.size(); + if (ring_size == 1) + continue; + + store_relative_ring(txn, dbi_rings, txin.k_image, txin.key_offsets, chacha_key); + } + + dbr = mdb_txn_commit(txn); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn adding ring to database: " + std::string(mdb_strerror(dbr))); + tx_active = false; + return true; +} + +bool ringdb::remove_rings(const crypto::chacha_key &chacha_key, const cryptonote::transaction_prefix &tx) +{ + MDB_txn *txn; + int dbr; + bool tx_active = false; + + dbr = resize_env(env, filename.c_str(), 0); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set env map size"); + dbr = mdb_txn_begin(env, NULL, 0, &txn); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + for (const auto &in: tx.vin) + { + if (in.type() != typeid(cryptonote::txin_to_key)) + continue; + const auto &txin = boost::get<cryptonote::txin_to_key>(in); + const uint32_t ring_size = txin.key_offsets.size(); + if (ring_size == 1) + continue; + + MDB_val key, data; + std::string key_ciphertext = encrypt(txin.k_image, chacha_key); + key.mv_data = (void*)key_ciphertext.data(); + key.mv_size = key_ciphertext.size(); + + dbr = mdb_get(txn, dbi_rings, &key, &data); + THROW_WALLET_EXCEPTION_IF(dbr && dbr != MDB_NOTFOUND, tools::error::wallet_internal_error, "Failed to look for key image in LMDB table: " + std::string(mdb_strerror(dbr))); + if (dbr == MDB_NOTFOUND) + continue; + THROW_WALLET_EXCEPTION_IF(data.mv_size <= 0, tools::error::wallet_internal_error, "Invalid ring data size"); + + MDEBUG("Removing ring data for key image " << txin.k_image); + dbr = mdb_del(txn, dbi_rings, &key, NULL); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to remove ring to database: " + std::string(mdb_strerror(dbr))); + } + + dbr = mdb_txn_commit(txn); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn removing ring to database: " + std::string(mdb_strerror(dbr))); + tx_active = false; + return true; +} + +bool ringdb::get_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, std::vector<uint64_t> &outs) +{ + MDB_txn *txn; + int dbr; + bool tx_active = false; + + dbr = resize_env(env, filename.c_str(), 0); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set env map size: " + std::string(mdb_strerror(dbr))); + dbr = mdb_txn_begin(env, NULL, 0, &txn); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + MDB_val key, data; + std::string key_ciphertext = encrypt(key_image, chacha_key); + key.mv_data = (void*)key_ciphertext.data(); + key.mv_size = key_ciphertext.size(); + dbr = mdb_get(txn, dbi_rings, &key, &data); + THROW_WALLET_EXCEPTION_IF(dbr && dbr != MDB_NOTFOUND, tools::error::wallet_internal_error, "Failed to look for key image in LMDB table: " + std::string(mdb_strerror(dbr))); + if (dbr == MDB_NOTFOUND) + return false; + THROW_WALLET_EXCEPTION_IF(data.mv_size <= 0, tools::error::wallet_internal_error, "Invalid ring data size"); + + std::string data_plaintext = decrypt(std::string((const char*)data.mv_data, data.mv_size), key_image, chacha_key); + outs = decompress_ring(data_plaintext); + MDEBUG("Found ring for key image " << key_image << ":"); + MDEBUG("Relative: " << boost::join(outs | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " ")); + outs = cryptonote::relative_output_offsets_to_absolute(outs); + MDEBUG("Absolute: " << boost::join(outs | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " ")); + + dbr = mdb_txn_commit(txn); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn getting ring from database: " + std::string(mdb_strerror(dbr))); + tx_active = false; + return true; +} + +bool ringdb::set_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, const std::vector<uint64_t> &outs, bool relative) +{ + MDB_txn *txn; + int dbr; + bool tx_active = false; + + dbr = resize_env(env, filename.c_str(), outs.size() * 64); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set env map size: " + std::string(mdb_strerror(dbr))); + dbr = mdb_txn_begin(env, NULL, 0, &txn); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + store_relative_ring(txn, dbi_rings, key_image, relative ? outs : cryptonote::absolute_output_offsets_to_relative(outs), chacha_key); + + dbr = mdb_txn_commit(txn); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn setting ring to database: " + std::string(mdb_strerror(dbr))); + tx_active = false; + return true; +} + +bool ringdb::blackball_worker(const crypto::public_key &output, int op) +{ + MDB_txn *txn; + MDB_cursor *cursor; + int dbr; + bool tx_active = false; + bool ret = true; + + dbr = resize_env(env, filename.c_str(), 32 * 2); // a pubkey, and some slack + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set env map size: " + std::string(mdb_strerror(dbr))); + dbr = mdb_txn_begin(env, NULL, 0, &txn); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + MDB_val key = zerokeyval; + MDB_val data; + data.mv_data = (void*)&output; + data.mv_size = sizeof(output); + + switch (op) + { + case BLACKBALL_BLACKBALL: + MDEBUG("Blackballing output " << output); + dbr = mdb_put(txn, dbi_blackballs, &key, &data, MDB_NODUPDATA); + if (dbr == MDB_KEYEXIST) + dbr = 0; + break; + case BLACKBALL_UNBLACKBALL: + MDEBUG("Unblackballing output " << output); + dbr = mdb_del(txn, dbi_blackballs, &key, &data); + if (dbr == MDB_NOTFOUND) + dbr = 0; + break; + case BLACKBALL_QUERY: + dbr = mdb_cursor_open(txn, dbi_blackballs, &cursor); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create cursor for blackballs table: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_get(cursor, &key, &data, MDB_GET_BOTH); + THROW_WALLET_EXCEPTION_IF(dbr && dbr != MDB_NOTFOUND, tools::error::wallet_internal_error, "Failed to lookup in blackballs table: " + std::string(mdb_strerror(dbr))); + ret = dbr != MDB_NOTFOUND; + if (dbr == MDB_NOTFOUND) + dbr = 0; + mdb_cursor_close(cursor); + break; + case BLACKBALL_CLEAR: + dbr = mdb_drop(txn, dbi_blackballs, 0); + break; + default: + THROW_WALLET_EXCEPTION(tools::error::wallet_internal_error, "Invalid blackball op"); + } + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to query blackballs table: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_txn_commit(txn); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn blackballing output to database: " + std::string(mdb_strerror(dbr))); + tx_active = false; + return ret; +} + +bool ringdb::blackball(const crypto::public_key &output) +{ + return blackball_worker(output, BLACKBALL_BLACKBALL); +} + +bool ringdb::unblackball(const crypto::public_key &output) +{ + return blackball_worker(output, BLACKBALL_UNBLACKBALL); +} + +bool ringdb::blackballed(const crypto::public_key &output) +{ + return blackball_worker(output, BLACKBALL_QUERY); +} + +bool ringdb::clear_blackballs() +{ + return blackball_worker(crypto::public_key(), BLACKBALL_CLEAR); +} + +} diff --git a/src/wallet/ringdb.h b/src/wallet/ringdb.h new file mode 100644 index 000000000..2bd1ac149 --- /dev/null +++ b/src/wallet/ringdb.h @@ -0,0 +1,65 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <string> +#include <vector> +#include <lmdb.h> +#include "wipeable_string.h" +#include "crypto/crypto.h" +#include "cryptonote_basic/cryptonote_basic.h" + +namespace tools +{ + class ringdb + { + public: + ringdb(std::string filename, const std::string &genesis); + ~ringdb(); + + bool add_rings(const crypto::chacha_key &chacha_key, const cryptonote::transaction_prefix &tx); + bool remove_rings(const crypto::chacha_key &chacha_key, const cryptonote::transaction_prefix &tx); + bool get_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, std::vector<uint64_t> &outs); + bool set_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, const std::vector<uint64_t> &outs, bool relative); + + bool blackball(const crypto::public_key &output); + bool unblackball(const crypto::public_key &output); + bool blackballed(const crypto::public_key &output); + bool clear_blackballs(); + + private: + bool blackball_worker(const crypto::public_key &output, int op); + + private: + std::string filename; + MDB_env *env; + MDB_dbi dbi_rings; + MDB_dbi dbi_blackballs; + }; +} diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index cb5e7bc67..6d941163e 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -66,6 +66,7 @@ using namespace epee; #include "memwipe.h" #include "common/base58.h" #include "ringct/rctSigs.h" +#include "ringdb.h" extern "C" { @@ -93,7 +94,9 @@ using namespace cryptonote; #define MULTISIG_UNSIGNED_TX_PREFIX "Monero multisig unsigned tx set\001" #define RECENT_OUTPUT_RATIO (0.5) // 50% of outputs are from the recent zone -#define RECENT_OUTPUT_ZONE ((time_t)(1.8 * 86400)) // last 1.8 day makes up the recent zone (taken from monerolink.pdf, Miller et al) +#define RECENT_OUTPUT_DAYS (1.8) // last 1.8 day makes up the recent zone (taken from monerolink.pdf, Miller et al) +#define RECENT_OUTPUT_ZONE ((time_t)(RECENT_OUTPUT_DAYS * 86400)) +#define RECENT_OUTPUT_BLOCKS (RECENT_OUTPUT_DAYS * 720) #define FEE_ESTIMATE_GRACE_BLOCKS 10 // estimate fee valid for that many blocks @@ -106,6 +109,24 @@ using namespace cryptonote; #define MULTISIG_EXPORT_FILE_MAGIC "Monero multisig export\001" +#define SEGREGATION_FORK_HEIGHT 1564965 +#define TESTNET_SEGREGATION_FORK_HEIGHT 1000000 +#define STAGENET_SEGREGATION_FORK_HEIGHT 1000000 +#define SEGREGATION_FORK_VICINITY 1500 /* blocks */ + + +namespace +{ + std::string get_default_ringdb_path() + { + boost::filesystem::path dir = tools::get_default_data_dir(); + // remove .bitmonero, replace with .shared-ringdb + dir = dir.remove_filename(); + dir /= ".shared-ringdb"; + return dir.string(); + } +} + namespace { // Create on-demand to prevent static initialization order fiasco issues. @@ -119,6 +140,16 @@ struct options { const command_line::arg_descriptor<bool> testnet = {"testnet", tools::wallet2::tr("For testnet. Daemon must also be launched with --testnet flag"), false}; const command_line::arg_descriptor<bool> stagenet = {"stagenet", tools::wallet2::tr("For stagenet. Daemon must also be launched with --stagenet flag"), false}; const command_line::arg_descriptor<bool> restricted = {"restricted-rpc", tools::wallet2::tr("Restricts to view-only commands"), false}; + const command_line::arg_descriptor<std::string, false, true> shared_ringdb_dir = { + "shared-ringdb-dir", tools::wallet2::tr("Set shared ring database path"), + get_default_ringdb_path(), + testnet, + [](bool testnet, bool defaulted, std::string val) { + if (testnet) + return (boost::filesystem::path(val) / "testnet").string(); + return val; + } + }; }; void do_prepare_file_names(const std::string& file_path, std::string& keys_file, std::string& wallet_file) @@ -196,6 +227,8 @@ std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variabl std::unique_ptr<tools::wallet2> wallet(new tools::wallet2(testnet ? TESTNET : stagenet ? STAGENET : MAINNET, restricted)); wallet->init(std::move(daemon_address), std::move(login)); + boost::filesystem::path ringdb_path = command_line::get_arg(vm, opts.shared_ringdb_dir); + wallet->set_ring_database(ringdb_path.string()); return wallet; } @@ -627,6 +660,8 @@ wallet2::wallet2(network_type nettype, bool restricted): m_confirm_backlog_threshold(0), m_confirm_export_overwrite(true), m_auto_low_priority(true), + m_segregate_pre_fork_outputs(true), + m_key_reuse_mitigation2(true), m_is_initialized(false), m_restricted(restricted), is_old_file_format(false), @@ -639,7 +674,13 @@ wallet2::wallet2(network_type nettype, bool restricted): m_light_wallet_connected(false), m_light_wallet_balance(0), m_light_wallet_unlocked_balance(0), - m_key_on_device(false) + m_key_on_device(false), + m_ring_history_saved(false), + m_ringdb() +{ +} + +wallet2::~wallet2() { } @@ -665,6 +706,7 @@ void wallet2::init_options(boost::program_options::options_description& desc_par command_line::add_arg(desc_params, opts.testnet); command_line::add_arg(desc_params, opts.stagenet); command_line::add_arg(desc_params, opts.restricted); + command_line::add_arg(desc_params, opts.shared_ringdb_dir); } std::unique_ptr<wallet2> wallet2::make_from_json(const boost::program_options::variables_map& vm, const std::string& json_file, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter) @@ -1479,9 +1521,19 @@ void wallet2::process_outgoing(const crypto::hash &txid, const cryptonote::trans entry.first->second.m_subaddr_account = subaddr_account; entry.first->second.m_subaddr_indices = subaddr_indices; } + + for (const auto &in: tx.vin) + { + if (in.type() != typeid(cryptonote::txin_to_key)) + continue; + const auto &txin = boost::get<cryptonote::txin_to_key>(in); + entry.first->second.m_rings.push_back(std::make_pair(txin.k_image, txin.key_offsets)); + } entry.first->second.m_block_height = height; entry.first->second.m_timestamp = ts; entry.first->second.m_unlock_time = tx.unlock_time; + + add_rings(tx); } //---------------------------------------------------------------------------------------------------- void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const crypto::hash& bl_id, uint64_t height, const cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices &o_indices) @@ -1852,6 +1904,7 @@ void wallet2::update_pool_state(bool refreshed) pit->second.m_state = wallet2::unconfirmed_transfer_details::failed; // the inputs aren't spent anymore, since the tx failed + remove_rings(pit->second.m_tx); for (size_t vini = 0; vini < pit->second.m_tx.vin.size(); ++vini) { if (pit->second.m_tx.vin[vini].type() == typeid(txin_to_key)) @@ -2266,6 +2319,73 @@ bool wallet2::refresh(uint64_t & blocks_fetched, bool& received_money, bool& ok) return ok; } //---------------------------------------------------------------------------------------------------- +bool wallet2::get_output_distribution(uint64_t &start_height, std::vector<uint64_t> &distribution) +{ + uint32_t rpc_version; + boost::optional<std::string> result = m_node_rpc_proxy.get_rpc_version(rpc_version); + // no error + if (!!result) + { + // empty string -> not connection + THROW_WALLET_EXCEPTION_IF(result->empty(), tools::error::no_connection_to_daemon, "getversion"); + THROW_WALLET_EXCEPTION_IF(*result == CORE_RPC_STATUS_BUSY, tools::error::daemon_busy, "getversion"); + if (*result != CORE_RPC_STATUS_OK) + { + MDEBUG("Cannot determine daemon RPC version, not requesting rct distribution"); + return false; + } + } + else + { + if (rpc_version >= MAKE_CORE_RPC_VERSION(1, 19)) + { + MDEBUG("Daemon is recent enough, requesting rct distribution"); + } + else + { + MDEBUG("Daemon is too old, not requesting rct distribution"); + return false; + } + } + + cryptonote::COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request req = AUTO_VAL_INIT(req); + cryptonote::COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response res = AUTO_VAL_INIT(res); + req.amounts.push_back(0); + req.from_height = 0; + req.cumulative = true; + m_daemon_rpc_mutex.lock(); + bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_distribution", req, res, m_http_client, rpc_timeout); + m_daemon_rpc_mutex.unlock(); + if (!r) + { + MWARNING("Failed to request output distribution: no connection to daemon"); + return false; + } + if (res.status == CORE_RPC_STATUS_BUSY) + { + MWARNING("Failed to request output distribution: daemon is busy"); + return false; + } + if (res.status != CORE_RPC_STATUS_OK) + { + MWARNING("Failed to request output distribution: " << res.status); + return false; + } + if (res.distributions.size() != 1) + { + MWARNING("Failed to request output distribution: not the expected single result"); + return false; + } + if (res.distributions[0].amount != 0) + { + MWARNING("Failed to request output distribution: results are not for amount 0"); + return false; + } + start_height = res.distributions[0].start_height; + distribution = std::move(res.distributions[0].distribution); + return true; +} +//---------------------------------------------------------------------------------------------------- void wallet2::detach_blockchain(uint64_t height) { LOG_PRINT_L0("Detaching blockchain on height " << height); @@ -2468,6 +2588,12 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable value2.SetUint(m_nettype); json.AddMember("nettype", value2, json.GetAllocator()); + value2.SetInt(m_segregate_pre_fork_outputs ? 1 : 0); + json.AddMember("segregate_pre_fork_outputs", value2, json.GetAllocator()); + + value2.SetInt(m_key_reuse_mitigation2 ? 1 : 0); + json.AddMember("key_reuse_mitigation2", value2, json.GetAllocator()); + // Serialize the JSON object rapidjson::StringBuffer buffer; rapidjson::Writer<rapidjson::StringBuffer> writer(buffer); @@ -3677,6 +3803,15 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass add_subaddress_account(tr("Primary account")); m_local_bc_height = m_blockchain.size(); + + try + { + find_and_save_rings(false); + } + catch (const std::exception &e) + { + MERROR("Failed to save rings, will try again next time"); + } } //---------------------------------------------------------------------------------------------------- void wallet2::trim_hashchain() @@ -4263,6 +4398,13 @@ void wallet2::add_unconfirmed_tx(const cryptonote::transaction& tx, uint64_t amo utd.m_timestamp = time(NULL); utd.m_subaddr_account = subaddr_account; utd.m_subaddr_indices = subaddr_indices; + for (const auto &in: tx.vin) + { + if (in.type() != typeid(cryptonote::txin_to_key)) + continue; + const auto &txin = boost::get<cryptonote::txin_to_key>(in); + utd.m_rings.push_back(std::make_pair(txin.k_image, txin.key_offsets)); + } } //---------------------------------------------------------------------------------------------------- @@ -5320,17 +5462,195 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions(std::vector<crypto } } +void wallet2::set_ring_database(const std::string &filename) +{ + m_ring_database = filename; + MINFO("ringdb path set to " << filename); + m_ringdb.reset(); + cryptonote::block b; + generate_genesis(b); + if (!m_ring_database.empty()) + m_ringdb.reset(new tools::ringdb(m_ring_database, epee::string_tools::pod_to_hex(get_block_hash(b)))); +} + +bool wallet2::add_rings(const crypto::chacha_key &key, const cryptonote::transaction_prefix &tx) +{ + if (!m_ringdb) + return true; + return m_ringdb->add_rings(key, tx); +} + +bool wallet2::add_rings(const cryptonote::transaction_prefix &tx) +{ + crypto::chacha_key key; + generate_chacha_key_from_secret_keys(key); + return add_rings(key, tx); +} + +bool wallet2::remove_rings(const cryptonote::transaction_prefix &tx) +{ + if (!m_ringdb) + return true; + crypto::chacha_key key; + generate_chacha_key_from_secret_keys(key); + return m_ringdb->remove_rings(key, tx); +} + +bool wallet2::get_ring(const crypto::chacha_key &key, const crypto::key_image &key_image, std::vector<uint64_t> &outs) +{ + if (!m_ringdb) + return true; + return m_ringdb->get_ring(key, key_image, outs); +} + +bool wallet2::get_rings(const crypto::hash &txid, std::vector<std::pair<crypto::key_image, std::vector<uint64_t>>> &outs) +{ + for (auto i: m_confirmed_txs) + { + if (txid == i.first) + { + for (const auto &x: i.second.m_rings) + outs.push_back({x.first, cryptonote::relative_output_offsets_to_absolute(x.second)}); + return true; + } + } + for (auto i: m_unconfirmed_txs) + { + if (txid == i.first) + { + for (const auto &x: i.second.m_rings) + outs.push_back({x.first, cryptonote::relative_output_offsets_to_absolute(x.second)}); + return true; + } + } + return false; +} + +bool wallet2::get_ring(const crypto::key_image &key_image, std::vector<uint64_t> &outs) +{ + crypto::chacha_key key; + generate_chacha_key_from_secret_keys(key); + + return get_ring(key, key_image, outs); +} + +bool wallet2::set_ring(const crypto::key_image &key_image, const std::vector<uint64_t> &outs, bool relative) +{ + if (!m_ringdb) + return true; + + crypto::chacha_key key; + generate_chacha_key_from_secret_keys(key); + + return m_ringdb->set_ring(key, key_image, outs, relative); +} + +bool wallet2::find_and_save_rings(bool force) +{ + if (!force && m_ring_history_saved) + return true; + if (!m_ringdb) + return true; + + COMMAND_RPC_GET_TRANSACTIONS::request req = AUTO_VAL_INIT(req); + COMMAND_RPC_GET_TRANSACTIONS::response res = AUTO_VAL_INIT(res); + + MDEBUG("Finding and saving rings..."); + + // get payments we made + std::list<std::pair<crypto::hash,wallet2::confirmed_transfer_details>> payments; + get_payments_out(payments, 0, std::numeric_limits<uint64_t>::max(), boost::none, std::set<uint32_t>()); + for (const std::pair<crypto::hash,wallet2::confirmed_transfer_details> &entry: payments) + { + const crypto::hash &txid = entry.first; + req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); + } + + MDEBUG("Found " << std::to_string(req.txs_hashes.size()) << " transactions"); + + // get those transactions from the daemon + req.decode_as_json = false; + bool r; + { + const boost::lock_guard<boost::mutex> lock{m_daemon_rpc_mutex}; + r = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client, rpc_timeout); + } + THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "gettransactions"); + THROW_WALLET_EXCEPTION_IF(res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "gettransactions"); + THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::wallet_internal_error, "gettransactions"); + THROW_WALLET_EXCEPTION_IF(res.txs.size() != req.txs_hashes.size(), error::wallet_internal_error, + "daemon returned wrong response for gettransactions, wrong txs count = " + + std::to_string(res.txs.size()) + ", expected " + std::to_string(req.txs_hashes.size())); + + MDEBUG("Scanning " << res.txs.size() << " transactions"); + + crypto::chacha_key key; + generate_chacha_key_from_secret_keys(key); + + auto it = req.txs_hashes.begin(); + for (size_t i = 0; i < res.txs.size(); ++i, ++it) + { + const auto &tx_info = res.txs[i]; + THROW_WALLET_EXCEPTION_IF(tx_info.tx_hash != *it, error::wallet_internal_error, "Wrong txid received"); + cryptonote::blobdata bd; + THROW_WALLET_EXCEPTION_IF(!epee::string_tools::parse_hexstr_to_binbuff(tx_info.as_hex, bd), error::wallet_internal_error, "failed to parse tx from hexstr"); + cryptonote::transaction tx; + crypto::hash tx_hash, tx_prefix_hash; + THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(bd, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, "failed to parse tx from blob"); + THROW_WALLET_EXCEPTION_IF(epee::string_tools::pod_to_hex(tx_hash) != tx_info.tx_hash, error::wallet_internal_error, "txid mismatch"); + THROW_WALLET_EXCEPTION_IF(!add_rings(key, tx), error::wallet_internal_error, "Failed to save ring"); + } + + MINFO("Found and saved rings for " << res.txs.size() << " transactions"); + m_ring_history_saved = true; + return true; +} + +bool wallet2::blackball_output(const crypto::public_key &output) +{ + if (!m_ringdb) + return true; + return m_ringdb->blackball(output); +} + +bool wallet2::set_blackballed_outputs(const std::vector<crypto::public_key> &outputs, bool add) +{ + if (!m_ringdb) + return true; + bool ret = true; + if (!add) + ret &= m_ringdb->clear_blackballs(); + for (const auto &output: outputs) + ret &= m_ringdb->blackball(output); + return ret; +} + +bool wallet2::unblackball_output(const crypto::public_key &output) +{ + if (!m_ringdb) + return true; + return m_ringdb->unblackball(output); +} -bool wallet2::tx_add_fake_output(std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, uint64_t global_index, const crypto::public_key& tx_public_key, const rct::key& mask, uint64_t real_index, bool unlocked) const +bool wallet2::is_output_blackballed(const crypto::public_key &output) const +{ + if (!m_ringdb) + return true; + return m_ringdb->blackballed(output); +} + +bool wallet2::tx_add_fake_output(std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, uint64_t global_index, const crypto::public_key& output_public_key, const rct::key& mask, uint64_t real_index, bool unlocked) const { if (!unlocked) // don't add locked outs return false; if (global_index == real_index) // don't re-add real one return false; - auto item = std::make_tuple(global_index, tx_public_key, mask); + auto item = std::make_tuple(global_index, output_public_key, mask); CHECK_AND_ASSERT_MES(!outs.empty(), false, "internal error: outs is empty"); if (std::find(outs.back().begin(), outs.back().end(), item) != outs.back().end()) // don't add duplicates return false; + if (is_output_blackballed(output_public_key)) // don't add blackballed outputs + return false; outs.back().push_back(item); return true; } @@ -5449,8 +5769,25 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> return; } + crypto::chacha_key key; + generate_chacha_key_from_secret_keys(key); + if (fake_outputs_count > 0) { + uint64_t segregation_fork_height; + switch (m_nettype) + { + case TESTNET: segregation_fork_height = TESTNET_SEGREGATION_FORK_HEIGHT; break; + case STAGENET: segregation_fork_height = STAGENET_SEGREGATION_FORK_HEIGHT; break; + case MAINNET: segregation_fork_height = SEGREGATION_FORK_HEIGHT; break; + default: THROW_WALLET_EXCEPTION(error::wallet_internal_error, "Invalid network type"); + } + // check whether we're shortly after the fork + uint64_t height; + boost::optional<std::string> result = m_node_rpc_proxy.get_height(height); + throw_on_rpc_response_error(result, "get_info"); + bool is_shortly_after_segregation_fork = height >= segregation_fork_height && height < segregation_fork_height + SEGREGATION_FORK_VICINITY; + // get histogram for the amounts we need cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request req_t = AUTO_VAL_INIT(req_t); cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response resp_t = AUTO_VAL_INIT(resp_t); @@ -5468,6 +5805,50 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> THROW_WALLET_EXCEPTION_IF(resp_t.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_output_histogram"); THROW_WALLET_EXCEPTION_IF(resp_t.status != CORE_RPC_STATUS_OK, error::get_histogram_error, resp_t.status); + // if we want to segregate fake outs pre or post fork, get distribution + std::unordered_map<uint64_t, std::pair<uint64_t, uint64_t>> segregation_limit; + if (m_segregate_pre_fork_outputs || m_key_reuse_mitigation2) + { + cryptonote::COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request req_t = AUTO_VAL_INIT(req_t); + cryptonote::COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response resp_t = AUTO_VAL_INIT(resp_t); + for(size_t idx: selected_transfers) + req_t.amounts.push_back(m_transfers[idx].is_rct() ? 0 : m_transfers[idx].amount()); + std::sort(req_t.amounts.begin(), req_t.amounts.end()); + auto end = std::unique(req_t.amounts.begin(), req_t.amounts.end()); + req_t.amounts.resize(std::distance(req_t.amounts.begin(), end)); + req_t.from_height = segregation_fork_height >= RECENT_OUTPUT_ZONE ? height >= (segregation_fork_height ? segregation_fork_height : height) - RECENT_OUTPUT_BLOCKS : 0; + req_t.cumulative = true; + m_daemon_rpc_mutex.lock(); + bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_distribution", req_t, resp_t, m_http_client, rpc_timeout); + m_daemon_rpc_mutex.unlock(); + THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "transfer_selected"); + THROW_WALLET_EXCEPTION_IF(resp_t.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_output_distribution"); + THROW_WALLET_EXCEPTION_IF(resp_t.status != CORE_RPC_STATUS_OK, error::get_output_distribution, resp_t.status); + + // check we got all data + for(size_t idx: selected_transfers) + { + const uint64_t amount = m_transfers[idx].is_rct() ? 0 : m_transfers[idx].amount(); + bool found = false; + for (const auto &d: resp_t.distributions) + { + if (d.amount == amount) + { + THROW_WALLET_EXCEPTION_IF(d.start_height > segregation_fork_height, error::get_output_distribution, "Distribution start_height too high"); + THROW_WALLET_EXCEPTION_IF(segregation_fork_height - d.start_height >= d.distribution.size(), error::get_output_distribution, "Distribution size too small"); + THROW_WALLET_EXCEPTION_IF(segregation_fork_height <= RECENT_OUTPUT_BLOCKS, error::wallet_internal_error, "Fork height too low"); + THROW_WALLET_EXCEPTION_IF(segregation_fork_height - RECENT_OUTPUT_BLOCKS < d.start_height, error::get_output_distribution, "Bad start height"); + uint64_t till_fork = d.distribution[segregation_fork_height - d.start_height]; + uint64_t recent = till_fork - d.distribution[segregation_fork_height - RECENT_OUTPUT_BLOCKS - d.start_height]; + segregation_limit[amount] = std::make_pair(till_fork, recent); + found = true; + break; + } + } + THROW_WALLET_EXCEPTION_IF(!found, error::get_output_distribution, "Requested amount not found in response"); + } + } + // we ask for more, to have spares if some outputs are still locked size_t base_requested_outputs_count = (size_t)((fake_outputs_count + 1) * 1.5 + 1); LOG_PRINT_L2("base_requested_outputs_count: " << base_requested_outputs_count); @@ -5487,37 +5868,126 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> size_t requested_outputs_count = base_requested_outputs_count + (td.is_rct() ? CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW - CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE : 0); size_t start = req.outputs.size(); - // if there are just enough outputs to mix with, use all of them. - // Eventually this should become impossible. + const bool output_is_pre_fork = td.m_block_height < segregation_fork_height; uint64_t num_outs = 0, num_recent_outs = 0; - for (const auto &he: resp_t.histogram) + uint64_t num_post_fork_outs = 0; + float pre_fork_num_out_ratio = 0.0f; + float post_fork_num_out_ratio = 0.0f; + + if (m_segregate_pre_fork_outputs && output_is_pre_fork) { - if (he.amount == amount) + num_outs = segregation_limit[amount].first; + num_recent_outs = segregation_limit[amount].second; + } + else + { + // if there are just enough outputs to mix with, use all of them. + // Eventually this should become impossible. + for (const auto &he: resp_t.histogram) { - LOG_PRINT_L2("Found " << print_money(amount) << ": " << he.total_instances << " total, " - << he.unlocked_instances << " unlocked, " << he.recent_instances << " recent"); - num_outs = he.unlocked_instances; - num_recent_outs = he.recent_instances; - break; + if (he.amount == amount) + { + LOG_PRINT_L2("Found " << print_money(amount) << ": " << he.total_instances << " total, " + << he.unlocked_instances << " unlocked, " << he.recent_instances << " recent"); + num_outs = he.unlocked_instances; + num_recent_outs = he.recent_instances; + break; + } } + if (m_key_reuse_mitigation2) + { + if (output_is_pre_fork) + { + if (is_shortly_after_segregation_fork) + { + pre_fork_num_out_ratio = 33.4/100.0f * (1.0f - RECENT_OUTPUT_RATIO); + } + else + { + pre_fork_num_out_ratio = 33.4/100.0f * (1.0f - RECENT_OUTPUT_RATIO); + post_fork_num_out_ratio = 33.4/100.0f * (1.0f - RECENT_OUTPUT_RATIO); + } + } + else + { + if (is_shortly_after_segregation_fork) + { + } + else + { + post_fork_num_out_ratio = 67.8/100.0f * (1.0f - RECENT_OUTPUT_RATIO); + } + } + } + num_post_fork_outs = num_outs - segregation_limit[amount].first; } + LOG_PRINT_L1("" << num_outs << " unlocked outputs of size " << print_money(amount)); THROW_WALLET_EXCEPTION_IF(num_outs == 0, error::wallet_internal_error, "histogram reports no unlocked outputs for " + boost::lexical_cast<std::string>(amount) + ", not even ours"); THROW_WALLET_EXCEPTION_IF(num_recent_outs > num_outs, error::wallet_internal_error, "histogram reports more recent outs than outs for " + boost::lexical_cast<std::string>(amount)); + // how many fake outs to draw on a pre-fork triangular distribution + size_t pre_fork_outputs_count = requested_outputs_count * pre_fork_num_out_ratio; + size_t post_fork_outputs_count = requested_outputs_count * post_fork_num_out_ratio; + // how many fake outs to draw otherwise + size_t normal_output_count = requested_outputs_count - pre_fork_outputs_count - post_fork_outputs_count; + // X% of those outs are to be taken from recent outputs - size_t recent_outputs_count = requested_outputs_count * RECENT_OUTPUT_RATIO; + size_t recent_outputs_count = normal_output_count * RECENT_OUTPUT_RATIO; if (recent_outputs_count == 0) recent_outputs_count = 1; // ensure we have at least one, if possible if (recent_outputs_count > num_recent_outs) recent_outputs_count = num_recent_outs; if (td.m_global_output_index >= num_outs - num_recent_outs && recent_outputs_count > 0) --recent_outputs_count; // if the real out is recent, pick one less recent fake out - LOG_PRINT_L1("Using " << recent_outputs_count << " recent outputs"); + LOG_PRINT_L1("Fake output makeup: " << requested_outputs_count << " requested: " << recent_outputs_count << " recent, " << + pre_fork_outputs_count << " pre-fork, " << post_fork_outputs_count << " post-fork, " << + (requested_outputs_count - recent_outputs_count - pre_fork_outputs_count - post_fork_outputs_count) << " full-chain"); - if (num_outs <= requested_outputs_count) + uint64_t num_found = 0; + + // if we have a known ring, use it + bool existing_ring_found = false; + if (td.m_key_image_known && !td.m_key_image_partial) + { + std::vector<uint64_t> ring; + if (get_ring(key, td.m_key_image, ring)) + { + MINFO("This output has a known ring, reusing (size " << ring.size() << ")"); + THROW_WALLET_EXCEPTION_IF(ring.size() > fake_outputs_count + 1, error::wallet_internal_error, + "An output in this transaction was previously spent on another chain with ring size " + + std::to_string(ring.size()) + ", it cannot be spent now with ring size " + + std::to_string(fake_outputs_count + 1) + " as it is smaller: use a higher ring size"); + bool own_found = false; + existing_ring_found = true; + for (const auto &out: ring) + { + MINFO("Ring has output " << out); + if (out < num_outs) + { + MINFO("Using it"); + req.outputs.push_back({amount, out}); + ++num_found; + seen_indices.emplace(out); + if (out == td.m_global_output_index) + { + MINFO("This is the real output"); + own_found = true; + } + } + else + { + MINFO("Ignoring output " << out << ", too recent"); + } + } + THROW_WALLET_EXCEPTION_IF(!own_found, error::wallet_internal_error, + "Known ring does not include the spent output: " + std::to_string(td.m_global_output_index)); + } + } + + if (num_outs <= requested_outputs_count && !existing_ring_found) { for (uint64_t i = 0; i < num_outs; i++) req.outputs.push_back({amount, i}); @@ -5530,10 +6000,13 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> else { // start with real one - uint64_t num_found = 1; - seen_indices.emplace(td.m_global_output_index); - req.outputs.push_back({amount, td.m_global_output_index}); - LOG_PRINT_L1("Selecting real output: " << td.m_global_output_index << " for " << print_money(amount)); + if (num_found == 0) + { + num_found = 1; + seen_indices.emplace(td.m_global_output_index); + req.outputs.push_back({amount, td.m_global_output_index}); + LOG_PRINT_L1("Selecting real output: " << td.m_global_output_index << " for " << print_money(amount)); + } // while we still need more mixins while (num_found < requested_outputs_count) @@ -5547,6 +6020,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> // list of output indices we've seen. uint64_t i; + const char *type = ""; if (num_found - 1 < recent_outputs_count) // -1 to account for the real one we seeded with { // triangular distribution over [a,b) with a=0, mode c=b=up_index_limit @@ -5556,7 +6030,29 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> // just in case rounding up to 1 occurs after calc if (i == num_outs) --i; - LOG_PRINT_L2("picking " << i << " as recent"); + type = "recent"; + } + else if (num_found -1 < recent_outputs_count + pre_fork_outputs_count) + { + // triangular distribution over [a,b) with a=0, mode c=b=up_index_limit + uint64_t r = crypto::rand<uint64_t>() % ((uint64_t)1 << 53); + double frac = std::sqrt((double)r / ((uint64_t)1 << 53)); + i = (uint64_t)(frac*segregation_limit[amount].first); + // just in case rounding up to 1 occurs after calc + if (i == num_outs) + --i; + type = " pre-fork"; + } + else if (num_found -1 < recent_outputs_count + pre_fork_outputs_count + post_fork_outputs_count) + { + // triangular distribution over [a,b) with a=0, mode c=b=up_index_limit + uint64_t r = crypto::rand<uint64_t>() % ((uint64_t)1 << 53); + double frac = std::sqrt((double)r / ((uint64_t)1 << 53)); + i = (uint64_t)(frac*num_post_fork_outs) + segregation_limit[amount].first; + // just in case rounding up to 1 occurs after calc + if (i == num_post_fork_outs+segregation_limit[amount].first) + --i; + type = "post-fork"; } else { @@ -5567,13 +6063,14 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> // just in case rounding up to 1 occurs after calc if (i == num_outs) --i; - LOG_PRINT_L2("picking " << i << " as triangular"); + type = "triangular"; } if (seen_indices.count(i)) continue; seen_indices.emplace(i); + LOG_PRINT_L2("picking " << i << " as " << type); req.outputs.push_back({amount, i}); ++num_found; } @@ -5609,6 +6106,20 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> outs.back().reserve(fake_outputs_count + 1); const rct::key mask = td.is_rct() ? rct::commit(td.amount(), td.m_mask) : rct::zeroCommit(td.amount()); + uint64_t num_outs = 0; + const uint64_t amount = td.is_rct() ? 0 : td.amount(); + const bool output_is_pre_fork = td.m_block_height < segregation_fork_height; + if (m_segregate_pre_fork_outputs && output_is_pre_fork) + num_outs = segregation_limit[amount].first; + else for (const auto &he: resp_t.histogram) + { + if (he.amount == amount) + { + num_outs = he.unlocked_instances; + break; + } + } + // make sure the real outputs we asked for are really included, along // with the correct key and mask: this guards against an active attack // where the node sends dummy data for all outputs, and we then send @@ -5629,6 +6140,38 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> // pick real out first (it will be sorted when done) outs.back().push_back(std::make_tuple(td.m_global_output_index, boost::get<txout_to_key>(td.m_tx.vout[td.m_internal_output_index].target).key, mask)); + // then pick outs from an existing ring, if any + bool existing_ring_found = false; + if (td.m_key_image_known && !td.m_key_image_partial) + { + std::vector<uint64_t> ring; + if (get_ring(key, td.m_key_image, ring)) + { + for (uint64_t out: ring) + { + if (out < num_outs) + { + if (out != td.m_global_output_index) + { + bool found = false; + for (size_t o = 0; o < requested_outputs_count; ++o) + { + size_t i = base + o; + if (req.outputs[i].index == out) + { + LOG_PRINT_L2("Index " << i << "/" << requested_outputs_count << ": idx " << req.outputs[i].index << " (real " << td.m_global_output_index << "), unlocked " << daemon_resp.outs[i].unlocked << ", key " << daemon_resp.outs[i].key << " (from existing ring)"); + tx_add_fake_output(outs, req.outputs[i].index, daemon_resp.outs[i].key, daemon_resp.outs[i].mask, td.m_global_output_index, daemon_resp.outs[i].unlocked); + found = true; + break; + } + } + THROW_WALLET_EXCEPTION_IF(!found, error::wallet_internal_error, "Falied to find existing ring output in daemon out data"); + } + } + } + } + } + // then pick others in random order till we reach the required number // since we use an equiprobable pick here, we don't upset the triangular distribution std::vector<size_t> order; diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index d2e484e05..46f1ddc2d 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -66,6 +66,8 @@ class Serialization_portability_wallet_Test; namespace tools { + class ringdb; + class i_wallet2_callback { public: @@ -166,6 +168,7 @@ namespace tools static bool verify_password(const std::string& keys_file_name, const epee::wipeable_string& password, bool no_spend_key, hw::device &hwdev); wallet2(cryptonote::network_type nettype = cryptonote::MAINNET, bool restricted = false); + ~wallet2(); struct multisig_info { @@ -286,6 +289,7 @@ namespace tools uint64_t m_timestamp; uint32_t m_subaddr_account; // subaddress account of your wallet to be used in this transfer std::set<uint32_t> m_subaddr_indices; // set of address indices used as inputs in this transfer + std::vector<std::pair<crypto::key_image, std::vector<uint64_t>>> m_rings; // relative }; struct confirmed_transfer_details @@ -300,10 +304,11 @@ namespace tools uint64_t m_unlock_time; uint32_t m_subaddr_account; // subaddress account of your wallet to be used in this transfer std::set<uint32_t> m_subaddr_indices; // set of address indices used as inputs in this transfer + std::vector<std::pair<crypto::key_image, std::vector<uint64_t>>> m_rings; // relative confirmed_transfer_details(): m_amount_in(0), m_amount_out(0), m_change((uint64_t)-1), m_block_height(0), m_payment_id(crypto::null_hash), m_timestamp(0), m_unlock_time(0), m_subaddr_account((uint32_t)-1) {} confirmed_transfer_details(const unconfirmed_transfer_details &utd, uint64_t height): - m_amount_in(utd.m_amount_in), m_amount_out(utd.m_amount_out), m_change(utd.m_change), m_block_height(height), m_dests(utd.m_dests), m_payment_id(utd.m_payment_id), m_timestamp(utd.m_timestamp), m_unlock_time(utd.m_tx.unlock_time), m_subaddr_account(utd.m_subaddr_account), m_subaddr_indices(utd.m_subaddr_indices) {} + m_amount_in(utd.m_amount_in), m_amount_out(utd.m_amount_out), m_change(utd.m_change), m_block_height(height), m_dests(utd.m_dests), m_payment_id(utd.m_payment_id), m_timestamp(utd.m_timestamp), m_unlock_time(utd.m_tx.unlock_time), m_subaddr_account(utd.m_subaddr_account), m_subaddr_indices(utd.m_subaddr_indices), m_rings(utd.m_rings) {} }; struct tx_construction_data @@ -810,6 +815,9 @@ namespace tools if(ver < 23) return; a & m_account_tags; + if(ver < 24) + return; + a & m_ring_history_saved; } /*! @@ -859,6 +867,10 @@ namespace tools void confirm_export_overwrite(bool always) { m_confirm_export_overwrite = always; } bool auto_low_priority() const { return m_auto_low_priority; } void auto_low_priority(bool value) { m_auto_low_priority = value; } + bool segregate_pre_fork_outputs() const { return m_segregate_pre_fork_outputs; } + void segregate_pre_fork_outputs(bool value) { m_segregate_pre_fork_outputs = value; } + bool key_reuse_mitigation2() const { return m_key_reuse_mitigation2; } + void key_reuse_mitigation2(bool value) { m_key_reuse_mitigation2 = value; } bool get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys) const; void check_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, const cryptonote::account_public_address &address, uint64_t &received, bool &in_pool, uint64_t &confirmations); @@ -1046,6 +1058,18 @@ namespace tools return epee::net_utils::invoke_http_json_rpc(uri, method_name, req, res, m_http_client, timeout, http_method, req_id); } + void set_ring_database(const std::string &filename); + const std::string get_ring_database() const { return m_ring_database; } + bool get_ring(const crypto::key_image &key_image, std::vector<uint64_t> &outs); + bool get_rings(const crypto::hash &txid, std::vector<std::pair<crypto::key_image, std::vector<uint64_t>>> &outs); + bool set_ring(const crypto::key_image &key_image, const std::vector<uint64_t> &outs, bool relative); + bool find_and_save_rings(bool force = true); + + bool blackball_output(const crypto::public_key &output); + bool set_blackballed_outputs(const std::vector<crypto::public_key> &outputs, bool add = false); + bool unblackball_output(const crypto::public_key &output); + bool is_output_blackballed(const crypto::public_key &output) const; + private: /*! * \brief Stores wallet information to wallet file. @@ -1102,6 +1126,12 @@ namespace tools rct::multisig_kLRki get_multisig_kLRki(size_t n, const rct::key &k) const; rct::key get_multisig_k(size_t idx, const std::unordered_set<rct::key> &used_L) const; void update_multisig_rescan_info(const std::vector<std::vector<rct::key>> &multisig_k, const std::vector<std::vector<tools::wallet2::multisig_info>> &info, size_t n); + bool add_rings(const crypto::chacha_key &key, const cryptonote::transaction_prefix &tx); + bool add_rings(const cryptonote::transaction_prefix &tx); + bool remove_rings(const cryptonote::transaction_prefix &tx); + bool get_ring(const crypto::chacha_key &key, const crypto::key_image &key_image, std::vector<uint64_t> &outs); + + bool get_output_distribution(uint64_t &start_height, std::vector<uint64_t> &distribution); cryptonote::account_base m_account; boost::optional<epee::net_utils::http::login> m_daemon_login; @@ -1167,6 +1197,8 @@ namespace tools uint32_t m_confirm_backlog_threshold; bool m_confirm_export_overwrite; bool m_auto_low_priority; + bool m_segregate_pre_fork_outputs; + bool m_key_reuse_mitigation2; bool m_is_initialized; NodeRPCProxy m_node_rpc_proxy; std::unordered_set<crypto::hash> m_scanned_pool_txs[2]; @@ -1185,17 +1217,21 @@ namespace tools std::unordered_map<crypto::hash, address_tx> m_light_wallet_address_txs; // store calculated key image for faster lookup std::unordered_map<crypto::public_key, std::map<uint64_t, crypto::key_image> > m_key_image_cache; + + std::string m_ring_database; + bool m_ring_history_saved; + std::unique_ptr<ringdb> m_ringdb; }; } -BOOST_CLASS_VERSION(tools::wallet2, 23) +BOOST_CLASS_VERSION(tools::wallet2, 24) BOOST_CLASS_VERSION(tools::wallet2::transfer_details, 9) BOOST_CLASS_VERSION(tools::wallet2::multisig_info, 1) BOOST_CLASS_VERSION(tools::wallet2::multisig_info::LR, 0) BOOST_CLASS_VERSION(tools::wallet2::multisig_tx_set, 1) BOOST_CLASS_VERSION(tools::wallet2::payment_details, 3) BOOST_CLASS_VERSION(tools::wallet2::pool_payment_details, 1) -BOOST_CLASS_VERSION(tools::wallet2::unconfirmed_transfer_details, 7) -BOOST_CLASS_VERSION(tools::wallet2::confirmed_transfer_details, 5) +BOOST_CLASS_VERSION(tools::wallet2::unconfirmed_transfer_details, 8) +BOOST_CLASS_VERSION(tools::wallet2::confirmed_transfer_details, 6) BOOST_CLASS_VERSION(tools::wallet2::address_book_row, 17) BOOST_CLASS_VERSION(tools::wallet2::reserve_proof_entry, 0) BOOST_CLASS_VERSION(tools::wallet2::unsigned_tx_set, 0) @@ -1395,6 +1431,9 @@ namespace boost } a & x.m_subaddr_account; a & x.m_subaddr_indices; + if (ver < 8) + return; + a & x.m_rings; } template <class Archive> @@ -1439,6 +1478,9 @@ namespace boost } a & x.m_subaddr_account; a & x.m_subaddr_indices; + if (ver < 6) + return; + a & x.m_rings; } template <class Archive> diff --git a/src/wallet/wallet_errors.h b/src/wallet/wallet_errors.h index 32a0231b1..892a7922c 100644 --- a/src/wallet/wallet_errors.h +++ b/src/wallet/wallet_errors.h @@ -36,6 +36,7 @@ #include <vector> #include "cryptonote_basic/cryptonote_format_utils.h" +#include "cryptonote_core/cryptonote_tx_utils.h" #include "rpc/core_rpc_server_commands_defs.h" #include "include_base_utils.h" @@ -85,6 +86,7 @@ namespace tools // no_connection_to_daemon // is_key_image_spent_error // get_histogram_error + // get_output_distribution // wallet_files_doesnt_correspond // // * - class with protected ctor @@ -757,6 +759,14 @@ namespace tools } }; //---------------------------------------------------------------------------------------------------- + struct get_output_distribution : public wallet_rpc_error + { + explicit get_output_distribution(std::string&& loc, const std::string& request) + : wallet_rpc_error(std::move(loc), "failed to get output distribution", request) + { + } + }; + //---------------------------------------------------------------------------------------------------- struct wallet_files_doesnt_correspond : public wallet_logic_error { explicit wallet_files_doesnt_correspond(std::string&& loc, const std::string& keys_file, const std::string& wallet_file) diff --git a/tests/unit_tests/hardfork.cpp b/tests/unit_tests/hardfork.cpp index a2038ff0c..3e2199217 100644 --- a/tests/unit_tests/hardfork.cpp +++ b/tests/unit_tests/hardfork.cpp @@ -109,7 +109,8 @@ public: virtual bool for_all_key_images(std::function<bool(const crypto::key_image&)>) const { return true; } virtual bool for_blocks_range(const uint64_t&, const uint64_t&, std::function<bool(uint64_t, const crypto::hash&, const cryptonote::block&)>) const { return true; } virtual bool for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)>) const { return true; } - virtual bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, size_t tx_idx)> f) const { return true; } + virtual bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, uint64_t height, size_t tx_idx)> f) const { return true; } + virtual bool for_all_outputs(uint64_t amount, const std::function<bool(uint64_t height)> &f) const { return true; } virtual bool is_read_only() const { return false; } virtual std::map<uint64_t, std::tuple<uint64_t, uint64_t, uint64_t>> get_output_histogram(const std::vector<uint64_t> &amounts, bool unlocked, uint64_t recent_cutoff) const { return std::map<uint64_t, std::tuple<uint64_t, uint64_t, uint64_t>>(); } |