diff options
Diffstat (limited to 'src')
25 files changed, 2551 insertions, 1296 deletions
diff --git a/src/blockchain_utilities/CMakeLists.txt b/src/blockchain_utilities/CMakeLists.txt index 338ec3e4b..fe57895e1 100644 --- a/src/blockchain_utilities/CMakeLists.txt +++ b/src/blockchain_utilities/CMakeLists.txt @@ -91,6 +91,28 @@ monero_private_headers(blockchain_usage +set(blockchain_ancestry_sources + blockchain_ancestry.cpp + ) + +set(blockchain_ancestry_private_headers) + +monero_private_headers(blockchain_ancestry + ${blockchain_ancestry_private_headers}) + + + +set(blockchain_depth_sources + blockchain_depth.cpp + ) + +set(blockchain_depth_private_headers) + +monero_private_headers(blockchain_depth + ${blockchain_depth_private_headers}) + + + monero_add_executable(blockchain_import ${blockchain_import_sources} ${blockchain_import_private_headers} @@ -183,3 +205,45 @@ set_property(TARGET blockchain_usage OUTPUT_NAME "monero-blockchain-usage") install(TARGETS blockchain_usage DESTINATION bin) +monero_add_executable(blockchain_ancestry + ${blockchain_ancestry_sources} + ${blockchain_ancestry_private_headers}) + +target_link_libraries(blockchain_ancestry + PRIVATE + cryptonote_core + blockchain_db + version + epee + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${Boost_THREAD_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES}) + +set_property(TARGET blockchain_ancestry + PROPERTY + OUTPUT_NAME "monero-blockchain-ancestry") +install(TARGETS blockchain_ancestry DESTINATION bin) + +monero_add_executable(blockchain_depth + ${blockchain_depth_sources} + ${blockchain_depth_private_headers}) + +target_link_libraries(blockchain_depth + PRIVATE + cryptonote_core + blockchain_db + version + epee + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${Boost_THREAD_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES}) + +set_property(TARGET blockchain_depth + PROPERTY + OUTPUT_NAME "monero-blockchain-depth") +install(TARGETS blockchain_depth DESTINATION bin) + diff --git a/src/blockchain_utilities/blockchain_ancestry.cpp b/src/blockchain_utilities/blockchain_ancestry.cpp new file mode 100644 index 000000000..2f0bbffd6 --- /dev/null +++ b/src/blockchain_utilities/blockchain_ancestry.cpp @@ -0,0 +1,772 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <unordered_map> +#include <unordered_set> +#include <boost/range/adaptor/transformed.hpp> +#include <boost/algorithm/string.hpp> +#include <boost/archive/portable_binary_iarchive.hpp> +#include <boost/archive/portable_binary_oarchive.hpp> +#include "common/unordered_containers_boost_serialization.h" +#include "common/command_line.h" +#include "common/varint.h" +#include "cryptonote_basic/cryptonote_boost_serialization.h" +#include "cryptonote_core/tx_pool.h" +#include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_core/blockchain.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/db_types.h" +#include "version.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "bcutil" + +namespace po = boost::program_options; +using namespace epee; +using namespace cryptonote; + +static bool stop_requested = false; + +struct ancestor +{ + uint64_t amount; + uint64_t offset; + + bool operator==(const ancestor &other) const { return amount == other.amount && offset == other.offset; } + + template <typename t_archive> void serialize(t_archive &a, const unsigned int ver) + { + a & amount; + a & offset; + } +}; +BOOST_CLASS_VERSION(ancestor, 0) + +namespace std +{ + template<> struct hash<ancestor> + { + size_t operator()(const ancestor &a) const + { + return a.amount ^ a.offset; // not that bad, since amount almost always have a high bit set, and offset doesn't + } + }; +} + +struct tx_data_t +{ + std::vector<std::pair<uint64_t, std::vector<uint64_t>>> vin; + std::vector<crypto::public_key> vout; + bool coinbase; + + tx_data_t(): coinbase(false) {} + tx_data_t(const cryptonote::transaction &tx) + { + coinbase = tx.vin.size() == 1 && tx.vin[0].type() == typeid(cryptonote::txin_gen); + if (!coinbase) + { + vin.reserve(tx.vin.size()); + for (size_t ring = 0; ring < tx.vin.size(); ++ring) + { + if (tx.vin[ring].type() == typeid(cryptonote::txin_to_key)) + { + const cryptonote::txin_to_key &txin = boost::get<cryptonote::txin_to_key>(tx.vin[ring]); + vin.push_back(std::make_pair(txin.amount, cryptonote::relative_output_offsets_to_absolute(txin.key_offsets))); + } + else + { + LOG_PRINT_L0("Bad vin type in txid " << get_transaction_hash(tx)); + throw std::runtime_error("Bad vin type"); + } + } + } + vout.reserve(tx.vout.size()); + for (size_t out = 0; out < tx.vout.size(); ++out) + { + if (tx.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get<cryptonote::txout_to_key>(tx.vout[out].target); + vout.push_back(txout.key); + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << get_transaction_hash(tx)); + throw std::runtime_error("Bad vout type"); + } + } + } + + template <typename t_archive> void serialize(t_archive &a, const unsigned int ver) + { + a & coinbase; + a & vin; + a & vout; + } +}; + +struct ancestry_state_t +{ + uint64_t height; + std::unordered_map<crypto::hash, std::unordered_set<ancestor>> ancestry; + std::unordered_map<ancestor, crypto::hash> output_cache; + std::unordered_map<crypto::hash, ::tx_data_t> tx_cache; + std::vector<cryptonote::block> block_cache; + + template <typename t_archive> void serialize(t_archive &a, const unsigned int ver) + { + a & height; + a & ancestry; + a & output_cache; + if (ver < 1) + { + std::unordered_map<crypto::hash, cryptonote::transaction> old_tx_cache; + a & old_tx_cache; + for (const auto i: old_tx_cache) + tx_cache.insert(std::make_pair(i.first, ::tx_data_t(i.second))); + } + else + { + a & tx_cache; + } + if (ver < 2) + { + std::unordered_map<uint64_t, cryptonote::block> old_block_cache; + a & old_block_cache; + block_cache.resize(old_block_cache.size()); + for (const auto i: old_block_cache) + block_cache[i.first] = i.second; + } + else + { + a & block_cache; + } + } +}; +BOOST_CLASS_VERSION(ancestry_state_t, 2) + +static void add_ancestor(std::unordered_map<ancestor, unsigned int> &ancestry, uint64_t amount, uint64_t offset) +{ + std::pair<std::unordered_map<ancestor, unsigned int>::iterator, bool> p = ancestry.insert(std::make_pair(ancestor{amount, offset}, 1)); + if (!p.second) + { + ++p.first->second; + } +} + +static size_t get_full_ancestry(const std::unordered_map<ancestor, unsigned int> &ancestry) +{ + size_t count = 0; + for (const auto &i: ancestry) + count += i.second; + return count; +} + +static size_t get_deduplicated_ancestry(const std::unordered_map<ancestor, unsigned int> &ancestry) +{ + return ancestry.size(); +} + +static void add_ancestry(std::unordered_map<crypto::hash, std::unordered_set<ancestor>> &ancestry, const crypto::hash &txid, const std::unordered_set<ancestor> &ancestors) +{ + std::pair<std::unordered_map<crypto::hash, std::unordered_set<ancestor>>::iterator, bool> p = ancestry.insert(std::make_pair(txid, ancestors)); + if (!p.second) + { + for (const auto &e: ancestors) + p.first->second.insert(e); + } +} + +static void add_ancestry(std::unordered_map<crypto::hash, std::unordered_set<ancestor>> &ancestry, const crypto::hash &txid, const ancestor &new_ancestor) +{ + std::pair<std::unordered_map<crypto::hash, std::unordered_set<ancestor>>::iterator, bool> p = ancestry.insert(std::make_pair(txid, std::unordered_set<ancestor>())); + p.first->second.insert(new_ancestor); +} + +static std::unordered_set<ancestor> get_ancestry(const std::unordered_map<crypto::hash, std::unordered_set<ancestor>> &ancestry, const crypto::hash &txid) +{ + std::unordered_map<crypto::hash, std::unordered_set<ancestor>>::const_iterator i = ancestry.find(txid); + if (i == ancestry.end()) + { + //MERROR("txid ancestry not found: " << txid); + //throw std::runtime_error("txid ancestry not found"); + return std::unordered_set<ancestor>(); + } + return i->second; +} + +int main(int argc, char* argv[]) +{ + TRY_ENTRY(); + + epee::string_tools::set_module_name_and_folder(argv[0]); + + std::string default_db_type = "lmdb"; + + std::string available_dbs = cryptonote::blockchain_db_types(", "); + available_dbs = "available: " + available_dbs; + + uint32_t log_level = 0; + + tools::on_startup(); + + boost::filesystem::path output_file_path; + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""}; + const command_line::arg_descriptor<std::string> arg_database = { + "database", available_dbs.c_str(), default_db_type + }; + const command_line::arg_descriptor<std::string> arg_txid = {"txid", "Get ancestry for this txid", ""}; + const command_line::arg_descriptor<uint64_t> arg_height = {"height", "Get ancestry for all txes at this height", 0}; + const command_line::arg_descriptor<bool> arg_all = {"all", "Include the whole chain", false}; + const command_line::arg_descriptor<bool> arg_cache_outputs = {"cache-outputs", "Cache outputs (memory hungry)", false}; + const command_line::arg_descriptor<bool> arg_cache_txes = {"cache-txes", "Cache txes (memory hungry)", false}; + const command_line::arg_descriptor<bool> arg_cache_blocks = {"cache-blocks", "Cache blocks (memory hungry)", false}; + const command_line::arg_descriptor<bool> arg_include_coinbase = {"include-coinbase", "Including coinbase tx", false}; + const command_line::arg_descriptor<bool> arg_show_cache_stats = {"show-cache-stats", "Show cache statistics", false}; + + command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_database); + command_line::add_arg(desc_cmd_sett, arg_txid); + command_line::add_arg(desc_cmd_sett, arg_height); + command_line::add_arg(desc_cmd_sett, arg_all); + command_line::add_arg(desc_cmd_sett, arg_cache_outputs); + command_line::add_arg(desc_cmd_sett, arg_cache_txes); + command_line::add_arg(desc_cmd_sett, arg_cache_blocks); + command_line::add_arg(desc_cmd_sett, arg_include_coinbase); + command_line::add_arg(desc_cmd_sett, arg_show_cache_stats); + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + auto parser = po::command_line_parser(argc, argv).options(desc_options); + po::store(parser.run(), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + mlog_configure(mlog_get_default_log_path("monero-blockchain-ancestry.log"), true); + if (!command_line::is_arg_defaulted(vm, arg_log_level)) + mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); + else + mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str()); + + LOG_PRINT_L0("Starting..."); + + std::string opt_data_dir = command_line::get_arg(vm, cryptonote::arg_data_dir); + bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); + bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); + network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; + std::string opt_txid_string = command_line::get_arg(vm, arg_txid); + uint64_t opt_height = command_line::get_arg(vm, arg_height); + bool opt_all = command_line::get_arg(vm, arg_all); + bool opt_cache_outputs = command_line::get_arg(vm, arg_cache_outputs); + bool opt_cache_txes = command_line::get_arg(vm, arg_cache_txes); + bool opt_cache_blocks = command_line::get_arg(vm, arg_cache_blocks); + bool opt_include_coinbase = command_line::get_arg(vm, arg_include_coinbase); + bool opt_show_cache_stats = command_line::get_arg(vm, arg_show_cache_stats); + + if ((!opt_txid_string.empty()) + !!opt_height + !!opt_all > 1) + { + std::cerr << "Only one of --txid, --height and --all can be given" << std::endl; + return 1; + } + crypto::hash opt_txid = crypto::null_hash; + if (!opt_txid_string.empty()) + { + if (!epee::string_tools::hex_to_pod(opt_txid_string, opt_txid)) + { + std::cerr << "Invalid txid" << std::endl; + return 1; + } + } + + std::string db_type = command_line::get_arg(vm, arg_database); + if (!cryptonote::blockchain_valid_db_type(db_type)) + { + std::cerr << "Invalid database type: " << db_type << std::endl; + return 1; + } + + // If we wanted to use the memory pool, we would set up a fake_core. + + // Use Blockchain instead of lower-level BlockchainDB for two reasons: + // 1. Blockchain has the init() method for easy setup + // 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash() + // + // cannot match blockchain_storage setup above with just one line, + // e.g. + // Blockchain* core_storage = new Blockchain(NULL); + // because unlike blockchain_storage constructor, which takes a pointer to + // tx_memory_pool, Blockchain's constructor takes tx_memory_pool object. + LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); + std::unique_ptr<Blockchain> core_storage; + tx_memory_pool m_mempool(*core_storage); + core_storage.reset(new Blockchain(m_mempool)); + BlockchainDB *db = new_db(db_type); + if (db == NULL) + { + LOG_ERROR("Attempted to use non-existent database type: " << db_type); + throw std::runtime_error("Attempting to use non-existent database type"); + } + LOG_PRINT_L0("database: " << db_type); + + const std::string filename = (boost::filesystem::path(opt_data_dir) / db->get_db_name()).string(); + LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); + + try + { + db->open(filename, DBF_RDONLY); + } + catch (const std::exception& e) + { + LOG_PRINT_L0("Error opening database: " << e.what()); + return 1; + } + r = core_storage->init(db, net_type); + + CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage"); + LOG_PRINT_L0("Source blockchain storage initialized OK"); + + std::vector<crypto::hash> start_txids; + + // forward method + if (opt_all) + { + uint64_t cached_txes = 0, cached_blocks = 0, cached_outputs = 0, total_txes = 0, total_blocks = 0, total_outputs = 0; + ancestry_state_t state; + + const std::string state_file_path = (boost::filesystem::path(opt_data_dir) / "ancestry-state.bin").string(); + LOG_PRINT_L0("Loading state data from " << state_file_path); + std::ifstream state_data_in; + state_data_in.open(state_file_path, std::ios_base::binary | std::ios_base::in); + if (!state_data_in.fail()) + { + try + { + boost::archive::portable_binary_iarchive a(state_data_in); + a >> state; + } + catch (const std::exception &e) + { + MERROR("Failed to load state data from " << state_file_path << ", restarting from scratch"); + state = ancestry_state_t(); + } + state_data_in.close(); + } + + tools::signal_handler::install([](int type) { + stop_requested = true; + }); + + MINFO("Starting from height " << state.height); + const uint64_t db_height = db->height(); + state.block_cache.reserve(db_height); + for (uint64_t h = state.height; h < db_height; ++h) + { + size_t block_ancestry_size = 0; + const crypto::hash block_hash = db->get_block_hash_from_height(h); + const cryptonote::blobdata bd = db->get_block_blob(block_hash); + ++total_blocks; + cryptonote::block b; + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + if (opt_cache_blocks) + { + state.block_cache.resize(h + 1); + state.block_cache[h] = b; + } + std::vector<crypto::hash> txids; + txids.reserve(1 + b.tx_hashes.size()); + if (opt_include_coinbase) + txids.push_back(cryptonote::get_transaction_hash(b.miner_tx)); + for (const auto &h: b.tx_hashes) + txids.push_back(h); + for (const crypto::hash &txid: txids) + { + printf("%lu/%lu \r", (unsigned long)h, (unsigned long)db_height); + fflush(stdout); + ::tx_data_t tx_data; + std::unordered_map<crypto::hash, ::tx_data_t>::const_iterator i = state.tx_cache.find(txid); + ++total_txes; + if (i != state.tx_cache.end()) + { + ++cached_txes; + tx_data = i->second; + } + else + { + cryptonote::blobdata bd; + if (!db->get_pruned_tx_blob(txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << txid << " from db"); + return 1; + } + cryptonote::transaction tx; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) + { + LOG_PRINT_L0("Bad tx: " << txid); + return 1; + } + tx_data = ::tx_data_t(tx); + if (opt_cache_txes) + state.tx_cache.insert(std::make_pair(txid, tx_data)); + } + if (tx_data.coinbase) + { + add_ancestry(state.ancestry, txid, std::unordered_set<ancestor>()); + } + else + { + for (size_t ring = 0; ring < tx_data.vin.size(); ++ring) + { + if (1) + { + const uint64_t amount = tx_data.vin[ring].first; + const std::vector<uint64_t> &absolute_offsets = tx_data.vin[ring].second; + for (uint64_t offset: absolute_offsets) + { + const output_data_t od = db->get_output_key(amount, offset); + add_ancestry(state.ancestry, txid, ancestor{amount, offset}); + cryptonote::block b; + ++total_blocks; + if (state.block_cache.size() > od.height && !state.block_cache[od.height].miner_tx.vin.empty()) + { + ++cached_blocks; + b = state.block_cache[od.height]; + } + else + { + const crypto::hash block_hash = db->get_block_hash_from_height(od.height); + cryptonote::blobdata bd = db->get_block_blob(block_hash); + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + if (opt_cache_blocks) + { + state.block_cache.resize(od.height + 1); + state.block_cache[od.height] = b; + } + } + // find the tx which created this output + bool found = false; + std::unordered_map<ancestor, crypto::hash>::const_iterator i = state.output_cache.find({amount, offset}); + ++total_outputs; + if (i != state.output_cache.end()) + { + ++cached_outputs; + add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, i->second)); + found = true; + } + else for (size_t out = 0; out < b.miner_tx.vout.size(); ++out) + { + if (b.miner_tx.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get<cryptonote::txout_to_key>(b.miner_tx.vout[out].target); + if (txout.key == od.pubkey) + { + found = true; + add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, cryptonote::get_transaction_hash(b.miner_tx))); + if (opt_cache_outputs) + state.output_cache.insert(std::make_pair(ancestor{amount, offset}, cryptonote::get_transaction_hash(b.miner_tx))); + break; + } + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << cryptonote::get_transaction_hash(b.miner_tx)); + return 1; + } + } + for (const crypto::hash &block_txid: b.tx_hashes) + { + if (found) + break; + ::tx_data_t tx_data2; + std::unordered_map<crypto::hash, ::tx_data_t>::const_iterator i = state.tx_cache.find(block_txid); + ++total_txes; + if (i != state.tx_cache.end()) + { + ++cached_txes; + tx_data2 = i->second; + } + else + { + cryptonote::blobdata bd; + if (!db->get_pruned_tx_blob(block_txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << block_txid << " from db"); + return 1; + } + cryptonote::transaction tx; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) + { + LOG_PRINT_L0("Bad tx: " << block_txid); + return 1; + } + tx_data2 = ::tx_data_t(tx); + if (opt_cache_txes) + state.tx_cache.insert(std::make_pair(block_txid, tx_data2)); + } + for (size_t out = 0; out < tx_data2.vout.size(); ++out) + { + if (tx_data2.vout[out] == od.pubkey) + { + found = true; + add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, block_txid)); + if (opt_cache_outputs) + state.output_cache.insert(std::make_pair(ancestor{amount, offset}, block_txid)); + break; + } + } + } + if (!found) + { + LOG_PRINT_L0("Output originating transaction not found"); + return 1; + } + } + } + } + } + const size_t ancestry_size = get_ancestry(state.ancestry, txid).size(); + block_ancestry_size += ancestry_size; + MINFO(txid << ": " << ancestry_size); + } + if (!txids.empty()) + { + std::string stats_msg; + if (opt_show_cache_stats) + stats_msg = std::string(", cache: txes ") + std::to_string(cached_txes*100./total_txes) + + ", blocks " + std::to_string(cached_blocks*100./total_blocks) + ", outputs " + + std::to_string(cached_outputs*100./total_outputs); + MINFO("Height " << h << ": " << (block_ancestry_size / txids.size()) << " average over " << txids.size() << stats_msg); + } + state.height = h; + if (stop_requested) + break; + } + + LOG_PRINT_L0("Saving state data to " << state_file_path); + std::ofstream state_data_out; + state_data_out.open(state_file_path, std::ios_base::binary | std::ios_base::out | std::ios::trunc); + if (!state_data_out.fail()) + { + try + { + boost::archive::portable_binary_oarchive a(state_data_out); + a << state; + } + catch (const std::exception &e) + { + MERROR("Failed to save state data to " << state_file_path); + } + state_data_out.close(); + } + + goto done; + } + + if (!opt_txid_string.empty()) + { + start_txids.push_back(opt_txid); + } + else + { + const crypto::hash block_hash = db->get_block_hash_from_height(opt_height); + const cryptonote::blobdata bd = db->get_block_blob(block_hash); + cryptonote::block b; + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + for (const crypto::hash &txid: b.tx_hashes) + start_txids.push_back(txid); + } + + if (start_txids.empty()) + { + LOG_PRINT_L0("No transaction(s) to check"); + return 1; + } + + for (const crypto::hash &start_txid: start_txids) + { + LOG_PRINT_L0("Checking ancestry for txid " << start_txid); + + std::unordered_map<ancestor, unsigned int> ancestry; + + std::list<crypto::hash> txids; + txids.push_back(start_txid); + while (!txids.empty()) + { + const crypto::hash txid = txids.front(); + txids.pop_front(); + + cryptonote::blobdata bd; + if (!db->get_pruned_tx_blob(txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << txid << " from db"); + return 1; + } + cryptonote::transaction tx; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) + { + LOG_PRINT_L0("Bad tx: " << txid); + return 1; + } + const bool coinbase = tx.vin.size() == 1 && tx.vin[0].type() == typeid(cryptonote::txin_gen); + if (coinbase) + continue; + + for (size_t ring = 0; ring < tx.vin.size(); ++ring) + { + if (tx.vin[ring].type() == typeid(cryptonote::txin_to_key)) + { + const cryptonote::txin_to_key &txin = boost::get<cryptonote::txin_to_key>(tx.vin[ring]); + const uint64_t amount = txin.amount; + auto absolute_offsets = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets); + for (uint64_t offset: absolute_offsets) + { + add_ancestor(ancestry, amount, offset); + const output_data_t od = db->get_output_key(amount, offset); + const crypto::hash block_hash = db->get_block_hash_from_height(od.height); + bd = db->get_block_blob(block_hash); + cryptonote::block b; + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + // find the tx which created this output + bool found = false; + for (size_t out = 0; out < b.miner_tx.vout.size(); ++out) + { + if (b.miner_tx.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get<cryptonote::txout_to_key>(b.miner_tx.vout[out].target); + if (txout.key == od.pubkey) + { + found = true; + txids.push_back(cryptonote::get_transaction_hash(b.miner_tx)); + MDEBUG("adding txid: " << cryptonote::get_transaction_hash(b.miner_tx)); + break; + } + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << cryptonote::get_transaction_hash(b.miner_tx)); + return 1; + } + } + for (const crypto::hash &block_txid: b.tx_hashes) + { + if (found) + break; + if (!db->get_pruned_tx_blob(block_txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << block_txid << " from db"); + return 1; + } + cryptonote::transaction tx2; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx2)) + { + LOG_PRINT_L0("Bad tx: " << block_txid); + return 1; + } + for (size_t out = 0; out < tx2.vout.size(); ++out) + { + if (tx2.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get<cryptonote::txout_to_key>(tx2.vout[out].target); + if (txout.key == od.pubkey) + { + found = true; + txids.push_back(block_txid); + MDEBUG("adding txid: " << block_txid); + break; + } + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << block_txid); + return 1; + } + } + } + if (!found) + { + LOG_PRINT_L0("Output originating transaction not found"); + return 1; + } + } + } + else + { + LOG_PRINT_L0("Bad vin type in txid " << txid); + return 1; + } + } + } + + MINFO("Ancestry for " << start_txid << ": " << get_deduplicated_ancestry(ancestry) << " / " << get_full_ancestry(ancestry)); + for (const auto &i: ancestry) + { + MINFO(cryptonote::print_money(i.first.amount) << "/" << i.first.offset << ": " << i.second); + } + } + +done: + core_storage->deinit(); + return 0; + + CATCH_ENTRY("Depth query error", 1); +} diff --git a/src/blockchain_utilities/blockchain_blackball.cpp b/src/blockchain_utilities/blockchain_blackball.cpp index 1653910fc..d9a179f64 100644 --- a/src/blockchain_utilities/blockchain_blackball.cpp +++ b/src/blockchain_utilities/blockchain_blackball.cpp @@ -50,64 +50,86 @@ namespace po = boost::program_options; using namespace epee; using namespace cryptonote; +static const char zerokey[8] = {0}; +static const MDB_val zerokval = { sizeof(zerokey), (void *)zerokey }; + +static uint64_t records_per_sync = 200; +static uint64_t db_flags = 0; +static MDB_dbi dbi_relative_rings; +static MDB_dbi dbi_outputs; +static MDB_dbi dbi_processed_txidx; +static MDB_dbi dbi_spent; +static MDB_dbi dbi_ring_instances; +static MDB_dbi dbi_stats; +static MDB_env *env = NULL; + struct output_data { uint64_t amount; - uint64_t index; - output_data(): amount(0), index(0) {} - output_data(uint64_t a, uint64_t i): amount(a), index(i) {} - bool operator==(const output_data &other) const { return other.amount == amount && other.index == index; } - template <typename t_archive> void serialize(t_archive &a, const unsigned int ver) - { - a & amount; - a & index; - } + uint64_t offset; + output_data(): amount(0), offset(0) {} + output_data(uint64_t a, uint64_t i): amount(a), offset(i) {} + bool operator==(const output_data &other) const { return other.amount == amount && other.offset == offset; } }; -BOOST_CLASS_VERSION(output_data, 0) -namespace std +// +// relative_rings: key_image -> vector<uint64_t> +// outputs: 128 bits -> set of key images +// processed_txidx: string -> uint64_t +// spent: amount -> offset +// ring_instances: vector<uint64_t> -> uint64_t +// stats: string -> arbitrary +// + +static bool parse_db_sync_mode(std::string db_sync_mode) { - template<> struct hash<output_data> + std::vector<std::string> options; + boost::trim(db_sync_mode); + boost::split(options, db_sync_mode, boost::is_any_of(" :")); + + for(const auto &option : options) + MDEBUG("option: " << option); + + // default to fast:async:1 + uint64_t DEFAULT_FLAGS = DBF_FAST; + + if(options.size() == 0) + { + // default to fast:async:1 + db_flags = DEFAULT_FLAGS; + } + + bool safemode = false; + if(options.size() >= 1) { - size_t operator()(const output_data &od) const + if(options[0] == "safe") { - const uint64_t data[2] = {od.amount, od.index}; - crypto::hash h; - crypto::cn_fast_hash(data, 2 * sizeof(uint64_t), h); - return reinterpret_cast<const std::size_t &>(h); + safemode = true; + db_flags = DBF_SAFE; } - }; - template<> struct hash<std::vector<uint64_t>> - { - size_t operator()(const std::vector<uint64_t> &v) const + else if(options[0] == "fast") { - crypto::hash h; - crypto::cn_fast_hash(v.data(), v.size() * sizeof(uint64_t), h); - return reinterpret_cast<const std::size_t &>(h); + db_flags = DBF_FAST; } - }; -} - -struct blackball_state_t -{ - std::unordered_map<crypto::key_image, std::vector<uint64_t>> relative_rings; - std::unordered_map<output_data, std::unordered_set<crypto::key_image>> outputs; - std::unordered_map<std::string, uint64_t> processed_heights; - std::unordered_set<output_data> spent; - std::unordered_map<std::vector<uint64_t>, size_t> ring_instances; + else if(options[0] == "fastest") + { + db_flags = DBF_FASTEST; + records_per_sync = 1000; // default to fastest:async:1000 + } + else + db_flags = DEFAULT_FLAGS; + } - template <typename t_archive> void serialize(t_archive &a, const unsigned int ver) + if(options.size() >= 2 && !safemode) { - a & relative_rings; - a & outputs; - a & processed_heights; - a & spent; - if (ver < 1) - return; - a & ring_instances; + char *endptr; + uint64_t bps = strtoull(options[1].c_str(), &endptr, 0); + if (*endptr == '\0') + records_per_sync = bps; } -}; -BOOST_CLASS_VERSION(blackball_state_t, 1) + + return true; +} static std::string get_default_db_path() { @@ -118,7 +140,195 @@ static std::string get_default_db_path() return dir.string(); } -static bool for_all_transactions(const std::string &filename, uint64_t &start_idx, const std::function<bool(const cryptonote::transaction_prefix&)> &f) +static std::string get_cache_filename(boost::filesystem::path filename) +{ + if (!boost::filesystem::is_directory(filename)) + filename.remove_filename(); + return filename.string(); +} + +static int compare_hash32(const MDB_val *a, const MDB_val *b) +{ + const uint32_t *va = (const uint32_t*) a->mv_data; + const uint32_t *vb = (const uint32_t*) b->mv_data; + for (int n = 7; n >= 0; n--) + { + if (va[n] == vb[n]) + continue; + return va[n] < vb[n] ? -1 : 1; + } + + return 0; +} + +int compare_uint64(const MDB_val *a, const MDB_val *b) +{ + const uint64_t va = *(const uint64_t *)a->mv_data; + const uint64_t vb = *(const uint64_t *)b->mv_data; + return (va < vb) ? -1 : va > vb; +} + +static int compare_double64(const MDB_val *a, const MDB_val *b) +{ + const uint64_t va = *(const uint64_t*) a->mv_data; + const uint64_t vb = *(const uint64_t*) b->mv_data; + if (va == vb) + { + const uint64_t va = ((const uint64_t*) a->mv_data)[1]; + const uint64_t vb = ((const uint64_t*) b->mv_data)[1]; + return va < vb ? -1 : va > vb; + } + return va < vb ? -1 : va > vb; +} + +static int resize_env(const char *db_path) +{ + MDB_envinfo mei; + MDB_stat mst; + int ret; + + size_t needed = 1000ul * 1024 * 1024; // at least 1000 MB + + ret = mdb_env_info(env, &mei); + if (ret) + return ret; + ret = mdb_env_stat(env, &mst); + if (ret) + return ret; + uint64_t size_used = mst.ms_psize * mei.me_last_pgno; + uint64_t mapsize = mei.me_mapsize; + if (size_used + needed > mei.me_mapsize) + { + try + { + boost::filesystem::path path(db_path); + boost::filesystem::space_info si = boost::filesystem::space(path); + if(si.available < needed) + { + MERROR("!! WARNING: Insufficient free space to extend database !!: " << (si.available >> 20L) << " MB available"); + return ENOSPC; + } + } + catch(...) + { + // print something but proceed. + MWARNING("Unable to query free disk space."); + } + + mapsize += needed; + } + return mdb_env_set_mapsize(env, mapsize); +} + +static void init(std::string cache_filename) +{ + MDB_txn *txn; + bool tx_active = false; + int dbr; + + MINFO("Creating blackball cache in " << cache_filename); + + tools::create_directories_if_necessary(cache_filename); + + int flags = 0; + if (db_flags & DBF_FAST) + flags |= MDB_NOSYNC; + if (db_flags & DBF_FASTEST) + flags |= MDB_NOSYNC | MDB_WRITEMAP | MDB_MAPASYNC; + + dbr = mdb_env_create(&env); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); + dbr = mdb_env_set_maxdbs(env, 6); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); + const std::string actual_filename = get_cache_filename(cache_filename); + dbr = mdb_env_open(env, actual_filename.c_str(), flags, 0664); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open rings database file '" + + actual_filename + "': " + std::string(mdb_strerror(dbr))); + + dbr = mdb_txn_begin(env, NULL, 0, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + dbr = mdb_dbi_open(txn, "relative_rings", MDB_CREATE | MDB_INTEGERKEY, &dbi_relative_rings); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_compare(txn, dbi_relative_rings, compare_hash32); + + dbr = mdb_dbi_open(txn, "outputs", MDB_CREATE | MDB_INTEGERKEY, &dbi_outputs); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_compare(txn, dbi_outputs, compare_double64); + + dbr = mdb_dbi_open(txn, "processed_txidx", MDB_CREATE, &dbi_processed_txidx); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_dbi_open(txn, "spent", MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi_spent); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_dupsort(txn, dbi_spent, compare_uint64); + + dbr = mdb_dbi_open(txn, "ring_instances", MDB_CREATE, &dbi_ring_instances); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_dbi_open(txn, "stats", MDB_CREATE, &dbi_stats); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); + tx_active = false; +} + +static void close() +{ + if (env) + { + mdb_dbi_close(env, dbi_relative_rings); + mdb_dbi_close(env, dbi_outputs); + mdb_dbi_close(env, dbi_processed_txidx); + mdb_dbi_close(env, dbi_spent); + mdb_dbi_close(env, dbi_ring_instances); + mdb_dbi_close(env, dbi_stats); + mdb_env_close(env); + env = NULL; + } +} + +static std::string compress_ring(const std::vector<uint64_t> &ring, std::string s = "") +{ + const size_t sz = s.size(); + s.resize(s.size() + 12 * ring.size()); + char *ptr = (char*)s.data() + sz; + for (uint64_t out: ring) + tools::write_varint(ptr, out); + if (ptr > s.data() + sz + 12 * ring.size()) + throw std::runtime_error("varint output overflow"); + s.resize(ptr - s.data()); + return s; +} + +static std::string compress_ring(uint64_t amount, const std::vector<uint64_t> &ring) +{ + char s[12], *ptr = s; + tools::write_varint(ptr, amount); + if (ptr > s + sizeof(s)) + throw std::runtime_error("varint output overflow"); + return compress_ring(ring, std::string(s, ptr-s)); +} + +static std::vector<uint64_t> decompress_ring(const std::string &s) +{ + std::vector<uint64_t> ring; + int read = 0; + for (std::string::const_iterator i = s.begin(); i != s.cend(); std::advance(i, read)) + { + uint64_t out; + std::string tmp(i, s.cend()); + read = tools::read_varint(tmp.begin(), tmp.end(), out); + CHECK_AND_ASSERT_THROW_MES(read > 0 && read <= 256, "Internal error decompressing ring"); + ring.push_back(out); + } + return ring; +} + +static bool for_all_transactions(const std::string &filename, uint64_t &start_idx, uint64_t &n_txes, const std::function<bool(const cryptonote::transaction_prefix&)> &f) { MDB_env *env; MDB_dbi dbi; @@ -126,6 +336,8 @@ static bool for_all_transactions(const std::string &filename, uint64_t &start_id MDB_cursor *cur; int dbr; bool tx_active = false; + MDB_val k; + MDB_val v; dbr = mdb_env_create(&env); if (dbr) throw std::runtime_error("Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); @@ -136,7 +348,7 @@ static bool for_all_transactions(const std::string &filename, uint64_t &start_id if (dbr) throw std::runtime_error("Failed to open rings database file '" + actual_filename + "': " + std::string(mdb_strerror(dbr))); - dbr = mdb_txn_begin(env, NULL, 0, &txn); + dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); tx_active = true; @@ -147,9 +359,11 @@ static bool for_all_transactions(const std::string &filename, uint64_t &start_id if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); dbr = mdb_cursor_open(txn, dbi, &cur); if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + MDB_stat stat; + dbr = mdb_stat(txn, dbi, &stat); + if (dbr) throw std::runtime_error("Failed to query m_block_info: " + std::string(mdb_strerror(dbr))); + n_txes = stat.ms_entries; - MDB_val k; - MDB_val v; bool fret = true; k.mv_size = sizeof(uint64_t); @@ -194,6 +408,77 @@ static bool for_all_transactions(const std::string &filename, uint64_t &start_id return fret; } +static uint64_t find_first_diverging_transaction(const std::string &first_filename, const std::string &second_filename) +{ + MDB_env *env[2]; + MDB_dbi dbi[2]; + MDB_txn *txn[2]; + MDB_cursor *cur[2]; + int dbr; + bool tx_active[2] = { false, false }; + uint64_t n_txes[2]; + MDB_val k; + MDB_val v[2]; + + epee::misc_utils::auto_scope_leave_caller txn_dtor[2]; + for (int i = 0; i < 2; ++i) + { + dbr = mdb_env_create(&env[i]); + if (dbr) throw std::runtime_error("Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); + dbr = mdb_env_set_maxdbs(env[i], 2); + if (dbr) throw std::runtime_error("Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); + const std::string actual_filename = i ? second_filename : first_filename; + dbr = mdb_env_open(env[i], actual_filename.c_str(), 0, 0664); + if (dbr) throw std::runtime_error("Failed to open rings database file '" + + actual_filename + "': " + std::string(mdb_strerror(dbr))); + + dbr = mdb_txn_begin(env[i], NULL, MDB_RDONLY, &txn[i]); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + txn_dtor[i] = epee::misc_utils::create_scope_leave_handler([&, i](){if (tx_active[i]) mdb_txn_abort(txn[i]);}); + tx_active[i] = true; + + dbr = mdb_dbi_open(txn[i], "txs_pruned", MDB_INTEGERKEY, &dbi[i]); + if (dbr) + dbr = mdb_dbi_open(txn[i], "txs", MDB_INTEGERKEY, &dbi[i]); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_open(txn[i], dbi[i], &cur[i]); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + MDB_stat stat; + dbr = mdb_stat(txn[i], dbi[i], &stat); + if (dbr) throw std::runtime_error("Failed to query m_block_info: " + std::string(mdb_strerror(dbr))); + n_txes[i] = stat.ms_entries; + } + + if (n_txes[0] == 0 || n_txes[1] == 0) + throw std::runtime_error("No transaction in the database"); + uint64_t lo = 0, hi = std::min(n_txes[0], n_txes[1]) - 1; + while (lo <= hi) + { + uint64_t mid = (lo + hi) / 2; + + k.mv_size = sizeof(uint64_t); + k.mv_data = (void*)∣ + dbr = mdb_cursor_get(cur[0], &k, &v[0], MDB_SET); + if (dbr) throw std::runtime_error("Failed to query transaction: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_get(cur[1], &k, &v[1], MDB_SET); + if (dbr) throw std::runtime_error("Failed to query transaction: " + std::string(mdb_strerror(dbr))); + if (v[0].mv_size == v[1].mv_size && !memcmp(v[0].mv_data, v[1].mv_data, v[0].mv_size)) + lo = mid + 1; + else + hi = mid - 1; + } + + for (int i = 0; i < 2; ++i) + { + mdb_cursor_close(cur[i]); + mdb_txn_commit(txn[i]); + tx_active[i] = false; + mdb_dbi_close(env[i], dbi[i]); + mdb_env_close(env[i]); + } + return hi; +} + static std::vector<uint64_t> canonicalize(const std::vector<uint64_t> &v) { std::vector<uint64_t> c; @@ -212,6 +497,499 @@ static std::vector<uint64_t> canonicalize(const std::vector<uint64_t> &v) return c; } +static uint64_t get_num_spent_outputs() +{ + MDB_txn *txn; + bool tx_active = false; + + int dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + MDB_cursor *cur; + dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for spent outputs: " + std::string(mdb_strerror(dbr))); + MDB_val k, v; + mdb_size_t count = 0, tmp; + + MDB_cursor_op op = MDB_FIRST; + while (1) + { + dbr = mdb_cursor_get(cur, &k, &v, op); + op = MDB_NEXT_NODUP; + if (dbr == MDB_NOTFOUND) + break; + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get first/next spent output: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_count(cur, &tmp); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to count entries: " + std::string(mdb_strerror(dbr))); + count += tmp; + } + + mdb_cursor_close(cur); + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn: " + std::string(mdb_strerror(dbr))); + tx_active = false; + + return count; +} + +static void add_spent_output(MDB_cursor *cur, const output_data &od) +{ + MDB_val k = {sizeof(od.amount), (void*)&od.amount}; + MDB_val v = {sizeof(od.offset), (void*)&od.offset}; + int dbr = mdb_cursor_put(cur, &k, &v, 0); + CHECK_AND_ASSERT_THROW_MES(!dbr || dbr == MDB_KEYEXIST, "Failed to add spent output: " + std::string(mdb_strerror(dbr))); +} + +static bool is_output_spent(MDB_cursor *cur, const output_data &od) +{ + MDB_val k = {sizeof(od.amount), (void*)&od.amount}; + MDB_val v = {sizeof(od.offset), (void*)&od.offset}; + int dbr = mdb_cursor_get(cur, &k, &v, MDB_GET_BOTH); + CHECK_AND_ASSERT_THROW_MES(!dbr || dbr == MDB_NOTFOUND, "Failed to get spent output: " + std::string(mdb_strerror(dbr))); + bool spent = dbr == 0; + return spent; +} + +static std::vector<output_data> get_spent_outputs(MDB_txn *txn) +{ + MDB_cursor *cur; + int dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for spent outputs: " + std::string(mdb_strerror(dbr))); + MDB_val k, v; + uint64_t count = 0; + dbr = mdb_cursor_get(cur, &k, &v, MDB_FIRST); + if (dbr != MDB_NOTFOUND) + { + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get first spent output: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_count(cur, &count); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to count entries: " + std::string(mdb_strerror(dbr))); + } + std::vector<output_data> outs; + outs.reserve(count); + while (1) + { + outs.push_back({*(const uint64_t*)k.mv_data, *(const uint64_t*)v.mv_data}); + dbr = mdb_cursor_get(cur, &k, &v, MDB_NEXT); + if (dbr == MDB_NOTFOUND) + break; + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get next spent output: " + std::string(mdb_strerror(dbr))); + } + mdb_cursor_close(cur); + return outs; +} + +static uint64_t get_processed_txidx(const std::string &name) +{ + MDB_txn *txn; + bool tx_active = false; + + int dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + uint64_t height = 0; + MDB_val k, v; + k.mv_data = (void*)name.c_str(); + k.mv_size = name.size(); + dbr = mdb_get(txn, dbi_processed_txidx, &k, &v); + if (dbr != MDB_NOTFOUND) + { + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get processed height: " + std::string(mdb_strerror(dbr))); + height = *(const uint64_t*)v.mv_data; + } + + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn: " + std::string(mdb_strerror(dbr))); + tx_active = false; + + return height; +} + +static void set_processed_txidx(MDB_txn *txn, const std::string &name, uint64_t height) +{ + MDB_val k, v; + k.mv_data = (void*)name.c_str(); + k.mv_size = name.size(); + v.mv_data = (void*)&height; + v.mv_size = sizeof(height); + int dbr = mdb_put(txn, dbi_processed_txidx, &k, &v, 0); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set processed height: " + std::string(mdb_strerror(dbr))); +} + +static bool get_relative_ring(MDB_txn *txn, const crypto::key_image &ki, std::vector<uint64_t> &ring) +{ + MDB_val k, v; + k.mv_data = (void*)&ki; + k.mv_size = sizeof(ki); + int dbr = mdb_get(txn, dbi_relative_rings, &k, &v); + if (dbr == MDB_NOTFOUND) + return false; + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get relative ring: " + std::string(mdb_strerror(dbr))); + ring = decompress_ring(std::string((const char*)v.mv_data, v.mv_size)); + return true; +} + +static void set_relative_ring(MDB_txn *txn, const crypto::key_image &ki, const std::vector<uint64_t> &ring) +{ + const std::string sring = compress_ring(ring); + MDB_val k, v; + k.mv_data = (void*)&ki; + k.mv_size = sizeof(ki); + v.mv_data = (void*)sring.c_str(); + v.mv_size = sring.size(); + int dbr = mdb_put(txn, dbi_relative_rings, &k, &v, 0); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set relative ring: " + std::string(mdb_strerror(dbr))); +} + +static std::string keep_under_511(const std::string &s) +{ + if (s.size() <= 511) + return s; + crypto::hash hash; + crypto::cn_fast_hash(s.data(), s.size(), hash); + return std::string((const char*)&hash, 32); +} + +static uint64_t get_ring_instances(MDB_txn *txn, uint64_t amount, const std::vector<uint64_t> &ring) +{ + const std::string sring = keep_under_511(compress_ring(amount, ring)); + MDB_val k, v; + k.mv_data = (void*)sring.data(); + k.mv_size = sring.size(); + int dbr = mdb_get(txn, dbi_ring_instances, &k, &v); + if (dbr == MDB_NOTFOUND) + return 0; + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get ring instances: " + std::string(mdb_strerror(dbr))); + return *(const uint64_t*)v.mv_data; +} + +static uint64_t get_ring_subset_instances(MDB_txn *txn, uint64_t amount, const std::vector<uint64_t> &ring) +{ + uint64_t instances = get_ring_instances(txn, amount, ring); + if (ring.size() > 11) + return instances; + + uint64_t extra = 0; + std::vector<uint64_t> subset; + subset.reserve(ring.size()); + for (uint64_t mask = 1; mask < (1u << ring.size()) - 1; ++mask) + { + subset.resize(0); + for (size_t i = 0; i < ring.size(); ++i) + if ((mask >> i) & 1) + subset.push_back(ring[i]); + extra += get_ring_instances(txn, amount, subset); + } + return instances + extra; +} + +static uint64_t inc_ring_instances(MDB_txn *txn, uint64_t amount, const std::vector<uint64_t> &ring) +{ + const std::string sring = keep_under_511(compress_ring(amount, ring)); + MDB_val k, v; + k.mv_data = (void*)sring.data(); + k.mv_size = sring.size(); + + int dbr = mdb_get(txn, dbi_ring_instances, &k, &v); + CHECK_AND_ASSERT_THROW_MES(!dbr || dbr == MDB_NOTFOUND, "Failed to get ring instances: " + std::string(mdb_strerror(dbr))); + + uint64_t count; + if (dbr == MDB_NOTFOUND) + count = 1; + else + count = 1 + *(const uint64_t*)v.mv_data; + + v.mv_data = &count; + v.mv_size = sizeof(count); + dbr = mdb_put(txn, dbi_ring_instances, &k, &v, 0); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set ring instances: " + std::string(mdb_strerror(dbr))); + + return count; +} + +static std::vector<crypto::key_image> get_key_images(MDB_txn *txn, const output_data &od) +{ + MDB_val k, v; + k.mv_data = (void*)&od; + k.mv_size = sizeof(od); + int dbr = mdb_get(txn, dbi_outputs, &k, &v); + CHECK_AND_ASSERT_THROW_MES(!dbr || dbr == MDB_NOTFOUND, "Failed to get output: " + std::string(mdb_strerror(dbr))); + if (dbr == MDB_NOTFOUND) + return {}; + CHECK_AND_ASSERT_THROW_MES(v.mv_size % 32 == 0, "Unexpected record size"); + std::vector<crypto::key_image> key_images; + key_images.reserve(v.mv_size / 32); + const crypto::key_image *ki = (const crypto::key_image*)v.mv_data; + for (size_t n = 0; n < v.mv_size / 32; ++n) + key_images.push_back(*ki++); + return key_images; +} + +static void add_key_image(MDB_txn *txn, const output_data &od, const crypto::key_image &ki) +{ + MDB_val k, v; + k.mv_data = (void*)&od; + k.mv_size = sizeof(od); + int dbr = mdb_get(txn, dbi_outputs, &k, &v); + CHECK_AND_ASSERT_THROW_MES(!dbr || dbr == MDB_NOTFOUND, "Failed to get output"); + std::string data; + if (!dbr) + { + CHECK_AND_ASSERT_THROW_MES(v.mv_size % 32 == 0, "Unexpected record size"); + data = std::string((const char*)v.mv_data, v.mv_size); + } + data += std::string((const char*)&ki, sizeof(ki)); + + v.mv_data = (void*)data.data(); + v.mv_size = data.size(); + dbr = mdb_put(txn, dbi_outputs, &k, &v, 0); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set outputs: " + std::string(mdb_strerror(dbr))); +} + +static bool get_stat(MDB_txn *txn, const char *key, uint64_t &data) +{ + MDB_val k, v; + k.mv_data = (void*)key; + k.mv_size = strlen(key); + int dbr = mdb_get(txn, dbi_stats, &k, &v); + if (dbr == MDB_NOTFOUND) + return false; + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get stat record"); + CHECK_AND_ASSERT_THROW_MES(v.mv_size == sizeof(uint64_t), "Unexpected record size"); + data = *(const uint64_t*)v.mv_data; + return true; +} + +static void set_stat(MDB_txn *txn, const char *key, uint64_t data) +{ + MDB_val k, v; + k.mv_data = (void*)key; + k.mv_size = strlen(key); + v.mv_data = (void*)&data; + v.mv_size = sizeof(uint64_t); + int dbr = mdb_put(txn, dbi_stats, &k, &v, 0); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set stat record"); +} + +static void inc_stat(MDB_txn *txn, const char *key) +{ + uint64_t data; + if (!get_stat(txn, key, data)) + data = 0; + ++data; + set_stat(txn, key, data); +} + +static void open_db(const std::string &filename, MDB_env **env, MDB_txn **txn, MDB_cursor **cur, MDB_dbi *dbi) +{ + tools::create_directories_if_necessary(filename); + + int flags = MDB_RDONLY; + if (db_flags & DBF_FAST) + flags |= MDB_NOSYNC; + if (db_flags & DBF_FASTEST) + flags |= MDB_NOSYNC | MDB_WRITEMAP | MDB_MAPASYNC; + + int dbr = mdb_env_create(env); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); + dbr = mdb_env_set_maxdbs(*env, 1); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); + const std::string actual_filename = filename; + MINFO("Opening monero blockchain at " << actual_filename); + dbr = mdb_env_open(*env, actual_filename.c_str(), flags, 0664); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open rings database file '" + + actual_filename + "': " + std::string(mdb_strerror(dbr))); + + dbr = mdb_txn_begin(*env, NULL, MDB_RDONLY, txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_dbi_open(*txn, "output_amounts", MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, dbi); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_dupsort(*txn, *dbi, compare_uint64); + + dbr = mdb_cursor_open(*txn, *dbi, cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); +} + +static void close_db(MDB_env *env, MDB_txn *txn, MDB_cursor *cur, MDB_dbi dbi) +{ + mdb_txn_abort(txn); + mdb_cursor_close(cur); + mdb_dbi_close(env, dbi); + mdb_env_close(env); +} + +static void get_num_outputs(MDB_txn *txn, MDB_cursor *cur, MDB_dbi dbi, uint64_t &pre_rct, uint64_t &rct) +{ + uint64_t amount = 0; + MDB_val k = { sizeof(amount), (void*)&amount }, v; + int dbr = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (dbr == MDB_NOTFOUND) + { + rct = 0; + } + else + { + if (dbr) throw std::runtime_error("Record 0 not found: " + std::string(mdb_strerror(dbr))); + mdb_size_t count = 0; + dbr = mdb_cursor_count(cur, &count); + if (dbr) throw std::runtime_error("Failed to count records: " + std::string(mdb_strerror(dbr))); + rct = count; + } + MDB_stat s; + dbr = mdb_stat(txn, dbi, &s); + if (dbr) throw std::runtime_error("Failed to count records: " + std::string(mdb_strerror(dbr))); + if (s.ms_entries < rct) throw std::runtime_error("Inconsistent records: " + std::string(mdb_strerror(dbr))); + pre_rct = s.ms_entries - rct; +} + +static crypto::hash get_genesis_block_hash(const std::string &filename) +{ + MDB_env *env; + MDB_dbi dbi; + MDB_txn *txn; + int dbr; + bool tx_active = false; + + dbr = mdb_env_create(&env); + if (dbr) throw std::runtime_error("Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); + dbr = mdb_env_set_maxdbs(env, 1); + if (dbr) throw std::runtime_error("Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); + const std::string actual_filename = filename; + dbr = mdb_env_open(env, actual_filename.c_str(), 0, 0664); + if (dbr) throw std::runtime_error("Failed to open rings database file '" + + actual_filename + "': " + std::string(mdb_strerror(dbr))); + + dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + dbr = mdb_dbi_open(txn, "block_info", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi); + mdb_set_dupsort(txn, dbi, compare_uint64); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + uint64_t zero = 0; + MDB_val k = { sizeof(uint64_t), (void*)&zero}, v; + dbr = mdb_get(txn, dbi, &k, &v); + if (dbr) throw std::runtime_error("Failed to retrieve genesis block: " + std::string(mdb_strerror(dbr))); + crypto::hash genesis_block_hash = *(const crypto::hash*)(((const uint64_t*)v.mv_data) + 5); + mdb_dbi_close(env, dbi); + mdb_txn_abort(txn); + mdb_env_close(env); + tx_active = false; + return genesis_block_hash; +} + +static std::vector<std::pair<uint64_t, uint64_t>> load_outputs(const std::string &filename) +{ + std::vector<std::pair<uint64_t, uint64_t>> outputs; + uint64_t amount = std::numeric_limits<uint64_t>::max(); + FILE *f; + + f = fopen(filename.c_str(), "r"); + if (!f) + { + MERROR("Failed to load outputs from " << filename << ": " << strerror(errno)); + return {}; + } + while (1) + { + char s[256]; + fgets(s, sizeof(s), f); + if (feof(f)) + break; + const size_t len = strlen(s); + if (len > 0 && s[len - 1] == '\n') + s[len - 1] = 0; + if (!s[0]) + continue; + std::pair<uint64_t, uint64_t> output; + uint64_t offset, num_offsets; + if (sscanf(s, "@%" PRIu64, &amount) == 1) + { + continue; + } + if (amount == std::numeric_limits<uint64_t>::max()) + { + MERROR("Bad format in " << filename); + continue; + } + if (sscanf(s, "%" PRIu64 "*%" PRIu64, &offset, &num_offsets) == 2 && num_offsets < std::numeric_limits<uint64_t>::max() - offset) + { + while (num_offsets-- > 0) + outputs.push_back(std::make_pair(amount, offset++)); + } + else if (sscanf(s, "%" PRIu64, &offset) == 1) + { + outputs.push_back(std::make_pair(amount, offset)); + } + else + { + MERROR("Bad format in " << filename); + continue; + } + } + fclose(f); + return outputs; +} + +static bool export_spent_outputs(MDB_cursor *cur, const std::string &filename) +{ + FILE *f = fopen(filename.c_str(), "w"); + if (!f) + { + MERROR("Failed to open " << filename << ": " << strerror(errno)); + return false; + } + + uint64_t pending_amount = std::numeric_limits<uint64_t>::max(); + std::vector<uint64_t> pending_offsets; + MDB_val k, v; + MDB_cursor_op op = MDB_FIRST; + while (1) + { + int dbr = mdb_cursor_get(cur, &k, &v, op); + if (dbr == MDB_NOTFOUND) + break; + op = MDB_NEXT; + if (dbr) + { + fclose(f); + MERROR("Failed to enumerate spent outputs: " << mdb_strerror(dbr)); + return false; + } + const uint64_t amount = *(const uint64_t*)k.mv_data; + const uint64_t offset = *(const uint64_t*)v.mv_data; + if (!pending_offsets.empty() && (amount != pending_amount || pending_offsets.back()+1 != offset)) + { + if (pending_offsets.size() == 1) + fprintf(f, "%" PRIu64 "\n", pending_offsets.front()); + else + fprintf(f, "%" PRIu64 "*%" PRIu64 "\n", pending_offsets.front(), pending_offsets.size()); + pending_offsets.clear(); + } + if (pending_amount != amount) + { + fprintf(f, "@%" PRIu64 "\n", amount); + pending_amount = amount; + } + pending_offsets.push_back(offset); + } + if (!pending_offsets.empty()) + { + if (pending_offsets.size() == 1) + fprintf(f, "%" PRIu64 "\n", pending_offsets.front()); + else + fprintf(f, "%" PRIu64 "*%" PRIu64 "\n", pending_offsets.front(), pending_offsets.size()); + pending_offsets.clear(); + } + fclose(f); + return true; +} + int main(int argc, char* argv[]) { TRY_ENTRY(); @@ -231,31 +1009,37 @@ int main(int argc, char* argv[]) po::options_description desc_cmd_only("Command line options"); po::options_description desc_cmd_sett("Command line options and settings options"); - const command_line::arg_descriptor<std::string, false, true, 2> arg_blackball_db_dir = { + const command_line::arg_descriptor<std::string> arg_blackball_db_dir = { "blackball-db-dir", "Specify blackball database directory", get_default_db_path(), - {{ &arg_testnet_on, &arg_stagenet_on }}, - [](std::array<bool, 2> testnet_stagenet, bool defaulted, std::string val)->std::string { - if (testnet_stagenet[0]) - return (boost::filesystem::path(val) / "testnet").string(); - else if (testnet_stagenet[1]) - return (boost::filesystem::path(val) / "stagenet").string(); - return val; - } }; const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""}; const command_line::arg_descriptor<std::string> arg_database = { "database", available_dbs.c_str(), default_db_type }; const command_line::arg_descriptor<bool> arg_rct_only = {"rct-only", "Only work on ringCT outputs", false}; + const command_line::arg_descriptor<bool> arg_check_subsets = {"check-subsets", "Check ring subsets (very expensive)", false}; + const command_line::arg_descriptor<bool> arg_verbose = {"verbose", "Verbose output)", false}; const command_line::arg_descriptor<std::vector<std::string> > arg_inputs = {"inputs", "Path to Monero DB, and path to any fork DBs"}; + const command_line::arg_descriptor<std::string> arg_db_sync_mode = { + "db-sync-mode" + , "Specify sync option, using format [safe|fast|fastest]:[nrecords_per_sync]." + , "fast:1000" + }; + const command_line::arg_descriptor<std::string> arg_extra_spent_list = {"extra-spent-list", "Optional list of known spent outputs",""}; + const command_line::arg_descriptor<std::string> arg_export = {"export", "Filename to export the backball list to"}; + const command_line::arg_descriptor<bool> arg_force_chain_reaction_pass = {"force-chain-reaction-pass", "Run the chain reaction pass even if no new blockchain data was processed"}; command_line::add_arg(desc_cmd_sett, arg_blackball_db_dir); - command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); - command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); command_line::add_arg(desc_cmd_sett, arg_log_level); command_line::add_arg(desc_cmd_sett, arg_database); command_line::add_arg(desc_cmd_sett, arg_rct_only); + command_line::add_arg(desc_cmd_sett, arg_check_subsets); + command_line::add_arg(desc_cmd_sett, arg_verbose); + command_line::add_arg(desc_cmd_sett, arg_db_sync_mode); + command_line::add_arg(desc_cmd_sett, arg_extra_spent_list); + command_line::add_arg(desc_cmd_sett, arg_export); + command_line::add_arg(desc_cmd_sett, arg_force_chain_reaction_pass); command_line::add_arg(desc_cmd_sett, arg_inputs); command_line::add_arg(desc_cmd_only, command_line::arg_help); @@ -291,11 +1075,14 @@ int main(int argc, char* argv[]) LOG_PRINT_L0("Starting..."); - bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); - bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); - network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; output_file_path = command_line::get_arg(vm, arg_blackball_db_dir); bool opt_rct_only = command_line::get_arg(vm, arg_rct_only); + bool opt_check_subsets = command_line::get_arg(vm, arg_check_subsets); + bool opt_verbose = command_line::get_arg(vm, arg_verbose); + bool opt_force_chain_reaction_pass = command_line::get_arg(vm, arg_force_chain_reaction_pass); + std::string opt_export = command_line::get_arg(vm, arg_export); + std::string extra_spent_list = command_line::get_arg(vm, arg_extra_spent_list); + std::vector<std::pair<uint64_t, uint64_t>> extra_spent_outputs = extra_spent_list.empty() ? std::vector<std::pair<uint64_t, uint64_t>>() : load_outputs(extra_spent_list); std::string db_type = command_line::get_arg(vm, arg_database); if (!cryptonote::blockchain_valid_db_type(db_type)) @@ -304,121 +1091,99 @@ int main(int argc, char* argv[]) return 1; } - // If we wanted to use the memory pool, we would set up a fake_core. + std::string db_sync_mode = command_line::get_arg(vm, arg_db_sync_mode); + if (!parse_db_sync_mode(db_sync_mode)) + { + MERROR("Invalid db sync mode: " << db_sync_mode); + return 1; + } - // Use Blockchain instead of lower-level BlockchainDB for two reasons: - // 1. Blockchain has the init() method for easy setup - // 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash() - // - // cannot match blockchain_storage setup above with just one line, - // e.g. - // Blockchain* core_storage = new Blockchain(NULL); - // because unlike blockchain_storage constructor, which takes a pointer to - // tx_memory_pool, Blockchain's constructor takes tx_memory_pool object. - LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); const std::vector<std::string> inputs = command_line::get_arg(vm, arg_inputs); if (inputs.empty()) { LOG_PRINT_L0("No inputs given"); return 1; } - std::vector<std::unique_ptr<Blockchain>> core_storage(inputs.size()); - Blockchain *blockchain = NULL; - tx_memory_pool m_mempool(*blockchain); - for (size_t n = 0; n < inputs.size(); ++n) - { - core_storage[n].reset(new Blockchain(m_mempool)); - BlockchainDB* db = new_db(db_type); - if (db == NULL) - { - LOG_ERROR("Attempted to use non-existent database type: " << db_type); - throw std::runtime_error("Attempting to use non-existent database type"); - } - LOG_PRINT_L0("database: " << db_type); + const std::string cache_dir = (output_file_path / "blackball-cache").string(); + init(cache_dir); - std::string filename = inputs[n]; - while (boost::ends_with(filename, "/") || boost::ends_with(filename, "\\")) - filename.pop_back(); - LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); + LOG_PRINT_L0("Scanning for blackballable outputs..."); - try - { - db->open(filename, DBF_RDONLY); - } - catch (const std::exception& e) - { - LOG_PRINT_L0("Error opening database: " << e.what()); - return 1; - } - r = core_storage[n]->init(db, net_type); + size_t done = 0; - CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage"); - LOG_PRINT_L0("Source blockchain storage initialized OK"); - } + const uint64_t start_blackballed_outputs = get_num_spent_outputs(); - boost::filesystem::path direc(output_file_path.string()); - if (boost::filesystem::exists(direc)) - { - if (!boost::filesystem::is_directory(direc)) - { - MERROR("LMDB needs a directory path, but a file was passed: " << output_file_path.string()); - return 1; - } - } - else - { - if (!boost::filesystem::create_directories(direc)) - { - MERROR("Failed to create directory: " << output_file_path.string()); - return 1; - } - } + tools::ringdb ringdb(output_file_path.string(), epee::string_tools::pod_to_hex(get_genesis_block_hash(inputs[0]))); - LOG_PRINT_L0("Scanning for blackballable outputs..."); + bool stop_requested = false; + tools::signal_handler::install([&stop_requested](int type) { + stop_requested = true; + }); - size_t done = 0; - blackball_state_t state; - std::unordered_set<output_data> newly_spent; - const std::string state_file_path = (boost::filesystem::path(output_file_path) / "blackball-state.bin").string(); + int dbr = resize_env(cache_dir.c_str()); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to resize LMDB database: " + std::string(mdb_strerror(dbr))); + + // open first db + MDB_env *env0; + MDB_txn *txn0; + MDB_dbi dbi0; + MDB_cursor *cur0; + open_db(inputs[0], &env0, &txn0, &cur0, &dbi0); - LOG_PRINT_L0("Loading state data from " << state_file_path); - std::ifstream state_data_in; - state_data_in.open(state_file_path, std::ios_base::binary | std::ios_base::in); - if (!state_data_in.fail()) + if (!extra_spent_outputs.empty()) { - try + MINFO("Adding " << extra_spent_outputs.size() << " extra spent outputs"); + MDB_txn *txn; + int dbr = mdb_txn_begin(env, NULL, 0, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + MDB_cursor *cur; + dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr))); + + std::vector<std::pair<uint64_t, uint64_t>> blackballs; + for (const std::pair<uint64_t, uint64_t> &output: extra_spent_outputs) { - boost::archive::portable_binary_iarchive a(state_data_in); - a >> state; + if (!is_output_spent(cur, output_data(output.first, output.second))) + { + blackballs.push_back(output); + add_spent_output(cur, output_data(output.first, output.second)); + inc_stat(txn, output.first ? "pre-rct-extra" : "rct-ring-extra"); + } } - catch (const std::exception &e) + if (!blackballs.empty()) { - MERROR("Failed to load state data from " << state_file_path << ", restarting from scratch"); - state = blackball_state_t(); + ringdb.blackball(blackballs); + blackballs.clear(); } - state_data_in.close(); + mdb_cursor_close(cur); + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); } - uint64_t start_blackballed_outputs = state.spent.size(); - - cryptonote::block b = core_storage[0]->get_db().get_block_from_height(0); - tools::ringdb ringdb(output_file_path.string(), epee::string_tools::pod_to_hex(get_block_hash(b))); - - bool stop_requested = false; - tools::signal_handler::install([&stop_requested](int type) { - stop_requested = true; - }); for (size_t n = 0; n < inputs.size(); ++n) { const std::string canonical = boost::filesystem::canonical(inputs[n]).string(); - uint64_t start_idx = 0; - auto it = state.processed_heights.find(canonical); - if (it != state.processed_heights.end()) - start_idx = it->second; + uint64_t start_idx = get_processed_txidx(canonical); + if (n > 0 && start_idx == 0) + { + start_idx = find_first_diverging_transaction(inputs[0], inputs[n]); + LOG_PRINT_L0("First diverging transaction at " << start_idx); + } LOG_PRINT_L0("Reading blockchain from " << inputs[n] << " from " << start_idx); - for_all_transactions(inputs[n], start_idx, [&](const cryptonote::transaction_prefix &tx)->bool + MDB_txn *txn; + int dbr = mdb_txn_begin(env, NULL, 0, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + MDB_cursor *cur; + dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr))); + size_t records = 0; + const std::string filename = inputs[n]; + std::vector<std::pair<uint64_t, uint64_t>> blackballs; + uint64_t n_txes; + for_all_transactions(filename, start_idx, n_txes, [&](const cryptonote::transaction_prefix &tx)->bool { + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; for (const auto &in: tx.vin) { if (in.type() != typeid(txin_to_key)) @@ -430,37 +1195,65 @@ int main(int argc, char* argv[]) const std::vector<uint64_t> absolute = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets); if (n == 0) for (uint64_t out: absolute) - state.outputs[output_data(txin.amount, out)].insert(txin.k_image); + add_key_image(txn, output_data(txin.amount, out), txin.k_image); + std::vector<uint64_t> relative_ring; std::vector<uint64_t> new_ring = canonicalize(txin.key_offsets); const uint32_t ring_size = txin.key_offsets.size(); - state.ring_instances[new_ring] += 1; - if (ring_size == 1) + const uint64_t instances = inc_ring_instances(txn, txin.amount, new_ring); + if (n == 0 && ring_size == 1) { - const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, absolute[0]); - MINFO("Blackballing output " << pkey << ", due to being used in a 1-ring"); - ringdb.blackball(pkey); - newly_spent.insert(output_data(txin.amount, absolute[0])); + const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, absolute[0]); + if (opt_verbose) + { + MINFO("Blackballing output " << output.first << "/" << output.second << ", due to being used in a 1-ring"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + } + blackballs.push_back(output); + add_spent_output(cur, output_data(txin.amount, absolute[0])); + inc_stat(txn, txin.amount ? "pre-rct-ring-size-1" : "rct-ring-size-1"); } - else if (state.ring_instances[new_ring] == new_ring.size()) + else if (n == 0 && instances == new_ring.size()) { for (size_t o = 0; o < new_ring.size(); ++o) { - const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, absolute[o]); - MINFO("Blackballing output " << pkey << ", due to being used in " << new_ring.size() << " identical " << new_ring.size() << "-rings"); - ringdb.blackball(pkey); - newly_spent.insert(output_data(txin.amount, absolute[o])); + const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, absolute[o]); + if (opt_verbose) + { + MINFO("Blackballing output " << output.first << "/" << output.second << ", due to being used in " << new_ring.size() << " identical " << new_ring.size() << "-rings"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + } + blackballs.push_back(output); + add_spent_output(cur, output_data(txin.amount, absolute[o])); + inc_stat(txn, txin.amount ? "pre-rct-duplicate-rings" : "rct-duplicate-rings"); } } - else if (state.relative_rings.find(txin.k_image) != state.relative_rings.end()) + else if (n == 0 && opt_check_subsets && get_ring_subset_instances(txn, txin.amount, new_ring) >= new_ring.size()) { - MINFO("Key image " << txin.k_image << " already seen: rings " << - boost::join(state.relative_rings[txin.k_image] | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " ") << + for (size_t o = 0; o < new_ring.size(); ++o) + { + const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, absolute[o]); + if (opt_verbose) + { + MINFO("Blackballing output " << output.first << "/" << output.second << ", due to being used in " << new_ring.size() << " subsets of " << new_ring.size() << "-rings"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + } + blackballs.push_back(output); + add_spent_output(cur, output_data(txin.amount, absolute[o])); + inc_stat(txn, txin.amount ? "pre-rct-subset-rings" : "rct-subset-rings"); + } + } + else if (n > 0 && get_relative_ring(txn, txin.k_image, relative_ring)) + { + MDEBUG("Key image " << txin.k_image << " already seen: rings " << + boost::join(relative_ring | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " ") << ", " << boost::join(txin.key_offsets | boost::adaptors::transformed([](uint64_t out){return std::to_string(out);}), " ")); - if (state.relative_rings[txin.k_image] != txin.key_offsets) + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + if (relative_ring != txin.key_offsets) { - MINFO("Rings are different"); - const std::vector<uint64_t> r0 = cryptonote::relative_output_offsets_to_absolute(state.relative_rings[txin.k_image]); + MDEBUG("Rings are different"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + const std::vector<uint64_t> r0 = cryptonote::relative_output_offsets_to_absolute(relative_ring); const std::vector<uint64_t> r1 = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets); std::vector<uint64_t> common; for (uint64_t out: r0) @@ -471,17 +1264,24 @@ int main(int argc, char* argv[]) if (common.empty()) { MERROR("Rings for the same key image are disjoint"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; } else if (common.size() == 1) { - const crypto::public_key pkey = core_storage[n]->get_output_key(txin.amount, common[0]); - MINFO("Blackballing output " << pkey << ", due to being used in rings with a single common element"); - ringdb.blackball(pkey); - newly_spent.insert(output_data(txin.amount, common[0])); + const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, common[0]); + if (opt_verbose) + { + MINFO("Blackballing output " << output.first << "/" << output.second << ", due to being used in rings with a single common element"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + } + blackballs.push_back(output); + add_spent_output(cur, output_data(txin.amount, common[0])); + inc_stat(txn, txin.amount ? "pre-rct-key-image-attack" : "rct-key-image-attack"); } else { - MINFO("The intersection has more than one element, it's still ok"); + MDEBUG("The intersection has more than one element, it's still ok"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; for (const auto &out: r0) if (std::find(common.begin(), common.end(), out) != common.end()) new_ring.push_back(out); @@ -489,78 +1289,172 @@ int main(int argc, char* argv[]) } } } - state.relative_rings[txin.k_image] = new_ring; + if (n == 0) + set_relative_ring(txn, txin.k_image, new_ring); + } + set_processed_txidx(txn, canonical, start_idx+1); + + ++records; + if (records >= records_per_sync) + { + if (!blackballs.empty()) + { + ringdb.blackball(blackballs); + blackballs.clear(); + } + mdb_cursor_close(cur); + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); + int dbr = resize_env(cache_dir.c_str()); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to resize LMDB database: " + std::string(mdb_strerror(dbr))); + dbr = mdb_txn_begin(env, NULL, 0, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr))); + records = 0; } + if (stop_requested) { - MINFO("Stopping scan, secondary passes will still happen..."); + MINFO("Stopping scan..."); return false; } return true; }); - LOG_PRINT_L0("blockchain from " << inputs[n] << " processed still height " << start_idx); - state.processed_heights[canonical] = start_idx; + mdb_cursor_close(cur); + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); + LOG_PRINT_L0("blockchain from " << inputs[n] << " processed till tx idx " << start_idx); if (stop_requested) break; } - while (!newly_spent.empty()) + std::vector<output_data> work_spent; + + if (stop_requested) + goto skip_secondary_passes; + + if (opt_force_chain_reaction_pass || get_num_spent_outputs() > start_blackballed_outputs) { - LOG_PRINT_L0("Secondary pass due to " << newly_spent.size() << " newly found spent outputs"); - std::unordered_set<output_data> work_spent = std::move(newly_spent); - newly_spent.clear(); + MDB_txn *txn; + dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + work_spent = get_spent_outputs(txn); + mdb_txn_abort(txn); + } + + while (!work_spent.empty()) + { + LOG_PRINT_L0("Secondary pass on " << work_spent.size() << " spent outputs"); - for (const auto &e: work_spent) - state.spent.insert(e); + int dbr = resize_env(cache_dir.c_str()); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to resize LMDB database: " + std::string(mdb_strerror(dbr))); - for (const output_data &od: work_spent) + MDB_txn *txn; + dbr = mdb_txn_begin(env, NULL, 0, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + MDB_cursor *cur; + dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr))); + + std::vector<std::pair<uint64_t, uint64_t>> blackballs; + std::vector<output_data> scan_spent = std::move(work_spent); + work_spent.clear(); + for (const output_data &od: scan_spent) { - for (const crypto::key_image &ki: state.outputs[od]) + std::vector<crypto::key_image> key_images = get_key_images(txn, od); + for (const crypto::key_image &ki: key_images) { - std::vector<uint64_t> absolute = cryptonote::relative_output_offsets_to_absolute(state.relative_rings[ki]); + std::vector<uint64_t> relative_ring; + CHECK_AND_ASSERT_THROW_MES(get_relative_ring(txn, ki, relative_ring), "Relative ring not found"); + std::vector<uint64_t> absolute = cryptonote::relative_output_offsets_to_absolute(relative_ring); size_t known = 0; uint64_t last_unknown = 0; for (uint64_t out: absolute) { output_data new_od(od.amount, out); - if (state.spent.find(new_od) != state.spent.end()) + if (is_output_spent(cur, new_od)) ++known; else last_unknown = out; } if (known == absolute.size() - 1) { - const crypto::public_key pkey = core_storage[0]->get_output_key(od.amount, last_unknown); - MINFO("Blackballing output " << pkey << ", due to being used in a " << - absolute.size() << "-ring where all other outputs are known to be spent"); - ringdb.blackball(pkey); - newly_spent.insert(output_data(od.amount, last_unknown)); + const std::pair<uint64_t, uint64_t> output = std::make_pair(od.amount, last_unknown); + if (opt_verbose) + { + MINFO("Blackballing output " << output.first << "/" << output.second << ", due to being used in a " << + absolute.size() << "-ring where all other outputs are known to be spent"); + } + blackballs.push_back(output); + add_spent_output(cur, output_data(od.amount, last_unknown)); + work_spent.push_back(output_data(od.amount, last_unknown)); + inc_stat(txn, od.amount ? "pre-rct-chain-reaction" : "rct-chain-reaction"); } } - } - } - LOG_PRINT_L0("Saving state data to " << state_file_path); - std::ofstream state_data_out; - state_data_out.open(state_file_path, std::ios_base::binary | std::ios_base::out | std::ios::trunc); - if (!state_data_out.fail()) - { - try - { - boost::archive::portable_binary_oarchive a(state_data_out); - a << state; + if (stop_requested) + { + MINFO("Stopping secondary passes. Secondary passes are not incremental, they will re-run fully."); + return false; + } } - catch (const std::exception &e) + if (!blackballs.empty()) { - MERROR("Failed to save state data to " << state_file_path); + ringdb.blackball(blackballs); + blackballs.clear(); } - state_data_out.close(); + mdb_cursor_close(cur); + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); + } + +skip_secondary_passes: + uint64_t diff = get_num_spent_outputs() - start_blackballed_outputs; + LOG_PRINT_L0(std::to_string(diff) << " new outputs blackballed, " << get_num_spent_outputs() << " total outputs blackballed"); + + MDB_txn *txn; + dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + uint64_t pre_rct = 0, rct = 0; + get_num_outputs(txn0, cur0, dbi0, pre_rct, rct); + MINFO("Total pre-rct outputs: " << pre_rct); + MINFO("Total rct outputs: " << rct); + static const struct { const char *key; uint64_t base; } stat_keys[] = { + { "pre-rct-ring-size-1", pre_rct }, { "rct-ring-size-1", rct }, + { "pre-rct-duplicate-rings", pre_rct }, { "rct-duplicate-rings", rct }, + { "pre-rct-subset-rings", pre_rct }, { "rct-subset-rings", rct }, + { "pre-rct-key-image-attack", pre_rct }, { "rct-key-image-attack", rct }, + { "pre-rct-extra", pre_rct }, { "rct-ring-extra", rct }, + { "pre-rct-chain-reaction", pre_rct }, { "rct-chain-reaction", rct }, + }; + for (const auto &key: stat_keys) + { + uint64_t data; + if (!get_stat(txn, key.key, data)) + data = 0; + float percent = key.base ? 100.0f * data / key.base : 0.0f; + MINFO(key.key << ": " << data << " (" << percent << "%)"); + } + mdb_txn_abort(txn); + + if (!opt_export.empty()) + { + MDB_txn *txn; + int dbr = mdb_txn_begin(env, NULL, 0, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + MDB_cursor *cur; + dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr))); + export_spent_outputs(cur, opt_export); + mdb_cursor_close(cur); + mdb_txn_abort(txn); } - uint64_t diff = state.spent.size() - start_blackballed_outputs; - LOG_PRINT_L0(std::to_string(diff) << " new outputs blackballed, " << state.spent.size() << " total outputs blackballed"); LOG_PRINT_L0("Blockchain blackball data exported OK"); + close_db(env0, txn0, cur0, dbi0); + close(); return 0; - CATCH_ENTRY("Export error", 1); + CATCH_ENTRY("Error", 1); } diff --git a/src/blockchain_utilities/blockchain_depth.cpp b/src/blockchain_utilities/blockchain_depth.cpp new file mode 100644 index 000000000..dd2387e5b --- /dev/null +++ b/src/blockchain_utilities/blockchain_depth.cpp @@ -0,0 +1,351 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <boost/range/adaptor/transformed.hpp> +#include <boost/algorithm/string.hpp> +#include "common/command_line.h" +#include "common/varint.h" +#include "cryptonote_core/tx_pool.h" +#include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_core/blockchain.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/db_types.h" +#include "version.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "bcutil" + +namespace po = boost::program_options; +using namespace epee; +using namespace cryptonote; + +int main(int argc, char* argv[]) +{ + TRY_ENTRY(); + + epee::string_tools::set_module_name_and_folder(argv[0]); + + std::string default_db_type = "lmdb"; + + std::string available_dbs = cryptonote::blockchain_db_types(", "); + available_dbs = "available: " + available_dbs; + + uint32_t log_level = 0; + + tools::on_startup(); + + boost::filesystem::path output_file_path; + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""}; + const command_line::arg_descriptor<std::string> arg_database = { + "database", available_dbs.c_str(), default_db_type + }; + const command_line::arg_descriptor<std::string> arg_txid = {"txid", "Get min depth for this txid", ""}; + const command_line::arg_descriptor<uint64_t> arg_height = {"height", "Get min depth for all txes at this height", 0}; + const command_line::arg_descriptor<bool> arg_include_coinbase = {"include-coinbase", "Include coinbase in the average", false}; + + command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_database); + command_line::add_arg(desc_cmd_sett, arg_txid); + command_line::add_arg(desc_cmd_sett, arg_height); + command_line::add_arg(desc_cmd_sett, arg_include_coinbase); + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + auto parser = po::command_line_parser(argc, argv).options(desc_options); + po::store(parser.run(), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + mlog_configure(mlog_get_default_log_path("monero-blockchain-depth.log"), true); + if (!command_line::is_arg_defaulted(vm, arg_log_level)) + mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); + else + mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str()); + + LOG_PRINT_L0("Starting..."); + + std::string opt_data_dir = command_line::get_arg(vm, cryptonote::arg_data_dir); + bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); + bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); + network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; + std::string opt_txid_string = command_line::get_arg(vm, arg_txid); + uint64_t opt_height = command_line::get_arg(vm, arg_height); + bool opt_include_coinbase = command_line::get_arg(vm, arg_include_coinbase); + + if (!opt_txid_string.empty() && opt_height) + { + std::cerr << "txid and height cannot be given at the same time" << std::endl; + return 1; + } + crypto::hash opt_txid = crypto::null_hash; + if (!opt_txid_string.empty()) + { + if (!epee::string_tools::hex_to_pod(opt_txid_string, opt_txid)) + { + std::cerr << "Invalid txid" << std::endl; + return 1; + } + } + + std::string db_type = command_line::get_arg(vm, arg_database); + if (!cryptonote::blockchain_valid_db_type(db_type)) + { + std::cerr << "Invalid database type: " << db_type << std::endl; + return 1; + } + + // If we wanted to use the memory pool, we would set up a fake_core. + + // Use Blockchain instead of lower-level BlockchainDB for two reasons: + // 1. Blockchain has the init() method for easy setup + // 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash() + // + // cannot match blockchain_storage setup above with just one line, + // e.g. + // Blockchain* core_storage = new Blockchain(NULL); + // because unlike blockchain_storage constructor, which takes a pointer to + // tx_memory_pool, Blockchain's constructor takes tx_memory_pool object. + LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); + std::unique_ptr<Blockchain> core_storage; + tx_memory_pool m_mempool(*core_storage); + core_storage.reset(new Blockchain(m_mempool)); + BlockchainDB *db = new_db(db_type); + if (db == NULL) + { + LOG_ERROR("Attempted to use non-existent database type: " << db_type); + throw std::runtime_error("Attempting to use non-existent database type"); + } + LOG_PRINT_L0("database: " << db_type); + + const std::string filename = (boost::filesystem::path(opt_data_dir) / db->get_db_name()).string(); + LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); + + try + { + db->open(filename, DBF_RDONLY); + } + catch (const std::exception& e) + { + LOG_PRINT_L0("Error opening database: " << e.what()); + return 1; + } + r = core_storage->init(db, net_type); + + CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage"); + LOG_PRINT_L0("Source blockchain storage initialized OK"); + + std::vector<crypto::hash> start_txids; + if (!opt_txid_string.empty()) + { + start_txids.push_back(opt_txid); + } + else + { + const crypto::hash block_hash = db->get_block_hash_from_height(opt_height); + const cryptonote::blobdata bd = db->get_block_blob(block_hash); + cryptonote::block b; + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + for (const crypto::hash &txid: b.tx_hashes) + start_txids.push_back(txid); + if (opt_include_coinbase) + start_txids.push_back(cryptonote::get_transaction_hash(b.miner_tx)); + } + + if (start_txids.empty()) + { + LOG_PRINT_L0("No transaction(s) to check"); + return 1; + } + + std::vector<uint64_t> depths; + for (const crypto::hash &start_txid: start_txids) + { + uint64_t depth = 0; + bool coinbase = false; + + LOG_PRINT_L0("Checking depth for txid " << start_txid); + std::vector<crypto::hash> txids(1, start_txid); + while (!coinbase) + { + LOG_PRINT_L0("Considering "<< txids.size() << " transaction(s) at depth " << depth); + std::vector<crypto::hash> new_txids; + for (const crypto::hash &txid: txids) + { + cryptonote::blobdata bd; + if (!db->get_pruned_tx_blob(txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << txid << " from db"); + return 1; + } + cryptonote::transaction tx; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) + { + LOG_PRINT_L0("Bad tx: " << txid); + return 1; + } + for (size_t ring = 0; ring < tx.vin.size(); ++ring) + { + if (tx.vin[ring].type() == typeid(cryptonote::txin_gen)) + { + MDEBUG(txid << " is a coinbase transaction"); + coinbase = true; + goto done; + } + if (tx.vin[ring].type() == typeid(cryptonote::txin_to_key)) + { + const cryptonote::txin_to_key &txin = boost::get<cryptonote::txin_to_key>(tx.vin[ring]); + const uint64_t amount = txin.amount; + auto absolute_offsets = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets); + for (uint64_t offset: absolute_offsets) + { + const output_data_t od = db->get_output_key(amount, offset); + const crypto::hash block_hash = db->get_block_hash_from_height(od.height); + bd = db->get_block_blob(block_hash); + cryptonote::block b; + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + // find the tx which created this output + bool found = false; + for (size_t out = 0; out < b.miner_tx.vout.size(); ++out) + { + if (b.miner_tx.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get<cryptonote::txout_to_key>(b.miner_tx.vout[out].target); + if (txout.key == od.pubkey) + { + found = true; + new_txids.push_back(cryptonote::get_transaction_hash(b.miner_tx)); + MDEBUG("adding txid: " << cryptonote::get_transaction_hash(b.miner_tx)); + break; + } + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << cryptonote::get_transaction_hash(b.miner_tx)); + return 1; + } + } + for (const crypto::hash &block_txid: b.tx_hashes) + { + if (found) + break; + if (!db->get_pruned_tx_blob(block_txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << block_txid << " from db"); + return 1; + } + cryptonote::transaction tx2; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx2)) + { + LOG_PRINT_L0("Bad tx: " << block_txid); + return 1; + } + for (size_t out = 0; out < tx2.vout.size(); ++out) + { + if (tx2.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get<cryptonote::txout_to_key>(tx2.vout[out].target); + if (txout.key == od.pubkey) + { + found = true; + new_txids.push_back(block_txid); + MDEBUG("adding txid: " << block_txid); + break; + } + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << block_txid); + return 1; + } + } + } + if (!found) + { + LOG_PRINT_L0("Output originating transaction not found"); + return 1; + } + } + } + else + { + LOG_PRINT_L0("Bad vin type in txid " << txid); + return 1; + } + } + } + if (!coinbase) + { + std::swap(txids, new_txids); + ++depth; + } + } +done: + LOG_PRINT_L0("Min depth for txid " << start_txid << ": " << depth); + depths.push_back(depth); + } + + uint64_t cumulative_depth = 0; + for (uint64_t depth: depths) + cumulative_depth += depth; + LOG_PRINT_L0("Average min depth for " << start_txids.size() << " transaction(s): " << cumulative_depth/(float)depths.size()); + LOG_PRINT_L0("Median min depth for " << start_txids.size() << " transaction(s): " << epee::misc_utils::median(depths)); + + core_storage->deinit(); + return 0; + + CATCH_ENTRY("Depth query error", 1); +} diff --git a/src/common/dns_utils.cpp b/src/common/dns_utils.cpp index 33f60bc3c..3f2bde620 100644 --- a/src/common/dns_utils.cpp +++ b/src/common/dns_utils.cpp @@ -97,11 +97,16 @@ get_builtin_cert(void) */ /** return the built in root DS trust anchor */ -static const char* +static const char* const* get_builtin_ds(void) { - return -". IN DS 19036 8 2 49AAC11D7B6F6446702E54A1607371607A1A41855200FD2CE1CDDE32F24E8FB5\n"; + static const char * const ds[] = + { + ". IN DS 19036 8 2 49AAC11D7B6F6446702E54A1607371607A1A41855200FD2CE1CDDE32F24E8FB5\n", + ". IN DS 20326 8 2 E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC683457104237C7F8EC8D\n", + NULL + }; + return ds; } /************************************************************ @@ -240,7 +245,12 @@ DNSResolver::DNSResolver() : m_data(new DNSResolverData()) ub_ctx_hosts(m_data->m_ub_context, NULL); } - ub_ctx_add_ta(m_data->m_ub_context, string_copy(::get_builtin_ds())); + const char * const *ds = ::get_builtin_ds(); + while (*ds) + { + MINFO("adding trust anchor: " << *ds); + ub_ctx_add_ta(m_data->m_ub_context, string_copy(*ds++)); + } } DNSResolver::~DNSResolver() diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index b20fe9869..c7e2f5a65 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -1717,17 +1717,6 @@ size_t Blockchain::get_alternative_blocks_count() const //------------------------------------------------------------------ // This function adds the output specified by <amount, i> to the result_outs container // unlocked and other such checks should be done by here. -void Blockchain::add_out_to_get_random_outs(COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i) const -{ - LOG_PRINT_L3("Blockchain::" << __func__); - CRITICAL_REGION_LOCAL(m_blockchain_lock); - - COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry& oen = *result_outs.outs.insert(result_outs.outs.end(), COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry()); - oen.global_amount_index = i; - output_data_t data = m_db->get_output_key(amount, i); - oen.out_key = data.pubkey; -} - uint64_t Blockchain::get_num_mature_outputs(uint64_t amount) const { uint64_t num_outs = m_db->get_num_outputs(amount); @@ -1745,74 +1734,6 @@ uint64_t Blockchain::get_num_mature_outputs(uint64_t amount) const return num_outs; } -std::vector<uint64_t> Blockchain::get_random_outputs(uint64_t amount, uint64_t count) const -{ - uint64_t num_outs = get_num_mature_outputs(amount); - - std::vector<uint64_t> indices; - - std::unordered_set<uint64_t> seen_indices; - - // if there aren't enough outputs to mix with (or just enough), - // use all of them. Eventually this should become impossible. - if (num_outs <= count) - { - for (uint64_t i = 0; i < num_outs; i++) - { - // get tx_hash, tx_out_index from DB - tx_out_index toi = m_db->get_output_tx_and_index(amount, i); - - // if tx is unlocked, add output to indices - if (is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first))) - { - indices.push_back(i); - } - } - } - else - { - // while we still need more mixins - while (indices.size() < count) - { - // if we've gone through every possible output, we've gotten all we can - if (seen_indices.size() == num_outs) - { - break; - } - - // get a random output index from the DB. If we've already seen it, - // return to the top of the loop and try again, otherwise add it to the - // list of output indices we've seen. - - // triangular distribution over [a,b) with a=0, mode c=b=up_index_limit - uint64_t r = crypto::rand<uint64_t>() % ((uint64_t)1 << 53); - double frac = std::sqrt((double)r / ((uint64_t)1 << 53)); - uint64_t i = (uint64_t)(frac*num_outs); - // just in case rounding up to 1 occurs after sqrt - if (i == num_outs) - --i; - - if (seen_indices.count(i)) - { - continue; - } - seen_indices.emplace(i); - - // get tx_hash, tx_out_index from DB - tx_out_index toi = m_db->get_output_tx_and_index(amount, i); - - // if the output's transaction is unlocked, add the output's index to - // our list. - if (is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first))) - { - indices.push_back(i); - } - } - } - - return indices; -} - crypto::public_key Blockchain::get_output_key(uint64_t amount, uint64_t global_index) const { output_data_t data = m_db->get_output_key(amount, global_index); @@ -1820,169 +1741,6 @@ crypto::public_key Blockchain::get_output_key(uint64_t amount, uint64_t global_i } //------------------------------------------------------------------ -// This function takes an RPC request for mixins and creates an RPC response -// with the requested mixins. -// TODO: figure out why this returns boolean / if we should be returning false -// in some cases -bool Blockchain::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) const -{ - LOG_PRINT_L3("Blockchain::" << __func__); - CRITICAL_REGION_LOCAL(m_blockchain_lock); - - // for each amount that we need to get mixins for, get <n> random outputs - // from BlockchainDB where <n> is req.outs_count (number of mixins). - for (uint64_t amount : req.amounts) - { - // create outs_for_amount struct and populate amount field - COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs = *res.outs.insert(res.outs.end(), COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount()); - result_outs.amount = amount; - - std::vector<uint64_t> indices = get_random_outputs(amount, req.outs_count); - - for (auto i : indices) - { - COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry& oe = *result_outs.outs.insert(result_outs.outs.end(), COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry()); - - oe.global_amount_index = i; - oe.out_key = get_output_key(amount, i); - } - } - return true; -} -//------------------------------------------------------------------ -// This function adds the ringct output at index i to the list -// unlocked and other such checks should be done by here. -void Blockchain::add_out_to_get_rct_random_outs(std::list<COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::out_entry>& outs, uint64_t amount, size_t i) const -{ - LOG_PRINT_L3("Blockchain::" << __func__); - CRITICAL_REGION_LOCAL(m_blockchain_lock); - - COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::out_entry& oen = *outs.insert(outs.end(), COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::out_entry()); - oen.amount = amount; - oen.global_amount_index = i; - output_data_t data = m_db->get_output_key(amount, i); - oen.out_key = data.pubkey; - oen.commitment = data.commitment; -} -//------------------------------------------------------------------ -// This function takes an RPC request for mixins and creates an RPC response -// with the requested mixins. -// TODO: figure out why this returns boolean / if we should be returning false -// in some cases -bool Blockchain::get_random_rct_outs(const COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::request& req, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::response& res) const -{ - LOG_PRINT_L3("Blockchain::" << __func__); - CRITICAL_REGION_LOCAL(m_blockchain_lock); - - // for each amount that we need to get mixins for, get <n> random outputs - // from BlockchainDB where <n> is req.outs_count (number of mixins). - auto num_outs = m_db->get_num_outputs(0); - // ensure we don't include outputs that aren't yet eligible to be used - // outpouts are sorted by height - while (num_outs > 0) - { - const tx_out_index toi = m_db->get_output_tx_and_index(0, num_outs - 1); - const uint64_t height = m_db->get_tx_block_height(toi.first); - if (height + CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE <= m_db->height()) - break; - --num_outs; - } - - std::unordered_set<uint64_t> seen_indices; - - // if there aren't enough outputs to mix with (or just enough), - // use all of them. Eventually this should become impossible. - if (num_outs <= req.outs_count) - { - for (uint64_t i = 0; i < num_outs; i++) - { - // get tx_hash, tx_out_index from DB - tx_out_index toi = m_db->get_output_tx_and_index(0, i); - - // if tx is unlocked, add output to result_outs - if (is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first))) - { - add_out_to_get_rct_random_outs(res.outs, 0, i); - } - } - } - else - { - // while we still need more mixins - while (res.outs.size() < req.outs_count) - { - // if we've gone through every possible output, we've gotten all we can - if (seen_indices.size() == num_outs) - { - break; - } - - // get a random output index from the DB. If we've already seen it, - // return to the top of the loop and try again, otherwise add it to the - // list of output indices we've seen. - - // triangular distribution over [a,b) with a=0, mode c=b=up_index_limit - uint64_t r = crypto::rand<uint64_t>() % ((uint64_t)1 << 53); - double frac = std::sqrt((double)r / ((uint64_t)1 << 53)); - uint64_t i = (uint64_t)(frac*num_outs); - // just in case rounding up to 1 occurs after sqrt - if (i == num_outs) - --i; - - if (seen_indices.count(i)) - { - continue; - } - seen_indices.emplace(i); - - // get tx_hash, tx_out_index from DB - tx_out_index toi = m_db->get_output_tx_and_index(0, i); - - // if the output's transaction is unlocked, add the output's index to - // our list. - if (is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first))) - { - add_out_to_get_rct_random_outs(res.outs, 0, i); - } - } - } - - if (res.outs.size() < req.outs_count) - return false; -#if 0 - // if we do not have enough RCT inputs, we can pick from the non RCT ones - // which will have a zero mask - if (res.outs.size() < req.outs_count) - { - LOG_PRINT_L0("Out of RCT inputs (" << res.outs.size() << "/" << req.outs_count << "), using regular ones"); - - // TODO: arbitrary selection, needs better - COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request req2 = AUTO_VAL_INIT(req2); - COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response res2 = AUTO_VAL_INIT(res2); - req2.outs_count = req.outs_count - res.outs.size(); - static const uint64_t amounts[] = {1, 10, 20, 50, 100, 200, 500, 1000, 10000}; - for (uint64_t a: amounts) - req2.amounts.push_back(a); - if (!get_random_outs_for_amounts(req2, res2)) - return false; - - // pick random ones from there - while (res.outs.size() < req.outs_count) - { - int list_idx = rand() % (sizeof(amounts)/sizeof(amounts[0])); - if (!res2.outs[list_idx].outs.empty()) - { - const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry oe = res2.outs[list_idx].outs.back(); - res2.outs[list_idx].outs.pop_back(); - add_out_to_get_rct_random_outs(res.outs, res2.outs[list_idx].amount, oe.global_amount_index); - } - } - } -#endif - - return true; -} -//------------------------------------------------------------------ bool Blockchain::get_outs(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res) const { LOG_PRINT_L3("Blockchain::" << __func__); diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index 7e2ba7a39..50ceccd0f 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -447,16 +447,6 @@ namespace cryptonote uint64_t get_num_mature_outputs(uint64_t amount) const; /** - * @brief get random outputs (indices) for an amount - * - * @param amount the amount - * @param count the number of random outputs to choose - * - * @return the outputs' amount-global indices - */ - std::vector<uint64_t> get_random_outputs(uint64_t amount, uint64_t count) const; - - /** * @brief get the public key for an output * * @param amount the output amount @@ -467,22 +457,6 @@ namespace cryptonote crypto::public_key get_output_key(uint64_t amount, uint64_t global_index) const; /** - * @brief gets random outputs to mix with - * - * This function takes an RPC request for outputs to mix with - * and creates an RPC response with the resultant output indices. - * - * Outputs to mix with are randomly selected from the utxo set - * for each output amount in the request. - * - * @param req the output amounts and number of mixins to select - * @param res return-by-reference the resultant output indices - * - * @return true - */ - bool get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) const; - - /** * @brief gets specific outputs to mix with * * This function takes an RPC request for outputs to mix with @@ -509,23 +483,6 @@ namespace cryptonote void get_output_key_mask_unlocked(const uint64_t& amount, const uint64_t& index, crypto::public_key& key, rct::key& mask, bool& unlocked) const; /** - * @brief gets random ringct outputs to mix with - * - * This function takes an RPC request for outputs to mix with - * and creates an RPC response with the resultant output indices - * and the matching keys. - * - * Outputs to mix with are randomly selected from the utxo set - * for each output amount in the request. - * - * @param req the output amounts and number of mixins to select - * @param res return-by-reference the resultant output indices - * - * @return true - */ - bool get_random_rct_outs(const COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::request& req, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::response& res) const; - - /** * @brief gets per block distribution of outputs of a given amount * * @param amount the amount to get a ditribution for @@ -1272,24 +1229,6 @@ namespace cryptonote void get_last_n_blocks_weights(std::vector<size_t>& weights, size_t count) const; /** - * @brief adds the given output to the requested set of random outputs - * - * @param result_outs return-by-reference the set the output is to be added to - * @param amount the output amount - * @param i the output index (indexed to amount) - */ - void add_out_to_get_random_outs(COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i) const; - - /** - * @brief adds the given output to the requested set of random ringct outputs - * - * @param outs return-by-reference the set the output is to be added to - * @param amount the output amount (0 for rct inputs) - * @param i the rct output index - */ - void add_out_to_get_rct_random_outs(std::list<COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::out_entry>& outs, uint64_t amount, size_t i) const; - - /** * @brief checks if a transaction is unlocked (its outputs spendable) * * This function checks to see if a transaction is unlocked. diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index c4eaa0cc4..7cbf414b7 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -1181,21 +1181,11 @@ namespace cryptonote return m_blockchain_storage.find_blockchain_supplement(req_start_block, qblock_ids, blocks, total_height, start_height, pruned, get_miner_tx_hash, max_count); } //----------------------------------------------------------------------------------------------- - bool core::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) const - { - return m_blockchain_storage.get_random_outs_for_amounts(req, res); - } - //----------------------------------------------------------------------------------------------- bool core::get_outs(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res) const { return m_blockchain_storage.get_outs(req, res); } //----------------------------------------------------------------------------------------------- - bool core::get_random_rct_outs(const COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::request& req, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::response& res) const - { - return m_blockchain_storage.get_random_rct_outs(req, res); - } - //----------------------------------------------------------------------------------------------- bool core::get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t to_height, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) const { return m_blockchain_storage.get_output_distribution(amount, from_height, to_height, start_height, distribution, base); diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 8b68f5e2b..b40575ae9 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -551,13 +551,6 @@ namespace cryptonote difficulty_type get_block_cumulative_difficulty(uint64_t height) const; /** - * @copydoc Blockchain::get_random_outs_for_amounts - * - * @note see Blockchain::get_random_outs_for_amounts - */ - bool get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) const; - - /** * @copydoc Blockchain::get_outs * * @note see Blockchain::get_outs @@ -565,14 +558,6 @@ namespace cryptonote bool get_outs(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res) const; /** - * - * @copydoc Blockchain::get_random_rct_outs - * - * @note see Blockchain::get_random_rct_outs - */ - bool get_random_rct_outs(const COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::request& req, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::response& res) const; - - /** * @copydoc Blockchain::get_output_distribution * * @brief get per block distribution of outputs of a given amount diff --git a/src/debug_utilities/cn_deserialize.cpp b/src/debug_utilities/cn_deserialize.cpp index 6c09b0f18..bf228fe14 100644 --- a/src/debug_utilities/cn_deserialize.cpp +++ b/src/debug_utilities/cn_deserialize.cpp @@ -43,6 +43,15 @@ using namespace epee; using namespace cryptonote; +static std::string extra_nonce_to_string(const cryptonote::tx_extra_nonce &extra_nonce) +{ + if (extra_nonce.nonce.size() == 9 && extra_nonce.nonce[0] == TX_EXTRA_NONCE_ENCRYPTED_PAYMENT_ID) + return "encrypted payment ID: " + epee::string_tools::buff_to_hex_nodelimer(extra_nonce.nonce.substr(1)); + if (extra_nonce.nonce.size() == 33 && extra_nonce.nonce[0] == TX_EXTRA_NONCE_PAYMENT_ID) + return "plaintext payment ID: " + epee::string_tools::buff_to_hex_nodelimer(extra_nonce.nonce.substr(1)); + return epee::string_tools::buff_to_hex_nodelimer(extra_nonce.nonce); +} + static void print_extra_fields(const std::vector<cryptonote::tx_extra_field> &fields) { std::cout << "tx_extra has " << fields.size() << " field(s)" << std::endl; @@ -51,7 +60,7 @@ static void print_extra_fields(const std::vector<cryptonote::tx_extra_field> &fi std::cout << "field " << n << ": "; if (typeid(cryptonote::tx_extra_padding) == fields[n].type()) std::cout << "extra padding: " << boost::get<cryptonote::tx_extra_padding>(fields[n]).size << " bytes"; else if (typeid(cryptonote::tx_extra_pub_key) == fields[n].type()) std::cout << "extra pub key: " << boost::get<cryptonote::tx_extra_pub_key>(fields[n]).pub_key; - else if (typeid(cryptonote::tx_extra_nonce) == fields[n].type()) std::cout << "extra nonce: " << epee::string_tools::buff_to_hex_nodelimer(boost::get<cryptonote::tx_extra_nonce>(fields[n]).nonce); + else if (typeid(cryptonote::tx_extra_nonce) == fields[n].type()) std::cout << "extra nonce: " << extra_nonce_to_string(boost::get<cryptonote::tx_extra_nonce>(fields[n])); else if (typeid(cryptonote::tx_extra_merge_mining_tag) == fields[n].type()) std::cout << "extra merge mining tag: depth " << boost::get<cryptonote::tx_extra_merge_mining_tag>(fields[n]).depth << ", merkle root " << boost::get<cryptonote::tx_extra_merge_mining_tag>(fields[n]).merkle_root; else if (typeid(cryptonote::tx_extra_additional_pub_keys) == fields[n].type()) std::cout << "additional tx pubkeys: " << boost::join(boost::get<cryptonote::tx_extra_additional_pub_keys>(fields[n]).data | boost::adaptors::transformed([](const crypto::public_key &key){ return epee::string_tools::pod_to_hex(key); }), ", " ); else if (typeid(cryptonote::tx_extra_mysterious_minergate) == fields[n].type()) std::cout << "extra minergate custom: " << epee::string_tools::buff_to_hex_nodelimer(boost::get<cryptonote::tx_extra_mysterious_minergate>(fields[n]).data); @@ -168,9 +177,12 @@ int main(int argc, char* argv[]) std::cout << "Parsed block:" << std::endl; std::cout << cryptonote::obj_to_json_str(block) << std::endl; } - else if (cryptonote::parse_and_validate_tx_from_blob(blob, tx)) + else if (cryptonote::parse_and_validate_tx_from_blob(blob, tx) || cryptonote::parse_and_validate_tx_base_from_blob(blob, tx)) { - std::cout << "Parsed transaction:" << std::endl; + if (tx.pruned) + std::cout << "Parsed pruned transaction:" << std::endl; + else + std::cout << "Parsed transaction:" << std::endl; std::cout << cryptonote::obj_to_json_str(tx) << std::endl; bool parsed = cryptonote::parse_tx_extra(tx.extra, fields); diff --git a/src/device/device_ledger.cpp b/src/device/device_ledger.cpp index 658b379e4..9b5ea0fd7 100644 --- a/src/device/device_ledger.cpp +++ b/src/device/device_ledger.cpp @@ -1354,7 +1354,7 @@ namespace hw { this->exchange(); //pseudoOuts - if ((type == rct::RCTTypeSimple) || (type == rct::RCTTypeBulletproof)) { + if (type == rct::RCTTypeSimple) { for ( i = 0; i < inputs_size; i++) { offset = set_command_header(INS_VALIDATE, 0x01, i+2); //options diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 4383ad190..db7f2dbaa 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -364,49 +364,6 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_random_outs(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) - { - PERF_TIMER(on_get_random_outs); - bool r; - if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS>(invoke_http_mode::BIN, "/getrandom_outs.bin", req, res, r)) - return r; - - res.status = "Failed"; - - if (m_restricted) - { - if (req.amounts.size() > 100 || req.outs_count > MAX_RESTRICTED_FAKE_OUTS_COUNT) - { - res.status = "Too many outs requested"; - return true; - } - } - - if(!m_core.get_random_outs_for_amounts(req, res)) - { - return true; - } - - res.status = CORE_RPC_STATUS_OK; - std::stringstream ss; - typedef COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount outs_for_amount; - typedef COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry out_entry; - std::for_each(res.outs.begin(), res.outs.end(), [&](outs_for_amount& ofa) - { - ss << "[" << ofa.amount << "]:"; - CHECK_AND_ASSERT_MES(ofa.outs.size(), ;, "internal error: ofa.outs.size() is empty for amount " << ofa.amount); - std::for_each(ofa.outs.begin(), ofa.outs.end(), [&](out_entry& oe) - { - ss << oe.global_amount_index << " "; - }); - ss << ENDL; - }); - std::string s = ss.str(); - LOG_PRINT_L2("COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS: " << ENDL << s); - res.status = CORE_RPC_STATUS_OK; - return true; - } - //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_get_outs_bin(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res) { PERF_TIMER(on_get_outs_bin); @@ -476,34 +433,6 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_random_rct_outs(const COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::request& req, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::response& res) - { - PERF_TIMER(on_get_random_rct_outs); - bool r; - if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS>(invoke_http_mode::BIN, "/getrandom_rctouts.bin", req, res, r)) - return r; - - res.status = "Failed"; - if(!m_core.get_random_rct_outs(req, res)) - { - return true; - } - - res.status = CORE_RPC_STATUS_OK; - std::stringstream ss; - typedef COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::out_entry out_entry; - CHECK_AND_ASSERT_MES(res.outs.size(), true, "internal error: res.outs.size() is empty"); - std::for_each(res.outs.begin(), res.outs.end(), [&](out_entry& oe) - { - ss << oe.global_amount_index << " "; - }); - ss << ENDL; - std::string s = ss.str(); - LOG_PRINT_L2("COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS: " << ENDL << s); - res.status = CORE_RPC_STATUS_OK; - return true; - } - //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_get_indexes(const COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::request& req, COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::response& res) { PERF_TIMER(on_get_indexes); diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index 5dbe44d24..3ba882b23 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -87,11 +87,7 @@ namespace cryptonote MAP_URI_AUTO_BIN2("/get_hashes.bin", on_get_hashes, COMMAND_RPC_GET_HASHES_FAST) MAP_URI_AUTO_BIN2("/gethashes.bin", on_get_hashes, COMMAND_RPC_GET_HASHES_FAST) MAP_URI_AUTO_BIN2("/get_o_indexes.bin", on_get_indexes, COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES) - MAP_URI_AUTO_BIN2("/get_random_outs.bin", on_get_random_outs, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS) - MAP_URI_AUTO_BIN2("/getrandom_outs.bin", on_get_random_outs, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS) MAP_URI_AUTO_BIN2("/get_outs.bin", on_get_outs_bin, COMMAND_RPC_GET_OUTPUTS_BIN) - MAP_URI_AUTO_BIN2("/get_random_rctouts.bin", on_get_random_rct_outs, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS) - MAP_URI_AUTO_BIN2("/getrandom_rctouts.bin", on_get_random_rct_outs, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS) MAP_URI_AUTO_JON2("/get_transactions", on_get_transactions, COMMAND_RPC_GET_TRANSACTIONS) MAP_URI_AUTO_JON2("/gettransactions", on_get_transactions, COMMAND_RPC_GET_TRANSACTIONS) MAP_URI_AUTO_JON2("/get_alt_blocks_hashes", on_get_alt_blocks_hashes, COMMAND_RPC_GET_ALT_BLOCKS_HASHES) @@ -171,10 +167,8 @@ namespace cryptonote bool on_start_mining(const COMMAND_RPC_START_MINING::request& req, COMMAND_RPC_START_MINING::response& res); bool on_stop_mining(const COMMAND_RPC_STOP_MINING::request& req, COMMAND_RPC_STOP_MINING::response& res); bool on_mining_status(const COMMAND_RPC_MINING_STATUS::request& req, COMMAND_RPC_MINING_STATUS::response& res); - bool on_get_random_outs(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res); bool on_get_outs_bin(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMAND_RPC_GET_OUTPUTS_BIN::response& res); bool on_get_outs(const COMMAND_RPC_GET_OUTPUTS::request& req, COMMAND_RPC_GET_OUTPUTS::response& res); - bool on_get_random_rct_outs(const COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::request& req, COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS::response& res); bool on_get_info(const COMMAND_RPC_GET_INFO::request& req, COMMAND_RPC_GET_INFO::response& res); bool on_save_bc(const COMMAND_RPC_SAVE_BC::request& req, COMMAND_RPC_SAVE_BC::response& res); bool on_get_peer_list(const COMMAND_RPC_GET_PEER_LIST::request& req, COMMAND_RPC_GET_PEER_LIST::response& res); diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index da4c8b6cf..1a84ee614 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -680,50 +680,6 @@ namespace cryptonote }; }; //----------------------------------------------- - struct COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS - { - struct request - { - std::vector<uint64_t> amounts; - uint64_t outs_count; - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(amounts) - KV_SERIALIZE(outs_count) - END_KV_SERIALIZE_MAP() - }; - -#pragma pack (push, 1) - struct out_entry - { - uint64_t global_amount_index; - crypto::public_key out_key; - }; -#pragma pack(pop) - - struct outs_for_amount - { - uint64_t amount; - std::list<out_entry> outs; - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(amount) - KV_SERIALIZE_CONTAINER_POD_AS_BLOB(outs) - END_KV_SERIALIZE_MAP() - }; - - struct response - { - std::vector<outs_for_amount> outs; - std::string status; - bool untrusted; - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(outs) - KV_SERIALIZE(status) - KV_SERIALIZE(untrusted) - END_KV_SERIALIZE_MAP() - }; - }; - //----------------------------------------------- struct get_outputs_out { uint64_t amount; @@ -818,39 +774,6 @@ namespace cryptonote END_KV_SERIALIZE_MAP() }; }; - - struct COMMAND_RPC_GET_RANDOM_RCT_OUTPUTS - { - struct request - { - uint64_t outs_count; - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(outs_count) - END_KV_SERIALIZE_MAP() - }; - -#pragma pack (push, 1) - struct out_entry - { - uint64_t amount; - uint64_t global_amount_index; - crypto::public_key out_key; - rct::key commitment; - }; -#pragma pack(pop) - - struct response - { - std::list<out_entry> outs; - std::string status; - bool untrusted; - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE_CONTAINER_POD_AS_BLOB(outs) - KV_SERIALIZE(status) - KV_SERIALIZE(untrusted) - END_KV_SERIALIZE_MAP() - }; - }; //----------------------------------------------- struct COMMAND_RPC_SEND_RAW_TX { diff --git a/src/rpc/daemon_handler.cpp b/src/rpc/daemon_handler.cpp index 26f102a8b..9d3b09b68 100644 --- a/src/rpc/daemon_handler.cpp +++ b/src/rpc/daemon_handler.cpp @@ -260,44 +260,6 @@ namespace rpc } - //TODO: handle "restricted" RPC - void DaemonHandler::handle(const GetRandomOutputsForAmounts::Request& req, GetRandomOutputsForAmounts::Response& res) - { - auto& chain = m_core.get_blockchain_storage(); - - try - { - for (const uint64_t& amount : req.amounts) - { - std::vector<uint64_t> indices = chain.get_random_outputs(amount, req.count); - - outputs_for_amount ofa; - - ofa.resize(indices.size()); - - for (size_t i = 0; i < indices.size(); i++) - { - crypto::public_key key = chain.get_output_key(amount, indices[i]); - ofa[i].amount_index = indices[i]; - ofa[i].key = key; - } - - amount_with_random_outputs amt; - amt.amount = amount; - amt.outputs = ofa; - - res.amounts_with_outputs.push_back(amt); - } - - res.status = Message::STATUS_OK; - } - catch (const std::exception& e) - { - res.status = Message::STATUS_FAILED; - res.error_details = e.what(); - } - } - void DaemonHandler::handle(const SendRawTx::Request& req, SendRawTx::Response& res) { auto tx_blob = cryptonote::tx_to_blob(req.tx); @@ -824,7 +786,6 @@ namespace rpc REQ_RESP_TYPES_MACRO(request_type, GetTransactions, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, KeyImagesSpent, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, GetTxGlobalOutputIndices, req_json, resp_message, handle); - REQ_RESP_TYPES_MACRO(request_type, GetRandomOutputsForAmounts, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, SendRawTx, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, GetInfo, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, StartMining, req_json, resp_message, handle); diff --git a/src/rpc/daemon_handler.h b/src/rpc/daemon_handler.h index f43711640..5f9687511 100644 --- a/src/rpc/daemon_handler.h +++ b/src/rpc/daemon_handler.h @@ -66,8 +66,6 @@ class DaemonHandler : public RpcHandler void handle(const GetTxGlobalOutputIndices::Request& req, GetTxGlobalOutputIndices::Response& res); - void handle(const GetRandomOutputsForAmounts::Request& req, GetRandomOutputsForAmounts::Response& res); - void handle(const SendRawTx::Request& req, SendRawTx::Response& res); void handle(const StartMining::Request& req, StartMining::Response& res); diff --git a/src/rpc/daemon_messages.cpp b/src/rpc/daemon_messages.cpp index e5fb9781c..56f6f6a8c 100644 --- a/src/rpc/daemon_messages.cpp +++ b/src/rpc/daemon_messages.cpp @@ -41,7 +41,6 @@ const char* const GetHashesFast::name = "get_hashes_fast"; const char* const GetTransactions::name = "get_transactions"; const char* const KeyImagesSpent::name = "key_images_spent"; const char* const GetTxGlobalOutputIndices::name = "get_tx_global_output_indices"; -const char* const GetRandomOutputsForAmounts::name = "get_random_outputs_for_amounts"; const char* const SendRawTx::name = "send_raw_tx"; const char* const StartMining::name = "start_mining"; const char* const StopMining::name = "stop_mining"; @@ -273,42 +272,6 @@ void GetTxGlobalOutputIndices::Response::fromJson(rapidjson::Value& val) GET_FROM_JSON_OBJECT(val, output_indices, output_indices); } - -rapidjson::Value GetRandomOutputsForAmounts::Request::toJson(rapidjson::Document& doc) const -{ - auto val = Message::toJson(doc); - - auto& al = doc.GetAllocator(); - - INSERT_INTO_JSON_OBJECT(val, doc, amounts, amounts); - INSERT_INTO_JSON_OBJECT(val, doc, count, count); - - return val; -} - -void GetRandomOutputsForAmounts::Request::fromJson(rapidjson::Value& val) -{ - GET_FROM_JSON_OBJECT(val, amounts, amounts); - GET_FROM_JSON_OBJECT(val, count, count); -} - -rapidjson::Value GetRandomOutputsForAmounts::Response::toJson(rapidjson::Document& doc) const -{ - auto val = Message::toJson(doc); - - auto& al = doc.GetAllocator(); - - INSERT_INTO_JSON_OBJECT(val, doc, amounts_with_outputs, amounts_with_outputs); - - return val; -} - -void GetRandomOutputsForAmounts::Response::fromJson(rapidjson::Value& val) -{ - GET_FROM_JSON_OBJECT(val, amounts_with_outputs, amounts_with_outputs); -} - - rapidjson::Value SendRawTx::Request::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index bcdf2a43f..602cdfe44 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -445,7 +445,7 @@ namespace LOG_ERROR("RPC error: " << e.to_string()); fail_msg_writer() << tr("RPC error: ") << e.what(); } - catch (const tools::error::get_random_outs_error &e) + catch (const tools::error::get_outs_error &e) { fail_msg_writer() << tr("failed to get random outputs to mix: ") << e.what(); } @@ -1627,23 +1627,23 @@ bool simple_wallet::set_ring(const std::vector<std::string> &args) bool simple_wallet::blackball(const std::vector<std::string> &args) { - crypto::public_key output; + uint64_t amount = std::numeric_limits<uint64_t>::max(), offset, num_offsets; if (args.size() == 0) { - fail_msg_writer() << tr("usage: blackball <output_public_key> | <filename> [add]"); + fail_msg_writer() << tr("usage: blackball <amount>/<offset> | <filename> [add]"); return true; } try { - if (epee::string_tools::hex_to_pod(args[0], output)) + if (sscanf(args[0].c_str(), "%" PRIu64 "/%" PRIu64, &amount, &offset) == 2) { - m_wallet->blackball_output(output); + m_wallet->blackball_output(std::make_pair(amount, offset)); } else if (epee::file_io_utils::is_file_exist(args[0])) { - std::vector<crypto::public_key> outputs; - char str[65]; + std::vector<std::pair<uint64_t, uint64_t>> outputs; + char str[256]; std::unique_ptr<FILE, tools::close_file> f(fopen(args[0].c_str(), "r")); if (f) @@ -1657,10 +1657,27 @@ bool simple_wallet::blackball(const std::vector<std::string> &args) str[len - 1] = 0; if (!str[0]) continue; - outputs.push_back(crypto::public_key()); - if (!epee::string_tools::hex_to_pod(str, outputs.back())) + if (sscanf(str, "@%" PRIu64, &amount) == 1) { - fail_msg_writer() << tr("Invalid public key: ") << str; + continue; + } + if (amount == std::numeric_limits<uint64_t>::max()) + { + fail_msg_writer() << tr("First line is not an amount"); + return true; + } + if (sscanf(str, "%" PRIu64 "*%" PRIu64, &offset, &num_offsets) == 2 && num_offsets <= std::numeric_limits<uint64_t>::max() - offset) + { + while (num_offsets--) + outputs.push_back(std::make_pair(amount, offset++)); + } + else if (sscanf(str, "%" PRIu64, &offset) == 1) + { + outputs.push_back(std::make_pair(amount, offset)); + } + else + { + fail_msg_writer() << tr("Invalid output: ") << str; return true; } } @@ -1685,7 +1702,7 @@ bool simple_wallet::blackball(const std::vector<std::string> &args) } else { - fail_msg_writer() << tr("Invalid public key, and file doesn't exist"); + fail_msg_writer() << tr("Invalid output key, and file doesn't exist"); return true; } } @@ -1699,16 +1716,16 @@ bool simple_wallet::blackball(const std::vector<std::string> &args) bool simple_wallet::unblackball(const std::vector<std::string> &args) { - crypto::public_key output; + std::pair<uint64_t, uint64_t> output; if (args.size() != 1) { - fail_msg_writer() << tr("usage: unblackball <output_public_key>"); + fail_msg_writer() << tr("usage: unblackball <amount>/<offset>"); return true; } - if (!epee::string_tools::hex_to_pod(args[0], output)) + if (sscanf(args[0].c_str(), "%" PRIu64 "/%" PRIu64, &output.first, &output.second) != 2) { - fail_msg_writer() << tr("Invalid public key"); + fail_msg_writer() << tr("Invalid output"); return true; } @@ -1726,25 +1743,25 @@ bool simple_wallet::unblackball(const std::vector<std::string> &args) bool simple_wallet::blackballed(const std::vector<std::string> &args) { - crypto::public_key output; + std::pair<uint64_t, uint64_t> output; if (args.size() != 1) { - fail_msg_writer() << tr("usage: blackballed <output_public_key>"); + fail_msg_writer() << tr("usage: blackballed <amount>/<offset>"); return true; } - if (!epee::string_tools::hex_to_pod(args[0], output)) + if (sscanf(args[0].c_str(), "%" PRIu64 "/%" PRIu64, &output.first, &output.second) != 2) { - fail_msg_writer() << tr("Invalid public key"); + fail_msg_writer() << tr("Invalid output"); return true; } try { if (m_wallet->is_output_blackballed(output)) - message_writer() << tr("Blackballed: ") << output; + message_writer() << tr("Blackballed: ") << output.first << "/" << output.second; else - message_writer() << tr("not blackballed: ") << output; + message_writer() << tr("not blackballed: ") << output.first << "/" << output.second; } catch (const std::exception &e) { @@ -2271,15 +2288,15 @@ simple_wallet::simple_wallet() tr("Show the blockchain height.")); m_cmd_binder.set_handler("transfer_original", boost::bind(&simple_wallet::transfer, this, _1), - tr("transfer_original [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> <amount> [<payment_id>]"), - tr("Transfer <amount> to <address> using an older transaction building algorithm. If the parameter \"index=<N1>[,<N2>,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. <priority> is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability. Multiple payments can be made at once by adding <address_2> <amount_2> etcetera (before the payment ID, if it's included)")); + tr("transfer_original [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] (<URI> | <address> <amount>) [<payment_id>]"), + tr("Transfer <amount> to <address> using an older transaction building algorithm. If the parameter \"index=<N1>[,<N2>,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. <priority> is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability. Multiple payments can be made at once by adding URI_2 or <address_2> <amount_2> etcetera (before the payment ID, if it's included)")); m_cmd_binder.set_handler("transfer", boost::bind(&simple_wallet::transfer_new, this, _1), - tr("transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> <amount> [<payment_id>]"), - tr("Transfer <amount> to <address>. If the parameter \"index=<N1>[,<N2>,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. <priority> is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability. Multiple payments can be made at once by adding <address_2> <amount_2> etcetera (before the payment ID, if it's included)")); + tr("transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] (<URI> | <address> <amount>) [<payment_id>]"), + tr("Transfer <amount> to <address>. If the parameter \"index=<N1>[,<N2>,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. <priority> is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability. Multiple payments can be made at once by adding URI_2 or <address_2> <amount_2> etcetera (before the payment ID, if it's included)")); m_cmd_binder.set_handler("locked_transfer", boost::bind(&simple_wallet::locked_transfer, this, _1), - tr("locked_transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <addr> <amount> <lockblocks> [<payment_id>]"), - tr("Transfer <amount> to <address> and lock it for <lockblocks> (max. 1000000). If the parameter \"index=<N1>[,<N2>,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. <priority> is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability. Multiple payments can be made at once by adding <address_2> <amount_2> etcetera (before the payment ID, if it's included)")); + tr("locked_transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] (<URI> | <addr> <amount>) <lockblocks> [<payment_id>]"), + tr("Transfer <amount> to <address> and lock it for <lockblocks> (max. 1000000). If the parameter \"index=<N1>[,<N2>,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. <priority> is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability. Multiple payments can be made at once by adding URI_2 or <address_2> <amount_2> etcetera (before the payment ID, if it's included)")); m_cmd_binder.set_handler("locked_sweep_all", boost::bind(&simple_wallet::locked_sweep_all, this, _1), tr("locked_sweep_all [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> <lockblocks> [<payment_id>]"), @@ -2559,15 +2576,15 @@ simple_wallet::simple_wallet() tr("Save known rings to the shared rings database")); m_cmd_binder.set_handler("blackball", boost::bind(&simple_wallet::blackball, this, _1), - tr("blackball <output public key> | <filename> [add]"), + tr("blackball <amount>/<offset> | <filename> [add]"), tr("Blackball output(s) so they never get selected as fake outputs in a ring")); m_cmd_binder.set_handler("unblackball", boost::bind(&simple_wallet::unblackball, this, _1), - tr("unblackball <output public key>"), + tr("unblackball <amount>/<offset>"), tr("Unblackballs an output so it may get selected as a fake output in a ring")); m_cmd_binder.set_handler("blackballed", boost::bind(&simple_wallet::blackballed, this, _1), - tr("blackballed <output public key>"), + tr("blackballed <amount>/<offset>"), tr("Checks whether an output is blackballed")); m_cmd_binder.set_handler("version", boost::bind(&simple_wallet::version, this, _1), @@ -4715,7 +4732,7 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri return true; } - const size_t min_args = (transfer_type == TransferLocked) ? 3 : 2; + const size_t min_args = (transfer_type == TransferLocked) ? 2 : 1; if(local_args.size() < min_args) { fail_msg_writer() << tr("wrong number of arguments"); @@ -4724,39 +4741,38 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri std::vector<uint8_t> extra; bool payment_id_seen = false; - bool expect_even = (transfer_type == TransferLocked); - if ((expect_even ? 0 : 1) == local_args.size() % 2) + if (!local_args.empty()) { std::string payment_id_str = local_args.back(); - local_args.pop_back(); - crypto::hash payment_id; - bool r = tools::wallet2::parse_long_payment_id(payment_id_str, payment_id); - if(r) + bool r = true; + if (tools::wallet2::parse_long_payment_id(payment_id_str, payment_id)) { std::string extra_nonce; set_payment_id_to_tx_extra_nonce(extra_nonce, payment_id); r = add_extra_nonce_to_tx_extra(extra, extra_nonce); + local_args.pop_back(); + payment_id_seen = true; + message_writer() << tr("Unencrypted payment IDs are bad for privacy: ask the recipient to use subaddresses instead"); } else { crypto::hash8 payment_id8; - r = tools::wallet2::parse_short_payment_id(payment_id_str, payment_id8); - if(r) + if (tools::wallet2::parse_short_payment_id(payment_id_str, payment_id8)) { std::string extra_nonce; set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, payment_id8); r = add_extra_nonce_to_tx_extra(extra, extra_nonce); + local_args.pop_back(); + payment_id_seen = true; } } if(!r) { - fail_msg_writer() << tr("payment id has invalid format, expected 16 or 64 character hex string: ") << payment_id_str; + fail_msg_writer() << tr("payment id failed to encode"); return true; } - payment_id_seen = true; - message_writer() << tr("Unencrypted payment IDs are bad for privacy: ask the recipient to use subaddresses instead"); } uint64_t locked_blocks = 0; @@ -4781,11 +4797,54 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri vector<cryptonote::tx_destination_entry> dsts; size_t num_subaddresses = 0; - for (size_t i = 0; i < local_args.size(); i += 2) + for (size_t i = 0; i < local_args.size(); ) { - cryptonote::address_parse_info info; cryptonote::tx_destination_entry de; - if (!cryptonote::get_account_address_from_str_or_url(info, m_wallet->nettype(), local_args[i], oa_prompter)) + cryptonote::address_parse_info info; + bool r = true; + + // check for a URI + std::string address_uri, payment_id_uri, tx_description, recipient_name, error; + std::vector<std::string> unknown_parameters; + uint64_t amount = 0; + bool has_uri = m_wallet->parse_uri(local_args[i], address_uri, payment_id_uri, amount, tx_description, recipient_name, unknown_parameters, error); + if (has_uri) + { + r = cryptonote::get_account_address_from_str_or_url(info, m_wallet->nettype(), address_uri, oa_prompter); + if (payment_id_uri.size() == 16) + { + if (!tools::wallet2::parse_short_payment_id(payment_id_uri, info.payment_id)) + { + fail_msg_writer() << tr("failed to parse short payment ID from URI"); + return true; + } + info.has_payment_id = true; + } + de.amount = amount; + ++i; + } + else if (i + 1 < local_args.size()) + { + r = cryptonote::get_account_address_from_str_or_url(info, m_wallet->nettype(), local_args[i], oa_prompter); + bool ok = cryptonote::parse_amount(de.amount, local_args[i + 1]); + if(!ok || 0 == de.amount) + { + fail_msg_writer() << tr("amount is wrong: ") << local_args[i] << ' ' << local_args[i + 1] << + ", " << tr("expected number from 0 to ") << print_money(std::numeric_limits<uint64_t>::max()); + return true; + } + i += 2; + } + else + { + if (boost::starts_with(local_args[i], "monero:")) + fail_msg_writer() << tr("Invalid last argument: ") << local_args.back() << ": " << error; + else + fail_msg_writer() << tr("Invalid last argument: ") << local_args.back(); + return true; + } + + if (!r) { fail_msg_writer() << tr("failed to parse address"); return true; @@ -4794,16 +4853,30 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri de.is_subaddress = info.is_subaddress; num_subaddresses += info.is_subaddress; - if (info.has_payment_id) + if (info.has_payment_id || !payment_id_uri.empty()) { if (payment_id_seen) { - fail_msg_writer() << tr("a single transaction cannot use more than one payment id: ") << local_args[i]; + fail_msg_writer() << tr("a single transaction cannot use more than one payment id"); return true; } + crypto::hash payment_id; std::string extra_nonce; - set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, info.payment_id); + if (info.has_payment_id) + { + set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, info.payment_id); + } + else if (tools::wallet2::parse_payment_id(payment_id_uri, payment_id)) + { + set_payment_id_to_tx_extra_nonce(extra_nonce, payment_id); + message_writer() << tr("Unencrypted payment IDs are bad for privacy: ask the recipient to use subaddresses instead"); + } + else + { + fail_msg_writer() << tr("failed to parse payment id, though it was detected"); + return true; + } bool r = add_extra_nonce_to_tx_extra(extra, extra_nonce); if(!r) { @@ -4813,14 +4886,6 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri payment_id_seen = true; } - bool ok = cryptonote::parse_amount(de.amount, local_args[i + 1]); - if(!ok || 0 == de.amount) - { - fail_msg_writer() << tr("amount is wrong: ") << local_args[i] << ' ' << local_args[i + 1] << - ", " << tr("expected number from 0 to ") << print_money(std::numeric_limits<uint64_t>::max()); - return true; - } - dsts.push_back(de); } @@ -4856,15 +4921,12 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri unlock_block = bc_height + locked_blocks; ptx_vector = m_wallet->create_transactions_2(dsts, fake_outs_count, unlock_block /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices); break; + default: + LOG_ERROR("Unknown transfer method, using default"); + /* FALLTHRU */ case TransferNew: ptx_vector = m_wallet->create_transactions_2(dsts, fake_outs_count, 0 /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices); break; - default: - LOG_ERROR("Unknown transfer method, using original"); - /* FALLTHRU */ - case TransferOriginal: - ptx_vector = m_wallet->create_transactions(dsts, fake_outs_count, 0 /* unlock_time */, priority, extra); - break; } if (ptx_vector.empty()) diff --git a/src/wallet/api/wallet.cpp b/src/wallet/api/wallet.cpp index 3780d7271..e9e9a4e67 100644 --- a/src/wallet/api/wallet.cpp +++ b/src/wallet/api/wallet.cpp @@ -1381,8 +1381,8 @@ PendingTransaction *WalletImpl::createTransaction(const string &dst_addr, const setStatusError(tr("no connection to daemon. Please make sure daemon is running.")); } catch (const tools::error::wallet_rpc_error& e) { setStatusError(tr("RPC error: ") + e.to_string()); - } catch (const tools::error::get_random_outs_error &e) { - setStatusError((boost::format(tr("failed to get random outputs to mix: %s")) % e.what()).str()); + } catch (const tools::error::get_outs_error &e) { + setStatusError((boost::format(tr("failed to get outputs to mix: %s")) % e.what()).str()); } catch (const tools::error::not_enough_unlocked_money& e) { std::ostringstream writer; @@ -1463,8 +1463,8 @@ PendingTransaction *WalletImpl::createSweepUnmixableTransaction() setStatusError(tr("no connection to daemon. Please make sure daemon is running.")); } catch (const tools::error::wallet_rpc_error& e) { setStatusError(tr("RPC error: ") + e.to_string()); - } catch (const tools::error::get_random_outs_error&) { - setStatusError(tr("failed to get random outputs to mix")); + } catch (const tools::error::get_outs_error&) { + setStatusError(tr("failed to get outputs to mix")); } catch (const tools::error::not_enough_unlocked_money& e) { setStatusError(""); std::ostringstream writer; diff --git a/src/wallet/ringdb.cpp b/src/wallet/ringdb.cpp index 3f2634c8b..e9fc6866d 100644 --- a/src/wallet/ringdb.cpp +++ b/src/wallet/ringdb.cpp @@ -55,6 +55,13 @@ static int compare_hash32(const MDB_val *a, const MDB_val *b) return 0; } +static int compare_uint64(const MDB_val *a, const MDB_val *b) +{ + const uint64_t va = *(const uint64_t*) a->mv_data; + const uint64_t vb = *(const uint64_t*) b->mv_data; + return va < vb ? -1 : va > vb; +} + static std::string compress_ring(const std::vector<uint64_t> &ring) { std::string s; @@ -146,7 +153,7 @@ static int resize_env(MDB_env *env, const char *db_path, size_t needed) MDB_stat mst; int ret; - needed = std::max(needed, (size_t)(2ul * 1024 * 1024)); // at least 2 MB + needed = std::max(needed, (size_t)(100ul * 1024 * 1024)); // at least 100 MB ret = mdb_env_info(env, &mei); if (ret) @@ -217,9 +224,9 @@ ringdb::ringdb(std::string filename, const std::string &genesis): THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); mdb_set_compare(txn, dbi_rings, compare_hash32); - dbr = mdb_dbi_open(txn, ("blackballs-" + genesis).c_str(), MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi_blackballs); + dbr = mdb_dbi_open(txn, ("blackballs2-" + genesis).c_str(), MDB_CREATE | MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi_blackballs); THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); - mdb_set_dupsort(txn, dbi_blackballs, compare_hash32); + mdb_set_dupsort(txn, dbi_blackballs, compare_uint64); dbr = mdb_txn_commit(txn); THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); @@ -374,7 +381,7 @@ bool ringdb::set_ring(const crypto::chacha_key &chacha_key, const crypto::key_im return true; } -bool ringdb::blackball_worker(const crypto::public_key &output, int op) +bool ringdb::blackball_worker(const std::vector<std::pair<uint64_t, uint64_t>> &outputs, int op) { MDB_txn *txn; MDB_cursor *cursor; @@ -382,49 +389,61 @@ bool ringdb::blackball_worker(const crypto::public_key &output, int op) bool tx_active = false; bool ret = true; - dbr = resize_env(env, filename.c_str(), 32 * 2); // a pubkey, and some slack + THROW_WALLET_EXCEPTION_IF(outputs.size() > 1 && op == BLACKBALL_QUERY, tools::error::wallet_internal_error, "Blackball query only makes sense for a single output"); + + dbr = resize_env(env, filename.c_str(), 32 * 2 * outputs.size()); // a pubkey, and some slack THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to set env map size: " + std::string(mdb_strerror(dbr))); dbr = mdb_txn_begin(env, NULL, 0, &txn); THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); tx_active = true; - MDB_val key = zerokeyval; - MDB_val data; - data.mv_data = (void*)&output; - data.mv_size = sizeof(output); - switch (op) + MDB_val key, data; + for (const std::pair<uint64_t, uint64_t> &output: outputs) + { + key.mv_data = (void*)&output.first; + key.mv_size = sizeof(output.first); + data.mv_data = (void*)&output.second; + data.mv_size = sizeof(output.second); + + switch (op) + { + case BLACKBALL_BLACKBALL: + MDEBUG("Blackballing output " << output.first << "/" << output.second); + dbr = mdb_put(txn, dbi_blackballs, &key, &data, MDB_APPENDDUP); + if (dbr == MDB_KEYEXIST) + dbr = 0; + break; + case BLACKBALL_UNBLACKBALL: + MDEBUG("Unblackballing output " << output.first << "/" << output.second); + dbr = mdb_del(txn, dbi_blackballs, &key, &data); + if (dbr == MDB_NOTFOUND) + dbr = 0; + break; + case BLACKBALL_QUERY: + dbr = mdb_cursor_open(txn, dbi_blackballs, &cursor); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create cursor for blackballs table: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_get(cursor, &key, &data, MDB_GET_BOTH); + THROW_WALLET_EXCEPTION_IF(dbr && dbr != MDB_NOTFOUND, tools::error::wallet_internal_error, "Failed to lookup in blackballs table: " + std::string(mdb_strerror(dbr))); + ret = dbr != MDB_NOTFOUND; + if (dbr == MDB_NOTFOUND) + dbr = 0; + mdb_cursor_close(cursor); + break; + case BLACKBALL_CLEAR: + break; + default: + THROW_WALLET_EXCEPTION(tools::error::wallet_internal_error, "Invalid blackball op"); + } + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to query blackballs table: " + std::string(mdb_strerror(dbr))); + } + + if (op == BLACKBALL_CLEAR) { - case BLACKBALL_BLACKBALL: - MDEBUG("Blackballing output " << output); - dbr = mdb_put(txn, dbi_blackballs, &key, &data, MDB_NODUPDATA); - if (dbr == MDB_KEYEXIST) - dbr = 0; - break; - case BLACKBALL_UNBLACKBALL: - MDEBUG("Unblackballing output " << output); - dbr = mdb_del(txn, dbi_blackballs, &key, &data); - if (dbr == MDB_NOTFOUND) - dbr = 0; - break; - case BLACKBALL_QUERY: - dbr = mdb_cursor_open(txn, dbi_blackballs, &cursor); - THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to create cursor for blackballs table: " + std::string(mdb_strerror(dbr))); - dbr = mdb_cursor_get(cursor, &key, &data, MDB_GET_BOTH); - THROW_WALLET_EXCEPTION_IF(dbr && dbr != MDB_NOTFOUND, tools::error::wallet_internal_error, "Failed to lookup in blackballs table: " + std::string(mdb_strerror(dbr))); - ret = dbr != MDB_NOTFOUND; - if (dbr == MDB_NOTFOUND) - dbr = 0; - mdb_cursor_close(cursor); - break; - case BLACKBALL_CLEAR: - dbr = mdb_drop(txn, dbi_blackballs, 0); - break; - default: - THROW_WALLET_EXCEPTION(tools::error::wallet_internal_error, "Invalid blackball op"); + dbr = mdb_drop(txn, dbi_blackballs, 0); + THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to clear blackballs table: " + std::string(mdb_strerror(dbr))); } - THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to query blackballs table: " + std::string(mdb_strerror(dbr))); dbr = mdb_txn_commit(txn); THROW_WALLET_EXCEPTION_IF(dbr, tools::error::wallet_internal_error, "Failed to commit txn blackballing output to database: " + std::string(mdb_strerror(dbr))); @@ -432,24 +451,32 @@ bool ringdb::blackball_worker(const crypto::public_key &output, int op) return ret; } -bool ringdb::blackball(const crypto::public_key &output) +bool ringdb::blackball(const std::vector<std::pair<uint64_t, uint64_t>> &outputs) +{ + return blackball_worker(outputs, BLACKBALL_BLACKBALL); +} + +bool ringdb::blackball(const std::pair<uint64_t, uint64_t> &output) { - return blackball_worker(output, BLACKBALL_BLACKBALL); + std::vector<std::pair<uint64_t, uint64_t>> outputs(1, output); + return blackball_worker(outputs, BLACKBALL_BLACKBALL); } -bool ringdb::unblackball(const crypto::public_key &output) +bool ringdb::unblackball(const std::pair<uint64_t, uint64_t> &output) { - return blackball_worker(output, BLACKBALL_UNBLACKBALL); + std::vector<std::pair<uint64_t, uint64_t>> outputs(1, output); + return blackball_worker(outputs, BLACKBALL_UNBLACKBALL); } -bool ringdb::blackballed(const crypto::public_key &output) +bool ringdb::blackballed(const std::pair<uint64_t, uint64_t> &output) { - return blackball_worker(output, BLACKBALL_QUERY); + std::vector<std::pair<uint64_t, uint64_t>> outputs(1, output); + return blackball_worker(outputs, BLACKBALL_QUERY); } bool ringdb::clear_blackballs() { - return blackball_worker(crypto::public_key(), BLACKBALL_CLEAR); + return blackball_worker(std::vector<std::pair<uint64_t, uint64_t>>(), BLACKBALL_CLEAR); } } diff --git a/src/wallet/ringdb.h b/src/wallet/ringdb.h index 6b4bce124..7b448b0d7 100644 --- a/src/wallet/ringdb.h +++ b/src/wallet/ringdb.h @@ -49,13 +49,14 @@ namespace tools bool get_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, std::vector<uint64_t> &outs); bool set_ring(const crypto::chacha_key &chacha_key, const crypto::key_image &key_image, const std::vector<uint64_t> &outs, bool relative); - bool blackball(const crypto::public_key &output); - bool unblackball(const crypto::public_key &output); - bool blackballed(const crypto::public_key &output); + bool blackball(const std::pair<uint64_t, uint64_t> &output); + bool blackball(const std::vector<std::pair<uint64_t, uint64_t>> &outputs); + bool unblackball(const std::pair<uint64_t, uint64_t> &output); + bool blackballed(const std::pair<uint64_t, uint64_t> &output); bool clear_blackballs(); private: - bool blackball_worker(const crypto::public_key &output, int op); + bool blackball_worker(const std::vector<std::pair<uint64_t, uint64_t>> &outputs, int op); private: std::string filename; diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index a50a88b59..f670febbf 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -2067,9 +2067,11 @@ void wallet2::pull_and_parse_next_blocks(uint64_t start_height, uint64_t &blocks { drop_from_short_history(short_chain_history, 3); + THROW_WALLET_EXCEPTION_IF(prev_blocks.size() != prev_parsed_blocks.size(), error::wallet_internal_error, "size mismatch"); + // prepend the last 3 blocks, should be enough to guard against a block or two's reorg std::vector<parsed_block>::const_reverse_iterator i = prev_parsed_blocks.rbegin(); - for (size_t n = 0; n < std::min((size_t)3, prev_blocks.size()); ++n) + for (size_t n = 0; n < std::min((size_t)3, prev_parsed_blocks.size()); ++n) { short_chain_history.push_front(i->hash); ++i; @@ -2389,6 +2391,7 @@ void wallet2::fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, while (missing_blocks-- > 0) m_blockchain.push_back(crypto::null_hash); // maybe a bit suboptimal, but deque won't do huge reallocs like vector m_blockchain.push_back(m_checkpoints.get_points().at(checkpoint_height)); + m_blockchain.trim(checkpoint_height); short_chain_history.clear(); get_short_chain_history(short_chain_history); } @@ -2600,10 +2603,6 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo break; } - // switch to the new blocks from the daemon - blocks_start_height = next_blocks_start_height; - blocks = std::move(next_blocks); - parsed_blocks = std::move(next_parsed_blocks); first = false; // handle error from async fetching thread @@ -2611,6 +2610,11 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo { throw std::runtime_error("proxy exception in refresh thread"); } + + // switch to the new blocks from the daemon + blocks_start_height = next_blocks_start_height; + blocks = std::move(next_blocks); + parsed_blocks = std::move(next_parsed_blocks); } catch (const tools::error::password_needed&) { @@ -4993,69 +4997,6 @@ void wallet2::add_unconfirmed_tx(const cryptonote::transaction& tx, uint64_t amo } //---------------------------------------------------------------------------------------------------- -void wallet2::transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outs_count, const std::vector<size_t> &unused_transfers_indices, - uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx& ptx) -{ - transfer(dsts, fake_outs_count, unused_transfers_indices, unlock_time, fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), tx, ptx); -} -//---------------------------------------------------------------------------------------------------- -void wallet2::transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outs_count, const std::vector<size_t> &unused_transfers_indices, - uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra) -{ - cryptonote::transaction tx; - pending_tx ptx; - transfer(dsts, fake_outs_count, unused_transfers_indices, unlock_time, fee, extra, tx, ptx); -} - -namespace { -// split_amounts(vector<cryptonote::tx_destination_entry> dsts, size_t num_splits) -// -// split amount for each dst in dsts into num_splits parts -// and make num_splits new vector<crypt...> instances to hold these new amounts -std::vector<std::vector<cryptonote::tx_destination_entry>> split_amounts( - std::vector<cryptonote::tx_destination_entry> dsts, size_t num_splits) -{ - std::vector<std::vector<cryptonote::tx_destination_entry>> retVal; - - if (num_splits <= 1) - { - retVal.push_back(dsts); - return retVal; - } - - // for each split required - for (size_t i=0; i < num_splits; i++) - { - std::vector<cryptonote::tx_destination_entry> new_dsts; - - // for each destination - for (size_t j=0; j < dsts.size(); j++) - { - cryptonote::tx_destination_entry de; - uint64_t amount; - - amount = dsts[j].amount; - amount = amount / num_splits; - - // if last split, add remainder - if (i + 1 == num_splits) - { - amount += dsts[j].amount % num_splits; - } - - de.addr = dsts[j].addr; - de.amount = amount; - - new_dsts.push_back(de); - } - - retVal.push_back(new_dsts); - } - - return retVal; -} -} // anonymous namespace -//---------------------------------------------------------------------------------------------------- crypto::hash wallet2::get_payment_id(const pending_tx &ptx) const { std::vector<tx_extra_field> tx_extra_fields; @@ -6055,116 +5996,6 @@ uint32_t wallet2::adjust_priority(uint32_t priority) return priority; } //---------------------------------------------------------------------------------------------------- -// separated the call(s) to wallet2::transfer into their own function -// -// this function will make multiple calls to wallet2::transfer if multiple -// transactions will be required -std::vector<wallet2::pending_tx> wallet2::create_transactions(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra) -{ - const std::vector<size_t> unused_transfers_indices = select_available_outputs_from_histogram(fake_outs_count + 1, true, true, true); - - const uint64_t base_fee = get_base_fee(); - const uint64_t fee_multiplier = get_fee_multiplier(priority, get_fee_algorithm()); - const bool use_per_byte_fee = use_fork_rules(HF_VERSION_PER_BYTE_FEE); - const uint64_t fee_quantization_mask = get_fee_quantization_mask(); - - // failsafe split attempt counter - size_t attempt_count = 0; - - for(attempt_count = 1; ;attempt_count++) - { - size_t num_tx = 0.5 + pow(1.7,attempt_count-1); - - auto split_values = split_amounts(dsts, num_tx); - - // Throw if split_amounts comes back with a vector of size different than it should - if (split_values.size() != num_tx) - { - throw std::runtime_error("Splitting transactions returned a number of potential tx not equal to what was requested"); - } - - std::vector<pending_tx> ptx_vector; - try - { - // for each new destination vector (i.e. for each new tx) - for (auto & dst_vector : split_values) - { - cryptonote::transaction tx; - pending_tx ptx; - - // loop until fee is met without increasing tx size to next KB boundary. - uint64_t needed_fee = estimate_fee(use_per_byte_fee, false, unused_transfers_indices.size(), fake_outs_count, dst_vector.size()+1, extra.size(), false, base_fee, fee_multiplier, fee_quantization_mask); - do - { - transfer(dst_vector, fake_outs_count, unused_transfers_indices, unlock_time, needed_fee, extra, tx, ptx); - auto txBlob = t_serializable_object_to_blob(ptx.tx); - needed_fee = calculate_fee(use_per_byte_fee, ptx.tx, txBlob.size(), base_fee, fee_multiplier, fee_quantization_mask); - } while (ptx.fee < needed_fee); - - ptx_vector.push_back(ptx); - - // mark transfers to be used as "spent" - for(size_t idx: ptx.selected_transfers) - { - set_spent(idx, 0); - } - } - - // if we made it this far, we've selected our transactions. committing them will mark them spent, - // so this is a failsafe in case they don't go through - // unmark pending tx transfers as spent - for (auto & ptx : ptx_vector) - { - // mark transfers to be used as not spent - for(size_t idx2: ptx.selected_transfers) - { - set_unspent(idx2); - } - - } - - // if we made it this far, we're OK to actually send the transactions - return ptx_vector; - - } - // only catch this here, other exceptions need to pass through to the calling function - catch (const tools::error::tx_too_big& e) - { - - // unmark pending tx transfers as spent - for (auto & ptx : ptx_vector) - { - // mark transfers to be used as not spent - for(size_t idx2: ptx.selected_transfers) - { - set_unspent(idx2); - } - } - - if (attempt_count >= MAX_SPLIT_ATTEMPTS) - { - throw; - } - } - catch (...) - { - // in case of some other exception, make sure any tx in queue are marked unspent again - - // unmark pending tx transfers as spent - for (auto & ptx : ptx_vector) - { - // mark transfers to be used as not spent - for(size_t idx2: ptx.selected_transfers) - { - set_unspent(idx2); - } - } - - throw; - } - } -} - bool wallet2::set_ring_database(const std::string &filename) { m_ring_database = filename; @@ -6337,7 +6168,7 @@ bool wallet2::find_and_save_rings(bool force) return true; } -bool wallet2::blackball_output(const crypto::public_key &output) +bool wallet2::blackball_output(const std::pair<uint64_t, uint64_t> &output) { if (!m_ringdb) return false; @@ -6345,7 +6176,7 @@ bool wallet2::blackball_output(const crypto::public_key &output) catch (const std::exception &e) { return false; } } -bool wallet2::set_blackballed_outputs(const std::vector<crypto::public_key> &outputs, bool add) +bool wallet2::set_blackballed_outputs(const std::vector<std::pair<uint64_t, uint64_t>> &outputs, bool add) { if (!m_ringdb) return false; @@ -6354,14 +6185,13 @@ bool wallet2::set_blackballed_outputs(const std::vector<crypto::public_key> &out bool ret = true; if (!add) ret &= m_ringdb->clear_blackballs(); - for (const auto &output: outputs) - ret &= m_ringdb->blackball(output); + ret &= m_ringdb->blackball(outputs); return ret; } catch (const std::exception &e) { return false; } } -bool wallet2::unblackball_output(const crypto::public_key &output) +bool wallet2::unblackball_output(const std::pair<uint64_t, uint64_t> &output) { if (!m_ringdb) return false; @@ -6369,7 +6199,7 @@ bool wallet2::unblackball_output(const crypto::public_key &output) catch (const std::exception &e) { return false; } } -bool wallet2::is_output_blackballed(const crypto::public_key &output) const +bool wallet2::is_output_blackballed(const std::pair<uint64_t, uint64_t> &output) const { if (!m_ringdb) return false; @@ -6414,8 +6244,8 @@ bool wallet2::tx_add_fake_output(std::vector<std::vector<tools::wallet2::get_out CHECK_AND_ASSERT_MES(!outs.empty(), false, "internal error: outs is empty"); if (std::find(outs.back().begin(), outs.back().end(), item) != outs.back().end()) // don't add duplicates return false; - if (is_output_blackballed(output_public_key)) // don't add blackballed outputs - return false; +// if (is_output_blackballed(output_public_key)) // don't add blackballed outputs +// return false; outs.back().push_back(item); return true; } @@ -6912,6 +6742,8 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> if (seen_indices.count(i)) continue; + if (is_output_blackballed(std::make_pair(amount, i))) // don't add blackballed outputs + continue; seen_indices.emplace(i); LOG_PRINT_L2("picking " << i << " as " << type); @@ -6934,7 +6766,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "get_outs.bin"); THROW_WALLET_EXCEPTION_IF(daemon_resp.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_outs.bin"); - THROW_WALLET_EXCEPTION_IF(daemon_resp.status != CORE_RPC_STATUS_OK, error::get_random_outs_error, daemon_resp.status); + THROW_WALLET_EXCEPTION_IF(daemon_resp.status != CORE_RPC_STATUS_OK, error::get_outs_error, daemon_resp.status); THROW_WALLET_EXCEPTION_IF(daemon_resp.outs.size() != req.outputs.size(), error::wallet_internal_error, "daemon returned wrong response for get_outs.bin, wrong amounts count = " + std::to_string(daemon_resp.outs.size()) + ", expected " + std::to_string(req.outputs.size())); diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index f9b516bff..0347092c0 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -713,12 +713,6 @@ namespace tools uint64_t balance_all() const; uint64_t unlocked_balance_all() const; template<typename T> - void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy); - template<typename T> - void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, cryptonote::transaction& tx, pending_tx& ptx); - void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra); - void transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx& ptx); - template<typename T> void transfer_selected(const std::vector<cryptonote::tx_destination_entry>& dsts, const std::vector<size_t>& selected_transfers, size_t fake_outputs_count, std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, cryptonote::transaction& tx, pending_tx &ptx); @@ -746,7 +740,6 @@ namespace tools bool parse_unsigned_tx_from_str(const std::string &unsigned_tx_st, unsigned_tx_set &exported_txs) const; bool load_tx(const std::string &signed_filename, std::vector<tools::wallet2::pending_tx> &ptx, std::function<bool(const signed_tx_set&)> accept_func = NULL); bool parse_tx_from_str(const std::string &signed_tx_st, std::vector<tools::wallet2::pending_tx> &ptx, std::function<bool(const signed_tx_set &)> accept_func); - std::vector<pending_tx> create_transactions(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra); std::vector<wallet2::pending_tx> create_transactions_2(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices); // pass subaddr_indices by value on purpose std::vector<wallet2::pending_tx> create_transactions_all(uint64_t below, const cryptonote::account_public_address &address, bool is_subaddress, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices); std::vector<wallet2::pending_tx> create_transactions_single(const crypto::key_image &ki, const cryptonote::account_public_address &address, bool is_subaddress, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra); @@ -1159,10 +1152,10 @@ namespace tools bool set_ring(const crypto::key_image &key_image, const std::vector<uint64_t> &outs, bool relative); bool find_and_save_rings(bool force = true); - bool blackball_output(const crypto::public_key &output); - bool set_blackballed_outputs(const std::vector<crypto::public_key> &outputs, bool add = false); - bool unblackball_output(const crypto::public_key &output); - bool is_output_blackballed(const crypto::public_key &output) const; + bool blackball_output(const std::pair<uint64_t, uint64_t> &output); + bool set_blackballed_outputs(const std::vector<std::pair<uint64_t, uint64_t>> &outputs, bool add = false); + bool unblackball_output(const std::pair<uint64_t, uint64_t> &output); + bool is_output_blackballed(const std::pair<uint64_t, uint64_t> &output) const; bool lock_keys_file(); bool unlock_keys_file(); @@ -1817,198 +1810,4 @@ namespace tools //---------------------------------------------------------------------------------------------------- } //---------------------------------------------------------------------------------------------------- - template<typename T> - void wallet2::transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outs_count, const std::vector<size_t> &unused_transfers_indices, - uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy) - { - pending_tx ptx; - cryptonote::transaction tx; - transfer(dsts, fake_outs_count, unused_transfers_indices, unlock_time, fee, extra, destination_split_strategy, dust_policy, tx, ptx); - } - - template<typename T> - void wallet2::transfer(const std::vector<cryptonote::tx_destination_entry>& dsts, const size_t fake_outputs_count, const std::vector<size_t> &unused_transfers_indices, - uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, cryptonote::transaction& tx, pending_tx &ptx) - { - using namespace cryptonote; - // throw if attempting a transaction with no destinations - THROW_WALLET_EXCEPTION_IF(dsts.empty(), error::zero_destination); - - THROW_WALLET_EXCEPTION_IF(m_multisig, error::wallet_internal_error, "Multisig wallets cannot spend non rct outputs"); - - uint64_t upper_transaction_weight_limit = get_upper_transaction_weight_limit(); - uint64_t needed_money = fee; - - // calculate total amount being sent to all destinations - // throw if total amount overflows uint64_t - for(auto& dt: dsts) - { - THROW_WALLET_EXCEPTION_IF(0 == dt.amount, error::zero_destination); - needed_money += dt.amount; - THROW_WALLET_EXCEPTION_IF(needed_money < dt.amount, error::tx_sum_overflow, dsts, fee, m_nettype); - } - - // randomly select inputs for transaction - // throw if requested send amount is greater than (unlocked) amount available to send - std::vector<size_t> selected_transfers; - uint64_t found_money = select_transfers(needed_money, unused_transfers_indices, selected_transfers); - THROW_WALLET_EXCEPTION_IF(found_money < needed_money, error::not_enough_unlocked_money, found_money, needed_money - fee, fee); - - uint32_t subaddr_account = m_transfers[*selected_transfers.begin()].m_subaddr_index.major; - for (auto i = ++selected_transfers.begin(); i != selected_transfers.end(); ++i) - THROW_WALLET_EXCEPTION_IF(subaddr_account != *i, error::wallet_internal_error, "the tx uses funds from multiple accounts"); - - typedef COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry out_entry; - typedef cryptonote::tx_source_entry::output_entry tx_output_entry; - - COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response daemon_resp = AUTO_VAL_INIT(daemon_resp); - if(fake_outputs_count) - { - COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request req = AUTO_VAL_INIT(req); - req.outs_count = fake_outputs_count + 1;// add one to make possible (if need) to skip real output key - for(size_t idx: selected_transfers) - { - const transfer_container::const_iterator it = m_transfers.begin() + idx; - THROW_WALLET_EXCEPTION_IF(it->m_tx.vout.size() <= it->m_internal_output_index, error::wallet_internal_error, - "m_internal_output_index = " + std::to_string(it->m_internal_output_index) + - " is greater or equal to outputs count = " + std::to_string(it->m_tx.vout.size())); - req.amounts.push_back(it->amount()); - } - - m_daemon_rpc_mutex.lock(); - bool r = epee::net_utils::invoke_http_bin("/getrandom_outs.bin", req, daemon_resp, m_http_client, rpc_timeout); - m_daemon_rpc_mutex.unlock(); - THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "getrandom_outs.bin"); - THROW_WALLET_EXCEPTION_IF(daemon_resp.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "getrandom_outs.bin"); - THROW_WALLET_EXCEPTION_IF(daemon_resp.status != CORE_RPC_STATUS_OK, error::get_random_outs_error, daemon_resp.status); - THROW_WALLET_EXCEPTION_IF(daemon_resp.outs.size() != selected_transfers.size(), error::wallet_internal_error, - "daemon returned wrong response for getrandom_outs.bin, wrong amounts count = " + - std::to_string(daemon_resp.outs.size()) + ", expected " + std::to_string(selected_transfers.size())); - - std::unordered_map<uint64_t, uint64_t> scanty_outs; - for(COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& amount_outs: daemon_resp.outs) - { - if (amount_outs.outs.size() < fake_outputs_count) - { - scanty_outs[amount_outs.amount] = amount_outs.outs.size(); - } - } - THROW_WALLET_EXCEPTION_IF(!scanty_outs.empty(), error::not_enough_outs_to_mix, scanty_outs, fake_outputs_count); - } - - //prepare inputs - size_t i = 0; - std::vector<cryptonote::tx_source_entry> sources; - for(size_t idx: selected_transfers) - { - sources.resize(sources.size()+1); - cryptonote::tx_source_entry& src = sources.back(); - const transfer_details& td = m_transfers[idx]; - src.amount = td.amount(); - src.rct = false; - //paste mixin transaction - if(daemon_resp.outs.size()) - { - daemon_resp.outs[i].outs.sort([](const out_entry& a, const out_entry& b){return a.global_amount_index < b.global_amount_index;}); - for(out_entry& daemon_oe: daemon_resp.outs[i].outs) - { - if(td.m_global_output_index == daemon_oe.global_amount_index) - continue; - tx_output_entry oe; - oe.first = daemon_oe.global_amount_index; - oe.second.dest = rct::pk2rct(daemon_oe.out_key); - oe.second.mask = rct::identity(); - src.outputs.push_back(oe); - if(src.outputs.size() >= fake_outputs_count) - break; - } - } - - //paste real transaction to the random index - auto it_to_insert = std::find_if(src.outputs.begin(), src.outputs.end(), [&](const tx_output_entry& a) - { - return a.first >= td.m_global_output_index; - }); - //size_t real_index = src.outputs.size() ? (rand() % src.outputs.size() ):0; - tx_output_entry real_oe; - real_oe.first = td.m_global_output_index; - real_oe.second.dest = rct::pk2rct(boost::get<txout_to_key>(td.m_tx.vout[td.m_internal_output_index].target).key); - real_oe.second.mask = rct::identity(); - auto interted_it = src.outputs.insert(it_to_insert, real_oe); - src.real_out_tx_key = get_tx_pub_key_from_extra(td.m_tx); - src.real_output = interted_it - src.outputs.begin(); - src.real_output_in_tx_index = td.m_internal_output_index; - src.multisig_kLRki = rct::multisig_kLRki({rct::zero(), rct::zero(), rct::zero(), rct::zero()}); - detail::print_source_entry(src); - ++i; - } - - cryptonote::tx_destination_entry change_dts = AUTO_VAL_INIT(change_dts); - if (needed_money < found_money) - { - change_dts.addr = get_subaddress({subaddr_account, 0}); - change_dts.amount = found_money - needed_money; - } - - std::vector<cryptonote::tx_destination_entry> splitted_dsts, dust_dsts; - uint64_t dust = 0; - destination_split_strategy(dsts, change_dts, dust_policy.dust_threshold, splitted_dsts, dust_dsts); - for(auto& d: dust_dsts) { - THROW_WALLET_EXCEPTION_IF(dust_policy.dust_threshold < d.amount, error::wallet_internal_error, "invalid dust value: dust = " + - std::to_string(d.amount) + ", dust_threshold = " + std::to_string(dust_policy.dust_threshold)); - } - for(auto& d: dust_dsts) { - if (!dust_policy.add_to_fee) - splitted_dsts.push_back(cryptonote::tx_destination_entry(d.amount, dust_policy.addr_for_dust, d.is_subaddress)); - dust += d.amount; - } - - crypto::secret_key tx_key; - std::vector<crypto::secret_key> additional_tx_keys; - rct::multisig_out msout; - bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sources, splitted_dsts, change_dts.addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, rct::RangeProofBorromean, m_multisig ? &msout : NULL); - THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sources, splitted_dsts, unlock_time, m_nettype); - THROW_WALLET_EXCEPTION_IF(upper_transaction_weight_limit <= get_transaction_weight(tx), error::tx_too_big, tx, upper_transaction_weight_limit); - - std::string key_images; - bool all_are_txin_to_key = std::all_of(tx.vin.begin(), tx.vin.end(), [&](const txin_v& s_e) -> bool - { - CHECKED_GET_SPECIFIC_VARIANT(s_e, const txin_to_key, in, false); - key_images += boost::to_string(in.k_image) + " "; - return true; - }); - THROW_WALLET_EXCEPTION_IF(!all_are_txin_to_key, error::unexpected_txin_type, tx); - - bool dust_sent_elsewhere = (dust_policy.addr_for_dust.m_view_public_key != change_dts.addr.m_view_public_key - || dust_policy.addr_for_dust.m_spend_public_key != change_dts.addr.m_spend_public_key); - - if (dust_policy.add_to_fee || dust_sent_elsewhere) change_dts.amount -= dust; - - ptx.key_images = key_images; - ptx.fee = (dust_policy.add_to_fee ? fee+dust : fee); - ptx.dust = ((dust_policy.add_to_fee || dust_sent_elsewhere) ? dust : 0); - ptx.dust_added_to_fee = dust_policy.add_to_fee; - ptx.tx = tx; - ptx.change_dts = change_dts; - ptx.selected_transfers = selected_transfers; - ptx.tx_key = tx_key; - ptx.additional_tx_keys = additional_tx_keys; - ptx.dests = dsts; - ptx.construction_data.sources = sources; - ptx.construction_data.change_dts = change_dts; - ptx.construction_data.splitted_dsts = splitted_dsts; - ptx.construction_data.selected_transfers = selected_transfers; - ptx.construction_data.extra = tx.extra; - ptx.construction_data.unlock_time = unlock_time; - ptx.construction_data.use_rct = false; - ptx.construction_data.use_bulletproofs = false; - ptx.construction_data.dests = dsts; - // record which subaddress indices are being used as inputs - ptx.construction_data.subaddr_account = subaddr_account; - ptx.construction_data.subaddr_indices.clear(); - for (size_t idx: selected_transfers) - ptx.construction_data.subaddr_indices.insert(m_transfers[idx].m_subaddr_index.minor); - } - - } diff --git a/src/wallet/wallet_errors.h b/src/wallet/wallet_errors.h index a30e807b1..bc518d04a 100644 --- a/src/wallet/wallet_errors.h +++ b/src/wallet/wallet_errors.h @@ -73,7 +73,7 @@ namespace tools // get_tx_pool_error // out_of_hashchain_bounds_error // transfer_error * - // get_random_outs_general_error + // get_outs_general_error // not_enough_unlocked_money // not_enough_money // tx_not_possible @@ -128,7 +128,7 @@ namespace tools get_blocks_error_message_index, get_hashes_error_message_index, get_out_indices_error_message_index, - get_random_outs_error_message_index + get_outs_error_message_index }; template<typename Base, int msg_index> @@ -427,7 +427,7 @@ namespace tools } }; //---------------------------------------------------------------------------------------------------- - typedef failed_rpc_request<transfer_error, get_random_outs_error_message_index> get_random_outs_error; + typedef failed_rpc_request<transfer_error, get_outs_error_message_index> get_outs_error; //---------------------------------------------------------------------------------------------------- struct not_enough_unlocked_money : public transfer_error { diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index a6fe4e244..0e0b2e4eb 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -155,16 +155,7 @@ namespace tools return false; m_vm = vm; - tools::wallet2 *walvars; - std::unique_ptr<tools::wallet2> tmpwal; - if (m_wallet) - walvars = m_wallet; - else - { - tmpwal = tools::wallet2::make_dummy(*m_vm, true, password_prompter); - walvars = tmpwal.get(); - } boost::optional<epee::net_utils::http::login> http_login{}; std::string bind_port = command_line::get_arg(*m_vm, arg_rpc_bind_port); const bool disable_auth = command_line::get_arg(*m_vm, arg_disable_rpc_login); |