From 769ae42a7b0e7e73a13ef578ec9a0ecb06df22c1 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Mon, 10 Sep 2018 13:08:34 +0000 Subject: wallet2: faster output and key image import/export --- src/wallet/wallet2.cpp | 94 ++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 76 insertions(+), 18 deletions(-) (limited to 'src/wallet/wallet2.cpp') diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index c27e4e820..31246d0b1 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -69,6 +69,7 @@ using namespace epee; #include "common/base58.h" #include "common/dns_utils.h" #include "common/notify.h" +#include "common/perf_timer.h" #include "ringct/rctSigs.h" #include "ringdb.h" #include "device/device_cold.hpp" @@ -10517,11 +10518,13 @@ crypto::public_key wallet2::get_tx_pub_key_from_received_outs(const tools::walle bool wallet2::export_key_images(const std::string &filename) const { + PERF_TIMER(export_key_images); std::vector> ski = export_key_images(); std::string magic(KEY_IMAGE_EXPORT_FILE_MAGIC, strlen(KEY_IMAGE_EXPORT_FILE_MAGIC)); const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address; std::string data; + data.reserve(ski.size() * (sizeof(crypto::key_image) + sizeof(crypto::signature)) + 2 * sizeof(crypto::public_key)); data += std::string((const char *)&keys.m_spend_public_key, sizeof(crypto::public_key)); data += std::string((const char *)&keys.m_view_public_key, sizeof(crypto::public_key)); for (const auto &i: ski) @@ -10531,6 +10534,7 @@ bool wallet2::export_key_images(const std::string &filename) const } // encrypt data, keep magic plaintext + PERF_TIMER(export_key_images_encrypt); std::string ciphertext = encrypt_with_view_secret_key(data); return epee::file_io_utils::save_string_to_file(filename, magic + ciphertext); } @@ -10538,6 +10542,7 @@ bool wallet2::export_key_images(const std::string &filename) const //---------------------------------------------------------------------------------------------------- std::vector> wallet2::export_key_images() const { + PERF_TIMER(export_key_images_raw); std::vector> ski; ski.reserve(m_transfers.size()); @@ -10590,6 +10595,7 @@ std::vector> wallet2::export_key uint64_t wallet2::import_key_images(const std::string &filename, uint64_t &spent, uint64_t &unspent) { + PERF_TIMER(import_key_images_fsu); std::string data; bool r = epee::file_io_utils::load_file_to_string(filename, data); @@ -10603,6 +10609,7 @@ uint64_t wallet2::import_key_images(const std::string &filename, uint64_t &spent try { + PERF_TIMER(import_key_images_decrypt); data = decrypt_with_view_secret_key(std::string(data, magiclen)); } catch (const std::exception &e) @@ -10641,6 +10648,7 @@ uint64_t wallet2::import_key_images(const std::string &filename, uint64_t &spent //---------------------------------------------------------------------------------------------------- uint64_t wallet2::import_key_images(const std::vector> &signed_key_images, uint64_t &spent, uint64_t &unspent, bool check_spent) { + PERF_TIMER(import_key_images_lots); COMMAND_RPC_IS_KEY_IMAGE_SPENT::request req = AUTO_VAL_INIT(req); COMMAND_RPC_IS_KEY_IMAGE_SPENT::response daemon_resp = AUTO_VAL_INIT(daemon_resp); @@ -10654,6 +10662,9 @@ uint64_t wallet2::import_key_images(const std::vector(out.target); const crypto::public_key pkey = o.key; - std::vector pkeys; - pkeys.push_back(&pkey); - THROW_WALLET_EXCEPTION_IF(!(rct::scalarmultKey(rct::ki2rct(key_image), rct::curveOrder()) == rct::identity()), - error::wallet_internal_error, "Key image out of validity domain: input " + boost::lexical_cast(n) + "/" - + boost::lexical_cast(signed_key_images.size()) + ", key image " + epee::string_tools::pod_to_hex(key_image)); - - THROW_WALLET_EXCEPTION_IF(!crypto::check_ring_signature((const crypto::hash&)key_image, key_image, pkeys, &signature), - error::signature_check_failed, boost::lexical_cast(n) + "/" - + boost::lexical_cast(signed_key_images.size()) + ", key image " + epee::string_tools::pod_to_hex(key_image) - + ", signature " + epee::string_tools::pod_to_hex(signature) + ", pubkey " + epee::string_tools::pod_to_hex(*pkeys[0])); + if (!td.m_key_image_known || !(key_image == td.m_key_image)) + { + std::vector pkeys; + pkeys.push_back(&pkey); + THROW_WALLET_EXCEPTION_IF(!(rct::scalarmultKey(rct::ki2rct(key_image), rct::curveOrder()) == rct::identity()), + error::wallet_internal_error, "Key image out of validity domain: input " + boost::lexical_cast(n) + "/" + + boost::lexical_cast(signed_key_images.size()) + ", key image " + epee::string_tools::pod_to_hex(key_image)); + THROW_WALLET_EXCEPTION_IF(!crypto::check_ring_signature((const crypto::hash&)key_image, key_image, pkeys, &signature), + error::signature_check_failed, boost::lexical_cast(n) + "/" + + boost::lexical_cast(signed_key_images.size()) + ", key image " + epee::string_tools::pod_to_hex(key_image) + + ", signature " + epee::string_tools::pod_to_hex(signature) + ", pubkey " + epee::string_tools::pod_to_hex(*pkeys[0])); + } req.key_images.push_back(epee::string_tools::pod_to_hex(key_image)); } + PERF_TIMER_STOP(import_key_images_A); + PERF_TIMER_START(import_key_images_B); for (size_t n = 0; n < signed_key_images.size(); ++n) { m_transfers[n].m_key_image = signed_key_images[n].first; @@ -10688,9 +10703,11 @@ uint64_t wallet2::import_key_images(const std::vector spent_key_images; + PERF_TIMER_START(import_key_images_C); for (const transfer_details &td: m_transfers) { for (const cryptonote::txin_v& in : td.m_tx.vin) @@ -10721,10 +10739,12 @@ uint64_t wallet2::import_key_images(const std::vector(in).k_image, td.m_txid)); } } + PERF_TIMER_STOP(import_key_images_C); + PERF_TIMER_START(import_key_images_D); for(size_t i = 0; i < signed_key_images.size(); ++i) { - transfer_details &td = m_transfers[i]; + const transfer_details &td = m_transfers[i]; uint64_t amount = td.amount(); if (td.m_spent) spent += amount; @@ -10742,6 +10762,8 @@ uint64_t wallet2::import_key_images(const std::vectorsecond); } } + PERF_TIMER_STOP(import_key_images_D); + MDEBUG("Total: " << print_money(spent) << " spent, " << print_money(unspent) << " unspent"); if (check_spent) @@ -10751,8 +10773,12 @@ uint64_t wallet2::import_key_images(const std::vector wallet2::export_outputs() const { + PERF_TIMER(export_outputs); std::vector outs; outs.reserve(m_transfers.size()); @@ -10970,29 +11002,53 @@ std::vector wallet2::export_outputs() const //---------------------------------------------------------------------------------------------------- std::string wallet2::export_outputs_to_str() const { - std::vector outs = export_outputs(); + PERF_TIMER(export_outputs_to_str); std::stringstream oss; boost::archive::portable_binary_oarchive ar(oss); - ar << outs; + ar << m_transfers; std::string magic(OUTPUT_EXPORT_FILE_MAGIC, strlen(OUTPUT_EXPORT_FILE_MAGIC)); const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address; std::string header; header += std::string((const char *)&keys.m_spend_public_key, sizeof(crypto::public_key)); header += std::string((const char *)&keys.m_view_public_key, sizeof(crypto::public_key)); + PERF_TIMER(export_outputs_encryption); std::string ciphertext = encrypt_with_view_secret_key(header + oss.str()); return magic + ciphertext; } //---------------------------------------------------------------------------------------------------- size_t wallet2::import_outputs(const std::vector &outputs) { - m_transfers.clear(); - m_transfers.reserve(outputs.size()); + PERF_TIMER(import_outputs); + const size_t original_size = m_transfers.size(); + m_transfers.resize(outputs.size()); for (size_t i = 0; i < outputs.size(); ++i) { transfer_details td = outputs[i]; + // skip those we've already imported, or which have different data + if (i < original_size) + { + // compare the data used to create the key image below + const transfer_details &org_td = m_transfers[i]; + if (!org_td.m_key_image_known) + goto process; +#define CMPF(f) if (!(td.f == org_td.f)) goto process + CMPF(m_txid); + CMPF(m_key_image); + CMPF(m_internal_output_index); +#undef CMPF + if (!(get_transaction_prefix_hash(td.m_tx) == get_transaction_prefix_hash(org_td.m_tx))) + goto process; + + // copy anyway, since the comparison does not include ancillary fields which may have changed + m_transfers[i] = std::move(td); + continue; + } + +process: + // the hot wallet wouldn't have known about key images (except if we already exported them) cryptonote::keypair in_ephemeral; @@ -11011,9 +11067,9 @@ size_t wallet2::import_outputs(const std::vector(i)); - m_key_images[td.m_key_image] = m_transfers.size(); - m_pub_keys[td.get_public_key()] = m_transfers.size(); - m_transfers.push_back(std::move(td)); + m_key_images[td.m_key_image] = i; + m_pub_keys[td.get_public_key()] = i; + m_transfers[i] = std::move(td); } return m_transfers.size(); @@ -11021,6 +11077,7 @@ size_t wallet2::import_outputs(const std::vector Date: Wed, 24 Oct 2018 18:24:11 +0000 Subject: wallet2: only export necessary outputs and key images and disable annoying test that requires ridiculous amounts of skullduggery every time some format changes --- src/wallet/wallet2.cpp | 122 ++++++++++++++++++++++++++++++------------------- 1 file changed, 76 insertions(+), 46 deletions(-) (limited to 'src/wallet/wallet2.cpp') diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 31246d0b1..a90a93321 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -113,11 +113,11 @@ using namespace cryptonote; #define SUBADDRESS_LOOKAHEAD_MAJOR 50 #define SUBADDRESS_LOOKAHEAD_MINOR 200 -#define KEY_IMAGE_EXPORT_FILE_MAGIC "Monero key image export\002" +#define KEY_IMAGE_EXPORT_FILE_MAGIC "Monero key image export\003" #define MULTISIG_EXPORT_FILE_MAGIC "Monero multisig export\001" -#define OUTPUT_EXPORT_FILE_MAGIC "Monero output export\003" +#define OUTPUT_EXPORT_FILE_MAGIC "Monero output export\004" #define SEGREGATION_FORK_HEIGHT 99999999 #define TESTNET_SEGREGATION_FORK_HEIGHT 99999999 @@ -1606,6 +1606,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote td.m_txid = txid; td.m_key_image = tx_scan_info[o].ki; td.m_key_image_known = !m_watch_only && !m_multisig; + td.m_key_image_requested = false; td.m_key_image_partial = m_multisig; td.m_amount = amount; td.m_pk_index = pk_index - 1; @@ -5466,7 +5467,7 @@ std::string wallet2::dump_tx_to_str(const std::vector &ptx_vector) c txs.txes.push_back(get_construction_data_with_decrypted_short_payment_id(tx, m_account.get_device())); } - txs.transfers = m_transfers; + txs.transfers = export_outputs(); // save as binary std::ostringstream oss; boost::archive::portable_binary_oarchive ar(oss); @@ -7952,6 +7953,7 @@ void wallet2::light_wallet_get_unspent_outs() td.m_key_image = unspent_key_image; td.m_key_image_known = !m_watch_only && !m_multisig; + td.m_key_image_requested = false; td.m_key_image_partial = m_multisig; td.m_amount = o.amount; td.m_pk_index = 0; @@ -9125,7 +9127,7 @@ void wallet2::cold_sign_tx(const std::vector& ptx_vector, signed_tx_ { txs.txes.push_back(get_construction_data_with_decrypted_short_payment_id(tx, m_account.get_device())); } - txs.transfers = m_transfers; + txs.transfers = std::make_pair(0, m_transfers); auto dev_cold = dynamic_cast<::hw::device_cold*>(&hwdev); CHECK_AND_ASSERT_THROW_MES(dev_cold, "Device does not implement cold signing interface"); @@ -9156,7 +9158,7 @@ uint64_t wallet2::cold_key_image_sync(uint64_t &spent, uint64_t &unspent) { dev_cold->ki_sync(&wallet_shim, m_transfers, ski); - return import_key_images(ski, spent, unspent); + return import_key_images(ski, 0, spent, unspent); } //---------------------------------------------------------------------------------------------------- void wallet2::get_hard_fork_info(uint8_t version, uint64_t &earliest_height) const @@ -10519,15 +10521,21 @@ crypto::public_key wallet2::get_tx_pub_key_from_received_outs(const tools::walle bool wallet2::export_key_images(const std::string &filename) const { PERF_TIMER(export_key_images); - std::vector> ski = export_key_images(); + std::pair>> ski = export_key_images(); std::string magic(KEY_IMAGE_EXPORT_FILE_MAGIC, strlen(KEY_IMAGE_EXPORT_FILE_MAGIC)); const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address; + const uint32_t offset = ski.first; std::string data; - data.reserve(ski.size() * (sizeof(crypto::key_image) + sizeof(crypto::signature)) + 2 * sizeof(crypto::public_key)); + data.reserve(4 + ski.second.size() * (sizeof(crypto::key_image) + sizeof(crypto::signature)) + 2 * sizeof(crypto::public_key)); + data.resize(4); + data[0] = offset & 0xff; + data[1] = (offset >> 8) & 0xff; + data[2] = (offset >> 16) & 0xff; + data[3] = (offset >> 24) & 0xff; data += std::string((const char *)&keys.m_spend_public_key, sizeof(crypto::public_key)); data += std::string((const char *)&keys.m_view_public_key, sizeof(crypto::public_key)); - for (const auto &i: ski) + for (const auto &i: ski.second) { data += std::string((const char *)&i.first, sizeof(crypto::key_image)); data += std::string((const char *)&i.second, sizeof(crypto::signature)); @@ -10540,13 +10548,17 @@ bool wallet2::export_key_images(const std::string &filename) const } //---------------------------------------------------------------------------------------------------- -std::vector> wallet2::export_key_images() const +std::pair>> wallet2::export_key_images() const { PERF_TIMER(export_key_images_raw); std::vector> ski; - ski.reserve(m_transfers.size()); - for (size_t n = 0; n < m_transfers.size(); ++n) + size_t offset = 0; + while (offset < m_transfers.size() && !m_transfers[offset].m_key_image_requested) + ++offset; + + ski.reserve(m_transfers.size() - offset); + for (size_t n = offset; n < m_transfers.size(); ++n) { const transfer_details &td = m_transfers[n]; @@ -10590,7 +10602,7 @@ std::vector> wallet2::export_key ski.push_back(std::make_pair(td.m_key_image, signature)); } - return ski; + return std::make_pair(offset, ski); } uint64_t wallet2::import_key_images(const std::string &filename, uint64_t &spent, uint64_t &unspent) @@ -10617,15 +10629,17 @@ uint64_t wallet2::import_key_images(const std::string &filename, uint64_t &spent THROW_WALLET_EXCEPTION(error::wallet_internal_error, std::string("Failed to decrypt ") + filename + ": " + e.what()); } - const size_t headerlen = 2 * sizeof(crypto::public_key); + const size_t headerlen = 4 + 2 * sizeof(crypto::public_key); THROW_WALLET_EXCEPTION_IF(data.size() < headerlen, error::wallet_internal_error, std::string("Bad data size from file ") + filename); - const crypto::public_key &public_spend_key = *(const crypto::public_key*)&data[0]; - const crypto::public_key &public_view_key = *(const crypto::public_key*)&data[sizeof(crypto::public_key)]; + const uint32_t offset = (uint8_t)data[0] | (((uint8_t)data[1]) << 8) | (((uint8_t)data[2]) << 16) | (((uint8_t)data[3]) << 24); + const crypto::public_key &public_spend_key = *(const crypto::public_key*)&data[4]; + const crypto::public_key &public_view_key = *(const crypto::public_key*)&data[4 + sizeof(crypto::public_key)]; const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address; if (public_spend_key != keys.m_spend_public_key || public_view_key != keys.m_view_public_key) { THROW_WALLET_EXCEPTION(error::wallet_internal_error, std::string( "Key images from ") + filename + " are for a different account"); } + THROW_WALLET_EXCEPTION_IF(offset > m_transfers.size(), error::wallet_internal_error, "Offset larger than known outputs"); const size_t record_size = sizeof(crypto::key_image) + sizeof(crypto::signature); THROW_WALLET_EXCEPTION_IF((data.size() - headerlen) % record_size, @@ -10642,20 +10656,21 @@ uint64_t wallet2::import_key_images(const std::string &filename, uint64_t &spent ski.push_back(std::make_pair(key_image, signature)); } - return import_key_images(ski, spent, unspent); + return import_key_images(ski, offset, spent, unspent); } //---------------------------------------------------------------------------------------------------- -uint64_t wallet2::import_key_images(const std::vector> &signed_key_images, uint64_t &spent, uint64_t &unspent, bool check_spent) +uint64_t wallet2::import_key_images(const std::vector> &signed_key_images, size_t offset, uint64_t &spent, uint64_t &unspent, bool check_spent) { PERF_TIMER(import_key_images_lots); COMMAND_RPC_IS_KEY_IMAGE_SPENT::request req = AUTO_VAL_INIT(req); COMMAND_RPC_IS_KEY_IMAGE_SPENT::response daemon_resp = AUTO_VAL_INIT(daemon_resp); - THROW_WALLET_EXCEPTION_IF(signed_key_images.size() > m_transfers.size(), error::wallet_internal_error, + THROW_WALLET_EXCEPTION_IF(offset > m_transfers.size(), error::wallet_internal_error, "Offset larger than known outputs"); + THROW_WALLET_EXCEPTION_IF(signed_key_images.size() > m_transfers.size() - offset, error::wallet_internal_error, "The blockchain is out of date compared to the signed key images"); - if (signed_key_images.empty()) + if (signed_key_images.empty() && offset == 0) { spent = 0; unspent = 0; @@ -10667,7 +10682,7 @@ uint64_t wallet2::import_key_images(const std::vector pkeys; pkeys.push_back(&pkey); THROW_WALLET_EXCEPTION_IF(!(rct::scalarmultKey(rct::ki2rct(key_image), rct::curveOrder()) == rct::identity()), - error::wallet_internal_error, "Key image out of validity domain: input " + boost::lexical_cast(n) + "/" + error::wallet_internal_error, "Key image out of validity domain: input " + boost::lexical_cast(n + offset) + "/" + boost::lexical_cast(signed_key_images.size()) + ", key image " + epee::string_tools::pod_to_hex(key_image)); THROW_WALLET_EXCEPTION_IF(!crypto::check_ring_signature((const crypto::hash&)key_image, key_image, pkeys, &signature), - error::signature_check_failed, boost::lexical_cast(n) + "/" + error::signature_check_failed, boost::lexical_cast(n + offset) + "/" + boost::lexical_cast(signed_key_images.size()) + ", key image " + epee::string_tools::pod_to_hex(key_image) + ", signature " + epee::string_tools::pod_to_hex(signature) + ", pubkey " + epee::string_tools::pod_to_hex(*pkeys[0])); } @@ -10698,10 +10713,11 @@ uint64_t wallet2::import_key_images(const std::vector key_images) td.m_key_image = key_images[i]; m_key_images[m_transfers[i].m_key_image] = i; td.m_key_image_known = true; + td.m_key_image_requested = false; td.m_key_image_partial = false; m_pub_keys[m_transfers[i].get_public_key()] = i; } @@ -10984,20 +11001,24 @@ void wallet2::import_blockchain(const std::tuple wallet2::export_outputs() const +std::pair> wallet2::export_outputs() const { PERF_TIMER(export_outputs); std::vector outs; - outs.reserve(m_transfers.size()); - for (size_t n = 0; n < m_transfers.size(); ++n) + size_t offset = 0; + while (offset < m_transfers.size() && m_transfers[offset].m_key_image_known) + ++offset; + + outs.reserve(m_transfers.size() - offset); + for (size_t n = offset; n < m_transfers.size(); ++n) { const transfer_details &td = m_transfers[n]; outs.push_back(td); } - return outs; + return std::make_pair(offset, outs); } //---------------------------------------------------------------------------------------------------- std::string wallet2::export_outputs_to_str() const @@ -11006,7 +11027,7 @@ std::string wallet2::export_outputs_to_str() const std::stringstream oss; boost::archive::portable_binary_oarchive ar(oss); - ar << m_transfers; + ar << export_outputs(); std::string magic(OUTPUT_EXPORT_FILE_MAGIC, strlen(OUTPUT_EXPORT_FILE_MAGIC)); const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address; @@ -11018,20 +11039,27 @@ std::string wallet2::export_outputs_to_str() const return magic + ciphertext; } //---------------------------------------------------------------------------------------------------- -size_t wallet2::import_outputs(const std::vector &outputs) +size_t wallet2::import_outputs(const std::pair> &outputs) { PERF_TIMER(import_outputs); + + THROW_WALLET_EXCEPTION_IF(outputs.first > m_transfers.size(), error::wallet_internal_error, + "Imported outputs omit more outputs that we know of"); + + const size_t offset = outputs.first; const size_t original_size = m_transfers.size(); - m_transfers.resize(outputs.size()); - for (size_t i = 0; i < outputs.size(); ++i) + m_transfers.resize(offset + outputs.second.size()); + for (size_t i = 0; i < offset; ++i) + m_transfers[i].m_key_image_requested = false; + for (size_t i = 0; i < outputs.second.size(); ++i) { - transfer_details td = outputs[i]; + transfer_details td = outputs.second[i]; // skip those we've already imported, or which have different data - if (i < original_size) + if (i + offset < original_size) { // compare the data used to create the key image below - const transfer_details &org_td = m_transfers[i]; + const transfer_details &org_td = m_transfers[i + offset]; if (!org_td.m_key_image_known) goto process; #define CMPF(f) if (!(td.f == org_td.f)) goto process @@ -11043,7 +11071,7 @@ size_t wallet2::import_outputs(const std::vector(i)); + THROW_WALLET_EXCEPTION_IF(td.m_tx.vout.empty(), error::wallet_internal_error, "tx with no outputs at index " + boost::lexical_cast(i + offset)); crypto::public_key tx_pub_key = get_tx_pub_key_from_received_outs(td); const std::vector additional_tx_pub_keys = get_additional_tx_pub_keys_from_extra(td.m_tx); @@ -11063,13 +11091,14 @@ process: THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "Failed to generate key image"); expand_subaddresses(td.m_subaddr_index); td.m_key_image_known = true; + td.m_key_image_requested = true; td.m_key_image_partial = false; THROW_WALLET_EXCEPTION_IF(in_ephemeral.pub != out_key, - error::wallet_internal_error, "key_image generated ephemeral public key not matched with output_key at index " + boost::lexical_cast(i)); + error::wallet_internal_error, "key_image generated ephemeral public key not matched with output_key at index " + boost::lexical_cast(i + offset)); - m_key_images[td.m_key_image] = i; - m_pub_keys[td.get_public_key()] = i; - m_transfers[i] = std::move(td); + m_key_images[td.m_key_image] = i + offset; + m_pub_keys[td.get_public_key()] = i + offset; + m_transfers[i + offset] = std::move(td); } return m_transfers.size(); @@ -11114,7 +11143,7 @@ size_t wallet2::import_outputs_from_str(const std::string &outputs_st) std::string body(data, headerlen); std::stringstream iss; iss << body; - std::vector outputs; + std::pair> outputs; try { boost::archive::portable_binary_iarchive ar(iss); @@ -11306,6 +11335,7 @@ void wallet2::update_multisig_rescan_info(const std::vector