From 55a2da7475db07d7ac453da0e5d0b3f6777ddd8d Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 22 Nov 2015 15:18:36 +0000 Subject: wallet2: speedup refresh a bit Use the NoodleDoodle threading technique to speedup a couple code blocks on the main path when refreshing blocks without any transactions for us. --- src/wallet/wallet2.cpp | 150 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 148 insertions(+), 2 deletions(-) (limited to 'src/wallet/wallet2.cpp') diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index f26809af8..005af0a9a 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -68,6 +68,13 @@ using namespace cryptonote; // arbitrary, used to generate different hashes from the same input #define CHACHA8_KEY_TAIL 0x8c +#define KILL_IOSERVICE() \ + do { \ + work.reset(); \ + threadpool.join_all(); \ + ioservice.stop(); \ + } while(0) + namespace { void do_prepare_file_names(const std::string& file_path, std::string& keys_file, std::string& wallet_file) @@ -148,6 +155,25 @@ bool wallet2::is_deprecated() const return is_old_file_format; } //---------------------------------------------------------------------------------------------------- +void wallet2::check_acc_out(const account_keys &acc, const tx_out &o, const crypto::public_key &tx_pub_key, size_t i, uint64_t &money_transfered, bool &error) const +{ + if (o.target.type() != typeid(txout_to_key)) + { + error = true; + LOG_ERROR("wrong type id in transaction out"); + return; + } + if(is_out_to_acc(acc, boost::get(o.target), tx_pub_key, i)) + { + money_transfered = o.amount; + } + else + { + money_transfered = 0; + } + error = false; +} +//---------------------------------------------------------------------------------------------------- void wallet2::process_new_transaction(const cryptonote::transaction& tx, uint64_t height) { process_unconfirmed(tx, height); @@ -175,7 +201,48 @@ void wallet2::process_new_transaction(const cryptonote::transaction& tx, uint64_ } tx_pub_key = pub_key_field.pub_key; - bool r = lookup_acc_outs(m_account.get_keys(), tx, tx_pub_key, outs, tx_money_got_in_outs); + bool r = true; + int threads = std::thread::hardware_concurrency(); + if (threads > 1) + { + boost::asio::io_service ioservice; + boost::thread_group threadpool; + std::auto_ptr < boost::asio::io_service::work > work(new boost::asio::io_service::work(ioservice)); + for (int i = 0; i < threads; i++) + { + threadpool.create_thread(boost::bind(&boost::asio::io_service::run, &ioservice)); + } + + const account_keys &keys = m_account.get_keys(); + uint64_t *money_transfered = new uint64_t[tx.vout.size()]; + bool *error = new bool[tx.vout.size()]; + for (size_t i = 0; i < tx.vout.size(); ++i) + { + ioservice.dispatch(boost::bind(&wallet2::check_acc_out, this, std::cref(keys), std::cref(tx.vout[i]), std::cref(tx_pub_key), i, + std::ref(money_transfered[i]), std::ref(error[i]))); + } + KILL_IOSERVICE(); + tx_money_got_in_outs = 0; + for (size_t i = 0; i < tx.vout.size(); ++i) + { + if (error[i]) + { + r = false; + break; + } + if (money_transfered[i]) + { + outs.push_back(i); + tx_money_got_in_outs += money_transfered[i]; + } + } + delete[] error; + delete[] money_transfered; + } + else + { + r = lookup_acc_outs(m_account.get_keys(), tx, tx_pub_key, outs, tx_money_got_in_outs); + } THROW_WALLET_EXCEPTION_IF(!r, error::acc_outs_lookup_error, tx, tx_pub_key, m_account.get_keys()); if(!outs.empty() && tx_money_got_in_outs) @@ -320,7 +387,7 @@ void wallet2::process_outgoing(const cryptonote::transaction &tx, uint64_t heigh ctd.m_block_height = height; } //---------------------------------------------------------------------------------------------------- -void wallet2::process_new_blockchain_entry(const cryptonote::block& b, cryptonote::block_complete_entry& bche, crypto::hash& bl_id, uint64_t height) +void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const crypto::hash& bl_id, uint64_t height) { //handle transactions from new block @@ -379,6 +446,13 @@ void wallet2::get_short_chain_history(std::list& ids) const ids.push_back(m_blockchain[0]); } //---------------------------------------------------------------------------------------------------- +void wallet2::parse_block_round(const cryptonote::blobdata &blob, cryptonote::block &bl, crypto::hash &bl_id, bool &error) const +{ + error = !cryptonote::parse_and_validate_block_from_blob(blob, bl); + if (!error) + bl_id = get_block_hash(bl); +} +//---------------------------------------------------------------------------------------------------- void wallet2::pull_blocks(uint64_t start_height, uint64_t& blocks_added) { blocks_added = 0; @@ -392,6 +466,77 @@ void wallet2::pull_blocks(uint64_t start_height, uint64_t& blocks_added) THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::get_blocks_error, res.status); size_t current_index = res.start_height; + + int threads = std::thread::hardware_concurrency(); + if (threads > 1) + { + crypto::hash *round_block_hashes = new crypto::hash[threads]; + cryptonote::block *round_blocks = new cryptonote::block[threads]; + bool *error = new bool[threads]; + const std::list &blocks = res.blocks; + size_t blocks_size = blocks.size(); + std::list::const_iterator blocki = blocks.begin(); + for (size_t b = 0; b < blocks_size; b += threads) + { + size_t round_size = std::min((size_t)threads, blocks_size - b); + + boost::asio::io_service ioservice; + boost::thread_group threadpool; + std::auto_ptr < boost::asio::io_service::work > work(new boost::asio::io_service::work(ioservice)); + for (size_t i = 0; i < round_size; i++) + { + threadpool.create_thread(boost::bind(&boost::asio::io_service::run, &ioservice)); + } + + std::list::const_iterator tmpblocki = blocki; + for (size_t i = 0; i < round_size; ++i) + { + ioservice.dispatch(boost::bind(&wallet2::parse_block_round, this, std::cref(tmpblocki->block), + std::ref(round_blocks[i]), std::ref(round_block_hashes[i]), std::ref(error[i]))); + ++tmpblocki; + } + KILL_IOSERVICE(); + tmpblocki = blocki; + for (size_t i = 0; i < round_size; ++i) + { + THROW_WALLET_EXCEPTION_IF(error[i], error::block_parse_error, tmpblocki->block); + ++tmpblocki; + } + for (size_t i = 0; i < round_size; ++i) + { + const crypto::hash &bl_id = round_block_hashes[i]; + cryptonote::block &bl = round_blocks[i]; + + if(current_index >= m_blockchain.size()) + { + process_new_blockchain_entry(bl, *blocki, bl_id, current_index); + ++blocks_added; + } + else if(bl_id != m_blockchain[current_index]) + { + //split detected here !!! + THROW_WALLET_EXCEPTION_IF(current_index == start_height, error::wallet_internal_error, + "wrong daemon response: split starts from the first block in response " + string_tools::pod_to_hex(bl_id) + + " (height " + std::to_string(start_height) + "), local block id at this height: " + + string_tools::pod_to_hex(m_blockchain[current_index])); + + detach_blockchain(current_index); + process_new_blockchain_entry(bl, *blocki, bl_id, current_index); + } + else + { + LOG_PRINT_L2("Block is already in blockchain: " << string_tools::pod_to_hex(bl_id)); + } + ++current_index; + ++blocki; + } + } + delete[] error; + delete[] round_blocks; + delete[] round_block_hashes; + } + else + { BOOST_FOREACH(auto& bl_entry, res.blocks) { cryptonote::block bl; @@ -422,6 +567,7 @@ void wallet2::pull_blocks(uint64_t start_height, uint64_t& blocks_added) ++current_index; } + } } //---------------------------------------------------------------------------------------------------- void wallet2::refresh() -- cgit v1.2.3 From ea707c779cd01c7c2a13d0c339b1fd7c11e95739 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 22 Nov 2015 15:20:26 +0000 Subject: wallet2: minor cleanup - use std::vector::std::deque to not leak when exceptions happen - use std::unique_ptr instead of the deprecated std::auto_ptr --- src/wallet/wallet2.cpp | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) (limited to 'src/wallet/wallet2.cpp') diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 005af0a9a..b51a23698 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -470,9 +470,9 @@ void wallet2::pull_blocks(uint64_t start_height, uint64_t& blocks_added) int threads = std::thread::hardware_concurrency(); if (threads > 1) { - crypto::hash *round_block_hashes = new crypto::hash[threads]; - cryptonote::block *round_blocks = new cryptonote::block[threads]; - bool *error = new bool[threads]; + std::vector round_block_hashes(threads); + std::vector round_blocks(threads); + std::deque error(threads); const std::list &blocks = res.blocks; size_t blocks_size = blocks.size(); std::list::const_iterator blocki = blocks.begin(); @@ -482,7 +482,7 @@ void wallet2::pull_blocks(uint64_t start_height, uint64_t& blocks_added) boost::asio::io_service ioservice; boost::thread_group threadpool; - std::auto_ptr < boost::asio::io_service::work > work(new boost::asio::io_service::work(ioservice)); + std::unique_ptr < boost::asio::io_service::work > work(new boost::asio::io_service::work(ioservice)); for (size_t i = 0; i < round_size; i++) { threadpool.create_thread(boost::bind(&boost::asio::io_service::run, &ioservice)); @@ -531,9 +531,6 @@ void wallet2::pull_blocks(uint64_t start_height, uint64_t& blocks_added) ++blocki; } } - delete[] error; - delete[] round_blocks; - delete[] round_block_hashes; } else { -- cgit v1.2.3 From d2c031332ec6197c1f3bd4b0147e6cd2c7e35ee5 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 22 Nov 2015 17:52:31 +0000 Subject: wallet: speedup refresh from daemon Assume the whole of a coinbase goes to the same address (so that if the first output isn't for us, none of it is), and only look for payment id when we received something in the transaction. --- src/wallet/wallet2.cpp | 95 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 58 insertions(+), 37 deletions(-) (limited to 'src/wallet/wallet2.cpp') diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index b51a23698..9f9dcb999 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -174,9 +174,10 @@ void wallet2::check_acc_out(const account_keys &acc, const tx_out &o, const cryp error = false; } //---------------------------------------------------------------------------------------------------- -void wallet2::process_new_transaction(const cryptonote::transaction& tx, uint64_t height) +void wallet2::process_new_transaction(const cryptonote::transaction& tx, uint64_t height, bool miner_tx) { - process_unconfirmed(tx, height); + if (!miner_tx) + process_unconfirmed(tx, height); std::vector outs; uint64_t tx_money_got_in_outs = 0; crypto::public_key tx_pub_key = null_pkey; @@ -202,8 +203,27 @@ void wallet2::process_new_transaction(const cryptonote::transaction& tx, uint64_ tx_pub_key = pub_key_field.pub_key; bool r = true; - int threads = std::thread::hardware_concurrency(); - if (threads > 1) + int threads; + if (miner_tx) + { + for (size_t i = 0; i < tx.vout.size(); ++i) + { + uint64_t money_transfered = 0; + bool error = false; + check_acc_out(m_account.get_keys(), tx.vout[i], tx_pub_key, i, money_transfered, error); + if (error) + { + r = false; + break; + } + // this assumes that the miner tx pays a single address + if (money_transfered == 0) + break; + outs.push_back(i); + tx_money_got_in_outs += money_transfered; + } + } + else if (tx.vout.size() > 1 && (threads = std::thread::hardware_concurrency()) > 1) { boost::asio::io_service ioservice; boost::thread_group threadpool; @@ -214,8 +234,8 @@ void wallet2::process_new_transaction(const cryptonote::transaction& tx, uint64_ } const account_keys &keys = m_account.get_keys(); - uint64_t *money_transfered = new uint64_t[tx.vout.size()]; - bool *error = new bool[tx.vout.size()]; + std::vector money_transfered(tx.vout.size()); + std::deque error(tx.vout.size()); for (size_t i = 0; i < tx.vout.size(); ++i) { ioservice.dispatch(boost::bind(&wallet2::check_acc_out, this, std::cref(keys), std::cref(tx.vout[i]), std::cref(tx_pub_key), i, @@ -236,8 +256,6 @@ void wallet2::process_new_transaction(const cryptonote::transaction& tx, uint64_ tx_money_got_in_outs += money_transfered[i]; } } - delete[] error; - delete[] money_transfered; } else { @@ -303,50 +321,53 @@ void wallet2::process_new_transaction(const cryptonote::transaction& tx, uint64_ } } - tx_extra_nonce extra_nonce; - crypto::hash payment_id = null_hash; - if (find_tx_extra_field_by_type(tx_extra_fields, extra_nonce)) + if (tx_money_spent_in_ins > 0) { - crypto::hash8 payment_id8 = null_hash8; - if(get_encrypted_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id8)) + process_outgoing(tx, height, tx_money_spent_in_ins, tx_money_got_in_outs); + } + + uint64_t received = (tx_money_spent_in_ins < tx_money_got_in_outs) ? tx_money_got_in_outs - tx_money_spent_in_ins : 0; + if (0 < received) + { + tx_extra_nonce extra_nonce; + crypto::hash payment_id = null_hash; + if (find_tx_extra_field_by_type(tx_extra_fields, extra_nonce)) { - // We got a payment ID to go with this tx - LOG_PRINT_L2("Found encrypted payment ID: " << payment_id8); - if (tx_pub_key != null_pkey) + crypto::hash8 payment_id8 = null_hash8; + if(get_encrypted_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id8)) { - if (!decrypt_payment_id(payment_id8, tx_pub_key, m_account.get_keys().m_view_secret_key)) + // We got a payment ID to go with this tx + LOG_PRINT_L2("Found encrypted payment ID: " << payment_id8); + if (tx_pub_key != null_pkey) { - LOG_PRINT_L0("Failed to decrypt payment ID: " << payment_id8); + if (!decrypt_payment_id(payment_id8, tx_pub_key, m_account.get_keys().m_view_secret_key)) + { + LOG_PRINT_L0("Failed to decrypt payment ID: " << payment_id8); + } + else + { + LOG_PRINT_L2("Decrypted payment ID: " << payment_id8); + // put the 64 bit decrypted payment id in the first 8 bytes + memcpy(payment_id.data, payment_id8.data, 8); + // rest is already 0, but guard against code changes above + memset(payment_id.data + 8, 0, 24); + } } else { - LOG_PRINT_L2("Decrypted payment ID: " << payment_id8); - // put the 64 bit decrypted payment id in the first 8 bytes - memcpy(payment_id.data, payment_id8.data, 8); - // rest is already 0, but guard against code changes above - memset(payment_id.data + 8, 0, 24); + LOG_PRINT_L1("No public key found in tx, unable to decrypt payment id"); } } - else + else if (get_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id)) { - LOG_PRINT_L1("No public key found in tx, unable to decrypt payment id"); + LOG_PRINT_L2("Found unencrypted payment ID: " << payment_id); } } else if (get_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id)) { LOG_PRINT_L2("Found unencrypted payment ID: " << payment_id); } - } - - uint64_t received = (tx_money_spent_in_ins < tx_money_got_in_outs) ? tx_money_got_in_outs - tx_money_spent_in_ins : 0; - - if (tx_money_spent_in_ins > 0) - { - process_outgoing(tx, height, tx_money_spent_in_ins, tx_money_got_in_outs); - } - if (0 < received) - { payment_details payment; payment.m_tx_hash = cryptonote::get_transaction_hash(tx); payment.m_amount = received; @@ -395,7 +416,7 @@ void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cry if(b.timestamp + 60*60*24 > m_account.get_createtime()) { TIME_MEASURE_START(miner_tx_handle_time); - process_new_transaction(b.miner_tx, height); + process_new_transaction(b.miner_tx, height, true); TIME_MEASURE_FINISH(miner_tx_handle_time); TIME_MEASURE_START(txs_handle_time); @@ -404,7 +425,7 @@ void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cry cryptonote::transaction tx; bool r = parse_and_validate_tx_from_blob(txblob, tx); THROW_WALLET_EXCEPTION_IF(!r, error::tx_parse_error, txblob); - process_new_transaction(tx, height); + process_new_transaction(tx, height, false); } TIME_MEASURE_FINISH(txs_handle_time); LOG_PRINT_L2("Processed block: " << bl_id << ", height " << height << ", " << miner_tx_handle_time + txs_handle_time << "(" << miner_tx_handle_time << "/" << txs_handle_time <<")ms"); -- cgit v1.2.3 From 9b945f52111cdb22fb17e7fe5cba223ebb8be2af Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 22 Nov 2015 19:03:10 +0000 Subject: wallet: make the refresh optimizations selectable via command line Take the opportunity to add a no-coinbase case too, for even faster sync when an address is known to never have mined to. --- src/wallet/wallet2.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'src/wallet/wallet2.cpp') diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 9f9dcb999..a6ac860c7 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -204,7 +204,11 @@ void wallet2::process_new_transaction(const cryptonote::transaction& tx, uint64_ tx_pub_key = pub_key_field.pub_key; bool r = true; int threads; - if (miner_tx) + if (miner_tx && m_refresh_type == RefreshNoCoinbase) + { + // assume coinbase isn't for us + } + else if (miner_tx && m_refresh_type == RefreshOptimizeCoinbase) { for (size_t i = 0; i < tx.vout.size(); ++i) { -- cgit v1.2.3