diff options
Diffstat (limited to 'src')
31 files changed, 882 insertions, 300 deletions
diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp index 1a6a19da5..01faf43c4 100644 --- a/src/blockchain_db/blockchain_db.cpp +++ b/src/blockchain_db/blockchain_db.cpp @@ -53,9 +53,7 @@ bool matches_category(relay_method method, relay_category category) noexcept case relay_category::all: return true; case relay_category::relayable: - if (method == relay_method::none) - return false; - return true; + return method != relay_method::none; case relay_category::broadcasted: case relay_category::legacy: break; @@ -65,6 +63,7 @@ bool matches_category(relay_method method, relay_category category) noexcept { default: case relay_method::local: + case relay_method::stem: return false; case relay_method::block: case relay_method::fluff: @@ -80,6 +79,7 @@ void txpool_tx_meta_t::set_relay_method(relay_method method) noexcept kept_by_block = 0; do_not_relay = 0; is_local = 0; + dandelionpp_stem = 0; switch (method) { @@ -92,6 +92,9 @@ void txpool_tx_meta_t::set_relay_method(relay_method method) noexcept default: case relay_method::fluff: break; + case relay_method::stem: + dandelionpp_stem = 1; + break; case relay_method::block: kept_by_block = 1; break; @@ -106,9 +109,26 @@ relay_method txpool_tx_meta_t::get_relay_method() const noexcept return relay_method::none; if (is_local) return relay_method::local; + if (dandelionpp_stem) + return relay_method::stem; return relay_method::fluff; } +bool txpool_tx_meta_t::upgrade_relay_method(relay_method method) noexcept +{ + static_assert(relay_method::none < relay_method::local, "bad relay_method value"); + static_assert(relay_method::local < relay_method::stem, "bad relay_method value"); + static_assert(relay_method::stem < relay_method::fluff, "bad relay_method value"); + static_assert(relay_method::fluff < relay_method::block, "bad relay_method value"); + + if (get_relay_method() < method) + { + set_relay_method(method); + return true; + } + return false; +} + const command_line::arg_descriptor<std::string> arg_db_sync_mode = { "db-sync-mode" , "Specify sync option, using format [safe|fast|fastest]:[sync|async]:[<nblocks_per_sync>[blocks]|<nbytes_per_sync>[bytes]]." diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index d3a218365..3e2387da4 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -160,7 +160,7 @@ struct txpool_tx_meta_t uint64_t max_used_block_height; uint64_t last_failed_height; uint64_t receive_time; - uint64_t last_relayed_time; + uint64_t last_relayed_time; //!< If Dandelion++ stem, randomized embargo timestamp. Otherwise, last relayed timestmap. // 112 bytes uint8_t kept_by_block; uint8_t relayed; @@ -168,13 +168,17 @@ struct txpool_tx_meta_t uint8_t double_spend_seen: 1; uint8_t pruned: 1; uint8_t is_local: 1; - uint8_t bf_padding: 5; + uint8_t dandelionpp_stem : 1; + uint8_t bf_padding: 4; uint8_t padding[76]; // till 192 bytes void set_relay_method(relay_method method) noexcept; relay_method get_relay_method() const noexcept; + //! \return True if `get_relay_method()` now returns `method`. + bool upgrade_relay_method(relay_method method) noexcept; + //! See `relay_category` description bool matches(const relay_category category) const noexcept { diff --git a/src/common/util.cpp b/src/common/util.cpp index 747235646..f1140d1d5 100644 --- a/src/common/util.cpp +++ b/src/common/util.cpp @@ -66,7 +66,6 @@ using namespace epee; #include "util.h" #include "stack_trace.h" #include "memwipe.h" -#include "cryptonote_config.h" #include "net/http_client.h" // epee::net_utils::... #include "readline_buffer.h" @@ -1074,16 +1073,33 @@ std::string get_nix_version_display_string() { if (seconds < 60) return std::to_string(seconds) + " seconds"; + std::stringstream ss; + ss << std::fixed << std::setprecision(1); if (seconds < 3600) - return std::to_string((uint64_t)(seconds / 60)) + " minutes"; + { + ss << seconds / 60.f; + return ss.str() + " minutes"; + } if (seconds < 3600 * 24) - return std::to_string((uint64_t)(seconds / 3600)) + " hours"; - if (seconds < 3600 * 24 * 30.5) - return std::to_string((uint64_t)(seconds / (3600 * 24))) + " days"; - if (seconds < 3600 * 24 * 365.25) - return std::to_string((uint64_t)(seconds / (3600 * 24 * 30.5))) + " months"; - if (seconds < 3600 * 24 * 365.25 * 100) - return std::to_string((uint64_t)(seconds / (3600 * 24 * 30.5 * 365.25))) + " years"; + { + ss << seconds / 3600.f; + return ss.str() + " hours"; + } + if (seconds < 3600 * 24 * 30.5f) + { + ss << seconds / (3600 * 24.f); + return ss.str() + " days"; + } + if (seconds < 3600 * 24 * 365.25f) + { + ss << seconds / (3600 * 24 * 30.5f); + return ss.str() + " months"; + } + if (seconds < 3600 * 24 * 365.25f * 100) + { + ss << seconds / (3600 * 24 * 365.25f); + return ss.str() + " years"; + } return "a long time"; } @@ -1279,6 +1295,86 @@ std::string get_nix_version_display_string() return lines; } + // Calculate a "sync weight" over ranges of blocks in the blockchain, suitable for + // calculating sync time estimates + uint64_t cumulative_block_sync_weight(cryptonote::network_type nettype, uint64_t start_block, uint64_t num_blocks) + { + if (nettype != cryptonote::MAINNET) + { + // No detailed data available except for Mainnet: Give back the number of blocks + // as a very simple and non-varying block sync weight for ranges of Testnet and + // Stagenet blocks + return num_blocks; + } + + // The following is a table of average blocks sizes in bytes over the Monero mainnet + // blockchain, where the block size is averaged over ranges of 10,000 blocks + // (about 2 weeks worth of blocks each). + // The first array entry of 442 thus means "The average byte size of the blocks + // 0 .. 9,999 is 442". The info "block_size" from the "get_block_header_by_height" + // RPC call was used for calculating this. This table (and the whole mechanism + // of calculating a "sync weight") is most important when estimating times for + // syncing from scratch. Without it the fast progress through the (in comparison) + // rather small blocks in the early blockchain) would lead to vastly underestimated + // total sync times. + // It's no big problem for estimates that this table will, over time, and if not + // updated, miss larger and larger parts at the top of the blockchain, as long + // as block size averages there do not differ wildly. + // Without time-consuming tests it's hard to say how much the estimates would + // improve if one would not only take block sizes into account, but also varying + // verification times i.e. the different CPU effort needed for the different + // transaction types (pre / post RingCT, pre / post Bulletproofs). + // Testnet and Stagenet are neglected here because of their much smaller + // importance. + static const uint32_t average_block_sizes[] = + { + 442, 1211, 1445, 1763, 2272, 8217, 5603, 9999, 16358, 10805, 5290, 4362, + 4325, 5584, 4515, 5008, 4789, 5196, 7660, 3829, 6034, 2925, 3762, 2545, + 2437, 2553, 2167, 2761, 2015, 1969, 2350, 1731, 2367, 2078, 2026, 3518, + 2214, 1908, 1780, 1640, 1976, 1647, 1921, 1716, 1895, 2150, 2419, 2451, + 2147, 2327, 2251, 1644, 1750, 1481, 1570, 1524, 1562, 1668, 1386, 1494, + 1637, 1880, 1431, 1472, 1637, 1363, 1762, 1597, 1999, 1564, 1341, 1388, + 1530, 1476, 1617, 1488, 1368, 1906, 1403, 1695, 1535, 1598, 1318, 1234, + 1358, 1406, 1698, 1554, 1591, 1758, 1426, 2389, 1946, 1533, 1308, 2701, + 1525, 1653, 3580, 1889, 2913, 8164, 5154, 3762, 3356, 4360, 3589, 4844, + 4232, 3781, 3882, 5924, 10790, 7185, 7442, 8214, 8509, 7484, 6939, 7391, + 8210, 15572, 39680, 44810, 53873, 54639, 68227, 63428, 62386, 68504, + 83073, 103858, 117573, 98089, 96793, 102337, 94714, 129568, 251584, + 132026, 94579, 94516, 95722, 106495, 121824, 153983, 162338, 136608, + 137104, 109872, 91114, 84757, 96339, 74251, 94314, 143216, 155837, + 129968, 120201, 109913, 101588, 97332, 104611, 95310, 93419, 113345, + 100743, 92152, 57565, 22533, 37564, 21823, 19980, 18277, 18402, 14344, + 12142, 15842, 13677, 17631, 18294, 22270, 41422, 39296, 36688, 33512, + 33831, 27582, 22276, 27516, 27317, 25505, 24426, 20566, 23045, 26766, + 28185, 26169, 27011, + 28642 // Blocks 1,990,000 to 1,999,999 in December 2019 + }; + const uint64_t block_range_size = 10000; + + uint64_t num_block_sizes = sizeof(average_block_sizes) / sizeof(average_block_sizes[0]); + uint64_t weight = 0; + uint64_t table_index = start_block / block_range_size; + for (;;) { + if (num_blocks == 0) + { + break; + } + if (table_index >= num_block_sizes) + { + // Take all blocks beyond our table as having the size of the blocks + // in the last table entry i.e. in the most recent known block range + weight += num_blocks * average_block_sizes[num_block_sizes - 1]; + break; + } + uint64_t portion_size = std::min(num_blocks, block_range_size - start_block % block_range_size); + weight += portion_size * average_block_sizes[table_index]; + table_index++; + num_blocks -= portion_size; + start_block += portion_size; + } + return weight; + } + std::vector<std::pair<std::string, size_t>> split_string_by_width(const std::string &s, size_t columns) { std::vector<std::string> lines; diff --git a/src/common/util.h b/src/common/util.h index b794d7908..25137ab64 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -46,6 +46,7 @@ #endif #include "crypto/hash.h" +#include "cryptonote_config.h" /*! \brief Various Tools * @@ -252,4 +253,6 @@ namespace tools void clear_screen(); std::vector<std::pair<std::string, size_t>> split_string_by_width(const std::string &s, size_t columns); + + uint64_t cumulative_block_sync_weight(cryptonote::network_type nettype, uint64_t start_block, uint64_t num_blocks); } diff --git a/src/crypto/duration.h b/src/crypto/duration.h new file mode 100644 index 000000000..493874288 --- /dev/null +++ b/src/crypto/duration.h @@ -0,0 +1,70 @@ +// Copyright (c) 2020, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <chrono> +#include "crypto/crypto.h" + +namespace crypto +{ + //! Generate poisson distributed values in discrete `D` time units. + template<typename D> + struct random_poisson_duration + { + using result_type = D; //!< std::chrono::duration time unit precision + using rep = typename result_type::rep; //!< Type used to represent duration value + + //! \param average for generated durations + explicit random_poisson_duration(result_type average) + : dist(average.count() < 0 ? 0 : average.count()) + {} + + //! Generate a crypto-secure random duration + result_type operator()() + { + crypto::random_device rand{}; + return result_type{dist(rand)}; + } + + private: + std::poisson_distribution<rep> dist; + }; + + /* A custom duration is used for subsecond precision because of the + variance. If 5000 milliseconds is given, 95% of the values fall between + 4859ms-5141ms in 1ms increments (not enough time variance). Providing 1/4 + seconds would yield 95% of the values between 3s-7.25s in 1/4s + increments. */ + + //! Generate random durations with 1 second precision + using random_poisson_seconds = random_poisson_duration<std::chrono::seconds>; + //! Generate random duration with 1/4 second precision + using random_poisson_subseconds = + random_poisson_duration<std::chrono::duration<std::chrono::milliseconds::rep, std::ratio<1, 4>>>; +} diff --git a/src/cryptonote_basic/verification_context.h b/src/cryptonote_basic/verification_context.h index 4d49b692c..ec5f604a5 100644 --- a/src/cryptonote_basic/verification_context.h +++ b/src/cryptonote_basic/verification_context.h @@ -29,6 +29,9 @@ // Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers #pragma once + +#include "cryptonote_protocol/enums.h" + namespace cryptonote { /************************************************************************/ @@ -36,7 +39,9 @@ namespace cryptonote /************************************************************************/ struct tx_verification_context { - bool m_should_be_relayed; + static_assert(unsigned(relay_method::none) == 0, "default m_relay initialization is not to relay_method::none"); + + relay_method m_relay; // gives indication on how tx should be relayed (if at all) bool m_verifivation_failed; //bad tx, should drop connection bool m_verifivation_impossible; //the transaction is related with an alternative blockchain bool m_added_to_pool; diff --git a/src/cryptonote_config.h b/src/cryptonote_config.h index 66af46a5f..81dc15dee 100644 --- a/src/cryptonote_config.h +++ b/src/cryptonote_config.h @@ -102,7 +102,12 @@ #define CRYPTONOTE_MEMPOOL_TX_FROM_ALT_BLOCK_LIVETIME 604800 //seconds, one week -#define CRYPTONOTE_DANDELIONPP_FLUSH_AVERAGE 5 // seconds +#define CRYPTONOTE_DANDELIONPP_STEMS 2 // number of outgoing stem connections per epoch +#define CRYPTONOTE_DANDELIONPP_FLUFF_PROBABILITY 10 // out of 100 +#define CRYPTONOTE_DANDELIONPP_MIN_EPOCH 10 // minutes +#define CRYPTONOTE_DANDELIONPP_EPOCH_RANGE 30 // seconds +#define CRYPTONOTE_DANDELIONPP_FLUSH_AVERAGE 5 // seconds average for poisson distributed fluff flush +#define CRYPTONOTE_DANDELIONPP_EMBARGO_AVERAGE 173 // seconds (see tx_pool.cpp for more info) // see src/cryptonote_protocol/levin_notify.cpp #define CRYPTONOTE_NOISE_MIN_EPOCH 5 // minutes diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 7fb232ad2..10bbff457 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -1284,6 +1284,7 @@ namespace cryptonote break; case relay_method::block: case relay_method::fluff: + case relay_method::stem: public_req.txs.push_back(std::move(std::get<1>(tx))); break; } @@ -1295,9 +1296,9 @@ namespace cryptonote re-relaying public and private _should_ be acceptable here. */ const boost::uuids::uuid source = boost::uuids::nil_uuid(); if (!public_req.txs.empty()) - get_protocol()->relay_transactions(public_req, source, epee::net_utils::zone::public_); + get_protocol()->relay_transactions(public_req, source, epee::net_utils::zone::public_, relay_method::fluff); if (!private_req.txs.empty()) - get_protocol()->relay_transactions(private_req, source, epee::net_utils::zone::invalid); + get_protocol()->relay_transactions(private_req, source, epee::net_utils::zone::invalid, relay_method::local); } return true; } diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index d7fc89d61..c49a3dabc 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -46,6 +46,7 @@ #include "warnings.h" #include "common/perf_timer.h" #include "crypto/hash.h" +#include "crypto/duration.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "txpool" @@ -58,6 +59,29 @@ namespace cryptonote { namespace { + /*! The Dandelion++ has formula for calculating the average embargo timeout: + (-k*(k-1)*hop)/(2*log(1-ep)) + where k is the number of hops before this node and ep is the probability + that one of the k hops hits their embargo timer, and hop is the average + time taken between hops. So decreasing ep will make it more probable + that "this" node is the first to expire the embargo timer. Increasing k + will increase the number of nodes that will be "hidden" as a prior + recipient of the tx. + + As example, k=5 and ep=0.1 means "this" embargo timer has a 90% + probability of being the first to expire amongst 5 nodes that saw the + tx before "this" one. These values are independent to the fluff + probability, but setting a low k with a low p (fluff probability) is + not ideal since a blackhole is more likely to reveal earlier nodes in + the chain. + + This value was calculated with k=10, ep=0.10, and hop = 175 ms. A + testrun from a recent Intel laptop took ~80ms to + receive+parse+proces+send transaction. At least 50ms will be added to + the latency if crossing an ocean. So 175ms is the fudge factor for + a single hop with 173s being the embargo timer. */ + constexpr const std::chrono::seconds dandelionpp_embargo_average{CRYPTONOTE_DANDELIONPP_EMBARGO_AVERAGE}; + //TODO: constants such as these should at least be in the header, // but probably somewhere more accessible to the rest of the // codebase. As it stands, it is at best nontrivial to test @@ -262,34 +286,51 @@ namespace cryptonote } }else { - //update transactions container - meta.weight = tx_weight; - meta.fee = fee; - meta.max_used_block_id = max_used_block_id; - meta.max_used_block_height = max_used_block_height; - meta.last_failed_height = 0; - meta.last_failed_id = null_hash; - meta.receive_time = receive_time; - meta.last_relayed_time = time(NULL); - meta.relayed = relayed; - meta.set_relay_method(tx_relay); - meta.double_spend_seen = false; - meta.pruned = tx.pruned; - meta.bf_padding = 0; - memset(meta.padding, 0, sizeof(meta.padding)); - try { if (kept_by_block) m_parsed_tx_cache.insert(std::make_pair(id, tx)); CRITICAL_REGION_LOCAL1(m_blockchain); LockedTXN lock(m_blockchain.get_db()); - m_blockchain.remove_txpool_tx(id); - if (!insert_key_images(tx, id, tx_relay)) - return false; - m_blockchain.add_txpool_tx(id, blob, meta); - m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)(tx_weight ? tx_weight : 1), receive_time), id); + const bool existing_tx = m_blockchain.get_txpool_tx_meta(id, meta); + if (existing_tx) + { + /* If Dandelion++ loop. Do not use txes in the `local` state in the + loop detection - txes in that state should be outgoing over i2p/tor + then routed back via public dandelion++ stem. Pretend to be + another stem node in that situation, a loop over the public + network hasn't been hit yet. */ + if (tx_relay == relay_method::stem && meta.dandelionpp_stem) + tx_relay = relay_method::fluff; + } + else + meta.set_relay_method(relay_method::none); + + if (meta.upgrade_relay_method(tx_relay) || !existing_tx) // synchronize with embargo timer or stem/fluff out-of-order messages + { + //update transactions container + meta.last_relayed_time = std::numeric_limits<decltype(meta.last_relayed_time)>::max(); + meta.receive_time = receive_time; + meta.weight = tx_weight; + meta.fee = fee; + meta.max_used_block_id = max_used_block_id; + meta.max_used_block_height = max_used_block_height; + meta.last_failed_height = 0; + meta.last_failed_id = null_hash; + meta.relayed = relayed; + meta.double_spend_seen = false; + meta.pruned = tx.pruned; + meta.bf_padding = 0; + memset(meta.padding, 0, sizeof(meta.padding)); + + if (!insert_key_images(tx, id, tx_relay)) + return false; + + m_blockchain.remove_txpool_tx(id); + m_blockchain.add_txpool_tx(id, blob, meta); + m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)(tx_weight ? tx_weight : 1), receive_time), id); + } lock.commit(); } catch (const std::exception &e) @@ -299,8 +340,9 @@ namespace cryptonote } tvc.m_added_to_pool = true; - if(meta.fee > 0 && tx_relay != relay_method::none) - tvc.m_should_be_relayed = true; + static_assert(unsigned(relay_method::none) == 0, "expected relay_method::none value to be zero"); + if(meta.fee > 0) + tvc.m_relay = tx_relay; } tvc.m_verifivation_failed = false; @@ -553,7 +595,7 @@ namespace cryptonote td.last_failed_height = meta.last_failed_height; td.last_failed_id = meta.last_failed_id; td.receive_time = meta.receive_time; - td.last_relayed_time = meta.last_relayed_time; + td.last_relayed_time = meta.dandelionpp_stem ? 0 : meta.last_relayed_time; td.relayed = meta.relayed; td.do_not_relay = meta.do_not_relay; td.double_spend_seen = meta.double_spend_seen; @@ -686,8 +728,13 @@ namespace cryptonote txs.reserve(m_blockchain.get_txpool_tx_count()); m_blockchain.for_all_txpool_txes([this, now, &txs](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *){ // 0 fee transactions are never relayed - if(!meta.pruned && meta.fee > 0 && !meta.do_not_relay && now - meta.last_relayed_time > get_relay_delay(now, meta.receive_time)) + if(!meta.pruned && meta.fee > 0 && !meta.do_not_relay) { + if (!meta.dandelionpp_stem && now - meta.last_relayed_time <= get_relay_delay(now, meta.receive_time)) + return true; + if (meta.dandelionpp_stem && meta.last_relayed_time < now) // for dandelion++ stem, this value is the embargo timeout + return true; + // if the tx is older than half the max lifetime, we don't re-relay it, to avoid a problem // mentioned by smooth where nodes would flush txes at slightly different times, causing // flushed txes to be re-added when received from a node which was just about to flush it @@ -712,9 +759,11 @@ namespace cryptonote //--------------------------------------------------------------------------------- void tx_memory_pool::set_relayed(const epee::span<const crypto::hash> hashes, const relay_method method) { + crypto::random_poisson_seconds embargo_duration{dandelionpp_embargo_average}; + const auto now = std::chrono::system_clock::now(); + CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); - const time_t now = time(NULL); LockedTXN lock(m_blockchain.get_db()); for (const auto& hash : hashes) { @@ -723,9 +772,15 @@ namespace cryptonote txpool_tx_meta_t meta; if (m_blockchain.get_txpool_tx_meta(hash, meta)) { + // txes can be received as "stem" or "fluff" in either order + meta.upgrade_relay_method(method); meta.relayed = true; - meta.last_relayed_time = now; - meta.set_relay_method(method); + + if (meta.dandelionpp_stem) + meta.last_relayed_time = std::chrono::system_clock::to_time_t(now + embargo_duration()); + else + meta.last_relayed_time = std::chrono::system_clock::to_time_t(now); + m_blockchain.update_txpool_tx(hash, meta); } } @@ -910,7 +965,7 @@ namespace cryptonote txi.receive_time = include_sensitive_data ? meta.receive_time : 0; txi.relayed = meta.relayed; // In restricted mode we do not include this data: - txi.last_relayed_time = include_sensitive_data ? meta.last_relayed_time : 0; + txi.last_relayed_time = (include_sensitive_data && !meta.dandelionpp_stem) ? meta.last_relayed_time : 0; txi.do_not_relay = meta.do_not_relay; txi.double_spend_seen = meta.double_spend_seen; tx_infos.push_back(std::move(txi)); @@ -962,7 +1017,7 @@ namespace cryptonote txi.last_failed_block_hash = meta.last_failed_id; txi.receive_time = meta.receive_time; txi.relayed = meta.relayed; - txi.last_relayed_time = meta.last_relayed_time; + txi.last_relayed_time = meta.dandelionpp_stem ? 0 : meta.last_relayed_time; txi.do_not_relay = meta.do_not_relay; txi.double_spend_seen = meta.double_spend_seen; tx_infos.push_back(txi); diff --git a/src/cryptonote_protocol/cryptonote_protocol_defs.h b/src/cryptonote_protocol/cryptonote_protocol_defs.h index f809bff74..76b57afd3 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_defs.h +++ b/src/cryptonote_protocol/cryptonote_protocol_defs.h @@ -197,10 +197,12 @@ namespace cryptonote { std::vector<blobdata> txs; std::string _; // padding + bool dandelionpp_fluff; //zero initialization defaults to stem mode BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(txs) KV_SERIALIZE(_) + KV_SERIALIZE_OPT(dandelionpp_fluff, true) // backwards compatible mode is fluff END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<request_t> request; diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.h b/src/cryptonote_protocol/cryptonote_protocol_handler.h index 2664716a8..e2ad3727f 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.h @@ -129,7 +129,7 @@ namespace cryptonote //----------------- i_bc_protocol_layout --------------------------------------- virtual bool relay_block(NOTIFY_NEW_BLOCK::request& arg, cryptonote_connection_context& exclude_context); - virtual bool relay_transactions(NOTIFY_NEW_TRANSACTIONS::request& arg, const boost::uuids::uuid& source, epee::net_utils::zone zone); + virtual bool relay_transactions(NOTIFY_NEW_TRANSACTIONS::request& arg, const boost::uuids::uuid& source, epee::net_utils::zone zone, relay_method tx_relay); //---------------------------------------------------------------------------------- //bool get_payload_sync_data(HANDSHAKE_DATA::request& hshd, cryptonote_connection_context& context); bool should_drop_connection(cryptonote_connection_context& context, uint32_t next_stripe); @@ -170,6 +170,14 @@ namespace cryptonote size_t m_block_download_max_size; bool m_sync_pruned_blocks; + // Values for sync time estimates + boost::posix_time::ptime m_sync_start_time; + boost::posix_time::ptime m_period_start_time; + uint64_t m_sync_start_height; + uint64_t m_period_start_height; + uint64_t get_estimated_remaining_sync_seconds(uint64_t current_blockchain_height, uint64_t target_blockchain_height); + std::string get_periodic_sync_estimate(uint64_t current_blockchain_height, uint64_t target_blockchain_height); + boost::mutex m_buffer_mutex; double get_avg_block_size(); boost::circular_buffer<size_t> m_avg_buffer = boost::circular_buffer<size_t>(10); diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index 3aacce421..f8e032fde 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -43,6 +43,7 @@ #include "profile_tools.h" #include "net/network_throttle-detail.hpp" #include "common/pruning.h" +#include "common/util.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net.cn" @@ -362,7 +363,7 @@ namespace cryptonote uint64_t last_block_v1 = m_core.get_nettype() == TESTNET ? 624633 : m_core.get_nettype() == MAINNET ? 1009826 : (uint64_t)-1; uint64_t diff_v2 = max_block_height > last_block_v1 ? std::min(abs_diff, max_block_height - last_block_v1) : 0; MCLOG(is_inital ? el::Level::Info : el::Level::Debug, "global", el::Color::Yellow, context << "Sync data returned a new top block candidate: " << m_core.get_current_blockchain_height() << " -> " << hshd.current_height - << " [Your node is " << abs_diff << " blocks (" << ((abs_diff - diff_v2) / (24 * 60 * 60 / DIFFICULTY_TARGET_V1)) + (diff_v2 / (24 * 60 * 60 / DIFFICULTY_TARGET_V2)) << " days) " + << " [Your node is " << abs_diff << " blocks (" << tools::get_human_readable_timespan((abs_diff - diff_v2) * DIFFICULTY_TARGET_V1 + diff_v2 * DIFFICULTY_TARGET_V2) << ") " << (0 <= diff ? std::string("behind") : std::string("ahead")) << "] " << ENDL << "SYNCHRONIZATION started"); if (hshd.current_height >= m_core.get_current_blockchain_height() + 5) // don't switch to unsafe mode just for a few blocks @@ -926,29 +927,60 @@ namespace cryptonote return 1; } - std::vector<cryptonote::blobdata> newtxs; - newtxs.reserve(arg.txs.size()); - for (size_t i = 0; i < arg.txs.size(); ++i) + relay_method tx_relay; + std::vector<blobdata> stem_txs{}; + std::vector<blobdata> fluff_txs{}; + if (arg.dandelionpp_fluff) { - cryptonote::tx_verification_context tvc{}; - m_core.handle_incoming_tx({arg.txs[i], crypto::null_hash}, tvc, relay_method::fluff, true); - if(tvc.m_verifivation_failed) + tx_relay = relay_method::fluff; + fluff_txs.reserve(arg.txs.size()); + } + else + { + tx_relay = relay_method::stem; + stem_txs.reserve(arg.txs.size()); + } + + for (auto& tx : arg.txs) + { + tx_verification_context tvc{}; + if (!m_core.handle_incoming_tx({tx, crypto::null_hash}, tvc, tx_relay, true)) { LOG_PRINT_CCONTEXT_L1("Tx verification failed, dropping connection"); drop_connection(context, false, false); return 1; } - if(tvc.m_should_be_relayed) - newtxs.push_back(std::move(arg.txs[i])); + + switch (tvc.m_relay) + { + case relay_method::local: + case relay_method::stem: + stem_txs.push_back(std::move(tx)); + break; + case relay_method::block: + case relay_method::fluff: + fluff_txs.push_back(std::move(tx)); + break; + default: + case relay_method::none: + break; + } } - arg.txs = std::move(newtxs); - if(arg.txs.size()) + if (!stem_txs.empty()) { //TODO: add announce usage here - relay_transactions(arg, context.m_connection_id, context.m_remote_address.get_zone()); + arg.dandelionpp_fluff = false; + arg.txs = std::move(stem_txs); + relay_transactions(arg, context.m_connection_id, context.m_remote_address.get_zone(), relay_method::stem); + } + if (!fluff_txs.empty()) + { + //TODO: add announce usage here + arg.dandelionpp_fluff = true; + arg.txs = std::move(fluff_txs); + relay_transactions(arg, context.m_connection_id, context.m_remote_address.get_zone(), relay_method::fluff); } - return 1; } //------------------------------------------------------------------------------------------------------------------------ @@ -1181,6 +1213,55 @@ namespace cryptonote return 1; } + // Get an estimate for the remaining sync time from given current to target blockchain height, in seconds + template<class t_core> + uint64_t t_cryptonote_protocol_handler<t_core>::get_estimated_remaining_sync_seconds(uint64_t current_blockchain_height, uint64_t target_blockchain_height) + { + // The average sync speed varies so much, even averaged over quite long time periods like 10 minutes, + // that using some sliding window would be difficult to implement without often leading to bad estimates. + // The simplest strategy - always average sync speed over the maximum available interval i.e. since sync + // started at all (from "m_sync_start_time" and "m_sync_start_height") - gives already useful results + // and seems to be quite robust. Some quite special cases like "Internet connection suddenly becoming + // much faster after syncing already a long time, and staying fast" are not well supported however. + + if (target_blockchain_height <= current_blockchain_height) + { + // Syncing stuck, or other special circumstance: Avoid errors, simply give back 0 + return 0; + } + + const boost::posix_time::ptime now = boost::posix_time::microsec_clock::universal_time(); + const boost::posix_time::time_duration sync_time = now - m_sync_start_time; + cryptonote::network_type nettype = m_core.get_nettype(); + + // Don't simply use remaining number of blocks for the estimate but "sync weight" as provided by + // "cumulative_block_sync_weight" which knows about strongly varying Monero mainnet block sizes + uint64_t synced_weight = tools::cumulative_block_sync_weight(nettype, m_sync_start_height, current_blockchain_height - m_sync_start_height); + float us_per_weight = (float)sync_time.total_microseconds() / (float)synced_weight; + uint64_t remaining_weight = tools::cumulative_block_sync_weight(nettype, current_blockchain_height, target_blockchain_height - current_blockchain_height); + float remaining_us = us_per_weight * (float)remaining_weight; + return (uint64_t)(remaining_us / 1e6); + } + + // Return a textual remaining sync time estimate, or the empty string if waiting period not yet over + template<class t_core> + std::string t_cryptonote_protocol_handler<t_core>::get_periodic_sync_estimate(uint64_t current_blockchain_height, uint64_t target_blockchain_height) + { + std::string text = ""; + const boost::posix_time::ptime now = boost::posix_time::microsec_clock::universal_time(); + boost::posix_time::time_duration period_sync_time = now - m_period_start_time; + if (period_sync_time > boost::posix_time::minutes(2)) + { + // Period is over, time to report another estimate + uint64_t remaining_seconds = get_estimated_remaining_sync_seconds(current_blockchain_height, target_blockchain_height); + text = tools::get_human_readable_timespan(remaining_seconds); + + // Start the new period + m_period_start_time = now; + } + return text; + } + template<class t_core> int t_cryptonote_protocol_handler<t_core>::try_add_next_blocks(cryptonote_connection_context& context) { @@ -1209,6 +1290,9 @@ namespace cryptonote if (!starting) m_last_add_end_time = tools::get_tick_count(); }); + m_sync_start_time = boost::posix_time::microsec_clock::universal_time(); + m_sync_start_height = m_core.get_current_blockchain_height(); + m_period_start_time = m_sync_start_time; while (1) { @@ -1459,7 +1543,16 @@ namespace cryptonote if (completion_percent == 100) // never show 100% if not actually up to date completion_percent = 99; progress_message = " (" + std::to_string(completion_percent) + "%, " - + std::to_string(target_blockchain_height - current_blockchain_height) + " left)"; + + std::to_string(target_blockchain_height - current_blockchain_height) + " left"; + std::string time_message = get_periodic_sync_estimate(current_blockchain_height, target_blockchain_height); + if (!time_message.empty()) + { + uint64_t total_blocks_to_sync = target_blockchain_height - m_sync_start_height; + uint64_t total_blocks_synced = current_blockchain_height - m_sync_start_height; + progress_message += ", " + std::to_string(total_blocks_synced * 100 / total_blocks_to_sync) + "% of total synced"; + progress_message += ", estimated " + time_message + " left"; + } + progress_message += ")"; } const uint32_t previous_stripe = tools::get_pruning_stripe(previous_height, target_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES); const uint32_t current_stripe = tools::get_pruning_stripe(current_blockchain_height, target_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES); @@ -2197,8 +2290,26 @@ skip: bool t_cryptonote_protocol_handler<t_core>::on_connection_synchronized() { bool val_expected = false; - if(!m_core.is_within_compiled_block_hash_area(m_core.get_current_blockchain_height()) && m_synchronized.compare_exchange_strong(val_expected, true)) + uint64_t current_blockchain_height = m_core.get_current_blockchain_height(); + if(!m_core.is_within_compiled_block_hash_area(current_blockchain_height) && m_synchronized.compare_exchange_strong(val_expected, true)) { + if ((current_blockchain_height > m_sync_start_height) && (m_sync_spans_downloaded > 0)) + { + uint64_t synced_blocks = current_blockchain_height - m_sync_start_height; + // Report only after syncing an "interesting" number of blocks: + if (synced_blocks > 20) + { + const boost::posix_time::ptime now = boost::posix_time::microsec_clock::universal_time(); + uint64_t synced_seconds = (now - m_sync_start_time).total_seconds(); + if (synced_seconds == 0) + { + synced_seconds = 1; + } + float blocks_per_second = (1000 * synced_blocks / synced_seconds) / 1000.0f; + MGINFO_YELLOW("Synced " << synced_blocks << " blocks in " + << tools::get_human_readable_timespan(synced_seconds) << " (" << blocks_per_second << " blocks per second)"); + } + } MGINFO_YELLOW(ENDL << "**********************************************************************" << ENDL << "You are now synchronized with the network. You may now start monero-wallet-cli." << ENDL << ENDL @@ -2387,14 +2498,14 @@ skip: } //------------------------------------------------------------------------------------------------------------------------ template<class t_core> - bool t_cryptonote_protocol_handler<t_core>::relay_transactions(NOTIFY_NEW_TRANSACTIONS::request& arg, const boost::uuids::uuid& source, epee::net_utils::zone zone) + bool t_cryptonote_protocol_handler<t_core>::relay_transactions(NOTIFY_NEW_TRANSACTIONS::request& arg, const boost::uuids::uuid& source, epee::net_utils::zone zone, relay_method tx_relay) { /* Push all outgoing transactions to this function. The behavior needs to identify how the transaction is going to be relayed, and then update the local mempool before doing the relay. The code was already updating the DB twice on received transactions - it is difficult to workaround this due to the internal design. */ - return m_p2p->send_txs(std::move(arg.txs), zone, source, m_core) != epee::net_utils::zone::invalid; + return m_p2p->send_txs(std::move(arg.txs), zone, source, m_core, tx_relay) != epee::net_utils::zone::invalid; } //------------------------------------------------------------------------------------------------------------------------ template<class t_core> diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler_common.h b/src/cryptonote_protocol/cryptonote_protocol_handler_common.h index 978a9ebf3..11184299d 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler_common.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler_common.h @@ -41,7 +41,7 @@ namespace cryptonote struct i_cryptonote_protocol { virtual bool relay_block(NOTIFY_NEW_BLOCK::request& arg, cryptonote_connection_context& exclude_context)=0; - virtual bool relay_transactions(NOTIFY_NEW_TRANSACTIONS::request& arg, const boost::uuids::uuid& source, epee::net_utils::zone zone)=0; + virtual bool relay_transactions(NOTIFY_NEW_TRANSACTIONS::request& arg, const boost::uuids::uuid& source, epee::net_utils::zone zone, relay_method tx_relay)=0; //virtual bool request_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, cryptonote_connection_context& context)=0; }; @@ -54,7 +54,7 @@ namespace cryptonote { return false; } - virtual bool relay_transactions(NOTIFY_NEW_TRANSACTIONS::request& arg, const boost::uuids::uuid& source, epee::net_utils::zone zone) + virtual bool relay_transactions(NOTIFY_NEW_TRANSACTIONS::request& arg, const boost::uuids::uuid& source, epee::net_utils::zone zone, relay_method tx_relay) { return false; } diff --git a/src/cryptonote_protocol/enums.h b/src/cryptonote_protocol/enums.h index 2ec622d94..a3a52b83f 100644 --- a/src/cryptonote_protocol/enums.h +++ b/src/cryptonote_protocol/enums.h @@ -37,7 +37,8 @@ namespace cryptonote { none = 0, //!< Received via RPC with `do_not_relay` set local, //!< Received via RPC; trying to send over i2p/tor, etc. - block, //!< Received in block, takes precedence over others - fluff //!< Received/sent over public networks + stem, //!< Received/send over network using Dandelion++ stem + fluff, //!< Received/sent over network using Dandelion++ fluff + block //!< Received in block, takes precedence over others }; } diff --git a/src/cryptonote_protocol/levin_notify.cpp b/src/cryptonote_protocol/levin_notify.cpp index 428b739bc..127801092 100644 --- a/src/cryptonote_protocol/levin_notify.cpp +++ b/src/cryptonote_protocol/levin_notify.cpp @@ -30,6 +30,7 @@ #include <boost/asio/steady_timer.hpp> #include <boost/system/system_error.hpp> +#include <boost/uuid/uuid_io.hpp> #include <chrono> #include <deque> #include <stdexcept> @@ -38,8 +39,10 @@ #include "common/expect.h" #include "common/varint.h" #include "cryptonote_config.h" -#include "crypto/random.h" +#include "crypto/crypto.h" +#include "crypto/duration.h" #include "cryptonote_basic/connection_context.h" +#include "cryptonote_core/i_core_events.h" #include "cryptonote_protocol/cryptonote_protocol_defs.h" #include "net/dandelionpp.h" #include "p2p/net_node.h" @@ -61,11 +64,14 @@ namespace levin { namespace { - constexpr std::size_t connection_id_reserve_size = 100; + constexpr const std::size_t connection_id_reserve_size = 100; constexpr const std::chrono::minutes noise_min_epoch{CRYPTONOTE_NOISE_MIN_EPOCH}; constexpr const std::chrono::seconds noise_epoch_range{CRYPTONOTE_NOISE_EPOCH_RANGE}; + constexpr const std::chrono::minutes dandelionpp_min_epoch{CRYPTONOTE_DANDELIONPP_MIN_EPOCH}; + constexpr const std::chrono::seconds dandelionpp_epoch_range{CRYPTONOTE_DANDELIONPP_EPOCH_RANGE}; + constexpr const std::chrono::seconds noise_min_delay{CRYPTONOTE_NOISE_MIN_DELAY}; constexpr const std::chrono::seconds noise_delay_range{CRYPTONOTE_NOISE_DELAY_RANGE}; @@ -83,22 +89,8 @@ namespace levin connections (Dandelion++ makes similar assumptions in its stem algorithm). The randomization yields 95% values between 1s-4s in 1/4s increments. */ - constexpr const fluff_stepsize fluff_average_out{fluff_stepsize{fluff_average_in} / 2}; - - class random_poisson - { - std::poisson_distribution<fluff_stepsize::rep> dist; - public: - explicit random_poisson(fluff_stepsize average) - : dist(average.count() < 0 ? 0 : average.count()) - {} - - fluff_stepsize operator()() - { - crypto::random_device rand{}; - return fluff_stepsize{dist(rand)}; - } - }; + using fluff_duration = crypto::random_poisson_subseconds::result_type; + constexpr const fluff_duration fluff_average_out{fluff_duration{fluff_average_in} / 2}; /*! Select a randomized duration from 0 to `range`. The precision will be to the systems `steady_clock`. As an example, supplying 3 seconds to this @@ -132,10 +124,11 @@ namespace levin return outs; } - std::string make_tx_payload(std::vector<blobdata>&& txs, const bool pad) + std::string make_tx_payload(std::vector<blobdata>&& txs, const bool pad, const bool fluff) { NOTIFY_NEW_TRANSACTIONS::request request{}; request.txs = std::move(txs); + request.dandelionpp_fluff = fluff; if (pad) { @@ -172,9 +165,9 @@ namespace levin return fullBlob; } - bool make_payload_send_txs(connections& p2p, std::vector<blobdata>&& txs, const boost::uuids::uuid& destination, const bool pad) + bool make_payload_send_txs(connections& p2p, std::vector<blobdata>&& txs, const boost::uuids::uuid& destination, const bool pad, const bool fluff) { - const cryptonote::blobdata blob = make_tx_payload(std::move(txs), pad); + const cryptonote::blobdata blob = make_tx_payload(std::move(txs), pad, fluff); p2p.for_connection(destination, [&blob](detail::p2p_context& context) { on_levin_traffic(context, true, true, false, blob.size(), get_command_from_message(blob)); return true; @@ -251,7 +244,8 @@ namespace levin flush_time(std::chrono::steady_clock::time_point::max()), connection_count(0), is_public(is_public), - pad_txs(pad_txs) + pad_txs(pad_txs), + fluffing(false) { for (std::size_t count = 0; !noise.empty() && count < CRYPTONOTE_NOISE_CHANNELS; ++count) channels.emplace_back(io_service); @@ -268,6 +262,7 @@ namespace levin std::atomic<std::size_t> connection_count; //!< Only update in strand, can be read at any time const bool is_public; //!< Zone is public ipv4/ipv6 connections const bool pad_txs; //!< Pad txs to the next boundary for privacy + bool fluffing; //!< Zone is in Dandelion++ fluff epoch }; } // detail @@ -362,10 +357,11 @@ namespace levin return true; }); + // Always send txs in stem mode over i2p/tor, see comments in `send_txs` below. for (auto& connection : connections) { std::sort(connection.first.begin(), connection.first.end()); // don't leak receive order - make_payload_send_txs(*zone_->p2p, std::move(connection.first), connection.second, zone_->pad_txs); + make_payload_send_txs(*zone_->p2p, std::move(connection.first), connection.second, zone_->pad_txs, zone_->is_public); } if (next_flush != std::chrono::steady_clock::time_point::max()) @@ -387,29 +383,38 @@ namespace levin void operator()() { - if (!zone_ || !zone_->p2p || txs_.empty()) + run(std::move(zone_), epee::to_span(txs_), source_); + } + + static void run(std::shared_ptr<detail::zone> zone, epee::span<const blobdata> txs, const boost::uuids::uuid& source) + { + if (!zone || !zone->p2p || txs.empty()) return; - assert(zone_->strand.running_in_this_thread()); + assert(zone->strand.running_in_this_thread()); const auto now = std::chrono::steady_clock::now(); auto next_flush = std::chrono::steady_clock::time_point::max(); - random_poisson in_duration(fluff_average_in); - random_poisson out_duration(fluff_average_out); + crypto::random_poisson_subseconds in_duration(fluff_average_in); + crypto::random_poisson_subseconds out_duration(fluff_average_out); + + + MDEBUG("Queueing " << txs.size() << " transaction(s) for Dandelion++ fluffing"); bool available = false; - zone_->p2p->foreach_connection([this, now, &in_duration, &out_duration, &next_flush, &available] (detail::p2p_context& context) + zone->p2p->foreach_connection([txs, now, &zone, &source, &in_duration, &out_duration, &next_flush, &available] (detail::p2p_context& context) { - if (this->source_ != context.m_connection_id && (this->zone_->is_public || !context.m_is_income)) + // When i2p/tor, only fluff to outbound connections + if (source != context.m_connection_id && (zone->is_public || !context.m_is_income)) { available = true; if (context.fluff_txs.empty()) context.flush_time = now + (context.m_is_income ? in_duration() : out_duration()); next_flush = std::min(next_flush, context.flush_time); - context.fluff_txs.reserve(context.fluff_txs.size() + this->txs_.size()); - for (const blobdata& tx : this->txs_) + context.fluff_txs.reserve(context.fluff_txs.size() + txs.size()); + for (const blobdata& tx : txs) context.fluff_txs.push_back(tx); // must copy instead of move (multiple conns) } return true; @@ -418,8 +423,8 @@ namespace levin if (!available) MWARNING("Unable to send transaction(s), no available connections"); - if (next_flush < zone_->flush_time) - fluff_flush::queue(std::move(zone_), next_flush); + if (next_flush < zone->flush_time) + fluff_flush::queue(std::move(zone), next_flush); } }; @@ -471,6 +476,11 @@ namespace levin assert(zone->strand.running_in_this_thread()); zone->connection_count = zone->map.size(); + + // only noise uses the "noise channels", only update when enabled + if (zone->noise.empty()) + return; + for (auto id = zone->map.begin(); id != zone->map.end(); ++id) { const std::size_t i = id - zone->map.begin(); @@ -479,26 +489,75 @@ namespace levin } //! \pre Called within `zone_->strand`. + static void run(std::shared_ptr<detail::zone> zone, std::vector<boost::uuids::uuid> out_connections) + { + if (!zone) + return; + + assert(zone->strand.running_in_this_thread()); + if (zone->map.update(std::move(out_connections))) + post(std::move(zone)); + } + + //! \pre Called within `zone_->strand`. void operator()() { - if (!zone_) + run(std::move(zone_), std::move(out_connections_)); + } + }; + + //! Checks fluff status for this node, and then does stem or fluff for txes + struct dandelionpp_notify + { + std::shared_ptr<detail::zone> zone_; + i_core_events* core_; + std::vector<blobdata> txs_; + boost::uuids::uuid source_; + + //! \pre Called in `zone_->strand` + void operator()() + { + if (!zone_ || !core_ || txs_.empty()) return; - assert(zone_->strand.running_in_this_thread()); - if (zone_->map.update(std::move(out_connections_))) - post(std::move(zone_)); + if (zone_->fluffing) + { + core_->on_transactions_relayed(epee::to_span(txs_), relay_method::fluff); + fluff_notify::run(std::move(zone_), epee::to_span(txs_), source_); + } + else // forward tx in stem + { + core_->on_transactions_relayed(epee::to_span(txs_), relay_method::stem); + for (int tries = 2; 0 < tries; tries--) + { + const boost::uuids::uuid destination = zone_->map.get_stem(source_); + if (!destination.is_nil() && make_payload_send_txs(*zone_->p2p, std::vector<blobdata>{txs_}, destination, zone_->pad_txs, false)) + { + /* Source is intentionally omitted in debug log for privacy - a + nil uuid indicates source is that node. */ + MDEBUG("Sent " << txs_.size() << " transaction(s) to " << destination << " using Dandelion++ stem"); + return; + } + + // connection list may be outdated, try again + update_channels::run(zone_, get_out_connections(*zone_->p2p)); + } + + MERROR("Unable to send transaction(s) via Dandelion++ stem"); + } } }; - //! Swaps out noise channels entirely; new epoch start. + //! Swaps out noise/dandelionpp channels entirely; new epoch start. class change_channels { std::shared_ptr<detail::zone> zone_; net::dandelionpp::connection_map map_; // Requires manual copy constructor + bool fluffing_; public: - explicit change_channels(std::shared_ptr<detail::zone> zone, net::dandelionpp::connection_map map) - : zone_(std::move(zone)), map_(std::move(map)) + explicit change_channels(std::shared_ptr<detail::zone> zone, net::dandelionpp::connection_map map, const bool fluffing) + : zone_(std::move(zone)), map_(std::move(map)), fluffing_(fluffing) {} change_channels(change_channels&&) = default; @@ -510,11 +569,15 @@ namespace levin void operator()() { if (!zone_) - return + return; assert(zone_->strand.running_in_this_thread()); + if (zone_->is_public) + MDEBUG("Starting new Dandelion++ epoch: " << (fluffing_ ? "fluff" : "stem")); + zone_->map = std::move(map_); + zone_->fluffing = fluffing_; update_channels::post(std::move(zone_)); } }; @@ -608,9 +671,10 @@ namespace levin if (error && error != boost::system::errc::operation_canceled) throw boost::system::system_error{error, "start_epoch timer failed"}; + const bool fluffing = crypto::rand_idx(unsigned(100)) < CRYPTONOTE_DANDELIONPP_FLUFF_PROBABILITY; const auto start = std::chrono::steady_clock::now(); zone_->strand.dispatch( - change_channels{zone_, net::dandelionpp::connection_map{get_out_connections(*(zone_->p2p)), count_}} + change_channels{zone_, net::dandelionpp::connection_map{get_out_connections(*(zone_->p2p)), count_}, fluffing} ); detail::zone& alias = *zone_; @@ -626,10 +690,16 @@ namespace levin if (!zone_->p2p) throw std::logic_error{"cryptonote::levin::notify cannot have nullptr p2p argument"}; - if (!zone_->noise.empty()) + const bool noise_enabled = !zone_->noise.empty(); + if (noise_enabled || is_public) { const auto now = std::chrono::steady_clock::now(); - start_epoch{zone_, noise_min_epoch, noise_epoch_range, CRYPTONOTE_NOISE_CHANNELS}(); + const auto min_epoch = noise_enabled ? noise_min_epoch : dandelionpp_min_epoch; + const auto epoch_range = noise_enabled ? noise_epoch_range : dandelionpp_epoch_range; + const std::size_t out_count = noise_enabled ? CRYPTONOTE_NOISE_CHANNELS : CRYPTONOTE_DANDELIONPP_STEMS; + + start_epoch{zone_, min_epoch, epoch_range, out_count}(); + for (std::size_t channel = 0; channel < zone_->channels.size(); ++channel) send_noise::wait(now, zone_, channel); } @@ -679,7 +749,7 @@ namespace levin zone_->flush_txs.cancel(); } - bool notify::send_txs(std::vector<blobdata> txs, const boost::uuids::uuid& source) + bool notify::send_txs(std::vector<blobdata> txs, const boost::uuids::uuid& source, i_core_events& core, relay_method tx_relay) { if (txs.empty()) return true; @@ -687,6 +757,17 @@ namespace levin if (!zone_) return false; + /* If noise is enabled in a zone, it always takes precedence. The technique + provides good protection against ISP adversaries, but not sybil + adversaries. Noise is currently only enabled over I2P/Tor - those + networks provide protection against sybil attacks (we only send to + outgoing connections). + + If noise is disabled, Dandelion++ is used for public networks only. + Dandelion++ over I2P/Tor should be an interesting case to investigate, + but the mempool/stempool needs to know the zone a tx originated from to + work properly. */ + if (!zone_->noise.empty() && !zone_->channels.empty()) { // covert send in "noise" channel @@ -694,8 +775,17 @@ namespace levin CRYPTONOTE_MAX_FRAGMENTS * CRYPTONOTE_NOISE_BYTES <= LEVIN_DEFAULT_MAX_PACKET_SIZE, "most nodes will reject this fragment setting" ); - // padding is not useful when using noise mode - const std::string payload = make_tx_payload(std::move(txs), false); + if (tx_relay == relay_method::stem) + { + MWARNING("Dandelion++ stem not supported over noise networks"); + tx_relay = relay_method::local; // do not put into stempool embargo (hopefully not there already!). + } + + core.on_transactions_relayed(epee::to_span(txs), tx_relay); + + // Padding is not useful when using noise mode. Send as stem so receiver + // forwards in Dandelion++ mode. + const std::string payload = make_tx_payload(std::move(txs), false, false); epee::byte_slice message = epee::levin::make_fragmented_notify( zone_->noise, NOTIFY_NEW_TRANSACTIONS::ID, epee::strspan<std::uint8_t>(payload) ); @@ -714,9 +804,31 @@ namespace levin } else { - zone_->strand.dispatch(fluff_notify{zone_, std::move(txs), source}); + switch (tx_relay) + { + default: + case relay_method::none: + case relay_method::block: + return false; + case relay_method::stem: + tx_relay = relay_method::fluff; // don't set stempool embargo when skipping to fluff + /* fallthrough */ + case relay_method::local: + if (zone_->is_public) + { + // this will change a local tx to stem or fluff ... + zone_->strand.dispatch( + dandelionpp_notify{zone_, std::addressof(core), std::move(txs), source} + ); + break; + } + /* fallthrough */ + case relay_method::fluff: + core.on_transactions_relayed(epee::to_span(txs), tx_relay); + zone_->strand.dispatch(fluff_notify{zone_, std::move(txs), source}); + break; + } } - return true; } } // levin diff --git a/src/cryptonote_protocol/levin_notify.h b/src/cryptonote_protocol/levin_notify.h index ce652d933..641f1f956 100644 --- a/src/cryptonote_protocol/levin_notify.h +++ b/src/cryptonote_protocol/levin_notify.h @@ -35,6 +35,7 @@ #include "byte_slice.h" #include "cryptonote_basic/blobdatatype.h" +#include "cryptonote_protocol/enums.h" #include "cryptonote_protocol/fwd.h" #include "net/enums.h" #include "span.h" @@ -122,7 +123,7 @@ namespace levin particular stem. \return True iff the notification is queued for sending. */ - bool send_txs(std::vector<blobdata> txs, const boost::uuids::uuid& source); + bool send_txs(std::vector<blobdata> txs, const boost::uuids::uuid& source, i_core_events& core, relay_method tx_relay); }; } // levin } // net diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index 31d8aad3f..5337106dd 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -334,7 +334,7 @@ namespace nodetool virtual void callback(p2p_connection_context& context); //----------------- i_p2p_endpoint ------------------------------------------------------------- virtual bool relay_notify_to_list(int command, const epee::span<const uint8_t> data_buff, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections); - virtual epee::net_utils::zone send_txs(std::vector<cryptonote::blobdata> txs, const epee::net_utils::zone origin, const boost::uuids::uuid& source, cryptonote::i_core_events& core); + virtual epee::net_utils::zone send_txs(std::vector<cryptonote::blobdata> txs, const epee::net_utils::zone origin, const boost::uuids::uuid& source, cryptonote::i_core_events& core, cryptonote::relay_method tx_relay); virtual bool invoke_command_to_peer(int command, const epee::span<const uint8_t> req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context); virtual bool invoke_notify_to_peer(int command, const epee::span<const uint8_t> req_buff, const epee::net_utils::connection_context_base& context); virtual bool drop_connection(const epee::net_utils::connection_context_base& context); diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index 09058c76a..4c253b673 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -1975,18 +1975,13 @@ namespace nodetool } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - epee::net_utils::zone node_server<t_payload_net_handler>::send_txs(std::vector<cryptonote::blobdata> txs, const epee::net_utils::zone origin, const boost::uuids::uuid& source, cryptonote::i_core_events& core) + epee::net_utils::zone node_server<t_payload_net_handler>::send_txs(std::vector<cryptonote::blobdata> txs, const epee::net_utils::zone origin, const boost::uuids::uuid& source, cryptonote::i_core_events& core, const cryptonote::relay_method tx_relay) { namespace enet = epee::net_utils; - const auto send = [&txs, &source, &core] (std::pair<const enet::zone, network_zone>& network) + const auto send = [&txs, &source, &core, tx_relay] (std::pair<const enet::zone, network_zone>& network) { - const bool is_public = (network.first == enet::zone::public_); - const cryptonote::relay_method tx_relay = is_public ? - cryptonote::relay_method::fluff : cryptonote::relay_method::local; - - core.on_transactions_relayed(epee::to_span(txs), tx_relay); - if (network.second.m_notifier.send_txs(std::move(txs), source)) + if (network.second.m_notifier.send_txs(std::move(txs), source, core, tx_relay)) return network.first; return enet::zone::invalid; }; diff --git a/src/p2p/net_node_common.h b/src/p2p/net_node_common.h index ed88aa28c..6a6100e0c 100644 --- a/src/p2p/net_node_common.h +++ b/src/p2p/net_node_common.h @@ -50,7 +50,7 @@ namespace nodetool struct i_p2p_endpoint { virtual bool relay_notify_to_list(int command, const epee::span<const uint8_t> data_buff, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections)=0; - virtual epee::net_utils::zone send_txs(std::vector<cryptonote::blobdata> txs, const epee::net_utils::zone origin, const boost::uuids::uuid& source, cryptonote::i_core_events& core)=0; + virtual epee::net_utils::zone send_txs(std::vector<cryptonote::blobdata> txs, const epee::net_utils::zone origin, const boost::uuids::uuid& source, cryptonote::i_core_events& core, cryptonote::relay_method tx_relay)=0; virtual bool invoke_command_to_peer(int command, const epee::span<const uint8_t> req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context)=0; virtual bool invoke_notify_to_peer(int command, const epee::span<const uint8_t> req_buff, const epee::net_utils::connection_context_base& context)=0; virtual bool drop_connection(const epee::net_utils::connection_context_base& context)=0; @@ -75,7 +75,7 @@ namespace nodetool { return false; } - virtual epee::net_utils::zone send_txs(std::vector<cryptonote::blobdata> txs, const epee::net_utils::zone origin, const boost::uuids::uuid& source, cryptonote::i_core_events& core) + virtual epee::net_utils::zone send_txs(std::vector<cryptonote::blobdata> txs, const epee::net_utils::zone origin, const boost::uuids::uuid& source, cryptonote::i_core_events& core, cryptonote::relay_method tx_relay) { return epee::net_utils::zone::invalid; } diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index f097c93fa..1fd0c037b 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -1152,7 +1152,7 @@ namespace cryptonote return true; } - if(!tvc.m_should_be_relayed) + if(tvc.m_relay == relay_method::none) { LOG_PRINT_L0("[on_send_raw_tx]: tx accepted, but not relayed"); res.reason = "Not relayed"; @@ -1162,8 +1162,8 @@ namespace cryptonote } NOTIFY_NEW_TRANSACTIONS::request r; - r.txs.push_back(tx_blob); - m_core.get_protocol()->relay_transactions(r, boost::uuids::nil_uuid(), epee::net_utils::zone::invalid); + r.txs.push_back(std::move(tx_blob)); + m_core.get_protocol()->relay_transactions(r, boost::uuids::nil_uuid(), epee::net_utils::zone::invalid, relay_method::local); //TODO: make sure that tx has reached other nodes here, probably wait to receive reflections from other nodes res.status = CORE_RPC_STATUS_OK; return true; @@ -2776,8 +2776,8 @@ namespace cryptonote if (!m_core.get_pool_transaction(txid, txblob, relay_category::legacy)) { NOTIFY_NEW_TRANSACTIONS::request r; - r.txs.push_back(txblob); - m_core.get_protocol()->relay_transactions(r, boost::uuids::nil_uuid(), epee::net_utils::zone::invalid); + r.txs.push_back(std::move(txblob)); + m_core.get_protocol()->relay_transactions(r, boost::uuids::nil_uuid(), epee::net_utils::zone::invalid, relay_method::local); //TODO: make sure that tx has reached other nodes here, probably wait to receive reflections from other nodes } else diff --git a/src/rpc/daemon_handler.cpp b/src/rpc/daemon_handler.cpp index 7292176b4..d05854e34 100644 --- a/src/rpc/daemon_handler.cpp +++ b/src/rpc/daemon_handler.cpp @@ -349,10 +349,10 @@ namespace rpc res.error_details = "Invalid hex"; return; } - handleTxBlob(tx_blob, req.relay, res); + handleTxBlob(std::move(tx_blob), req.relay, res); } - void DaemonHandler::handleTxBlob(const std::string& tx_blob, bool relay, SendRawTx::Response& res) + void DaemonHandler::handleTxBlob(std::string&& tx_blob, bool relay, SendRawTx::Response& res) { if (!m_p2p.get_payload_object().is_synchronized()) { @@ -423,7 +423,7 @@ namespace rpc return; } - if(!tvc.m_should_be_relayed || !relay) + if(tvc.m_relay == relay_method::none || !relay) { MERROR("[SendRawTx]: tx accepted, but not relayed"); res.error_details = "Not relayed"; @@ -434,8 +434,8 @@ namespace rpc } NOTIFY_NEW_TRANSACTIONS::request r; - r.txs.push_back(tx_blob); - m_core.get_protocol()->relay_transactions(r, boost::uuids::nil_uuid(), epee::net_utils::zone::invalid); + r.txs.push_back(std::move(tx_blob)); + m_core.get_protocol()->relay_transactions(r, boost::uuids::nil_uuid(), epee::net_utils::zone::invalid, relay_method::local); //TODO: make sure that tx has reached other nodes here, probably wait to receive reflections from other nodes res.status = Message::STATUS_OK; diff --git a/src/rpc/daemon_handler.h b/src/rpc/daemon_handler.h index c33f608ab..61eac17f0 100644 --- a/src/rpc/daemon_handler.h +++ b/src/rpc/daemon_handler.h @@ -138,7 +138,7 @@ class DaemonHandler : public RpcHandler bool getBlockHeaderByHash(const crypto::hash& hash_in, cryptonote::rpc::BlockHeaderResponse& response); - void handleTxBlob(const std::string& tx_blob, bool relay, SendRawTx::Response& res); + void handleTxBlob(std::string&& tx_blob, bool relay, SendRawTx::Response& res); cryptonote::core& m_core; t_p2p& m_p2p; diff --git a/src/wallet/message_store.cpp b/src/wallet/message_store.cpp index 6e2cb933f..1bd462ef5 100644 --- a/src/wallet/message_store.cpp +++ b/src/wallet/message_store.cpp @@ -48,7 +48,7 @@ namespace mms { -message_store::message_store() +message_store::message_store(std::unique_ptr<epee::net_utils::http::abstract_http_client> http_client) : m_transporter(std::move(http_client)) { m_active = false; m_auto_send = false; diff --git a/src/wallet/message_store.h b/src/wallet/message_store.h index 637bd29a1..d40daf186 100644 --- a/src/wallet/message_store.h +++ b/src/wallet/message_store.h @@ -43,6 +43,7 @@ #include "common/i18n.h" #include "common/command_line.h" #include "wipeable_string.h" +#include "net/abstract_http_client.h" #include "message_transporter.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -202,7 +203,8 @@ namespace mms class message_store { public: - message_store(); + message_store(std::unique_ptr<epee::net_utils::http::abstract_http_client> http_client); + // Initialize and start to use the MMS, set the first signer, this wallet itself // Filename, if not null and not empty, is used to create the ".mms" file // reset it if already used, with deletion of all signers and messages diff --git a/src/wallet/message_transporter.cpp b/src/wallet/message_transporter.cpp index cf9b45b37..4dd4b8f01 100644 --- a/src/wallet/message_transporter.cpp +++ b/src/wallet/message_transporter.cpp @@ -80,7 +80,7 @@ namespace bitmessage_rpc } -message_transporter::message_transporter() +message_transporter::message_transporter(std::unique_ptr<epee::net_utils::http::abstract_http_client> http_client) : m_http_client(std::move(http_client)) { m_run = true; } @@ -96,7 +96,7 @@ void message_transporter::set_options(const std::string &bitmessage_address, con } m_bitmessage_login = bitmessage_login; - m_http_client.set_server(address_parts.host, std::to_string(address_parts.port), boost::none); + m_http_client->set_server(address_parts.host, std::to_string(address_parts.port), boost::none); } bool message_transporter::receive_messages(const std::vector<std::string> &destination_transport_addresses, @@ -256,7 +256,7 @@ bool message_transporter::post_request(const std::string &request, std::string & additional_params.push_back(std::make_pair("Content-Type", "application/xml; charset=utf-8")); const epee::net_utils::http::http_response_info* response = NULL; std::chrono::milliseconds timeout = std::chrono::seconds(15); - bool r = m_http_client.invoke("/", "POST", request, timeout, std::addressof(response), std::move(additional_params)); + bool r = m_http_client->invoke("/", "POST", request, timeout, std::addressof(response), std::move(additional_params)); if (r) { answer = response->m_body; @@ -266,7 +266,7 @@ bool message_transporter::post_request(const std::string &request, std::string & LOG_ERROR("POST request to Bitmessage failed: " << request.substr(0, 300)); THROW_WALLET_EXCEPTION(tools::error::no_connection_to_bitmessage, m_bitmessage_url); } - m_http_client.disconnect(); // see comment above + m_http_client->disconnect(); // see comment above std::string string_value = get_str_between_tags(answer, "<string>", "</string>"); if ((string_value.find("API Error") == 0) || (string_value.find("RPC ") == 0)) { diff --git a/src/wallet/message_transporter.h b/src/wallet/message_transporter.h index 28c099d87..84a2e9bae 100644 --- a/src/wallet/message_transporter.h +++ b/src/wallet/message_transporter.h @@ -34,9 +34,9 @@ #include "cryptonote_basic/cryptonote_basic.h" #include "net/http_server_impl_base.h" #include "net/http_client.h" +#include "net/abstract_http_client.h" #include "common/util.h" #include "wipeable_string.h" -#include "serialization/keyvalue_serialization.h" #include <vector> namespace mms @@ -83,7 +83,7 @@ typedef epee::misc_utils::struct_init<transport_message_t> transport_message; class message_transporter { public: - message_transporter(); + message_transporter(std::unique_ptr<epee::net_utils::http::abstract_http_client> http_client); void set_options(const std::string &bitmessage_address, const epee::wipeable_string &bitmessage_login); bool send_message(const transport_message &message); bool receive_messages(const std::vector<std::string> &destination_transport_addresses, @@ -94,7 +94,7 @@ public: bool delete_transport_address(const std::string &transport_address); private: - epee::net_utils::http::http_simple_client m_http_client; + const std::unique_ptr<epee::net_utils::http::abstract_http_client> m_http_client; std::string m_bitmessage_url; epee::wipeable_string m_bitmessage_login; std::atomic<bool> m_run; diff --git a/src/wallet/node_rpc_proxy.cpp b/src/wallet/node_rpc_proxy.cpp index f3698b599..873c2ee51 100644 --- a/src/wallet/node_rpc_proxy.cpp +++ b/src/wallet/node_rpc_proxy.cpp @@ -51,7 +51,7 @@ namespace tools static const std::chrono::seconds rpc_timeout = std::chrono::minutes(3) + std::chrono::seconds(30); -NodeRPCProxy::NodeRPCProxy(epee::net_utils::http::http_simple_client &http_client, rpc_payment_state_t &rpc_payment_state, boost::recursive_mutex &mutex) +NodeRPCProxy::NodeRPCProxy(epee::net_utils::http::abstract_http_client &http_client, rpc_payment_state_t &rpc_payment_state, boost::recursive_mutex &mutex) : m_http_client(http_client) , m_rpc_payment_state(rpc_payment_state) , m_daemon_rpc_mutex(mutex) diff --git a/src/wallet/node_rpc_proxy.h b/src/wallet/node_rpc_proxy.h index 65ca40640..b053659e9 100644 --- a/src/wallet/node_rpc_proxy.h +++ b/src/wallet/node_rpc_proxy.h @@ -31,7 +31,7 @@ #include <string> #include <boost/thread/mutex.hpp> #include "include_base_utils.h" -#include "net/http_client.h" +#include "net/abstract_http_client.h" #include "rpc/core_rpc_server_commands_defs.h" #include "wallet_rpc_helpers.h" @@ -41,7 +41,7 @@ namespace tools class NodeRPCProxy { public: - NodeRPCProxy(epee::net_utils::http::http_simple_client &http_client, rpc_payment_state_t &rpc_payment_state, boost::recursive_mutex &mutex); + NodeRPCProxy(epee::net_utils::http::abstract_http_client &http_client, rpc_payment_state_t &rpc_payment_state, boost::recursive_mutex &mutex); void set_client_secret_key(const crypto::secret_key &skey) { m_client_id_secret_key = skey; } void invalidate(); @@ -72,7 +72,7 @@ private: private: boost::optional<std::string> get_info(); - epee::net_utils::http::http_simple_client &m_http_client; + epee::net_utils::http::abstract_http_client &m_http_client; rpc_payment_state_t &m_rpc_payment_state; boost::recursive_mutex &m_daemon_rpc_mutex; crypto::secret_key m_client_id_secret_key; diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index bc8219c69..4220f18be 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -1122,7 +1122,8 @@ void wallet_device_callback::on_progress(const hw::device_progress& event) wallet->on_device_progress(event); } -wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended): +wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended, std::unique_ptr<epee::net_utils::http::http_client_factory> http_client_factory): + m_http_client(std::move(http_client_factory->create())), m_multisig_rescan_info(NULL), m_multisig_rescan_k(NULL), m_upper_transaction_weight_limit(0), @@ -1167,7 +1168,7 @@ wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended): m_watch_only(false), m_multisig(false), m_multisig_threshold(0), - m_node_rpc_proxy(m_http_client, m_rpc_payment_state, m_daemon_rpc_mutex), + m_node_rpc_proxy(*m_http_client, m_rpc_payment_state, m_daemon_rpc_mutex), m_account_public_address{crypto::null_pkey, crypto::null_pkey}, m_subaddress_lookahead_major(SUBADDRESS_LOOKAHEAD_MAJOR), m_subaddress_lookahead_minor(SUBADDRESS_LOOKAHEAD_MINOR), @@ -1178,7 +1179,7 @@ wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended): m_light_wallet_balance(0), m_light_wallet_unlocked_balance(0), m_original_keys_available(false), - m_message_store(), + m_message_store(http_client_factory->create()), m_key_device_type(hw::device::device_type::SOFTWARE), m_ring_history_saved(false), m_ringdb(), @@ -1298,8 +1299,8 @@ bool wallet2::set_daemon(std::string daemon_address, boost::optional<epee::net_u { boost::lock_guard<boost::recursive_mutex> lock(m_daemon_rpc_mutex); - if(m_http_client.is_connected()) - m_http_client.disconnect(); + if(m_http_client->is_connected()) + m_http_client->disconnect(); const bool changed = m_daemon_address != daemon_address; m_daemon_address = std::move(daemon_address); m_daemon_login = std::move(daemon_login); @@ -1313,7 +1314,7 @@ bool wallet2::set_daemon(std::string daemon_address, boost::optional<epee::net_u const std::string address = get_daemon_address(); MINFO("setting daemon to " << address); - bool ret = m_http_client.set_server(address, get_daemon_login(), std::move(ssl_options)); + bool ret = m_http_client->set_server(address, get_daemon_login(), std::move(ssl_options)); if (ret) { CRITICAL_REGION_LOCAL(default_daemon_address_lock); @@ -1328,7 +1329,12 @@ bool wallet2::init(std::string daemon_address, boost::optional<epee::net_utils:: m_is_initialized = true; m_upper_transaction_weight_limit = upper_transaction_weight_limit; if (proxy != boost::asio::ip::tcp::endpoint{}) - m_http_client.set_connector(net::socks::connector{std::move(proxy)}); + { + epee::net_utils::http::abstract_http_client* abstract_http_client = m_http_client.get(); + epee::net_utils::http::http_simple_client* http_simple_client = dynamic_cast<epee::net_utils::http::http_simple_client*>(abstract_http_client); + CHECK_AND_ASSERT_MES(http_simple_client != nullptr, false, "http_simple_client must be used to set proxy"); + http_simple_client->set_connector(net::socks::connector{std::move(proxy)}); + } return set_daemon(daemon_address, daemon_login, trusted_daemon, std::move(ssl_options)); } //---------------------------------------------------------------------------------------------------- @@ -2593,7 +2599,7 @@ void wallet2::pull_blocks(uint64_t start_height, uint64_t &blocks_start_height, const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - bool r = net_utils::invoke_http_bin("/getblocks.bin", req, res, m_http_client, rpc_timeout); + bool r = net_utils::invoke_http_bin("/getblocks.bin", req, res, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR(r, {}, res, "getblocks.bin", error::get_blocks_error, get_rpc_status(res.status)); THROW_WALLET_EXCEPTION_IF(res.blocks.size() != res.output_indices.size(), error::wallet_internal_error, "mismatched blocks (" + boost::lexical_cast<std::string>(res.blocks.size()) + ") and output_indices (" + @@ -2622,7 +2628,7 @@ void wallet2::pull_hashes(uint64_t start_height, uint64_t &blocks_start_height, const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; req.client = get_client_signature(); uint64_t pre_call_credits = m_rpc_payment_state.credits; - bool r = net_utils::invoke_http_bin("/gethashes.bin", req, res, m_http_client, rpc_timeout); + bool r = net_utils::invoke_http_bin("/gethashes.bin", req, res, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR(r, {}, res, "gethashes.bin", error::get_hashes_error, get_rpc_status(res.status)); check_rpc_cost("/gethashes.bin", res.credits, pre_call_credits, 1 + res.m_block_ids.size() * COST_PER_BLOCK_HASH); } @@ -2907,7 +2913,7 @@ void wallet2::update_pool_state(std::vector<std::tuple<cryptonote::transaction, const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - bool r = epee::net_utils::invoke_http_json("/get_transaction_pool_hashes.bin", req, res, m_http_client, rpc_timeout); + bool r = epee::net_utils::invoke_http_json("/get_transaction_pool_hashes.bin", req, res, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR(r, {}, res, "get_transaction_pool_hashes.bin", error::get_tx_pool_error); check_rpc_cost("/get_transaction_pool_hashes.bin", res.credits, pre_call_credits, 1 + res.tx_hashes.size() * COST_PER_POOL_HASH); } @@ -3052,7 +3058,7 @@ void wallet2::update_pool_state(std::vector<std::tuple<cryptonote::transaction, const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - r = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client, rpc_timeout); + r = epee::net_utils::invoke_http_json("/gettransactions", req, res, *m_http_client, rpc_timeout); if (r && res.status == CORE_RPC_STATUS_OK) check_rpc_cost("/gettransactions", res.credits, pre_call_credits, res.txs.size() * COST_PER_TX); } @@ -3538,7 +3544,7 @@ bool wallet2::get_rct_distribution(uint64_t &start_height, std::vector<uint64_t> const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - r = net_utils::invoke_http_bin("/get_output_distribution.bin", req, res, m_http_client, rpc_timeout); + r = net_utils::invoke_http_bin("/get_output_distribution.bin", req, res, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR_GENERIC(r, {}, res, "/get_output_distribution.bin"); check_rpc_cost("/get_output_distribution.bin", res.credits, pre_call_credits, COST_PER_OUTPUT_DISTRIBUTION_0); } @@ -3697,6 +3703,30 @@ void wallet2::clear_soft(bool keep_key_images) */ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable_string& password, bool watch_only) { + boost::optional<wallet2::keys_file_data> keys_file_data = get_keys_file_data(password, watch_only); + CHECK_AND_ASSERT_MES(keys_file_data != boost::none, false, "failed to generate wallet keys data"); + + std::string tmp_file_name = keys_file_name + ".new"; + std::string buf; + bool r = ::serialization::dump_binary(keys_file_data.get(), buf); + r = r && save_to_file(tmp_file_name, buf); + CHECK_AND_ASSERT_MES(r, false, "failed to generate wallet keys file " << tmp_file_name); + + unlock_keys_file(); + std::error_code e = tools::replace_file(tmp_file_name, keys_file_name); + lock_keys_file(); + + if (e) { + boost::filesystem::remove(tmp_file_name); + LOG_ERROR("failed to update wallet keys file " << keys_file_name); + return false; + } + + return true; +} +//---------------------------------------------------------------------------------------------------- +boost::optional<wallet2::keys_file_data> wallet2::get_keys_file_data(const epee::wipeable_string& password, bool watch_only) +{ std::string account_data; std::string multisig_signers; std::string multisig_derivations; @@ -3717,8 +3747,8 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable account.encrypt_keys(key); bool r = epee::serialization::store_t_to_binary(account, account_data); - CHECK_AND_ASSERT_MES(r, false, "failed to serialize wallet keys"); - wallet2::keys_file_data keys_file_data = {}; + CHECK_AND_ASSERT_MES(r, boost::none, "failed to serialize wallet keys"); + boost::optional<wallet2::keys_file_data> keys_file_data = (wallet2::keys_file_data) {}; // Create a JSON object with "key_data" and "seed_language" as keys. rapidjson::Document json; @@ -3749,12 +3779,12 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable if (m_multisig) { bool r = ::serialization::dump_binary(m_multisig_signers, multisig_signers); - CHECK_AND_ASSERT_MES(r, false, "failed to serialize wallet multisig signers"); + CHECK_AND_ASSERT_MES(r, boost::none, "failed to serialize wallet multisig signers"); value.SetString(multisig_signers.c_str(), multisig_signers.length()); json.AddMember("multisig_signers", value, json.GetAllocator()); r = ::serialization::dump_binary(m_multisig_derivations, multisig_derivations); - CHECK_AND_ASSERT_MES(r, false, "failed to serialize wallet multisig derivations"); + CHECK_AND_ASSERT_MES(r, boost::none, "failed to serialize wallet multisig derivations"); value.SetString(multisig_derivations.c_str(), multisig_derivations.length()); json.AddMember("multisig_derivations", value, json.GetAllocator()); @@ -3897,27 +3927,10 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable // Encrypt the entire JSON object. std::string cipher; cipher.resize(account_data.size()); - keys_file_data.iv = crypto::rand<crypto::chacha_iv>(); - crypto::chacha20(account_data.data(), account_data.size(), key, keys_file_data.iv, &cipher[0]); - keys_file_data.account_data = cipher; - - std::string tmp_file_name = keys_file_name + ".new"; - std::string buf; - r = ::serialization::dump_binary(keys_file_data, buf); - r = r && save_to_file(tmp_file_name, buf); - CHECK_AND_ASSERT_MES(r, false, "failed to generate wallet keys file " << tmp_file_name); - - unlock_keys_file(); - std::error_code e = tools::replace_file(tmp_file_name, keys_file_name); - lock_keys_file(); - - if (e) { - boost::filesystem::remove(tmp_file_name); - LOG_ERROR("failed to update wallet keys file " << keys_file_name); - return false; - } - - return true; + keys_file_data.get().iv = crypto::rand<crypto::chacha_iv>(); + crypto::chacha20(account_data.data(), account_data.size(), key, keys_file_data.get().iv, &cipher[0]); + keys_file_data.get().account_data = cipher; + return keys_file_data; } //---------------------------------------------------------------------------------------------------- void wallet2::setup_keys(const epee::wipeable_string &password) @@ -3957,16 +3970,51 @@ void wallet2::change_password(const std::string &filename, const epee::wipeable_ */ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_string& password) { - rapidjson::Document json; - wallet2::keys_file_data keys_file_data; - std::string buf; - bool encrypted_secret_keys = false; - bool r = load_from_file(keys_file_name, buf); + std::string keys_file_buf; + bool r = load_from_file(keys_file_name, keys_file_buf); THROW_WALLET_EXCEPTION_IF(!r, error::file_read_error, keys_file_name); + // Load keys from buffer + boost::optional<crypto::chacha_key> keys_to_encrypt; + try { + r = wallet2::load_keys_buf(keys_file_buf, password, keys_to_encrypt); + } catch (const std::exception& e) { + std::size_t found = string(e.what()).find("failed to deserialize keys buffer"); + THROW_WALLET_EXCEPTION_IF(found != std::string::npos, error::wallet_internal_error, "internal error: failed to deserialize \"" + keys_file_name + '\"'); + throw e; + } + + // Rewrite with encrypted keys if unencrypted, ignore errors + if (r && keys_to_encrypt != boost::none) + { + if (m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only) + encrypt_keys(keys_to_encrypt.get()); + bool saved_ret = store_keys(keys_file_name, password, m_watch_only); + if (!saved_ret) + { + // just moan a bit, but not fatal + MERROR("Error saving keys file with encrypted keys, not fatal"); + } + if (m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only) + decrypt_keys(keys_to_encrypt.get()); + m_keys_file_locker.reset(); + } + return r; +} +//---------------------------------------------------------------------------------------------------- +bool wallet2::load_keys_buf(const std::string& keys_buf, const epee::wipeable_string& password) { + boost::optional<crypto::chacha_key> keys_to_encrypt; + return wallet2::load_keys_buf(keys_buf, password, keys_to_encrypt); +} +//---------------------------------------------------------------------------------------------------- +bool wallet2::load_keys_buf(const std::string& keys_buf, const epee::wipeable_string& password, boost::optional<crypto::chacha_key>& keys_to_encrypt) { + // Decrypt the contents - r = ::serialization::parse_binary(buf, keys_file_data); - THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "internal error: failed to deserialize \"" + keys_file_name + '\"'); + rapidjson::Document json; + wallet2::keys_file_data keys_file_data; + bool encrypted_secret_keys = false; + bool r = ::serialization::parse_binary(keys_buf, keys_file_data); + THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "internal error: failed to deserialize keys buffer"); crypto::chacha_key key; crypto::generate_chacha_key(password.data(), password.size(), key, m_kdf_rounds); std::string account_data; @@ -4250,8 +4298,8 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ } else { - THROW_WALLET_EXCEPTION(error::wallet_internal_error, "invalid password"); - return false; + THROW_WALLET_EXCEPTION(error::wallet_internal_error, "invalid password"); + return false; } r = epee::serialization::load_t_from_binary(m_account, account_data); @@ -4285,24 +4333,13 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ } else { - // rewrite with encrypted keys, ignore errors - if (m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only) - encrypt_keys(key); - bool saved_ret = store_keys(keys_file_name, password, m_watch_only); - if (!saved_ret) - { - // just moan a bit, but not fatal - MERROR("Error saving keys file with encrypted keys, not fatal"); - } - if (m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only) - decrypt_keys(key); - m_keys_file_locker.reset(); + keys_to_encrypt = key; } } const cryptonote::account_keys& keys = m_account.get_keys(); hw::device &hwdev = m_account.get_device(); r = r && hwdev.verify_keys(keys.m_view_secret_key, keys.m_account_address.m_view_public_key); - if(!m_watch_only && !m_multisig && hwdev.device_protocol() != hw::device::PROTOCOL_COLD) + if (!m_watch_only && !m_multisig && hwdev.device_protocol() != hw::device::PROTOCOL_COLD) r = r && hwdev.verify_keys(keys.m_spend_secret_key, keys.m_account_address.m_spend_public_key); THROW_WALLET_EXCEPTION_IF(!r, error::wallet_files_doesnt_correspond, m_keys_file, m_wallet_file); @@ -4921,7 +4958,8 @@ std::string wallet2::make_multisig(const epee::wipeable_string &password, // re-encrypt keys keys_reencryptor = epee::misc_utils::auto_scope_leave_caller(); - create_keys_file(m_wallet_file, false, password, boost::filesystem::exists(m_wallet_file + ".address.txt")); + if (!m_wallet_file.empty()) + create_keys_file(m_wallet_file, false, password, boost::filesystem::exists(m_wallet_file + ".address.txt")); setup_new_blockchain(); @@ -5061,7 +5099,9 @@ std::string wallet2::exchange_multisig_keys(const epee::wipeable_string &passwor ++m_multisig_rounds_passed; - create_keys_file(m_wallet_file, false, password, boost::filesystem::exists(m_wallet_file + ".address.txt")); + if (!m_wallet_file.empty()) + create_keys_file(m_wallet_file, false, password, boost::filesystem::exists(m_wallet_file + ".address.txt")); + return extra_multisig_info; } @@ -5435,13 +5475,13 @@ bool wallet2::check_connection(uint32_t *version, bool *ssl, uint32_t timeout) { boost::lock_guard<boost::recursive_mutex> lock(m_daemon_rpc_mutex); - if(!m_http_client.is_connected(ssl)) + if(!m_http_client->is_connected(ssl)) { m_rpc_version = 0; m_node_rpc_proxy.invalidate(); - if (!m_http_client.connect(std::chrono::milliseconds(timeout))) + if (!m_http_client->connect(std::chrono::milliseconds(timeout))) return false; - if(!m_http_client.is_connected(ssl)) + if(!m_http_client->is_connected(ssl)) return false; } } @@ -5469,12 +5509,12 @@ void wallet2::set_offline(bool offline) { m_offline = offline; m_node_rpc_proxy.set_offline(offline); - m_http_client.set_auto_connect(!offline); + m_http_client->set_auto_connect(!offline); if (offline) { boost::lock_guard<boost::recursive_mutex> lock(m_daemon_rpc_mutex); - if(m_http_client.is_connected()) - m_http_client.disconnect(); + if(m_http_client->is_connected()) + m_http_client->disconnect(); } } //---------------------------------------------------------------------------------------------------- @@ -5489,48 +5529,63 @@ void wallet2::generate_chacha_key_from_password(const epee::wipeable_string &pas crypto::generate_chacha_key(pass.data(), pass.size(), key, m_kdf_rounds); } //---------------------------------------------------------------------------------------------------- -void wallet2::load(const std::string& wallet_, const epee::wipeable_string& password) +void wallet2::load(const std::string& wallet_, const epee::wipeable_string& password, const std::string& keys_buf, const std::string& cache_buf) { clear(); prepare_file_names(wallet_); + // determine if loading from file system or string buffer + bool use_fs = !wallet_.empty(); + THROW_WALLET_EXCEPTION_IF((use_fs && !keys_buf.empty()) || (!use_fs && keys_buf.empty()), error::file_read_error, "must load keys either from file system or from buffer");\ + boost::system::error_code e; - bool exists = boost::filesystem::exists(m_keys_file, e); - THROW_WALLET_EXCEPTION_IF(e || !exists, error::file_not_found, m_keys_file); - lock_keys_file(); - THROW_WALLET_EXCEPTION_IF(!is_keys_file_locked(), error::wallet_internal_error, "internal error: \"" + m_keys_file + "\" is opened by another wallet program"); + if (use_fs) + { + bool exists = boost::filesystem::exists(m_keys_file, e); + THROW_WALLET_EXCEPTION_IF(e || !exists, error::file_not_found, m_keys_file); + lock_keys_file(); + THROW_WALLET_EXCEPTION_IF(!is_keys_file_locked(), error::wallet_internal_error, "internal error: \"" + m_keys_file + "\" is opened by another wallet program"); - // this temporary unlocking is necessary for Windows (otherwise the file couldn't be loaded). - unlock_keys_file(); - if (!load_keys(m_keys_file, password)) + // this temporary unlocking is necessary for Windows (otherwise the file couldn't be loaded). + unlock_keys_file(); + if (!load_keys(m_keys_file, password)) + { + THROW_WALLET_EXCEPTION_IF(true, error::file_read_error, m_keys_file); + } + LOG_PRINT_L0("Loaded wallet keys file, with public address: " << m_account.get_public_address_str(m_nettype)); + lock_keys_file(); + } + else if (!load_keys_buf(keys_buf, password)) { - THROW_WALLET_EXCEPTION_IF(true, error::file_read_error, m_keys_file); + THROW_WALLET_EXCEPTION_IF(true, error::file_read_error, "failed to load keys from buffer"); } - LOG_PRINT_L0("Loaded wallet keys file, with public address: " << m_account.get_public_address_str(m_nettype)); - lock_keys_file(); wallet_keys_unlocker unlocker(*this, m_ask_password == AskPasswordToDecrypt && !m_unattended && !m_watch_only, password); //keys loaded ok! //try to load wallet file. but even if we failed, it is not big problem - if(!boost::filesystem::exists(m_wallet_file, e) || e) + if (use_fs && (!boost::filesystem::exists(m_wallet_file, e) || e)) { LOG_PRINT_L0("file not found: " << m_wallet_file << ", starting with empty blockchain"); m_account_public_address = m_account.get_keys().m_account_address; } - else + else if (use_fs || !cache_buf.empty()) { wallet2::cache_file_data cache_file_data; - std::string buf; - bool r = load_from_file(m_wallet_file, buf, std::numeric_limits<size_t>::max()); - THROW_WALLET_EXCEPTION_IF(!r, error::file_read_error, m_wallet_file); + std::string cache_file_buf; + bool r = true; + if (use_fs) + { + load_from_file(m_wallet_file, cache_file_buf, std::numeric_limits<size_t>::max()); + THROW_WALLET_EXCEPTION_IF(!r, error::file_read_error, m_wallet_file); + } // try to read it as an encrypted cache try { LOG_PRINT_L1("Trying to decrypt cache data"); - r = ::serialization::parse_binary(buf, cache_file_data); + r = ::serialization::parse_binary(use_fs ? cache_file_buf : cache_buf, cache_file_data); THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "internal error: failed to deserialize \"" + m_wallet_file + '\"'); std::string cache_data; cache_data.resize(cache_file_data.cache_data.size()); @@ -5567,7 +5622,7 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass catch (...) { LOG_PRINT_L0("Failed to open portable binary, trying unportable"); - boost::filesystem::copy_file(m_wallet_file, m_wallet_file + ".unportable", boost::filesystem::copy_option::overwrite_if_exists); + if (use_fs) boost::filesystem::copy_file(m_wallet_file, m_wallet_file + ".unportable", boost::filesystem::copy_option::overwrite_if_exists); std::stringstream iss; iss.str(""); iss << cache_data; @@ -5582,17 +5637,17 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass LOG_PRINT_L1("Failed to load encrypted cache, trying unencrypted"); try { std::stringstream iss; - iss << buf; + iss << cache_file_buf; boost::archive::portable_binary_iarchive ar(iss); ar >> *this; } catch (...) { LOG_PRINT_L0("Failed to open portable binary, trying unportable"); - boost::filesystem::copy_file(m_wallet_file, m_wallet_file + ".unportable", boost::filesystem::copy_option::overwrite_if_exists); + if (use_fs) boost::filesystem::copy_file(m_wallet_file, m_wallet_file + ".unportable", boost::filesystem::copy_option::overwrite_if_exists); std::stringstream iss; iss.str(""); - iss << buf; + iss << cache_file_buf; boost::archive::binary_iarchive ar(iss); ar >> *this; } @@ -5636,7 +5691,8 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass try { - m_message_store.read_from_file(get_multisig_wallet_state(), m_mms_file); + if (use_fs) + m_message_store.read_from_file(get_multisig_wallet_state(), m_mms_file); } catch (const std::exception &e) { @@ -5664,7 +5720,7 @@ void wallet2::trim_hashchain() req.height = m_blockchain.size() - 1; uint64_t pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - r = net_utils::invoke_http_json_rpc("/json_rpc", "getblockheaderbyheight", req, res, m_http_client, rpc_timeout); + r = net_utils::invoke_http_json_rpc("/json_rpc", "getblockheaderbyheight", req, res, *m_http_client, rpc_timeout); if (r && res.status == CORE_RPC_STATUS_OK) check_rpc_cost("getblockheaderbyheight", res.credits, pre_call_credits, COST_PER_BLOCK_HEADER); } @@ -5739,18 +5795,10 @@ void wallet2::store_to(const std::string &path, const epee::wipeable_string &pas } } } - // preparing wallet data - std::stringstream oss; - boost::archive::portable_binary_oarchive ar(oss); - ar << *this; - wallet2::cache_file_data cache_file_data = {}; - cache_file_data.cache_data = oss.str(); - std::string cipher; - cipher.resize(cache_file_data.cache_data.size()); - cache_file_data.iv = crypto::rand<crypto::chacha_iv>(); - crypto::chacha20(cache_file_data.cache_data.data(), cache_file_data.cache_data.size(), m_cache_key, cache_file_data.iv, &cipher[0]); - cache_file_data.cache_data = cipher; + // get wallet cache data + boost::optional<wallet2::cache_file_data> cache_file_data = get_cache_file_data(password); + THROW_WALLET_EXCEPTION_IF(cache_file_data == boost::none, error::wallet_internal_error, "failed to generate wallet cache data"); const std::string new_file = same_file ? m_wallet_file + ".new" : path; const std::string old_file = m_wallet_file; @@ -5801,7 +5849,7 @@ void wallet2::store_to(const std::string &path, const epee::wipeable_string &pas // The price to pay is temporary higher memory consumption for string stream + binary archive std::ostringstream oss; binary_archive<true> oar(oss); - bool success = ::serialization::serialize(oar, cache_file_data); + bool success = ::serialization::serialize(oar, cache_file_data.get()); if (success) { success = save_to_file(new_file, oss.str()); } @@ -5810,7 +5858,7 @@ void wallet2::store_to(const std::string &path, const epee::wipeable_string &pas std::ofstream ostr; ostr.open(new_file, std::ios_base::binary | std::ios_base::out | std::ios_base::trunc); binary_archive<true> oar(ostr); - bool success = ::serialization::serialize(oar, cache_file_data); + bool success = ::serialization::serialize(oar, cache_file_data.get()); ostr.close(); THROW_WALLET_EXCEPTION_IF(!success || !ostr.good(), error::file_save_error, new_file); #endif @@ -5826,7 +5874,30 @@ void wallet2::store_to(const std::string &path, const epee::wipeable_string &pas // store should only exist if the MMS is really active m_message_store.write_to_file(get_multisig_wallet_state(), m_mms_file); } - +} +//---------------------------------------------------------------------------------------------------- +boost::optional<wallet2::cache_file_data> wallet2::get_cache_file_data(const epee::wipeable_string &passwords) +{ + trim_hashchain(); + try + { + std::stringstream oss; + boost::archive::portable_binary_oarchive ar(oss); + ar << *this; + + boost::optional<wallet2::cache_file_data> cache_file_data = (wallet2::cache_file_data) {}; + cache_file_data.get().cache_data = oss.str(); + std::string cipher; + cipher.resize(cache_file_data.get().cache_data.size()); + cache_file_data.get().iv = crypto::rand<crypto::chacha_iv>(); + crypto::chacha20(cache_file_data.get().cache_data.data(), cache_file_data.get().cache_data.size(), m_cache_key, cache_file_data.get().iv, &cipher[0]); + cache_file_data.get().cache_data = cipher; + return cache_file_data; + } + catch(...) + { + return boost::none; + } } //---------------------------------------------------------------------------------------------------- uint64_t wallet2::balance(uint32_t index_major, bool strict) const @@ -6030,7 +6101,7 @@ void wallet2::rescan_spent() const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - bool r = epee::net_utils::invoke_http_json("/is_key_image_spent", req, daemon_resp, m_http_client, rpc_timeout); + bool r = epee::net_utils::invoke_http_json("/is_key_image_spent", req, daemon_resp, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR(r, {}, daemon_resp, "is_key_image_spent", error::is_key_image_spent_error, get_rpc_status(daemon_resp.status)); THROW_WALLET_EXCEPTION_IF(daemon_resp.spent_status.size() != n_outputs, error::wallet_internal_error, "daemon returned wrong response for is_key_image_spent, wrong amounts count = " + @@ -6359,7 +6430,7 @@ void wallet2::commit_tx(pending_tx& ptx) oreq.tx = epee::string_tools::buff_to_hex_nodelimer(tx_to_blob(ptx.tx)); { const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; - bool r = epee::net_utils::invoke_http_json("/submit_raw_tx", oreq, ores, m_http_client, rpc_timeout, "POST"); + bool r = epee::net_utils::invoke_http_json("/submit_raw_tx", oreq, ores, *m_http_client, rpc_timeout, "POST"); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "submit_raw_tx"); // MyMonero and OpenMonero use different status strings THROW_WALLET_EXCEPTION_IF(ores.status != "OK" && ores.status != "success" , error::tx_rejected, ptx.tx, get_rpc_status(ores.status), ores.error); @@ -6378,7 +6449,7 @@ void wallet2::commit_tx(pending_tx& ptx) const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - bool r = epee::net_utils::invoke_http_json("/sendrawtransaction", req, daemon_send_resp, m_http_client, rpc_timeout); + bool r = epee::net_utils::invoke_http_json("/sendrawtransaction", req, daemon_send_resp, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR(r, {}, daemon_send_resp, "sendrawtransaction", error::tx_rejected, ptx.tx, get_rpc_status(daemon_send_resp.status), get_text_reason(daemon_send_resp)); check_rpc_cost("/sendrawtransaction", daemon_send_resp.credits, pre_call_credits, COST_PER_TX_RELAY); } @@ -7350,7 +7421,7 @@ uint32_t wallet2::adjust_priority(uint32_t priority) const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; getbh_req.client = get_client_signature(); - bool r = net_utils::invoke_http_json_rpc("/json_rpc", "getblockheadersrange", getbh_req, getbh_res, m_http_client, rpc_timeout); + bool r = net_utils::invoke_http_json_rpc("/json_rpc", "getblockheadersrange", getbh_req, getbh_res, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR(r, {}, getbh_res, "getblockheadersrange", error::get_blocks_error, get_rpc_status(getbh_res.status)); check_rpc_cost("/sendrawtransaction", getbh_res.credits, pre_call_credits, N * COST_PER_BLOCK_HEADER); } @@ -7576,7 +7647,7 @@ bool wallet2::find_and_save_rings(bool force) const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - bool r = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client, rpc_timeout); + bool r = epee::net_utils::invoke_http_json("/gettransactions", req, res, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR_GENERIC(r, {}, res, "/gettransactions"); THROW_WALLET_EXCEPTION_IF(res.txs.size() != req.txs_hashes.size(), error::wallet_internal_error, "daemon returned wrong response for gettransactions, wrong txs count = " + @@ -7724,7 +7795,7 @@ void wallet2::light_wallet_get_outs(std::vector<std::vector<tools::wallet2::get_ { const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; - bool r = epee::net_utils::invoke_http_json("/get_random_outs", oreq, ores, m_http_client, rpc_timeout, "POST"); + bool r = epee::net_utils::invoke_http_json("/get_random_outs", oreq, ores, *m_http_client, rpc_timeout, "POST"); m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "get_random_outs"); THROW_WALLET_EXCEPTION_IF(ores.amount_outs.empty() , error::wallet_internal_error, "No outputs received from light wallet node. Error: " + ores.Error); @@ -7911,7 +7982,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; req_t.client = get_client_signature(); - bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_histogram", req_t, resp_t, m_http_client, rpc_timeout); + bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_histogram", req_t, resp_t, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR(r, {}, resp_t, "get_output_histogram", error::get_histogram_error, get_rpc_status(resp_t.status)); check_rpc_cost("get_output_histogram", resp_t.credits, pre_call_credits, COST_PER_OUTPUT_HISTOGRAM * req_t.amounts.size()); } @@ -7937,7 +8008,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; req_t.client = get_client_signature(); - bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_distribution", req_t, resp_t, m_http_client, rpc_timeout * 1000); + bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_distribution", req_t, resp_t, *m_http_client, rpc_timeout * 1000); THROW_ON_RPC_RESPONSE_ERROR(r, {}, resp_t, "get_output_distribution", error::get_output_distribution, get_rpc_status(resp_t.status)); uint64_t expected_cost = 0; for (uint64_t amount: req_t.amounts) expected_cost += (amount ? COST_PER_OUTPUT_DISTRIBUTION : COST_PER_OUTPUT_DISTRIBUTION_0); @@ -8291,7 +8362,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - bool r = epee::net_utils::invoke_http_bin("/get_outs.bin", req, daemon_resp, m_http_client, rpc_timeout); + bool r = epee::net_utils::invoke_http_bin("/get_outs.bin", req, daemon_resp, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR(r, {}, daemon_resp, "get_outs.bin", error::get_outs_error, get_rpc_status(daemon_resp.status)); THROW_WALLET_EXCEPTION_IF(daemon_resp.outs.size() != req.outputs.size(), error::wallet_internal_error, "daemon returned wrong response for get_outs.bin, wrong amounts count = " + @@ -10500,7 +10571,7 @@ uint8_t wallet2::get_current_hard_fork() m_daemon_rpc_mutex.lock(); req_t.version = 0; - bool r = net_utils::invoke_http_json_rpc("/json_rpc", "hard_fork_info", req_t, resp_t, m_http_client, rpc_timeout); + bool r = net_utils::invoke_http_json_rpc("/json_rpc", "hard_fork_info", req_t, resp_t, *m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, tools::error::no_connection_to_daemon, "hard_fork_info"); THROW_WALLET_EXCEPTION_IF(resp_t.status == CORE_RPC_STATUS_BUSY, tools::error::daemon_busy, "hard_fork_info"); @@ -10595,7 +10666,7 @@ std::vector<size_t> wallet2::select_available_outputs_from_histogram(uint64_t co const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; req_t.client = get_client_signature(); - bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_histogram", req_t, resp_t, m_http_client, rpc_timeout); + bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_histogram", req_t, resp_t, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR(r, {}, resp_t, "get_output_histogram", error::get_histogram_error, resp_t.status); uint64_t cost = req_t.amounts.empty() ? COST_PER_FULL_OUTPUT_HISTOGRAM : (COST_PER_OUTPUT_HISTOGRAM * req_t.amounts.size()); check_rpc_cost("get_output_histogram", resp_t.credits, pre_call_credits, cost); @@ -10637,7 +10708,7 @@ uint64_t wallet2::get_num_rct_outputs() const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; req_t.client = get_client_signature(); - bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_histogram", req_t, resp_t, m_http_client, rpc_timeout); + bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_histogram", req_t, resp_t, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR(r, {}, resp_t, "get_output_histogram", error::get_histogram_error, resp_t.status); THROW_WALLET_EXCEPTION_IF(resp_t.histogram.size() != 1, error::get_histogram_error, "Expected exactly one response"); THROW_WALLET_EXCEPTION_IF(resp_t.histogram[0].amount != 0, error::get_histogram_error, "Expected 0 amount"); @@ -10768,7 +10839,7 @@ bool wallet2::get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, s const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; req.client = get_client_signature(); uint64_t pre_call_credits = m_rpc_payment_state.credits; - bool ok = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client); + bool ok = epee::net_utils::invoke_http_json("/gettransactions", req, res, *m_http_client); THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1), error::wallet_internal_error, "Failed to get transaction from daemon"); check_rpc_cost("/gettransactions", res.credits, pre_call_credits, res.txs.size() * COST_PER_TX); @@ -10821,7 +10892,7 @@ void wallet2::set_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_ const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - r = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client, rpc_timeout); + r = epee::net_utils::invoke_http_json("/gettransactions", req, res, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR_GENERIC(r, {}, res, "/gettransactions"); THROW_WALLET_EXCEPTION_IF(res.txs.size() != 1, error::wallet_internal_error, "daemon returned wrong response for gettransactions, wrong txs count = " + @@ -10874,7 +10945,7 @@ std::string wallet2::get_spend_proof(const crypto::hash &txid, const std::string const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - r = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client, rpc_timeout); + r = epee::net_utils::invoke_http_json("/gettransactions", req, res, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR_GENERIC(r, {}, res, "gettransactions"); THROW_WALLET_EXCEPTION_IF(res.txs.size() != 1, error::wallet_internal_error, "daemon returned wrong response for gettransactions, wrong txs count = " + @@ -10938,7 +11009,7 @@ std::string wallet2::get_spend_proof(const crypto::hash &txid, const std::string const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - r = epee::net_utils::invoke_http_bin("/get_outs.bin", req, res, m_http_client, rpc_timeout); + r = epee::net_utils::invoke_http_bin("/get_outs.bin", req, res, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR(r, {}, res, "get_outs.bin", error::get_outs_error, res.status); THROW_WALLET_EXCEPTION_IF(res.outs.size() != ring_size, error::wallet_internal_error, "daemon returned wrong response for get_outs.bin, wrong amounts count = " + @@ -10996,7 +11067,7 @@ bool wallet2::check_spend_proof(const crypto::hash &txid, const std::string &mes const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - r = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client, rpc_timeout); + r = epee::net_utils::invoke_http_json("/gettransactions", req, res, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR_GENERIC(r, {}, res, "gettransactions"); THROW_WALLET_EXCEPTION_IF(res.txs.size() != 1, error::wallet_internal_error, "daemon returned wrong response for gettransactions, wrong txs count = " + @@ -11071,7 +11142,7 @@ bool wallet2::check_spend_proof(const crypto::hash &txid, const std::string &mes const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - r = epee::net_utils::invoke_http_bin("/get_outs.bin", req, res, m_http_client, rpc_timeout); + r = epee::net_utils::invoke_http_bin("/get_outs.bin", req, res, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR(r, {}, res, "get_outs.bin", error::get_outs_error, res.status); THROW_WALLET_EXCEPTION_IF(res.outs.size() != req.outputs.size(), error::wallet_internal_error, "daemon returned wrong response for get_outs.bin, wrong amounts count = " + @@ -11173,7 +11244,7 @@ void wallet2::check_tx_key_helper(const crypto::hash &txid, const crypto::key_de const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - ok = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client); + ok = epee::net_utils::invoke_http_json("/gettransactions", req, res, *m_http_client); THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1), error::wallet_internal_error, "Failed to get transaction from daemon"); check_rpc_cost("/gettransactions", res.credits, pre_call_credits, COST_PER_TX); @@ -11228,7 +11299,7 @@ std::string wallet2::get_tx_proof(const crypto::hash &txid, const cryptonote::ac const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - ok = net_utils::invoke_http_json("/gettransactions", req, res, m_http_client); + ok = net_utils::invoke_http_json("/gettransactions", req, res, *m_http_client); THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1), error::wallet_internal_error, "Failed to get transaction from daemon"); check_rpc_cost("/gettransactions", res.credits, pre_call_credits, COST_PER_TX); @@ -11389,7 +11460,7 @@ bool wallet2::check_tx_proof(const crypto::hash &txid, const cryptonote::account const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - ok = net_utils::invoke_http_json("/gettransactions", req, res, m_http_client); + ok = net_utils::invoke_http_json("/gettransactions", req, res, *m_http_client); THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1), error::wallet_internal_error, "Failed to get transaction from daemon"); check_rpc_cost("/gettransactions", res.credits, pre_call_credits, COST_PER_TX); @@ -11686,7 +11757,7 @@ bool wallet2::check_reserve_proof(const cryptonote::account_public_address &addr const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; gettx_req.client = get_client_signature(); - bool ok = net_utils::invoke_http_json("/gettransactions", gettx_req, gettx_res, m_http_client); + bool ok = net_utils::invoke_http_json("/gettransactions", gettx_req, gettx_res, *m_http_client); THROW_WALLET_EXCEPTION_IF(!ok || gettx_res.txs.size() != proofs.size(), error::wallet_internal_error, "Failed to get transaction from daemon"); check_rpc_cost("/gettransactions", gettx_res.credits, pre_call_credits, gettx_res.txs.size() * COST_PER_TX); @@ -11703,7 +11774,7 @@ bool wallet2::check_reserve_proof(const cryptonote::account_public_address &addr const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; kispent_req.client = get_client_signature(); - ok = epee::net_utils::invoke_http_json("/is_key_image_spent", kispent_req, kispent_res, m_http_client, rpc_timeout); + ok = epee::net_utils::invoke_http_json("/is_key_image_spent", kispent_req, kispent_res, *m_http_client, rpc_timeout); THROW_WALLET_EXCEPTION_IF(!ok || kispent_res.spent_status.size() != proofs.size(), error::wallet_internal_error, "Failed to get key image spent status from daemon"); check_rpc_cost("/is_key_image_spent", kispent_res.credits, pre_call_credits, kispent_res.spent_status.size() * COST_PER_KEY_IMAGE); @@ -12277,7 +12348,7 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - bool r = epee::net_utils::invoke_http_json("/is_key_image_spent", req, daemon_resp, m_http_client, rpc_timeout); + bool r = epee::net_utils::invoke_http_json("/is_key_image_spent", req, daemon_resp, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR_GENERIC(r, {}, daemon_resp, "is_key_image_spent"); THROW_WALLET_EXCEPTION_IF(daemon_resp.spent_status.size() != signed_key_images.size(), error::wallet_internal_error, "daemon returned wrong response for is_key_image_spent, wrong amounts count = " + @@ -12366,7 +12437,7 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; gettxs_req.client = get_client_signature(); uint64_t pre_call_credits = m_rpc_payment_state.credits; - bool r = epee::net_utils::invoke_http_json("/gettransactions", gettxs_req, gettxs_res, m_http_client, rpc_timeout); + bool r = epee::net_utils::invoke_http_json("/gettransactions", gettxs_req, gettxs_res, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR_GENERIC(r, {}, gettxs_res, "gettransactions"); THROW_WALLET_EXCEPTION_IF(gettxs_res.txs.size() != spent_txids.size(), error::wallet_internal_error, "daemon returned wrong response for gettransactions, wrong count = " + std::to_string(gettxs_res.txs.size()) + ", expected " + std::to_string(spent_txids.size())); @@ -13307,7 +13378,7 @@ uint64_t wallet2::get_blockchain_height_by_date(uint16_t year, uint8_t month, ui const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - r = net_utils::invoke_http_bin("/getblocks_by_height.bin", req, res, m_http_client, rpc_timeout); + r = net_utils::invoke_http_bin("/getblocks_by_height.bin", req, res, *m_http_client, rpc_timeout); if (r && res.status == CORE_RPC_STATUS_OK) check_rpc_cost("/getblocks_by_height.bin", res.credits, pre_call_credits, 3 * COST_PER_BLOCK); } @@ -13385,7 +13456,7 @@ std::vector<std::pair<uint64_t, uint64_t>> wallet2::estimate_backlog(const std:: const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); - bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_txpool_backlog", req, res, m_http_client, rpc_timeout); + bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_txpool_backlog", req, res, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR(r, {}, res, "get_txpool_backlog", error::get_tx_pool_error); check_rpc_cost("get_txpool_backlog", res.credits, pre_call_credits, COST_PER_TX_POOL_STATS * res.backlog.size()); } @@ -13724,12 +13795,12 @@ void wallet2::finish_rescan_bc_keep_key_images(uint64_t transfer_height, const c //---------------------------------------------------------------------------------------------------- uint64_t wallet2::get_bytes_sent() const { - return m_http_client.get_bytes_sent(); + return m_http_client->get_bytes_sent(); } //---------------------------------------------------------------------------------------------------- uint64_t wallet2::get_bytes_received() const { - return m_http_client.get_bytes_received(); + return m_http_client->get_bytes_received(); } //---------------------------------------------------------------------------------------------------- std::vector<cryptonote::public_node> wallet2::get_public_nodes(bool white_only) @@ -13742,7 +13813,7 @@ std::vector<cryptonote::public_node> wallet2::get_public_nodes(bool white_only) { const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; - bool r = epee::net_utils::invoke_http_json("/get_public_nodes", req, res, m_http_client, rpc_timeout); + bool r = epee::net_utils::invoke_http_json("/get_public_nodes", req, res, *m_http_client, rpc_timeout); THROW_ON_RPC_RESPONSE_ERROR_GENERIC(r, {}, res, "/get_public_nodes"); } diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 7620d09d8..1c3c00152 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -269,7 +269,7 @@ private: static bool verify_password(const std::string& keys_file_name, const epee::wipeable_string& password, bool no_spend_key, hw::device &hwdev, uint64_t kdf_rounds); static bool query_device(hw::device::device_type& device_type, const std::string& keys_file_name, const epee::wipeable_string& password, uint64_t kdf_rounds = 1); - wallet2(cryptonote::network_type nettype = cryptonote::MAINNET, uint64_t kdf_rounds = 1, bool unattended = false); + wallet2(cryptonote::network_type nettype = cryptonote::MAINNET, uint64_t kdf_rounds = 1, bool unattended = false, std::unique_ptr<epee::net_utils::http::http_client_factory> http_client_factory = std::unique_ptr<epee::net_utils::http::http_simple_client_factory>(new epee::net_utils::http::http_simple_client_factory())); ~wallet2(); struct multisig_info @@ -708,7 +708,7 @@ private: */ void rewrite(const std::string& wallet_name, const epee::wipeable_string& password); void write_watch_only_wallet(const std::string& wallet_name, const epee::wipeable_string& password, std::string &new_keys_filename); - void load(const std::string& wallet, const epee::wipeable_string& password); + void load(const std::string& wallet, const epee::wipeable_string& password, const std::string& keys_buf = "", const std::string& cache_buf = ""); void store(); /*! * \brief store_to Stores wallet to another file(s), deleting old ones @@ -716,6 +716,19 @@ private: * \param password Password to protect new wallet (TODO: probably better save the password in the wallet object?) */ void store_to(const std::string &path, const epee::wipeable_string &password); + /*! + * \brief get_keys_file_data Get wallet keys data which can be stored to a wallet file. + * \param password Password of the encrypted wallet buffer (TODO: probably better save the password in the wallet object?) + * \param watch_only true to include only view key, false to include both spend and view keys + * \return Encrypted wallet keys data which can be stored to a wallet file + */ + boost::optional<wallet2::keys_file_data> get_keys_file_data(const epee::wipeable_string& password, bool watch_only); + /*! + * \brief get_cache_file_data Get wallet cache data which can be stored to a wallet file. + * \param password Password to protect the wallet cache data (TODO: probably better save the password in the wallet object?) + * \return Encrypted wallet cache data which can be stored to a wallet file + */ + boost::optional<wallet2::cache_file_data> get_cache_file_data(const epee::wipeable_string& password); std::string path() const; @@ -1319,25 +1332,25 @@ private: crypto::public_key get_multisig_signing_public_key(const crypto::secret_key &skey) const; template<class t_request, class t_response> - inline bool invoke_http_json(const boost::string_ref uri, const t_request& req, t_response& res, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref http_method = "GET") + inline bool invoke_http_json(const boost::string_ref uri, const t_request& req, t_response& res, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref http_method = "POST") { if (m_offline) return false; boost::lock_guard<boost::recursive_mutex> lock(m_daemon_rpc_mutex); - return epee::net_utils::invoke_http_json(uri, req, res, m_http_client, timeout, http_method); + return epee::net_utils::invoke_http_json(uri, req, res, *m_http_client, timeout, http_method); } template<class t_request, class t_response> - inline bool invoke_http_bin(const boost::string_ref uri, const t_request& req, t_response& res, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref http_method = "GET") + inline bool invoke_http_bin(const boost::string_ref uri, const t_request& req, t_response& res, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref http_method = "POST") { if (m_offline) return false; boost::lock_guard<boost::recursive_mutex> lock(m_daemon_rpc_mutex); - return epee::net_utils::invoke_http_bin(uri, req, res, m_http_client, timeout, http_method); + return epee::net_utils::invoke_http_bin(uri, req, res, *m_http_client, timeout, http_method); } template<class t_request, class t_response> - inline bool invoke_http_json_rpc(const boost::string_ref uri, const std::string& method_name, const t_request& req, t_response& res, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref http_method = "GET", const std::string& req_id = "0") + inline bool invoke_http_json_rpc(const boost::string_ref uri, const std::string& method_name, const t_request& req, t_response& res, std::chrono::milliseconds timeout = std::chrono::seconds(15), const boost::string_ref http_method = "POST", const std::string& req_id = "0") { if (m_offline) return false; boost::lock_guard<boost::recursive_mutex> lock(m_daemon_rpc_mutex); - return epee::net_utils::invoke_http_json_rpc(uri, method_name, req, res, m_http_client, timeout, http_method, req_id); + return epee::net_utils::invoke_http_json_rpc(uri, method_name, req, res, *m_http_client, timeout, http_method, req_id); } bool set_ring_database(const std::string &filename); @@ -1403,11 +1416,18 @@ private: */ bool store_keys(const std::string& keys_file_name, const epee::wipeable_string& password, bool watch_only = false); /*! - * \brief Load wallet information from wallet file. + * \brief Load wallet keys information from wallet file. * \param keys_file_name Name of wallet file * \param password Password of wallet file */ bool load_keys(const std::string& keys_file_name, const epee::wipeable_string& password); + /*! + * \brief Load wallet keys information from a string buffer. + * \param keys_buf Keys buffer to load + * \param password Password of keys buffer + */ + bool load_keys_buf(const std::string& keys_buf, const epee::wipeable_string& password); + bool load_keys_buf(const std::string& keys_buf, const epee::wipeable_string& password, boost::optional<crypto::chacha_key>& keys_to_encrypt); void process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector<uint64_t> &o_indices, uint64_t height, uint8_t block_version, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen, const tx_cache_data &tx_cache_data, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL); bool should_skip_block(const cryptonote::block &b, uint64_t height) const; void process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const parsed_block &parsed_block, const crypto::hash& bl_id, uint64_t height, const std::vector<tx_cache_data> &tx_cache_data, size_t tx_cache_data_offset, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL); @@ -1502,7 +1522,7 @@ private: std::string m_wallet_file; std::string m_keys_file; std::string m_mms_file; - epee::net_utils::http::http_simple_client m_http_client; + const std::unique_ptr<epee::net_utils::http::abstract_http_client> m_http_client; hashchain m_blockchain; std::unordered_map<crypto::hash, unconfirmed_transfer_details> m_unconfirmed_txs; std::unordered_map<crypto::hash, confirmed_transfer_details> m_confirmed_txs; diff --git a/src/wallet/wallet_rpc_payments.cpp b/src/wallet/wallet_rpc_payments.cpp index 41696d13b..4f5364269 100644 --- a/src/wallet/wallet_rpc_payments.cpp +++ b/src/wallet/wallet_rpc_payments.cpp @@ -85,7 +85,7 @@ bool wallet2::make_rpc_payment(uint32_t nonce, uint32_t cookie, uint64_t &credit uint64_t pre_call_credits = m_rpc_payment_state.credits; req.client = get_client_signature(); epee::json_rpc::error error; - bool r = epee::net_utils::invoke_http_json_rpc("/json_rpc", "rpc_access_submit_nonce", req, res, error, m_http_client, rpc_timeout); + bool r = epee::net_utils::invoke_http_json_rpc("/json_rpc", "rpc_access_submit_nonce", req, res, error, *m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); THROW_ON_RPC_RESPONSE_ERROR_GENERIC(r, error, res, "rpc_access_submit_nonce"); THROW_WALLET_EXCEPTION_IF(res.credits < pre_call_credits, error::wallet_internal_error, "RPC payment did not increase balance"); |