diff options
Diffstat (limited to 'src')
93 files changed, 1653 insertions, 500 deletions
diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp index a9a7d035f..5e12fa8ec 100644 --- a/src/blockchain_db/blockchain_db.cpp +++ b/src/blockchain_db/blockchain_db.cpp @@ -287,7 +287,7 @@ uint64_t BlockchainDB::add_block( const std::pair<block, blobdata>& blck num_rct_outs += blk.miner_tx.vout.size(); int tx_i = 0; crypto::hash tx_hash = crypto::null_hash; - for (const std::pair<transaction, blobdata_ref>& tx : txs) + for (const std::pair<transaction, blobdata>& tx : txs) { tx_hash = blk.tx_hashes[tx_i]; add_transaction(blk_hash, tx, &tx_hash); diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 0aa2aee6f..5f3b495b0 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -897,7 +897,6 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons throw0(DB_ERROR(lmdb_error("Failed to add tx data to db transaction: ", result).c_str())); const cryptonote::blobdata_ref &blob = txp.second; - MDB_val_sized(blobval, blob); unsigned int unprunable_size = tx.unprunable_size; if (unprunable_size == 0) @@ -3190,9 +3189,8 @@ bool BlockchainLMDB::get_blocks_from(uint64_t start_height, size_t min_block_cou uint64_t size = 0; size_t num_txes = 0; MDB_val_copy<uint64_t> key(start_height); - MDB_val k, v, val_tx_id; + MDB_val v, val_tx_id; uint64_t tx_id = ~0; - MDB_cursor_op op = MDB_SET; for (uint64_t h = start_height; h < blockchain_height && blocks.size() < max_block_count && (size < max_size || blocks.size() < min_block_count); ++h) { MDB_cursor_op op = h == start_height ? MDB_SET : MDB_NEXT; @@ -3314,7 +3312,7 @@ bool BlockchainLMDB::get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::h RCURSOR(txs_prunable_hash); MDB_val_set(v, tx_hash); - MDB_val result, val_tx_prunable_hash; + MDB_val result; auto get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); if (get_result == 0) { @@ -4311,7 +4309,6 @@ bool BlockchainLMDB::get_output_distribution(uint64_t amount, uint64_t from_heig return false; distribution.resize(db_height - from_height, 0); - bool fret = true; MDB_val_set(k, amount); MDB_val v; MDB_cursor_op op = MDB_SET; @@ -5106,11 +5103,10 @@ void BlockchainLMDB::migrate_0_1() void BlockchainLMDB::migrate_1_2() { LOG_PRINT_L3("BlockchainLMDB::" << __func__); - uint64_t i, z; + uint64_t i; int result; mdb_txn_safe txn(false); - MDB_val k, v; - char *ptr; + MDB_val v; MGINFO_YELLOW("Migrating blockchain from DB version 1 to 2 - this may take a while:"); MINFO("updating txs_pruned and txs_prunable tables..."); @@ -5311,7 +5307,6 @@ void BlockchainLMDB::migrate_2_3() if (result) throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_info: ", result).c_str())); if (!i) { - MDB_stat db_stat; result = mdb_stat(txn, m_block_info, &db_stats); if (result) throw0(DB_ERROR(lmdb_error("Failed to query m_block_info: ", result).c_str())); @@ -5443,7 +5438,6 @@ void BlockchainLMDB::migrate_3_4() if (result) throw0(DB_ERROR(lmdb_error("Failed to open a cursor for blocks: ", result).c_str())); if (!i) { - MDB_stat db_stat; result = mdb_stat(txn, m_block_info, &db_stats); if (result) throw0(DB_ERROR(lmdb_error("Failed to query m_block_info: ", result).c_str())); @@ -5597,7 +5591,6 @@ void BlockchainLMDB::migrate_4_5() if (result) throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_info: ", result).c_str())); if (!i) { - MDB_stat db_stat; result = mdb_stat(txn, m_block_info, &db_stats); if (result) throw0(DB_ERROR(lmdb_error("Failed to query m_block_info: ", result).c_str())); diff --git a/src/blockchain_utilities/blockchain_ancestry.cpp b/src/blockchain_utilities/blockchain_ancestry.cpp index 89b932e4f..99a84606d 100644 --- a/src/blockchain_utilities/blockchain_ancestry.cpp +++ b/src/blockchain_utilities/blockchain_ancestry.cpp @@ -32,6 +32,7 @@ #include <boost/algorithm/string.hpp> #include <boost/archive/portable_binary_iarchive.hpp> #include <boost/archive/portable_binary_oarchive.hpp> +#include <boost/filesystem/path.hpp> #include "common/unordered_containers_boost_serialization.h" #include "common/command_line.h" #include "common/varint.h" @@ -149,7 +150,7 @@ struct ancestry_state_t { std::unordered_map<crypto::hash, cryptonote::transaction> old_tx_cache; a & old_tx_cache; - for (const auto i: old_tx_cache) + for (const auto& i: old_tx_cache) tx_cache.insert(std::make_pair(i.first, ::tx_data_t(i.second))); } else @@ -161,7 +162,7 @@ struct ancestry_state_t std::unordered_map<uint64_t, cryptonote::block> old_block_cache; a & old_block_cache; block_cache.resize(old_block_cache.size()); - for (const auto i: old_block_cache) + for (const auto& i: old_block_cache) block_cache[i.first] = i.second; } else @@ -575,7 +576,6 @@ int main(int argc, char* argv[]) { add_ancestry(state.ancestry, txid, ancestor{amount, offset}); // find the tx which created this output - bool found = false; crypto::hash output_txid; if (!get_output_txid(state, db, amount, offset, output_txid)) { @@ -693,7 +693,6 @@ int main(int argc, char* argv[]) add_ancestor(ancestry, amount, offset); // find the tx which created this output - bool found = false; crypto::hash output_txid; if (!get_output_txid(state, db, amount, offset, output_txid)) { diff --git a/src/blockchain_utilities/blockchain_blackball.cpp b/src/blockchain_utilities/blockchain_blackball.cpp index a8197483f..18a37434c 100644 --- a/src/blockchain_utilities/blockchain_blackball.cpp +++ b/src/blockchain_utilities/blockchain_blackball.cpp @@ -28,6 +28,7 @@ #include <boost/range/adaptor/transformed.hpp> #include <boost/algorithm/string.hpp> +#include <boost/filesystem.hpp> #include "common/unordered_containers_boost_serialization.h" #include "common/command_line.h" #include "common/varint.h" @@ -47,9 +48,6 @@ namespace po = boost::program_options; using namespace epee; using namespace cryptonote; -static const char zerokey[8] = {0}; -static const MDB_val zerokval = { sizeof(zerokey), (void *)zerokey }; - static uint64_t records_per_sync = 200; static uint64_t db_flags = 0; static MDB_dbi dbi_relative_rings; @@ -703,7 +701,6 @@ static void get_per_amount_outputs(MDB_txn *txn, uint64_t amount, uint64_t &tota int dbr = mdb_cursor_open(txn, dbi_per_amount, &cur); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for per amount outputs: " + std::string(mdb_strerror(dbr))); MDB_val k, v; - mdb_size_t count = 0; k.mv_size = sizeof(uint64_t); k.mv_data = (void*)&amount; dbr = mdb_cursor_get(cur, &k, &v, MDB_SET); @@ -726,7 +723,6 @@ static void inc_per_amount_outputs(MDB_txn *txn, uint64_t amount, uint64_t total int dbr = mdb_cursor_open(txn, dbi_per_amount, &cur); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for per amount outputs: " + std::string(mdb_strerror(dbr))); MDB_val k, v; - mdb_size_t count = 0; k.mv_size = sizeof(uint64_t); k.mv_data = (void*)&amount; dbr = mdb_cursor_get(cur, &k, &v, MDB_SET); @@ -1077,7 +1073,6 @@ static std::vector<std::pair<uint64_t, uint64_t>> load_outputs(const std::string s[len - 1] = 0; if (!s[0]) continue; - std::pair<uint64_t, uint64_t> output; uint64_t offset, num_offsets; if (sscanf(s, "@%" PRIu64, &amount) == 1) { @@ -1269,8 +1264,6 @@ int main(int argc, char* argv[]) LOG_PRINT_L0("Scanning for spent outputs..."); - size_t done = 0; - const uint64_t start_blackballed_outputs = get_num_spent_outputs(); tools::ringdb ringdb(output_file_path.string(), epee::string_tools::pod_to_hex(get_genesis_block_hash(inputs[0]))); diff --git a/src/blockchain_utilities/blockchain_depth.cpp b/src/blockchain_utilities/blockchain_depth.cpp index 6199586bf..8c3c3a009 100644 --- a/src/blockchain_utilities/blockchain_depth.cpp +++ b/src/blockchain_utilities/blockchain_depth.cpp @@ -27,6 +27,7 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <boost/range/adaptor/transformed.hpp> +#include <boost/filesystem/path.hpp> #include <boost/algorithm/string.hpp> #include "common/command_line.h" #include "common/varint.h" diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp index 60c069c3b..df2662444 100644 --- a/src/blockchain_utilities/blockchain_import.cpp +++ b/src/blockchain_utilities/blockchain_import.cpp @@ -227,7 +227,7 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path return false; } - uint64_t block_first, block_last; + uint64_t block_first; uint64_t start_height = 1, seek_height; if (opt_resume) start_height = core.get_blockchain_storage().get_current_blockchain_height(); diff --git a/src/blockchain_utilities/blockchain_prune.cpp b/src/blockchain_utilities/blockchain_prune.cpp index 9a9d58c46..b1c599f3a 100644 --- a/src/blockchain_utilities/blockchain_prune.cpp +++ b/src/blockchain_utilities/blockchain_prune.cpp @@ -29,6 +29,8 @@ #include <array> #include <lmdb.h> #include <boost/algorithm/string.hpp> +#include <boost/system/error_code.hpp> +#include <boost/filesystem.hpp> #include "common/command_line.h" #include "common/pruning.h" #include "cryptonote_core/cryptonote_core.h" diff --git a/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp b/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp index f8763710e..78a662134 100644 --- a/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp +++ b/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp @@ -27,6 +27,7 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <boost/algorithm/string.hpp> +#include <boost/filesystem.hpp> #include "common/command_line.h" #include "serialization/crypto.h" #include "cryptonote_core/tx_pool.h" @@ -66,7 +67,6 @@ static std::map<uint64_t, uint64_t> load_outputs(const std::string &filename) s[len - 1] = 0; if (!s[0]) continue; - std::pair<uint64_t, uint64_t> output; uint64_t offset, num_offsets; if (sscanf(s, "@%" PRIu64, &amount) == 1) { diff --git a/src/blockchain_utilities/blockchain_stats.cpp b/src/blockchain_utilities/blockchain_stats.cpp index 1f728b4e5..5f5ca6abf 100644 --- a/src/blockchain_utilities/blockchain_stats.cpp +++ b/src/blockchain_utilities/blockchain_stats.cpp @@ -27,6 +27,7 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <boost/algorithm/string.hpp> +#include <boost/filesystem.hpp> #include "common/command_line.h" #include "common/varint.h" #include "cryptonote_basic/cryptonote_boost_serialization.h" diff --git a/src/blockchain_utilities/blockchain_usage.cpp b/src/blockchain_utilities/blockchain_usage.cpp index 6e87a0974..8356ef420 100644 --- a/src/blockchain_utilities/blockchain_usage.cpp +++ b/src/blockchain_utilities/blockchain_usage.cpp @@ -28,6 +28,7 @@ #include <boost/range/adaptor/transformed.hpp> #include <boost/algorithm/string.hpp> +#include <boost/filesystem/path.hpp> #include "common/command_line.h" #include "common/varint.h" #include "cryptonote_core/tx_pool.h" @@ -180,7 +181,6 @@ int main(int argc, char* argv[]) LOG_PRINT_L0("Building usage patterns..."); - size_t done = 0; std::unordered_map<output_data, std::list<reference>> outputs; std::unordered_map<uint64_t,uint64_t> indices; @@ -195,7 +195,7 @@ int main(int argc, char* argv[]) { if (opt_rct_only && out.amount) continue; - uint64_t index = indices[out.amount]++; + indices[out.amount]++; output_data od(out.amount, indices[out.amount], coinbase, height); auto itb = outputs.emplace(od, std::list<reference>()); itb.first->first.info(coinbase, height); diff --git a/src/checkpoints/checkpoints.cpp b/src/checkpoints/checkpoints.cpp index c88a630cc..6b48d8723 100644 --- a/src/checkpoints/checkpoints.cpp +++ b/src/checkpoints/checkpoints.cpp @@ -34,6 +34,8 @@ #include "string_tools.h" #include "storages/portable_storage_template_helper.h" // epee json include #include "serialization/keyvalue_serialization.h" +#include <boost/system/error_code.hpp> +#include <boost/filesystem.hpp> #include <functional> #include <vector> diff --git a/src/common/dns_utils.cpp b/src/common/dns_utils.cpp index 4f4efcd81..f0b617798 100644 --- a/src/common/dns_utils.cpp +++ b/src/common/dns_utils.cpp @@ -37,6 +37,7 @@ #include <boost/thread/mutex.hpp> #include <boost/algorithm/string/join.hpp> #include <boost/optional.hpp> +#include <boost/utility/string_ref.hpp> using namespace epee; #undef MONERO_DEFAULT_LOG_CATEGORY @@ -124,6 +125,7 @@ static const char *get_record_name(int record_type) case DNS_TYPE_A: return "A"; case DNS_TYPE_TXT: return "TXT"; case DNS_TYPE_AAAA: return "AAAA"; + case DNS_TYPE_TLSA: return "TLSA"; default: return "unknown"; } } @@ -186,6 +188,13 @@ boost::optional<std::string> txt_to_string(const char* src, size_t len) return std::string(src+1, len-1); } +boost::optional<std::string> tlsa_to_string(const char* src, size_t len) +{ + if (len < 4) + return boost::none; + return std::string(src, len); +} + // custom smart pointer. // TODO: see if std::auto_ptr and the like support custom destructors template<typename type, void (*freefunc)(type*)> @@ -326,11 +335,15 @@ std::vector<std::string> DNSResolver::get_record(const std::string& url, int rec // destructor takes care of cleanup ub_result_ptr result; + MDEBUG("Performing DNSSEC " << get_record_name(record_type) << " record query for " << url); + // call DNS resolver, blocking. if return value not zero, something went wrong if (!ub_resolve(m_data->m_ub_context, string_copy(url.c_str()), record_type, DNS_CLASS_IN, &result)) { dnssec_available = (result->secure || result->bogus); dnssec_valid = result->secure && !result->bogus; + if (dnssec_available && !dnssec_valid) + MWARNING("Invalid DNSSEC " << get_record_name(record_type) << " record signature for " << url << ": " << result->why_bogus); if (result->havedata) { for (size_t i=0; result->data[i] != NULL; i++) @@ -338,8 +351,9 @@ std::vector<std::string> DNSResolver::get_record(const std::string& url, int rec boost::optional<std::string> res = (*reader)(result->data[i], result->len[i]); if (res) { - MINFO("Found \"" << *res << "\" in " << get_record_name(record_type) << " record for " << url); - addresses.push_back(*res); + // do not dump dns record directly from dns into log + MINFO("Found " << get_record_name(record_type) << " record for " << url); + addresses.push_back(std::move(*res)); } } } @@ -363,6 +377,17 @@ std::vector<std::string> DNSResolver::get_txt_record(const std::string& url, boo return get_record(url, DNS_TYPE_TXT, txt_to_string, dnssec_available, dnssec_valid); } +std::vector<std::string> DNSResolver::get_tlsa_tcp_record(const boost::string_ref url, const boost::string_ref port, bool& dnssec_available, bool& dnssec_valid) +{ + std::string service_addr; + service_addr.reserve(url.size() + port.size() + 7); + service_addr.push_back('_'); + service_addr.append(port.data(), port.size()); + service_addr.append("._tcp."); + service_addr.append(url.data(), url.size()); + return get_record(service_addr, DNS_TYPE_TLSA, tlsa_to_string, dnssec_available, dnssec_valid); +} + std::string DNSResolver::get_dns_format_from_oa_address(const std::string& oa_addr) { std::string addr(oa_addr); @@ -484,36 +509,12 @@ std::string get_account_address_as_str_from_url(const std::string& url, bool& dn return dns_confirm(url, addresses, dnssec_valid); } -namespace -{ - bool dns_records_match(const std::vector<std::string>& a, const std::vector<std::string>& b) - { - if (a.size() != b.size()) return false; - - for (const auto& record_in_a : a) - { - bool ok = false; - for (const auto& record_in_b : b) - { - if (record_in_a == record_in_b) - { - ok = true; - break; - } - } - if (!ok) return false; - } - - return true; - } -} - bool load_txt_records_from_dns(std::vector<std::string> &good_records, const std::vector<std::string> &dns_urls) { // Prevent infinite recursion when distributing if (dns_urls.empty()) return false; - std::vector<std::vector<std::string> > records; + std::vector<std::set<std::string> > records; records.resize(dns_urls.size()); size_t first_index = crypto::rand_idx(dns_urls.size()); @@ -525,7 +526,9 @@ bool load_txt_records_from_dns(std::vector<std::string> &good_records, const std for (size_t n = 0; n < dns_urls.size(); ++n) { tpool.submit(&waiter,[n, dns_urls, &records, &avail, &valid](){ - records[n] = tools::DNSResolver::instance().get_txt_record(dns_urls[n], avail[n], valid[n]); + const auto res = tools::DNSResolver::instance().get_txt_record(dns_urls[n], avail[n], valid[n]); + for (const auto &s: res) + records[n].insert(s); }); } waiter.wait(); @@ -568,29 +571,31 @@ bool load_txt_records_from_dns(std::vector<std::string> &good_records, const std return false; } - int good_records_index = -1; - for (size_t i = 0; i < records.size() - 1; ++i) + typedef std::map<std::set<std::string>, uint32_t> map_t; + map_t record_count; + for (const auto &e: records) { - if (records[i].size() == 0) continue; + if (!e.empty()) + ++record_count[e]; + } - for (size_t j = i + 1; j < records.size(); ++j) - { - if (dns_records_match(records[i], records[j])) - { - good_records_index = i; - break; - } - } - if (good_records_index >= 0) break; + map_t::const_iterator good_record = record_count.end(); + for (map_t::const_iterator i = record_count.begin(); i != record_count.end(); ++i) + { + if (good_record == record_count.end() || i->second > good_record->second) + good_record = i; } - if (good_records_index < 0) + MDEBUG("Found " << (good_record == record_count.end() ? 0 : good_record->second) << "/" << dns_urls.size() << " matching records from " << num_valid_records << " valid records"); + if (good_record == record_count.end() || good_record->second < dns_urls.size() / 2 + 1) { - LOG_PRINT_L0("WARNING: no two DNS TXT records matched"); + LOG_PRINT_L0("WARNING: no majority of DNS TXT records matched (only " << good_record->second << "/" << dns_urls.size() << ")"); return false; } - good_records = records[good_records_index]; + good_records = {}; + for (const auto &s: good_record->first) + good_records.push_back(s); return true; } diff --git a/src/common/dns_utils.h b/src/common/dns_utils.h index 30c4cced2..99e91bc54 100644 --- a/src/common/dns_utils.h +++ b/src/common/dns_utils.h @@ -31,15 +31,17 @@ #include <string> #include <functional> #include <boost/optional/optional_fwd.hpp> +#include <boost/utility/string_ref_fwd.hpp> namespace tools { // RFC defines for record types and classes for DNS, gleaned from ldns source -const static int DNS_CLASS_IN = 1; -const static int DNS_TYPE_A = 1; -const static int DNS_TYPE_TXT = 16; -const static int DNS_TYPE_AAAA = 8; +constexpr const int DNS_CLASS_IN = 1; +constexpr const int DNS_TYPE_A = 1; +constexpr const int DNS_TYPE_TXT = 16; +constexpr const int DNS_TYPE_AAAA = 8; +constexpr const int DNS_TYPE_TLSA = 52; struct DNSResolverData; @@ -106,6 +108,17 @@ public: std::vector<std::string> get_txt_record(const std::string& url, bool& dnssec_available, bool& dnssec_valid); /** + * @brief gets all TLSA TCP records from a DNS query for the supplied URL; + * if no TLSA record present returns an empty vector. + * + * @param url A string containing a URL to query for + * @param port The service port number (as string) to query + * + * @return A vector of strings containing all TLSA records; or an empty vector + */ + std::vector<std::string> get_tlsa_tcp_record(boost::string_ref url, boost::string_ref port, bool& dnssec_available, bool& dnssec_valid); + + /** * @brief Gets a DNS address from OpenAlias format * * If the address looks good, but contains one @ symbol, replace that with a . diff --git a/src/common/i18n.cpp b/src/common/i18n.cpp index ebc367b3a..051220ee1 100644 --- a/src/common/i18n.cpp +++ b/src/common/i18n.cpp @@ -35,6 +35,10 @@ #include "common/i18n.h" #include "translation_files.h" +#include <boost/system/error_code.hpp> +#include <boost/filesystem.hpp> +#include <algorithm> + #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "i18n" diff --git a/src/common/updates.cpp b/src/common/updates.cpp index 61a76f5da..af38d7a54 100644 --- a/src/common/updates.cpp +++ b/src/common/updates.cpp @@ -48,7 +48,10 @@ namespace tools static const std::vector<std::string> dns_urls = { "updates.moneropulse.org", "updates.moneropulse.net", - "updates.moneropulse.co", + "updates.moneropulse.fr", + "updates.moneropulse.de", + "updates.moneropulse.no", + "updates.moneropulse.ch", "updates.moneropulse.se" }; diff --git a/src/crypto/hash-ops.h b/src/crypto/hash-ops.h index 7dfc5151d..1cd502994 100644 --- a/src/crypto/hash-ops.h +++ b/src/crypto/hash-ops.h @@ -87,6 +87,10 @@ void hash_extra_jh(const void *data, size_t length, char *hash); void hash_extra_skein(const void *data, size_t length, char *hash); void tree_hash(const char (*hashes)[HASH_SIZE], size_t count, char *root_hash); +bool tree_path(size_t count, size_t idx, uint32_t *path); +bool tree_branch(const char (*hashes)[HASH_SIZE], size_t count, const char *hash, char (*branch)[HASH_SIZE], size_t *depth, uint32_t *path); +bool tree_branch_hash(const char hash[HASH_SIZE], const char (*branch)[HASH_SIZE], size_t depth, uint32_t path, char root[HASH_SIZE]); +bool is_branch_in_tree(const char hash[HASH_SIZE], const char root[HASH_SIZE], const char (*branch)[HASH_SIZE], size_t depth, uint32_t path); #define RX_BLOCK_VERSION 12 void rx_slow_hash_allocate_state(void); diff --git a/src/crypto/rx-slow-hash.c b/src/crypto/rx-slow-hash.c index fa35a32e2..801987e37 100644 --- a/src/crypto/rx-slow-hash.c +++ b/src/crypto/rx-slow-hash.c @@ -264,12 +264,14 @@ void rx_slow_hash(const uint64_t mainheight, const uint64_t seedheight, const ch cache = rx_sp->rs_cache; if (cache == NULL) { - if (cache == NULL) { + if (!(disabled_flags() & RANDOMX_FLAG_LARGE_PAGES)) { cache = randomx_alloc_cache(flags | RANDOMX_FLAG_LARGE_PAGES); if (cache == NULL) { mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX cache"); - cache = randomx_alloc_cache(flags); } + } + if (cache == NULL) { + cache = randomx_alloc_cache(flags); if (cache == NULL) local_abort("Couldn't allocate RandomX cache"); } @@ -291,11 +293,14 @@ void rx_slow_hash(const uint64_t mainheight, const uint64_t seedheight, const ch CTHR_MUTEX_LOCK(rx_dataset_mutex); if (!rx_dataset_nomem) { if (rx_dataset == NULL) { - rx_dataset = randomx_alloc_dataset(RANDOMX_FLAG_LARGE_PAGES); - if (rx_dataset == NULL) { - mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX dataset"); - rx_dataset = randomx_alloc_dataset(RANDOMX_FLAG_DEFAULT); + if (!(disabled_flags() & RANDOMX_FLAG_LARGE_PAGES)) { + rx_dataset = randomx_alloc_dataset(RANDOMX_FLAG_LARGE_PAGES); + if (rx_dataset == NULL) { + mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX dataset"); + } } + if (rx_dataset == NULL) + rx_dataset = randomx_alloc_dataset(RANDOMX_FLAG_DEFAULT); if (rx_dataset != NULL) rx_initdata(rx_sp->rs_cache, miners, seedheight); } @@ -311,11 +316,14 @@ void rx_slow_hash(const uint64_t mainheight, const uint64_t seedheight, const ch } CTHR_MUTEX_UNLOCK(rx_dataset_mutex); } - rx_vm = randomx_create_vm(flags | RANDOMX_FLAG_LARGE_PAGES, rx_sp->rs_cache, rx_dataset); - if(rx_vm == NULL) { //large pages failed - mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX VM"); - rx_vm = randomx_create_vm(flags, rx_sp->rs_cache, rx_dataset); + if (!(disabled_flags() & RANDOMX_FLAG_LARGE_PAGES)) { + rx_vm = randomx_create_vm(flags | RANDOMX_FLAG_LARGE_PAGES, rx_sp->rs_cache, rx_dataset); + if(rx_vm == NULL) { //large pages failed + mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX VM"); + } } + if (rx_vm == NULL) + rx_vm = randomx_create_vm(flags, rx_sp->rs_cache, rx_dataset); if(rx_vm == NULL) {//fallback if everything fails flags = RANDOMX_FLAG_DEFAULT | (miners ? RANDOMX_FLAG_FULL_MEM : 0); rx_vm = randomx_create_vm(flags, rx_sp->rs_cache, rx_dataset); diff --git a/src/crypto/tree-hash.c b/src/crypto/tree-hash.c index 643e95121..8f3ea3339 100644 --- a/src/crypto/tree-hash.c +++ b/src/crypto/tree-hash.c @@ -104,3 +104,154 @@ void tree_hash(const char (*hashes)[HASH_SIZE], size_t count, char *root_hash) { free(ints); } } + +bool tree_path(size_t count, size_t idx, uint32_t *path) +{ + if (count == 0) + return false; + + if (count == 1) { + *path = 0; + } else if (count == 2) { + *path = idx == 0 ? 0 : 1; + } else { + size_t i, j; + + *path = 0; + size_t cnt = tree_hash_cnt( count ); + + for (i = 2 * cnt - count, j = 2 * cnt - count; j < cnt; i += 2, ++j) { + if (idx == i || idx == i+1) + { + *path = (*path << 1) | (idx == i ? 0 : 1); + idx = j; + } + } + assert(i == count); + + while (cnt > 2) { + cnt >>= 1; + for (i = 0, j = 0; j < cnt; i += 2, ++j) { + if (idx == i || idx == i + 1) + { + *path = (*path << 1) | (idx == i ? 0 : 1); + idx = j; + } + } + } + + if (idx == 0 || idx == 1) + { + *path = (*path << 1) | (idx == 0 ? 0 : 1); + idx = 0; + } + } + return true; +} + +bool tree_branch(const char (*hashes)[HASH_SIZE], size_t count, const char *hash, char (*branch)[HASH_SIZE], size_t *depth, uint32_t *path) +{ + size_t idx; + + if (count == 0) + return false; + + for (idx = 0; idx < count; ++idx) + if (!memcmp(hash, hashes[idx], HASH_SIZE)) + break; + if (idx == count) + return false; + + assert(count > 0); + if (count == 1) { + *depth = 0; + *path = 0; + } else if (count == 2) { + *depth = 1; + *path = idx == 0 ? 0 : 1; + memcpy(branch[0], hashes[idx ^ 1], HASH_SIZE); + } else { + size_t i, j; + + *depth = 0; + *path = 0; + size_t cnt = tree_hash_cnt( count ); + + char *ints = calloc(cnt, HASH_SIZE); // zero out as extra protection for using uninitialized mem + assert(ints); + + memcpy(ints, hashes, (2 * cnt - count) * HASH_SIZE); + + for (i = 2 * cnt - count, j = 2 * cnt - count; j < cnt; i += 2, ++j) { + if (idx == i || idx == i+1) + { + memcpy(branch[*depth], hashes[idx == i ? i + 1 : i], HASH_SIZE); + ++*depth; + *path = (*path << 1) | (idx == i ? 0 : 1); + idx = j; + } + cn_fast_hash(hashes[i], 64, ints + j * HASH_SIZE); + } + assert(i == count); + + while (cnt > 2) { + cnt >>= 1; + for (i = 0, j = 0; j < cnt; i += 2, ++j) { + if (idx == i || idx == i + 1) + { + memcpy(branch[*depth], ints + (idx == i ? i + 1 : i) * HASH_SIZE, HASH_SIZE); + ++*depth; + *path = (*path << 1) | (idx == i ? 0 : 1); + idx = j; + } + cn_fast_hash(ints + i * HASH_SIZE, 64, ints + j * HASH_SIZE); + } + } + + if (idx == 0 || idx == 1) + { + memcpy(branch[*depth], ints + (idx == 0 ? 1 : 0) * HASH_SIZE, HASH_SIZE); + ++*depth; + *path = (*path << 1) | (idx == 0 ? 0 : 1); + idx = 0; + } + + free(ints); + } + return true; +} + +bool tree_branch_hash(const char hash[HASH_SIZE], const char (*branch)[HASH_SIZE], size_t depth, uint32_t path, char root[HASH_SIZE]) +{ + size_t d; + char partial[HASH_SIZE]; + + memcpy(partial, hash, HASH_SIZE); + + for (d = 0; d < depth; ++d) + { + char buffer[2 * HASH_SIZE]; + if ((path >> (depth - d - 1)) & 1) + { + memcpy(buffer, branch[d], HASH_SIZE); + memcpy(buffer + HASH_SIZE, partial, HASH_SIZE); + } + else + { + memcpy(buffer, partial, HASH_SIZE); + memcpy(buffer + HASH_SIZE, branch[d], HASH_SIZE); + } + cn_fast_hash(buffer, 2 * HASH_SIZE, partial); + } + + memcpy(root, partial, HASH_SIZE); + return true; +} + +bool is_branch_in_tree(const char hash[HASH_SIZE], const char root[HASH_SIZE], const char (*branch)[HASH_SIZE], size_t depth, uint32_t path) +{ + char res[HASH_SIZE]; + if (!tree_branch_hash(hash, branch, depth, path, res)) + return false; + return memcmp(res, root, HASH_SIZE) == 0; +} diff --git a/src/cryptonote_basic/CMakeLists.txt b/src/cryptonote_basic/CMakeLists.txt index 113fd9d86..c9fb1433c 100644 --- a/src/cryptonote_basic/CMakeLists.txt +++ b/src/cryptonote_basic/CMakeLists.txt @@ -38,10 +38,12 @@ endif() set(cryptonote_basic_sources account.cpp + connection_context.cpp cryptonote_basic_impl.cpp cryptonote_format_utils.cpp difficulty.cpp hardfork.cpp + merge_mining.cpp miner.cpp) set(cryptonote_basic_headers) @@ -56,6 +58,7 @@ set(cryptonote_basic_private_headers cryptonote_format_utils.h difficulty.h hardfork.h + merge_mining.h miner.h tx_extra.h verification_context.h) diff --git a/src/cryptonote_basic/connection_context.cpp b/src/cryptonote_basic/connection_context.cpp new file mode 100644 index 000000000..a0b8ca1f1 --- /dev/null +++ b/src/cryptonote_basic/connection_context.cpp @@ -0,0 +1,71 @@ +// Copyright (c) 2020, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "connection_context.h" + +#include "cryptonote_protocol/cryptonote_protocol_defs.h" +#include "p2p/p2p_protocol_defs.h" + +namespace cryptonote +{ + std::size_t cryptonote_connection_context::get_max_bytes(const int command) noexcept + { + switch (command) + { + case nodetool::COMMAND_HANDSHAKE_T<cryptonote::CORE_SYNC_DATA>::ID: + return 65536; + case nodetool::COMMAND_TIMED_SYNC_T<cryptonote::CORE_SYNC_DATA>::ID: + return 65536; + case nodetool::COMMAND_PING::ID: + return 4096; + case nodetool::COMMAND_REQUEST_SUPPORT_FLAGS::ID: + return 4096; + case cryptonote::NOTIFY_NEW_BLOCK::ID: + return 1024 * 1024 * 128; // 128 MB (max packet is a bit less than 100 MB though) + case cryptonote::NOTIFY_NEW_TRANSACTIONS::ID: + return 1024 * 1024 * 128; // 128 MB (max packet is a bit less than 100 MB though) + case cryptonote::NOTIFY_REQUEST_GET_OBJECTS::ID: + return 1024 * 1024 * 2; // 2 MB + case cryptonote::NOTIFY_RESPONSE_GET_OBJECTS::ID: + return 1024 * 1024 * 128; // 128 MB (max packet is a bit less than 100 MB though) + case cryptonote::NOTIFY_REQUEST_CHAIN::ID: + return 512 * 1024; // 512 kB + case cryptonote::NOTIFY_RESPONSE_CHAIN_ENTRY::ID: + return 1024 * 1024 * 4; // 4 MB + case cryptonote::NOTIFY_NEW_FLUFFY_BLOCK::ID: + return 1024 * 1024 * 4; // 4 MB, but it does not includes transaction data + case cryptonote::NOTIFY_REQUEST_FLUFFY_MISSING_TX::ID: + return 1024 * 1024; // 1 MB + case cryptonote::NOTIFY_GET_TXPOOL_COMPLEMENT::ID: + return 1024 * 1024 * 4; // 4 MB + default: + break; + }; + return std::numeric_limits<size_t>::max(); + } +} // cryptonote diff --git a/src/cryptonote_basic/connection_context.h b/src/cryptonote_basic/connection_context.h index 9e012f8f5..ee26a0c07 100644 --- a/src/cryptonote_basic/connection_context.h +++ b/src/cryptonote_basic/connection_context.h @@ -31,6 +31,7 @@ #pragma once #include <unordered_set> #include <atomic> +#include <algorithm> #include <boost/date_time/posix_time/posix_time.hpp> #include "net/net_utils_base.h" #include "copyable_atomic.h" @@ -38,7 +39,6 @@ namespace cryptonote { - struct cryptonote_connection_context: public epee::net_utils::connection_context_base { cryptonote_connection_context(): m_state(state_before_handshake), m_remote_blockchain_height(0), m_last_response_height(0), @@ -55,6 +55,12 @@ namespace cryptonote state_normal }; + static constexpr int handshake_command() noexcept { return 1001; } + bool handshake_complete() const noexcept { return m_state != state_before_handshake; } + + //! \return Maximum number of bytes permissible for `command`. + static size_t get_max_bytes(int command) noexcept; + state m_state; std::vector<std::pair<crypto::hash, uint64_t>> m_needed_objects; std::unordered_set<crypto::hash> m_requested_objects; @@ -71,6 +77,8 @@ namespace cryptonote int m_expect_response; uint64_t m_expect_height; size_t m_num_requested; + epee::copyable_atomic m_new_stripe_notification{0}; + epee::copyable_atomic m_idle_peer_notification{0}; }; inline std::string get_protocol_state_string(cryptonote_connection_context::state s) diff --git a/src/cryptonote_basic/cryptonote_format_utils.cpp b/src/cryptonote_basic/cryptonote_format_utils.cpp index fcc96883b..3e4532d4e 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.cpp +++ b/src/cryptonote_basic/cryptonote_format_utils.cpp @@ -104,7 +104,6 @@ namespace cryptonote uint64_t get_transaction_weight_clawback(const transaction &tx, size_t n_padded_outputs) { - const rct::rctSig &rv = tx.rct_signatures; const uint64_t bp_base = 368; const size_t n_outputs = tx.vout.size(); if (n_padded_outputs <= 2) @@ -728,6 +727,25 @@ namespace cryptonote return true; } //--------------------------------------------------------------- + bool add_mm_merkle_root_to_tx_extra(std::vector<uint8_t>& tx_extra, const crypto::hash& mm_merkle_root, size_t mm_merkle_tree_depth) + { + CHECK_AND_ASSERT_MES(mm_merkle_tree_depth < 32, false, "merge mining merkle tree depth should be less than 32"); + size_t start_pos = tx_extra.size(); + tx_extra.resize(tx_extra.size() + 3 + 32); + //write tag + tx_extra[start_pos] = TX_EXTRA_MERGE_MINING_TAG; + //write data size + ++start_pos; + tx_extra[start_pos] = 33; + //write depth varint (always one byte here) + ++start_pos; + tx_extra[start_pos] = mm_merkle_tree_depth; + //write data + ++start_pos; + memcpy(&tx_extra[start_pos], &mm_merkle_root, 32); + return true; + } + //--------------------------------------------------------------- bool remove_field_from_tx_extra(std::vector<uint8_t>& tx_extra, const std::type_info &type) { if (tx_extra.empty()) diff --git a/src/cryptonote_basic/cryptonote_format_utils.h b/src/cryptonote_basic/cryptonote_format_utils.h index 636a88b9a..b311bd2b2 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.h +++ b/src/cryptonote_basic/cryptonote_format_utils.h @@ -83,6 +83,7 @@ namespace cryptonote std::vector<crypto::public_key> get_additional_tx_pub_keys_from_extra(const transaction_prefix& tx); bool add_additional_tx_pub_keys_to_extra(std::vector<uint8_t>& tx_extra, const std::vector<crypto::public_key>& additional_pub_keys); bool add_extra_nonce_to_tx_extra(std::vector<uint8_t>& tx_extra, const blobdata& extra_nonce); + bool add_mm_merkle_root_to_tx_extra(std::vector<uint8_t>& tx_extra, const crypto::hash& mm_merkle_root, size_t mm_merkle_tree_depth); bool remove_field_from_tx_extra(std::vector<uint8_t>& tx_extra, const std::type_info &type); void set_payment_id_to_tx_extra_nonce(blobdata& extra_nonce, const crypto::hash& payment_id); void set_encrypted_payment_id_to_tx_extra_nonce(blobdata& extra_nonce, const crypto::hash8& payment_id); diff --git a/src/cryptonote_basic/merge_mining.cpp b/src/cryptonote_basic/merge_mining.cpp new file mode 100644 index 000000000..fcc74859f --- /dev/null +++ b/src/cryptonote_basic/merge_mining.cpp @@ -0,0 +1,95 @@ +// Copyright (c) 2020, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <string.h> +#include "misc_log_ex.h" +#include "int-util.h" +#include "crypto/crypto.h" +#include "common/util.h" +#include "merge_mining.h" + +using namespace epee; + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "cn.mm" + +using namespace crypto; + +namespace cryptonote +{ + +//--------------------------------------------------------------- +uint32_t get_aux_slot(const crypto::hash &id, uint32_t nonce, uint32_t n_aux_chains) +{ + CHECK_AND_ASSERT_THROW_MES(n_aux_chains > 0, "n_aux_chains is 0"); + + uint8_t buf[HASH_SIZE + sizeof(uint32_t) + 1]; + memcpy(buf, &id, HASH_SIZE); + uint32_t v = SWAP32LE(nonce); + memcpy(buf + HASH_SIZE, &v, sizeof(uint32_t)); + buf[HASH_SIZE + sizeof(uint32_t)] = config::HASH_KEY_MM_SLOT; + + crypto::hash res; + tools::sha256sum(buf, sizeof(buf), res); + v = *((const uint32_t*)&res); + return SWAP32LE(v) % n_aux_chains; +} +//--------------------------------------------------------------- +uint32_t get_path_from_aux_slot(uint32_t slot, uint32_t n_aux_chains) +{ + CHECK_AND_ASSERT_THROW_MES(n_aux_chains > 0, "n_aux_chains is 0"); + CHECK_AND_ASSERT_THROW_MES(slot < n_aux_chains, "slot >= n_aux_chains"); + + uint32_t path = 0; + CHECK_AND_ASSERT_THROW_MES(tree_path(n_aux_chains, slot, &path), "Failed to get path from aux slot"); + return path; +} +//--------------------------------------------------------------- +uint32_t encode_mm_depth(uint32_t n_aux_chains, uint32_t nonce) +{ + CHECK_AND_ASSERT_THROW_MES(n_aux_chains > 0, "n_aux_chains is 0"); + + // how many bits to we need to representing n_aux_chains - 1 + uint32_t n_bits = 1; + while ((1u << n_bits) < n_aux_chains && n_bits < 16) + ++n_bits; + CHECK_AND_ASSERT_THROW_MES(n_bits <= 16, "Way too many bits required"); + + const uint32_t depth = (n_bits - 1) | ((n_aux_chains - 1) << 3) | (nonce << (3 + n_bits)); + return depth; +} +//--------------------------------------------------------------- +bool decode_mm_depth(uint32_t depth, uint32_t &n_aux_chains, uint32_t &nonce) +{ + const uint32_t n_bits = 1 + (depth & 7); + n_aux_chains = 1 + (depth >> 3 & ((1 << n_bits) - 1)); + nonce = depth >> (3 + n_bits); + return true; +} +//--------------------------------------------------------------- +} diff --git a/src/cryptonote_basic/merge_mining.h b/src/cryptonote_basic/merge_mining.h new file mode 100644 index 000000000..378438f7c --- /dev/null +++ b/src/cryptonote_basic/merge_mining.h @@ -0,0 +1,40 @@ +// Copyright (c) 2020, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <stdint.h> +#include "crypto/crypto.h" + +namespace cryptonote +{ + uint32_t get_aux_slot(const crypto::hash &id, uint32_t nonce, uint32_t n_aux_chains); + uint32_t get_path_from_aux_slot(uint32_t slot, uint32_t n_aux_chains); + uint32_t encode_mm_depth(uint32_t n_aux_chains, uint32_t nonce); + bool decode_mm_depth(uint32_t depth, uint32_t &n_aux_chains, uint32_t &nonce); +} diff --git a/src/cryptonote_basic/miner.cpp b/src/cryptonote_basic/miner.cpp index 29f6dce5a..ae514aac6 100644 --- a/src/cryptonote_basic/miner.cpp +++ b/src/cryptonote_basic/miner.cpp @@ -44,6 +44,7 @@ #include "string_tools.h" #include "storages/portable_storage_template_helper.h" #include "boost/logic/tribool.hpp" +#include <boost/filesystem.hpp> #ifdef __APPLE__ #include <sys/times.h> diff --git a/src/cryptonote_config.h b/src/cryptonote_config.h index 2cb28b2b1..915835d1b 100644 --- a/src/cryptonote_config.h +++ b/src/cryptonote_config.h @@ -194,7 +194,6 @@ #define CRYPTONOTE_PRUNING_STRIPE_SIZE 4096 // the smaller, the smoother the increase #define CRYPTONOTE_PRUNING_LOG_STRIPES 3 // the higher, the more space saved #define CRYPTONOTE_PRUNING_TIP_BLOCKS 5500 // the smaller, the more space saved -//#define CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED #define RPC_CREDITS_PER_HASH_SCALE ((float)(1<<24)) @@ -235,6 +234,7 @@ namespace config const unsigned char HASH_KEY_CLSAG_AGG_0[] = "CLSAG_agg_0"; const unsigned char HASH_KEY_CLSAG_AGG_1[] = "CLSAG_agg_1"; const char HASH_KEY_MESSAGE_SIGNING[] = "MoneroMessageSignature"; + const unsigned char HASH_KEY_MM_SLOT = 'm'; namespace testnet { diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 5b3b0527b..a3d695b85 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -57,6 +57,7 @@ #include "common/notify.h" #include "common/varint.h" #include "common/pruning.h" +#include "time_helper.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "blockchain" @@ -973,7 +974,7 @@ start: std::pair<bool, uint64_t> Blockchain::check_difficulty_checkpoints() const { uint64_t res = 0; - for (const std::pair<uint64_t, difficulty_type>& i : m_checkpoints.get_difficulty_points()) + for (const std::pair<const uint64_t, difficulty_type>& i : m_checkpoints.get_difficulty_points()) { if (i.first >= m_db->height()) break; @@ -2758,32 +2759,44 @@ void Blockchain::flush_invalid_blocks() m_invalid_blocks.clear(); } //------------------------------------------------------------------ -bool Blockchain::have_block(const crypto::hash& id) const +bool Blockchain::have_block_unlocked(const crypto::hash& id, int *where) const { + // WARNING: this function does not take m_blockchain_lock, and thus should only call read only + // m_db functions which do not depend on one another (ie, no getheight + gethash(height-1), as + // well as not accessing class members, even read only (ie, m_invalid_blocks). The caller must + // lock if it is otherwise needed. LOG_PRINT_L3("Blockchain::" << __func__); - CRITICAL_REGION_LOCAL(m_blockchain_lock); if(m_db->block_exists(id)) { LOG_PRINT_L2("block " << id << " found in main chain"); + if (where) *where = HAVE_BLOCK_MAIN_CHAIN; return true; } if(m_db->get_alt_block(id, NULL, NULL)) { LOG_PRINT_L2("block " << id << " found in alternative chains"); + if (where) *where = HAVE_BLOCK_ALT_CHAIN; return true; } if(m_invalid_blocks.count(id)) { LOG_PRINT_L2("block " << id << " found in m_invalid_blocks"); + if (where) *where = HAVE_BLOCK_INVALID; return true; } return false; } //------------------------------------------------------------------ +bool Blockchain::have_block(const crypto::hash& id, int *where) const +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + return have_block_unlocked(id, where); +} +//------------------------------------------------------------------ bool Blockchain::handle_block_to_main_chain(const block& bl, block_verification_context& bvc, bool notify/* = true*/) { LOG_PRINT_L3("Blockchain::" << __func__); @@ -3426,7 +3439,6 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, bool failed = false; for (size_t i = 0; i < tx.vin.size(); i++) { - const txin_to_key& in_to_key = boost::get<txin_to_key>(tx.vin[i]); if(!failed && !results[i]) failed = true; } @@ -4716,6 +4728,8 @@ uint64_t Blockchain::prevalidate_block_hashes(uint64_t height, const std::vector CHECK_AND_ASSERT_MES(weights.empty() || weights.size() == hashes.size(), 0, "Unexpected weights size"); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + // easy case: height >= hashes if (height >= m_blocks_hash_of_hashes.size() * HASH_OF_HASHES_STEP) return hashes.size(); diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index 07238b719..5291f1338 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -377,10 +377,12 @@ namespace cryptonote * for a block with the given hash * * @param id the hash to search for + * @param where the type of block, if non NULL * * @return true if the block is known, else false */ - bool have_block(const crypto::hash& id) const; + bool have_block_unlocked(const crypto::hash& id, int *where = NULL) const; + bool have_block(const crypto::hash& id, int *where = NULL) const; /** * @brief gets the total number of transactions on the main chain diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 507c47a51..beae55b35 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -57,6 +57,8 @@ using namespace epee; #include "hardforks/hardforks.h" #include "version.h" +#include <boost/filesystem.hpp> + #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "cn" @@ -630,7 +632,7 @@ namespace cryptonote void operator()(std::uint64_t, epee::span<const block> blocks) const { - for (const block bl : blocks) + for (const block& bl : blocks) cmdline.notify("%s", epee::string_tools::pod_to_hex(get_block_hash(bl)).c_str(), NULL); } }; @@ -1625,9 +1627,14 @@ namespace cryptonote return m_mempool.get_transactions_count(include_sensitive_txes); } //----------------------------------------------------------------------------------------------- - bool core::have_block(const crypto::hash& id) const + bool core::have_block_unlocked(const crypto::hash& id, int *where) const + { + return m_blockchain_storage.have_block_unlocked(id, where); + } + //----------------------------------------------------------------------------------------------- + bool core::have_block(const crypto::hash& id, int *where) const { - return m_blockchain_storage.have_block(id); + return m_blockchain_storage.have_block(id, where); } //----------------------------------------------------------------------------------------------- bool core::parse_tx_from_blob(transaction& tx, crypto::hash& tx_hash, const blobdata& blob) const diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 24d0f9e76..8891540a9 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -55,6 +55,8 @@ PUSH_WARNINGS DISABLE_VS_WARNINGS(4355) +enum { HAVE_BLOCK_MAIN_CHAIN, HAVE_BLOCK_ALT_CHAIN, HAVE_BLOCK_INVALID }; + namespace cryptonote { struct test_options { @@ -543,7 +545,8 @@ namespace cryptonote * * @note see Blockchain::have_block */ - bool have_block(const crypto::hash& id) const; + bool have_block_unlocked(const crypto::hash& id, int *where = NULL) const; + bool have_block(const crypto::hash& id, int *where = NULL) const; /** * @copydoc Blockchain::get_short_chain_history diff --git a/src/cryptonote_core/cryptonote_tx_utils.h b/src/cryptonote_core/cryptonote_tx_utils.h index dbdf409b5..73cdd31cd 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.h +++ b/src/cryptonote_core/cryptonote_tx_utils.h @@ -44,10 +44,10 @@ namespace cryptonote typedef std::pair<uint64_t, rct::ctkey> output_entry; std::vector<output_entry> outputs; //index + key + optional ringct commitment - size_t real_output; //index in outputs vector of real output_entry + uint64_t real_output; //index in outputs vector of real output_entry crypto::public_key real_out_tx_key; //incoming real tx public key std::vector<crypto::public_key> real_out_additional_tx_keys; //incoming real tx additional public keys - size_t real_output_in_tx_index; //index in transaction outputs vector + uint64_t real_output_in_tx_index; //index in transaction outputs vector uint64_t amount; //money bool rct; //true if the output is rct rct::key mask; //ringct amount mask diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index d059ab78f..a7e96e23a 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -383,7 +383,6 @@ namespace cryptonote bool tx_memory_pool::add_tx(transaction &tx, tx_verification_context& tvc, relay_method tx_relay, bool relayed, uint8_t version) { crypto::hash h = null_hash; - size_t blob_size = 0; cryptonote::blobdata bl; t_serializable_object_to_blob(tx, bl); if (bl.size() == 0 || !get_transaction_hash(tx, h)) @@ -1041,7 +1040,6 @@ namespace cryptonote return true; }, true, category); - txpool_tx_meta_t meta; for (const key_images_container::value_type& kee : m_spent_key_images) { const crypto::key_image& k_image = kee.first; const std::unordered_set<crypto::hash>& kei_image_set = kee.second; diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.h b/src/cryptonote_protocol/cryptonote_protocol_handler.h index 28530f3e7..80dd2bc39 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.h @@ -46,6 +46,8 @@ #include "block_queue.h" #include "common/perf_timer.h" #include "cryptonote_basic/connection_context.h" +#include "net/levin_base.h" +#include "p2p/net_node_common.h" #include <boost/circular_buffer.hpp> PUSH_WARNINGS @@ -195,10 +197,11 @@ namespace cryptonote bool post_notify(typename t_parameter::request& arg, cryptonote_connection_context& context) { LOG_PRINT_L2("[" << epee::net_utils::print_connection_context_short(context) << "] post " << typeid(t_parameter).name() << " -->"); - epee::byte_slice blob; - epee::serialization::store_t_to_binary(arg, blob, 256 * 1024); // optimize for block responses + + epee::levin::message_writer out{256 * 1024}; // optimize for block responses + epee::serialization::store_t_to_binary(arg, out.buffer); //handler_response_blocks_now(blob.size()); // XXX - return m_p2p->invoke_notify_to_peer(t_parameter::ID, epee::to_span(blob), context); + return m_p2p->invoke_notify_to_peer(t_parameter::ID, std::move(out), context); } }; diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index 22e87465f..afc81f552 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -137,7 +137,42 @@ namespace cryptonote CHECK_AND_ASSERT_MES_CC( context.m_callback_request_count > 0, false, "false callback fired, but context.m_callback_request_count=" << context.m_callback_request_count); --context.m_callback_request_count; - if(context.m_state == cryptonote_connection_context::state_synchronizing) + uint32_t notified = true; + if (context.m_idle_peer_notification.compare_exchange_strong(notified, not notified)) + { + if (context.m_state == cryptonote_connection_context::state_synchronizing && context.m_last_request_time != boost::date_time::not_a_date_time) + { + const boost::posix_time::ptime now = boost::posix_time::microsec_clock::universal_time(); + const boost::posix_time::time_duration dt = now - context.m_last_request_time; + const auto ms = dt.total_microseconds(); + if (ms > IDLE_PEER_KICK_TIME || (context.m_expect_response && ms > NON_RESPONSIVE_PEER_KICK_TIME)) + { + if (context.m_score-- >= 0) + { + MINFO(context << " kicking idle peer, last update " << (dt.total_microseconds() / 1.e6) << " seconds ago, expecting " << (int)context.m_expect_response); + context.m_last_request_time = boost::date_time::not_a_date_time; + context.m_expect_response = 0; + context.m_expect_height = 0; + context.m_state = cryptonote_connection_context::state_standby; // we'll go back to adding, then (if we can't), download + } + else + { + MINFO(context << "dropping idle peer with negative score"); + drop_connection_with_score(context, context.m_expect_response == 0 ? 1 : 5, false); + return false; + } + } + } + } + + notified = true; + if (context.m_new_stripe_notification.compare_exchange_strong(notified, not notified)) + { + if (context.m_state == cryptonote_connection_context::state_normal) + context.m_state = cryptonote_connection_context::state_synchronizing; + } + + if(context.m_state == cryptonote_connection_context::state_synchronizing && context.m_last_request_time == boost::posix_time::not_a_date_time) { NOTIFY_REQUEST_CHAIN::request r = {}; context.m_needed_objects.clear(); @@ -338,10 +373,6 @@ namespace cryptonote } context.m_remote_blockchain_height = hshd.current_height; context.m_pruning_seed = hshd.pruning_seed; -#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED - context.m_pruning_seed = tools::make_pruning_seed(1 + (context.m_remote_address.as<epee::net_utils::ipv4_network_address>().ip()) % (1 << CRYPTONOTE_PRUNING_LOG_STRIPES), CRYPTONOTE_PRUNING_LOG_STRIPES); - LOG_INFO_CC(context, "New connection posing as pruning seed " << epee::string_tools::to_string_hex(context.m_pruning_seed) << ", seed address " << &context.m_pruning_seed); -#endif uint64_t target = m_core.get_target_blockchain_height(); if (target == 0) @@ -549,6 +580,7 @@ namespace cryptonote } std::vector<tx_blob_entry> have_tx; + have_tx.reserve(new_block.tx_hashes.size()); // Instead of requesting missing transactions by hash like BTC, // we do it by index (thanks to a suggestion from moneromooo) because @@ -557,6 +589,7 @@ namespace cryptonote // Also, remember to pepper some whitespace changes around to bother // moneromooo ... only because I <3 him. std::vector<uint64_t> need_tx_indices; + need_tx_indices.reserve(new_block.tx_hashes.size()); transaction tx; crypto::hash tx_hash; @@ -829,6 +862,7 @@ namespace cryptonote } std::vector<crypto::hash> txids; + txids.reserve(b.tx_hashes.size()); NOTIFY_NEW_FLUFFY_BLOCK::request fluffy_response; fluffy_response.b.block = t_serializable_object_to_blob(b); fluffy_response.current_blockchain_height = arg.current_blockchain_height; @@ -1688,7 +1722,7 @@ skip: const uint32_t peer_stripe = tools::get_pruning_stripe(context.m_pruning_seed); if (stripe && peer_stripe && peer_stripe != stripe) return true; - context.m_state = cryptonote_connection_context::state_synchronizing; + context.m_new_stripe_notification = true; LOG_PRINT_CCONTEXT_L2("requesting callback"); ++context.m_callback_request_count; m_p2p->request_callback(context); @@ -1711,7 +1745,6 @@ skip: bool t_cryptonote_protocol_handler<t_core>::kick_idle_peers() { MTRACE("Checking for idle peers..."); - std::vector<std::pair<boost::uuids::uuid, unsigned>> idle_peers; m_p2p->for_each_connection([&](cryptonote_connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags)->bool { if (context.m_state == cryptonote_connection_context::state_synchronizing && context.m_last_request_time != boost::date_time::not_a_date_time) @@ -1721,36 +1754,16 @@ skip: const auto ms = dt.total_microseconds(); if (ms > IDLE_PEER_KICK_TIME || (context.m_expect_response && ms > NON_RESPONSIVE_PEER_KICK_TIME)) { - if (context.m_score-- >= 0) - { - MINFO(context << " kicking idle peer, last update " << (dt.total_microseconds() / 1.e6) << " seconds ago, expecting " << (int)context.m_expect_response); - LOG_PRINT_CCONTEXT_L2("requesting callback"); - context.m_last_request_time = boost::date_time::not_a_date_time; - context.m_expect_response = 0; - context.m_expect_height = 0; - context.m_state = cryptonote_connection_context::state_standby; // we'll go back to adding, then (if we can't), download - ++context.m_callback_request_count; - m_p2p->request_callback(context); - } - else - { - idle_peers.push_back(std::make_pair(context.m_connection_id, context.m_expect_response == 0 ? 1 : 5)); - } + context.m_idle_peer_notification = true; + LOG_PRINT_CCONTEXT_L2("requesting callback"); + ++context.m_callback_request_count; + m_p2p->request_callback(context); + MLOG_PEER_STATE("requesting callback"); } } return true; }); - for (const auto &e: idle_peers) - { - const auto &uuid = e.first; - m_p2p->for_connection(uuid, [&](cryptonote_connection_context& ctx, nodetool::peerid_type peer_id, uint32_t f)->bool{ - MINFO(ctx << "dropping idle peer with negative score"); - drop_connection_with_score(ctx, e.second, false); - return true; - }); - } - return true; } //------------------------------------------------------------------------------------------------------------------------ @@ -1846,10 +1859,8 @@ skip: bool t_cryptonote_protocol_handler<t_core>::should_download_next_span(cryptonote_connection_context& context, bool standby) { std::vector<crypto::hash> hashes; - boost::uuids::uuid span_connection_id; boost::posix_time::ptime request_time; boost::uuids::uuid connection_id; - std::pair<uint64_t, uint64_t> span; bool filled; const uint64_t blockchain_height = m_core.get_current_blockchain_height(); @@ -1875,7 +1886,6 @@ skip: // in standby, be ready to double download early since we're idling anyway // let the fastest peer trigger first - long threshold; const double dl_speed = context.m_max_speed_down; if (standby && dt >= REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD_STANDBY && dl_speed > 0) { @@ -2189,6 +2199,7 @@ skip: if (span.second > 0) { is_next = true; + req.blocks.reserve(hashes.size()); for (const auto &hash: hashes) { req.blocks.push_back(hash); @@ -2247,6 +2258,7 @@ skip: if (span.second > 0) { is_next = true; + req.blocks.reserve(hashes.size()); for (const auto &hash: hashes) { req.blocks.push_back(hash); @@ -2280,6 +2292,7 @@ skip: return false; } + req.blocks.reserve(req.blocks.size() + span.second); for (size_t n = 0; n < span.second; ++n) { req.blocks.push_back(context.m_needed_objects[n].first); @@ -2570,17 +2583,6 @@ skip: return 1; } - std::unordered_set<crypto::hash> hashes; - for (const auto &h: arg.m_block_ids) - { - if (!hashes.insert(h).second) - { - LOG_ERROR_CCONTEXT("sent duplicate block, dropping connection"); - drop_connection(context, true, false); - return 1; - } - } - uint64_t n_use_blocks = m_core.prevalidate_block_hashes(arg.start_height, arg.m_block_ids, arg.m_block_weights); if (n_use_blocks == 0 || n_use_blocks + HASH_OF_HASHES_STEP <= arg.m_block_ids.size()) { @@ -2590,20 +2592,77 @@ skip: } context.m_needed_objects.clear(); + context.m_needed_objects.reserve(arg.m_block_ids.size()); uint64_t added = 0; std::unordered_set<crypto::hash> blocks_found; + bool first = true; + bool expect_unknown = false; for (size_t i = 0; i < arg.m_block_ids.size(); ++i) { if (!blocks_found.insert(arg.m_block_ids[i]).second) { LOG_ERROR_CCONTEXT("Duplicate blocks in chain entry response, dropping connection"); - drop_connection(context, true, false); + drop_connection_with_score(context, 5, false); return 1; } + int where; + const bool have_block = m_core.have_block_unlocked(arg.m_block_ids[i], &where); + if (first) + { + if (!have_block && !m_block_queue.requested(arg.m_block_ids[i]) && !m_block_queue.have(arg.m_block_ids[i])) + { + LOG_ERROR_CCONTEXT("First block hash is unknown, dropping connection"); + drop_connection_with_score(context, 5, false); + return 1; + } + if (!have_block) + expect_unknown = true; + } + if (!first) + { + // after the first, blocks may be known or unknown, but if they are known, + // they should be at the same height if on the main chain + if (have_block) + { + switch (where) + { + default: + case HAVE_BLOCK_INVALID: + LOG_ERROR_CCONTEXT("Block is invalid or known without known type, dropping connection"); + drop_connection(context, true, false); + return 1; + case HAVE_BLOCK_MAIN_CHAIN: + if (expect_unknown) + { + LOG_ERROR_CCONTEXT("Block is on the main chain, but we did not expect a known block, dropping connection"); + drop_connection_with_score(context, 5, false); + return 1; + } + if (m_core.get_block_id_by_height(arg.start_height + i) != arg.m_block_ids[i]) + { + LOG_ERROR_CCONTEXT("Block is on the main chain, but not at the expected height, dropping connection"); + drop_connection_with_score(context, 5, false); + return 1; + } + break; + case HAVE_BLOCK_ALT_CHAIN: + if (expect_unknown) + { + LOG_ERROR_CCONTEXT("Block is on the main chain, but we did not expect a known block, dropping connection"); + drop_connection_with_score(context, 5, false); + return 1; + } + break; + } + } + else + expect_unknown = true; + } const uint64_t block_weight = arg.m_block_weights.empty() ? 0 : arg.m_block_weights[i]; context.m_needed_objects.push_back(std::make_pair(arg.m_block_ids[i], block_weight)); if (++added == n_use_blocks) break; + first = false; } context.m_last_response_height -= arg.m_block_ids.size() - n_use_blocks; @@ -2634,6 +2693,7 @@ skip: std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> fullConnections, fluffyConnections; m_p2p->for_each_connection([this, &exclude_context, &fullConnections, &fluffyConnections](connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags) { + // peer_id also filters out connections before handshake if (peer_id && exclude_context.m_connection_id != context.m_connection_id && context.m_remote_address.get_zone() == epee::net_utils::zone::public_) { if(m_core.fluffy_blocks_enabled() && (support_flags & P2P_SUPPORT_FLAG_FLUFFY_BLOCKS)) @@ -2653,15 +2713,15 @@ skip: // send fluffy ones first, we want to encourage people to run that if (!fluffyConnections.empty()) { - epee::byte_slice fluffyBlob; - epee::serialization::store_t_to_binary(fluffy_arg, fluffyBlob, 32 * 1024); - m_p2p->relay_notify_to_list(NOTIFY_NEW_FLUFFY_BLOCK::ID, epee::to_span(fluffyBlob), std::move(fluffyConnections)); + epee::levin::message_writer fluffyBlob{32 * 1024}; + epee::serialization::store_t_to_binary(fluffy_arg, fluffyBlob.buffer); + m_p2p->relay_notify_to_list(NOTIFY_NEW_FLUFFY_BLOCK::ID, std::move(fluffyBlob), std::move(fluffyConnections)); } if (!fullConnections.empty()) { - epee::byte_slice fullBlob; - epee::serialization::store_t_to_binary(arg, fullBlob, 128 * 1024); - m_p2p->relay_notify_to_list(NOTIFY_NEW_BLOCK::ID, epee::to_span(fullBlob), std::move(fullConnections)); + epee::levin::message_writer fullBlob{128 * 1024}; + epee::serialization::store_t_to_binary(arg, fullBlob.buffer); + m_p2p->relay_notify_to_list(NOTIFY_NEW_BLOCK::ID, std::move(fullBlob), std::move(fullConnections)); } return true; diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler_common.h b/src/cryptonote_protocol/cryptonote_protocol_handler_common.h index 79c2edf1d..57b1d049c 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler_common.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler_common.h @@ -30,8 +30,8 @@ #pragma once -#include "p2p/net_node_common.h" #include "cryptonote_protocol/cryptonote_protocol_defs.h" +#include "cryptonote_protocol/enums.h" #include "cryptonote_basic/connection_context.h" namespace cryptonote { diff --git a/src/cryptonote_protocol/levin_notify.cpp b/src/cryptonote_protocol/levin_notify.cpp index 21363972d..0b065c3c3 100644 --- a/src/cryptonote_protocol/levin_notify.cpp +++ b/src/cryptonote_protocol/levin_notify.cpp @@ -51,14 +51,6 @@ #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net.p2p.tx" -namespace -{ - int get_command_from_message(const epee::byte_slice &msg) - { - return msg.size() >= sizeof(epee::levin::bucket_head2) ? SWAP32LE(((epee::levin::bucket_head2*)msg.data())->m_command) : 0; - } -} - namespace cryptonote { namespace levin @@ -167,7 +159,7 @@ namespace levin return get_out_connections(p2p, get_blockchain_height(p2p, core)); } - epee::byte_slice make_tx_payload(std::vector<blobdata>&& txs, const bool pad, const bool fluff) + epee::levin::message_writer make_tx_message(std::vector<blobdata>&& txs, const bool pad, const bool fluff) { NOTIFY_NEW_TRANSACTIONS::request request{}; request.txs = std::move(txs); @@ -201,21 +193,17 @@ namespace levin // if the size of _ moved enough, we might lose byte in size encoding, we don't care } - epee::byte_slice fullBlob; - if (!epee::serialization::store_t_to_binary(request, fullBlob)) + epee::levin::message_writer out; + if (!epee::serialization::store_t_to_binary(request, out.buffer)) throw std::runtime_error{"Failed to serialize to epee binary format"}; - return fullBlob; + return out; } bool make_payload_send_txs(connections& p2p, std::vector<blobdata>&& txs, const boost::uuids::uuid& destination, const bool pad, const bool fluff) { - const epee::byte_slice blob = make_tx_payload(std::move(txs), pad, fluff); - p2p.for_connection(destination, [&blob](detail::p2p_context& context) { - on_levin_traffic(context, true, true, false, blob.size(), get_command_from_message(blob)); - return true; - }); - return p2p.notify(NOTIFY_NEW_TRANSACTIONS::ID, epee::to_span(blob), destination); + epee::byte_slice blob = make_tx_message(std::move(txs), pad, fluff).finalize_notify(NOTIFY_NEW_TRANSACTIONS::ID); + return p2p.send(std::move(blob), destination); } /* The current design uses `asio::strand`s. The documentation isn't as clear @@ -443,7 +431,7 @@ namespace levin zone->p2p->foreach_connection([txs, now, &zone, &source, &in_duration, &out_duration, &next_flush] (detail::p2p_context& context) { // When i2p/tor, only fluff to outbound connections - if (source != context.m_connection_id && (zone->nzone == epee::net_utils::zone::public_ || !context.m_is_income)) + if (context.handshake_complete() && source != context.m_connection_id && (zone->nzone == epee::net_utils::zone::public_ || !context.m_is_income)) { if (context.fluff_txs.empty()) context.flush_time = now + (context.m_is_income ? in_duration() : out_duration()); @@ -661,10 +649,6 @@ namespace levin else message = zone_->noise.clone(); - zone_->p2p->for_connection(channel.connection, [&](detail::p2p_context& context) { - on_levin_traffic(context, true, true, false, message.size(), "noise"); - return true; - }); if (zone_->p2p->send(std::move(message), channel.connection)) { if (!channel.queue.empty() && channel.active.empty()) @@ -824,9 +808,8 @@ namespace levin // Padding is not useful when using noise mode. Send as stem so receiver // forwards in Dandelion++ mode. - const epee::byte_slice payload = make_tx_payload(std::move(txs), false, false); epee::byte_slice message = epee::levin::make_fragmented_notify( - zone_->noise, NOTIFY_NEW_TRANSACTIONS::ID, epee::to_span(payload) + zone_->noise.size(), NOTIFY_NEW_TRANSACTIONS::ID, make_tx_message(std::move(txs), false, false) ); if (CRYPTONOTE_MAX_FRAGMENTS * zone_->noise.size() < message.size()) { diff --git a/src/daemon/command_parser_executor.cpp b/src/daemon/command_parser_executor.cpp index 5a7560874..abca614cc 100644 --- a/src/daemon/command_parser_executor.cpp +++ b/src/daemon/command_parser_executor.cpp @@ -30,6 +30,7 @@ #include "common/command_line.h" #include "net/parse.h" #include "daemon/command_parser_executor.h" +#include <boost/filesystem.hpp> #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "daemon" @@ -986,17 +987,67 @@ bool t_command_parser_executor::check_blockchain_pruning(const std::vector<std:: bool t_command_parser_executor::set_bootstrap_daemon(const std::vector<std::string>& args) { - const size_t args_count = args.size(); - if (args_count < 1 || args_count > 3) + struct parsed_t + { + std::string address; + std::string user; + std::string password; + std::string proxy; + }; + + boost::optional<parsed_t> parsed = [&args]() -> boost::optional<parsed_t> { + const size_t args_count = args.size(); + if (args_count == 0) + { + return {}; + } + if (args[0] == "auto") + { + if (args_count == 1) + { + return {{args[0], "", "", ""}}; + } + if (args_count == 2) + { + return {{args[0], "", "", args[1]}}; + } + } + else if (args[0] == "none") + { + if (args_count == 1) + { + return {{"", "", "", ""}}; + } + } + else + { + if (args_count == 1) + { + return {{args[0], "", "", ""}}; + } + if (args_count == 2) + { + return {{args[0], "", "", args[1]}}; + } + if (args_count == 3) + { + return {{args[0], args[1], args[2], ""}}; + } + if (args_count == 4) + { + return {{args[0], args[1], args[2], args[3]}}; + } + } + return {}; + }(); + + if (!parsed) { std::cout << "Invalid syntax: Wrong number of parameters. For more details, use the help command." << std::endl; return true; } - return m_executor.set_bootstrap_daemon( - args[0] != "none" ? args[0] : std::string(), - args_count > 1 ? args[1] : std::string(), - args_count > 2 ? args[2] : std::string()); + return m_executor.set_bootstrap_daemon(parsed->address, parsed->user, parsed->password, parsed->proxy); } bool t_command_parser_executor::flush_cache(const std::vector<std::string>& args) diff --git a/src/daemon/command_server.cpp b/src/daemon/command_server.cpp index 4768bb842..63f44c4cd 100644 --- a/src/daemon/command_server.cpp +++ b/src/daemon/command_server.cpp @@ -326,7 +326,7 @@ t_command_server::t_command_server( m_command_lookup.set_handler( "set_bootstrap_daemon" , std::bind(&t_command_parser_executor::set_bootstrap_daemon, &m_parser, p::_1) - , "set_bootstrap_daemon (auto | none | host[:port] [username] [password])" + , "set_bootstrap_daemon (auto | none | host[:port] [username] [password]) [proxy_ip:proxy_port]" , "URL of a 'bootstrap' remote daemon that the connected wallets can use while this daemon is still not fully synced.\n" "Use 'auto' to enable automatic public nodes discovering and bootstrap daemon switching" ); diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp index 04feb55fd..16e6a304c 100644 --- a/src/daemon/rpc_command_executor.cpp +++ b/src/daemon/rpc_command_executor.cpp @@ -1059,7 +1059,6 @@ bool t_rpc_command_executor::print_transaction(crypto::hash transaction_hash, // Print json if requested if (include_json) { - crypto::hash tx_hash, tx_prefix_hash; cryptonote::transaction tx; cryptonote::blobdata blob; std::string source = as_hex.empty() ? pruned_as_hex + prunable_as_hex : as_hex; @@ -2275,6 +2274,7 @@ bool t_rpc_command_executor::sync_info() tools::success_msg_writer() << "Next needed pruning seed: " << res.next_needed_pruning_seed; tools::success_msg_writer() << std::to_string(res.peers.size()) << " peers"; + tools::success_msg_writer() << "Remote Host Peer_ID State Prune_Seed Height DL kB/s, Queued Blocks / MB"; for (const auto &p: res.peers) { std::string address = epee::string_tools::pad_string(p.info.address, 24); @@ -2405,7 +2405,8 @@ bool t_rpc_command_executor::check_blockchain_pruning() bool t_rpc_command_executor::set_bootstrap_daemon( const std::string &address, const std::string &username, - const std::string &password) + const std::string &password, + const std::string &proxy) { cryptonote::COMMAND_RPC_SET_BOOTSTRAP_DAEMON::request req; cryptonote::COMMAND_RPC_SET_BOOTSTRAP_DAEMON::response res; @@ -2414,6 +2415,7 @@ bool t_rpc_command_executor::set_bootstrap_daemon( req.address = address; req.username = username; req.password = password; + req.proxy = proxy; if (m_is_rpc) { diff --git a/src/daemon/rpc_command_executor.h b/src/daemon/rpc_command_executor.h index 6fb5d6903..118f04731 100644 --- a/src/daemon/rpc_command_executor.h +++ b/src/daemon/rpc_command_executor.h @@ -168,7 +168,8 @@ public: bool set_bootstrap_daemon( const std::string &address, const std::string &username, - const std::string &password); + const std::string &password, + const std::string &proxy); bool rpc_payments(); diff --git a/src/debug_utilities/cn_deserialize.cpp b/src/debug_utilities/cn_deserialize.cpp index dd4701e4f..c039b93c5 100644 --- a/src/debug_utilities/cn_deserialize.cpp +++ b/src/debug_utilities/cn_deserialize.cpp @@ -133,6 +133,18 @@ int main(int argc, char* argv[]) { std::cout << "Parsed block:" << std::endl; std::cout << cryptonote::obj_to_json_str(block) << std::endl; + bool parsed = cryptonote::parse_tx_extra(block.miner_tx.extra, fields); + if (!parsed) + std::cout << "Failed to parse tx_extra" << std::endl; + + if (!fields.empty()) + { + print_extra_fields(fields); + } + else + { + std::cout << "No fields were found in tx_extra" << std::endl; + } } else if (cryptonote::parse_and_validate_tx_from_blob(blob, tx) || cryptonote::parse_and_validate_tx_base_from_blob(blob, tx)) { diff --git a/src/debug_utilities/dns_checks.cpp b/src/debug_utilities/dns_checks.cpp index 76b66c6cb..138cd4fc1 100644 --- a/src/debug_utilities/dns_checks.cpp +++ b/src/debug_utilities/dns_checks.cpp @@ -131,7 +131,7 @@ int main(int argc, char* argv[]) lookup(LOOKUP_A, {"seeds.moneroseeds.se", "seeds.moneroseeds.ae.org", "seeds.moneroseeds.ch", "seeds.moneroseeds.li"}); - lookup(LOOKUP_TXT, {"updates.moneropulse.org", "updates.moneropulse.net", "updates.moneropulse.co", "updates.moneropulse.se"}); + lookup(LOOKUP_TXT, {"updates.moneropulse.org", "updates.moneropulse.net", "updates.moneropulse.co", "updates.moneropulse.se", "updates.moneropulse.fr", "updates.moneropulse.de", "updates.moneropulse.no", "updates.moneropulse.ch"}); lookup(LOOKUP_TXT, {"checkpoints.moneropulse.org", "checkpoints.moneropulse.net", "checkpoints.moneropulse.co", "checkpoints.moneropulse.se"}); diff --git a/src/device/device_io_hid.cpp b/src/device/device_io_hid.cpp index 7c61c3b1a..7aa5b39bf 100644 --- a/src/device/device_io_hid.cpp +++ b/src/device/device_io_hid.cpp @@ -181,7 +181,6 @@ namespace hw { unsigned char padding_buffer[MAX_BLOCK+1]; unsigned int result; int hid_ret; - unsigned int sw_offset; unsigned int remaining; unsigned int offset = 0; diff --git a/src/device_trezor/device_trezor_base.cpp b/src/device_trezor/device_trezor_base.cpp index f59be1573..70dc7f539 100644 --- a/src/device_trezor/device_trezor_base.cpp +++ b/src/device_trezor/device_trezor_base.cpp @@ -365,15 +365,14 @@ namespace trezor { void device_trezor_base::device_state_initialize_unsafe() { require_connected(); - std::string tmp_session_id; auto initMsg = std::make_shared<messages::management::Initialize>(); const auto data_cleaner = epee::misc_utils::create_scope_leave_handler([&]() { - memwipe(&tmp_session_id[0], tmp_session_id.size()); + if (initMsg->has_session_id()) + memwipe(&(*initMsg->mutable_session_id())[0], initMsg->mutable_session_id()->size()); }); if(!m_device_session_id.empty()) { - tmp_session_id.assign(m_device_session_id.data(), m_device_session_id.size()); - initMsg->set_allocated_session_id(&tmp_session_id); + initMsg->set_allocated_session_id(new std::string(m_device_session_id.data(), m_device_session_id.size())); } m_features = this->client_exchange<messages::management::Features>(initMsg); @@ -382,8 +381,6 @@ namespace trezor { } else { m_device_session_id.clear(); } - - initMsg->release_session_id(); } void device_trezor_base::device_state_reset() @@ -453,18 +450,14 @@ namespace trezor { pin = m_pin; } - std::string pin_field; messages::common::PinMatrixAck m; if (pin) { - pin_field.assign(pin->data(), pin->size()); - m.set_allocated_pin(&pin_field); + m.set_allocated_pin(new std::string(pin->data(), pin->size())); } const auto data_cleaner = epee::misc_utils::create_scope_leave_handler([&]() { - m.release_pin(); - if (!pin_field.empty()){ - memwipe(&pin_field[0], pin_field.size()); - } + if (m.has_pin()) + memwipe(&(*m.mutable_pin())[0], m.mutable_pin()->size()); }); resp = call_raw(&m); @@ -499,7 +492,6 @@ namespace trezor { boost::optional<epee::wipeable_string> passphrase; TREZOR_CALLBACK_GET(passphrase, on_passphrase_request, on_device); - std::string passphrase_field; messages::common::PassphraseAck m; m.set_on_device(on_device); if (!on_device) { @@ -512,16 +504,13 @@ namespace trezor { } if (passphrase) { - passphrase_field.assign(passphrase->data(), passphrase->size()); - m.set_allocated_passphrase(&passphrase_field); + m.set_allocated_passphrase(new std::string(passphrase->data(), passphrase->size())); } } const auto data_cleaner = epee::misc_utils::create_scope_leave_handler([&]() { - m.release_passphrase(); - if (!passphrase_field.empty()){ - memwipe(&passphrase_field[0], passphrase_field.size()); - } + if (m.has_passphrase()) + memwipe(&(m.mutable_passphrase())[0], m.mutable_passphrase()->size()); }); resp = call_raw(&m); diff --git a/src/device_trezor/device_trezor_base.hpp b/src/device_trezor/device_trezor_base.hpp index 4db8f0c8e..0162b23df 100644 --- a/src/device_trezor/device_trezor_base.hpp +++ b/src/device_trezor/device_trezor_base.hpp @@ -165,7 +165,7 @@ namespace trezor { // Scoped session closer BOOST_SCOPE_EXIT_ALL(&, this) { - if (open_session){ + if (open_session && this->get_transport()){ this->get_transport()->close(); } }; diff --git a/src/device_trezor/trezor/protocol.cpp b/src/device_trezor/trezor/protocol.cpp index 288f3ddca..92150b579 100644 --- a/src/device_trezor/trezor/protocol.cpp +++ b/src/device_trezor/trezor/protocol.cpp @@ -502,21 +502,9 @@ namespace tx { } void Signer::compute_integrated_indices(TsxData * tsx_data){ - if (m_aux_data == nullptr || m_aux_data->tx_recipients.empty()){ - return; - } - auto & chg = tsx_data->change_dts(); std::string change_hash = hash_addr(&chg.addr(), chg.amount(), chg.is_subaddress()); - std::vector<uint32_t> integrated_indices; - std::set<std::string> integrated_hashes; - for (auto & cur : m_aux_data->tx_recipients){ - if (!cur.has_payment_id){ - continue; - } - integrated_hashes.emplace(hash_addr(&cur.address.m_spend_public_key, &cur.address.m_view_public_key)); - } ssize_t idx = -1; for (auto & cur : tsx_data->outputs()){ @@ -527,8 +515,7 @@ namespace tx { continue; } - c_hash = hash_addr(&cur.addr()); - if (integrated_hashes.find(c_hash) != integrated_hashes.end()){ + if (cur.is_integrated()){ integrated_indices.push_back((uint32_t)idx); } } diff --git a/src/gen_multisig/gen_multisig.cpp b/src/gen_multisig/gen_multisig.cpp index 15f7e5c0a..4aa21b149 100644 --- a/src/gen_multisig/gen_multisig.cpp +++ b/src/gen_multisig/gen_multisig.cpp @@ -74,8 +74,6 @@ namespace const command_line::arg_descriptor<bool, false> arg_testnet = {"testnet", genms::tr("Create testnet multisig wallets"), false}; const command_line::arg_descriptor<bool, false> arg_stagenet = {"stagenet", genms::tr("Create stagenet multisig wallets"), false}; const command_line::arg_descriptor<bool, false> arg_create_address_file = {"create-address-file", genms::tr("Create an address file for new wallets"), false}; - - const command_line::arg_descriptor< std::vector<std::string> > arg_command = {"command", ""}; } static bool generate_multisig(uint32_t threshold, uint32_t total, const std::string &basename, network_type nettype, bool create_address_file) diff --git a/src/mnemonics/electrum-words.cpp b/src/mnemonics/electrum-words.cpp index bda18d950..8c79a53ca 100644 --- a/src/mnemonics/electrum-words.cpp +++ b/src/mnemonics/electrum-words.cpp @@ -194,7 +194,6 @@ namespace { epee::wipeable_string trimmed_words = "", word; - const auto &word_map = language->get_word_map(); const auto &trimmed_word_map = language->get_trimmed_word_map(); const uint32_t unique_prefix_length = language->get_unique_prefix_length(); for (std::vector<epee::wipeable_string>::const_iterator it = word_list.begin(); it != word_list.end(); it++) @@ -492,6 +491,14 @@ namespace crypto return "<language not found>"; } + bool is_valid_language(const std::string &language) + { + const std::vector<const Language::Base*> language_instances = get_language_list(); + for (std::vector<const Language::Base*>::const_iterator it = language_instances.begin(); it != language_instances.end(); it++) + if ((*it)->get_english_language_name() == language || (*it)->get_language_name() == language) + return true; + return false; + } } } diff --git a/src/mnemonics/electrum-words.h b/src/mnemonics/electrum-words.h index 8d4c5be66..eb0c99e0b 100644 --- a/src/mnemonics/electrum-words.h +++ b/src/mnemonics/electrum-words.h @@ -124,6 +124,8 @@ namespace crypto * \return the name of the language in English */ std::string get_english_name_for(const std::string &name); + + bool is_valid_language(const std::string &language); } } diff --git a/src/mnemonics/language_base.h b/src/mnemonics/language_base.h index bf8793aa2..1aa869e45 100644 --- a/src/mnemonics/language_base.h +++ b/src/mnemonics/language_base.h @@ -129,7 +129,7 @@ namespace Language if ((*it).size() < unique_prefix_length)
{
if (flags & ALLOW_SHORT_WORDS)
- MWARNING(language_name << " word '" << *it << "' is shorter than its prefix length, " << unique_prefix_length);
+ MINFO(language_name << " word '" << *it << "' is shorter than its prefix length, " << unique_prefix_length);
else
throw std::runtime_error("Too short word in " + language_name + " word list: " + *it);
}
diff --git a/src/net/CMakeLists.txt b/src/net/CMakeLists.txt index afcd42ef7..e93e27bcd 100644 --- a/src/net/CMakeLists.txt +++ b/src/net/CMakeLists.txt @@ -26,10 +26,10 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -set(net_sources dandelionpp.cpp error.cpp http.cpp i2p_address.cpp parse.cpp socks.cpp - socks_connect.cpp tor_address.cpp zmq.cpp) -set(net_headers dandelionpp.h error.h http.cpp i2p_address.h parse.h socks.h socks_connect.h - tor_address.h zmq.h) +set(net_sources dandelionpp.cpp error.cpp http.cpp i2p_address.cpp parse.cpp resolve.cpp + socks.cpp socks_connect.cpp tor_address.cpp zmq.cpp) +set(net_headers dandelionpp.h error.h http.cpp i2p_address.h parse.h socks.h resolve.h + socks_connect.h tor_address.h zmq.h) monero_add_library(net ${net_sources} ${net_headers}) target_link_libraries(net common epee ${ZMQ_LIB} ${Boost_ASIO_LIBRARY}) diff --git a/src/net/error.cpp b/src/net/error.cpp index 037f44d52..d2e713bc5 100644 --- a/src/net/error.cpp +++ b/src/net/error.cpp @@ -47,12 +47,18 @@ namespace { switch (net::error(value)) { + case net::error::bogus_dnssec: + return "Invalid response signature from DNSSEC enabled domain"; + case net::error::dns_query_failure: + return "Failed to retrieve desired DNS record"; case net::error::expected_tld: return "Expected top-level domain"; case net::error::invalid_host: return "Host value is not valid"; case net::error::invalid_i2p_address: return "Invalid I2P address"; + case net::error::invalid_mask: + return "CIDR netmask outside of 0-32 range"; case net::error::invalid_port: return "Invalid port value (expected 0-65535)"; case net::error::invalid_tor_address: @@ -71,6 +77,7 @@ namespace switch (net::error(value)) { case net::error::invalid_port: + case net::error::invalid_mask: return std::errc::result_out_of_range; case net::error::expected_tld: case net::error::invalid_tor_address: diff --git a/src/net/error.h b/src/net/error.h index 7c852dd20..746eb0ecb 100644 --- a/src/net/error.h +++ b/src/net/error.h @@ -37,13 +37,16 @@ namespace net enum class error : int { // 0 reserved for success (as per expect<T>) - expected_tld = 1, //!< Expected a tld + bogus_dnssec = 1, //!< Invalid response signature from DNSSEC enabled domain + dns_query_failure, //!< Failed to retrieve desired DNS record + expected_tld, //!< Expected a tld invalid_host, //!< Hostname is not valid invalid_i2p_address, + invalid_mask, //!< Outside of 0-32 range invalid_port, //!< Outside of 0-65535 range invalid_tor_address,//!< Invalid base32 or length unsupported_address,//!< Type not supported by `get_network_address` - invalid_mask, //!< Outside of 0-32 range + }; //! \return `std::error_category` for `net` namespace. diff --git a/src/net/resolve.cpp b/src/net/resolve.cpp new file mode 100644 index 000000000..1b43cf6c4 --- /dev/null +++ b/src/net/resolve.cpp @@ -0,0 +1,71 @@ +// Copyright (c) 2020, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "net/resolve.h" + +#include <boost/utility/string_ref.hpp> +#include "common/dns_utils.h" +#include "common/expect.h" +#include "net/error.h" + +namespace net +{ +namespace dnssec +{ + expect<service_response> resolve_hostname(const std::string& addr, const std::string& tlsa_port) + { + // use basic (blocking) unbound for now, possibly refactor later + tools::DNSResolver& resolver = tools::DNSResolver::instance(); + + bool dnssec_available = false; + bool dnssec_valid = false; + std::vector<std::string> ip_records = resolver.get_ipv4(addr, dnssec_available, dnssec_valid); + + if (dnssec_available && !dnssec_valid) + return {net::error::bogus_dnssec}; + + if (ip_records.empty()) + { + ip_records = resolver.get_ipv6(addr, dnssec_available, dnssec_valid); + if (dnssec_available && !dnssec_valid) + return {net::error::bogus_dnssec}; + if (ip_records.empty()) + return {net::error::dns_query_failure}; + } + + std::vector<std::string> tlsa{}; + if (dnssec_available && !tlsa_port.empty()) + { + tlsa = resolver.get_tlsa_tcp_record(addr, tlsa_port, dnssec_available, dnssec_valid); + if (!dnssec_valid) + return {net::error::bogus_dnssec}; + } + return {{std::move(ip_records), std::move(tlsa)}}; + } +} // dnssec +} // net diff --git a/src/net/resolve.h b/src/net/resolve.h new file mode 100644 index 000000000..46bd8e617 --- /dev/null +++ b/src/net/resolve.h @@ -0,0 +1,47 @@ +// Copyright (c) 2020, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <string> +#include <vector> + +template<typename> class expect; + +namespace net +{ +namespace dnssec +{ + struct service_response + { + std::vector<std::string> ip; //!< IPv4/6 records in dotted or semicolon notation + std::vector<std::string> tlsa; //!< DANE/TLSA records + }; + + //! \return IP + (optionally) DANE/TLSA records, failing if DNSSEC signature is "bogus" + expect<service_response> resolve_hostname(const std::string& addr, const std::string& tlsa_port = {}); +} // dnssec +} // net diff --git a/src/net/socks.cpp b/src/net/socks.cpp index 3463f452c..c2330bd41 100644 --- a/src/net/socks.cpp +++ b/src/net/socks.cpp @@ -48,7 +48,6 @@ namespace socks { namespace { - constexpr const unsigned v4_reply_size = 8; constexpr const std::uint8_t v4_connect_command = 1; constexpr const std::uint8_t v4tor_resolve_command = 0xf0; constexpr const std::uint8_t v4_request_granted = 90; diff --git a/src/p2p/net_node.cpp b/src/p2p/net_node.cpp index 8dd551d1e..84cc1581e 100644 --- a/src/p2p/net_node.cpp +++ b/src/p2p/net_node.cpp @@ -166,6 +166,7 @@ namespace nodetool const command_line::arg_descriptor<bool> arg_pad_transactions = { "pad-transactions", "Pad relayed transactions to help defend against traffic volume analysis", false }; + const command_line::arg_descriptor<uint32_t> arg_max_connections_per_ip = {"max-connections-per-ip", "Maximum number of connections allowed from the same IP address", 1}; boost::optional<std::vector<proxy>> get_proxies(boost::program_options::variables_map const& vm) { diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index 9fba5d636..f2888674b 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -139,6 +139,7 @@ namespace nodetool typedef COMMAND_HANDSHAKE_T<typename t_payload_net_handler::payload_type> COMMAND_HANDSHAKE; typedef COMMAND_TIMED_SYNC_T<typename t_payload_net_handler::payload_type> COMMAND_TIMED_SYNC; + static_assert(p2p_connection_context::handshake_command() == COMMAND_HANDSHAKE::ID, "invalid handshake command id"); typedef epee::net_utils::boosted_tcp_server<epee::levin::async_protocol_handler<p2p_connection_context>> net_server; @@ -257,7 +258,8 @@ namespace nodetool m_igd(no_igd), m_offline(false), is_closing(false), - m_network_id() + m_network_id(), + max_connections(1) {} virtual ~node_server(); @@ -342,10 +344,9 @@ namespace nodetool virtual void on_connection_close(p2p_connection_context& context); virtual void callback(p2p_connection_context& context); //----------------- i_p2p_endpoint ------------------------------------------------------------- - virtual bool relay_notify_to_list(int command, const epee::span<const uint8_t> data_buff, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections); + virtual bool relay_notify_to_list(int command, epee::levin::message_writer message, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections) final; virtual epee::net_utils::zone send_txs(std::vector<cryptonote::blobdata> txs, const epee::net_utils::zone origin, const boost::uuids::uuid& source, cryptonote::relay_method tx_relay); - virtual bool invoke_command_to_peer(int command, const epee::span<const uint8_t> req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context); - virtual bool invoke_notify_to_peer(int command, const epee::span<const uint8_t> req_buff, const epee::net_utils::connection_context_base& context); + virtual bool invoke_notify_to_peer(int command, epee::levin::message_writer message, const epee::net_utils::connection_context_base& context) final; virtual bool drop_connection(const epee::net_utils::connection_context_base& context); virtual void request_callback(const epee::net_utils::connection_context_base& context); virtual void for_each_connection(std::function<bool(typename t_payload_net_handler::connection_context&, peerid_type, uint32_t)> f); @@ -516,6 +517,8 @@ namespace nodetool epee::net_utils::ssl_support_t m_ssl_support; bool m_enable_dns_blocklist; + + uint32_t max_connections; }; const int64_t default_limit_up = P2P_DEFAULT_LIMIT_RATE_UP; // kB/s @@ -550,6 +553,7 @@ namespace nodetool extern const command_line::arg_descriptor<int64_t> arg_limit_rate_down; extern const command_line::arg_descriptor<int64_t> arg_limit_rate; extern const command_line::arg_descriptor<bool> arg_pad_transactions; + extern const command_line::arg_descriptor<uint32_t> arg_max_connections_per_ip; } POP_WARNINGS diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index 315ebfdde..e1d6d1e10 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -131,6 +131,7 @@ namespace nodetool command_line::add_arg(desc, arg_limit_rate_down); command_line::add_arg(desc, arg_limit_rate); command_line::add_arg(desc, arg_pad_transactions); + command_line::add_arg(desc, arg_max_connections_per_ip); } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> @@ -234,6 +235,7 @@ namespace nodetool return false; const time_t now = time(nullptr); + bool added = false; CRITICAL_REGION_LOCAL(m_blocked_hosts_lock); time_t limit; @@ -244,7 +246,10 @@ namespace nodetool const std::string host_str = addr.host_str(); auto it = m_blocked_hosts.find(host_str); if (it == m_blocked_hosts.end()) + { m_blocked_hosts[host_str] = limit; + added = true; + } else if (it->second < limit || !add_only) it->second = limit; @@ -275,7 +280,10 @@ namespace nodetool conns.clear(); } - MCLOG_CYAN(el::Level::Info, "global", "Host " << host_str << " blocked."); + if (added) + MCLOG_CYAN(el::Level::Info, "global", "Host " << host_str << " blocked."); + else + MINFO("Host " << host_str << " block time updated."); return true; } //----------------------------------------------------------------------------------- @@ -608,6 +616,8 @@ namespace nodetool return false; } + max_connections = command_line::get_arg(vm, arg_max_connections_per_ip); + return true; } //----------------------------------------------------------------------------------- @@ -887,32 +897,6 @@ namespace nodetool for(const auto& p: m_command_line_peers) m_network_zones.at(p.adr.get_zone()).m_peerlist.append_with_peer_white(p); -// all peers are now setup -#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED - for (auto& zone : m_network_zones) - { - std::list<peerlist_entry> plw; - while (zone.second.m_peerlist.get_white_peers_count()) - { - plw.push_back(peerlist_entry()); - zone.second.m_peerlist.get_white_peer_by_index(plw.back(), 0); - zone.second.m_peerlist.remove_from_peer_white(plw.back()); - } - for (auto &e:plw) - zone.second.m_peerlist.append_with_peer_white(e); - - std::list<peerlist_entry> plg; - while (zone.second.m_peerlist.get_gray_peers_count()) - { - plg.push_back(peerlist_entry()); - zone.second.m_peerlist.get_gray_peer_by_index(plg.back(), 0); - zone.second.m_peerlist.remove_from_peer_gray(plg.back()); - } - for (auto &e:plg) - zone.second.m_peerlist.append_with_peer_gray(e); - } -#endif - //only in case if we really sure that we have external visible ip m_have_address = true; @@ -1155,6 +1139,7 @@ namespace nodetool pi = context.peer_id = rsp.node_data.peer_id; context.m_rpc_port = rsp.node_data.rpc_port; context.m_rpc_credits_per_hash = rsp.node_data.rpc_credits_per_hash; + context.support_flags = rsp.node_data.support_flags; const auto azone = context.m_remote_address.get_zone(); network_zone& zone = m_network_zones.at(azone); zone.m_peerlist.set_peer_just_seen(rsp.node_data.peer_id, context.m_remote_address, context.m_pruning_seed, context.m_rpc_port, context.m_rpc_credits_per_hash); @@ -1188,10 +1173,11 @@ namespace nodetool } else if (!just_take_peerlist) { - try_get_support_flags(context_, [](p2p_connection_context& flags_context, const uint32_t& support_flags) - { - flags_context.support_flags = support_flags; - }); + if (context_.support_flags == 0) + try_get_support_flags(context_, [](p2p_connection_context& flags_context, const uint32_t& support_flags) + { + flags_context.support_flags = support_flags; + }); } return hsh_result; @@ -2014,15 +2000,24 @@ namespace nodetool boost::split(ips, record, boost::is_any_of(";")); for (const auto &ip: ips) { + if (ip.empty()) + continue; + auto subnet = net::get_ipv4_subnet_address(ip); + if (subnet) + { + block_subnet(*subnet, DNS_BLOCKLIST_LIFETIME); + ++good; + continue; + } const expect<epee::net_utils::network_address> parsed_addr = net::get_network_address(ip, 0); - if (!parsed_addr) + if (parsed_addr) { - MWARNING("Invalid IP address from DNS blocklist: " << ip << " - " << parsed_addr.error()); - ++bad; + block_host(*parsed_addr, DNS_BLOCKLIST_LIFETIME, true); + ++good; continue; } - block_host(*parsed_addr, DNS_BLOCKLIST_LIFETIME, true); - ++good; + MWARNING("Invalid IP address or subnet from DNS blocklist: " << ip << " - " << parsed_addr.error()); + ++bad; } } if (good > 0) @@ -2117,10 +2112,6 @@ namespace nodetool continue; } local_peerlist[i].last_seen = 0; - -#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED - be.pruning_seed = tools::make_pruning_seed(1 + (be.adr.as<epee::net_utils::ipv4_network_address>().ip()) % (1ul << CRYPTONOTE_PRUNING_LOG_STRIPES), CRYPTONOTE_PRUNING_LOG_STRIPES); -#endif } return true; } @@ -2149,6 +2140,7 @@ namespace nodetool LOG_DEBUG_CC(context, "REMOTE PEERLIST: remote peerlist size=" << peerlist_.size()); LOG_TRACE_CC(context, "REMOTE PEERLIST: " << ENDL << print_peerlist_to_string(peerlist_)); + CRITICAL_REGION_LOCAL(m_blocked_hosts_lock); return m_network_zones.at(context.m_remote_address.get_zone()).m_peerlist.merge_peerlist(peerlist_, [this](const peerlist_entry &pe) { return !is_addr_recently_failed(pe.adr) && is_remote_host_allowed(pe.adr); }); @@ -2165,6 +2157,7 @@ namespace nodetool node_data.rpc_port = zone.m_can_pingback ? m_rpc_port : 0; node_data.rpc_credits_per_hash = zone.m_can_pingback ? m_rpc_credits_per_hash : 0; node_data.network_id = m_network_id; + node_data.support_flags = zone.m_config.m_support_flags; return true; } //----------------------------------------------------------------------------------- @@ -2182,8 +2175,9 @@ namespace nodetool } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::relay_notify_to_list(int command, const epee::span<const uint8_t> data_buff, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections) + bool node_server<t_payload_net_handler>::relay_notify_to_list(int command, epee::levin::message_writer data_buff, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections) { + epee::byte_slice message = data_buff.finalize_notify(command); std::sort(connections.begin(), connections.end()); auto zone = m_network_zones.begin(); for(const auto& c_id: connections) @@ -2201,7 +2195,7 @@ namespace nodetool ++zone; } if (zone->first == c_id.first) - zone->second.m_net_server.get_config_object().notify(command, data_buff, c_id.second); + zone->second.m_net_server.get_config_object().send(message.clone(), c_id.second); } return true; } @@ -2268,24 +2262,13 @@ namespace nodetool } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::invoke_notify_to_peer(int command, const epee::span<const uint8_t> req_buff, const epee::net_utils::connection_context_base& context) + bool node_server<t_payload_net_handler>::invoke_notify_to_peer(const int command, epee::levin::message_writer message, const epee::net_utils::connection_context_base& context) { if(is_filtered_command(context.m_remote_address, command)) return false; network_zone& zone = m_network_zones.at(context.m_remote_address.get_zone()); - int res = zone.m_net_server.get_config_object().notify(command, req_buff, context.m_connection_id); - return res > 0; - } - //----------------------------------------------------------------------------------- - template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::invoke_command_to_peer(int command, const epee::span<const uint8_t> req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context) - { - if(is_filtered_command(context.m_remote_address, command)) - return false; - - network_zone& zone = m_network_zones.at(context.m_remote_address.get_zone()); - int res = zone.m_net_server.get_config_object().invoke(command, req_buff, resp_buff, context.m_connection_id); + int res = zone.m_net_server.get_config_object().send(message.finalize_notify(command), context.m_connection_id); return res > 0; } //----------------------------------------------------------------------------------- @@ -2533,6 +2516,7 @@ namespace nodetool context.m_in_timedsync = false; context.m_rpc_port = arg.node_data.rpc_port; context.m_rpc_credits_per_hash = arg.node_data.rpc_credits_per_hash; + context.support_flags = arg.node_data.support_flags; if(arg.node_data.my_port && zone.m_can_pingback) { @@ -2566,10 +2550,11 @@ namespace nodetool }); } - try_get_support_flags(context, [](p2p_connection_context& flags_context, const uint32_t& support_flags) - { - flags_context.support_flags = support_flags; - }); + if (context.support_flags == 0) + try_get_support_flags(context, [](p2p_connection_context& flags_context, const uint32_t& support_flags) + { + flags_context.support_flags = support_flags; + }); //fill response zone.m_peerlist.get_peerlist_head(rsp.local_peerlist_new, true); @@ -2835,8 +2820,7 @@ namespace nodetool if (address.get_zone() != epee::net_utils::zone::public_) return false; // Unable to determine how many connections from host - const size_t max_connections = 1; - size_t count = 0; + uint32_t count = 0; m_network_zones.at(epee::net_utils::zone::public_).m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) { diff --git a/src/p2p/net_node_common.h b/src/p2p/net_node_common.h index 0da758ad4..92b7596ae 100644 --- a/src/p2p/net_node_common.h +++ b/src/p2p/net_node_common.h @@ -40,6 +40,8 @@ #include "net/net_utils_base.h" #include "p2p_protocol_defs.h" +namespace epee { namespace levin { class message_writer; } } + namespace nodetool { @@ -49,10 +51,9 @@ namespace nodetool template<class t_connection_context> struct i_p2p_endpoint { - virtual bool relay_notify_to_list(int command, const epee::span<const uint8_t> data_buff, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections)=0; + virtual bool relay_notify_to_list(int command, epee::levin::message_writer message, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections)=0; virtual epee::net_utils::zone send_txs(std::vector<cryptonote::blobdata> txs, const epee::net_utils::zone origin, const boost::uuids::uuid& source, cryptonote::relay_method tx_relay)=0; - virtual bool invoke_command_to_peer(int command, const epee::span<const uint8_t> req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context)=0; - virtual bool invoke_notify_to_peer(int command, const epee::span<const uint8_t> req_buff, const epee::net_utils::connection_context_base& context)=0; + virtual bool invoke_notify_to_peer(int command, epee::levin::message_writer message, const epee::net_utils::connection_context_base& context)=0; virtual bool drop_connection(const epee::net_utils::connection_context_base& context)=0; virtual void request_callback(const epee::net_utils::connection_context_base& context)=0; virtual uint64_t get_public_connections_count()=0; @@ -71,7 +72,7 @@ namespace nodetool template<class t_connection_context> struct p2p_endpoint_stub: public i_p2p_endpoint<t_connection_context> { - virtual bool relay_notify_to_list(int command, const epee::span<const uint8_t> data_buff, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections) + virtual bool relay_notify_to_list(int command, epee::levin::message_writer message, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections) { return false; } @@ -79,11 +80,7 @@ namespace nodetool { return epee::net_utils::zone::invalid; } - virtual bool invoke_command_to_peer(int command, const epee::span<const uint8_t> req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context) - { - return false; - } - virtual bool invoke_notify_to_peer(int command, const epee::span<const uint8_t> req_buff, const epee::net_utils::connection_context_base& context) + virtual bool invoke_notify_to_peer(int command, epee::levin::message_writer message, const epee::net_utils::connection_context_base& context) { return true; } diff --git a/src/p2p/net_peerlist.h b/src/p2p/net_peerlist.h index c794b0f3b..d8de6abe9 100644 --- a/src/p2p/net_peerlist.h +++ b/src/p2p/net_peerlist.h @@ -110,7 +110,7 @@ namespace nodetool bool get_gray_peer_by_index(peerlist_entry& p, size_t i); template<typename F> bool foreach(bool white, const F &f); void evict_host_from_white_peerlist(const peerlist_entry& pr); - bool append_with_peer_white(const peerlist_entry& pr); + bool append_with_peer_white(const peerlist_entry& pr, bool trust_last_seen = false); bool append_with_peer_gray(const peerlist_entry& pr); bool append_with_peer_anchor(const anchor_peerlist_entry& ple); bool set_peer_just_seen(peerid_type peer, const epee::net_utils::network_address& addr, uint32_t pruning_seed, uint16_t rpc_port, uint32_t rpc_credits_per_hash); @@ -329,12 +329,12 @@ namespace nodetool ple.pruning_seed = pruning_seed; ple.rpc_port = rpc_port; ple.rpc_credits_per_hash = rpc_credits_per_hash; - return append_with_peer_white(ple); + return append_with_peer_white(ple, true); CATCH_ENTRY_L0("peerlist_manager::set_peer_just_seen()", false); } //-------------------------------------------------------------------------------------------------- inline - bool peerlist_manager::append_with_peer_white(const peerlist_entry& ple) + bool peerlist_manager::append_with_peer_white(const peerlist_entry& ple, bool trust_last_seen) { TRY_ENTRY(); if(!is_host_allowed(ple.adr)) @@ -357,7 +357,8 @@ namespace nodetool new_ple.pruning_seed = by_addr_it_wt->pruning_seed; if (by_addr_it_wt->rpc_port && ple.rpc_port == 0) // guard against older nodes not passing RPC port around new_ple.rpc_port = by_addr_it_wt->rpc_port; - new_ple.last_seen = by_addr_it_wt->last_seen; // do not overwrite the last seen timestamp, incoming peer list are untrusted + if (!trust_last_seen) + new_ple.last_seen = by_addr_it_wt->last_seen; // do not overwrite the last seen timestamp, incoming peer lists are untrusted m_peers_white.replace(by_addr_it_wt, new_ple); } //remove from gray list, if need diff --git a/src/p2p/net_peerlist_boost_serialization.h b/src/p2p/net_peerlist_boost_serialization.h index 37a85d526..1cf4cb026 100644 --- a/src/p2p/net_peerlist_boost_serialization.h +++ b/src/p2p/net_peerlist_boost_serialization.h @@ -38,10 +38,6 @@ #include "net/i2p_address.h" #include "p2p/p2p_protocol_defs.h" -#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED -#include "common/pruning.h" -#endif - BOOST_CLASS_VERSION(nodetool::peerlist_entry, 3) namespace boost @@ -228,12 +224,6 @@ namespace boost return; } a & pl.pruning_seed; -#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED - if (!typename Archive::is_saving()) - { - pl.pruning_seed = tools::make_pruning_seed(1+pl.adr.as<epee::net_utils::ipv4_network_address>().ip() % (1<<CRYPTONOTE_PRUNING_LOG_STRIPES), CRYPTONOTE_PRUNING_LOG_STRIPES); - } -#endif if (ver < 2) { if (!typename Archive::is_saving()) diff --git a/src/p2p/p2p_protocol_defs.h b/src/p2p/p2p_protocol_defs.h index b439dc47e..12763d4ee 100644 --- a/src/p2p/p2p_protocol_defs.h +++ b/src/p2p/p2p_protocol_defs.h @@ -188,6 +188,7 @@ namespace nodetool uint16_t rpc_port; uint32_t rpc_credits_per_hash; peerid_type peer_id; + uint32_t support_flags; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE_VAL_POD_AS_BLOB(network_id) @@ -195,6 +196,7 @@ namespace nodetool KV_SERIALIZE(my_port) KV_SERIALIZE_OPT(rpc_port, (uint16_t)(0)) KV_SERIALIZE_OPT(rpc_credits_per_hash, (uint32_t)0) + KV_SERIALIZE_OPT(support_flags, (uint32_t)0) END_KV_SERIALIZE_MAP() }; diff --git a/src/ringct/multiexp.cc b/src/ringct/multiexp.cc index f69b4a12c..620f7d0dd 100644 --- a/src/ringct/multiexp.cc +++ b/src/ringct/multiexp.cc @@ -372,7 +372,6 @@ std::shared_ptr<straus_cached_data> straus_init_cache(const std::vector<Multiexp if (N == 0) N = data.size(); CHECK_AND_ASSERT_THROW_MES(N <= data.size(), "Bad cache base data"); - ge_cached cached; ge_p1p1 p1; ge_p3 p3; std::shared_ptr<straus_cached_data> cache(new straus_cached_data()); @@ -454,7 +453,6 @@ rct::key straus(const std::vector<MultiexpData> &data, const std::shared_ptr<str std::shared_ptr<straus_cached_data> local_cache = cache == NULL ? straus_init_cache(data) : cache; ge_cached cached; ge_p1p1 p1; - ge_p3 p3; #ifdef TRACK_STRAUS_ZERO_IDENTITY MULTIEXP_PERF(PERF_TIMER_START_UNIT(skip, 1000000)); @@ -587,7 +585,6 @@ std::shared_ptr<pippenger_cached_data> pippenger_init_cache(const std::vector<Mu if (N == 0) N = data.size() - start_offset; CHECK_AND_ASSERT_THROW_MES(N <= data.size() - start_offset, "Bad cache base data"); - ge_cached cached; std::shared_ptr<pippenger_cached_data> cache(new pippenger_cached_data()); cache->size = N; diff --git a/src/ringct/rctSigs.cpp b/src/ringct/rctSigs.cpp index 93eb52d4e..f5950c53c 100644 --- a/src/ringct/rctSigs.cpp +++ b/src/ringct/rctSigs.cpp @@ -725,7 +725,6 @@ namespace rct { CHECK_AND_ASSERT_THROW_MES((kLRki && mscout) || (!kLRki && !mscout), "Only one of kLRki/mscout is present"); keyV tmp(rows + 1); keyV sk(rows + 1); - size_t i; keyM M(cols, tmp); keyV P, C, C_nonzero; @@ -899,7 +898,6 @@ namespace rct { key R; geDsmp P_precomp; geDsmp C_precomp; - geDsmp H_precomp; size_t i = 0; ge_p3 hash8_p3; geDsmp hash_precomp; diff --git a/src/rpc/bootstrap_daemon.cpp b/src/rpc/bootstrap_daemon.cpp index 2fdd28406..ffea906d5 100644 --- a/src/rpc/bootstrap_daemon.cpp +++ b/src/rpc/bootstrap_daemon.cpp @@ -7,6 +7,7 @@ #include "crypto/crypto.h" #include "cryptonote_core/cryptonote_core.h" #include "misc_log_ex.h" +#include "net/parse.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "daemon.rpc.bootstrap_daemon" @@ -16,19 +17,23 @@ namespace cryptonote bootstrap_daemon::bootstrap_daemon( std::function<std::map<std::string, bool>()> get_public_nodes, - bool rpc_payment_enabled) + bool rpc_payment_enabled, + const std::string &proxy) : m_selector(new bootstrap_node::selector_auto(std::move(get_public_nodes))) , m_rpc_payment_enabled(rpc_payment_enabled) { + set_proxy(proxy); } bootstrap_daemon::bootstrap_daemon( const std::string &address, boost::optional<epee::net_utils::http::login> credentials, - bool rpc_payment_enabled) + bool rpc_payment_enabled, + const std::string &proxy) : m_selector(nullptr) , m_rpc_payment_enabled(rpc_payment_enabled) { + set_proxy(proxy); if (!set_server(address, std::move(credentials))) { throw std::runtime_error("invalid bootstrap daemon address or credentials"); @@ -78,6 +83,18 @@ namespace cryptonote return success; } + void bootstrap_daemon::set_proxy(const std::string &address) + { + if (!address.empty() && !net::get_tcp_endpoint(address)) + { + throw std::runtime_error("invalid proxy address format"); + } + if (!m_http_client.set_proxy(address)) + { + throw std::runtime_error("failed to set proxy address"); + } + } + bool bootstrap_daemon::set_server(const std::string &address, const boost::optional<epee::net_utils::http::login> &credentials /* = boost::none */) { if (!m_http_client.set_server(address, credentials)) diff --git a/src/rpc/bootstrap_daemon.h b/src/rpc/bootstrap_daemon.h index d54042b11..1e4477123 100644 --- a/src/rpc/bootstrap_daemon.h +++ b/src/rpc/bootstrap_daemon.h @@ -8,7 +8,7 @@ #include <boost/thread/mutex.hpp> #include <boost/utility/string_ref.hpp> -#include "net/http_client.h" +#include "net/http.h" #include "storages/http_abstract_invoke.h" #include "bootstrap_node_selector.h" @@ -21,11 +21,13 @@ namespace cryptonote public: bootstrap_daemon( std::function<std::map<std::string, bool>()> get_public_nodes, - bool rpc_payment_enabled); + bool rpc_payment_enabled, + const std::string &proxy); bootstrap_daemon( const std::string &address, boost::optional<epee::net_utils::http::login> credentials, - bool rpc_payment_enabled); + bool rpc_payment_enabled, + const std::string &proxy); std::string address() const noexcept; boost::optional<std::pair<uint64_t, uint64_t>> get_height(); @@ -72,12 +74,14 @@ namespace cryptonote return handle_result(result, result_struct.status); } + void set_proxy(const std::string &address); + private: bool set_server(const std::string &address, const boost::optional<epee::net_utils::http::login> &credentials = boost::none); bool switch_server_if_needed(); private: - epee::net_utils::http::http_simple_client m_http_client; + net::http::client m_http_client; const bool m_rpc_payment_enabled; const std::unique_ptr<bootstrap_node::selector> m_selector; boost::mutex m_selector_mutex; diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 5b2043de6..ac21a3a6b 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -30,6 +30,7 @@ #include <boost/preprocessor/stringize.hpp> #include <boost/uuid/nil_generator.hpp> +#include <boost/filesystem.hpp> #include "include_base_utils.h" #include "string_tools.h" using namespace epee; @@ -44,6 +45,7 @@ using namespace epee; #include "cryptonote_basic/cryptonote_format_utils.h" #include "cryptonote_basic/account.h" #include "cryptonote_basic/cryptonote_basic_impl.h" +#include "cryptonote_basic/merge_mining.h" #include "cryptonote_core/tx_sanity_check.h" #include "misc_language.h" #include "net/parse.h" @@ -154,6 +156,7 @@ namespace cryptonote command_line::add_arg(desc, arg_restricted_rpc); command_line::add_arg(desc, arg_bootstrap_daemon_address); command_line::add_arg(desc, arg_bootstrap_daemon_login); + command_line::add_arg(desc, arg_bootstrap_daemon_proxy); cryptonote::rpc_args::init_options(desc, true); command_line::add_arg(desc, arg_rpc_payment_address); command_line::add_arg(desc, arg_rpc_payment_difficulty); @@ -172,7 +175,10 @@ namespace cryptonote , m_rpc_payment_allow_free_loopback(false) {} //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::set_bootstrap_daemon(const std::string &address, const std::string &username_password) + bool core_rpc_server::set_bootstrap_daemon( + const std::string &address, + const std::string &username_password, + const std::string &proxy) { boost::optional<epee::net_utils::http::login> credentials; const auto loc = username_password.find(':'); @@ -180,7 +186,7 @@ namespace cryptonote { credentials = epee::net_utils::http::login(username_password.substr(0, loc), username_password.substr(loc + 1)); } - return set_bootstrap_daemon(address, credentials); + return set_bootstrap_daemon(address, credentials, proxy); } //------------------------------------------------------------------------------------------------------------------------------ std::map<std::string, bool> core_rpc_server::get_public_nodes(uint32_t credits_per_hash_threshold/* = 0*/) @@ -217,7 +223,10 @@ namespace cryptonote return result; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::set_bootstrap_daemon(const std::string &address, const boost::optional<epee::net_utils::http::login> &credentials) + bool core_rpc_server::set_bootstrap_daemon( + const std::string &address, + const boost::optional<epee::net_utils::http::login> &credentials, + const std::string &proxy) { boost::unique_lock<boost::shared_mutex> lock(m_bootstrap_daemon_mutex); @@ -233,11 +242,11 @@ namespace cryptonote auto get_nodes = [this]() { return get_public_nodes(credits_per_hash_threshold); }; - m_bootstrap_daemon.reset(new bootstrap_daemon(std::move(get_nodes), rpc_payment_enabled)); + m_bootstrap_daemon.reset(new bootstrap_daemon(std::move(get_nodes), rpc_payment_enabled, proxy)); } else { - m_bootstrap_daemon.reset(new bootstrap_daemon(address, credentials, rpc_payment_enabled)); + m_bootstrap_daemon.reset(new bootstrap_daemon(address, credentials, rpc_payment_enabled, proxy)); } m_should_use_bootstrap_daemon = m_bootstrap_daemon.get() != nullptr; @@ -278,6 +287,7 @@ namespace cryptonote } } disable_rpc_ban = rpc_config->disable_rpc_ban; + const std::string data_dir{command_line::get_arg(vm, cryptonote::arg_data_dir)}; std::string address = command_line::get_arg(vm, arg_rpc_payment_address); if (!address.empty() && allow_rpc_payment) { @@ -306,7 +316,7 @@ namespace cryptonote } m_rpc_payment_allow_free_loopback = command_line::get_arg(vm, arg_rpc_payment_allow_free_loopback); m_rpc_payment.reset(new rpc_payment(info.address, diff, credits)); - m_rpc_payment->load(command_line::get_arg(vm, cryptonote::arg_data_dir)); + m_rpc_payment->load(data_dir); m_p2p.set_rpc_credits_per_hash(RPC_CREDITS_PER_HASH_SCALE * (credits / (float)diff)); } @@ -318,8 +328,10 @@ namespace cryptonote MWARNING("The RPC server is accessible from the outside, but no RPC payment was setup. RPC access will be free for all."); } - if (!set_bootstrap_daemon(command_line::get_arg(vm, arg_bootstrap_daemon_address), - command_line::get_arg(vm, arg_bootstrap_daemon_login))) + if (!set_bootstrap_daemon( + command_line::get_arg(vm, arg_bootstrap_daemon_address), + command_line::get_arg(vm, arg_bootstrap_daemon_login), + command_line::get_arg(vm, arg_bootstrap_daemon_proxy))) { MFATAL("Failed to parse bootstrap daemon address"); return false; @@ -333,12 +345,32 @@ namespace cryptonote if (m_rpc_payment) m_net_server.add_idle_handler([this](){ return m_rpc_payment->on_idle(); }, 60 * 1000); + bool store_ssl_key = !restricted && rpc_config->ssl_options && rpc_config->ssl_options.auth.certificate_path.empty(); + const auto ssl_base_path = (boost::filesystem::path{data_dir} / "rpc_ssl").string(); + if (store_ssl_key && boost::filesystem::exists(ssl_base_path + ".crt")) + { + // load key from previous run, password prompted by OpenSSL + store_ssl_key = false; + rpc_config->ssl_options.auth = + epee::net_utils::ssl_authentication_t{ssl_base_path + ".key", ssl_base_path + ".crt"}; + } + auto rng = [](size_t len, uint8_t *ptr){ return crypto::rand(len, ptr); }; - return epee::http_server_impl_base<core_rpc_server, connection_context>::init( + const bool inited = epee::http_server_impl_base<core_rpc_server, connection_context>::init( rng, std::move(port), std::move(bind_ip_str), std::move(bind_ipv6_str), std::move(rpc_config->use_ipv6), std::move(rpc_config->require_ipv4), std::move(rpc_config->access_control_origins), std::move(http_login), std::move(rpc_config->ssl_options) ); + + if (store_ssl_key && inited) + { + // new keys were generated, store for next run + const auto error = epee::net_utils::store_ssl_keys(m_net_server.get_ssl_context(), ssl_base_path); + if (error) + MFATAL("Failed to store HTTP SSL cert/key for " << (restricted ? "restricted " : "") << "RPC server: " << error.message()); + return !bool(error); + } + return inited; } //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::check_payment(const std::string &client_message, uint64_t payment, const std::string &rpc, bool same_ts, std::string &message, uint64_t &credits, std::string &top_hash) @@ -367,7 +399,6 @@ namespace cryptonote message = "Client signature does not verify for " + rpc; return false; } - crypto::public_key local_client; if (!m_rpc_payment->pay(client, ts, payment, rpc, same_ts, credits)) { message = CORE_RPC_STATUS_PAYMENT_REQUIRED; @@ -452,7 +483,7 @@ namespace cryptonote m_core.get_blockchain_top(res.height, top_hash); ++res.height; // turn top block height into blockchain height res.top_block_hash = string_tools::pod_to_hex(top_hash); - res.target_height = m_core.get_target_blockchain_height(); + res.target_height = m_p2p.get_payload_object().is_synchronized() ? 0 : m_core.get_target_blockchain_height(); store_difficulty(m_core.get_blockchain_storage().get_difficulty_for_next_block(), res.difficulty, res.wide_difficulty, res.difficulty_top64); res.target = m_core.get_blockchain_storage().get_difficulty_target(); res.tx_count = m_core.get_blockchain_storage().get_total_transactions() - res.height; //without coinbase @@ -1596,15 +1627,15 @@ namespace cryptonote { credentials = epee::net_utils::http::login(req.username, req.password); } - - if (set_bootstrap_daemon(req.address, credentials)) + + if (set_bootstrap_daemon(req.address, credentials, req.proxy)) { res.status = CORE_RPC_STATUS_OK; } else { res.status = "Failed to set bootstrap daemon"; - } + } return true; } @@ -1806,7 +1837,6 @@ namespace cryptonote return false; } } - uint64_t seed_height; crypto::hash seed_hash, next_seed_hash; if (!get_block_template(info.address, req.prev_block.empty() ? NULL : &prev_block, blob_reserve, reserved_offset, wdiff, res.height, res.expected_reward, b, res.seed_height, seed_hash, next_seed_hash, error_resp)) return false; @@ -1828,6 +1858,125 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_add_aux_pow(const COMMAND_RPC_ADD_AUX_POW::request& req, COMMAND_RPC_ADD_AUX_POW::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) + { + RPC_TRACKER(add_aux_pow); + bool r; + if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_ADD_AUX_POW>(invoke_http_mode::JON_RPC, "add_aux_pow", req, res, r)) + return r; + + if (req.aux_pow.empty()) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_PARAM; + error_resp.message = "Empty aux pow hash vector"; + return false; + } + + crypto::hash merkle_root; + size_t merkle_tree_depth = 0; + std::vector<std::pair<crypto::hash, crypto::hash>> aux_pow; + std::vector<crypto::hash> aux_pow_raw; + aux_pow.reserve(req.aux_pow.size()); + aux_pow_raw.reserve(req.aux_pow.size()); + for (const auto &s: req.aux_pow) + { + aux_pow.push_back({}); + if (!epee::string_tools::hex_to_pod(s.id, aux_pow.back().first)) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_PARAM; + error_resp.message = "Invalid aux pow id"; + return false; + } + if (!epee::string_tools::hex_to_pod(s.hash, aux_pow.back().second)) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_PARAM; + error_resp.message = "Invalid aux pow hash"; + return false; + } + aux_pow_raw.push_back(aux_pow.back().second); + } + + size_t path_domain = 1; + while ((1u << path_domain) < aux_pow.size()) + ++path_domain; + uint32_t nonce; + const uint32_t max_nonce = 65535; + bool collision = true; + for (nonce = 0; nonce <= max_nonce; ++nonce) + { + std::vector<bool> slots(aux_pow.size(), false); + collision = false; + for (size_t idx = 0; idx < aux_pow.size(); ++idx) + { + const uint32_t slot = cryptonote::get_aux_slot(aux_pow[idx].first, nonce, aux_pow.size()); + if (slot >= aux_pow.size()) + { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = "Computed slot is out of range"; + return false; + } + if (slots[slot]) + { + collision = true; + break; + } + slots[slot] = true; + } + if (!collision) + break; + } + if (collision) + { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = "Failed to find a suitable nonce"; + return false; + } + + crypto::tree_hash((const char(*)[crypto::HASH_SIZE])aux_pow_raw.data(), aux_pow_raw.size(), merkle_root.data); + res.merkle_root = epee::string_tools::pod_to_hex(merkle_root); + res.merkle_tree_depth = cryptonote::encode_mm_depth(aux_pow.size(), nonce); + + blobdata blocktemplate_blob; + if (!epee::string_tools::parse_hexstr_to_binbuff(req.blocktemplate_blob, blocktemplate_blob)) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_PARAM; + error_resp.message = "Invalid blocktemplate_blob"; + return false; + } + + block b; + if (!parse_and_validate_block_from_blob(blocktemplate_blob, b)) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_BLOCKBLOB; + error_resp.message = "Wrong blocktemplate_blob"; + return false; + } + + if (!remove_field_from_tx_extra(b.miner_tx.extra, typeid(cryptonote::tx_extra_merge_mining_tag))) + { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = "Error removing existing merkle root"; + return false; + } + if (!add_mm_merkle_root_to_tx_extra(b.miner_tx.extra, merkle_root, merkle_tree_depth)) + { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = "Error adding merkle root"; + return false; + } + b.invalidate_hashes(); + b.miner_tx.invalidate_hashes(); + + const blobdata block_blob = t_serializable_object_to_blob(b); + const blobdata hashing_blob = get_block_hashing_blob(b); + + res.blocktemplate_blob = string_tools::buff_to_hex_nodelimer(block_blob); + res.blockhashing_blob = string_tools::buff_to_hex_nodelimer(hashing_blob); + res.aux_pow = req.aux_pow; + res.status = CORE_RPC_STATUS_OK; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_submitblock(const COMMAND_RPC_SUBMITBLOCK::request& req, COMMAND_RPC_SUBMITBLOCK::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { RPC_TRACKER(submitblock); @@ -2913,7 +3062,7 @@ namespace cryptonote crypto::hash top_hash; m_core.get_blockchain_top(res.height, top_hash); ++res.height; // turn top block height into blockchain height - res.target_height = m_core.get_target_blockchain_height(); + res.target_height = m_p2p.get_payload_object().is_synchronized() ? 0 : m_core.get_target_blockchain_height(); res.next_needed_pruning_seed = m_p2p.get_payload_object().get_next_needed_pruning_stripe().second; for (const auto &c: m_p2p.get_payload_object().get_connections()) @@ -3349,6 +3498,12 @@ namespace cryptonote , "" }; + const command_line::arg_descriptor<std::string> core_rpc_server::arg_bootstrap_daemon_proxy = { + "bootstrap-daemon-proxy" + , "<ip>:<port> socks proxy to use for bootstrap daemon connections" + , "" + }; + const command_line::arg_descriptor<std::string> core_rpc_server::arg_rpc_payment_address = { "rpc-payment-address" , "Restrict RPC to clients sending micropayment to this address" diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index dcf6b4e4b..b21e43ab0 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -72,6 +72,7 @@ namespace cryptonote static const command_line::arg_descriptor<bool> arg_rpc_ssl_allow_any_cert; static const command_line::arg_descriptor<std::string> arg_bootstrap_daemon_address; static const command_line::arg_descriptor<std::string> arg_bootstrap_daemon_login; + static const command_line::arg_descriptor<std::string> arg_bootstrap_daemon_proxy; static const command_line::arg_descriptor<std::string> arg_rpc_payment_address; static const command_line::arg_descriptor<uint64_t> arg_rpc_payment_difficulty; static const command_line::arg_descriptor<uint64_t> arg_rpc_payment_credits; @@ -146,6 +147,7 @@ namespace cryptonote MAP_JON_RPC_WE("on_getblockhash", on_getblockhash, COMMAND_RPC_GETBLOCKHASH) MAP_JON_RPC_WE("get_block_template", on_getblocktemplate, COMMAND_RPC_GETBLOCKTEMPLATE) MAP_JON_RPC_WE("getblocktemplate", on_getblocktemplate, COMMAND_RPC_GETBLOCKTEMPLATE) + MAP_JON_RPC_WE("add_aux_pow", on_add_aux_pow, COMMAND_RPC_ADD_AUX_POW) MAP_JON_RPC_WE("submit_block", on_submitblock, COMMAND_RPC_SUBMITBLOCK) MAP_JON_RPC_WE("submitblock", on_submitblock, COMMAND_RPC_SUBMITBLOCK) MAP_JON_RPC_WE_IF("generateblocks", on_generateblocks, COMMAND_RPC_GENERATEBLOCKS, !m_restricted) @@ -226,6 +228,7 @@ namespace cryptonote bool on_getblockcount(const COMMAND_RPC_GETBLOCKCOUNT::request& req, COMMAND_RPC_GETBLOCKCOUNT::response& res, const connection_context *ctx = NULL); bool on_getblockhash(const COMMAND_RPC_GETBLOCKHASH::request& req, COMMAND_RPC_GETBLOCKHASH::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_getblocktemplate(const COMMAND_RPC_GETBLOCKTEMPLATE::request& req, COMMAND_RPC_GETBLOCKTEMPLATE::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_add_aux_pow(const COMMAND_RPC_ADD_AUX_POW::request& req, COMMAND_RPC_ADD_AUX_POW::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_submitblock(const COMMAND_RPC_SUBMITBLOCK::request& req, COMMAND_RPC_SUBMITBLOCK::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_generateblocks(const COMMAND_RPC_GENERATEBLOCKS::request& req, COMMAND_RPC_GENERATEBLOCKS::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_get_last_block_header(const COMMAND_RPC_GET_LAST_BLOCK_HEADER::request& req, COMMAND_RPC_GET_LAST_BLOCK_HEADER::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); @@ -268,8 +271,14 @@ private: uint64_t get_block_reward(const block& blk); bool fill_block_header_response(const block& blk, bool orphan_status, uint64_t height, const crypto::hash& hash, block_header_response& response, bool fill_pow_hash); std::map<std::string, bool> get_public_nodes(uint32_t credits_per_hash_threshold = 0); - bool set_bootstrap_daemon(const std::string &address, const std::string &username_password); - bool set_bootstrap_daemon(const std::string &address, const boost::optional<epee::net_utils::http::login> &credentials); + bool set_bootstrap_daemon( + const std::string &address, + const std::string &username_password, + const std::string &proxy); + bool set_bootstrap_daemon( + const std::string &address, + const boost::optional<epee::net_utils::http::login> &credentials, + const std::string &proxy); enum invoke_http_mode { JON, BIN, JON_RPC }; template <typename COMMAND_TYPE> bool use_bootstrap_daemon_if_necessary(const invoke_http_mode &mode, const std::string &command_name, const typename COMMAND_TYPE::request& req, typename COMMAND_TYPE::response& res, bool &r); diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index bbcb27f1c..5ebe4f654 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -88,7 +88,7 @@ namespace cryptonote // advance which version they will stop working with // Don't go over 32767 for any of these #define CORE_RPC_VERSION_MAJOR 3 -#define CORE_RPC_VERSION_MINOR 5 +#define CORE_RPC_VERSION_MINOR 6 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR) @@ -938,6 +938,52 @@ namespace cryptonote typedef epee::misc_utils::struct_init<response_t> response; }; + struct COMMAND_RPC_ADD_AUX_POW + { + struct aux_pow_t + { + std::string id; + std::string hash; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(id) + KV_SERIALIZE(hash) + END_KV_SERIALIZE_MAP() + }; + + struct request_t: public rpc_request_base + { + blobdata blocktemplate_blob; + std::vector<aux_pow_t> aux_pow; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_PARENT(rpc_request_base) + KV_SERIALIZE(blocktemplate_blob) + KV_SERIALIZE(aux_pow) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<request_t> request; + + struct response_t: public rpc_response_base + { + blobdata blocktemplate_blob; + blobdata blockhashing_blob; + std::string merkle_root; + uint32_t merkle_tree_depth; + std::vector<aux_pow_t> aux_pow; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_PARENT(rpc_response_base) + KV_SERIALIZE(blocktemplate_blob) + KV_SERIALIZE(blockhashing_blob) + KV_SERIALIZE(merkle_root) + KV_SERIALIZE(merkle_tree_depth) + KV_SERIALIZE(aux_pow) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<response_t> response; + }; + struct COMMAND_RPC_SUBMITBLOCK { typedef std::vector<std::string> request; @@ -1613,11 +1659,13 @@ namespace cryptonote std::string address; std::string username; std::string password; + std::string proxy; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(address) KV_SERIALIZE(username) KV_SERIALIZE(password) + KV_SERIALIZE(proxy) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<request_t> request; diff --git a/src/rpc/rpc_payment.cpp b/src/rpc/rpc_payment.cpp index 176f11fa3..bf6584f72 100644 --- a/src/rpc/rpc_payment.cpp +++ b/src/rpc/rpc_payment.cpp @@ -27,6 +27,7 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <boost/archive/portable_binary_iarchive.hpp> +#include <boost/filesystem.hpp> #include "cryptonote_config.h" #include "include_base_utils.h" #include "string_tools.h" diff --git a/src/serialization/binary_archive.h b/src/serialization/binary_archive.h index 80104c8f7..49ca8aa57 100644 --- a/src/serialization/binary_archive.h +++ b/src/serialization/binary_archive.h @@ -98,7 +98,7 @@ template <> struct binary_archive<false> : public binary_archive_base<std::istream, false> { - explicit binary_archive(stream_type &s) : base_type(s) { + explicit binary_archive(stream_type &s) : base_type(s), varint_bug_backward_compatibility_(false) { stream_type::pos_type pos = stream_.tellg(); stream_.seekg(0, std::ios_base::end); eof_pos_ = stream_.tellg(); @@ -173,8 +173,13 @@ struct binary_archive<false> : public binary_archive_base<std::istream, false> assert(stream_.tellg() <= eof_pos_); return eof_pos_ - stream_.tellg(); } + + void enable_varint_bug_backward_compatibility() { varint_bug_backward_compatibility_ = true; } + bool varint_bug_backward_compatibility_enabled() const { return varint_bug_backward_compatibility_; } + protected: std::streamoff eof_pos_; + bool varint_bug_backward_compatibility_; }; template <> @@ -227,6 +232,8 @@ struct binary_archive<true> : public binary_archive_base<std::ostream, true> void write_variant_tag(variant_tag_type t) { serialize_int(t); } + + bool varint_bug_backward_compatibility_enabled() const { return false; } }; POP_WARNINGS diff --git a/src/serialization/container.h b/src/serialization/container.h index d5e75bb4f..a4997c8ae 100644 --- a/src/serialization/container.h +++ b/src/serialization/container.h @@ -32,22 +32,27 @@ namespace serialization { namespace detail { - template <typename Archive, class T> - bool serialize_container_element(Archive& ar, T& e) + template<typename T> + inline constexpr bool use_container_varint() noexcept { - return ::do_serialize(ar, e); + return std::is_integral<T>::value && std::is_unsigned<T>::value && sizeof(T) > 1; } - template <typename Archive> - bool serialize_container_element(Archive& ar, uint32_t& e) + template <typename Archive, class T> + typename std::enable_if<!use_container_varint<T>(), bool>::type + serialize_container_element(Archive& ar, T& e) { - ar.serialize_varint(e); - return true; + return ::do_serialize(ar, e); } - template <typename Archive> - bool serialize_container_element(Archive& ar, uint64_t& e) + template<typename Archive, typename T> + typename std::enable_if<use_container_varint<T>(), bool>::type + serialize_container_element(Archive& ar, T& e) { + static constexpr const bool previously_varint = std::is_same<uint64_t, T>() || std::is_same<uint32_t, T>(); + + if (!previously_varint && ar.varint_bug_backward_compatibility_enabled() && !typename Archive::is_saving()) + return ::do_serialize(ar, e); ar.serialize_varint(e); return true; } diff --git a/src/serialization/json_archive.h b/src/serialization/json_archive.h index 50dd5bbd0..3f98b5101 100644 --- a/src/serialization/json_archive.h +++ b/src/serialization/json_archive.h @@ -84,6 +84,8 @@ struct json_archive_base void end_variant() { end_object(); } Stream &stream() { return stream_; } + bool varint_bug_backward_compatibility_enabled() const { return false; } + protected: void make_indent() { diff --git a/src/serialization/pair.h b/src/serialization/pair.h index 18280d837..44aafa04d 100644 --- a/src/serialization/pair.h +++ b/src/serialization/pair.h @@ -30,21 +30,34 @@ #pragma once #include <memory> +#include <boost/type_traits/make_unsigned.hpp> #include "serialization.h" namespace serialization { namespace detail { + template<typename T> + inline constexpr bool use_pair_varint() noexcept + { + return std::is_integral<T>::value && std::is_unsigned<T>::value && sizeof(T) > 1; + } + template <typename Archive, class T> - bool serialize_pair_element(Archive& ar, T& e) + typename std::enable_if<!use_pair_varint<T>(), bool>::type + serialize_pair_element(Archive& ar, T& e) { return ::do_serialize(ar, e); } - template <typename Archive> - bool serialize_pair_element(Archive& ar, uint64_t& e) + template<typename Archive, typename T> + typename std::enable_if<use_pair_varint<T>(), bool>::type + serialize_pair_element(Archive& ar, T& e) { + static constexpr const bool previously_varint = std::is_same<uint64_t, T>(); + + if (!previously_varint && ar.varint_bug_backward_compatibility_enabled() && !typename Archive::is_saving()) + return ::do_serialize(ar, e); ar.serialize_varint(e); return true; } diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index 0635520c6..da6501183 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -50,6 +50,7 @@ #include <boost/format.hpp> #include <boost/regex.hpp> #include <boost/range/adaptor/transformed.hpp> +#include <boost/filesystem.hpp> #include "include_base_utils.h" #include "console_handler.h" #include "common/i18n.h" @@ -195,7 +196,7 @@ namespace const char* USAGE_SWEEP_BELOW("sweep_below <amount_threshold> [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> [<payment_id (obsolete)>]"); const char* USAGE_SWEEP_SINGLE("sweep_single [<priority>] [<ring_size>] [outputs=<N>] <key_image> <address> [<payment_id (obsolete)>]"); const char* USAGE_DONATE("donate [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <amount> [<payment_id (obsolete)>]"); - const char* USAGE_SIGN_TRANSFER("sign_transfer [export_raw]"); + const char* USAGE_SIGN_TRANSFER("sign_transfer [export_raw] [<filename>]"); const char* USAGE_SET_LOG("set_log <level>|{+,-,}<categories>"); const char* USAGE_ACCOUNT("account\n" " account new <label text with white spaces allowed>\n" @@ -276,12 +277,13 @@ namespace const char* USAGE_PUBLIC_NODES("public_nodes"); const char* USAGE_WELCOME("welcome"); const char* USAGE_RPC_PAYMENT_INFO("rpc_payment_info"); - const char* USAGE_START_MINING_FOR_RPC("start_mining_for_rpc"); + const char* USAGE_START_MINING_FOR_RPC("start_mining_for_rpc [<number_of_threads>]"); const char* USAGE_STOP_MINING_FOR_RPC("stop_mining_for_rpc"); const char* USAGE_SHOW_QR_CODE("show_qr_code [<subaddress_index>]"); const char* USAGE_VERSION("version"); const char* USAGE_HELP("help [<command> | all]"); const char* USAGE_APROPOS("apropos <keyword> [<keyword> ...]"); + const char* USAGE_SCAN_TX("scan_tx <txid> [<txid> ...]"); std::string input_line(const std::string& prompt, bool yesno = false) { @@ -1360,7 +1362,7 @@ bool simple_wallet::import_multisig_main(const std::vector<std::string> &args, b size_t n_outputs = m_wallet->import_multisig(info); // Clear line "Height xxx of xxx" std::cout << "\r \r"; - success_msg_writer() << tr("Multisig info imported"); + success_msg_writer() << tr("Multisig info imported. Number of outputs updated: ") << n_outputs; } catch (const std::exception &e) { @@ -2360,6 +2362,24 @@ bool simple_wallet::start_mining_for_rpc(const std::vector<std::string> &args) if (!try_connect_to_daemon()) return true; + bool ok = true; + if(args.size() >= 1) + { + uint16_t num = 0; + ok = string_tools::get_xtype_from_string(num, args[0]); + m_rpc_payment_threads = num; + } + else + { + m_rpc_payment_threads = 0; + } + + if (!ok) + { + PRINT_USAGE(USAGE_START_MINING_FOR_RPC); + return true; + } + LOCK_IDLE_SCOPE(); bool payment_required; @@ -3214,6 +3234,45 @@ bool simple_wallet::apropos(const std::vector<std::string> &args) return true; } +bool simple_wallet::scan_tx(const std::vector<std::string> &args) +{ + if (args.empty()) + { + PRINT_USAGE(USAGE_SCAN_TX); + return true; + } + + // Parse and dedup args + std::unordered_set<crypto::hash> txids; + for (const auto &s : args) { + crypto::hash txid; + if (!epee::string_tools::hex_to_pod(s, txid)) { + fail_msg_writer() << tr("Invalid txid specified: ") << s; + return true; + } + txids.insert(txid); + } + std::vector<crypto::hash> txids_v(txids.begin(), txids.end()); + + if (!m_wallet->is_trusted_daemon()) { + message_writer(console_color_red, true) << tr("WARNING: this operation may reveal the txids to the remote node and affect your privacy"); + if (!command_line::is_yes(input_line("Do you want to continue?", true))) { + message_writer() << tr("You have canceled the operation"); + return true; + } + } + + LOCK_IDLE_SCOPE(); + m_in_manual_refresh.store(true); + try { + m_wallet->scan_tx(txids_v); + } catch (const std::exception &e) { + fail_msg_writer() << e.what(); + } + m_in_manual_refresh.store(false); + return true; +} + simple_wallet::simple_wallet() : m_allow_mismatched_daemon_version(false) , m_refresh_progress_reporter(*this) @@ -3301,7 +3360,8 @@ simple_wallet::simple_wallet() m_cmd_binder.set_handler("sign_transfer", boost::bind(&simple_wallet::on_command, this, &simple_wallet::sign_transfer, _1), tr(USAGE_SIGN_TRANSFER), - tr("Sign a transaction from a file. If the parameter \"export_raw\" is specified, transaction raw hex data suitable for the daemon RPC /sendrawtransaction is exported.")); + tr("Sign a transaction from a file. If the parameter \"export_raw\" is specified, transaction raw hex data suitable for the daemon RPC /sendrawtransaction is exported.\n" + "Use the parameter <filename> to specify the file to read from. If not specified, the default \"unsigned_monero_tx\" will be used.")); m_cmd_binder.set_handler("submit_transfer", boost::bind(&simple_wallet::on_command, this, &simple_wallet::submit_transfer, _1), tr("Submit a signed transaction from a file.")); @@ -3763,6 +3823,10 @@ simple_wallet::simple_wallet() boost::bind(&simple_wallet::on_command, this, &simple_wallet::apropos, _1), tr(USAGE_APROPOS), tr("Search all command descriptions for keyword(s)")); + m_cmd_binder.set_handler("scan_tx", + boost::bind(&simple_wallet::on_command, this, &simple_wallet::scan_tx, _1), + tr(USAGE_SCAN_TX), + tr("Scan the transactions given by <txid>(s), processing them and looking for outputs")); m_cmd_binder.set_unknown_command_handler(boost::bind(&simple_wallet::on_command, this, &simple_wallet::on_unknown_command, _1)); m_cmd_binder.set_empty_command_handler(boost::bind(&simple_wallet::on_empty_command, this)); m_cmd_binder.set_cancel_handler(boost::bind(&simple_wallet::on_cancelled_command, this)); @@ -4519,7 +4583,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) password = *r; welcome = true; // if no block_height is specified, assume its a new account and start it "now" - if(m_wallet->get_refresh_from_block_height() == 0) { + if (command_line::is_arg_defaulted(vm, arg_restore_height)) { { tools::scoped_message_writer wrt = tools::msg_writer(); wrt << tr("No restore height is specified.") << " "; @@ -4765,9 +4829,14 @@ bool simple_wallet::try_connect_to_daemon(bool silent, uint32_t* version) if (!m_wallet->check_connection(version)) { if (!silent) - fail_msg_writer() << tr("wallet failed to connect to daemon: ") << m_wallet->get_daemon_address() << ". " << - tr("Daemon either is not started or wrong port was passed. " - "Please make sure daemon is running or change the daemon address using the 'set_daemon' command."); + { + if (m_wallet->is_offline()) + fail_msg_writer() << tr("wallet failed to connect to daemon, because it is set to offline mode"); + else + fail_msg_writer() << tr("wallet failed to connect to daemon: ") << m_wallet->get_daemon_address() << ". " << + tr("Daemon either is not started or wrong port was passed. " + "Please make sure daemon is running or change the daemon address using the 'set_daemon' command."); + } return false; } if (!m_allow_mismatched_daemon_version && ((*version >> 16) != CORE_RPC_VERSION_MAJOR)) @@ -5932,7 +6001,7 @@ bool simple_wallet::show_balance_unlocked(bool detailed) if (m_wallet->has_multisig_partial_key_images()) extra = tr(" (Some owned outputs have partial key images - import_multisig_info needed)"); else if (m_wallet->has_unknown_key_images()) - extra += tr(" (Some owned outputs have missing key images - import_key_images needed)"); + extra += tr(" (Some owned outputs have missing key images - export_outputs, import_outputs, export_key_images, and import_key_images needed)"); success_msg_writer() << tr("Currently selected account: [") << m_current_subaddress_account << tr("] ") << m_wallet->get_subaddress_label({m_current_subaddress_account, 0}); const std::string tag = m_wallet->get_account_tags().second[m_current_subaddress_account]; success_msg_writer() << tr("Tag: ") << (tag.empty() ? std::string{tr("(No tag assigned)")} : tag); @@ -6463,8 +6532,6 @@ void simple_wallet::check_for_inactivity_lock(bool user) //---------------------------------------------------------------------------------------------------- bool simple_wallet::on_command(bool (simple_wallet::*cmd)(const std::vector<std::string>&), const std::vector<std::string> &args) { - const time_t now = time(NULL); - time_t dt = now - m_last_activity_time; m_last_activity_time = time(NULL); m_in_command = true; @@ -7430,7 +7497,6 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) if (local_args.size() == 3) { crypto::hash payment_id; - crypto::hash8 payment_id8; std::string extra_nonce; if (tools::wallet2::parse_long_payment_id(local_args.back(), payment_id)) { @@ -7886,19 +7952,33 @@ bool simple_wallet::sign_transfer(const std::vector<std::string> &args_) fail_msg_writer() << tr("This is a watch only wallet"); return true; } - if (args_.size() > 1 || (args_.size() == 1 && args_[0] != "export_raw")) + + bool export_raw = false; + std::string unsigned_filename = "unsigned_monero_tx"; + if (args_.size() > 2 || (args_.size() == 2 && args_[0] != "export_raw")) { PRINT_USAGE(USAGE_SIGN_TRANSFER); return true; } + else if (args_.size() == 2) + { + export_raw = true; + unsigned_filename = args_[1]; + } + else if (args_.size() == 1) + { + if (args_[0] == "export_raw") + export_raw = true; + else + unsigned_filename = args_[0]; + } SCOPED_WALLET_UNLOCK(); - const bool export_raw = args_.size() == 1; std::vector<tools::wallet2::pending_tx> ptx; try { - bool r = m_wallet->sign_tx("unsigned_monero_tx", "signed_monero_tx", ptx, [&](const tools::wallet2::unsigned_tx_set &tx){ return accept_loaded_tx(tx); }, export_raw); + bool r = m_wallet->sign_tx(unsigned_filename, "signed_monero_tx", ptx, [&](const tools::wallet2::unsigned_tx_set &tx){ return accept_loaded_tx(tx); }, export_raw); if (!r) { fail_msg_writer() << tr("Failed to sign transaction"); @@ -8596,7 +8676,6 @@ bool simple_wallet::get_transfers(std::vector<std::string>& local_args, std::vec if (!unlocked) { locked_msg = "locked"; - const uint64_t unlock_time = pd.m_unlock_time; if (pd.m_unlock_time < CRYPTONOTE_MAX_BLOCK_NUMBER) { uint64_t bh = std::max(pd.m_unlock_time, pd.m_block_height + CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE); @@ -9267,7 +9346,7 @@ bool simple_wallet::check_rpc_payment() fail_msg_writer() << tr("Error mining to daemon: ") << error; m_cmd_binder.print_prompt(); }; - bool ret = m_wallet->search_for_rpc_payment(target, startfunc, contfunc, foundfunc, errorfunc); + bool ret = m_wallet->search_for_rpc_payment(target, m_rpc_payment_threads, startfunc, contfunc, foundfunc, errorfunc); if (!ret) { fail_msg_writer() << tr("Failed to start mining for RPC payment"); @@ -9298,7 +9377,7 @@ bool simple_wallet::run() refresh_main(0, ResetNone, true); - m_auto_refresh_enabled = m_wallet->auto_refresh(); + m_auto_refresh_enabled = !m_wallet->is_offline() && m_wallet->auto_refresh(); m_idle_thread = boost::thread([&]{wallet_idle_thread();}); message_writer(console_color_green, false) << "Background refresh thread started"; @@ -9461,7 +9540,7 @@ void simple_wallet::print_accounts() { const std::pair<std::map<std::string, std::string>, std::vector<std::string>>& account_tags = m_wallet->get_account_tags(); size_t num_untagged_accounts = m_wallet->get_num_subaddress_accounts(); - for (const std::pair<std::string, std::string>& p : account_tags.first) + for (const std::pair<const std::string, std::string>& p : account_tags.first) { const std::string& tag = p.first; print_accounts(tag); @@ -11175,7 +11254,6 @@ void simple_wallet::mms_next(const std::vector<std::string> &args) void simple_wallet::mms_sync(const std::vector<std::string> &args) { - mms::message_store& ms = m_wallet->get_message_store(); if (args.size() != 0) { fail_msg_writer() << tr("Usage: mms sync"); @@ -11273,7 +11351,6 @@ void simple_wallet::mms_export(const std::vector<std::string> &args) return; } LOCK_IDLE_SCOPE(); - mms::message_store& ms = m_wallet->get_message_store(); mms::message m; bool valid_id = get_message_from_arg(args[0], m); if (valid_id) @@ -11342,7 +11419,6 @@ void simple_wallet::mms_show(const std::vector<std::string> &args) return; } LOCK_IDLE_SCOPE(); - mms::message_store& ms = m_wallet->get_message_store(); mms::message m; bool valid_id = get_message_from_arg(args[0], m); if (valid_id) @@ -11648,4 +11724,3 @@ bool simple_wallet::mms(const std::vector<std::string> &args) return true; } // End MMS ------------------------------------------------------------------------------------------------ - diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h index 61104c87f..8780bee1d 100644 --- a/src/simplewallet/simplewallet.h +++ b/src/simplewallet/simplewallet.h @@ -158,6 +158,7 @@ namespace cryptonote bool set_credits_target(const std::vector<std::string> &args = std::vector<std::string>()); bool help(const std::vector<std::string> &args = std::vector<std::string>()); bool apropos(const std::vector<std::string> &args); + bool scan_tx(const std::vector<std::string> &args); bool start_mining(const std::vector<std::string> &args); bool stop_mining(const std::vector<std::string> &args); bool set_daemon(const std::vector<std::string> &args); @@ -463,6 +464,7 @@ namespace cryptonote std::atomic<bool> m_need_payment; boost::posix_time::ptime m_last_rpc_payment_mining_time; bool m_rpc_payment_mining_requested; + uint32_t m_rpc_payment_threads = 0; bool m_daemon_rpc_payment_message_displayed; float m_rpc_payment_hash_rate; std::atomic<bool> m_suspend_rpc_payment_mining; diff --git a/src/wallet/api/pending_transaction.cpp b/src/wallet/api/pending_transaction.cpp index 24f6d37db..b28ffd64c 100644 --- a/src/wallet/api/pending_transaction.cpp +++ b/src/wallet/api/pending_transaction.cpp @@ -40,6 +40,7 @@ #include <vector> #include <sstream> #include <boost/format.hpp> +#include <boost/filesystem.hpp> using namespace std; diff --git a/src/wallet/api/transaction_history.cpp b/src/wallet/api/transaction_history.cpp index b6d52c58d..d8e4aab65 100644 --- a/src/wallet/api/transaction_history.cpp +++ b/src/wallet/api/transaction_history.cpp @@ -139,6 +139,7 @@ void TransactionHistoryImpl::refresh() ti->m_paymentid = payment_id; ti->m_coinbase = pd.m_coinbase; ti->m_amount = pd.m_amount; + ti->m_fee = pd.m_fee; ti->m_direction = TransactionInfo::Direction_In; ti->m_hash = string_tools::pod_to_hex(pd.m_tx_hash); ti->m_blockheight = pd.m_block_height; diff --git a/src/wallet/api/wallet.cpp b/src/wallet/api/wallet.cpp index 9acc8484c..3bbd9ce0b 100644 --- a/src/wallet/api/wallet.cpp +++ b/src/wallet/api/wallet.cpp @@ -45,10 +45,8 @@ #include <sstream> #include <unordered_map> -#ifdef WIN32 #include <boost/locale.hpp> #include <boost/filesystem.hpp> -#endif using namespace std; using namespace cryptonote; @@ -791,11 +789,11 @@ bool WalletImpl::close(bool store) return result; } -std::string WalletImpl::seed() const +std::string WalletImpl::seed(const std::string& seed_offset) const { epee::wipeable_string seed; if (m_wallet) - m_wallet->get_seed(seed); + m_wallet->get_seed(seed, seed_offset); return std::string(seed.data(), seed.size()); // TODO } @@ -1168,7 +1166,7 @@ bool WalletImpl::submitTransaction(const string &fileName) { return true; } -bool WalletImpl::exportKeyImages(const string &filename) +bool WalletImpl::exportKeyImages(const string &filename, bool all) { if (m_wallet->watch_only()) { @@ -1178,7 +1176,7 @@ bool WalletImpl::exportKeyImages(const string &filename) try { - if (!m_wallet->export_key_images(filename)) + if (!m_wallet->export_key_images(filename), all) { setStatusError(tr("failed to save file ") + filename); return false; @@ -2104,6 +2102,11 @@ bool WalletImpl::watchOnly() const return m_wallet->watch_only(); } +bool WalletImpl::isDeterministic() const +{ + return m_wallet->is_deterministic(); +} + void WalletImpl::clearStatus() const { boost::lock_guard<boost::mutex> l(m_statusMutex); diff --git a/src/wallet/api/wallet.h b/src/wallet/api/wallet.h index 3bf3e759b..0f3614bb4 100644 --- a/src/wallet/api/wallet.h +++ b/src/wallet/api/wallet.h @@ -81,7 +81,7 @@ public: const std::string &device_name); Device getDeviceType() const override; bool close(bool store = true); - std::string seed() const override; + std::string seed(const std::string& seed_offset = "") const override; std::string getSeedLanguage() const override; void setSeedLanguage(const std::string &arg) override; // void setListener(Listener *) {} @@ -129,6 +129,7 @@ public: void setRecoveringFromDevice(bool recoveringFromDevice) override; void setSubaddressLookahead(uint32_t major, uint32_t minor) override; bool watchOnly() const override; + bool isDeterministic() const override; bool rescanSpent() override; NetworkType nettype() const override {return static_cast<NetworkType>(m_wallet->nettype());} void hardForkInfo(uint8_t &version, uint64_t &earliest_height) const override; @@ -164,7 +165,7 @@ public: virtual PendingTransaction * createSweepUnmixableTransaction() override; bool submitTransaction(const std::string &fileName) override; virtual UnsignedTransaction * loadUnsignedTx(const std::string &unsigned_filename) override; - bool exportKeyImages(const std::string &filename) override; + bool exportKeyImages(const std::string &filename, bool all = false) override; bool importKeyImages(const std::string &filename) override; virtual void disposeTransaction(PendingTransaction * t) override; diff --git a/src/wallet/api/wallet2_api.h b/src/wallet/api/wallet2_api.h index 44928a422..b1cebedaf 100644 --- a/src/wallet/api/wallet2_api.h +++ b/src/wallet/api/wallet2_api.h @@ -446,7 +446,7 @@ struct Wallet }; virtual ~Wallet() = 0; - virtual std::string seed() const = 0; + virtual std::string seed(const std::string& seed_offset = "") const = 0; virtual std::string getSeedLanguage() const = 0; virtual void setSeedLanguage(const std::string &arg) = 0; //! returns wallet status (Status_Ok | Status_Error) @@ -627,6 +627,12 @@ struct Wallet virtual bool watchOnly() const = 0; /** + * @brief isDeterministic - checks if wallet keys are deterministic + * @return - true if deterministic + */ + virtual bool isDeterministic() const = 0; + + /** * @brief blockChainHeight - returns current blockchain height * @return */ @@ -901,9 +907,10 @@ struct Wallet /*! * \brief exportKeyImages - exports key images to file * \param filename + * \param all - export all key images or only those that have not yet been exported * \return - true on success */ - virtual bool exportKeyImages(const std::string &filename) = 0; + virtual bool exportKeyImages(const std::string &filename, bool all = false) = 0; /*! * \brief importKeyImages - imports key images from file diff --git a/src/wallet/message_store.cpp b/src/wallet/message_store.cpp index 303b576c7..b7b29420b 100644 --- a/src/wallet/message_store.cpp +++ b/src/wallet/message_store.cpp @@ -30,6 +30,8 @@ #include <boost/archive/portable_binary_iarchive.hpp> #include <boost/format.hpp> #include <boost/algorithm/string.hpp> +#include <boost/system/error_code.hpp> +#include <boost/filesystem.hpp> #include <fstream> #include <sstream> #include "file_io_utils.h" @@ -1340,7 +1342,10 @@ bool message_store::check_for_messages(const multisig_wallet_state &state, std:: } } std::vector<transport_message> transport_messages; - bool r = m_transporter.receive_messages(destinations, transport_messages); + if (!m_transporter.receive_messages(destinations, transport_messages)) + { + return false; + } if (!m_run.load(std::memory_order_relaxed)) { // Stop was called, don't waste time processing the messages diff --git a/src/wallet/node_rpc_proxy.cpp b/src/wallet/node_rpc_proxy.cpp index 48a602bf3..a576c267c 100644 --- a/src/wallet/node_rpc_proxy.cpp +++ b/src/wallet/node_rpc_proxy.cpp @@ -32,6 +32,8 @@ #include "rpc/rpc_payment_costs.h" #include "storages/http_abstract_invoke.h" +#include <boost/thread.hpp> + #define RETURN_ON_RPC_RESPONSE_ERROR(r, error, res, method) \ do { \ CHECK_AND_ASSERT_MES(error.code == 0, error.message, error.message); \ diff --git a/src/wallet/node_rpc_proxy.h b/src/wallet/node_rpc_proxy.h index 51b7f01dd..f5e3fca5f 100644 --- a/src/wallet/node_rpc_proxy.h +++ b/src/wallet/node_rpc_proxy.h @@ -30,6 +30,7 @@ #include <string> #include <boost/thread/mutex.hpp> +#include <boost/thread/recursive_mutex.hpp> #include "include_base_utils.h" #include "net/abstract_http_client.h" #include "rpc/core_rpc_server_commands_defs.h" diff --git a/src/wallet/ringdb.cpp b/src/wallet/ringdb.cpp index dfeb987ca..025a2037f 100644 --- a/src/wallet/ringdb.cpp +++ b/src/wallet/ringdb.cpp @@ -42,9 +42,6 @@ #define V1TAG ((uint64_t)798237759845202) -static const char zerokey[8] = {0}; -static const MDB_val zerokeyval = { sizeof(zerokey), (void *)zerokey }; - static int compare_hash32(const MDB_val *a, const MDB_val *b) { uint32_t *va = (uint32_t*) a->mv_data; diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index d80335db8..0af896c76 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -30,6 +30,7 @@ #include <numeric> #include <tuple> +#include <queue> #include <boost/format.hpp> #include <boost/optional/optional.hpp> #include <boost/algorithm/string/classification.hpp> @@ -344,8 +345,6 @@ std::string get_weight_string(const cryptonote::transaction &tx, size_t blob_siz std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variables_map& vm, bool unattended, const options& opts, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter) { - namespace ip = boost::asio::ip; - const bool testnet = command_line::get_arg(vm, opts.testnet); const bool stagenet = command_line::get_arg(vm, opts.stagenet); const network_type nettype = testnet ? TESTNET : stagenet ? STAGENET : MAINNET; @@ -367,7 +366,7 @@ std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variabl // user specified CA file or fingeprints implies enabled SSL by default epee::net_utils::ssl_options_t ssl_options = epee::net_utils::ssl_support_t::e_ssl_support_enabled; - if (command_line::get_arg(vm, opts.daemon_ssl_allow_any_cert)) + if (daemon_ssl_allow_any_cert) ssl_options.verification = epee::net_utils::ssl_verification_t::none; else if (!daemon_ssl_ca_file.empty() || !daemon_ssl_allowed_fingerprints.empty()) { @@ -1004,9 +1003,6 @@ bool get_pruned_tx(const cryptonote::COMMAND_RPC_GET_TRANSACTIONS::entry &entry, namespace tools { -// for now, limit to 30 attempts. TODO: discuss a good number to limit to. -const size_t MAX_SPLIT_ATTEMPTS = 30; - constexpr const std::chrono::seconds wallet2::rpc_timeout; const char* wallet2::tr(const char* str) { return i18n_translate(str, "tools::wallet2"); } @@ -1602,6 +1598,47 @@ std::string wallet2::get_subaddress_label(const cryptonote::subaddress_index& in return m_subaddress_labels[index.major][index.minor]; } //---------------------------------------------------------------------------------------------------- +void wallet2::scan_tx(const std::vector<crypto::hash> &txids) +{ + // Get the transactions from daemon in batches and add them to a priority queue ordered in chronological order + auto cmp_tx_entry = [](const cryptonote::COMMAND_RPC_GET_TRANSACTIONS::entry& l, const cryptonote::COMMAND_RPC_GET_TRANSACTIONS::entry& r) + { return l.block_height > r.block_height; }; + + std::priority_queue<cryptonote::COMMAND_RPC_GET_TRANSACTIONS::entry, std::vector<COMMAND_RPC_GET_TRANSACTIONS::entry>, decltype(cmp_tx_entry)> txq(cmp_tx_entry); + const size_t SLICE_SIZE = 100; // RESTRICTED_TRANSACTIONS_COUNT as defined in rpc/core_rpc_server.cpp, hardcoded in daemon code + for(size_t slice = 0; slice < txids.size(); slice += SLICE_SIZE) { + cryptonote::COMMAND_RPC_GET_TRANSACTIONS::request req = AUTO_VAL_INIT(req); + cryptonote::COMMAND_RPC_GET_TRANSACTIONS::response res = AUTO_VAL_INIT(res); + req.decode_as_json = false; + req.prune = true; + + size_t ntxes = slice + SLICE_SIZE > txids.size() ? txids.size() - slice : SLICE_SIZE; + for (size_t i = slice; i < slice + ntxes; ++i) + req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txids[i])); + + { + const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; + req.client = get_client_signature(); + bool r = epee::net_utils::invoke_http_json("/gettransactions", req, res, *m_http_client, rpc_timeout); + THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "Failed to get transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(res.txs.size() != req.txs_hashes.size(), error::wallet_internal_error, "Failed to get transaction from daemon"); + } + + for (auto& tx_info : res.txs) + txq.push(tx_info); + } + + // Process the transactions in chronologically ascending order + while(!txq.empty()) { + auto& tx_info = txq.top(); + cryptonote::transaction tx; + crypto::hash tx_hash; + THROW_WALLET_EXCEPTION_IF(!get_pruned_tx(tx_info, tx, tx_hash), error::wallet_internal_error, "Failed to get transaction from daemon (2)"); + process_new_transaction(tx_hash, tx, tx_info.output_indices, tx_info.block_height, 0, tx_info.block_timestamp, false, tx_info.in_pool, tx_info.double_spend_seen, {}, {}); + txq.pop(); + } +} +//---------------------------------------------------------------------------------------------------- void wallet2::set_subaddress_label(const cryptonote::subaddress_index& index, const std::string &label) { THROW_WALLET_EXCEPTION_IF(index.major >= m_subaddress_labels.size(), error::account_index_outofbound); @@ -5669,6 +5706,16 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass if (::serialization::serialize(ar, *this)) if (::serialization::check_stream_state(ar)) loaded = true; + if (!loaded) + { + std::stringstream iss; + iss << cache_data; + binary_archive<false> ar(iss); + ar.enable_varint_bug_backward_compatibility(); + if (::serialization::serialize(ar, *this)) + if (::serialization::check_stream_state(ar)) + loaded = true; + } } catch(...) { } @@ -5988,7 +6035,7 @@ uint64_t wallet2::balance(uint32_t index_major, bool strict) const { uint64_t amount = 0; if(m_light_wallet) - return m_light_wallet_unlocked_balance; + return m_light_wallet_balance; for (const auto& i : balance_per_subaddress(index_major, strict)) amount += i.second; return amount; @@ -6002,7 +6049,7 @@ uint64_t wallet2::unlocked_balance(uint32_t index_major, bool strict, uint64_t * if (time_to_unlock) *time_to_unlock = 0; if(m_light_wallet) - return m_light_wallet_balance; + return m_light_wallet_unlocked_balance; for (const auto& i : unlocked_balance_per_subaddress(index_major, strict)) { amount += i.second.first; @@ -6042,6 +6089,14 @@ std::map<uint32_t, uint64_t> wallet2::balance_per_subaddress(uint32_t index_majo found->second += utx.second.m_change; } } + + for (const auto& utx: m_unconfirmed_payments) + { + if (utx.second.m_pd.m_subaddr_index.major == index_major) + { + amount_per_subaddr[utx.second.m_pd.m_subaddr_index.minor] += utx.second.m_pd.m_amount; + } + } } return amount_per_subaddr; } @@ -7984,7 +8039,6 @@ void wallet2::light_wallet_get_outs(std::vector<std::vector<tools::wallet2::get_ // Check if we got enough outputs for each amount for(auto& out: ores.amount_outs) { - const uint64_t out_amount = boost::lexical_cast<uint64_t>(out.amount); THROW_WALLET_EXCEPTION_IF(out.outputs.size() < light_wallet_requested_outputs_count , error::wallet_internal_error, "Not enough outputs for amount: " + boost::lexical_cast<std::string>(out.amount)); MDEBUG(out.outputs.size() << " outputs for amount "+ boost::lexical_cast<std::string>(out.amount) + " received from light wallet node"); } @@ -9778,13 +9832,18 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp TX() : weight(0), needed_fee(0) {} - void add(const cryptonote::tx_destination_entry &de, uint64_t amount, unsigned int original_output_index, bool merge_destinations) { + /* Add an output to the transaction. + * Returns True if the output was added, False if there are no more available output slots. + */ + bool add(const cryptonote::tx_destination_entry &de, uint64_t amount, unsigned int original_output_index, bool merge_destinations, size_t max_dsts) { if (merge_destinations) { std::vector<cryptonote::tx_destination_entry>::iterator i; i = std::find_if(dsts.begin(), dsts.end(), [&](const cryptonote::tx_destination_entry &d) { return !memcmp (&d.addr, &de.addr, sizeof(de.addr)); }); if (i == dsts.end()) { + if (dsts.size() >= max_dsts) + return false; dsts.push_back(de); i = dsts.end() - 1; i->amount = 0; @@ -9797,12 +9856,15 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp std::string("original_output_index too large: ") + std::to_string(original_output_index) + " > " + std::to_string(dsts.size())); if (original_output_index == dsts.size()) { + if (dsts.size() >= max_dsts) + return false; dsts.push_back(de); dsts.back().amount = 0; } THROW_WALLET_EXCEPTION_IF(memcmp(&dsts[original_output_index].addr, &de.addr, sizeof(de.addr)), error::wallet_internal_error, "Mismatched destination address"); dsts[original_output_index].amount += amount; } + return true; } }; std::vector<TX> txes; @@ -10072,6 +10134,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp // clear any fake outs we'd already gathered, since we'll need a new set outs.clear(); + bool out_slots_exhausted = false; if (adding_fee) { LOG_PRINT_L2("We need more fee, adding it to fee"); @@ -10084,20 +10147,32 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp // we can fully pay that destination LOG_PRINT_L2("We can fully pay " << get_account_address_as_str(m_nettype, dsts[0].is_subaddress, dsts[0].addr) << " for " << print_money(dsts[0].amount)); - tx.add(dsts[0], dsts[0].amount, original_output_index, m_merge_destinations); + if (!tx.add(dsts[0], dsts[0].amount, original_output_index, m_merge_destinations, BULLETPROOF_MAX_OUTPUTS-1)) + { + LOG_PRINT_L2("Didn't pay: ran out of output slots"); + out_slots_exhausted = true; + break; + } available_amount -= dsts[0].amount; dsts[0].amount = 0; pop_index(dsts, 0); ++original_output_index; } - if (available_amount > 0 && !dsts.empty() && estimate_tx_weight(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size()+1, extra.size(), bulletproof, clsag) < TX_WEIGHT_TARGET(upper_transaction_weight_limit)) { + if (!out_slots_exhausted && available_amount > 0 && !dsts.empty() && estimate_tx_weight(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size()+1, extra.size(), bulletproof, clsag) < TX_WEIGHT_TARGET(upper_transaction_weight_limit)) { // we can partially fill that destination LOG_PRINT_L2("We can partially pay " << get_account_address_as_str(m_nettype, dsts[0].is_subaddress, dsts[0].addr) << " for " << print_money(available_amount) << "/" << print_money(dsts[0].amount)); - tx.add(dsts[0], available_amount, original_output_index, m_merge_destinations); - dsts[0].amount -= available_amount; - available_amount = 0; + if (tx.add(dsts[0], available_amount, original_output_index, m_merge_destinations, BULLETPROOF_MAX_OUTPUTS-1)) + { + dsts[0].amount -= available_amount; + available_amount = 0; + } + else + { + LOG_PRINT_L2("Didn't pay: ran out of output slots"); + out_slots_exhausted = true; + } } } @@ -10105,8 +10180,15 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp LOG_PRINT_L2("Considering whether to create a tx now, " << tx.selected_transfers.size() << " inputs, tx limit " << upper_transaction_weight_limit); bool try_tx = false; + + // If the new transaction is full, create it and start a new one + if (out_slots_exhausted) + { + LOG_PRINT_L2("Transaction is full, will create it and start a new tx"); + try_tx = true; + } // if we have preferred picks, but haven't yet used all of them, continue - if (preferred_inputs.empty()) + else if (preferred_inputs.empty()) { if (adding_fee) { @@ -10303,8 +10385,6 @@ bool wallet2::sanity_check(const std::vector<wallet2::pending_tx> &ptx_vector, s { MDEBUG("sanity_check: " << ptx_vector.size() << " txes, " << dsts.size() << " destinations"); - hw::device &hwdev = m_account.get_device(); - THROW_WALLET_EXCEPTION_IF(ptx_vector.empty(), error::wallet_internal_error, "No transactions"); // check every party in there does receive at least the required amount @@ -10341,7 +10421,6 @@ bool wallet2::sanity_check(const std::vector<wallet2::pending_tx> &ptx_vector, s for (const auto &r: required) { const account_public_address &address = r.first; - const crypto::public_key &view_pkey = address.m_view_public_key; uint64_t total_received = 0; for (const auto &ptx: ptx_vector) @@ -12459,7 +12538,7 @@ crypto::public_key wallet2::get_tx_pub_key_from_received_outs(const tools::walle bool wallet2::export_key_images(const std::string &filename, bool all) const { PERF_TIMER(export_key_images); - std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> ski = export_key_images(all); + std::pair<uint64_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> ski = export_key_images(all); std::string magic(KEY_IMAGE_EXPORT_FILE_MAGIC, strlen(KEY_IMAGE_EXPORT_FILE_MAGIC)); const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address; const uint32_t offset = ski.first; @@ -12486,7 +12565,7 @@ bool wallet2::export_key_images(const std::string &filename, bool all) const } //---------------------------------------------------------------------------------------------------- -std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> wallet2::export_key_images(bool all) const +std::pair<uint64_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> wallet2::export_key_images(bool all) const { PERF_TIMER(export_key_images_raw); std::vector<std::pair<crypto::key_image, crypto::signature>> ski; @@ -12983,7 +13062,7 @@ void wallet2::import_blockchain(const std::tuple<size_t, crypto::hash, std::vect m_last_block_reward = cryptonote::get_outs_money_amount(genesis.miner_tx); } //---------------------------------------------------------------------------------------------------- -std::pair<size_t, std::vector<tools::wallet2::transfer_details>> wallet2::export_outputs(bool all) const +std::pair<uint64_t, std::vector<tools::wallet2::transfer_details>> wallet2::export_outputs(bool all) const { PERF_TIMER(export_outputs); std::vector<tools::wallet2::transfer_details> outs; @@ -13023,7 +13102,7 @@ std::string wallet2::export_outputs_to_str(bool all) const return magic + ciphertext; } //---------------------------------------------------------------------------------------------------- -size_t wallet2::import_outputs(const std::pair<size_t, std::vector<tools::wallet2::transfer_details>> &outputs) +size_t wallet2::import_outputs(const std::pair<uint64_t, std::vector<tools::wallet2::transfer_details>> &outputs) { PERF_TIMER(import_outputs); @@ -13129,7 +13208,7 @@ size_t wallet2::import_outputs_from_str(const std::string &outputs_st) try { std::string body(data, headerlen); - std::pair<size_t, std::vector<tools::wallet2::transfer_details>> outputs; + std::pair<uint64_t, std::vector<tools::wallet2::transfer_details>> outputs; try { std::stringstream iss; @@ -13229,7 +13308,6 @@ rct::multisig_kLRki wallet2::get_multisig_composite_kLRki(size_t n, const std::u { CHECK_AND_ASSERT_THROW_MES(n < m_transfers.size(), "Bad transfer index"); - const transfer_details &td = m_transfers[n]; rct::multisig_kLRki kLRki = get_multisig_kLRki(n, rct::skGen()); // pick a L/R pair from every other participant but one diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 68f03db72..e96a6b51c 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -327,7 +327,7 @@ private: uint64_t m_block_height; cryptonote::transaction_prefix m_tx; crypto::hash m_txid; - size_t m_internal_output_index; + uint64_t m_internal_output_index; uint64_t m_global_output_index; bool m_spent; bool m_frozen; @@ -338,7 +338,7 @@ private: bool m_rct; bool m_key_image_known; bool m_key_image_request; // view wallets: we want to request it; cold wallets: it was requested - size_t m_pk_index; + uint64_t m_pk_index; cryptonote::subaddress_index m_subaddr_index; bool m_key_image_partial; std::vector<rct::key> m_multisig_k; @@ -1264,6 +1264,8 @@ private: std::string get_spend_proof(const crypto::hash &txid, const std::string &message); bool check_spend_proof(const crypto::hash &txid, const std::string &message, const std::string &sig_str); + void scan_tx(const std::vector<crypto::hash> &txids); + /*! * \brief Generates a proof that proves the reserve of unspent funds * \param account_minreserve When specified, collect outputs only belonging to the given account and prove the smallest reserve above the given amount @@ -1370,9 +1372,9 @@ private: bool verify_with_public_key(const std::string &data, const crypto::public_key &public_key, const std::string &signature) const; // Import/Export wallet data - std::pair<size_t, std::vector<tools::wallet2::transfer_details>> export_outputs(bool all = false) const; + std::pair<uint64_t, std::vector<tools::wallet2::transfer_details>> export_outputs(bool all = false) const; std::string export_outputs_to_str(bool all = false) const; - size_t import_outputs(const std::pair<size_t, std::vector<tools::wallet2::transfer_details>> &outputs); + size_t import_outputs(const std::pair<uint64_t, std::vector<tools::wallet2::transfer_details>> &outputs); size_t import_outputs_from_str(const std::string &outputs_st); payment_container export_payments() const; void import_payments(const payment_container &payments); @@ -1380,7 +1382,7 @@ private: std::tuple<size_t, crypto::hash, std::vector<crypto::hash>> export_blockchain() const; void import_blockchain(const std::tuple<size_t, crypto::hash, std::vector<crypto::hash>> &bc); bool export_key_images(const std::string &filename, bool all = false) const; - std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> export_key_images(bool all = false) const; + std::pair<uint64_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> export_key_images(bool all = false) const; uint64_t import_key_images(const std::vector<std::pair<crypto::key_image, crypto::signature>> &signed_key_images, size_t offset, uint64_t &spent, uint64_t &unspent, bool check_spent = true); uint64_t import_key_images(const std::string &filename, uint64_t &spent, uint64_t &unspent); bool import_key_images(std::vector<crypto::key_image> key_images, size_t offset=0, boost::optional<std::unordered_set<size_t>> selected_transfers=boost::none); @@ -1426,7 +1428,7 @@ private: bool get_rpc_payment_info(bool mining, bool &payment_required, uint64_t &credits, uint64_t &diff, uint64_t &credits_per_hash_found, cryptonote::blobdata &hashing_blob, uint64_t &height, uint64_t &seed_height, crypto::hash &seed_hash, crypto::hash &next_seed_hash, uint32_t &cookie); bool daemon_requires_payment(); bool make_rpc_payment(uint32_t nonce, uint32_t cookie, uint64_t &credits, uint64_t &balance); - bool search_for_rpc_payment(uint64_t credits_target, const std::function<bool(uint64_t, uint64_t)> &startfunc, const std::function<bool(unsigned)> &contfunc, const std::function<bool(uint64_t)> &foundfunc = NULL, const std::function<void(const std::string&)> &errorfunc = NULL); + bool search_for_rpc_payment(uint64_t credits_target, uint32_t n_threads, const std::function<bool(uint64_t, uint64_t)> &startfunc, const std::function<bool(unsigned)> &contfunc, const std::function<bool(uint64_t)> &foundfunc = NULL, const std::function<void(const std::string&)> &errorfunc = NULL); template<typename T> void handle_payment_changes(const T &res, std::true_type) { if (res.status == CORE_RPC_STATUS_OK || res.status == CORE_RPC_STATUS_PAYMENT_REQUIRED) m_rpc_payment_state.credits = res.credits; @@ -1549,6 +1551,7 @@ private: void finish_rescan_bc_keep_key_images(uint64_t transfer_height, const crypto::hash &hash); void enable_dns(bool enable) { m_use_dns = enable; } void set_offline(bool offline = true); + bool is_offline() const { return m_offline; } uint64_t credits() const { return m_rpc_payment_state.credits; } void credit_report(uint64_t &expected_spent, uint64_t &discrepancy) const { expected_spent = m_rpc_payment_state.expected_spent; discrepancy = m_rpc_payment_state.discrepancy; } diff --git a/src/wallet/wallet_rpc_payments.cpp b/src/wallet/wallet_rpc_payments.cpp index 6527b1384..bf278f695 100644 --- a/src/wallet/wallet_rpc_payments.cpp +++ b/src/wallet/wallet_rpc_payments.cpp @@ -42,6 +42,7 @@ #include "cryptonote_basic/blobdatatype.h" #include "common/i18n.h" #include "common/util.h" +#include "common/threadpool.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "wallet.wallet2.rpc_payments" @@ -101,7 +102,7 @@ bool wallet2::make_rpc_payment(uint32_t nonce, uint32_t cookie, uint64_t &credit return true; } //---------------------------------------------------------------------------------------------------- -bool wallet2::search_for_rpc_payment(uint64_t credits_target, const std::function<bool(uint64_t, uint64_t)> &startfunc, const std::function<bool(unsigned)> &contfunc, const std::function<bool(uint64_t)> &foundfunc, const std::function<void(const std::string&)> &errorfunc) +bool wallet2::search_for_rpc_payment(uint64_t credits_target, uint32_t n_threads, const std::function<bool(uint64_t, uint64_t)> &startfunc, const std::function<bool(unsigned)> &contfunc, const std::function<bool(uint64_t)> &foundfunc, const std::function<void(const std::string&)> &errorfunc) { bool need_payment = false; bool payment_required; @@ -139,49 +140,65 @@ bool wallet2::search_for_rpc_payment(uint64_t credits_target, const std::functio continue; } - crypto::hash hash; - const uint32_t local_nonce = nonce++; // wrapping's OK - *(uint32_t*)(hashing_blob.data() + 39) = SWAP32LE(local_nonce); - const uint8_t major_version = hashing_blob[0]; - if (major_version >= RX_BLOCK_VERSION) - { - const int miners = 1; - crypto::rx_slow_hash(height, seed_height, seed_hash.data, hashing_blob.data(), hashing_blob.size(), hash.data, miners, 0); - } - else + if(n_threads == 0) + n_threads = boost::thread::hardware_concurrency(); + + std::vector<crypto::hash> hash(n_threads); + tools::threadpool& tpool = tools::threadpool::getInstance(); + tools::threadpool::waiter waiter(tpool); + + const uint32_t local_nonce = nonce += n_threads; // wrapping's OK + for (size_t i = 0; i < n_threads; i++) { - int cn_variant = hashing_blob[0] >= 7 ? hashing_blob[0] - 6 : 0; - crypto::cn_slow_hash(hashing_blob.data(), hashing_blob.size(), hash, cn_variant, height); + tpool.submit(&waiter, [&, i] { + *(uint32_t*)(hashing_blob.data() + 39) = SWAP32LE(local_nonce-i); + const uint8_t major_version = hashing_blob[0]; + if (major_version >= RX_BLOCK_VERSION) + { + const int miners = 1; + crypto::rx_slow_hash(height, seed_height, seed_hash.data, hashing_blob.data(), hashing_blob.size(), hash[i].data, miners, 0); + } + else + { + int cn_variant = hashing_blob[0] >= 7 ? hashing_blob[0] - 6 : 0; + crypto::cn_slow_hash(hashing_blob.data(), hashing_blob.size(), hash[i], cn_variant, height); + } + }); } - ++n_hashes; - if (cryptonote::check_hash(hash, diff)) + waiter.wait(); + n_hashes += n_threads; + + for(size_t i=0; i < n_threads; i++) { - uint64_t credits, balance; - try + if (cryptonote::check_hash(hash[i], diff)) { - make_rpc_payment(local_nonce, cookie, credits, balance); - if (credits != credits_per_hash_found) + uint64_t credits, balance; + try { - MERROR("Found nonce, but daemon did not credit us with the expected amount"); + make_rpc_payment(local_nonce-i, cookie, credits, balance); + if (credits != credits_per_hash_found) + { + MERROR("Found nonce, but daemon did not credit us with the expected amount"); + if (errorfunc) + errorfunc("Found nonce, but daemon did not credit us with the expected amount"); + return false; + } + MDEBUG("Found nonce " << local_nonce-i << " at diff " << diff << ", gets us " << credits_per_hash_found << ", now " << balance << " credits"); + if (!foundfunc(credits)) + break; + } + catch (const tools::error::wallet_coded_rpc_error &e) + { + MWARNING("Found a local_nonce at diff " << diff << ", but failed to send it to the daemon"); if (errorfunc) - errorfunc("Found nonce, but daemon did not credit us with the expected amount"); - return false; + errorfunc("Found nonce, but daemon errored out with error " + std::to_string(e.code()) + ": " + e.status() + ", continuing"); + } + catch (const std::exception &e) + { + MWARNING("Found a local_nonce at diff " << diff << ", but failed to send it to the daemon"); + if (errorfunc) + errorfunc("Found nonce, but daemon errored out with: '" + std::string(e.what()) + "', continuing"); } - MDEBUG("Found nonce " << local_nonce << " at diff " << diff << ", gets us " << credits_per_hash_found << ", now " << balance << " credits"); - if (!foundfunc(credits)) - break; - } - catch (const tools::error::wallet_coded_rpc_error &e) - { - MWARNING("Found a local_nonce at diff " << diff << ", but failed to send it to the daemon"); - if (errorfunc) - errorfunc("Found nonce, but daemon errored out with error " + std::to_string(e.code()) + ": " + e.status() + ", continuing"); - } - catch (const std::exception &e) - { - MWARNING("Found a local_nonce at diff " << diff << ", but failed to send it to the daemon"); - if (errorfunc) - errorfunc("Found nonce, but daemon errored out with: '" + std::string(e.what()) + "', continuing"); } } } diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index 03db8b70f..b39a40b64 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -687,7 +687,7 @@ namespace tools { if (!m_wallet) return not_open(er); const std::pair<std::map<std::string, std::string>, std::vector<std::string>> account_tags = m_wallet->get_account_tags(); - for (const std::pair<std::string, std::string>& p : account_tags.first) + for (const std::pair<const std::string, std::string>& p : account_tags.first) { res.account_tags.resize(res.account_tags.size() + 1); auto& info = res.account_tags.back(); @@ -1282,7 +1282,6 @@ namespace tools dests.erase(cd.change_dts.addr); } - size_t n_dummy_outputs = 0; for (auto i = dests.begin(); i != dests.end(); ) { if (i->second.second > 0) @@ -1888,6 +1887,7 @@ namespace tools rpc_transfers.tx_hash = epee::string_tools::pod_to_hex(td.m_txid); rpc_transfers.subaddr_index = {td.m_subaddr_index.major, td.m_subaddr_index.minor}; rpc_transfers.key_image = td.m_key_image_known ? epee::string_tools::pod_to_hex(td.m_key_image) : ""; + rpc_transfers.pubkey = epee::string_tools::pod_to_hex(td.get_public_key()); rpc_transfers.block_height = td.m_block_height; rpc_transfers.frozen = td.m_frozen; rpc_transfers.unlocked = m_wallet->is_transfer_unlocked(td); @@ -2702,7 +2702,7 @@ namespace tools if (!m_wallet) return not_open(er); try { - std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> ski = m_wallet->export_key_images(req.all); + std::pair<uint64_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> ski = m_wallet->export_key_images(req.all); res.offset = ski.first; res.signed_key_images.resize(ski.second.size()); for (size_t n = 0; n < ski.second.size(); ++n) @@ -3014,6 +3014,41 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_scan_tx(const wallet_rpc::COMMAND_RPC_SCAN_TX::request& req, wallet_rpc::COMMAND_RPC_SCAN_TX::response& res, epee::json_rpc::error& er, const connection_context *ctx) + { + if (!m_wallet) return not_open(er); + if (m_restricted) + { + er.code = WALLET_RPC_ERROR_CODE_DENIED; + er.message = "Command unavailable in restricted mode."; + return false; + } + + std::vector<crypto::hash> txids; + std::list<std::string>::const_iterator i = req.txids.begin(); + while (i != req.txids.end()) + { + cryptonote::blobdata txid_blob; + if(!epee::string_tools::parse_hexstr_to_binbuff(*i++, txid_blob) || txid_blob.size() != sizeof(crypto::hash)) + { + er.code = WALLET_RPC_ERROR_CODE_WRONG_TXID; + er.message = "TX ID has invalid format"; + return false; + } + + crypto::hash txid = *reinterpret_cast<const crypto::hash*>(txid_blob.data()); + txids.push_back(txid); + } + + try { + m_wallet->scan_tx(txids); + } catch (const std::exception &e) { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_rescan_spent(const wallet_rpc::COMMAND_RPC_RESCAN_SPENT::request& req, wallet_rpc::COMMAND_RPC_RESCAN_SPENT::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); @@ -3119,17 +3154,7 @@ namespace tools } std::string wallet_file = req.filename.empty() ? "" : (m_wallet_dir + "/" + req.filename); { - std::vector<std::string> languages; - crypto::ElectrumWords::get_language_list(languages, false); - std::vector<std::string>::iterator it; - - it = std::find(languages.begin(), languages.end(), req.language); - if (it == languages.end()) - { - crypto::ElectrumWords::get_language_list(languages, true); - it = std::find(languages.begin(), languages.end(), req.language); - } - if (it == languages.end()) + if (!crypto::ElectrumWords::is_valid_language(req.language)) { er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; er.message = "Unknown language: " + req.language; @@ -3547,6 +3572,17 @@ namespace tools return false; } + if (!req.language.empty()) + { + if (!crypto::ElectrumWords::is_valid_language(req.language)) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "The specified seed language is invalid."; + return false; + } + wal->set_seed_language(req.language); + } + // set blockheight if given try { @@ -3690,12 +3726,7 @@ namespace tools er.message = "Wallet was using the old seed language. You need to specify a new seed language."; return false; } - std::vector<std::string> language_list; - std::vector<std::string> language_list_en; - crypto::ElectrumWords::get_language_list(language_list); - crypto::ElectrumWords::get_language_list(language_list_en, true); - if (std::find(language_list.begin(), language_list.end(), req.language) == language_list.end() && - std::find(language_list_en.begin(), language_list_en.end(), req.language) == language_list_en.end()) + if (!crypto::ElectrumWords::is_valid_language(req.language)) { er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; er.message = "Wallet was using the old seed language, and the specified new seed language is invalid."; @@ -4438,7 +4469,14 @@ public: wal->stop(); }); - wal->refresh(wal->is_trusted_daemon()); + try + { + wal->refresh(wal->is_trusted_daemon()); + } + catch (const std::exception& e) + { + LOG_ERROR(tools::wallet_rpc_server::tr("Initial refresh failed: ") << e.what()); + } // if we ^C during potentially length load/refresh, there's no server loop yet if (quit) { diff --git a/src/wallet/wallet_rpc_server.h b/src/wallet/wallet_rpc_server.h index 6e39eca1e..9f9e3c134 100644 --- a/src/wallet/wallet_rpc_server.h +++ b/src/wallet/wallet_rpc_server.h @@ -131,6 +131,7 @@ namespace tools MAP_JON_RPC_WE("delete_address_book",on_delete_address_book,wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY) MAP_JON_RPC_WE("refresh", on_refresh, wallet_rpc::COMMAND_RPC_REFRESH) MAP_JON_RPC_WE("auto_refresh", on_auto_refresh, wallet_rpc::COMMAND_RPC_AUTO_REFRESH) + MAP_JON_RPC_WE("scan_tx", on_scan_tx, wallet_rpc::COMMAND_RPC_SCAN_TX) MAP_JON_RPC_WE("rescan_spent", on_rescan_spent, wallet_rpc::COMMAND_RPC_RESCAN_SPENT) MAP_JON_RPC_WE("start_mining", on_start_mining, wallet_rpc::COMMAND_RPC_START_MINING) MAP_JON_RPC_WE("stop_mining", on_stop_mining, wallet_rpc::COMMAND_RPC_STOP_MINING) @@ -218,6 +219,7 @@ namespace tools bool on_delete_address_book(const wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_refresh(const wallet_rpc::COMMAND_RPC_REFRESH::request& req, wallet_rpc::COMMAND_RPC_REFRESH::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_auto_refresh(const wallet_rpc::COMMAND_RPC_AUTO_REFRESH::request& req, wallet_rpc::COMMAND_RPC_AUTO_REFRESH::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_scan_tx(const wallet_rpc::COMMAND_RPC_SCAN_TX::request& req, wallet_rpc::COMMAND_RPC_SCAN_TX::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_rescan_spent(const wallet_rpc::COMMAND_RPC_RESCAN_SPENT::request& req, wallet_rpc::COMMAND_RPC_RESCAN_SPENT::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_start_mining(const wallet_rpc::COMMAND_RPC_START_MINING::request& req, wallet_rpc::COMMAND_RPC_START_MINING::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_stop_mining(const wallet_rpc::COMMAND_RPC_STOP_MINING::request& req, wallet_rpc::COMMAND_RPC_STOP_MINING::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); diff --git a/src/wallet/wallet_rpc_server_commands_defs.h b/src/wallet/wallet_rpc_server_commands_defs.h index 81f83fb18..0002508a2 100644 --- a/src/wallet/wallet_rpc_server_commands_defs.h +++ b/src/wallet/wallet_rpc_server_commands_defs.h @@ -47,7 +47,7 @@ // advance which version they will stop working with // Don't go over 32767 for any of these #define WALLET_RPC_VERSION_MAJOR 1 -#define WALLET_RPC_VERSION_MINOR 20 +#define WALLET_RPC_VERSION_MINOR 21 #define MAKE_WALLET_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define WALLET_RPC_VERSION MAKE_WALLET_RPC_VERSION(WALLET_RPC_VERSION_MAJOR, WALLET_RPC_VERSION_MINOR) namespace tools @@ -1010,6 +1010,7 @@ namespace wallet_rpc std::string tx_hash; cryptonote::subaddress_index subaddr_index; std::string key_image; + std::string pubkey; // owned output public key found uint64_t block_height; bool frozen; bool unlocked; @@ -1021,6 +1022,7 @@ namespace wallet_rpc KV_SERIALIZE(tx_hash) KV_SERIALIZE(subaddr_index) KV_SERIALIZE(key_image) + KV_SERIALIZE(pubkey); KV_SERIALIZE(block_height) KV_SERIALIZE(frozen) KV_SERIALIZE(unlocked) @@ -2028,6 +2030,26 @@ namespace wallet_rpc typedef epee::misc_utils::struct_init<response_t> response; }; + struct COMMAND_RPC_SCAN_TX + { + struct request_t + { + std::list<std::string> txids; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(txids) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<request_t> request; + + struct response_t + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<response_t> response; + }; + struct COMMAND_RPC_START_MINING { struct request_t @@ -2192,6 +2214,7 @@ namespace wallet_rpc std::string viewkey; std::string password; bool autosave_current; + std::string language; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE_OPT(restore_height, (uint64_t)0) @@ -2201,6 +2224,7 @@ namespace wallet_rpc KV_SERIALIZE(viewkey) KV_SERIALIZE(password) KV_SERIALIZE_OPT(autosave_current, true) + KV_SERIALIZE(language) END_KV_SERIALIZE_MAP() }; |