diff options
Diffstat (limited to 'src')
82 files changed, 4991 insertions, 966 deletions
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2ae72ee81..da6d76d97 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -109,6 +109,7 @@ add_subdirectory(ringct) add_subdirectory(checkpoints) add_subdirectory(cryptonote_basic) add_subdirectory(cryptonote_core) +add_subdirectory(lmdb) add_subdirectory(multisig) add_subdirectory(net) if(NOT IOS) diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 313e716e0..d2fe39fc2 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -947,6 +947,17 @@ public: virtual size_t get_block_weight(const uint64_t& height) const = 0; /** + * @brief fetch the last N blocks' weights + * + * If there are fewer than N blocks, the returned array will be smaller than N + * + * @param count the number of blocks requested + * + * @return the weights + */ + virtual std::vector<uint64_t> get_block_weights(uint64_t start_height, size_t count) const = 0; + + /** * @brief fetch a block's cumulative difficulty * * The subclass should return the cumulative difficulty of the block with the @@ -1000,6 +1011,17 @@ public: virtual uint64_t get_block_long_term_weight(const uint64_t& height) const = 0; /** + * @brief fetch the last N blocks' long term weights + * + * If there are fewer than N blocks, the returned array will be smaller than N + * + * @param count the number of blocks requested + * + * @return the weights + */ + virtual std::vector<uint64_t> get_long_term_block_weights(uint64_t start_height, size_t count) const = 0; + + /** * @brief fetch a block's hash * * The subclass should return hash of the block with the diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index becffed16..9f71fd068 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -54,7 +54,7 @@ using epee::string_tools::pod_to_hex; using namespace crypto; // Increase when the DB structure changes -#define VERSION 4 +#define VERSION 5 namespace { @@ -274,7 +274,7 @@ typedef struct mdb_block_info_1 uint64_t bi_timestamp; uint64_t bi_coins; uint64_t bi_weight; // a size_t really but we need 32-bit compat - difficulty_type bi_diff; + uint64_t bi_diff; crypto::hash bi_hash; } mdb_block_info_1; @@ -284,7 +284,7 @@ typedef struct mdb_block_info_2 uint64_t bi_timestamp; uint64_t bi_coins; uint64_t bi_weight; // a size_t really but we need 32-bit compat - difficulty_type bi_diff; + uint64_t bi_diff; crypto::hash bi_hash; uint64_t bi_cum_rct; } mdb_block_info_2; @@ -295,13 +295,26 @@ typedef struct mdb_block_info_3 uint64_t bi_timestamp; uint64_t bi_coins; uint64_t bi_weight; // a size_t really but we need 32-bit compat - difficulty_type bi_diff; + uint64_t bi_diff; crypto::hash bi_hash; uint64_t bi_cum_rct; uint64_t bi_long_term_block_weight; } mdb_block_info_3; -typedef mdb_block_info_3 mdb_block_info; +typedef struct mdb_block_info_4 +{ + uint64_t bi_height; + uint64_t bi_timestamp; + uint64_t bi_coins; + uint64_t bi_weight; // a size_t really but we need 32-bit compat + uint64_t bi_diff_lo; + uint64_t bi_diff_hi; + crypto::hash bi_hash; + uint64_t bi_cum_rct; + uint64_t bi_long_term_block_weight; +} mdb_block_info_4; + +typedef mdb_block_info_4 mdb_block_info; typedef struct blk_height { crypto::hash bh_hash; @@ -757,7 +770,8 @@ void BlockchainLMDB::add_block(const block& blk, size_t block_weight, uint64_t l bi.bi_timestamp = blk.timestamp; bi.bi_coins = coins_generated; bi.bi_weight = block_weight; - bi.bi_diff = cumulative_difficulty; + bi.bi_diff_hi = (cumulative_difficulty >> 64).convert_to<uint64_t>(); + bi.bi_diff_lo = (cumulative_difficulty << 64 >> 64).convert_to<uint64_t>(); bi.bi_hash = blk_hash; bi.bi_cum_rct = num_rct_outs; if (blk.major_version >= 4) @@ -2445,6 +2459,70 @@ size_t BlockchainLMDB::get_block_weight(const uint64_t& height) const return ret; } +std::vector<uint64_t> BlockchainLMDB::get_block_info_64bit_fields(uint64_t start_height, size_t count, off_t offset) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + RCURSOR(block_info); + + const uint64_t h = height(); + if (start_height >= h) + throw0(DB_ERROR(("Height " + std::to_string(start_height) + " not in blockchain").c_str())); + + std::vector<uint64_t> ret; + ret.reserve(count); + + MDB_val v; + uint64_t range_begin = 0, range_end = 0; + for (uint64_t height = start_height; height < h && count--; ++height) + { + if (height >= range_begin && height < range_end) + { + // nothing to do + } + else + { + int result = 0; + if (range_end > 0) + { + MDB_val k2; + result = mdb_cursor_get(m_cur_block_info, &k2, &v, MDB_NEXT_MULTIPLE); + range_begin = ((const mdb_block_info*)v.mv_data)->bi_height; + range_end = range_begin + v.mv_size / sizeof(mdb_block_info); // whole records please + if (height < range_begin || height >= range_end) + throw0(DB_ERROR(("Height " + std::to_string(height) + " not included in multiple record range: " + std::to_string(range_begin) + "-" + std::to_string(range_end)).c_str())); + } + else + { + v.mv_size = sizeof(uint64_t); + v.mv_data = (void*)&height; + result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); + range_begin = height; + range_end = range_begin + 1; + } + if (result) + throw0(DB_ERROR(lmdb_error("Error attempting to retrieve block_info from the db: ", result).c_str())); + } + const mdb_block_info *bi = ((const mdb_block_info *)v.mv_data) + (height - range_begin); + ret.push_back(*(const uint64_t*)(((const char*)bi) + offset)); + } + + TXN_POSTFIX_RDONLY(); + return ret; +} + +std::vector<uint64_t> BlockchainLMDB::get_block_weights(uint64_t start_height, size_t count) const +{ + return get_block_info_64bit_fields(start_height, count, offsetof(mdb_block_info, bi_weight)); +} + +std::vector<uint64_t> BlockchainLMDB::get_long_term_block_weights(uint64_t start_height, size_t count) const +{ + return get_block_info_64bit_fields(start_height, count, offsetof(mdb_block_info, bi_long_term_block_weight)); +} + difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& height) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " height: " << height); @@ -2463,7 +2541,9 @@ difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& throw0(DB_ERROR("Error attempting to retrieve a cumulative difficulty from the db")); mdb_block_info *bi = (mdb_block_info *)result.mv_data; - difficulty_type ret = bi->bi_diff; + difficulty_type ret = bi->bi_diff_hi; + ret <<= 64; + ret |= bi->bi_diff_lo; TXN_POSTFIX_RDONLY(); return ret; } @@ -4976,6 +5056,133 @@ void BlockchainLMDB::migrate_3_4() txn.commit(); } +void BlockchainLMDB::migrate_4_5() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + uint64_t i; + int result; + mdb_txn_safe txn(false); + MDB_val k, v; + char *ptr; + + MGINFO_YELLOW("Migrating blockchain from DB version 4 to 5 - this may take a while:"); + + do { + LOG_PRINT_L1("migrating block info:"); + + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + + MDB_stat db_stats; + if ((result = mdb_stat(txn, m_blocks, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str())); + const uint64_t blockchain_height = db_stats.ms_entries; + + /* the block_info table name is the same but the old version and new version + * have incompatible data. Create a new table. We want the name to be similar + * to the old name so that it will occupy the same location in the DB. + */ + MDB_dbi o_block_info = m_block_info; + lmdb_db_open(txn, "block_infn", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn"); + mdb_set_dupsort(txn, m_block_info, compare_uint64); + + + MDB_cursor *c_blocks; + result = mdb_cursor_open(txn, m_blocks, &c_blocks); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for blocks: ", result).c_str())); + + MDB_cursor *c_old, *c_cur; + i = 0; + while(1) { + if (!(i % 1000)) { + if (i) { + LOGIF(el::Level::Info) { + std::cout << i << " / " << blockchain_height << " \r" << std::flush; + } + txn.commit(); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + } + result = mdb_cursor_open(txn, m_block_info, &c_cur); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_infn: ", result).c_str())); + result = mdb_cursor_open(txn, o_block_info, &c_old); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_info: ", result).c_str())); + if (!i) { + MDB_stat db_stat; + result = mdb_stat(txn, m_block_info, &db_stats); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to query m_block_info: ", result).c_str())); + i = db_stats.ms_entries; + } + } + result = mdb_cursor_get(c_old, &k, &v, MDB_NEXT); + if (result == MDB_NOTFOUND) { + txn.commit(); + break; + } + else if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from block_info: ", result).c_str())); + const mdb_block_info_3 *bi_old = (const mdb_block_info_3*)v.mv_data; + mdb_block_info_4 bi; + bi.bi_height = bi_old->bi_height; + bi.bi_timestamp = bi_old->bi_timestamp; + bi.bi_coins = bi_old->bi_coins; + bi.bi_weight = bi_old->bi_weight; + bi.bi_diff_lo = bi_old->bi_diff; + bi.bi_diff_hi = 0; + bi.bi_hash = bi_old->bi_hash; + bi.bi_cum_rct = bi_old->bi_cum_rct; + bi.bi_long_term_block_weight = bi_old->bi_long_term_block_weight; + + MDB_val_set(nv, bi); + result = mdb_cursor_put(c_cur, (MDB_val *)&zerokval, &nv, MDB_APPENDDUP); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to put a record into block_infn: ", result).c_str())); + /* we delete the old records immediately, so the overall DB and mapsize should not grow. + * This is a little slower than just letting mdb_drop() delete it all at the end, but + * it saves a significant amount of disk space. + */ + result = mdb_cursor_del(c_old, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_info: ", result).c_str())); + i++; + } + + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + /* Delete the old table */ + result = mdb_drop(txn, o_block_info, 1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete old block_info table: ", result).c_str())); + + RENAME_DB("block_infn"); + mdb_dbi_close(m_env, m_block_info); + + lmdb_db_open(txn, "block_info", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn"); + mdb_set_dupsort(txn, m_block_info, compare_uint64); + + txn.commit(); + } while(0); + + uint32_t version = 5; + v.mv_data = (void *)&version; + v.mv_size = sizeof(version); + MDB_val_str(vk, "version"); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + result = mdb_put(txn, m_properties, &vk, &v, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to update version for the db: ", result).c_str())); + txn.commit(); +} + void BlockchainLMDB::migrate(const uint32_t oldversion) { if (oldversion < 1) @@ -4986,6 +5193,8 @@ void BlockchainLMDB::migrate(const uint32_t oldversion) migrate_2_3(); if (oldversion < 4) migrate_3_4(); + if (oldversion < 5) + migrate_4_5(); } } // namespace cryptonote diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 7a04c5f0c..2f89b77ac 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -219,6 +219,8 @@ public: virtual size_t get_block_weight(const uint64_t& height) const; + virtual std::vector<uint64_t> get_block_weights(uint64_t start_height, size_t count) const; + virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const; virtual difficulty_type get_block_difficulty(const uint64_t& height) const; @@ -227,6 +229,8 @@ public: virtual uint64_t get_block_long_term_weight(const uint64_t& height) const; + virtual std::vector<uint64_t> get_long_term_block_weights(uint64_t start_height, size_t count) const; + virtual crypto::hash get_block_hash_from_height(const uint64_t& height) const; virtual std::vector<block> get_blocks_range(const uint64_t& h1, const uint64_t& h2) const; @@ -394,6 +398,8 @@ private: virtual uint64_t get_database_size() const; + std::vector<uint64_t> get_block_info_64bit_fields(uint64_t start_height, size_t count, off_t offset) const; + // fix up anything that may be wrong due to past bugs virtual void fixup(); @@ -412,6 +418,9 @@ private: // migrate from DB version 3 to 4 void migrate_3_4(); + // migrate from DB version 4 to 5 + void migrate_4_5(); + void cleanup_batch(); private: diff --git a/src/blockchain_db/testdb.h b/src/blockchain_db/testdb.h index eb9c3e45a..7916364c5 100644 --- a/src/blockchain_db/testdb.h +++ b/src/blockchain_db/testdb.h @@ -72,10 +72,12 @@ public: virtual std::vector<uint64_t> get_block_cumulative_rct_outputs(const std::vector<uint64_t> &heights) const { return {}; } virtual uint64_t get_top_block_timestamp() const { return 0; } virtual size_t get_block_weight(const uint64_t& height) const { return 128; } + virtual std::vector<uint64_t> get_block_weights(uint64_t start_height, size_t count) const { return {}; } virtual cryptonote::difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const { return 10; } virtual cryptonote::difficulty_type get_block_difficulty(const uint64_t& height) const { return 0; } virtual uint64_t get_block_already_generated_coins(const uint64_t& height) const { return 10000000000; } virtual uint64_t get_block_long_term_weight(const uint64_t& height) const { return 128; } + virtual std::vector<uint64_t> get_long_term_block_weights(uint64_t start_height, size_t count) const { return {}; } virtual crypto::hash get_block_hash_from_height(const uint64_t& height) const { return crypto::hash(); } virtual std::vector<cryptonote::block> get_blocks_range(const uint64_t& h1, const uint64_t& h2) const { return std::vector<cryptonote::block>(); } virtual std::vector<crypto::hash> get_hashes_range(const uint64_t& h1, const uint64_t& h2) const { return std::vector<crypto::hash>(); } diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp index e4efdc3cb..8454595ac 100644 --- a/src/blockchain_utilities/blockchain_import.cpp +++ b/src/blockchain_utilities/blockchain_import.cpp @@ -294,7 +294,8 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path } // 4 byte magic + (currently) 1024 byte header structures - bootstrap.seek_to_first_chunk(import_file); + uint8_t major_version, minor_version; + bootstrap.seek_to_first_chunk(import_file, major_version, minor_version); std::string str1; char buffer1[1024]; @@ -415,7 +416,23 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path { str1.assign(buffer_block, chunk_size); bootstrap::block_package bp; - if (! ::serialization::parse_binary(str1, bp)) + bool res; + if (major_version == 0) + { + bootstrap::block_package_1 bp1; + res = ::serialization::parse_binary(str1, bp1); + if (res) + { + bp.block = std::move(bp1.block); + bp.txs = std::move(bp1.txs); + bp.block_weight = bp1.block_weight; + bp.cumulative_difficulty = bp1.cumulative_difficulty; + bp.coins_generated = bp1.coins_generated; + } + } + else + res = ::serialization::parse_binary(str1, bp); + if (!res) throw std::runtime_error("Error in deserialization of chunk"); int display_interval = 1000; diff --git a/src/blockchain_utilities/bootstrap_file.cpp b/src/blockchain_utilities/bootstrap_file.cpp index fb9a24f5d..252c79776 100644 --- a/src/blockchain_utilities/bootstrap_file.cpp +++ b/src/blockchain_utilities/bootstrap_file.cpp @@ -124,8 +124,8 @@ bool BootstrapFile::initialize_file() *m_raw_data_file << blob; bootstrap::file_info bfi; - bfi.major_version = 0; - bfi.minor_version = 1; + bfi.major_version = 1; + bfi.minor_version = 0; bfi.header_size = header_size; bootstrap::blocks_info bbi; @@ -323,7 +323,7 @@ bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem return BootstrapFile::close(); } -uint64_t BootstrapFile::seek_to_first_chunk(std::ifstream& import_file) +uint64_t BootstrapFile::seek_to_first_chunk(std::ifstream& import_file, uint8_t &major_version, uint8_t &minor_version) { uint32_t file_magic; @@ -371,6 +371,8 @@ uint64_t BootstrapFile::seek_to_first_chunk(std::ifstream& import_file) uint64_t full_header_size = sizeof(file_magic) + bfi.header_size; import_file.seekg(full_header_size); + major_version = bfi.major_version; + minor_version = bfi.minor_version; return full_header_size; } @@ -461,7 +463,8 @@ uint64_t BootstrapFile::count_blocks(const std::string& import_file_path, std::s } uint64_t full_header_size; // 4 byte magic + length of header structures - full_header_size = seek_to_first_chunk(import_file); + uint8_t major_version, minor_version; + full_header_size = seek_to_first_chunk(import_file, major_version, minor_version); MINFO("Scanning blockchain from bootstrap file..."); bool quit = false; diff --git a/src/blockchain_utilities/bootstrap_file.h b/src/blockchain_utilities/bootstrap_file.h index 5fb2cf366..1e6ef5d81 100644 --- a/src/blockchain_utilities/bootstrap_file.h +++ b/src/blockchain_utilities/bootstrap_file.h @@ -60,7 +60,7 @@ public: uint64_t count_bytes(std::ifstream& import_file, uint64_t blocks, uint64_t& h, bool& quit); uint64_t count_blocks(const std::string& dir_path, std::streampos& start_pos, uint64_t& seek_height); uint64_t count_blocks(const std::string& dir_path); - uint64_t seek_to_first_chunk(std::ifstream& import_file); + uint64_t seek_to_first_chunk(std::ifstream& import_file, uint8_t &major_version, uint8_t &minor_version); bool store_blockchain_raw(cryptonote::Blockchain* cs, cryptonote::tx_memory_pool* txp, boost::filesystem::path& output_file, uint64_t use_block_height=0); diff --git a/src/blockchain_utilities/bootstrap_serialization.h b/src/blockchain_utilities/bootstrap_serialization.h index 554c6d56e..70b3eea7e 100644 --- a/src/blockchain_utilities/bootstrap_serialization.h +++ b/src/blockchain_utilities/bootstrap_serialization.h @@ -29,7 +29,7 @@ #pragma once #include "cryptonote_basic/cryptonote_boost_serialization.h" -#include "cryptonote_basic/difficulty.h" +#include "serialization/difficulty_type.h" namespace cryptonote @@ -66,6 +66,23 @@ namespace cryptonote END_SERIALIZE() }; + struct block_package_1 + { + cryptonote::block block; + std::vector<transaction> txs; + size_t block_weight; + uint64_t cumulative_difficulty; + uint64_t coins_generated; + + BEGIN_SERIALIZE() + FIELD(block) + FIELD(txs) + VARINT_FIELD(block_weight) + VARINT_FIELD(cumulative_difficulty) + VARINT_FIELD(coins_generated) + END_SERIALIZE() + }; + struct block_package { cryptonote::block block; @@ -78,7 +95,7 @@ namespace cryptonote FIELD(block) FIELD(txs) VARINT_FIELD(block_weight) - VARINT_FIELD(cumulative_difficulty) + FIELD(cumulative_difficulty) VARINT_FIELD(coins_generated) END_SERIALIZE() }; diff --git a/src/common/dns_utils.cpp b/src/common/dns_utils.cpp index 58227dfa7..1a1155c7c 100644 --- a/src/common/dns_utils.cpp +++ b/src/common/dns_utils.cpp @@ -32,9 +32,9 @@ #include <stdlib.h> #include "include_base_utils.h" +#include "common/threadpool.h" #include <random> #include <boost/thread/mutex.hpp> -#include <boost/thread/thread.hpp> #include <boost/algorithm/string/join.hpp> #include <boost/optional.hpp> using namespace epee; @@ -232,13 +232,24 @@ public: char *str; }; +static void add_anchors(ub_ctx *ctx) +{ + const char * const *ds = ::get_builtin_ds(); + while (*ds) + { + MINFO("adding trust anchor: " << *ds); + ub_ctx_add_ta(ctx, string_copy(*ds++)); + } +} + DNSResolver::DNSResolver() : m_data(new DNSResolverData()) { int use_dns_public = 0; std::vector<std::string> dns_public_addr; - if (auto res = getenv("DNS_PUBLIC")) + const char *DNS_PUBLIC = getenv("DNS_PUBLIC"); + if (DNS_PUBLIC) { - dns_public_addr = tools::dns_utils::parse_dns_public(res); + dns_public_addr = tools::dns_utils::parse_dns_public(DNS_PUBLIC); if (!dns_public_addr.empty()) { MGINFO("Using public DNS server(s): " << boost::join(dns_public_addr, ", ") << " (TCP)"); @@ -266,11 +277,27 @@ DNSResolver::DNSResolver() : m_data(new DNSResolverData()) ub_ctx_hosts(m_data->m_ub_context, NULL); } - const char * const *ds = ::get_builtin_ds(); - while (*ds) + add_anchors(m_data->m_ub_context); + + if (!DNS_PUBLIC) { - MINFO("adding trust anchor: " << *ds); - ub_ctx_add_ta(m_data->m_ub_context, string_copy(*ds++)); + // if no DNS_PUBLIC specified, we try a lookup to what we know + // should be a valid DNSSEC record, and switch to known good + // DNSSEC resolvers if verification fails + bool available, valid; + static const char *probe_hostname = "updates.moneropulse.org"; + auto records = get_txt_record(probe_hostname, available, valid); + if (!valid) + { + MINFO("Failed to verify DNSSEC record from " << probe_hostname << ", falling back to TCP with well known DNSSEC resolvers"); + ub_ctx_delete(m_data->m_ub_context); + m_data->m_ub_context = ub_ctx_create(); + add_anchors(m_data->m_ub_context); + for (const auto &ip: DEFAULT_DNS_PUBLIC_ADDR) + ub_ctx_set_fwd(m_data->m_ub_context, string_copy(ip)); + ub_ctx_set_option(m_data->m_ub_context, string_copy("do-udp:"), string_copy("no")); + ub_ctx_set_option(m_data->m_ub_context, string_copy("do-tcp:"), string_copy("yes")); + } } } @@ -496,16 +523,16 @@ bool load_txt_records_from_dns(std::vector<std::string> &good_records, const std size_t first_index = dis(gen); // send all requests in parallel - std::vector<boost::thread> threads(dns_urls.size()); std::deque<bool> avail(dns_urls.size(), false), valid(dns_urls.size(), false); + tools::threadpool& tpool = tools::threadpool::getInstance(); + tools::threadpool::waiter waiter; for (size_t n = 0; n < dns_urls.size(); ++n) { - threads[n] = boost::thread([n, dns_urls, &records, &avail, &valid](){ + tpool.submit(&waiter,[n, dns_urls, &records, &avail, &valid](){ records[n] = tools::DNSResolver::instance().get_txt_record(dns_urls[n], avail[n], valid[n]); }); } - for (size_t n = 0; n < dns_urls.size(); ++n) - threads[n].join(); + waiter.wait(&tpool); size_t cur_index = first_index; do diff --git a/src/crypto/CMakeLists.txt b/src/crypto/CMakeLists.txt index 84f002929..d22d59b36 100644 --- a/src/crypto/CMakeLists.txt +++ b/src/crypto/CMakeLists.txt @@ -39,6 +39,7 @@ set(crypto_sources hash-extra-jh.c hash-extra-skein.c hash.c + hmac-keccak.c jh.c keccak.c oaes_lib.c @@ -64,6 +65,7 @@ set(crypto_private_headers groestl_tables.h hash-ops.h hash.h + hmac-keccak.h initializer.h jh.h keccak.h diff --git a/src/crypto/CryptonightR_JIT.c b/src/crypto/CryptonightR_JIT.c index 68258a959..ee8f3f36f 100644 --- a/src/crypto/CryptonightR_JIT.c +++ b/src/crypto/CryptonightR_JIT.c @@ -4,6 +4,9 @@ #include <string.h> #include <stdio.h> #include <unistd.h> +#if !(defined(_MSC_VER) || defined(__MINGW32__)) +#include <sys/mman.h> +#endif #include "int-util.h" #include "hash-ops.h" @@ -58,6 +61,11 @@ int v4_generate_JIT_code(const struct V4_Instruction* code, v4_random_math_JIT_f uint8_t* JIT_code = (uint8_t*) buf; const uint8_t* JIT_code_end = JIT_code + buf_size; +#if !(defined(_MSC_VER) || defined(__MINGW32__)) + if (mprotect((void*)buf, buf_size, PROT_READ | PROT_WRITE)) + return 1; +#endif + APPEND_CODE(prologue, sizeof(prologue)); uint32_t prev_rot_src = 0xFFFFFFFFU; @@ -101,6 +109,11 @@ int v4_generate_JIT_code(const struct V4_Instruction* code, v4_random_math_JIT_f APPEND_CODE(epilogue, sizeof(epilogue)); +#if !(defined(_MSC_VER) || defined(__MINGW32__)) + if (mprotect((void*)buf, buf_size, PROT_READ | PROT_EXEC)) + return 1; +#endif + __builtin___clear_cache((char*)buf, (char*)JIT_code); return 0; diff --git a/src/crypto/aesb.c b/src/crypto/aesb.c index efdeef8d1..6d4905ad5 100644 --- a/src/crypto/aesb.c +++ b/src/crypto/aesb.c @@ -33,11 +33,11 @@ extern "C" #define RC_LENGTH (5 * (AES_BLOCK_SIZE / 4 - 2)) #if defined(_MSC_VER) -#define ALIGN __declspec(align(TABLE_ALIGN)) +#define LOCAL_ALIGN __declspec(align(TABLE_ALIGN)) #elif defined(__GNUC__) -#define ALIGN __attribute__ ((aligned(16))) +#define LOCAL_ALIGN __attribute__ ((aligned(16))) #else -#define ALIGN +#define LOCAL_ALIGN #endif #define rf1(r,c) (r) @@ -131,7 +131,7 @@ extern "C" #define t_set(m,n) t_##m##n #define t_use(m,n) t_##m##n -#define d_4(t,n,b,e,f,g,h) ALIGN const t n[4][256] = { b(e), b(f), b(g), b(h) } +#define d_4(t,n,b,e,f,g,h) LOCAL_ALIGN const t n[4][256] = { b(e), b(f), b(g), b(h) } #define four_tables(x,tab,vf,rf,c) \ (tab[0][bval(vf(x,0,c),rf(0,c))] \ diff --git a/src/crypto/hmac-keccak.c b/src/crypto/hmac-keccak.c new file mode 100644 index 000000000..edcb2065e --- /dev/null +++ b/src/crypto/hmac-keccak.c @@ -0,0 +1,81 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "hmac-keccak.h" +#include "memwipe.h" + +#define KECCAK_BLOCKLEN 136 +#define HASH_SIZE 32 + +void hmac_keccak_init(hmac_keccak_state *S, const uint8_t *_key, size_t keylen) { + const uint8_t *key = _key; + uint8_t keyhash[HASH_SIZE]; + uint8_t pad[KECCAK_BLOCKLEN]; + uint64_t i; + + if (keylen > KECCAK_BLOCKLEN) { + keccak(key, keylen, keyhash, HASH_SIZE); + key = keyhash; + keylen = HASH_SIZE; + } + + keccak_init(&S->inner); + memset(pad, 0x36, KECCAK_BLOCKLEN); + for (i = 0; i < keylen; ++i) { + pad[i] ^= key[i]; + } + keccak_update(&S->inner, pad, KECCAK_BLOCKLEN); + + keccak_init(&S->outer); + memset(pad, 0x5c, KECCAK_BLOCKLEN); + for (i = 0; i < keylen; ++i) { + pad[i] ^= key[i]; + } + keccak_update(&S->outer, pad, KECCAK_BLOCKLEN); + + memwipe(keyhash, HASH_SIZE); +} + +void hmac_keccak_update(hmac_keccak_state *S, const uint8_t *data, size_t datalen) { + keccak_update(&S->inner, data, datalen); +} + +void hmac_keccak_finish(hmac_keccak_state *S, uint8_t *digest) { + uint8_t ihash[HASH_SIZE]; + keccak_finish(&S->inner, ihash); + keccak_update(&S->outer, ihash, HASH_SIZE); + keccak_finish(&S->outer, digest); + memwipe(ihash, HASH_SIZE); +} + +void hmac_keccak_hash(uint8_t *out, const uint8_t *key, size_t keylen, const uint8_t *in, size_t inlen) { + hmac_keccak_state S; + hmac_keccak_init(&S, key, keylen); + hmac_keccak_update(&S, in, inlen); + hmac_keccak_finish(&S, out); +} diff --git a/src/crypto/hmac-keccak.h b/src/crypto/hmac-keccak.h new file mode 100644 index 000000000..c450860d4 --- /dev/null +++ b/src/crypto/hmac-keccak.h @@ -0,0 +1,59 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef HMAC_KECCAK_H +#define HMAC_KECCAK_H + +#include "keccak.h" + +// HMAC RFC 2104 with Keccak-256 base hash function +// +// B = KECCAK_BLOCKLEN = 136 B +// L = HASH_SIZE = 32 B +// +// Note this is not HMAC-SHA3 as SHA3 and Keccak differs in +// the padding constant. + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + KECCAK_CTX inner; + KECCAK_CTX outer; +} hmac_keccak_state; + +void hmac_keccak_init(hmac_keccak_state *S, const uint8_t *_key, size_t keylen); +void hmac_keccak_update(hmac_keccak_state *S, const uint8_t *data, size_t datalen); +void hmac_keccak_finish(hmac_keccak_state *S, uint8_t *digest); +void hmac_keccak_hash(uint8_t *out, const uint8_t *key, size_t keylen, const uint8_t *in, size_t inlen); + +#ifdef __cplusplus +} +#endif +#endif //HMAC_KECCAK_H diff --git a/src/crypto/slow-hash.c b/src/crypto/slow-hash.c index 9dc6f5038..7f36c9dc3 100644 --- a/src/crypto/slow-hash.c +++ b/src/crypto/slow-hash.c @@ -76,7 +76,7 @@ static inline int use_v4_jit(void) const char *env = getenv("MONERO_USE_CNV4_JIT"); if (!env) { - use_v4_jit_flag = 0; + use_v4_jit_flag = 1; } else if (!strcmp(env, "0") || !strcmp(env, "no")) { use_v4_jit_flag = 0; @@ -274,10 +274,10 @@ static inline int use_v4_jit(void) #define VARIANT2_2() \ do if (variant == 2 || variant == 3) \ { \ - *U64(hp_state + (j ^ 0x10)) ^= SWAP64LE(hi); \ - *(U64(hp_state + (j ^ 0x10)) + 1) ^= SWAP64LE(lo); \ - hi ^= SWAP64LE(*U64(hp_state + (j ^ 0x20))); \ - lo ^= SWAP64LE(*(U64(hp_state + (j ^ 0x20)) + 1)); \ + *U64(local_hp_state + (j ^ 0x10)) ^= SWAP64LE(hi); \ + *(U64(local_hp_state + (j ^ 0x10)) + 1) ^= SWAP64LE(lo); \ + hi ^= SWAP64LE(*U64(local_hp_state + (j ^ 0x20))); \ + lo ^= SWAP64LE(*(U64(local_hp_state + (j ^ 0x20)) + 1)); \ } while (0) #define V4_REG_LOAD(dst, src) \ @@ -405,7 +405,7 @@ static inline int use_v4_jit(void) #define pre_aes() \ j = state_index(a); \ - _c = _mm_load_si128(R128(&hp_state[j])); \ + _c = _mm_load_si128(R128(&local_hp_state[j])); \ _a = _mm_load_si128(R128(a)); \ /* @@ -418,20 +418,20 @@ static inline int use_v4_jit(void) * This code is based upon an optimized implementation by dga. */ #define post_aes() \ - VARIANT2_SHUFFLE_ADD_SSE2(hp_state, j); \ + VARIANT2_SHUFFLE_ADD_SSE2(local_hp_state, j); \ _mm_store_si128(R128(c), _c); \ - _mm_store_si128(R128(&hp_state[j]), _mm_xor_si128(_b, _c)); \ - VARIANT1_1(&hp_state[j]); \ + _mm_store_si128(R128(&local_hp_state[j]), _mm_xor_si128(_b, _c)); \ + VARIANT1_1(&local_hp_state[j]); \ j = state_index(c); \ - p = U64(&hp_state[j]); \ + p = U64(&local_hp_state[j]); \ b[0] = p[0]; b[1] = p[1]; \ VARIANT2_INTEGER_MATH_SSE2(b, c); \ VARIANT4_RANDOM_MATH(a, b, r, &_b, &_b1); \ __mul(); \ VARIANT2_2(); \ - VARIANT2_SHUFFLE_ADD_SSE2(hp_state, j); \ + VARIANT2_SHUFFLE_ADD_SSE2(local_hp_state, j); \ a[0] += hi; a[1] += lo; \ - p = U64(&hp_state[j]); \ + p = U64(&local_hp_state[j]); \ p[0] = a[0]; p[1] = a[1]; \ a[0] ^= b[0]; a[1] ^= b[1]; \ VARIANT1_2(p + 1); \ @@ -756,10 +756,10 @@ void slow_hash_allocate_state(void) #if defined(__APPLE__) || defined(__FreeBSD__) || defined(__OpenBSD__) || \ defined(__DragonFly__) || defined(__NetBSD__) hp_state = mmap(0, MEMORY, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANON, 0, 0); + MAP_PRIVATE | MAP_ANON, -1, 0); #else hp_state = mmap(0, MEMORY, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, 0, 0); + MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); #endif if(hp_state == MAP_FAILED) hp_state = NULL; @@ -778,11 +778,16 @@ void slow_hash_allocate_state(void) #else #if defined(__APPLE__) || defined(__FreeBSD__) || defined(__OpenBSD__) || \ defined(__DragonFly__) || defined(__NetBSD__) - hp_jitfunc_memory = mmap(0, 4096 + 4095, PROT_READ | PROT_WRITE | PROT_EXEC, - MAP_PRIVATE | MAP_ANON, 0, 0); +#ifdef __NetBSD__ +#define RESERVED_FLAGS PROT_MPROTECT(PROT_EXEC) +#else +#define RESERVED_FLAGS 0 +#endif + hp_jitfunc_memory = mmap(0, 4096 + 4096, PROT_READ | PROT_WRITE | RESERVED_FLAGS, + MAP_PRIVATE | MAP_ANON, -1, 0); #else - hp_jitfunc_memory = mmap(0, 4096 + 4095, PROT_READ | PROT_WRITE | PROT_EXEC, - MAP_PRIVATE | MAP_ANONYMOUS, 0, 0); + hp_jitfunc_memory = mmap(0, 4096 + 4096, PROT_READ | PROT_WRITE | PROT_EXEC, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); #endif if(hp_jitfunc_memory == MAP_FAILED) hp_jitfunc_memory = NULL; @@ -794,9 +799,6 @@ void slow_hash_allocate_state(void) hp_jitfunc_memory = malloc(4096 + 4095); } hp_jitfunc = (v4_random_math_JIT_func)((size_t)(hp_jitfunc_memory + 4095) & ~4095); -#if !(defined(_MSC_VER) || defined(__MINGW32__)) - mprotect(hp_jitfunc, 4096, PROT_READ | PROT_WRITE | PROT_EXEC); -#endif } /** @@ -893,6 +895,10 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int if(hp_state == NULL) slow_hash_allocate_state(); + // locals to avoid constant TLS dereferencing + uint8_t *local_hp_state = hp_state; + v4_random_math_JIT_func local_hp_jitfunc = hp_jitfunc; + /* CryptoNight Step 1: Use Keccak1600 to initialize the 'state' (and 'text') buffers from the data. */ if (prehashed) { memcpy(&state.hs, data, length); @@ -915,7 +921,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int for(i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) { aes_pseudo_round(text, text, expandedKey, INIT_SIZE_BLK); - memcpy(&hp_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE); + memcpy(&local_hp_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE); } } else @@ -927,7 +933,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int for(j = 0; j < INIT_SIZE_BLK; j++) aesb_pseudo_round(&text[AES_BLOCK_SIZE * j], &text[AES_BLOCK_SIZE * j], aes_ctx->key->exp_data); - memcpy(&hp_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE); + memcpy(&local_hp_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE); } } @@ -975,7 +981,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int for(i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) { // add the xor to the pseudo round - aes_pseudo_round_xor(text, text, expandedKey, &hp_state[i * INIT_SIZE_BYTE], INIT_SIZE_BLK); + aes_pseudo_round_xor(text, text, expandedKey, &local_hp_state[i * INIT_SIZE_BYTE], INIT_SIZE_BLK); } } else @@ -985,7 +991,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int { for(j = 0; j < INIT_SIZE_BLK; j++) { - xor_blocks(&text[j * AES_BLOCK_SIZE], &hp_state[i * INIT_SIZE_BYTE + j * AES_BLOCK_SIZE]); + xor_blocks(&text[j * AES_BLOCK_SIZE], &local_hp_state[i * INIT_SIZE_BYTE + j * AES_BLOCK_SIZE]); aesb_pseudo_round(&text[AES_BLOCK_SIZE * j], &text[AES_BLOCK_SIZE * j], aes_ctx->key->exp_data); } } @@ -1065,24 +1071,24 @@ union cn_slow_hash_state #define pre_aes() \ j = state_index(a); \ - _c = vld1q_u8(&hp_state[j]); \ + _c = vld1q_u8(&local_hp_state[j]); \ _a = vld1q_u8((const uint8_t *)a); \ #define post_aes() \ - VARIANT2_SHUFFLE_ADD_NEON(hp_state, j); \ + VARIANT2_SHUFFLE_ADD_NEON(local_hp_state, j); \ vst1q_u8((uint8_t *)c, _c); \ - vst1q_u8(&hp_state[j], veorq_u8(_b, _c)); \ - VARIANT1_1(&hp_state[j]); \ + vst1q_u8(&local_hp_state[j], veorq_u8(_b, _c)); \ + VARIANT1_1(&local_hp_state[j]); \ j = state_index(c); \ - p = U64(&hp_state[j]); \ + p = U64(&local_hp_state[j]); \ b[0] = p[0]; b[1] = p[1]; \ VARIANT2_PORTABLE_INTEGER_MATH(b, c); \ VARIANT4_RANDOM_MATH(a, b, r, &_b, &_b1); \ __mul(); \ VARIANT2_2(); \ - VARIANT2_SHUFFLE_ADD_NEON(hp_state, j); \ + VARIANT2_SHUFFLE_ADD_NEON(local_hp_state, j); \ a[0] += hi; a[1] += lo; \ - p = U64(&hp_state[j]); \ + p = U64(&local_hp_state[j]); \ p[0] = a[0]; p[1] = a[1]; \ a[0] ^= b[0]; a[1] ^= b[1]; \ VARIANT1_2(p + 1); \ @@ -1245,9 +1251,9 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int RDATA_ALIGN16 uint8_t expandedKey[240]; #ifndef FORCE_USE_HEAP - RDATA_ALIGN16 uint8_t hp_state[MEMORY]; + RDATA_ALIGN16 uint8_t local_hp_state[MEMORY]; #else - uint8_t *hp_state = (uint8_t *)aligned_malloc(MEMORY,16); + uint8_t *local_hp_state = (uint8_t *)aligned_malloc(MEMORY,16); #endif uint8_t text[INIT_SIZE_BYTE]; @@ -1287,7 +1293,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int for(i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) { aes_pseudo_round(text, text, expandedKey, INIT_SIZE_BLK); - memcpy(&hp_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE); + memcpy(&local_hp_state[i * INIT_SIZE_BYTE], text, INIT_SIZE_BYTE); } U64(a)[0] = U64(&state.k[0])[0] ^ U64(&state.k[32])[0]; @@ -1322,7 +1328,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int for(i = 0; i < MEMORY / INIT_SIZE_BYTE; i++) { // add the xor to the pseudo round - aes_pseudo_round_xor(text, text, expandedKey, &hp_state[i * INIT_SIZE_BYTE], INIT_SIZE_BLK); + aes_pseudo_round_xor(text, text, expandedKey, &local_hp_state[i * INIT_SIZE_BYTE], INIT_SIZE_BLK); } /* CryptoNight Step 5: Apply Keccak to the state again, and then @@ -1337,7 +1343,7 @@ void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int extra_hashes[state.hs.b[0] & 3](&state, 200, hash); #ifdef FORCE_USE_HEAP - aligned_free(hp_state); + aligned_free(local_hp_state); #endif } #else /* aarch64 && crypto */ diff --git a/src/cryptonote_basic/cryptonote_boost_serialization.h b/src/cryptonote_basic/cryptonote_boost_serialization.h index 4ce9f3eeb..3dd98f0c6 100644 --- a/src/cryptonote_basic/cryptonote_boost_serialization.h +++ b/src/cryptonote_basic/cryptonote_boost_serialization.h @@ -40,6 +40,7 @@ #include <boost/archive/portable_binary_iarchive.hpp> #include <boost/archive/portable_binary_oarchive.hpp> #include "cryptonote_basic.h" +#include "difficulty.h" #include "common/unordered_containers_boost_serialization.h" #include "crypto/crypto.h" #include "ringct/rctTypes.h" @@ -339,6 +340,41 @@ namespace boost if (x.type == rct::RCTTypeBulletproof || x.type == rct::RCTTypeBulletproof2) a & x.p.pseudoOuts; } + + template <class Archive> + inline void serialize(Archive &a, rct::RCTConfig &x, const boost::serialization::version_type ver) + { + a & x.range_proof_type; + a & x.bp_version; + } + + template <class Archive> + inline void serialize(Archive &a, cryptonote::difficulty_type &x, const boost::serialization::version_type ver) + { + if (Archive::is_loading::value) + { + // load high part + uint64_t v = 0; + a & v; + x = v; + // load low part + x = x << 64; + a & v; + x += v; + } + else + { + // store high part + cryptonote::difficulty_type x_ = x >> 64; + uint64_t v = x_.convert_to<uint64_t>(); + a & v; + // store low part + x_ = x << 64 >> 64; + v = x_.convert_to<uint64_t>(); + a & v; + } + } + } } diff --git a/src/cryptonote_basic/cryptonote_format_utils.cpp b/src/cryptonote_basic/cryptonote_format_utils.cpp index f40464bd1..094057b1f 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.cpp +++ b/src/cryptonote_basic/cryptonote_format_utils.cpp @@ -282,6 +282,11 @@ namespace cryptonote //--------------------------------------------------------------- bool generate_key_image_helper_precomp(const account_keys& ack, const crypto::public_key& out_key, const crypto::key_derivation& recv_derivation, size_t real_output_index, const subaddress_index& received_index, keypair& in_ephemeral, crypto::key_image& ki, hw::device &hwdev) { + if (hwdev.compute_key_image(ack, out_key, recv_derivation, real_output_index, received_index, in_ephemeral, ki)) + { + return true; + } + if (ack.m_spend_secret_key == crypto::null_skey) { // for watch-only wallet, simply copy the known output pubkey @@ -982,15 +987,11 @@ namespace cryptonote { if (t.version == 1) return false; - static const crypto::hash empty_hash = { (char)0x70, (char)0xa4, (char)0x85, (char)0x5d, (char)0x04, (char)0xd8, (char)0xfa, (char)0x7b, (char)0x3b, (char)0x27, (char)0x82, (char)0xca, (char)0x53, (char)0xb6, (char)0x00, (char)0xe5, (char)0xc0, (char)0x03, (char)0xc7, (char)0xdc, (char)0xb2, (char)0x7d, (char)0x7e, (char)0x92, (char)0x3c, (char)0x23, (char)0xf7, (char)0x86, (char)0x01, (char)0x46, (char)0xd2, (char)0xc5 }; const unsigned int unprunable_size = t.unprunable_size; if (blob && unprunable_size) { CHECK_AND_ASSERT_MES(unprunable_size <= blob->size(), false, "Inconsistent transaction unprunable and blob sizes"); - if (blob->size() - unprunable_size == 0) - res = empty_hash; - else - cryptonote::get_blob_hash(epee::span<const char>(blob->data() + unprunable_size, blob->size() - unprunable_size), res); + cryptonote::get_blob_hash(epee::span<const char>(blob->data() + unprunable_size, blob->size() - unprunable_size), res); } else { @@ -1002,10 +1003,7 @@ namespace cryptonote const size_t mixin = t.vin.empty() ? 0 : t.vin[0].type() == typeid(txin_to_key) ? boost::get<txin_to_key>(t.vin[0]).key_offsets.size() - 1 : 0; bool r = tt.rct_signatures.p.serialize_rctsig_prunable(ba, t.rct_signatures.type, inputs, outputs, mixin); CHECK_AND_ASSERT_MES(r, false, "Failed to serialize rct signatures prunable"); - if (ss.str().empty()) - res = empty_hash; - else - cryptonote::get_blob_hash(ss.str(), res); + cryptonote::get_blob_hash(ss.str(), res); } return true; } @@ -1042,7 +1040,10 @@ namespace cryptonote } // prunable rct - hashes[2] = pruned_data_hash; + if (t.rct_signatures.type == rct::RCTTypeNull) + hashes[2] = crypto::null_hash; + else + hashes[2] = pruned_data_hash; // the tx hash is the hash of the 3 hashes crypto::hash res = cn_fast_hash(hashes, sizeof(hashes)); diff --git a/src/cryptonote_basic/difficulty.cpp b/src/cryptonote_basic/difficulty.cpp index 89b748a83..5162e53e6 100644 --- a/src/cryptonote_basic/difficulty.cpp +++ b/src/cryptonote_basic/difficulty.cpp @@ -102,7 +102,7 @@ namespace cryptonote { return a + b < a || (c && a + b == (uint64_t) -1); } - bool check_hash(const crypto::hash &hash, difficulty_type difficulty) { + bool check_hash_64(const crypto::hash &hash, uint64_t difficulty) { uint64_t low, high, top, cur; // First check the highest word, this will most likely fail for a random hash. mul(swap64le(((const uint64_t *) &hash)[3]), difficulty, top, high); @@ -119,7 +119,7 @@ namespace cryptonote { return !carry; } - difficulty_type next_difficulty(std::vector<std::uint64_t> timestamps, std::vector<difficulty_type> cumulative_difficulties, size_t target_seconds) { + uint64_t next_difficulty_64(std::vector<std::uint64_t> timestamps, std::vector<uint64_t> cumulative_difficulties, size_t target_seconds) { if(timestamps.size() > DIFFICULTY_WINDOW) { @@ -150,7 +150,7 @@ namespace cryptonote { if (time_span == 0) { time_span = 1; } - difficulty_type total_work = cumulative_difficulties[cut_end - 1] - cumulative_difficulties[cut_begin]; + uint64_t total_work = cumulative_difficulties[cut_end - 1] - cumulative_difficulties[cut_begin]; assert(total_work > 0); uint64_t low, high; mul(total_work, target_seconds, low, high); @@ -162,4 +162,81 @@ namespace cryptonote { return (low + time_span - 1) / time_span; } +#if defined(_MSC_VER) +#ifdef max +#undef max +#endif +#endif + + const difficulty_type max64bit(std::numeric_limits<std::uint64_t>::max()); + const boost::multiprecision::uint256_t max128bit(std::numeric_limits<boost::multiprecision::uint128_t>::max()); + const boost::multiprecision::uint512_t max256bit(std::numeric_limits<boost::multiprecision::uint256_t>::max()); + +#define FORCE_FULL_128_BITS + + bool check_hash_128(const crypto::hash &hash, difficulty_type difficulty) { +#ifndef FORCE_FULL_128_BITS + // fast check + if (difficulty >= max64bit && ((const uint64_t *) &hash)[3] > 0) + return false; +#endif + // usual slow check + boost::multiprecision::uint512_t hashVal = 0; +#ifdef FORCE_FULL_128_BITS + for(int i = 0; i < 4; i++) { // highest word is zero +#else + for(int i = 1; i < 4; i++) { // highest word is zero +#endif + hashVal <<= 64; + hashVal |= swap64le(((const uint64_t *) &hash)[3 - i]); + } + return hashVal * difficulty <= max256bit; + } + + bool check_hash(const crypto::hash &hash, difficulty_type difficulty) { + if (difficulty <= max64bit) // if can convert to small difficulty - do it + return check_hash_64(hash, difficulty.convert_to<std::uint64_t>()); + else + return check_hash_128(hash, difficulty); + } + + difficulty_type next_difficulty(std::vector<uint64_t> timestamps, std::vector<difficulty_type> cumulative_difficulties, size_t target_seconds) { + //cutoff DIFFICULTY_LAG + if(timestamps.size() > DIFFICULTY_WINDOW) + { + timestamps.resize(DIFFICULTY_WINDOW); + cumulative_difficulties.resize(DIFFICULTY_WINDOW); + } + + + size_t length = timestamps.size(); + assert(length == cumulative_difficulties.size()); + if (length <= 1) { + return 1; + } + static_assert(DIFFICULTY_WINDOW >= 2, "Window is too small"); + assert(length <= DIFFICULTY_WINDOW); + sort(timestamps.begin(), timestamps.end()); + size_t cut_begin, cut_end; + static_assert(2 * DIFFICULTY_CUT <= DIFFICULTY_WINDOW - 2, "Cut length is too large"); + if (length <= DIFFICULTY_WINDOW - 2 * DIFFICULTY_CUT) { + cut_begin = 0; + cut_end = length; + } else { + cut_begin = (length - (DIFFICULTY_WINDOW - 2 * DIFFICULTY_CUT) + 1) / 2; + cut_end = cut_begin + (DIFFICULTY_WINDOW - 2 * DIFFICULTY_CUT); + } + assert(/*cut_begin >= 0 &&*/ cut_begin + 2 <= cut_end && cut_end <= length); + uint64_t time_span = timestamps[cut_end - 1] - timestamps[cut_begin]; + if (time_span == 0) { + time_span = 1; + } + difficulty_type total_work = cumulative_difficulties[cut_end - 1] - cumulative_difficulties[cut_begin]; + assert(total_work > 0); + boost::multiprecision::uint256_t res = (boost::multiprecision::uint256_t(total_work) * target_seconds + time_span - 1) / time_span; + if(res > max128bit) + return 0; // to behave like previous implementation, may be better return max128bit? + return res.convert_to<difficulty_type>(); + } + } diff --git a/src/cryptonote_basic/difficulty.h b/src/cryptonote_basic/difficulty.h index 8da355b22..f7a9376fb 100644 --- a/src/cryptonote_basic/difficulty.h +++ b/src/cryptonote_basic/difficulty.h @@ -32,12 +32,13 @@ #include <cstdint> #include <vector> +#include <boost/multiprecision/cpp_int.hpp> #include "crypto/hash.h" namespace cryptonote { - typedef std::uint64_t difficulty_type; + typedef boost::multiprecision::uint128_t difficulty_type; /** * @brief checks if a hash fits the given difficulty @@ -51,6 +52,10 @@ namespace cryptonote * * @return true if valid, else false */ + bool check_hash_64(const crypto::hash &hash, uint64_t difficulty); + uint64_t next_difficulty_64(std::vector<std::uint64_t> timestamps, std::vector<uint64_t> cumulative_difficulties, size_t target_seconds); + + bool check_hash_128(const crypto::hash &hash, difficulty_type difficulty); bool check_hash(const crypto::hash &hash, difficulty_type difficulty); difficulty_type next_difficulty(std::vector<std::uint64_t> timestamps, std::vector<difficulty_type> cumulative_difficulties, size_t target_seconds); } diff --git a/src/cryptonote_basic/hardfork.cpp b/src/cryptonote_basic/hardfork.cpp index ebfe45f37..89bca2f09 100644 --- a/src/cryptonote_basic/hardfork.cpp +++ b/src/cryptonote_basic/hardfork.cpp @@ -317,9 +317,12 @@ void HardFork::on_block_popped(uint64_t nblocks) uint64_t height; for (height = old_chain_height - 1; height >= new_chain_height; --height) { + version = versions.back(); + last_versions[version]--; versions.pop_back(); version = db.get_hard_fork_version(height); versions.push_front(version); + last_versions[version]++; } // does not take voting into account diff --git a/src/cryptonote_basic/miner.cpp b/src/cryptonote_basic/miner.cpp index 6628c8448..4e2edc20f 100644 --- a/src/cryptonote_basic/miner.cpp +++ b/src/cryptonote_basic/miner.cpp @@ -435,14 +435,15 @@ namespace cryptonote { MTRACE("Miner has received stop signal"); - if (!is_mining()) + CRITICAL_REGION_LOCAL(m_threads_lock); + bool mining = !m_threads.empty(); + if (!mining) { MTRACE("Not mining - nothing to stop" ); return true; } send_stop_signal(); - CRITICAL_REGION_LOCAL(m_threads_lock); // In case background mining was active and the miner threads are waiting // on the background miner to signal start. diff --git a/src/cryptonote_config.h b/src/cryptonote_config.h index 9beafe7c5..b6087de22 100644 --- a/src/cryptonote_config.h +++ b/src/cryptonote_config.h @@ -116,6 +116,7 @@ #define P2P_DEFAULT_HANDSHAKE_INVOKE_TIMEOUT 5000 //5 seconds #define P2P_DEFAULT_WHITELIST_CONNECTIONS_PERCENT 70 #define P2P_DEFAULT_ANCHOR_CONNECTIONS_COUNT 2 +#define P2P_DEFAULT_SYNC_SEARCH_CONNECTIONS_COUNT 2 #define P2P_DEFAULT_LIMIT_RATE_UP 2048 // kB/s #define P2P_DEFAULT_LIMIT_RATE_DOWN 8192 // kB/s diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 3e903cd12..263227148 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -645,6 +645,8 @@ block Blockchain::pop_block_from_blockchain() block popped_block; std::vector<transaction> popped_txs; + CHECK_AND_ASSERT_THROW_MES(m_db->height() > 1, "Cannot pop the genesis block"); + try { m_db->pop_block(popped_block, popped_txs); @@ -1269,15 +1271,17 @@ void Blockchain::get_last_n_blocks_weights(std::vector<uint64_t>& weights, size_ if(h == 0) return; - m_db->block_txn_start(true); // add weight of last <count> blocks to vector <weights> (or less, if blockchain size < count) size_t start_offset = h - std::min<size_t>(h, count); - weights.reserve(weights.size() + h - start_offset); - for(size_t i = start_offset; i < h; i++) - { - weights.push_back(m_db->get_block_weight(i)); - } - m_db->block_txn_stop(); + weights = m_db->get_block_weights(start_offset, count); +} +//------------------------------------------------------------------ +void Blockchain::get_long_term_block_weights(std::vector<uint64_t>& weights, uint64_t start_height, size_t count) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + weights = m_db->get_long_term_block_weights(start_height, count); } //------------------------------------------------------------------ uint64_t Blockchain::get_current_cumulative_block_weight_limit() const @@ -1995,7 +1999,7 @@ bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qbloc return true; } //------------------------------------------------------------------ -uint64_t Blockchain::block_difficulty(uint64_t i) const +difficulty_type Blockchain::block_difficulty(uint64_t i) const { LOG_PRINT_L3("Blockchain::" << __func__); // WARNING: this function does not take m_blockchain_lock, and thus should only call read only @@ -2194,7 +2198,11 @@ bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qbloc bool result = find_blockchain_supplement(qblock_ids, resp.m_block_ids, resp.start_height, resp.total_height); if (result) - resp.cumulative_difficulty = m_db->get_block_cumulative_difficulty(resp.total_height - 1); + { + cryptonote::difficulty_type wide_cumulative_difficulty = m_db->get_block_cumulative_difficulty(resp.total_height - 1); + resp.cumulative_difficulty = (wide_cumulative_difficulty << 64 >> 64).convert_to<uint64_t>(); + resp.cumulative_difficulty_top64 = (wide_cumulative_difficulty >> 64).convert_to<uint64_t>(); + } return result; } @@ -3804,9 +3812,7 @@ uint64_t Blockchain::get_next_long_term_block_weight(uint64_t block_weight) cons return block_weight; std::vector<uint64_t> weights; - weights.resize(nblocks); - for (uint64_t h = 0; h < nblocks; ++h) - weights[h] = m_db->get_block_long_term_weight(db_height - nblocks + h); + get_long_term_block_weights(weights, db_height - nblocks, nblocks); uint64_t long_term_median = epee::misc_utils::median(weights); uint64_t long_term_effective_median_block_weight = std::max<uint64_t>(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, long_term_median); @@ -3850,9 +3856,7 @@ bool Blockchain::update_next_cumulative_weight_limit(uint64_t *long_term_effecti uint64_t nblocks = std::min<uint64_t>(m_long_term_block_weights_window, db_height); if (nblocks == db_height) --nblocks; - weights.resize(nblocks); - for (uint64_t h = 0; h < nblocks; ++h) - weights[h] = m_db->get_block_long_term_weight(db_height - nblocks + h - 1); + get_long_term_block_weights(weights, db_height - nblocks - 1, nblocks); new_weights = weights; long_term_median = epee::misc_utils::median(weights); } diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index 4744defc5..89d8e7572 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -653,7 +653,7 @@ namespace cryptonote * * @return the difficulty */ - uint64_t block_difficulty(uint64_t i) const; + difficulty_type block_difficulty(uint64_t i) const; /** * @brief gets blocks based on a list of block hashes @@ -1288,14 +1288,25 @@ namespace cryptonote /** * @brief gets recent block weights for median calculation * - * get the block weights of the last <count> blocks, and return by reference <sz>. + * get the block weights of the last <count> blocks, and return by reference <weights>. * - * @param sz return-by-reference the list of weights + * @param weights return-by-reference the list of weights * @param count the number of blocks to get weights for */ void get_last_n_blocks_weights(std::vector<uint64_t>& weights, size_t count) const; /** + * @brief gets recent block long term weights for median calculation + * + * get the block long term weights of the last <count> blocks, and return by reference <weights>. + * + * @param weights return-by-reference the list of weights + * @param start_height the block height of the first block to query + * @param count the number of blocks to get weights for + */ + void get_long_term_block_weights(std::vector<uint64_t>& weights, uint64_t start_height, size_t count) const; + + /** * @brief checks if a transaction is unlocked (its outputs spendable) * * This function checks to see if a transaction is unlocked. diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 58acdb6bb..387203cc0 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -921,6 +921,7 @@ namespace cryptonote catch (const std::exception &e) { MERROR_VER("Exception in handle_incoming_tx_pre: " << e.what()); + tvc[i].m_verifivation_failed = true; results[i].res = false; } }); @@ -951,6 +952,7 @@ namespace cryptonote catch (const std::exception &e) { MERROR_VER("Exception in handle_incoming_tx_post: " << e.what()); + tvc[i].m_verifivation_failed = true; results[i].res = false; } }); diff --git a/src/cryptonote_protocol/cryptonote_protocol_defs.h b/src/cryptonote_protocol/cryptonote_protocol_defs.h index d582e3e9c..3083a5b4c 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_defs.h +++ b/src/cryptonote_protocol/cryptonote_protocol_defs.h @@ -34,6 +34,7 @@ #include "serialization/keyvalue_serialization.h" #include "cryptonote_basic/cryptonote_basic.h" #include "cryptonote_basic/blobdatatype.h" + namespace cryptonote { @@ -208,6 +209,7 @@ namespace cryptonote { uint64_t current_height; uint64_t cumulative_difficulty; + uint64_t cumulative_difficulty_top64; crypto::hash top_id; uint8_t top_version; uint32_t pruning_seed; @@ -215,6 +217,7 @@ namespace cryptonote BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(current_height) KV_SERIALIZE(cumulative_difficulty) + KV_SERIALIZE(cumulative_difficulty_top64) KV_SERIALIZE_VAL_POD_AS_BLOB(top_id) KV_SERIALIZE_OPT(top_version, (uint8_t)0) KV_SERIALIZE_OPT(pruning_seed, (uint32_t)0) @@ -245,12 +248,14 @@ namespace cryptonote uint64_t start_height; uint64_t total_height; uint64_t cumulative_difficulty; + uint64_t cumulative_difficulty_top64; std::vector<crypto::hash> m_block_ids; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(start_height) KV_SERIALIZE(total_height) KV_SERIALIZE(cumulative_difficulty) + KV_SERIALIZE(cumulative_difficulty_top64) KV_SERIALIZE_CONTAINER_POD_AS_BLOB(m_block_ids) END_KV_SERIALIZE_MAP() }; diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.h b/src/cryptonote_protocol/cryptonote_protocol_handler.h index fc7db629c..0927b5d7f 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.h @@ -111,6 +111,7 @@ namespace cryptonote void stop(); void on_connection_close(cryptonote_connection_context &context); void set_max_out_peers(unsigned int max) { m_max_out_peers = max; } + void set_no_sync(bool value) { m_no_sync = value; } std::string get_peers_overview() const; std::pair<uint32_t, uint32_t> get_next_needed_pruning_stripe() const; bool needs_new_sync_connections() const; @@ -138,6 +139,7 @@ namespace cryptonote void drop_connection(cryptonote_connection_context &context, bool add_fail, bool flush_all_spans); bool kick_idle_peers(); bool check_standby_peers(); + bool update_sync_search(); int try_add_next_blocks(cryptonote_connection_context &context); void notify_new_stripe(cryptonote_connection_context &context, uint32_t stripe); void skip_unneeded_hashes(cryptonote_connection_context& context, bool check_block_queue) const; @@ -149,10 +151,12 @@ namespace cryptonote std::atomic<uint32_t> m_syncronized_connections_count; std::atomic<bool> m_synchronized; std::atomic<bool> m_stopping; + std::atomic<bool> m_no_sync; boost::mutex m_sync_lock; block_queue m_block_queue; epee::math_helper::once_a_time_seconds<30> m_idle_peer_kicker; epee::math_helper::once_a_time_milliseconds<100> m_standby_checker; + epee::math_helper::once_a_time_seconds<101> m_sync_search_checker; std::atomic<unsigned int> m_max_out_peers; tools::PerformanceTimer m_sync_timer, m_add_timer; uint64_t m_last_add_end_time; diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index 2317aee84..32f0afceb 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -72,7 +72,8 @@ namespace cryptonote m_p2p(p_net_layout), m_syncronized_connections_count(0), m_synchronized(offline), - m_stopping(false) + m_stopping(false), + m_no_sync(false) { if(!m_p2p) @@ -376,6 +377,13 @@ namespace cryptonote m_core.set_target_blockchain_height((hshd.current_height)); } MINFO(context << "Remote blockchain height: " << hshd.current_height << ", id: " << hshd.top_id); + + if (m_no_sync) + { + context.m_state = cryptonote_connection_context::state_normal; + return true; + } + context.m_state = cryptonote_connection_context::state_synchronizing; //let the socket to send response to handshake, but request callback, to let send request data after response LOG_PRINT_CCONTEXT_L2("requesting callback"); @@ -390,7 +398,9 @@ namespace cryptonote { m_core.get_blockchain_top(hshd.current_height, hshd.top_id); hshd.top_version = m_core.get_ideal_hard_fork_version(hshd.current_height); - hshd.cumulative_difficulty = m_core.get_block_cumulative_difficulty(hshd.current_height); + difficulty_type wide_cumulative_difficulty = m_core.get_block_cumulative_difficulty(hshd.current_height); + hshd.cumulative_difficulty = (wide_cumulative_difficulty << 64 >> 64).convert_to<uint64_t>(); + hshd.cumulative_difficulty_top64 = (wide_cumulative_difficulty >> 64).convert_to<uint64_t>(); hshd.current_height +=1; hshd.pruning_seed = m_core.get_blockchain_pruning_seed(); return true; @@ -1398,6 +1408,7 @@ skip: { m_idle_peer_kicker.do_call(boost::bind(&t_cryptonote_protocol_handler<t_core>::kick_idle_peers, this)); m_standby_checker.do_call(boost::bind(&t_cryptonote_protocol_handler<t_core>::check_standby_peers, this)); + m_sync_search_checker.do_call(boost::bind(&t_cryptonote_protocol_handler<t_core>::update_sync_search, this)); return m_core.on_idle(); } //------------------------------------------------------------------------------------------------------------------------ @@ -1427,6 +1438,47 @@ skip: } //------------------------------------------------------------------------------------------------------------------------ template<class t_core> + bool t_cryptonote_protocol_handler<t_core>::update_sync_search() + { + const uint64_t target = m_core.get_target_blockchain_height(); + const uint64_t height = m_core.get_current_blockchain_height(); + if (target > height) // if we're not synced yet, don't do it + return true; + + MTRACE("Checking for outgoing syncing peers..."); + unsigned n_syncing = 0, n_synced = 0; + boost::uuids::uuid last_synced_peer_id(boost::uuids::nil_uuid()); + m_p2p->for_each_connection([&](cryptonote_connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags)->bool + { + if (!peer_id || context.m_is_income) // only consider connected outgoing peers + return true; + if (context.m_state == cryptonote_connection_context::state_synchronizing) + ++n_syncing; + if (context.m_state == cryptonote_connection_context::state_normal) + { + ++n_synced; + if (!context.m_anchor) + last_synced_peer_id = context.m_connection_id; + } + return true; + }); + MTRACE(n_syncing << " syncing, " << n_synced << " synced"); + + // if we're at max out peers, and not enough are syncing + if (n_synced + n_syncing >= m_max_out_peers && n_syncing < P2P_DEFAULT_SYNC_SEARCH_CONNECTIONS_COUNT && last_synced_peer_id != boost::uuids::nil_uuid()) + { + if (!m_p2p->for_connection(last_synced_peer_id, [&](cryptonote_connection_context& ctx, nodetool::peerid_type peer_id, uint32_t f)->bool{ + MINFO(ctx << "dropping synced peer, " << n_syncing << " syncing, " << n_synced << " synced"); + drop_connection(ctx, false, false); + return true; + })) + MDEBUG("Failed to find peer we wanted to drop"); + } + + return true; + } + //------------------------------------------------------------------------------------------------------------------------ + template<class t_core> bool t_cryptonote_protocol_handler<t_core>::check_standby_peers() { m_p2p->for_each_connection([&](cryptonote_connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags)->bool diff --git a/src/debug_utilities/cn_deserialize.cpp b/src/debug_utilities/cn_deserialize.cpp index 87ded76bd..89c9db02e 100644 --- a/src/debug_utilities/cn_deserialize.cpp +++ b/src/debug_utilities/cn_deserialize.cpp @@ -80,11 +80,9 @@ int main(int argc, char* argv[]) po::options_description desc_cmd_only("Command line options"); po::options_description desc_cmd_sett("Command line options and settings options"); - const command_line::arg_descriptor<std::string> arg_output_file = {"output-file", "Specify output file", "", true}; const command_line::arg_descriptor<uint32_t> arg_log_level = {"log-level", "", log_level}; const command_line::arg_descriptor<std::string> arg_input = {"input", "Specify input has a hexadecimal string", ""}; - command_line::add_arg(desc_cmd_sett, arg_output_file); command_line::add_arg(desc_cmd_sett, arg_log_level); command_line::add_arg(desc_cmd_sett, arg_input); @@ -120,52 +118,10 @@ int main(int argc, char* argv[]) mlog_configure("", true); - std::string m_config_folder; - - std::ostream *output; - std::ofstream *raw_data_file = NULL; - if (command_line::has_arg(vm, arg_output_file)) - { - output_file_path = boost::filesystem::path(command_line::get_arg(vm, arg_output_file)); - - const boost::filesystem::path dir_path = output_file_path.parent_path(); - if (!dir_path.empty()) - { - if (boost::filesystem::exists(dir_path)) - { - if (!boost::filesystem::is_directory(dir_path)) - { - std::cerr << "output directory path is a file: " << dir_path << std::endl; - return 1; - } - } - else - { - if (!boost::filesystem::create_directory(dir_path)) - { - std::cerr << "Failed to create directory " << dir_path << std::endl; - return 1; - } - } - } - - raw_data_file = new std::ofstream(); - raw_data_file->open(output_file_path.string(), std::ios_base::out | std::ios::trunc); - if (raw_data_file->fail()) - return 1; - output = raw_data_file; - } - else - { - output_file_path = ""; - output = &std::cout; - } - cryptonote::blobdata blob; if (!epee::string_tools::parse_hexstr_to_binbuff(input, blob)) { std::cerr << "Invalid hex input" << std::endl; - std::cerr << "Invalid hex input: " << input << std::endl; return 1; } @@ -212,11 +168,5 @@ int main(int argc, char* argv[]) - if (output->fail()) - return 1; - output->flush(); - if (raw_data_file) - delete raw_data_file; - return 0; } diff --git a/src/device/device.hpp b/src/device/device.hpp index 7f1a60c9f..65b38361b 100644 --- a/src/device/device.hpp +++ b/src/device/device.hpp @@ -70,6 +70,7 @@ namespace cryptonote struct account_keys; struct subaddress_index; struct tx_destination_entry; + struct keypair; } namespace hw { @@ -81,11 +82,18 @@ namespace hw { return false; } + class device_progress { + public: + virtual double progress() const { return 0; } + virtual bool indeterminate() const { return false; } + }; + class i_device_callback { public: - virtual void on_button_request() {} - virtual void on_pin_request(epee::wipeable_string & pin) {} - virtual void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) {} + virtual void on_button_request(uint64_t code=0) {} + virtual boost::optional<epee::wipeable_string> on_pin_request() { return boost::none; } + virtual boost::optional<epee::wipeable_string> on_passphrase_request(bool on_device) { return boost::none; } + virtual void on_progress(const device_progress& event) {} virtual ~i_device_callback() = default; }; @@ -141,6 +149,9 @@ namespace hw { virtual void set_callback(i_device_callback * callback) {}; virtual void set_derivation_path(const std::string &derivation_path) {}; + virtual void set_pin(const epee::wipeable_string & pin) {} + virtual void set_passphrase(const epee::wipeable_string & passphrase) {} + /* ======================================================================= */ /* LOCKER */ /* ======================================================================= */ @@ -229,7 +240,9 @@ namespace hw { virtual bool has_ki_cold_sync(void) const { return false; } virtual bool has_tx_cold_sign(void) const { return false; } - + virtual bool has_ki_live_refresh(void) const { return true; } + virtual bool compute_key_image(const cryptonote::account_keys& ack, const crypto::public_key& out_key, const crypto::key_derivation& recv_derivation, size_t real_output_index, const cryptonote::subaddress_index& received_index, cryptonote::keypair& in_ephemeral, crypto::key_image& ki) { return false; } + virtual void computing_key_images(bool started) {}; virtual void set_network_type(cryptonote::network_type network_type) { } protected: diff --git a/src/device/device_cold.hpp b/src/device/device_cold.hpp index 8b8cdf6d2..31b1504ab 100644 --- a/src/device/device_cold.hpp +++ b/src/device/device_cold.hpp @@ -31,6 +31,7 @@ #define MONERO_DEVICE_COLD_H #include "wallet/wallet2.h" +#include <boost/optional/optional.hpp> #include <boost/function.hpp> @@ -44,6 +45,8 @@ namespace hw { public: std::vector<std::string> tx_device_aux; // device generated aux data std::vector<cryptonote::address_parse_info> tx_recipients; // as entered by user + boost::optional<int> bp_version; // BP version to use + boost::optional<unsigned> client_version; // Signing client version to use (testing) }; class device_cold { @@ -51,6 +54,53 @@ namespace hw { using exported_key_image = std::vector<std::pair<crypto::key_image, crypto::signature>>; + class op_progress : public hw::device_progress { + public: + op_progress():m_progress(0), m_indeterminate(false) {}; + explicit op_progress(double progress, bool indeterminate=false): m_progress(progress), m_indeterminate(indeterminate){} + + double progress() const override { return m_progress; } + bool indeterminate() const override { return m_indeterminate; } + protected: + double m_progress; + bool m_indeterminate; + }; + + class tx_progress : public op_progress { + public: + tx_progress(): + m_cur_tx(0), m_max_tx(1), + m_cur_step(0), m_max_step(1), + m_cur_substep(0), m_max_substep(1){}; + + tx_progress(size_t cur_tx, size_t max_tx, size_t cur_step, size_t max_step, size_t cur_substep, size_t max_substep): + m_cur_tx(cur_tx), m_max_tx(max_tx), + m_cur_step(cur_tx), m_max_step(max_tx), + m_cur_substep(cur_tx), m_max_substep(max_tx){} + + double progress() const override { + return std::max(1.0, (double)m_cur_tx / m_max_tx + + (double)m_cur_step / (m_max_tx * m_max_step) + + (double)m_cur_substep / (m_max_tx * m_max_step * m_max_substep)); + } + bool indeterminate() const override { return false; } + + protected: + size_t m_cur_tx; + size_t m_max_tx; + size_t m_cur_step; + size_t m_max_step; + size_t m_cur_substep; + size_t m_max_substep; + }; + + typedef struct { + std::string salt1; + std::string salt2; + std::string tx_enc_keys; + std::string tx_prefix_hash; + } tx_key_data_t; + /** * Key image sync with the cold protocol. */ @@ -65,6 +115,52 @@ namespace hw { const ::tools::wallet2::unsigned_tx_set & unsigned_tx, ::tools::wallet2::signed_tx_set & signed_tx, tx_aux_data & aux_data) =0; + + /** + * Get tx key support check. + */ + virtual bool is_get_tx_key_supported() const { return false; } + + /** + * Loads TX aux data required for tx key. + */ + virtual void load_tx_key_data(tx_key_data_t & res, const std::string & tx_aux_data) =0; + + /** + * Decrypts TX keys. + */ + virtual void get_tx_key( + std::vector<::crypto::secret_key> & tx_keys, + const tx_key_data_t & tx_aux_data, + const ::crypto::secret_key & view_key_priv) =0; + + /** + * Live refresh support check + */ + virtual bool is_live_refresh_supported() const { return false; }; + + /** + * Starts live refresh process with the device + */ + virtual void live_refresh_start() =0; + + /** + * One live refresh step + */ + virtual void live_refresh( + const ::crypto::secret_key & view_key_priv, + const crypto::public_key& out_key, + const crypto::key_derivation& recv_derivation, + size_t real_output_index, + const cryptonote::subaddress_index& received_index, + cryptonote::keypair& in_ephemeral, + crypto::key_image& ki + ) =0; + + /** + * Live refresh process termination + */ + virtual void live_refresh_finish() =0; }; } diff --git a/src/device_trezor/device_trezor.cpp b/src/device_trezor/device_trezor.cpp index ceb6111e0..b4a80cf2c 100644 --- a/src/device_trezor/device_trezor.cpp +++ b/src/device_trezor/device_trezor.cpp @@ -57,7 +57,9 @@ namespace trezor { } device_trezor::device_trezor() { - + m_live_refresh_in_progress = false; + m_live_refresh_enabled = true; + m_live_refresh_thread_running = false; } device_trezor::~device_trezor() { @@ -69,6 +71,89 @@ namespace trezor { } } + bool device_trezor::init() + { + m_live_refresh_in_progress = false; + bool r = device_trezor_base::init(); + if (r && !m_live_refresh_thread) + { + m_live_refresh_thread_running = true; + m_live_refresh_thread.reset(new boost::thread(boost::bind(&device_trezor::live_refresh_thread_main, this))); + } + return r; + } + + bool device_trezor::release() + { + m_live_refresh_in_progress = false; + m_live_refresh_thread_running = false; + if (m_live_refresh_thread) + { + m_live_refresh_thread->join(); + m_live_refresh_thread = nullptr; + } + return device_trezor_base::release(); + } + + bool device_trezor::disconnect() + { + m_live_refresh_in_progress = false; + return device_trezor_base::disconnect(); + } + + void device_trezor::device_state_reset_unsafe() + { + require_connected(); + if (m_live_refresh_in_progress) + { + try + { + live_refresh_finish_unsafe(); + } + catch(const std::exception & e) + { + MERROR("Live refresh could not be terminated: " << e.what()); + } + } + + m_live_refresh_in_progress = false; + device_trezor_base::device_state_reset_unsafe(); + } + + void device_trezor::live_refresh_thread_main() + { + while(m_live_refresh_thread_running) + { + boost::this_thread::sleep_for(boost::chrono::milliseconds(100)); + if (!m_live_refresh_in_progress) + { + continue; + } + + TREZOR_AUTO_LOCK_DEVICE(); + if (!m_transport || !m_live_refresh_in_progress) + { + continue; + } + + auto current_time = std::chrono::steady_clock::now(); + if (current_time - m_last_live_refresh_time <= std::chrono::seconds(20)) + { + continue; + } + + MTRACE("Closing live refresh process due to inactivity"); + try + { + live_refresh_finish(); + } + catch(const std::exception &e) + { + MWARNING("Live refresh auto-finish failed: " << e.what()); + } + } + } + /* ======================================================================= */ /* WALLET & ADDRESS */ /* ======================================================================= */ @@ -126,7 +211,7 @@ namespace trezor { std::shared_ptr<messages::monero::MoneroAddress> device_trezor::get_address( const boost::optional<std::vector<uint32_t>> & path, const boost::optional<cryptonote::network_type> & network_type){ - AUTO_LOCK_CMD(); + TREZOR_AUTO_LOCK_CMD(); require_connected(); device_state_reset_unsafe(); require_initialized(); @@ -142,7 +227,7 @@ namespace trezor { std::shared_ptr<messages::monero::MoneroWatchKey> device_trezor::get_view_key( const boost::optional<std::vector<uint32_t>> & path, const boost::optional<cryptonote::network_type> & network_type){ - AUTO_LOCK_CMD(); + TREZOR_AUTO_LOCK_CMD(); require_connected(); device_state_reset_unsafe(); require_initialized(); @@ -155,11 +240,43 @@ namespace trezor { return response; } + bool device_trezor::is_get_tx_key_supported() const + { + require_initialized(); + return get_version() > pack_version(2, 0, 10); + } + + void device_trezor::load_tx_key_data(::hw::device_cold::tx_key_data_t & res, const std::string & tx_aux_data) + { + protocol::tx::load_tx_key_data(res, tx_aux_data); + } + + void device_trezor::get_tx_key( + std::vector<::crypto::secret_key> & tx_keys, + const ::hw::device_cold::tx_key_data_t & tx_aux_data, + const ::crypto::secret_key & view_key_priv) + { + TREZOR_AUTO_LOCK_CMD(); + require_connected(); + device_state_reset_unsafe(); + require_initialized(); + + auto req = protocol::tx::get_tx_key(tx_aux_data); + this->set_msg_addr<messages::monero::MoneroGetTxKeyRequest>(req.get()); + + auto response = this->client_exchange<messages::monero::MoneroGetTxKeyAck>(req); + MTRACE("Get TX key response received"); + + protocol::tx::get_tx_key_ack(tx_keys, tx_aux_data.tx_prefix_hash, view_key_priv, response); + } + void device_trezor::ki_sync(wallet_shim * wallet, const std::vector<tools::wallet2::transfer_details> & transfers, hw::device_cold::exported_key_image & ski) { - AUTO_LOCK_CMD(); +#define EVENT_PROGRESS(P) do { if (m_callback) {(m_callback)->on_progress(device_cold::op_progress(P)); } }while(0) + + TREZOR_AUTO_LOCK_CMD(); require_connected(); device_state_reset_unsafe(); require_initialized(); @@ -171,6 +288,7 @@ namespace trezor { protocol::ki::key_image_data(wallet, transfers, mtds); protocol::ki::generate_commitment(mtds, transfers, req); + EVENT_PROGRESS(0.); this->set_msg_addr<messages::monero::MoneroKeyImageExportInitRequest>(req.get()); auto ack1 = this->client_exchange<messages::monero::MoneroKeyImageExportInitAck>(req); @@ -194,27 +312,160 @@ namespace trezor { } MTRACE("Batch " << cur << " / " << num_batches << " batches processed"); + EVENT_PROGRESS((double)cur * batch_size / mtds.size()); } + EVENT_PROGRESS(1.); auto final_req = std::make_shared<messages::monero::MoneroKeyImageSyncFinalRequest>(); auto final_ack = this->client_exchange<messages::monero::MoneroKeyImageSyncFinalAck>(final_req); ski.reserve(kis.size()); for(auto & sub : kis){ - char buff[32*3]; + ::crypto::signature sig{}; + ::crypto::key_image ki; + char buff[sizeof(ki.data)*3]; + + size_t buff_len = sizeof(buff); + protocol::crypto::chacha::decrypt(sub.blob().data(), sub.blob().size(), reinterpret_cast<const uint8_t *>(final_ack->enc_key().data()), - reinterpret_cast<const uint8_t *>(sub.iv().data()), buff); + reinterpret_cast<const uint8_t *>(sub.iv().data()), buff, &buff_len); + CHECK_AND_ASSERT_THROW_MES(buff_len == sizeof(buff), "Plaintext size invalid"); - ::crypto::signature sig{}; - ::crypto::key_image ki; - memcpy(ki.data, buff, 32); - memcpy(sig.c.data, buff + 32, 32); - memcpy(sig.r.data, buff + 64, 32); + memcpy(ki.data, buff, sizeof(ki.data)); + memcpy(sig.c.data, buff + sizeof(ki.data), sizeof(ki.data)); + memcpy(sig.r.data, buff + 2*sizeof(ki.data), sizeof(ki.data)); ski.push_back(std::make_pair(ki, sig)); } +#undef EVENT_PROGRESS + } + + bool device_trezor::is_live_refresh_supported() const + { + require_initialized(); + return get_version() > pack_version(2, 0, 10); } + bool device_trezor::is_live_refresh_enabled() const + { + return is_live_refresh_supported() && (mode == NONE || mode == TRANSACTION_PARSE) && m_live_refresh_enabled; + } + + bool device_trezor::has_ki_live_refresh() const + { + try{ + return is_live_refresh_enabled(); + } catch(const std::exception & e){ + MERROR("Could not detect if live refresh is enabled: " << e.what()); + } + return false; + } + + void device_trezor::live_refresh_start() + { + TREZOR_AUTO_LOCK_CMD(); + require_connected(); + live_refresh_start_unsafe(); + } + + void device_trezor::live_refresh_start_unsafe() + { + device_state_reset_unsafe(); + require_initialized(); + + auto req = std::make_shared<messages::monero::MoneroLiveRefreshStartRequest>(); + this->set_msg_addr<messages::monero::MoneroLiveRefreshStartRequest>(req.get()); + this->client_exchange<messages::monero::MoneroLiveRefreshStartAck>(req); + m_live_refresh_in_progress = true; + m_last_live_refresh_time = std::chrono::steady_clock::now(); + } + + void device_trezor::live_refresh( + const ::crypto::secret_key & view_key_priv, + const crypto::public_key& out_key, + const crypto::key_derivation& recv_derivation, + size_t real_output_index, + const cryptonote::subaddress_index& received_index, + cryptonote::keypair& in_ephemeral, + crypto::key_image& ki + ) + { + TREZOR_AUTO_LOCK_CMD(); + require_connected(); + + if (!m_live_refresh_in_progress) + { + live_refresh_start_unsafe(); + } + + m_last_live_refresh_time = std::chrono::steady_clock::now(); + + auto req = std::make_shared<messages::monero::MoneroLiveRefreshStepRequest>(); + req->set_out_key(out_key.data, 32); + req->set_recv_deriv(recv_derivation.data, 32); + req->set_real_out_idx(real_output_index); + req->set_sub_addr_major(received_index.major); + req->set_sub_addr_minor(received_index.minor); + + auto ack = this->client_exchange<messages::monero::MoneroLiveRefreshStepAck>(req); + protocol::ki::live_refresh_ack(view_key_priv, out_key, ack, in_ephemeral, ki); + } + + void device_trezor::live_refresh_finish_unsafe() + { + auto req = std::make_shared<messages::monero::MoneroLiveRefreshFinalRequest>(); + this->client_exchange<messages::monero::MoneroLiveRefreshFinalAck>(req); + m_live_refresh_in_progress = false; + } + + void device_trezor::live_refresh_finish() + { + TREZOR_AUTO_LOCK_CMD(); + require_connected(); + if (m_live_refresh_in_progress) + { + live_refresh_finish_unsafe(); + } + } + + void device_trezor::computing_key_images(bool started) + { + try + { + if (!is_live_refresh_enabled()) + { + return; + } + + // React only on termination as the process can auto-start itself. + if (!started && m_live_refresh_in_progress) + { + live_refresh_finish(); + } + } + catch(const std::exception & e) + { + MWARNING("KI computation state change failed, started: " << started << ", e: " << e.what()); + } + } + + bool device_trezor::compute_key_image( + const ::cryptonote::account_keys& ack, + const ::crypto::public_key& out_key, + const ::crypto::key_derivation& recv_derivation, + size_t real_output_index, + const ::cryptonote::subaddress_index& received_index, + ::cryptonote::keypair& in_ephemeral, + ::crypto::key_image& ki) + { + if (!is_live_refresh_enabled()) + { + return false; + } + + live_refresh(ack.m_view_secret_key, out_key, recv_derivation, real_output_index, received_index, in_ephemeral, ki); + return true; + } void device_trezor::tx_sign(wallet_shim * wallet, const tools::wallet2::unsigned_tx_set & unsigned_tx, @@ -222,7 +473,15 @@ namespace trezor { hw::tx_aux_data & aux_data) { CHECK_AND_ASSERT_THROW_MES(unsigned_tx.transfers.first == 0, "Unsuported non zero offset"); - size_t num_tx = unsigned_tx.txes.size(); + + TREZOR_AUTO_LOCK_CMD(); + require_connected(); + device_state_reset_unsafe(); + require_initialized(); + transaction_versions_check(unsigned_tx, aux_data); + + const size_t num_tx = unsigned_tx.txes.size(); + m_num_transations_to_sign = num_tx; signed_tx.key_images.clear(); signed_tx.key_images.resize(unsigned_tx.transfers.second.size()); @@ -267,6 +526,10 @@ namespace trezor { cpend.key_images = key_images; // KI sync + for(size_t cidx=0, trans_max=unsigned_tx.transfers.second.size(); cidx < trans_max; ++cidx){ + signed_tx.key_images[cidx] = unsigned_tx.transfers.second[cidx].m_key_image; + } + size_t num_sources = cdata.tx_data.sources.size(); CHECK_AND_ASSERT_THROW_MES(num_sources == cdata.source_permutation.size(), "Invalid permutation size"); CHECK_AND_ASSERT_THROW_MES(num_sources == cdata.tx.vin.size(), "Invalid tx.vin size"); @@ -276,12 +539,19 @@ namespace trezor { CHECK_AND_ASSERT_THROW_MES(src_idx < cdata.tx.vin.size(), "Invalid idx_mapped"); size_t idx_map_src = cdata.tx_data.selected_transfers[idx_mapped]; - auto vini = boost::get<cryptonote::txin_to_key>(cdata.tx.vin[src_idx]); + CHECK_AND_ASSERT_THROW_MES(idx_map_src >= unsigned_tx.transfers.first, "Invalid offset"); + idx_map_src -= unsigned_tx.transfers.first; CHECK_AND_ASSERT_THROW_MES(idx_map_src < signed_tx.key_images.size(), "Invalid key image index"); + + const auto vini = boost::get<cryptonote::txin_to_key>(cdata.tx.vin[src_idx]); signed_tx.key_images[idx_map_src] = vini.k_image; } } + + if (m_callback){ + m_callback->on_progress(device_cold::tx_progress(m_num_transations_to_sign, m_num_transations_to_sign, 1, 1, 1, 1)); + } } void device_trezor::tx_sign(wallet_shim * wallet, @@ -290,10 +560,16 @@ namespace trezor { hw::tx_aux_data & aux_data, std::shared_ptr<protocol::tx::Signer> & signer) { - AUTO_LOCK_CMD(); +#define EVENT_PROGRESS(S, SUB, SUBMAX) do { if (m_callback) { \ + (m_callback)->on_progress(device_cold::tx_progress(idx, m_num_transations_to_sign, S, 10, SUB, SUBMAX)); \ +} }while(0) + require_connected(); - device_state_reset_unsafe(); + if (idx > 0) + device_state_reset_unsafe(); + require_initialized(); + EVENT_PROGRESS(0, 1, 1); CHECK_AND_ASSERT_THROW_MES(idx < unsigned_tx.txes.size(), "Invalid transaction index"); signer = std::make_shared<protocol::tx::Signer>(wallet, &unsigned_tx, idx, &aux_data); @@ -305,6 +581,7 @@ namespace trezor { auto init_msg = signer->step_init(); this->set_msg_addr(init_msg.get()); transaction_pre_check(init_msg); + EVENT_PROGRESS(1, 1, 1); auto response = this->client_exchange<messages::monero::MoneroTransactionInitAck>(init_msg); signer->step_init_ack(response); @@ -314,6 +591,7 @@ namespace trezor { auto src = signer->step_set_input(cur_src); auto ack = this->client_exchange<messages::monero::MoneroTransactionSetInputAck>(src); signer->step_set_input_ack(ack); + EVENT_PROGRESS(2, cur_src, num_sources); } // Step: sort @@ -322,44 +600,82 @@ namespace trezor { auto perm_ack = this->client_exchange<messages::monero::MoneroTransactionInputsPermutationAck>(perm_req); signer->step_permutation_ack(perm_ack); } + EVENT_PROGRESS(3, 1, 1); // Step: input_vini - if (!signer->in_memory()){ - for(size_t cur_src = 0; cur_src < num_sources; ++cur_src){ - auto src = signer->step_set_vini_input(cur_src); - auto ack = this->client_exchange<messages::monero::MoneroTransactionInputViniAck>(src); - signer->step_set_vini_input_ack(ack); - } + for(size_t cur_src = 0; cur_src < num_sources; ++cur_src){ + auto src = signer->step_set_vini_input(cur_src); + auto ack = this->client_exchange<messages::monero::MoneroTransactionInputViniAck>(src); + signer->step_set_vini_input_ack(ack); + EVENT_PROGRESS(4, cur_src, num_sources); } // Step: all inputs set auto all_inputs_set = signer->step_all_inputs_set(); auto ack_all_inputs = this->client_exchange<messages::monero::MoneroTransactionAllInputsSetAck>(all_inputs_set); signer->step_all_inputs_set_ack(ack_all_inputs); + EVENT_PROGRESS(5, 1, 1); // Step: outputs for(size_t cur_dst = 0; cur_dst < num_outputs; ++cur_dst){ auto src = signer->step_set_output(cur_dst); auto ack = this->client_exchange<messages::monero::MoneroTransactionSetOutputAck>(src); signer->step_set_output_ack(ack); + + // If BP is offloaded to host, another step with computed BP may be needed. + auto offloaded_bp = signer->step_rsig(cur_dst); + if (offloaded_bp){ + auto bp_ack = this->client_exchange<messages::monero::MoneroTransactionSetOutputAck>(offloaded_bp); + signer->step_set_rsig_ack(ack); + } + + EVENT_PROGRESS(6, cur_dst, num_outputs); } // Step: all outs set auto all_out_set = signer->step_all_outs_set(); auto ack_all_out_set = this->client_exchange<messages::monero::MoneroTransactionAllOutSetAck>(all_out_set); signer->step_all_outs_set_ack(ack_all_out_set, *this); + EVENT_PROGRESS(7, 1, 1); // Step: sign each input for(size_t cur_src = 0; cur_src < num_sources; ++cur_src){ auto src = signer->step_sign_input(cur_src); auto ack_sign = this->client_exchange<messages::monero::MoneroTransactionSignInputAck>(src); signer->step_sign_input_ack(ack_sign); + EVENT_PROGRESS(8, cur_src, num_sources); } // Step: final auto final_msg = signer->step_final(); auto ack_final = this->client_exchange<messages::monero::MoneroTransactionFinalAck>(final_msg); signer->step_final_ack(ack_final); + EVENT_PROGRESS(9, 1, 1); +#undef EVENT_PROGRESS + } + + void device_trezor::transaction_versions_check(const ::tools::wallet2::unsigned_tx_set & unsigned_tx, hw::tx_aux_data & aux_data) + { + auto trezor_version = get_version(); + unsigned client_version = 1; // default client version for tx + + if (trezor_version <= pack_version(2, 0, 10)){ + client_version = 0; + } + + if (aux_data.client_version){ + auto wanted_client_version = aux_data.client_version.get(); + if (wanted_client_version > client_version){ + throw exc::TrezorException("Trezor firmware 2.0.10 and lower does not support current transaction sign protocol. Please update."); + } else { + client_version = wanted_client_version; + } + } + aux_data.client_version = client_version; + + if (client_version == 0 && aux_data.bp_version && aux_data.bp_version.get() != 1){ + throw exc::TrezorException("Trezor firmware 2.0.10 and lower does not support current transaction sign protocol (BPv2+). Please update."); + } } void device_trezor::transaction_pre_check(std::shared_ptr<messages::monero::MoneroTransactionInitRequest> init_msg) diff --git a/src/device_trezor/device_trezor.hpp b/src/device_trezor/device_trezor.hpp index 75f6d5875..0e91847dc 100644 --- a/src/device_trezor/device_trezor.hpp +++ b/src/device_trezor/device_trezor.hpp @@ -30,18 +30,21 @@ #ifndef MONERO_DEVICE_TREZOR_H #define MONERO_DEVICE_TREZOR_H +#include "trezor.hpp" +#include "device/device.hpp" +#ifdef WITH_DEVICE_TREZOR #include <cstddef> #include <string> -#include "device/device.hpp" -#include "device/device_default.hpp" -#include "device/device_cold.hpp" #include <boost/scope_exit.hpp> #include <boost/thread/mutex.hpp> #include <boost/thread/recursive_mutex.hpp> + +#include "device/device_default.hpp" +#include "device/device_cold.hpp" #include "cryptonote_config.h" -#include "trezor.hpp" #include "device_trezor_base.hpp" +#endif namespace hw { namespace trezor { @@ -57,8 +60,29 @@ namespace trezor { */ class device_trezor : public hw::trezor::device_trezor_base, public hw::device_cold { protected: + std::atomic<bool> m_live_refresh_in_progress; + std::chrono::steady_clock::time_point m_last_live_refresh_time; + std::unique_ptr<boost::thread> m_live_refresh_thread; + std::atomic<bool> m_live_refresh_thread_running; + bool m_live_refresh_enabled; + size_t m_num_transations_to_sign; + + void transaction_versions_check(const ::tools::wallet2::unsigned_tx_set & unsigned_tx, hw::tx_aux_data & aux_data); void transaction_pre_check(std::shared_ptr<messages::monero::MoneroTransactionInitRequest> init_msg); void transaction_check(const protocol::tx::TData & tdata, const hw::tx_aux_data & aux_data); + void device_state_reset_unsafe() override; + void live_refresh_start_unsafe(); + void live_refresh_finish_unsafe(); + void live_refresh_thread_main(); + + /** + * Signs particular transaction idx in the unsigned set, keeps state in the signer + */ + virtual void tx_sign(wallet_shim * wallet, + const ::tools::wallet2::unsigned_tx_set & unsigned_tx, + size_t idx, + hw::tx_aux_data & aux_data, + std::shared_ptr<protocol::tx::Signer> & signer); public: device_trezor(); @@ -69,11 +93,17 @@ namespace trezor { explicit operator bool() const override {return true;} + bool init() override; + bool release() override; + bool disconnect() override; + device_protocol_t device_protocol() const override { return PROTOCOL_COLD; }; bool has_ki_cold_sync() const override { return true; } bool has_tx_cold_sign() const override { return true; } void set_network_type(cryptonote::network_type network_type) override { this->network_type = network_type; } + void set_live_refresh_enabled(bool enabled) { m_live_refresh_enabled = enabled; } + bool live_refresh_enabled() const { return m_live_refresh_enabled; } /* ======================================================================= */ /* WALLET & ADDRESS */ @@ -100,20 +130,68 @@ namespace trezor { const boost::optional<cryptonote::network_type> & network_type = boost::none); /** + * Get_tx_key support check + */ + bool is_get_tx_key_supported() const override; + + /** + * Loads tx aux data + */ + void load_tx_key_data(::hw::device_cold::tx_key_data_t & res, const std::string & tx_aux_data) override; + + /** + * TX key load with the Trezor + */ + void get_tx_key( + std::vector<::crypto::secret_key> & tx_keys, + const ::hw::device_cold::tx_key_data_t & tx_aux_data, + const ::crypto::secret_key & view_key_priv) override; + + /** * Key image sync with the Trezor. */ void ki_sync(wallet_shim * wallet, const std::vector<::tools::wallet2::transfer_details> & transfers, hw::device_cold::exported_key_image & ski) override; + bool is_live_refresh_supported() const override; + + bool is_live_refresh_enabled() const; + + bool has_ki_live_refresh() const override; + + void live_refresh_start() override; + + void live_refresh( + const ::crypto::secret_key & view_key_priv, + const crypto::public_key& out_key, + const crypto::key_derivation& recv_derivation, + size_t real_output_index, + const cryptonote::subaddress_index& received_index, + cryptonote::keypair& in_ephemeral, + crypto::key_image& ki + ) override; + + void live_refresh_finish() override; + /** - * Signs particular transaction idx in the unsigned set, keeps state in the signer + * Letting device know the KI computation started / ended. + * During refresh */ - void tx_sign(wallet_shim * wallet, - const ::tools::wallet2::unsigned_tx_set & unsigned_tx, - size_t idx, - hw::tx_aux_data & aux_data, - std::shared_ptr<protocol::tx::Signer> & signer); + void computing_key_images(bool started) override; + + /** + * Implements hw::device interface + * called from generate_key_image_helper_precomp() + */ + bool compute_key_image( + const ::cryptonote::account_keys& ack, + const ::crypto::public_key& out_key, + const ::crypto::key_derivation& recv_derivation, + size_t real_output_index, + const ::cryptonote::subaddress_index& received_index, + ::cryptonote::keypair& in_ephemeral, + ::crypto::key_image& ki) override; /** * Signs unsigned transaction with the Trezor. diff --git a/src/device_trezor/device_trezor_base.cpp b/src/device_trezor/device_trezor_base.cpp index 1892b47d9..f3d15c5e2 100644 --- a/src/device_trezor/device_trezor_base.cpp +++ b/src/device_trezor/device_trezor_base.cpp @@ -109,6 +109,7 @@ namespace trezor { disconnect(); // Enumerate all available devices + TREZOR_AUTO_LOCK_DEVICE(); try { hw::trezor::t_transport_vect trans; @@ -145,6 +146,7 @@ namespace trezor { } bool device_trezor_base::disconnect() { + TREZOR_AUTO_LOCK_DEVICE(); m_device_state.clear(); m_features.reset(); @@ -203,13 +205,13 @@ namespace trezor { /* Helpers */ /* ======================================================================= */ - void device_trezor_base::require_connected(){ + void device_trezor_base::require_connected() const { if (!m_transport){ throw exc::NotConnectedException(); } } - void device_trezor_base::require_initialized(){ + void device_trezor_base::require_initialized() const { if (!m_features){ throw exc::TrezorException("Device state not initialized"); } @@ -330,7 +332,7 @@ namespace trezor { /* ======================================================================= */ bool device_trezor_base::ping() { - AUTO_LOCK_CMD(); + TREZOR_AUTO_LOCK_CMD(); if (!m_transport){ MINFO("Ping failed, device not connected"); return false; @@ -364,7 +366,7 @@ namespace trezor { void device_trezor_base::device_state_reset() { - AUTO_LOCK_CMD(); + TREZOR_AUTO_LOCK_CMD(); device_state_reset_unsafe(); } @@ -373,6 +375,10 @@ namespace trezor { if (m_debug_callback) m_debug_callback->method(__VA_ARGS__); \ if (m_callback) m_callback->method(__VA_ARGS__); \ }while(0) +#define TREZOR_CALLBACK_GET(VAR, method, ...) do { \ + if (m_debug_callback) VAR = m_debug_callback->method(__VA_ARGS__); \ + if (m_callback) VAR = m_callback->method(__VA_ARGS__); \ +}while(0) void device_trezor_base::setup_debug(){ if (!m_debug){ @@ -392,6 +398,7 @@ namespace trezor { #else #define TREZOR_CALLBACK(method, ...) do { if (m_callback) m_callback->method(__VA_ARGS__); } while(0) +#define TREZOR_CALLBACK_GET(VAR, method, ...) VAR = (m_callback ? m_callback->method(__VA_ARGS__) : boost::none) #endif void device_trezor_base::on_button_request(GenericMessage & resp, const messages::common::ButtonRequest * msg) @@ -402,7 +409,7 @@ namespace trezor { messages::common::ButtonAck ack; write_raw(&ack); - TREZOR_CALLBACK(on_button_request); + TREZOR_CALLBACK(on_button_request, msg->code()); resp = read_raw(); } @@ -411,13 +418,18 @@ namespace trezor { MDEBUG("on_pin_request"); CHECK_AND_ASSERT_THROW_MES(msg, "Empty message"); - epee::wipeable_string pin; + boost::optional<epee::wipeable_string> pin; + TREZOR_CALLBACK_GET(pin, on_pin_request); - TREZOR_CALLBACK(on_pin_request, pin); + if (!pin && m_pin){ + pin = m_pin; + } // TODO: remove PIN from memory messages::common::PinMatrixAck m; - m.set_pin(pin.data(), pin.size()); + if (pin) { + m.set_pin(pin.get().data(), pin.get().size()); + } resp = call_raw(&m); } @@ -425,14 +437,19 @@ namespace trezor { { CHECK_AND_ASSERT_THROW_MES(msg, "Empty message"); MDEBUG("on_passhprase_request, on device: " << msg->on_device()); - epee::wipeable_string passphrase; + boost::optional<epee::wipeable_string> passphrase; + TREZOR_CALLBACK_GET(passphrase, on_passphrase_request, msg->on_device()); - TREZOR_CALLBACK(on_passphrase_request, msg->on_device(), passphrase); + if (!passphrase && m_passphrase){ + passphrase = m_passphrase; + } + + m_passphrase = boost::none; messages::common::PassphraseAck m; - if (!msg->on_device()){ + if (!msg->on_device() && passphrase){ // TODO: remove passphrase from memory - m.set_passphrase(passphrase.data(), passphrase.size()); + m.set_passphrase(passphrase.get().data(), passphrase.get().size()); } if (!m_device_state.empty()){ @@ -494,16 +511,16 @@ namespace trezor { m_debug_link->init(debug_transport); } - void trezor_debug_callback::on_button_request() { + void trezor_debug_callback::on_button_request(uint64_t code) { if (m_debug_link) m_debug_link->press_yes(); } - void trezor_debug_callback::on_pin_request(epee::wipeable_string &pin) { - + boost::optional<epee::wipeable_string> trezor_debug_callback::on_pin_request() { + return boost::none; } - void trezor_debug_callback::on_passphrase_request(bool on_device, epee::wipeable_string &passphrase) { - + boost::optional<epee::wipeable_string> trezor_debug_callback::on_passphrase_request(bool on_device) { + return boost::none; } void trezor_debug_callback::on_passphrase_state_request(const std::string &state) { diff --git a/src/device_trezor/device_trezor_base.hpp b/src/device_trezor/device_trezor_base.hpp index 8942df797..8c3c14b29 100644 --- a/src/device_trezor/device_trezor_base.hpp +++ b/src/device_trezor/device_trezor_base.hpp @@ -47,14 +47,15 @@ #endif //automatic lock one more level on device ensuring the current thread is allowed to use it -#define AUTO_LOCK_CMD() \ +#define TREZOR_AUTO_LOCK_CMD() \ /* lock both mutexes without deadlock*/ \ boost::lock(device_locker, command_locker); \ /* make sure both already-locked mutexes are unlocked at the end of scope */ \ boost::lock_guard<boost::recursive_mutex> lock1(device_locker, boost::adopt_lock); \ boost::lock_guard<boost::mutex> lock2(command_locker, boost::adopt_lock) - +#define TREZOR_AUTO_LOCK_DEVICE() boost::lock_guard<boost::recursive_mutex> lock1_device(device_locker) + namespace hw { namespace trezor { @@ -62,14 +63,14 @@ namespace trezor { class device_trezor_base; #ifdef WITH_TREZOR_DEBUGGING - class trezor_debug_callback { + class trezor_debug_callback : public hw::i_device_callback { public: trezor_debug_callback()=default; explicit trezor_debug_callback(std::shared_ptr<Transport> & debug_transport); - void on_button_request(); - void on_pin_request(epee::wipeable_string &pin); - void on_passphrase_request(bool on_device, epee::wipeable_string &passphrase); + void on_button_request(uint64_t code=0) override; + boost::optional<epee::wipeable_string> on_pin_request() override; + boost::optional<epee::wipeable_string> on_passphrase_request(bool on_device) override; void on_passphrase_state_request(const std::string &state); void on_disconnect(); protected: @@ -95,6 +96,8 @@ namespace trezor { std::vector<unsigned int> m_wallet_deriv_path; std::string m_device_state; // returned after passphrase entry, session std::shared_ptr<messages::management::Features> m_features; // features from the last device reset + boost::optional<epee::wipeable_string> m_pin; + boost::optional<epee::wipeable_string> m_passphrase; cryptonote::network_type network_type; @@ -109,11 +112,11 @@ namespace trezor { // Internal methods // - void require_connected(); - void require_initialized(); + void require_connected() const; + void require_initialized() const; void call_ping_unsafe(); void test_ping(); - void device_state_reset_unsafe(); + virtual void device_state_reset_unsafe(); void ensure_derivation_path() noexcept; // Communication methods @@ -265,6 +268,15 @@ namespace trezor { void set_derivation_path(const std::string &deriv_path) override; + virtual bool has_ki_live_refresh(void) const override { return false; } + + virtual void set_pin(const epee::wipeable_string & pin) override { + m_pin = pin; + } + virtual void set_passphrase(const epee::wipeable_string & passphrase) override { + m_passphrase = passphrase; + } + /* ======================================================================= */ /* SETUP/TEARDOWN */ /* ======================================================================= */ diff --git a/src/device_trezor/trezor/protocol.cpp b/src/device_trezor/trezor/protocol.cpp index 79f6adac9..5fe08abbe 100644 --- a/src/device_trezor/trezor/protocol.cpp +++ b/src/device_trezor/trezor/protocol.cpp @@ -33,6 +33,8 @@ #include <utility> #include <boost/endian/conversion.hpp> #include <common/apply_permutation.h> +#include <common/json_util.h> +#include <crypto/hmac-keccak.h> #include <ringct/rctSigs.h> #include <ringct/bulletproofs.h> #include "cryptonote_config.h" @@ -40,6 +42,37 @@ #include <sodium/crypto_verify_32.h> #include <sodium/crypto_aead_chacha20poly1305.h> +#define GET_FIELD_STRING(name, type, jtype) field_##name = std::string(json[#name].GetString(), json[#name].GetStringLength()) +#define GET_FIELD_OTHER(name, type, jtype) field_##name = static_cast<type>(json[#name].Get##jtype()) + +#define GET_STRING_FROM_JSON(json, name, type, mandatory, def) \ + GET_FIELD_FROM_JSON_EX(json, name, type, String, mandatory, def, GET_FIELD_STRING) + +#define GET_FIELD_FROM_JSON(json, name, type, jtype, mandatory, def) \ + GET_FIELD_FROM_JSON_EX(json, name, type, jtype, mandatory, def, GET_FIELD_OTHER) + +#define GET_FIELD_FROM_JSON_EX(json, name, type, jtype, mandatory, def, VAL) \ + type field_##name = static_cast<type>(def); \ + bool field_##name##_found = false; \ + (void)field_##name##_found; \ + do if (json.HasMember(#name)) \ + { \ + if (json[#name].Is##jtype()) \ + { \ + VAL(name, type, jtype); \ + field_##name##_found = true; \ + } \ + else \ + { \ + throw std::invalid_argument("Field " #name " found in JSON, but not " #jtype); \ + } \ + } \ + else if (mandatory) \ + { \ + throw std::invalid_argument("Field " #name " not found in JSON");\ + } while(0) + + namespace hw{ namespace trezor{ namespace protocol{ @@ -84,19 +117,22 @@ namespace protocol{ namespace crypto { namespace chacha { - void decrypt(const void* ciphertext, size_t length, const uint8_t* key, const uint8_t* iv, char* plaintext){ - if (length < 16){ - throw std::invalid_argument("Ciphertext length too small"); - } + void decrypt(const void* ciphertext, size_t length, const uint8_t* key, const uint8_t* iv, char* plaintext, size_t *plaintext_len){ + CHECK_AND_ASSERT_THROW_MES(length >= TAG_SIZE, "Ciphertext length too small"); + CHECK_AND_ASSERT_THROW_MES(!plaintext_len || *plaintext_len >= (length - TAG_SIZE), "Plaintext length too small"); - unsigned long long int cip_len = length; + unsigned long long int res_len = plaintext_len ? *plaintext_len : length; auto r = crypto_aead_chacha20poly1305_ietf_decrypt( - reinterpret_cast<unsigned char *>(plaintext), &cip_len, nullptr, + reinterpret_cast<unsigned char *>(plaintext), &res_len, nullptr, static_cast<const unsigned char *>(ciphertext), length, nullptr, 0, iv, key); if (r != 0){ throw exc::Poly1305TagInvalid(); } + + if (plaintext_len){ + *plaintext_len = (size_t) res_len; + } } } @@ -185,6 +221,49 @@ namespace ki { } } + void live_refresh_ack(const ::crypto::secret_key & view_key_priv, + const ::crypto::public_key& out_key, + const std::shared_ptr<messages::monero::MoneroLiveRefreshStepAck> & ack, + ::cryptonote::keypair& in_ephemeral, + ::crypto::key_image& ki) + { + std::string str_out_key(out_key.data, sizeof(out_key.data)); + auto enc_key = protocol::tx::compute_enc_key(view_key_priv, str_out_key, ack->salt()); + + const size_t len_ciphertext = ack->key_image().size(); // IV || keys + CHECK_AND_ASSERT_THROW_MES(len_ciphertext > crypto::chacha::IV_SIZE + crypto::chacha::TAG_SIZE, "Invalid size"); + + size_t ki_len = len_ciphertext - crypto::chacha::IV_SIZE - crypto::chacha::TAG_SIZE; + std::unique_ptr<uint8_t[]> plaintext(new uint8_t[ki_len]); + uint8_t * buff = plaintext.get(); + + protocol::crypto::chacha::decrypt( + ack->key_image().data() + crypto::chacha::IV_SIZE, + len_ciphertext - crypto::chacha::IV_SIZE, + reinterpret_cast<const uint8_t *>(enc_key.data), + reinterpret_cast<const uint8_t *>(ack->key_image().data()), + reinterpret_cast<char *>(buff), &ki_len); + + CHECK_AND_ASSERT_THROW_MES(ki_len == 3*32, "Invalid size"); + ::crypto::signature sig{}; + memcpy(ki.data, buff, 32); + memcpy(sig.c.data, buff + 32, 32); + memcpy(sig.r.data, buff + 64, 32); + in_ephemeral.pub = out_key; + in_ephemeral.sec = ::crypto::null_skey; + + // Verification + std::vector<const ::crypto::public_key*> pkeys; + pkeys.push_back(&out_key); + + CHECK_AND_ASSERT_THROW_MES(rct::scalarmultKey(rct::ki2rct(ki), rct::curveOrder()) == rct::identity(), + "Key image out of validity domain: key image " << epee::string_tools::pod_to_hex(ki)); + + CHECK_AND_ASSERT_THROW_MES(::crypto::check_ring_signature((const ::crypto::hash&)ki, ki, pkeys, &sig), + "Signature failed for key image " << epee::string_tools::pod_to_hex(ki) + << ", signature " + epee::string_tools::pod_to_hex(sig) + << ", pubkey " + epee::string_tools::pod_to_hex(*pkeys[0])); + } } // Cold transaction signing @@ -198,6 +277,8 @@ namespace tx { void translate_dst_entry(MoneroTransactionDestinationEntry * dst, const cryptonote::tx_destination_entry * src){ dst->set_amount(src->amount); dst->set_is_subaddress(src->is_subaddress); + dst->set_is_integrated(src->is_integrated); + dst->set_original(src->original); translate_address(dst->mutable_addr(), &(src->addr)); } @@ -267,9 +348,29 @@ namespace tx { return std::string(buff, offset); } + ::crypto::secret_key compute_enc_key(const ::crypto::secret_key & private_view_key, const std::string & aux, const std::string & salt) + { + uint8_t hash[32]; + KECCAK_CTX ctx; + ::crypto::secret_key res; + + keccak_init(&ctx); + keccak_update(&ctx, (const uint8_t *) private_view_key.data, sizeof(private_view_key.data)); + if (!aux.empty()){ + keccak_update(&ctx, (const uint8_t *) aux.data(), aux.size()); + } + keccak_finish(&ctx, hash); + keccak(hash, sizeof(hash), hash, sizeof(hash)); + + hmac_keccak_hash(hash, (const uint8_t *) salt.data(), salt.size(), hash, sizeof(hash)); + memcpy(res.data, hash, sizeof(hash)); + memwipe(hash, sizeof(hash)); + return res; + } + TData::TData() { - in_memory = false; rsig_type = 0; + bp_version = 0; cur_input_idx = 0; cur_output_idx = 0; cur_batch_idx = 0; @@ -283,6 +384,7 @@ namespace tx { m_tx_idx = tx_idx; m_ct.tx_data = cur_tx(); m_multisig = false; + m_client_version = 1; } void Signer::extract_payment_id(){ @@ -308,8 +410,8 @@ namespace tx { } } - static unsigned get_rsig_type(bool use_bulletproof, size_t num_outputs){ - if (!use_bulletproof){ + static unsigned get_rsig_type(const rct::RCTConfig &rct_config, size_t num_outputs){ + if (rct_config.range_proof_type == rct::RangeProofBorromean){ return rct::RangeProofBorromean; } else if (num_outputs > BULLETPROOF_MAX_OUTPUTS){ return rct::RangeProofMultiOutputBulletproof; @@ -392,8 +494,10 @@ namespace tx { m_ct.tx.version = 2; m_ct.tx.unlock_time = tx.unlock_time; + m_client_version = (m_aux_data->client_version ? m_aux_data->client_version.get() : 1); tsx_data.set_version(1); + tsx_data.set_client_version(client_version()); tsx_data.set_unlock_time(tx.unlock_time); tsx_data.set_num_inputs(static_cast<google::protobuf::uint32>(tx.sources.size())); tsx_data.set_mixin(static_cast<google::protobuf::uint32>(tx.sources[0].outputs.size() - 1)); @@ -402,8 +506,12 @@ namespace tx { // Rsig decision auto rsig_data = tsx_data.mutable_rsig_data(); - m_ct.rsig_type = get_rsig_type(tx.use_bulletproofs, tx.splitted_dsts.size()); + m_ct.rsig_type = get_rsig_type(tx.rct_config, tx.splitted_dsts.size()); rsig_data->set_rsig_type(m_ct.rsig_type); + if (tx.rct_config.range_proof_type != rct::RangeProofBorromean){ + m_ct.bp_version = (m_aux_data->bp_version ? m_aux_data->bp_version.get() : 1); + rsig_data->set_bp_version((uint32_t) m_ct.bp_version); + } generate_rsig_batch_sizes(m_ct.grouping_vct, m_ct.rsig_type, tx.splitted_dsts.size()); assign_to_repeatable(rsig_data->mutable_grouping(), m_ct.grouping_vct.begin(), m_ct.grouping_vct.end()); @@ -437,7 +545,6 @@ namespace tx { } void Signer::step_init_ack(std::shared_ptr<const messages::monero::MoneroTransactionInitAck> ack){ - m_ct.in_memory = false; if (ack->has_rsig_data()){ m_ct.rsig_param = std::make_shared<MoneroRsigData>(ack->rsig_data()); } @@ -505,10 +612,6 @@ namespace tx { std::shared_ptr<messages::monero::MoneroTransactionInputsPermutationRequest> Signer::step_permutation(){ sort_ki(); - if (in_memory()){ - return nullptr; - } - auto res = std::make_shared<messages::monero::MoneroTransactionInputsPermutationRequest>(); assign_to_repeatable(res->mutable_perm(), m_ct.source_permutation.begin(), m_ct.source_permutation.end()); @@ -516,15 +619,10 @@ namespace tx { } void Signer::step_permutation_ack(std::shared_ptr<const messages::monero::MoneroTransactionInputsPermutationAck> ack){ - if (in_memory()){ - return; - } + } std::shared_ptr<messages::monero::MoneroTransactionInputViniRequest> Signer::step_set_vini_input(size_t idx){ - if (in_memory()){ - return nullptr; - } CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_data.sources.size(), "Invalid transaction index"); CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx.vin.size(), "Invalid transaction index"); CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_in_hmacs.size(), "Invalid transaction index"); @@ -536,7 +634,8 @@ namespace tx { translate_src_entry(res->mutable_src_entr(), &(tx.sources[idx])); res->set_vini(cryptonote::t_serializable_object_to_blob(vini)); res->set_vini_hmac(m_ct.tx_in_hmacs[idx]); - if (!in_memory()) { + + if (client_version() == 0) { CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs.size(), "Invalid transaction index"); CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs_hmac.size(), "Invalid transaction index"); res->set_pseudo_out(m_ct.pseudo_outs[idx]); @@ -547,9 +646,7 @@ namespace tx { } void Signer::step_set_vini_input_ack(std::shared_ptr<const messages::monero::MoneroTransactionInputViniAck> ack){ - if (in_memory()){ - return; - } + } std::shared_ptr<messages::monero::MoneroTransactionAllInputsSetRequest> Signer::step_all_inputs_set(){ @@ -557,34 +654,37 @@ namespace tx { } void Signer::step_all_inputs_set_ack(std::shared_ptr<const messages::monero::MoneroTransactionAllInputsSetAck> ack){ - if (is_offloading()){ - // If offloading, expect rsig configuration. - if (!ack->has_rsig_data()){ - throw exc::ProtocolException("Rsig offloading requires rsig param"); - } + if (client_version() > 0 || !is_offloading()){ + return; + } - auto & rsig_data = ack->rsig_data(); - if (!rsig_data.has_mask()){ - throw exc::ProtocolException("Gamma masks not present in offloaded version"); - } + // If offloading, expect rsig configuration. + if (!ack->has_rsig_data()){ + throw exc::ProtocolException("Rsig offloading requires rsig param"); + } - auto & mask = rsig_data.mask(); - if (mask.size() != 32 * num_outputs()){ - throw exc::ProtocolException("Invalid number of gamma masks"); - } + auto & rsig_data = ack->rsig_data(); + if (!rsig_data.has_mask()){ + throw exc::ProtocolException("Gamma masks not present in offloaded version"); + } - m_ct.rsig_gamma.reserve(num_outputs()); - for(size_t c=0; c < num_outputs(); ++c){ - rct::key cmask{}; - memcpy(cmask.bytes, mask.data() + c * 32, 32); - m_ct.rsig_gamma.emplace_back(cmask); - } + auto & mask = rsig_data.mask(); + if (mask.size() != 32 * num_outputs()){ + throw exc::ProtocolException("Invalid number of gamma masks"); + } + + m_ct.rsig_gamma.reserve(num_outputs()); + for(size_t c=0; c < num_outputs(); ++c){ + rct::key cmask{}; + memcpy(cmask.bytes, mask.data() + c * 32, 32); + m_ct.rsig_gamma.emplace_back(cmask); } } std::shared_ptr<messages::monero::MoneroTransactionSetOutputRequest> Signer::step_set_output(size_t idx){ CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_data.splitted_dsts.size(), "Invalid transaction index"); CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_out_entr_hmacs.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(is_req_bulletproof(), "Borromean rsig not supported"); m_ct.cur_output_idx = idx; m_ct.cur_output_in_batch_idx += 1; // assumes sequential call to step_set_output() @@ -595,48 +695,11 @@ namespace tx { res->set_dst_entr_hmac(m_ct.tx_out_entr_hmacs[idx]); // Range sig offloading to the host - if (!is_offloading()) { - return res; - } - - CHECK_AND_ASSERT_THROW_MES(m_ct.cur_batch_idx < m_ct.grouping_vct.size(), "Invalid batch index"); - if (m_ct.grouping_vct[m_ct.cur_batch_idx] > m_ct.cur_output_in_batch_idx) { - return res; - } - - auto rsig_data = res->mutable_rsig_data(); - auto batch_size = m_ct.grouping_vct[m_ct.cur_batch_idx]; - - if (!is_req_bulletproof()){ - if (batch_size > 1){ - throw std::invalid_argument("Borromean cannot batch outputs"); - } - - CHECK_AND_ASSERT_THROW_MES(idx < m_ct.rsig_gamma.size(), "Invalid gamma index"); - rct::key C{}, mask = m_ct.rsig_gamma[idx]; - auto genRsig = rct::proveRange(C, mask, cur_dst.amount); // TODO: rsig with given mask - auto serRsig = cn_serialize(genRsig); - m_ct.tx_out_rsigs.emplace_back(genRsig); - rsig_data->set_rsig(serRsig); - - } else { - std::vector<uint64_t> amounts; - rct::keyV masks; - CHECK_AND_ASSERT_THROW_MES(idx + 1 >= batch_size, "Invalid index for batching"); - - for(size_t i = 0; i < batch_size; ++i){ - const size_t bidx = 1 + idx - batch_size + i; - CHECK_AND_ASSERT_THROW_MES(bidx < m_ct.tx_data.splitted_dsts.size(), "Invalid gamma index"); - CHECK_AND_ASSERT_THROW_MES(bidx < m_ct.rsig_gamma.size(), "Invalid gamma index"); - - amounts.push_back(m_ct.tx_data.splitted_dsts[bidx].amount); - masks.push_back(m_ct.rsig_gamma[bidx]); - } - - auto bp = bulletproof_PROVE(amounts, masks); - auto serRsig = cn_serialize(bp); - m_ct.tx_out_rsigs.emplace_back(bp); - rsig_data->set_rsig(serRsig); + // ClientV0 sends offloaded BP with the last message in the batch. + // ClientV1 needs additional message after the last message in the batch as BP uses deterministic masks. + if (client_version() == 0 && is_offloading() && should_compute_bp_now()) { + auto rsig_data = res->mutable_rsig_data(); + compute_bproof(*rsig_data); } return res; @@ -644,7 +707,6 @@ namespace tx { void Signer::step_set_output_ack(std::shared_ptr<const messages::monero::MoneroTransactionSetOutputAck> ack){ cryptonote::tx_out tx_out; - rct::rangeSig range_sig{}; rct::Bulletproof bproof{}; rct::ctkey out_pk{}; rct::ecdhTuple ecdh{}; @@ -658,12 +720,12 @@ namespace tx { if (rsig_data.has_rsig() && !rsig_data.rsig().empty()){ has_rsig = true; rsig_buff = rsig_data.rsig(); + } - } else if (rsig_data.rsig_parts_size() > 0){ - has_rsig = true; - for (const auto &it : rsig_data.rsig_parts()) { - rsig_buff += it; - } + if (client_version() >= 1 && rsig_data.has_mask()){ + rct::key cmask{}; + string_to_key(cmask, rsig_data.mask()); + m_ct.rsig_gamma.emplace_back(cmask); } } @@ -675,12 +737,13 @@ namespace tx { throw exc::ProtocolException("Cannot deserialize out_pk"); } - if (!cn_deserialize(ack->ecdh_info(), ecdh)){ - throw exc::ProtocolException("Cannot deserialize ecdhtuple"); - } - - if (has_rsig && !is_req_bulletproof() && !cn_deserialize(rsig_buff, range_sig)){ - throw exc::ProtocolException("Cannot deserialize rangesig"); + if (m_ct.bp_version <= 1) { + if (!cn_deserialize(ack->ecdh_info(), ecdh)){ + throw exc::ProtocolException("Cannot deserialize ecdhtuple"); + } + } else { + CHECK_AND_ASSERT_THROW_MES(8 == ack->ecdh_info().size(), "Invalid ECDH.amount size"); + memcpy(ecdh.amount.bytes, ack->ecdh_info().data(), 8); } if (has_rsig && is_req_bulletproof() && !cn_deserialize(rsig_buff, bproof)){ @@ -692,35 +755,77 @@ namespace tx { m_ct.tx_out_pk.emplace_back(out_pk); m_ct.tx_out_ecdh.emplace_back(ecdh); - if (!has_rsig){ + // ClientV0, if no rsig was generated on Trezor, do not continue. + // ClientV1+ generates BP after all masks in the current batch are generated + if (!has_rsig || (client_version() >= 1 && is_offloading())){ return; } - if (is_req_bulletproof()){ - CHECK_AND_ASSERT_THROW_MES(m_ct.cur_batch_idx < m_ct.grouping_vct.size(), "Invalid batch index"); - auto batch_size = m_ct.grouping_vct[m_ct.cur_batch_idx]; - for (size_t i = 0; i < batch_size; ++i){ - const size_t bidx = 1 + m_ct.cur_output_idx - batch_size + i; - CHECK_AND_ASSERT_THROW_MES(bidx < m_ct.tx_out_pk.size(), "Invalid out index"); + process_bproof(bproof); + m_ct.cur_batch_idx += 1; + m_ct.cur_output_in_batch_idx = 0; + } - rct::key commitment = m_ct.tx_out_pk[bidx].mask; - commitment = rct::scalarmultKey(commitment, rct::INV_EIGHT); - bproof.V.push_back(commitment); - } + bool Signer::should_compute_bp_now() const { + CHECK_AND_ASSERT_THROW_MES(m_ct.cur_batch_idx < m_ct.grouping_vct.size(), "Invalid batch index"); + return m_ct.grouping_vct[m_ct.cur_batch_idx] <= m_ct.cur_output_in_batch_idx; + } - m_ct.tx_out_rsigs.emplace_back(bproof); - if (!rct::bulletproof_VERIFY(boost::get<rct::Bulletproof>(m_ct.tx_out_rsigs.back()))) { - throw exc::ProtocolException("Returned range signature is invalid"); - } + void Signer::compute_bproof(messages::monero::MoneroTransactionRsigData & rsig_data){ + auto batch_size = m_ct.grouping_vct[m_ct.cur_batch_idx]; + std::vector<uint64_t> amounts; + rct::keyV masks; + CHECK_AND_ASSERT_THROW_MES(m_ct.cur_output_idx + 1 >= batch_size, "Invalid index for batching"); - } else { - m_ct.tx_out_rsigs.emplace_back(range_sig); + for(size_t i = 0; i < batch_size; ++i){ + const size_t bidx = 1 + m_ct.cur_output_idx - batch_size + i; + CHECK_AND_ASSERT_THROW_MES(bidx < m_ct.tx_data.splitted_dsts.size(), "Invalid gamma index"); + CHECK_AND_ASSERT_THROW_MES(bidx < m_ct.rsig_gamma.size(), "Invalid gamma index"); - if (!rct::verRange(out_pk.mask, boost::get<rct::rangeSig>(m_ct.tx_out_rsigs.back()))) { - throw exc::ProtocolException("Returned range signature is invalid"); - } + amounts.push_back(m_ct.tx_data.splitted_dsts[bidx].amount); + masks.push_back(m_ct.rsig_gamma[bidx]); } + auto bp = bulletproof_PROVE(amounts, masks); + auto serRsig = cn_serialize(bp); + m_ct.tx_out_rsigs.emplace_back(bp); + rsig_data.set_rsig(serRsig); + } + + void Signer::process_bproof(rct::Bulletproof & bproof){ + CHECK_AND_ASSERT_THROW_MES(m_ct.cur_batch_idx < m_ct.grouping_vct.size(), "Invalid batch index"); + auto batch_size = m_ct.grouping_vct[m_ct.cur_batch_idx]; + for (size_t i = 0; i < batch_size; ++i){ + const size_t bidx = 1 + m_ct.cur_output_idx - batch_size + i; + CHECK_AND_ASSERT_THROW_MES(bidx < m_ct.tx_out_pk.size(), "Invalid out index"); + + rct::key commitment = m_ct.tx_out_pk[bidx].mask; + commitment = rct::scalarmultKey(commitment, rct::INV_EIGHT); + bproof.V.push_back(commitment); + } + + m_ct.tx_out_rsigs.emplace_back(bproof); + if (!rct::bulletproof_VERIFY(boost::get<rct::Bulletproof>(m_ct.tx_out_rsigs.back()))) { + throw exc::ProtocolException("Returned range signature is invalid"); + } + } + + std::shared_ptr<messages::monero::MoneroTransactionSetOutputRequest> Signer::step_rsig(size_t idx){ + if (client_version() == 0 || !is_offloading() || !should_compute_bp_now()){ + return nullptr; + } + + auto res = std::make_shared<messages::monero::MoneroTransactionSetOutputRequest>(); + auto & cur_dst = m_ct.tx_data.splitted_dsts[idx]; + translate_dst_entry(res->mutable_dst_entr(), &cur_dst); + res->set_dst_entr_hmac(m_ct.tx_out_entr_hmacs[idx]); + + compute_bproof(*(res->mutable_rsig_data())); + res->set_is_offloaded_bp(true); + return res; + } + + void Signer::step_set_rsig_ack(std::shared_ptr<const messages::monero::MoneroTransactionSetOutputAck> ack){ m_ct.cur_batch_idx += 1; m_ct.cur_output_in_batch_idx = 0; } @@ -814,12 +919,11 @@ namespace tx { res->set_vini_hmac(m_ct.tx_in_hmacs[idx]); res->set_pseudo_out_alpha(m_ct.alphas[idx]); res->set_spend_key(m_ct.spend_encs[idx]); - if (!in_memory()){ - CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs.size(), "Invalid transaction index"); - CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs_hmac.size(), "Invalid transaction index"); - res->set_pseudo_out(m_ct.pseudo_outs[idx]); - res->set_pseudo_out_hmac(m_ct.pseudo_outs_hmac[idx]); - } + + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs_hmac.size(), "Invalid transaction index"); + res->set_pseudo_out(m_ct.pseudo_outs[idx]); + res->set_pseudo_out_hmac(m_ct.pseudo_outs_hmac[idx]); return res; } @@ -829,6 +933,19 @@ namespace tx { throw exc::ProtocolException("Cannot deserialize mg[i]"); } + // Sync updated pseudo_outputs, client_version>=1, HF10+ + if (client_version() >= 1 && ack->has_pseudo_out()){ + CHECK_AND_ASSERT_THROW_MES(m_ct.cur_input_idx < m_ct.pseudo_outs.size(), "Invalid pseudo-out index"); + m_ct.pseudo_outs[m_ct.cur_input_idx] = ack->pseudo_out(); + if (is_bulletproof()){ + CHECK_AND_ASSERT_THROW_MES(m_ct.cur_input_idx < m_ct.rv->p.pseudoOuts.size(), "Invalid pseudo-out index"); + string_to_key(m_ct.rv->p.pseudoOuts[m_ct.cur_input_idx], ack->pseudo_out()); + } else { + CHECK_AND_ASSERT_THROW_MES(m_ct.cur_input_idx < m_ct.rv->pseudoOuts.size(), "Invalid pseudo-out index"); + string_to_key(m_ct.rv->pseudoOuts[m_ct.cur_input_idx], ack->pseudo_out()); + } + } + m_ct.rv->p.MGs.push_back(mg); } @@ -841,14 +958,14 @@ namespace tx { if (m_multisig){ auto & cout_key = ack->cout_key(); for(auto & cur : m_ct.couts){ - if (cur.size() != 12 + 32){ + if (cur.size() != crypto::chacha::IV_SIZE + 32){ throw std::invalid_argument("Encrypted cout has invalid length"); } char buff[32]; auto data = cur.data(); - crypto::chacha::decrypt(data + 12, 32, reinterpret_cast<const uint8_t *>(cout_key.data()), reinterpret_cast<const uint8_t *>(data), buff); + crypto::chacha::decrypt(data + crypto::chacha::IV_SIZE, 32, reinterpret_cast<const uint8_t *>(cout_key.data()), reinterpret_cast<const uint8_t *>(data), buff); m_ct.couts_dec.emplace_back(buff, 32); } } @@ -887,6 +1004,82 @@ namespace tx { return sb.GetString(); } + void load_tx_key_data(hw::device_cold::tx_key_data_t & res, const std::string & data) + { + rapidjson::Document json; + + // The contents should be JSON if the wallet follows the new format. + if (json.Parse(data.c_str()).HasParseError()) + { + throw std::invalid_argument("Data parsing error"); + } + else if(!json.IsObject()) + { + throw std::invalid_argument("Data parsing error - not an object"); + } + + GET_FIELD_FROM_JSON(json, version, int, Int, true, -1); + GET_STRING_FROM_JSON(json, salt1, std::string, true, std::string()); + GET_STRING_FROM_JSON(json, salt2, std::string, true, std::string()); + GET_STRING_FROM_JSON(json, enc_keys, std::string, true, std::string()); + GET_STRING_FROM_JSON(json, tx_prefix_hash, std::string, false, std::string()); + + if (field_version != 1) + { + throw std::invalid_argument("Unknown version"); + } + + res.salt1 = field_salt1; + res.salt2 = field_salt2; + res.tx_enc_keys = field_enc_keys; + res.tx_prefix_hash = field_tx_prefix_hash; + } + + std::shared_ptr<messages::monero::MoneroGetTxKeyRequest> get_tx_key( + const hw::device_cold::tx_key_data_t & tx_data) + { + auto req = std::make_shared<messages::monero::MoneroGetTxKeyRequest>(); + req->set_salt1(tx_data.salt1); + req->set_salt2(tx_data.salt2); + req->set_tx_enc_keys(tx_data.tx_enc_keys); + req->set_tx_prefix_hash(tx_data.tx_prefix_hash); + req->set_reason(0); + + return req; + } + + void get_tx_key_ack( + std::vector<::crypto::secret_key> & tx_keys, + const std::string & tx_prefix_hash, + const ::crypto::secret_key & view_key_priv, + std::shared_ptr<const messages::monero::MoneroGetTxKeyAck> ack + ) + { + auto enc_key = protocol::tx::compute_enc_key(view_key_priv, tx_prefix_hash, ack->salt()); + auto & encrypted_keys = ack->has_tx_derivations() ? ack->tx_derivations() : ack->tx_keys(); + + const size_t len_ciphertext = encrypted_keys.size(); // IV || keys || TAG + CHECK_AND_ASSERT_THROW_MES(len_ciphertext > crypto::chacha::IV_SIZE + crypto::chacha::TAG_SIZE, "Invalid size"); + + size_t keys_len = len_ciphertext - crypto::chacha::IV_SIZE - crypto::chacha::TAG_SIZE; + std::unique_ptr<uint8_t[]> plaintext(new uint8_t[keys_len]); + + protocol::crypto::chacha::decrypt( + encrypted_keys.data() + crypto::chacha::IV_SIZE, + len_ciphertext - crypto::chacha::IV_SIZE, + reinterpret_cast<const uint8_t *>(enc_key.data), + reinterpret_cast<const uint8_t *>(encrypted_keys.data()), + reinterpret_cast<char *>(plaintext.get()), &keys_len); + + CHECK_AND_ASSERT_THROW_MES(keys_len % 32 == 0, "Invalid size"); + tx_keys.resize(keys_len / 32); + + for(unsigned i = 0; i < keys_len / 32; ++i) + { + memcpy(tx_keys[i].data, plaintext.get() + 32 * i, 32); + } + memwipe(plaintext.get(), keys_len); + } } } diff --git a/src/device_trezor/trezor/protocol.hpp b/src/device_trezor/trezor/protocol.hpp index 23f94f948..f58bf1039 100644 --- a/src/device_trezor/trezor/protocol.hpp +++ b/src/device_trezor/trezor/protocol.hpp @@ -92,11 +92,14 @@ namespace protocol{ // Crypto / encryption namespace crypto { namespace chacha { + // Constants as defined in RFC 7539. + const unsigned IV_SIZE = 12; + const unsigned TAG_SIZE = 16; // crypto_aead_chacha20poly1305_IETF_ABYTES; /** * Chacha20Poly1305 decryption with tag verification. RFC 7539. */ - void decrypt(const void* ciphertext, size_t length, const uint8_t* key, const uint8_t* iv, char* plaintext); + void decrypt(const void* ciphertext, size_t length, const uint8_t* key, const uint8_t* iv, char* plaintext, size_t *plaintext_len=nullptr); } } @@ -129,6 +132,14 @@ namespace ki { const std::vector<tools::wallet2::transfer_details> & transfers, std::shared_ptr<messages::monero::MoneroKeyImageExportInitRequest> & req); + /** + * Processes Live refresh step response, parses KI, checks the signature + */ + void live_refresh_ack(const ::crypto::secret_key & view_key_priv, + const ::crypto::public_key& out_key, + const std::shared_ptr<messages::monero::MoneroLiveRefreshStepAck> & ack, + ::cryptonote::keypair& in_ephemeral, + ::crypto::key_image& ki); } // Cold transaction signing @@ -153,6 +164,7 @@ namespace tx { std::string hash_addr(const MoneroAccountPublicAddress * addr, boost::optional<uint64_t> amount = boost::none, boost::optional<bool> is_subaddr = boost::none); std::string hash_addr(const std::string & spend_key, const std::string & view_key, boost::optional<uint64_t> amount = boost::none, boost::optional<bool> is_subaddr = boost::none); std::string hash_addr(const ::crypto::public_key * spend_key, const ::crypto::public_key * view_key, boost::optional<uint64_t> amount = boost::none, boost::optional<bool> is_subaddr = boost::none); + ::crypto::secret_key compute_enc_key(const ::crypto::secret_key & private_view_key, const std::string & aux, const std::string & salt); typedef boost::variant<rct::rangeSig, rct::Bulletproof> rsig_v; @@ -164,8 +176,8 @@ namespace tx { TsxData tsx_data; tx_construction_data tx_data; cryptonote::transaction tx; - bool in_memory; unsigned rsig_type; + int bp_version; std::vector<uint64_t> grouping_vct; std::shared_ptr<MoneroRsigData> rsig_param; size_t cur_input_idx; @@ -206,6 +218,7 @@ namespace tx { const unsigned_tx_set * m_unsigned_tx; hw::tx_aux_data * m_aux_data; + unsigned m_client_version; bool m_multisig; const tx_construction_data & cur_tx(){ @@ -215,6 +228,9 @@ namespace tx { void extract_payment_id(); void compute_integrated_indices(TsxData * tsx_data); + bool should_compute_bp_now() const; + void compute_bproof(messages::monero::MoneroTransactionRsigData & rsig_data); + void process_bproof(rct::Bulletproof & bproof); public: Signer(wallet_shim * wallet2, const unsigned_tx_set * unsigned_tx, size_t tx_idx = 0, hw::tx_aux_data * aux_data = nullptr); @@ -238,6 +254,9 @@ namespace tx { std::shared_ptr<messages::monero::MoneroTransactionSetOutputRequest> step_set_output(size_t idx); void step_set_output_ack(std::shared_ptr<const messages::monero::MoneroTransactionSetOutputAck> ack); + std::shared_ptr<messages::monero::MoneroTransactionSetOutputRequest> step_rsig(size_t idx); + void step_set_rsig_ack(std::shared_ptr<const messages::monero::MoneroTransactionSetOutputAck> ack); + std::shared_ptr<messages::monero::MoneroTransactionAllOutSetRequest> step_all_outs_set(); void step_all_outs_set_ack(std::shared_ptr<const messages::monero::MoneroTransactionAllOutSetAck> ack, hw::device &hwdev); @@ -249,8 +268,8 @@ namespace tx { std::string store_tx_aux_info(); - bool in_memory() const { - return m_ct.in_memory; + unsigned client_version() const { + return m_client_version; } bool is_simple() const { @@ -262,7 +281,7 @@ namespace tx { } bool is_req_bulletproof() const { - return m_ct.tx_data.use_bulletproofs; + return m_ct.tx_data.rct_config.range_proof_type != rct::RangeProofBorromean; } bool is_bulletproof() const { @@ -290,6 +309,18 @@ namespace tx { } }; + // TX Key decryption + void load_tx_key_data(hw::device_cold::tx_key_data_t & res, const std::string & data); + + std::shared_ptr<messages::monero::MoneroGetTxKeyRequest> get_tx_key( + const hw::device_cold::tx_key_data_t & tx_data); + + void get_tx_key_ack( + std::vector<::crypto::secret_key> & tx_keys, + const std::string & tx_prefix_hash, + const ::crypto::secret_key & view_key_priv, + std::shared_ptr<const messages::monero::MoneroGetTxKeyAck> ack + ); } } diff --git a/src/lmdb/CMakeLists.txt b/src/lmdb/CMakeLists.txt new file mode 100644 index 000000000..1f369f114 --- /dev/null +++ b/src/lmdb/CMakeLists.txt @@ -0,0 +1,33 @@ +# Copyright (c) 2014-2018, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set(lmdb_sources database.cpp error.cpp table.cpp value_stream.cpp) +set(lmdb_headers database.h error.h key_stream.h table.h transaction.h util.h value_stream.h) + +monero_add_library(lmdb_lib ${lmdb_sources} ${lmdb_headers}) +target_link_libraries(lmdb_lib common ${LMDB_LIBRARY}) diff --git a/src/lmdb/database.cpp b/src/lmdb/database.cpp new file mode 100644 index 000000000..c6b244671 --- /dev/null +++ b/src/lmdb/database.cpp @@ -0,0 +1,187 @@ +// Copyright (c) 2014-2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include "database.h" +#include "lmdb/error.h" +#include "lmdb/util.h" + +#ifdef _WIN32 +namespace +{ + constexpr const mdb_mode_t open_flags = 0; +} +#else +#include <sys/stat.h> + +namespace +{ + constexpr const mdb_mode_t open_flags = (S_IRUSR | S_IWUSR); +} +#endif + +namespace lmdb +{ + namespace + { + constexpr const std::size_t max_resize = 1 * 1024 * 1024 * 1024; // 1 GB + void acquire_context(context& ctx) noexcept + { + while (ctx.lock.test_and_set()); + ++(ctx.active); + ctx.lock.clear(); + } + + void release_context(context& ctx) noexcept + { + --(ctx.active); + } + } + + void release_read_txn::operator()(MDB_txn* ptr) const noexcept + { + if (ptr) + { + MDB_env* const env = mdb_txn_env(ptr); + abort_txn{}(ptr); + if (env) + { + context* ctx = reinterpret_cast<context*>(mdb_env_get_userctx(env)); + if (ctx) + release_context(*ctx); + } + } + } + + expect<environment> open_environment(const char* path, MDB_dbi max_dbs) noexcept + { + MONERO_PRECOND(path != nullptr); + + MDB_env* obj = nullptr; + MONERO_LMDB_CHECK(mdb_env_create(std::addressof(obj))); + environment out{obj}; + + MONERO_LMDB_CHECK(mdb_env_set_maxdbs(out.get(), max_dbs)); + MONERO_LMDB_CHECK(mdb_env_open(out.get(), path, 0, open_flags)); + return {std::move(out)}; + } + + expect<write_txn> database::do_create_txn(unsigned int flags) noexcept + { + MONERO_PRECOND(handle() != nullptr); + + for (unsigned attempts = 0; attempts < 3; ++attempts) + { + acquire_context(ctx); + + MDB_txn* txn = nullptr; + const int err = + mdb_txn_begin(handle(), nullptr, flags, &txn); + if (!err && txn != nullptr) + return write_txn{txn}; + + release_context(ctx); + if (err != MDB_MAP_RESIZED) + return {lmdb::error(err)}; + MONERO_CHECK(this->resize()); + } + return {lmdb::error(MDB_MAP_RESIZED)}; + } + + database::database(environment env) + : env(std::move(env)), ctx{{}, ATOMIC_FLAG_INIT} + { + if (handle()) + { + const int err = mdb_env_set_userctx(handle(), std::addressof(ctx)); + if (err) + MONERO_THROW(lmdb::error(err), "Failed to set user context"); + } + } + + database::~database() noexcept + { + while (ctx.active); + } + + expect<void> database::resize() noexcept + { + MONERO_PRECOND(handle() != nullptr); + + while (ctx.lock.test_and_set()); + while (ctx.active); + + MDB_envinfo info{}; + MONERO_LMDB_CHECK(mdb_env_info(handle(), &info)); + + const std::size_t resize = std::min(info.me_mapsize, max_resize); + const int err = mdb_env_set_mapsize(handle(), info.me_mapsize + resize); + ctx.lock.clear(); + if (err) + return {lmdb::error(err)}; + return success(); + } + + expect<read_txn> database::create_read_txn(suspended_txn txn) noexcept + { + if (txn) + { + acquire_context(ctx); + const int err = mdb_txn_renew(txn.get()); + if (err) + { + release_context(ctx); + return {lmdb::error(err)}; + } + return read_txn{txn.release()}; + } + auto new_txn = do_create_txn(MDB_RDONLY); + if (new_txn) + return read_txn{new_txn->release()}; + return new_txn.error(); + } + + expect<suspended_txn> database::reset_txn(read_txn txn) noexcept + { + MONERO_PRECOND(txn != nullptr); + mdb_txn_reset(txn.get()); + release_context(ctx); + return suspended_txn{txn.release()}; + } + + expect<write_txn> database::create_write_txn() noexcept + { + return do_create_txn(0); + } + + expect<void> database::commit(write_txn txn) noexcept + { + MONERO_PRECOND(txn != nullptr); + MONERO_LMDB_CHECK(mdb_txn_commit(txn.get())); + txn.release(); + release_context(ctx); + return success(); + } +} // lmdb diff --git a/src/lmdb/database.h b/src/lmdb/database.h new file mode 100644 index 000000000..269f8c8a1 --- /dev/null +++ b/src/lmdb/database.h @@ -0,0 +1,138 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include <atomic> +#include <cstddef> +#include <lmdb.h> +#include <memory> +#include <type_traits> + +#include "common/expect.h" +#include "lmdb/error.h" +#include "lmdb/transaction.h" + +namespace lmdb +{ + //! Closes LMDB environment handle. + struct close_env + { + void operator()(MDB_env* ptr) const noexcept + { + if (ptr) + mdb_env_close(ptr); + } + }; + + using environment = std::unique_ptr<MDB_env, close_env>; + + //! \return LMDB environment at `path` with a max of `max_dbs` tables. + expect<environment> open_environment(const char* path, MDB_dbi max_dbs) noexcept; + + //! Context given to LMDB. + struct context + { + std::atomic<std::size_t> active; + std::atomic_flag lock; + }; + + //! Manages a LMDB environment for safe memory-map resizing. Thread-safe. + class database + { + environment env; + context ctx; + + //! \return The LMDB environment associated with the object. + MDB_env* handle() const noexcept { return env.get(); } + + expect<write_txn> do_create_txn(unsigned int flags) noexcept; + + public: + database(environment env); + + database(database&&) = delete; + database(database const&) = delete; + + virtual ~database() noexcept; + + database& operator=(database&&) = delete; + database& operator=(database const&) = delete; + + /*! + Resize the memory map for the LMDB environment. Will block until + all reads/writes on the environment complete. + */ + expect<void> resize() noexcept; + + //! \return A read only LMDB transaction, reusing `txn` if provided. + expect<read_txn> create_read_txn(suspended_txn txn = nullptr) noexcept; + + //! \return `txn` after releasing context. + expect<suspended_txn> reset_txn(read_txn txn) noexcept; + + //! \return A read-write LMDB transaction. + expect<write_txn> create_write_txn() noexcept; + + //! Commit the read-write transaction. + expect<void> commit(write_txn txn) noexcept; + + /*! + Create a write transaction, pass it to `f`, then try to commit + the write if `f` succeeds. + + \tparam F must be callable with signature `expect<T>(MDB_txn&)`. + \param f must be re-startable if `lmdb::error(MDB_MAP_FULL)`. + + \return The result of calling `f`. + */ + template<typename F> + typename std::result_of<F(MDB_txn&)>::type try_write(F f, unsigned attempts = 3) + { + for (unsigned i = 0; i < attempts; ++i) + { + expect<write_txn> txn = create_write_txn(); + if (!txn) + return txn.error(); + + MONERO_PRECOND(*txn != nullptr); + const auto wrote = f(*(*txn)); + if (wrote) + { + MONERO_CHECK(commit(std::move(*txn))); + return wrote; + } + if (wrote != lmdb::error(MDB_MAP_FULL)) + return wrote; + + txn->reset(); + MONERO_CHECK(this->resize()); + } + return {lmdb::error(MDB_MAP_FULL)}; + } + }; +} // lmdb + diff --git a/src/lmdb/error.cpp b/src/lmdb/error.cpp new file mode 100644 index 000000000..359677064 --- /dev/null +++ b/src/lmdb/error.cpp @@ -0,0 +1,98 @@ +// Copyright (c) 2014-2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "error.h" + +#include <lmdb.h> +#include <string> + +namespace { + struct category final : std::error_category + { + virtual const char* name() const noexcept override final + { + return "lmdb::error_category()"; + } + + virtual std::string message(int value) const override final + { + char const* const msg = mdb_strerror(value); + if (msg) + return msg; + return "Unknown lmdb::error_category() value"; + } + + virtual std::error_condition default_error_condition(int value) const noexcept override final + { + switch (value) + { + case MDB_KEYEXIST: + case MDB_NOTFOUND: + break; // map to nothing generic + case MDB_PAGE_NOTFOUND: + case MDB_CORRUPTED: + return std::errc::state_not_recoverable; + case MDB_PANIC: + case MDB_VERSION_MISMATCH: + case MDB_INVALID: + break; // map to nothing generic + case MDB_MAP_FULL: + return std::errc::no_buffer_space; + case MDB_DBS_FULL: + break; // map to nothing generic + case MDB_READERS_FULL: + case MDB_TLS_FULL: + return std::errc::no_lock_available; + case MDB_TXN_FULL: + case MDB_CURSOR_FULL: + case MDB_PAGE_FULL: + case MDB_MAP_RESIZED: + break; // map to nothing generic + case MDB_INCOMPATIBLE: + return std::errc::invalid_argument; + case MDB_BAD_RSLOT: + case MDB_BAD_TXN: + case MDB_BAD_VALSIZE: + case MDB_BAD_DBI: + return std::errc::invalid_argument; + default: + return std::error_condition{value, std::generic_category()}; + } + return std::error_condition{value, *this}; + } + }; +} + +namespace lmdb +{ + std::error_category const& error_category() noexcept + { + static const category instance{}; + return instance; + } +} + diff --git a/src/lmdb/error.h b/src/lmdb/error.h new file mode 100644 index 000000000..2944adf78 --- /dev/null +++ b/src/lmdb/error.h @@ -0,0 +1,64 @@ +// Copyright (c) 2014-2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include <system_error> +#include <type_traits> + +//! Executes a LMDB command, and returns errors via `lmdb::error` enum. +#define MONERO_LMDB_CHECK(...) \ + do \ + { \ + const int err = __VA_ARGS__ ; \ + if (err) \ + return {lmdb::error(err)}; \ + } while (0) + +namespace lmdb +{ + //! Tracks LMDB error codes. + enum class error : int + { + // 0 is reserved for no error, as per expect<T> + // All other errors are the values reported by LMDB + }; + + std::error_category const& error_category() noexcept; + + inline std::error_code make_error_code(error value) noexcept + { + return std::error_code{int(value), error_category()}; + } +} + +namespace std +{ + template<> + struct is_error_code_enum<::lmdb::error> + : true_type + {}; +} diff --git a/src/lmdb/key_stream.h b/src/lmdb/key_stream.h new file mode 100644 index 000000000..40434d3a1 --- /dev/null +++ b/src/lmdb/key_stream.h @@ -0,0 +1,264 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include <boost/range/iterator_range.hpp> +#include <cstdint> +#include <cstring> +#include <iterator> +#include <lmdb.h> +#include <utility> + +#include "lmdb/value_stream.h" +#include "span.h" + +namespace lmdb +{ + + /*! + An InputIterator for a fixed-sized LMDB key and value. `operator++` + iterates over keys. + + \tparam K Key type in database records. + \tparam V Value type in database records. + + \note This meets requirements for an InputIterator only. The iterator + can only be incremented and dereferenced. All copies of an iterator + share the same LMDB cursor, and therefore incrementing any copy will + change the cursor state for all (incrementing an iterator will + invalidate all prior copies of the iterator). Usage is identical + to `std::istream_iterator`. + */ + template<typename K, typename V> + class key_iterator + { + MDB_cursor* cur; + epee::span<const std::uint8_t> key; + + void increment() + { + // MDB_NEXT_MULTIPLE doesn't work if only one value is stored :/ + if (cur) + key = lmdb::stream::get(*cur, MDB_NEXT_NODUP, sizeof(K), sizeof(V)).first; + } + + public: + using value_type = std::pair<K, boost::iterator_range<value_iterator<V>>>; + using reference = value_type; + using pointer = void; + using difference_type = std::size_t; + using iterator_category = std::input_iterator_tag; + + //! Construct an "end" iterator. + key_iterator() noexcept + : cur(nullptr), key() + {} + + /*! + \param cur Iterate over keys starting at this cursor position. + \throw std::system_error if unexpected LMDB error. This can happen + if `cur` is invalid. + */ + key_iterator(MDB_cursor* cur) + : cur(cur), key() + { + if (cur) + key = lmdb::stream::get(*cur, MDB_GET_CURRENT, sizeof(K), sizeof(V)).first; + } + + //! \return True if `this` is one-past the last key. + bool is_end() const noexcept { return key.empty(); } + + //! \return True iff `rhs` is referencing `this` key. + bool equal(key_iterator const& rhs) const noexcept + { + return + (key.empty() && rhs.key.empty()) || + key.data() == rhs.key.data(); + } + + /*! + Moves iterator to next key or end. Invalidates all prior copies of + the iterator. + */ + key_iterator& operator++() + { + increment(); + return *this; + } + + /*! + Moves iterator to next key or end. + + \return A copy that is already invalidated, ignore + */ + key_iterator operator++(int) + { + key_iterator out{*this}; + increment(); + return out; + } + + //! \pre `!is_end()` \return {current key, current value range} + value_type operator*() const + { + return {get_key(), make_value_range()}; + } + + //! \pre `!is_end()` \return Current key + K get_key() const noexcept + { + assert(!is_end()); + K out; + std::memcpy(std::addressof(out), key.data(), sizeof(out)); + return out; + } + + /*! + Return a C++ iterator over database values from current cursor + position that will reach `.is_end()` after the last duplicate key + record. Calling `make_iterator()` will return an iterator whose + `operator*` will return an entire value (`V`). + `make_iterator<MONERO_FIELD(account, id)>()` will return an + iterator whose `operator*` will return a `decltype(account.id)` + object - the other fields in the struct `account` are never copied + from the database. + + \throw std::system_error if LMDB has unexpected errors. + \return C++ iterator starting at current cursor position. + */ + template<typename T = V, typename F = T, std::size_t offset = 0> + value_iterator<T, F, offset> make_value_iterator() const + { + static_assert(std::is_same<T, V>(), "bad MONERO_FIELD usage?"); + return {cur}; + } + + /*! + Return a range from current cursor position until last duplicate + key record. Useful in for-each range loops or in templated code + expecting a range of elements. Calling `make_range()` will return + a range of `T` objects. `make_range<MONERO_FIELD(account, id)>()` + will return a range of `decltype(account.id)` objects - the other + fields in the struct `account` are never copied from the database. + + \throw std::system_error if LMDB has unexpected errors. + \return An InputIterator range over values at cursor position. + */ + template<typename T = V, typename F = T, std::size_t offset = 0> + boost::iterator_range<value_iterator<T, F, offset>> make_value_range() const + { + return {make_value_iterator<T, F, offset>(), value_iterator<T, F, offset>{}}; + } + }; + + /*! + C++ wrapper for a LMDB read-only cursor on a fixed-sized key `K` and + value `V`. + + \tparam K key type being stored by each record. + \tparam V value type being stored by each record. + \tparam D cleanup functor for the cursor; usually unique per db/table. + */ + template<typename K, typename V, typename D> + class key_stream + { + std::unique_ptr<MDB_cursor, D> cur; + public: + + //! Take ownership of `cur` without changing position. `nullptr` valid. + explicit key_stream(std::unique_ptr<MDB_cursor, D> cur) + : cur(std::move(cur)) + {} + + key_stream(key_stream&&) = default; + key_stream(key_stream const&) = delete; + ~key_stream() = default; + key_stream& operator=(key_stream&&) = default; + key_stream& operator=(key_stream const&) = delete; + + /*! + Give up ownership of the cursor. `make_iterator()` and + `make_range()` can still be invoked, but return the empty set. + + \return Currently owned LMDB cursor. + */ + std::unique_ptr<MDB_cursor, D> give_cursor() noexcept + { + return {std::move(cur)}; + } + + /*! + Place the stream back at the first key/value. Newly created + iterators will start at the first value again. + + \note Invalidates all current iterators, including those created + with `make_iterator` or `make_range`. Also invalidates all + `value_iterator`s created with `key_iterator`. + */ + void reset() + { + if (cur) + lmdb::stream::get(*cur, MDB_FIRST, 0, 0); + } + + /*! + \throw std::system_error if LMDB has unexpected errors. + \return C++ iterator over database keys from current cursor + position that will reach `.is_end()` after the last key. + */ + key_iterator<K, V> make_iterator() const + { + return {cur.get()}; + } + + /*! + \throw std::system_error if LMDB has unexpected errors. + \return Range from current cursor position until last key record. + Useful in for-each range loops or in templated code + */ + boost::iterator_range<key_iterator<K, V>> make_range() const + { + return {make_iterator(), key_iterator<K, V>{}}; + } + }; + + template<typename K, typename V> + inline + bool operator==(key_iterator<K, V> const& lhs, key_iterator<K, V> const& rhs) noexcept + { + return lhs.equal(rhs); + } + + template<typename K, typename V> + inline + bool operator!=(key_iterator<K, V> const& lhs, key_iterator<K, V> const& rhs) noexcept + { + return !lhs.equal(rhs); + } +} // lmdb + diff --git a/src/lmdb/table.cpp b/src/lmdb/table.cpp new file mode 100644 index 000000000..0818b74e6 --- /dev/null +++ b/src/lmdb/table.cpp @@ -0,0 +1,43 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include "table.h" + +namespace lmdb +{ + expect<MDB_dbi> table::open(MDB_txn& write_txn) const noexcept + { + MONERO_PRECOND(name != nullptr); + + MDB_dbi out; + MONERO_LMDB_CHECK(mdb_dbi_open(&write_txn, name, flags, &out)); + if (key_cmp && !(flags & MDB_INTEGERKEY)) + MONERO_LMDB_CHECK(mdb_set_compare(&write_txn, out, key_cmp)); + if (value_cmp && !(flags & MDB_INTEGERDUP)) + MONERO_LMDB_CHECK(mdb_set_dupsort(&write_txn, out, value_cmp)); + return out; + } +} diff --git a/src/lmdb/table.h b/src/lmdb/table.h new file mode 100644 index 000000000..41a3de296 --- /dev/null +++ b/src/lmdb/table.h @@ -0,0 +1,120 @@ +#pragma once + +#include <utility> + +#include "common/expect.h" +#include "lmdb/error.h" +#include "lmdb/key_stream.h" +#include "lmdb/util.h" +#include "lmdb/value_stream.h" + +namespace lmdb +{ + //! Helper for grouping typical LMDB DBI options. + struct table + { + char const* const name; + const unsigned flags; + MDB_cmp_func const* const key_cmp; + MDB_cmp_func const* const value_cmp; + + //! \pre `name != nullptr` \return Open table. + expect<MDB_dbi> open(MDB_txn& write_txn) const noexcept; + }; + + //! Helper for grouping typical LMDB DBI options when key and value are fixed types. + template<typename K, typename V> + struct basic_table : table + { + using key_type = K; + using value_type = V; + + //! \return Additional LMDB flags based on `flags` value. + static constexpr unsigned compute_flags(const unsigned flags) noexcept + { + return flags | ((flags & MDB_DUPSORT) ? MDB_DUPFIXED : 0); + } + + constexpr explicit basic_table(const char* name, unsigned flags = 0, MDB_cmp_func value_cmp = nullptr) noexcept + : table{name, compute_flags(flags), &lmdb::less<lmdb::native_type<K>>, value_cmp} + {} + + /*! + \tparam U must be same as `V`; used for sanity checking. + \tparam F is the type within `U` that is being extracted. + \tparam offset to `F` within `U`. + + \note If using `F` and `offset` to retrieve a specific field, use + `MONERO_FIELD` macro in `src/lmdb/util.h` which calculates the + offset automatically. + + \return Value of type `F` at `offset` within `value` which has + type `U`. + */ + template<typename U, typename F = U, std::size_t offset = 0> + static expect<F> get_value(MDB_val value) noexcept + { + static_assert(std::is_same<U, V>(), "bad MONERO_FIELD?"); + static_assert(std::is_pod<F>(), "F must be POD"); + static_assert(sizeof(F) + offset <= sizeof(U), "bad field type and/or offset"); + + if (value.mv_size != sizeof(U)) + return {lmdb::error(MDB_BAD_VALSIZE)}; + + F out; + std::memcpy(std::addressof(out), static_cast<char*>(value.mv_data) + offset, sizeof(out)); + return out; + } + + /*! + \pre `cur != nullptr`. + \param cur Active cursor on table. Returned in object on success, + otherwise destroyed. + \return A handle to the first key/value in the table linked + to `cur` or an empty `key_stream`. + */ + template<typename D> + expect<key_stream<K, V, D>> + static get_key_stream(std::unique_ptr<MDB_cursor, D> cur) noexcept + { + MONERO_PRECOND(cur != nullptr); + + MDB_val key; + MDB_val value; + const int err = mdb_cursor_get(cur.get(), &key, &value, MDB_FIRST); + if (err) + { + if (err != MDB_NOTFOUND) + return {lmdb::error(err)}; + cur.reset(); // return empty set + } + return key_stream<K, V, D>{std::move(cur)}; + } + + /*! + \pre `cur != nullptr`. + \param cur Active cursor on table. Returned in object on success, + otherwise destroyed. + \return A handle to the first value at `key` in the table linked + to `cur` or an empty `value_stream`. + */ + template<typename D> + expect<value_stream<V, D>> + static get_value_stream(K const& key, std::unique_ptr<MDB_cursor, D> cur) noexcept + { + MONERO_PRECOND(cur != nullptr); + + MDB_val key_bytes = lmdb::to_val(key); + MDB_val value; + const int err = mdb_cursor_get(cur.get(), &key_bytes, &value, MDB_SET); + if (err) + { + if (err != MDB_NOTFOUND) + return {lmdb::error(err)}; + cur.reset(); // return empty set + } + return value_stream<V, D>{std::move(cur)}; + } + }; +} // lmdb + diff --git a/src/lmdb/transaction.h b/src/lmdb/transaction.h new file mode 100644 index 000000000..cdd80696c --- /dev/null +++ b/src/lmdb/transaction.h @@ -0,0 +1,95 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include <lmdb.h> +#include <memory> + +#include "lmdb/error.h" + +//! Uses C++ type system to differentiate between cursors +#define MONERO_CURSOR(name) \ + struct close_ ## name : ::lmdb::close_cursor {}; \ + using name = std::unique_ptr< MDB_cursor, close_ ## name >; + +namespace lmdb +{ + struct abort_txn + { + void operator()(MDB_txn* ptr) const noexcept + { + if (ptr) + mdb_txn_abort(ptr); + } + }; + + /*! + Only valid if used via `create_read_txn()`. Decrements active count in + associated `context`, and aborts a LMDB transaction (`mdb_txn_abort`). + */ + struct release_read_txn + { + void operator()(MDB_txn* ptr) const noexcept; + // implementation in database.cpp + }; + + /*! + Only valid if used via `create_write_txn()`. Decrements active count in + associated `context`, and aborts a LMDB transaction (`mdb_txn_abort`). + */ + struct abort_write_txn + { + void operator()(MDB_txn* ptr) const noexcept + { + release_read_txn{}(ptr); + } + }; + + struct close_cursor + { + void operator()(MDB_cursor* ptr) const noexcept + { + if (ptr) + mdb_cursor_close(ptr); + } + }; + + template<typename D> + inline expect<std::unique_ptr<MDB_cursor, D>> + open_cursor(MDB_txn& txn, MDB_dbi tbl) noexcept + { + MDB_cursor* cur = nullptr; + MONERO_LMDB_CHECK(mdb_cursor_open(&txn, tbl, &cur)); + return std::unique_ptr<MDB_cursor, D>{cur}; + } + + // The below use the C++ type system to designate `MDB_txn` status. + + using suspended_txn = std::unique_ptr<MDB_txn, abort_txn>; + using read_txn = std::unique_ptr<MDB_txn, release_read_txn>; + using write_txn = std::unique_ptr<MDB_txn, abort_write_txn>; +} // lmdb diff --git a/src/lmdb/util.h b/src/lmdb/util.h new file mode 100644 index 000000000..50162b7c8 --- /dev/null +++ b/src/lmdb/util.h @@ -0,0 +1,149 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include <cstddef> +#include <cstring> +#include <lmdb.h> +#include <type_traits> +#include <utility> + +#include "span.h" + +/*! Calculates types and offset of struct field. Use in template arguments for + `table::get_value`, `value_iterator::get_value`, + `value_stream::make_iterator`, or `value_stream::make_range`. */ +#define MONERO_FIELD(obj, field) \ + obj , decltype(std::declval<obj>().field) , offsetof(obj, field) + +//! Expands to `lmdb::less` for the value `field` within `obj`. +#define MONERO_SORT_BY(obj, field) \ + &::lmdb::less< \ + lmdb::native_type<decltype(std::declval<obj>().field)>, \ + offsetof(obj, field) \ + > + +//! Expands to `lmdb::compare` for the value `field` within `obj`. +#define MONERO_COMPARE(obj, field) \ + &::lmdb::compare< \ + decltype(std::declval<obj>().field), \ + offsetof(obj, field) \ + > + +namespace lmdb +{ + //! Prevent instantiation of `std::underlying_type<T>` when `T` is not enum. + template<typename T> + struct identity + { + using type = T; + }; + + /*! + Get the native type for enums, or return `T` unchanged. Useful for + merging generated machine code for templated functions that use enums + with identical size-widths without relying on aggressive identical + comdat folding (ICF) support in linker. So with enum defintion + `enum class enum_foo : unsigned long {};` will always yield + `assert(&func_foo<unsigned long> == &func_foo<native_type<enum_foo>>)`. + */ + template<typename T> + using native_type = typename std::conditional< + std::is_enum<T>::value, std::underlying_type<T>, identity<T> + >::type::type; + + //! \return `value` as its native type. + template<typename T, typename U = typename std::underlying_type<T>::type> + inline constexpr U to_native(T value) noexcept + { + return U(value); + } + + //! \return `value` bytes in a LMDB `MDB_val` object. + template<typename T> + inline MDB_val to_val(T&& value) noexcept + { + // lmdb does not touch user data, so const_cast is acceptable + static_assert(!std::is_rvalue_reference<T&&>(), "cannot use temporary value"); + void const* const temp = reinterpret_cast<void const*>(std::addressof(value)); + return MDB_val{sizeof(value), const_cast<void*>(temp)}; + } + + //! \return A span over the same chunk of memory as `value`. + inline constexpr epee::span<const std::uint8_t> to_byte_span(MDB_val value) noexcept + { + return {static_cast<const std::uint8_t*>(value.mv_data), value.mv_size}; + } + + /*! + A LMDB comparison function that uses `operator<`. + + \tparam T has a defined `operator<` . + \tparam offset to `T` within the value. + + \return -1 if `left < right`, 1 if `right < left`, and 0 otherwise. + */ + template<typename T, std::size_t offset = 0> + inline int less(MDB_val const* left, MDB_val const* right) noexcept + { + if (!left || !right || left->mv_size < sizeof(T) + offset || right->mv_size < sizeof(T) + offset) + { + assert("invalid use of custom comparison" == 0); + return -1; + } + + T left_val; + T right_val; + std::memcpy(std::addressof(left_val), static_cast<char*>(left->mv_data) + offset, sizeof(T)); + std::memcpy(std::addressof(right_val), static_cast<char*>(right->mv_data) + offset, sizeof(T)); + return left_val < right_val ? -1 : bool(right_val < left_val); + } + + /*! + A LMDB comparison function that uses `std::memcmp`. + + \toaram T is `!epee::has_padding` + \tparam offset to `T` within the value. + + \return The result of `std::memcmp` over the value. + */ + template<typename T, std::size_t offset = 0> + inline int compare(MDB_val const* left, MDB_val const* right) noexcept + { + static_assert(!epee::has_padding<T>(), "memcmp will not work"); + if (!left || !right || left->mv_size < sizeof(T) + offset || right->mv_size < sizeof(T) + offset) + { + assert("invalid use of custom comparison" == 0); + return -1; + } + return std::memcmp( + static_cast<char*>(left->mv_data) + offset, + static_cast<char*>(right->mv_data) + offset, + sizeof(T) + ); + } +} // lmdb diff --git a/src/lmdb/value_stream.cpp b/src/lmdb/value_stream.cpp new file mode 100644 index 000000000..1024deb06 --- /dev/null +++ b/src/lmdb/value_stream.cpp @@ -0,0 +1,74 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include "value_stream.h" + +#include <stdexcept> + +#include "common/expect.h" +#include "lmdb/error.h" +#include "lmdb/util.h" + +namespace lmdb +{ + namespace stream + { + std::size_t count(MDB_cursor* cur) + { + std::size_t out = 0; + if (cur) + { + const int rc = mdb_cursor_count(cur, &out); + if (rc) + MONERO_THROW(lmdb::error(rc), "mdb_cursor_count"); + } + return out; + } + + std::pair<epee::span<const std::uint8_t>, epee::span<const std::uint8_t>> + get(MDB_cursor& cur, MDB_cursor_op op, std::size_t key, std::size_t value) + { + MDB_val key_bytes{}; + MDB_val value_bytes{}; + const int rc = mdb_cursor_get(&cur, &key_bytes, &value_bytes, op); + if (rc) + { + if (rc == MDB_NOTFOUND) + return {}; + MONERO_THROW(lmdb::error(rc), "mdb_cursor_get"); + } + + if (key && key != key_bytes.mv_size) + MONERO_THROW(lmdb::error(MDB_BAD_VALSIZE), "mdb_cursor_get key"); + + if (value && (value_bytes.mv_size % value != 0 || value_bytes.mv_size == 0)) + MONERO_THROW(lmdb::error(MDB_BAD_VALSIZE), "mdb_cursor_get value"); + + return {lmdb::to_byte_span(key_bytes), lmdb::to_byte_span(value_bytes)}; + } + } +} + diff --git a/src/lmdb/value_stream.h b/src/lmdb/value_stream.h new file mode 100644 index 000000000..c9977221f --- /dev/null +++ b/src/lmdb/value_stream.h @@ -0,0 +1,287 @@ +// Copyright (c) 2018, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once + +#include <boost/range/iterator_range.hpp> +#include <cstdint> +#include <cstring> +#include <iterator> +#include <lmdb.h> +#include <utility> + +#include "span.h" + +namespace lmdb +{ + namespace stream + { + /* + \throw std::system_error if unexpected LMDB error. + \return 0 if `cur == nullptr`, otherwise count of values at current key. + */ + std::size_t count(MDB_cursor* cur); + + /*! + Calls `mdb_cursor_get` and does some error checking. + + \param cur is given to `mdb_cursor_get` without modification. + \param op is passed to `mdb_cursor_get` without modification. + \param key expected key size or 0 to skip key size check. + \param value expected value size or 0 to skip value size check. + + \throw std::system_error if `key != 0` and `key_.mv_size != key`. + \throw std::system_error if `value != 0` and `value_.mv_size != value`. + \throw std::system_error if `mdb_cursor_get` returns any error + other than `MDB_NOTFOUND`. + + \return {key bytes, value bytes} or two empty spans if `MDB_NOTFOUND`. + */ + std::pair<epee::span<const std::uint8_t>, epee::span<const std::uint8_t>> + get(MDB_cursor& cur, MDB_cursor_op op, std::size_t key, std::size_t value); + } + + /*! + An InputIterator for a fixed-sized LMDB value at a specific key. + + \tparam T The value type at the specific key. + \tparam F The value type being returned when dereferenced. + \tparam offset to `F` within `T`. + + \note This meets requirements for an InputIterator only. The iterator + can only be incremented and dereferenced. All copies of an iterator + share the same LMDB cursor, and therefore incrementing any copy will + change the cursor state for all (incrementing an iterator will + invalidate all prior copies of the iterator). Usage is identical + to `std::istream_iterator`. + */ + template<typename T, typename F = T, std::size_t offset = 0> + class value_iterator + { + MDB_cursor* cur; + epee::span<const std::uint8_t> values; + + void increment() + { + values.remove_prefix(sizeof(T)); + if (values.empty() && cur) + values = lmdb::stream::get(*cur, MDB_NEXT_DUP, 0, sizeof(T)).second; + } + + public: + using value_type = F; + using reference = value_type; + using pointer = void; + using difference_type = std::size_t; + using iterator_category = std::input_iterator_tag; + + //! Construct an "end" iterator. + value_iterator() noexcept + : cur(nullptr), values() + {} + + /*! + \param cur Iterate over values starting at this cursor position. + \throw std::system_error if unexpected LMDB error. This can happen + if `cur` is invalid. + */ + value_iterator(MDB_cursor* cur) + : cur(cur), values() + { + if (cur) + values = lmdb::stream::get(*cur, MDB_GET_CURRENT, 0, sizeof(T)).second; + } + + value_iterator(value_iterator const&) = default; + ~value_iterator() = default; + value_iterator& operator=(value_iterator const&) = default; + + //! \return True if `this` is one-past the last value. + bool is_end() const noexcept { return values.empty(); } + + //! \return True iff `rhs` is referencing `this` value. + bool equal(value_iterator const& rhs) const noexcept + { + return + (values.empty() && rhs.values.empty()) || + values.data() == rhs.values.data(); + } + + //! Invalidates all prior copies of the iterator. + value_iterator& operator++() + { + increment(); + return *this; + } + + //! \return A copy that is already invalidated, ignore + value_iterator operator++(int) + { + value_iterator out{*this}; + increment(); + return out; + } + + /*! + Get a specific field within `F`. Default behavior is to return + the entirety of `U`, despite the filtering logic of `operator*`. + + \pre `!is_end()` + + \tparam U must match `T`, used for `MONERO_FIELD` sanity checking. + \tparam G field type to extract from the value + \tparam uoffset to `G` type, or `0` when `std::is_same<U, G>()`. + + \return The field `G`, at `uoffset` within `U`. + */ + template<typename U, typename G = U, std::size_t uoffset = 0> + G get_value() const noexcept + { + static_assert(std::is_same<U, T>(), "bad MONERO_FIELD usage?"); + static_assert(std::is_pod<U>(), "value type must be pod"); + static_assert(std::is_pod<G>(), "field type must be pod"); + static_assert(sizeof(G) + uoffset <= sizeof(U), "bad field and/or offset"); + assert(sizeof(G) + uoffset <= values.size()); + assert(!is_end()); + + G value; + std::memcpy(std::addressof(value), values.data() + uoffset, sizeof(value)); + return value; + } + + //! \pre `!is_end()` \return The field `F`, at `offset`, within `T`. + value_type operator*() const noexcept { return get_value<T, F, offset>(); } + }; + + /*! + C++ wrapper for a LMDB read-only cursor on a fixed-sized value `T`. + + \tparam T value type being stored by each record. + \tparam D cleanup functor for the cursor; usually unique per db/table. + */ + template<typename T, typename D> + class value_stream + { + std::unique_ptr<MDB_cursor, D> cur; + public: + + //! Take ownership of `cur` without changing position. `nullptr` valid. + explicit value_stream(std::unique_ptr<MDB_cursor, D> cur) + : cur(std::move(cur)) + {} + + value_stream(value_stream&&) = default; + value_stream(value_stream const&) = delete; + ~value_stream() = default; + value_stream& operator=(value_stream&&) = default; + value_stream& operator=(value_stream const&) = delete; + + /*! + Give up ownership of the cursor. `count()`, `make_iterator()` and + `make_range()` can still be invoked, but return the empty set. + + \return Currently owned LMDB cursor. + */ + std::unique_ptr<MDB_cursor, D> give_cursor() noexcept + { + return {std::move(cur)}; + } + + /*! + Place the stream back at the first value. Newly created iterators + will start at the first value again. + + \note Invalidates all current iterators from `this`, including + those created with `make_iterator` or `make_range`. + */ + void reset() + { + if (cur) + lmdb::stream::get(*cur, MDB_FIRST_DUP, 0, 0); + } + + /*! + \throw std::system_error if LMDB has unexpected errors. + \return Number of values at this key. + */ + std::size_t count() const + { + return lmdb::stream::count(cur.get()); + } + + /*! + Return a C++ iterator over database values from current cursor + position that will reach `.is_end()` after the last duplicate key + record. Calling `make_iterator()` will return an iterator whose + `operator*` will return entire value (`T`). + `make_iterator<MONERO_FIELD(account, id)>()` will return an + iterator whose `operator*` will return a `decltype(account.id)` + object - the other fields in the struct `account` are never copied + from the database. + + \throw std::system_error if LMDB has unexpected errors. + \return C++ iterator starting at current cursor position. + */ + template<typename U = T, typename F = U, std::size_t offset = 0> + value_iterator<U, F, offset> make_iterator() const + { + static_assert(std::is_same<U, T>(), "was MONERO_FIELD used with wrong type?"); + return {cur.get()}; + } + + /*! + Return a range from current cursor position until last duplicate + key record. Useful in for-each range loops or in templated code + expecting a range of elements. Calling `make_range()` will return + a range of `T` objects. `make_range<MONERO_FIELD(account, id)>()` + will return a range of `decltype(account.id)` objects - the other + fields in the struct `account` are never copied from the database. + + \throw std::system_error if LMDB has unexpected errors. + \return An InputIterator range over values at cursor position. + */ + template<typename U = T, typename F = U, std::size_t offset = 0> + boost::iterator_range<value_iterator<U, F, offset>> make_range() const + { + return {make_iterator<U, F, offset>(), value_iterator<U, F, offset>{}}; + } + }; + + template<typename T, typename F, std::size_t offset> + inline + bool operator==(value_iterator<T, F, offset> const& lhs, value_iterator<T, F, offset> const& rhs) noexcept + { + return lhs.equal(rhs); + } + + template<typename T, typename F, std::size_t offset> + inline + bool operator!=(value_iterator<T, F, offset> const& lhs, value_iterator<T, F, offset> const& rhs) noexcept + { + return !lhs.equal(rhs); + } +} // lmdb + diff --git a/src/mnemonics/electrum-words.cpp b/src/mnemonics/electrum-words.cpp index 48c9ab1ba..2dd40cc9a 100644 --- a/src/mnemonics/electrum-words.cpp +++ b/src/mnemonics/electrum-words.cpp @@ -340,9 +340,7 @@ namespace crypto const size_t expected = len * 3 / 32; if (seed.size() == expected/2) { - dst += ' '; // if electrum 12-word seed, duplicate - dst += dst; // if electrum 12-word seed, duplicate - dst.pop_back(); // trailing space + dst.append(dst.data(), dst.size()); // if electrum 12-word seed, duplicate } } diff --git a/src/net/CMakeLists.txt b/src/net/CMakeLists.txt index 8a3ee9e6f..738f858f0 100644 --- a/src/net/CMakeLists.txt +++ b/src/net/CMakeLists.txt @@ -26,8 +26,8 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -set(net_sources error.cpp parse.cpp socks.cpp tor_address.cpp i2p_address.cpp) -set(net_headers error.h parse.h socks.h tor_address.h i2p_address.h) +set(net_sources error.cpp i2p_address.cpp parse.cpp socks.cpp socks_connect.cpp tor_address.cpp) +set(net_headers error.h i2p_address.h parse.h socks.h socks_connect.h tor_address.h) monero_add_library(net ${net_sources} ${net_headers}) target_link_libraries(net common epee ${Boost_ASIO_LIBRARY}) diff --git a/src/net/socks.cpp b/src/net/socks.cpp index 53154369b..5a27e16f4 100644 --- a/src/net/socks.cpp +++ b/src/net/socks.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2018, The Monero Project +// Copyright (c) 2018-2019, The Monero Project // // All rights reserved. // @@ -193,7 +193,7 @@ namespace socks else if (bytes < self.buffer().size()) self.done(socks::error::bad_write, std::move(self_)); else - boost::asio::async_read(self.proxy_, get_buffer(self), completed{std::move(self_)}); + boost::asio::async_read(self.proxy_, get_buffer(self), self.strand_.wrap(completed{std::move(self_)})); } } }; @@ -215,13 +215,13 @@ namespace socks if (error) self.done(error, std::move(self_)); else - boost::asio::async_write(self.proxy_, get_buffer(self), read{std::move(self_)}); + boost::asio::async_write(self.proxy_, get_buffer(self), self.strand_.wrap(read{std::move(self_)})); } } }; client::client(stream_type::socket&& proxy, socks::version ver) - : proxy_(std::move(proxy)), buffer_size_(0), buffer_(), ver_(ver) + : proxy_(std::move(proxy)), strand_(GET_IO_SERVICE(proxy_)), buffer_size_(0), buffer_(), ver_(ver) {} client::~client() {} @@ -296,7 +296,7 @@ namespace socks if (self && !self->buffer().empty()) { client& alias = *self; - alias.proxy_.async_connect(proxy_address, write{std::move(self)}); + alias.proxy_.async_connect(proxy_address, alias.strand_.wrap(write{std::move(self)})); return true; } return false; @@ -307,10 +307,26 @@ namespace socks if (self && !self->buffer().empty()) { client& alias = *self; - boost::asio::async_write(alias.proxy_, write::get_buffer(alias), read{std::move(self)}); + boost::asio::async_write(alias.proxy_, write::get_buffer(alias), alias.strand_.wrap(read{std::move(self)})); return true; } return false; } + + void client::async_close::operator()(boost::system::error_code error) + { + if (self_ && error != boost::system::errc::operation_canceled) + { + const std::shared_ptr<client> self = std::move(self_); + self->strand_.dispatch([self] () + { + if (self && self->proxy_.is_open()) + { + self->proxy_.shutdown(boost::asio::ip::tcp::socket::shutdown_both); + self->proxy_.close(); + } + }); + } + } } // socks } // net diff --git a/src/net/socks.h b/src/net/socks.h index 825937792..4d1d34e9e 100644 --- a/src/net/socks.h +++ b/src/net/socks.h @@ -1,4 +1,4 @@ -// Copyright (c) 2018, The Monero Project +// Copyright (c) 2018-2019, The Monero Project // // All rights reserved. // @@ -31,6 +31,7 @@ #include <cstdint> #include <boost/asio/ip/tcp.hpp> #include <boost/asio/io_service.hpp> +#include <boost/asio/strand.hpp> #include <boost/system/error_code.hpp> #include <boost/type_traits/integral_constant.hpp> #include <boost/utility/string_ref.hpp> @@ -92,6 +93,7 @@ namespace socks class client { boost::asio::ip::tcp::socket proxy_; + boost::asio::io_service::strand strand_; std::uint16_t buffer_size_; std::uint8_t buffer_[1024]; socks::version ver_; @@ -168,6 +170,8 @@ namespace socks \note Must use one of the `self->set_*_command` calls before using this function. + \note Only `async_close` can be invoked on `self` until the `done` + callback is invoked. \param self ownership of object is given to function. \param proxy_address of the socks server. @@ -182,11 +186,21 @@ namespace socks \note Must use one of the `self->set_*_command` calls before using the function. + \note Only `async_close` can be invoked on `self` until the `done` + callback is invoked. \param self ownership of object is given to function. \return False if `self->buffer().empty()` (no command set). */ static bool send(std::shared_ptr<client> self); + + /*! Callback for closing socket. Thread-safe with `*send` functions; + never blocks (uses strands). */ + struct async_close + { + std::shared_ptr<client> self_; + void operator()(boost::system::error_code error = boost::system::error_code{}); + }; }; template<typename Handler> diff --git a/src/net/socks_connect.cpp b/src/net/socks_connect.cpp new file mode 100644 index 000000000..a5557f6f8 --- /dev/null +++ b/src/net/socks_connect.cpp @@ -0,0 +1,90 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "socks_connect.h" + +#include <boost/system/error_code.hpp> +#include <boost/system/system_error.hpp> +#include <cstdint> +#include <memory> +#include <system_error> + +#include "net/error.h" +#include "net/net_utils_base.h" +#include "net/socks.h" +#include "string_tools.h" + +namespace net +{ +namespace socks +{ + boost::unique_future<boost::asio::ip::tcp::socket> + connector::operator()(const std::string& remote_host, const std::string& remote_port, boost::asio::steady_timer& timeout) const + { + struct future_socket + { + boost::promise<boost::asio::ip::tcp::socket> result_; + + void operator()(boost::system::error_code error, boost::asio::ip::tcp::socket&& socket) + { + if (error) + result_.set_exception(boost::system::system_error{error}); + else + result_.set_value(std::move(socket)); + } + }; + + boost::unique_future<boost::asio::ip::tcp::socket> out{}; + { + std::uint16_t port = 0; + if (!epee::string_tools::get_xtype_from_string(port, remote_port)) + throw std::system_error{net::error::invalid_port, "Remote port for socks proxy"}; + + bool is_set = false; + std::uint32_t ip_address = 0; + boost::promise<boost::asio::ip::tcp::socket> result{}; + out = result.get_future(); + const auto proxy = net::socks::make_connect_client( + boost::asio::ip::tcp::socket{GET_IO_SERVICE(timeout)}, net::socks::version::v4a, future_socket{std::move(result)} + ); + + if (epee::string_tools::get_ip_int32_from_string(ip_address, remote_host)) + is_set = proxy->set_connect_command(epee::net_utils::ipv4_network_address{ip_address, port}); + else + is_set = proxy->set_connect_command(remote_host, port); + + if (!is_set || !net::socks::client::connect_and_send(proxy, proxy_address)) + throw std::system_error{net::error::invalid_host, "Address for socks proxy"}; + + timeout.async_wait(net::socks::client::async_close{std::move(proxy)}); + } + + return out; + } +} // socks +} // net diff --git a/src/net/socks_connect.h b/src/net/socks_connect.h new file mode 100644 index 000000000..44b0fa2b3 --- /dev/null +++ b/src/net/socks_connect.h @@ -0,0 +1,55 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <boost/asio/ip/tcp.hpp> +#include <boost/asio/steady_timer.hpp> +#include <boost/thread/future.hpp> +#include <string> + +namespace net +{ +namespace socks +{ + //! Primarily for use with `epee::net_utils::http_client`. + struct connector + { + boost::asio::ip::tcp::endpoint proxy_address; + + /*! Creates a new socket, asynchronously connects to `proxy_address`, + and requests a connection to `remote_host` on `remote_port`. Sets + socket as closed if `timeout` is reached. + + \return The socket if successful, and exception in the future with + error otherwise. */ + boost::unique_future<boost::asio::ip::tcp::socket> + operator()(const std::string& remote_host, const std::string& remote_port, boost::asio::steady_timer& timeout) const; + }; +} // socks +} // net diff --git a/src/p2p/net_node.cpp b/src/p2p/net_node.cpp index e0f1de14a..fcbcce58c 100644 --- a/src/p2p/net_node.cpp +++ b/src/p2p/net_node.cpp @@ -132,6 +132,7 @@ namespace nodetool const command_line::arg_descriptor<std::vector<std::string> > arg_proxy = {"proxy", "<network-type>,<socks-ip:port>[,max_connections] i.e. \"tor,127.0.0.1:9050,100\""}; const command_line::arg_descriptor<std::vector<std::string> > arg_anonymous_inbound = {"anonymous-inbound", "<hidden-service-address>,<[bind-ip:]port>[,max_connections] i.e. \"x.onion,127.0.0.1:18083,100\""}; const command_line::arg_descriptor<bool> arg_p2p_hide_my_port = {"hide-my-port", "Do not announce yourself as peerlist candidate", false, true}; + const command_line::arg_descriptor<bool> arg_no_sync = {"no-sync", "Don't synchronize the blockchain with other peers", false}; const command_line::arg_descriptor<bool> arg_no_igd = {"no-igd", "Disable UPnP port mapping"}; const command_line::arg_descriptor<int64_t> arg_out_peers = {"out-peers", "set max number of out peers", -1}; diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index 7840e3832..42bb3b061 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -489,6 +489,7 @@ namespace nodetool extern const command_line::arg_descriptor<std::vector<std::string> > arg_proxy; extern const command_line::arg_descriptor<std::vector<std::string> > arg_anonymous_inbound; extern const command_line::arg_descriptor<bool> arg_p2p_hide_my_port; + extern const command_line::arg_descriptor<bool> arg_no_sync; extern const command_line::arg_descriptor<bool> arg_no_igd; extern const command_line::arg_descriptor<bool> arg_offline; diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index 3542357dc..ba6e79d3f 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -103,6 +103,7 @@ namespace nodetool command_line::add_arg(desc, arg_proxy); command_line::add_arg(desc, arg_anonymous_inbound); command_line::add_arg(desc, arg_p2p_hide_my_port); + command_line::add_arg(desc, arg_no_sync); command_line::add_arg(desc, arg_no_igd); command_line::add_arg(desc, arg_out_peers); command_line::add_arg(desc, arg_in_peers); @@ -310,6 +311,9 @@ namespace nodetool if(command_line::has_arg(vm, arg_p2p_hide_my_port)) m_hide_my_port = true; + if (command_line::has_arg(vm, arg_no_sync)) + m_payload_handler.set_no_sync(true); + if ( !set_max_out_peers(public_zone, command_line::get_arg(vm, arg_out_peers) ) ) return false; else diff --git a/src/ringct/rctOps.cpp b/src/ringct/rctOps.cpp index e39ba16fd..b5499262f 100644 --- a/src/ringct/rctOps.cpp +++ b/src/ringct/rctOps.cpp @@ -408,10 +408,10 @@ namespace rct { return res; } - //Computes aL where L is the curve order - bool isInMainSubgroup(const key & a) { + //Computes lA where l is the curve order + bool isInMainSubgroup(const key & A) { ge_p3 p3; - return toPointCheckOrder(&p3, a.bytes); + return toPointCheckOrder(&p3, A.bytes); } //Curve addition / subtractions diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 8c6ea7cd1..e6a09606b 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -70,6 +70,13 @@ namespace { return (value + quantum - 1) / quantum * quantum; } + + void store_difficulty(cryptonote::difficulty_type difficulty, uint64_t &sdiff, std::string &swdiff, uint64_t &stop64) + { + sdiff = (difficulty << 64 >> 64).convert_to<uint64_t>(); + swdiff = difficulty.convert_to<std::string>(); + stop64 = (difficulty >> 64).convert_to<uint64_t>(); + } } namespace cryptonote @@ -219,7 +226,7 @@ namespace cryptonote ++res.height; // turn top block height into blockchain height res.top_block_hash = string_tools::pod_to_hex(top_hash); res.target_height = m_core.get_target_blockchain_height(); - res.difficulty = m_core.get_blockchain_storage().get_difficulty_for_next_block(); + store_difficulty(m_core.get_blockchain_storage().get_difficulty_for_next_block(), res.difficulty, res.wide_difficulty, res.difficulty_top64); res.target = m_core.get_blockchain_storage().get_difficulty_target(); res.tx_count = m_core.get_blockchain_storage().get_total_transactions() - res.height; //without coinbase res.tx_pool_size = m_core.get_pool_transactions_count(); @@ -236,7 +243,8 @@ namespace cryptonote res.testnet = net_type == TESTNET; res.stagenet = net_type == STAGENET; res.nettype = net_type == MAINNET ? "mainnet" : net_type == TESTNET ? "testnet" : net_type == STAGENET ? "stagenet" : "fakechain"; - res.cumulative_difficulty = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.height - 1); + store_difficulty(m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.height - 1), + res.cumulative_difficulty, res.wide_cumulative_difficulty, res.cumulative_difficulty_top64); res.block_size_limit = res.block_weight_limit = m_core.get_blockchain_storage().get_current_cumulative_block_weight_limit(); res.block_size_median = res.block_weight_median = m_core.get_blockchain_storage().get_current_cumulative_block_weight_median(); res.status = CORE_RPC_STATUS_OK; @@ -1196,13 +1204,15 @@ namespace cryptonote block b; cryptonote::blobdata blob_reserve; blob_reserve.resize(req.reserve_size, 0); - if(!m_core.get_block_template(b, info.address, res.difficulty, res.height, res.expected_reward, blob_reserve)) + cryptonote::difficulty_type wdiff; + if(!m_core.get_block_template(b, info.address, wdiff, res.height, res.expected_reward, blob_reserve)) { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; error_resp.message = "Internal error: failed to create block template"; LOG_ERROR("Failed to create block template"); return false; } + store_difficulty(wdiff, res.difficulty, res.wide_difficulty, res.difficulty_top64); blobdata block_blob = t_serializable_object_to_blob(b); crypto::public_key tx_pub_key = cryptonote::get_tx_pub_key_from_extra(b.miner_tx); if(tx_pub_key == crypto::null_pkey) @@ -1375,8 +1385,10 @@ namespace cryptonote response.height = height; response.depth = m_core.get_current_blockchain_height() - height - 1; response.hash = string_tools::pod_to_hex(hash); - response.difficulty = m_core.get_blockchain_storage().block_difficulty(height); - response.cumulative_difficulty = response.block_weight = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(height); + store_difficulty(m_core.get_blockchain_storage().block_difficulty(height), + response.difficulty, response.wide_difficulty, response.difficulty_top64); + store_difficulty(m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(height), + response.cumulative_difficulty, response.wide_cumulative_difficulty, response.cumulative_difficulty_top64); response.reward = get_block_reward(blk); response.block_size = response.block_weight = m_core.get_blockchain_storage().get_db().get_block_weight(height); response.num_txes = blk.tx_hashes.size(); @@ -1471,7 +1483,8 @@ namespace cryptonote error_resp.message = "Internal error: can't get last block."; return false; } - bool response_filled = fill_block_header_response(last_block, false, last_block_height, last_block_hash, res.block_header, req.fill_pow_hash); + const bool restricted = m_restricted && ctx; + bool response_filled = fill_block_header_response(last_block, false, last_block_height, last_block_hash, res.block_header, req.fill_pow_hash && !restricted); if (!response_filled) { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; @@ -1513,7 +1526,8 @@ namespace cryptonote return false; } uint64_t block_height = boost::get<txin_gen>(blk.miner_tx.vin.front()).height; - bool response_filled = fill_block_header_response(blk, orphan, block_height, block_hash, res.block_header, req.fill_pow_hash); + const bool restricted = m_restricted && ctx; + bool response_filled = fill_block_header_response(blk, orphan, block_height, block_hash, res.block_header, req.fill_pow_hash && !restricted); if (!response_filled) { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; @@ -1563,7 +1577,8 @@ namespace cryptonote return false; } res.headers.push_back(block_header_response()); - bool response_filled = fill_block_header_response(blk, false, block_height, block_hash, res.headers.back(), req.fill_pow_hash); + const bool restricted = m_restricted && ctx; + bool response_filled = fill_block_header_response(blk, false, block_height, block_hash, res.headers.back(), req.fill_pow_hash && !restricted); if (!response_filled) { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; @@ -1597,7 +1612,8 @@ namespace cryptonote error_resp.message = "Internal error: can't get block by height. Height = " + std::to_string(req.height) + '.'; return false; } - bool response_filled = fill_block_header_response(blk, false, req.height, block_hash, res.block_header, req.fill_pow_hash); + const bool restricted = m_restricted && ctx; + bool response_filled = fill_block_header_response(blk, false, req.height, block_hash, res.block_header, req.fill_pow_hash && !restricted); if (!response_filled) { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; @@ -1652,7 +1668,8 @@ namespace cryptonote return false; } uint64_t block_height = boost::get<txin_gen>(blk.miner_tx.vin.front()).height; - bool response_filled = fill_block_header_response(blk, orphan, block_height, block_hash, res.block_header, req.fill_pow_hash); + const bool restricted = m_restricted && ctx; + bool response_filled = fill_block_header_response(blk, orphan, block_height, block_hash, res.block_header, req.fill_pow_hash && !restricted); if (!response_filled) { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; @@ -1702,7 +1719,8 @@ namespace cryptonote ++res.height; // turn top block height into blockchain height res.top_block_hash = string_tools::pod_to_hex(top_hash); res.target_height = m_core.get_target_blockchain_height(); - res.difficulty = m_core.get_blockchain_storage().get_difficulty_for_next_block(); + store_difficulty(m_core.get_blockchain_storage().get_difficulty_for_next_block(), + res.difficulty, res.wide_difficulty, res.difficulty_top64); res.target = m_core.get_blockchain_storage().get_current_hard_fork_version() < 2 ? DIFFICULTY_TARGET_V1 : DIFFICULTY_TARGET_V2; res.tx_count = m_core.get_blockchain_storage().get_total_transactions() - res.height; //without coinbase res.tx_pool_size = m_core.get_pool_transactions_count(); @@ -1720,7 +1738,8 @@ namespace cryptonote res.stagenet = net_type == STAGENET; res.nettype = net_type == MAINNET ? "mainnet" : net_type == TESTNET ? "testnet" : net_type == STAGENET ? "stagenet" : "fakechain"; - res.cumulative_difficulty = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.height - 1); + store_difficulty(m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.height - 1), + res.cumulative_difficulty, res.wide_cumulative_difficulty, res.cumulative_difficulty_top64); res.block_size_limit = res.block_weight_limit = m_core.get_blockchain_storage().get_current_cumulative_block_weight_limit(); res.block_size_median = res.block_weight_median = m_core.get_blockchain_storage().get_current_cumulative_block_weight_median(); res.status = CORE_RPC_STATUS_OK; @@ -1944,7 +1963,9 @@ namespace cryptonote std::list<std::pair<Blockchain::block_extended_info, std::vector<crypto::hash>>> chains = m_core.get_blockchain_storage().get_alternative_chains(); for (const auto &i: chains) { - res.chains.push_back(COMMAND_RPC_GET_ALTERNATE_CHAINS::chain_info{epee::string_tools::pod_to_hex(get_block_hash(i.first.bl)), i.first.height, i.second.size(), i.first.cumulative_difficulty, {}, std::string()}); + difficulty_type wdiff = i.first.cumulative_difficulty; + res.chains.push_back(COMMAND_RPC_GET_ALTERNATE_CHAINS::chain_info{epee::string_tools::pod_to_hex(get_block_hash(i.first.bl)), i.first.height, i.second.size(), 0, "", 0, {}, std::string()}); + store_difficulty(wdiff, res.chains.back().difficulty, res.chains.back().wide_difficulty, res.chains.back().difficulty_top64); res.chains.back().block_hashes.reserve(i.second.size()); for (const crypto::hash &block_id: i.second) res.chains.back().block_hashes.push_back(epee::string_tools::pod_to_hex(block_id)); diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index f65c7c8dd..4786f7a9b 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -252,223 +252,6 @@ namespace cryptonote }; //----------------------------------------------- - struct COMMAND_RPC_GET_ADDRESS_TXS - { - struct request_t - { - std::string address; - std::string view_key; - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(address) - KV_SERIALIZE(view_key) - END_KV_SERIALIZE_MAP() - }; - typedef epee::misc_utils::struct_init<request_t> request; - - struct spent_output { - uint64_t amount; - std::string key_image; - std::string tx_pub_key; - uint64_t out_index; - uint32_t mixin; - - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(amount) - KV_SERIALIZE(key_image) - KV_SERIALIZE(tx_pub_key) - KV_SERIALIZE(out_index) - KV_SERIALIZE(mixin) - END_KV_SERIALIZE_MAP() - }; - - struct transaction - { - uint64_t id; - std::string hash; - uint64_t timestamp; - uint64_t total_received; - uint64_t total_sent; - uint64_t unlock_time; - uint64_t height; - std::list<spent_output> spent_outputs; - std::string payment_id; - bool coinbase; - bool mempool; - uint32_t mixin; - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(id) - KV_SERIALIZE(hash) - KV_SERIALIZE(timestamp) - KV_SERIALIZE(total_received) - KV_SERIALIZE(total_sent) - KV_SERIALIZE(unlock_time) - KV_SERIALIZE(height) - KV_SERIALIZE(spent_outputs) - KV_SERIALIZE(payment_id) - KV_SERIALIZE(coinbase) - KV_SERIALIZE(mempool) - KV_SERIALIZE(mixin) - END_KV_SERIALIZE_MAP() - }; - - - struct response_t - { - //std::list<std::string> txs_as_json; - uint64_t total_received; - uint64_t total_received_unlocked = 0; // OpenMonero only - uint64_t scanned_height; - std::vector<transaction> transactions; - uint64_t blockchain_height; - uint64_t scanned_block_height; - std::string status; - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(total_received) - KV_SERIALIZE(total_received_unlocked) - KV_SERIALIZE(scanned_height) - KV_SERIALIZE(transactions) - KV_SERIALIZE(blockchain_height) - KV_SERIALIZE(scanned_block_height) - KV_SERIALIZE(status) - END_KV_SERIALIZE_MAP() - }; - typedef epee::misc_utils::struct_init<response_t> response; - }; - - //----------------------------------------------- - struct COMMAND_RPC_GET_ADDRESS_INFO - { - struct request_t - { - std::string address; - std::string view_key; - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(address) - KV_SERIALIZE(view_key) - END_KV_SERIALIZE_MAP() - }; - typedef epee::misc_utils::struct_init<request_t> request; - - struct spent_output - { - uint64_t amount; - std::string key_image; - std::string tx_pub_key; - uint64_t out_index; - uint32_t mixin; - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(amount) - KV_SERIALIZE(key_image) - KV_SERIALIZE(tx_pub_key) - KV_SERIALIZE(out_index) - KV_SERIALIZE(mixin) - END_KV_SERIALIZE_MAP() - }; - - struct response_t - { - uint64_t locked_funds; - uint64_t total_received; - uint64_t total_sent; - uint64_t scanned_height; - uint64_t scanned_block_height; - uint64_t start_height; - uint64_t transaction_height; - uint64_t blockchain_height; - std::list<spent_output> spent_outputs; - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(locked_funds) - KV_SERIALIZE(total_received) - KV_SERIALIZE(total_sent) - KV_SERIALIZE(scanned_height) - KV_SERIALIZE(scanned_block_height) - KV_SERIALIZE(start_height) - KV_SERIALIZE(transaction_height) - KV_SERIALIZE(blockchain_height) - KV_SERIALIZE(spent_outputs) - END_KV_SERIALIZE_MAP() - }; - typedef epee::misc_utils::struct_init<response_t> response; - }; - - //----------------------------------------------- - struct COMMAND_RPC_GET_UNSPENT_OUTS - { - struct request_t - { - std::string amount; - std::string address; - std::string view_key; - // OpenMonero specific - uint64_t mixin; - bool use_dust; - std::string dust_threshold; - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(amount) - KV_SERIALIZE(address) - KV_SERIALIZE(view_key) - KV_SERIALIZE(mixin) - KV_SERIALIZE(use_dust) - KV_SERIALIZE(dust_threshold) - END_KV_SERIALIZE_MAP() - }; - typedef epee::misc_utils::struct_init<request_t> request; - - - struct output { - uint64_t amount; - std::string public_key; - uint64_t index; - uint64_t global_index; - std::string rct; - std::string tx_hash; - std::string tx_pub_key; - std::string tx_prefix_hash; - std::vector<std::string> spend_key_images; - uint64_t timestamp; - uint64_t height; - - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(amount) - KV_SERIALIZE(public_key) - KV_SERIALIZE(index) - KV_SERIALIZE(global_index) - KV_SERIALIZE(rct) - KV_SERIALIZE(tx_hash) - KV_SERIALIZE(tx_pub_key) - KV_SERIALIZE(tx_prefix_hash) - KV_SERIALIZE(spend_key_images) - KV_SERIALIZE(timestamp) - KV_SERIALIZE(height) - END_KV_SERIALIZE_MAP() - }; - - struct response_t - { - uint64_t amount; - std::list<output> outputs; - uint64_t per_kb_fee; - std::string status; - std::string reason; - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(amount) - KV_SERIALIZE(outputs) - KV_SERIALIZE(per_kb_fee) - KV_SERIALIZE(status) - KV_SERIALIZE(reason) - END_KV_SERIALIZE_MAP() - }; - typedef epee::misc_utils::struct_init<response_t> response; - }; - - //----------------------------------------------- struct COMMAND_RPC_GET_RANDOM_OUTS { struct request_t @@ -548,72 +331,6 @@ namespace cryptonote typedef epee::misc_utils::struct_init<response_t> response; }; //----------------------------------------------- - struct COMMAND_RPC_LOGIN - { - struct request_t - { - std::string address; - std::string view_key; - bool create_account; - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(address) - KV_SERIALIZE(view_key) - KV_SERIALIZE(create_account) - END_KV_SERIALIZE_MAP() - }; - typedef epee::misc_utils::struct_init<request_t> request; - - struct response_t - { - std::string status; - std::string reason; - bool new_address; - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(status) - KV_SERIALIZE(reason) - KV_SERIALIZE(new_address) - END_KV_SERIALIZE_MAP() - }; - typedef epee::misc_utils::struct_init<response_t> response; - }; - //----------------------------------------------- - struct COMMAND_RPC_IMPORT_WALLET_REQUEST - { - struct request_t - { - std::string address; - std::string view_key; - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(address) - KV_SERIALIZE(view_key) - END_KV_SERIALIZE_MAP() - }; - typedef epee::misc_utils::struct_init<request_t> request; - - struct response_t - { - std::string payment_id; - uint64_t import_fee; - bool new_request; - bool request_fulfilled; - std::string payment_address; - std::string status; - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(payment_id) - KV_SERIALIZE(import_fee) - KV_SERIALIZE(new_request) - KV_SERIALIZE(request_fulfilled) - KV_SERIALIZE(payment_address) - KV_SERIALIZE(status) - END_KV_SERIALIZE_MAP() - }; - typedef epee::misc_utils::struct_init<response_t> response; - }; - //----------------------------------------------- struct COMMAND_RPC_GET_TRANSACTIONS { struct request_t @@ -943,6 +660,8 @@ namespace cryptonote uint64_t height; uint64_t target_height; uint64_t difficulty; + std::string wide_difficulty; + uint64_t difficulty_top64; uint64_t target; uint64_t tx_count; uint64_t tx_pool_size; @@ -958,6 +677,8 @@ namespace cryptonote std::string nettype; std::string top_block_hash; uint64_t cumulative_difficulty; + std::string wide_cumulative_difficulty; + uint64_t cumulative_difficulty_top64; uint64_t block_size_limit; uint64_t block_weight_limit; uint64_t block_size_median; @@ -978,6 +699,8 @@ namespace cryptonote KV_SERIALIZE(height) KV_SERIALIZE(target_height) KV_SERIALIZE(difficulty) + KV_SERIALIZE(wide_difficulty) + KV_SERIALIZE(difficulty_top64) KV_SERIALIZE(target) KV_SERIALIZE(tx_count) KV_SERIALIZE(tx_pool_size) @@ -993,6 +716,8 @@ namespace cryptonote KV_SERIALIZE(nettype) KV_SERIALIZE(top_block_hash) KV_SERIALIZE(cumulative_difficulty) + KV_SERIALIZE(wide_cumulative_difficulty) + KV_SERIALIZE(cumulative_difficulty_top64) KV_SERIALIZE(block_size_limit) KV_SERIALIZE_OPT(block_weight_limit, (uint64_t)0) KV_SERIALIZE(block_size_median) @@ -1149,6 +874,8 @@ namespace cryptonote struct response_t { uint64_t difficulty; + std::string wide_difficulty; + uint64_t difficulty_top64; uint64_t height; uint64_t reserved_offset; uint64_t expected_reward; @@ -1160,6 +887,8 @@ namespace cryptonote BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(difficulty) + KV_SERIALIZE(wide_difficulty) + KV_SERIALIZE(difficulty_top64) KV_SERIALIZE(height) KV_SERIALIZE(reserved_offset) KV_SERIALIZE(expected_reward) @@ -1226,8 +955,12 @@ namespace cryptonote uint64_t height; uint64_t depth; std::string hash; - difficulty_type difficulty; - difficulty_type cumulative_difficulty; + uint64_t difficulty; + std::string wide_difficulty; + uint64_t difficulty_top64; + uint64_t cumulative_difficulty; + std::string wide_cumulative_difficulty; + uint64_t cumulative_difficulty_top64; uint64_t reward; uint64_t block_size; uint64_t block_weight; @@ -1246,7 +979,11 @@ namespace cryptonote KV_SERIALIZE(depth) KV_SERIALIZE(hash) KV_SERIALIZE(difficulty) + KV_SERIALIZE(wide_difficulty) + KV_SERIALIZE(difficulty_top64) KV_SERIALIZE(cumulative_difficulty) + KV_SERIALIZE(wide_cumulative_difficulty) + KV_SERIALIZE(cumulative_difficulty_top64) KV_SERIALIZE(reward) KV_SERIALIZE(block_size) KV_SERIALIZE_OPT(block_weight, (uint64_t)0) @@ -2248,6 +1985,8 @@ namespace cryptonote uint64_t height; uint64_t length; uint64_t difficulty; + std::string wide_difficulty; + uint64_t difficulty_top64; std::vector<std::string> block_hashes; std::string main_chain_parent_block; @@ -2256,6 +1995,8 @@ namespace cryptonote KV_SERIALIZE(height) KV_SERIALIZE(length) KV_SERIALIZE(difficulty) + KV_SERIALIZE(wide_difficulty) + KV_SERIALIZE(difficulty_top64) KV_SERIALIZE(block_hashes) KV_SERIALIZE(main_chain_parent_block) END_KV_SERIALIZE_MAP() diff --git a/src/rpc/daemon_handler.cpp b/src/rpc/daemon_handler.cpp index 14b492786..540afe6b9 100644 --- a/src/rpc/daemon_handler.cpp +++ b/src/rpc/daemon_handler.cpp @@ -263,20 +263,43 @@ namespace rpc void DaemonHandler::handle(const SendRawTx::Request& req, SendRawTx::Response& res) { - auto tx_blob = cryptonote::tx_to_blob(req.tx); + handleTxBlob(cryptonote::tx_to_blob(req.tx), req.relay, res); + } + + void DaemonHandler::handle(const SendRawTxHex::Request& req, SendRawTxHex::Response& res) + { + std::string tx_blob; + if(!epee::string_tools::parse_hexstr_to_binbuff(req.tx_as_hex, tx_blob)) + { + MERROR("[SendRawTxHex]: Failed to parse tx from hexbuff: " << req.tx_as_hex); + res.status = Message::STATUS_FAILED; + res.error_details = "Invalid hex"; + return; + } + handleTxBlob(tx_blob, req.relay, res); + } + + void DaemonHandler::handleTxBlob(const std::string& tx_blob, bool relay, SendRawTx::Response& res) + { + if (!m_p2p.get_payload_object().is_synchronized()) + { + res.status = Message::STATUS_FAILED; + res.error_details = "Not ready to accept transactions; try again later"; + return; + } cryptonote_connection_context fake_context = AUTO_VAL_INIT(fake_context); tx_verification_context tvc = AUTO_VAL_INIT(tvc); - if(!m_core.handle_incoming_tx(tx_blob, tvc, false, false, !req.relay) || tvc.m_verifivation_failed) + if(!m_core.handle_incoming_tx(tx_blob, tvc, false, false, !relay) || tvc.m_verifivation_failed) { if (tvc.m_verifivation_failed) { - LOG_PRINT_L0("[on_send_raw_tx]: tx verification failed"); + MERROR("[SendRawTx]: tx verification failed"); } else { - LOG_PRINT_L0("[on_send_raw_tx]: Failed to process tx"); + MERROR("[SendRawTx]: Failed to process tx"); } res.status = Message::STATUS_FAILED; res.error_details = ""; @@ -328,9 +351,9 @@ namespace rpc return; } - if(!tvc.m_should_be_relayed || !req.relay) + if(!tvc.m_should_be_relayed || !relay) { - LOG_PRINT_L0("[on_send_raw_tx]: tx accepted, but not relayed"); + MERROR("[SendRawTx]: tx accepted, but not relayed"); res.error_details = "Not relayed"; res.relayed = false; res.status = Message::STATUS_OK; @@ -413,7 +436,8 @@ namespace rpc auto& chain = m_core.get_blockchain_storage(); - res.info.difficulty = chain.get_difficulty_for_next_block(); + res.info.wide_difficulty = chain.get_difficulty_for_next_block(); + res.info.difficulty = (res.info.wide_difficulty << 64 >> 64).convert_to<uint64_t>(); res.info.target = chain.get_difficulty_target(); @@ -434,7 +458,8 @@ namespace rpc res.info.mainnet = m_core.get_nettype() == MAINNET; res.info.testnet = m_core.get_nettype() == TESTNET; res.info.stagenet = m_core.get_nettype() == STAGENET; - res.info.cumulative_difficulty = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.info.height - 1); + res.info.wide_cumulative_difficulty = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.info.height - 1); + res.info.cumulative_difficulty = (res.info.wide_cumulative_difficulty << 64 >> 64).convert_to<uint64_t>(); res.info.block_size_limit = res.info.block_weight_limit = m_core.get_blockchain_storage().get_current_cumulative_block_weight_limit(); res.info.block_size_median = res.info.block_weight_median = m_core.get_blockchain_storage().get_current_cumulative_block_weight_median(); res.info.start_time = (uint64_t)m_core.get_start_time(); @@ -803,7 +828,8 @@ namespace rpc header.reward += out.amount; } - header.difficulty = m_core.get_blockchain_storage().block_difficulty(header.height); + header.wide_difficulty = m_core.get_blockchain_storage().block_difficulty(header.height); + header.difficulty = (header.wide_difficulty << 64 >> 64).convert_to<uint64_t>(); return true; } @@ -830,6 +856,7 @@ namespace rpc REQ_RESP_TYPES_MACRO(request_type, KeyImagesSpent, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, GetTxGlobalOutputIndices, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, SendRawTx, req_json, resp_message, handle); + REQ_RESP_TYPES_MACRO(request_type, SendRawTxHex, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, GetInfo, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, StartMining, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, StopMining, req_json, resp_message, handle); diff --git a/src/rpc/daemon_handler.h b/src/rpc/daemon_handler.h index 87161784e..34723f676 100644 --- a/src/rpc/daemon_handler.h +++ b/src/rpc/daemon_handler.h @@ -68,6 +68,8 @@ class DaemonHandler : public RpcHandler void handle(const SendRawTx::Request& req, SendRawTx::Response& res); + void handle(const SendRawTxHex::Request& req, SendRawTxHex::Response& res); + void handle(const StartMining::Request& req, StartMining::Response& res); void handle(const GetInfo::Request& req, GetInfo::Response& res); @@ -136,6 +138,8 @@ class DaemonHandler : public RpcHandler bool getBlockHeaderByHash(const crypto::hash& hash_in, cryptonote::rpc::BlockHeaderResponse& response); + void handleTxBlob(const std::string& tx_blob, bool relay, SendRawTx::Response& res); + cryptonote::core& m_core; t_p2p& m_p2p; }; diff --git a/src/rpc/daemon_messages.cpp b/src/rpc/daemon_messages.cpp index ecd3683e5..cf0f5ece1 100644 --- a/src/rpc/daemon_messages.cpp +++ b/src/rpc/daemon_messages.cpp @@ -42,6 +42,7 @@ const char* const GetTransactions::name = "get_transactions"; const char* const KeyImagesSpent::name = "key_images_spent"; const char* const GetTxGlobalOutputIndices::name = "get_tx_global_output_indices"; const char* const SendRawTx::name = "send_raw_tx"; +const char* const SendRawTxHex::name = "send_raw_tx_hex"; const char* const StartMining::name = "start_mining"; const char* const StopMining::name = "stop_mining"; const char* const MiningStatus::name = "mining_status"; @@ -292,6 +293,22 @@ void SendRawTx::Response::fromJson(rapidjson::Value& val) GET_FROM_JSON_OBJECT(val, relayed, relayed); } +rapidjson::Value SendRawTxHex::Request::toJson(rapidjson::Document& doc) const +{ + auto val = Message::toJson(doc); + + INSERT_INTO_JSON_OBJECT(val, doc, tx_as_hex, tx_as_hex); + INSERT_INTO_JSON_OBJECT(val, doc, relay, relay); + + return val; +} + +void SendRawTxHex::Request::fromJson(rapidjson::Value& val) +{ + GET_FROM_JSON_OBJECT(val, tx_as_hex, tx_as_hex); + GET_FROM_JSON_OBJECT(val, relay, relay); +} + rapidjson::Value StartMining::Request::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); diff --git a/src/rpc/daemon_messages.h b/src/rpc/daemon_messages.h index 8ce56b6af..c0d9aed0a 100644 --- a/src/rpc/daemon_messages.h +++ b/src/rpc/daemon_messages.h @@ -171,6 +171,14 @@ BEGIN_RPC_MESSAGE_CLASS(SendRawTx); END_RPC_MESSAGE_RESPONSE; END_RPC_MESSAGE_CLASS; +BEGIN_RPC_MESSAGE_CLASS(SendRawTxHex); + BEGIN_RPC_MESSAGE_REQUEST; + RPC_MESSAGE_MEMBER(std::string, tx_as_hex); + RPC_MESSAGE_MEMBER(bool, relay); + END_RPC_MESSAGE_REQUEST; + using Response = SendRawTx::Response; +END_RPC_MESSAGE_CLASS; + BEGIN_RPC_MESSAGE_CLASS(StartMining); BEGIN_RPC_MESSAGE_REQUEST; RPC_MESSAGE_MEMBER(std::string, miner_address); diff --git a/src/rpc/message_data_structs.h b/src/rpc/message_data_structs.h index 26c5038f6..2a43811cf 100644 --- a/src/rpc/message_data_structs.h +++ b/src/rpc/message_data_structs.h @@ -30,6 +30,7 @@ #include "crypto/hash.h" #include "cryptonote_basic/cryptonote_basic.h" +#include "cryptonote_basic/difficulty.h" #include "ringct/rctSigs.h" #include "rpc/rpc_handler.h" @@ -165,6 +166,7 @@ namespace rpc uint64_t height; uint64_t depth; crypto::hash hash; + cryptonote::difficulty_type wide_difficulty; uint64_t difficulty; uint64_t reward; }; @@ -173,6 +175,7 @@ namespace rpc { uint64_t height; uint64_t target_height; + cryptonote::difficulty_type wide_difficulty; uint64_t difficulty; uint64_t target; uint64_t tx_count; @@ -187,6 +190,7 @@ namespace rpc bool stagenet; std::string nettype; crypto::hash top_block_hash; + cryptonote::difficulty_type wide_cumulative_difficulty; uint64_t cumulative_difficulty; uint64_t block_size_limit; uint64_t block_weight_limit; diff --git a/src/serialization/difficulty_type.h b/src/serialization/difficulty_type.h new file mode 100644 index 000000000..e32e24b78 --- /dev/null +++ b/src/serialization/difficulty_type.h @@ -0,0 +1,65 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include "cryptonote_basic/difficulty.h" +#include "serialization.h" + +template<> struct is_basic_type<cryptonote::difficulty_type> { typedef boost::true_type type; }; + +template <template <bool> class Archive> +inline bool do_serialize(Archive<false>& ar, cryptonote::difficulty_type &diff) +{ + uint64_t hi, lo; + ar.serialize_varint(hi); + if (!ar.stream().good()) + return false; + ar.serialize_varint(lo); + if (!ar.stream().good()) + return false; + diff = hi; + diff <<= 64; + diff += lo; + return true; +} + +template <template <bool> class Archive> +inline bool do_serialize(Archive<true>& ar, cryptonote::difficulty_type &diff) +{ + if (!ar.stream().good()) + return false; + const uint64_t hi = (diff >> 64).convert_to<uint64_t>(); + const uint64_t lo = (diff << 64 >> 64).convert_to<uint64_t>(); + ar.serialize_varint(hi); + ar.serialize_varint(lo); + if (!ar.stream().good()) + return false; + return true; +} + diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index 8952a43dd..91a8f6348 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -116,7 +116,9 @@ typedef cryptonote::simple_wallet sw; #define LONG_PAYMENT_ID_SUPPORT_CHECK() \ do { \ if (!m_long_payment_id_support) { \ - fail_msg_writer() << tr("Long payment IDs are obsolete. Use --long-payment-id-support if you really must use one, and warn the recipient they are using an obsolete feature that will disappear in the future."); \ + fail_msg_writer() << tr("Warning: Long payment IDs are obsolete."); \ + fail_msg_writer() << tr("Long payment IDs are not encrypted on the blockchain, and will harm your privacy."); \ + fail_msg_writer() << tr("Use --long-payment-id-support if you really must use one, and warn the recipient they are using an obsolete feature that will disappear in the future."); \ return true; \ } \ } while(0) @@ -149,7 +151,7 @@ namespace const command_line::arg_descriptor<bool> arg_create_address_file = {"create-address-file", sw::tr("Create an address file for new wallets"), false}; const command_line::arg_descriptor<std::string> arg_subaddress_lookahead = {"subaddress-lookahead", tools::wallet2::tr("Set subaddress lookahead sizes to <major>:<minor>"), ""}; const command_line::arg_descriptor<bool> arg_use_english_language_names = {"use-english-language-names", sw::tr("Display English language names"), false}; - const command_line::arg_descriptor<bool> arg_long_payment_id_support = {"long-payment-id-support", sw::tr("Support obsolete long (unencrypted) payment ids"), false}; + const command_line::arg_descriptor<bool> arg_long_payment_id_support = {"long-payment-id-support-bad-for-privacy", sw::tr("Support obsolete long (unencrypted) payment ids (using them harms your privacy)"), false}; const command_line::arg_descriptor< std::vector<std::string> > arg_command = {"command", ""}; @@ -190,7 +192,7 @@ namespace const char* USAGE_CHECK_RESERVE_PROOF("check_reserve_proof <address> <signature_file> [<message>]"); const char* USAGE_SHOW_TRANSFERS("show_transfers [in|out|pending|failed|pool|coinbase] [index=<N1>[,<N2>,...]] [<min_height> [<max_height>]]"); const char* USAGE_UNSPENT_OUTPUTS("unspent_outputs [index=<N1>[,<N2>,...]] [<min_amount> [<max_amount>]]"); - const char* USAGE_RESCAN_BC("rescan_bc [hard]"); + const char* USAGE_RESCAN_BC("rescan_bc [hard|soft|keep_ki] [start_height=0]"); const char* USAGE_SET_TX_NOTE("set_tx_note <txid> [free text note]"); const char* USAGE_GET_TX_NOTE("get_tx_note <txid>"); const char* USAGE_GET_DESCRIPTION("get_description"); @@ -2049,7 +2051,7 @@ bool simple_wallet::cold_sign_tx(const std::vector<tools::wallet2::pending_tx>& m_wallet->cold_tx_aux_import(exported_txs.ptx, tx_aux); // import key images - return m_wallet->import_key_images(exported_txs.key_images); + return m_wallet->import_key_images(exported_txs, 0, true); } bool simple_wallet::set_always_confirm_transfers(const std::vector<std::string> &args/* = std::vector<std::string>()*/) @@ -4686,12 +4688,12 @@ boost::optional<epee::wipeable_string> simple_wallet::on_get_password(const char return pwd_container->password(); } //---------------------------------------------------------------------------------------------------- -void simple_wallet::on_button_request() +void simple_wallet::on_device_button_request(uint64_t code) { message_writer(console_color_white, false) << tr("Device requires attention"); } //---------------------------------------------------------------------------------------------------- -void simple_wallet::on_pin_request(epee::wipeable_string & pin) +boost::optional<epee::wipeable_string> simple_wallet::on_device_pin_request() { #ifdef HAVE_READLINE rdln::suspend_readline pause_readline; @@ -4699,14 +4701,14 @@ void simple_wallet::on_pin_request(epee::wipeable_string & pin) std::string msg = tr("Enter device PIN"); auto pwd_container = tools::password_container::prompt(false, msg.c_str()); THROW_WALLET_EXCEPTION_IF(!pwd_container, tools::error::password_entry_failed, tr("Failed to read device PIN")); - pin = pwd_container->password(); + return pwd_container->password(); } //---------------------------------------------------------------------------------------------------- -void simple_wallet::on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) +boost::optional<epee::wipeable_string> simple_wallet::on_device_passphrase_request(bool on_device) { if (on_device){ message_writer(console_color_white, true) << tr("Please enter the device passphrase on the device"); - return; + return boost::none; } #ifdef HAVE_READLINE @@ -4715,13 +4717,13 @@ void simple_wallet::on_passphrase_request(bool on_device, epee::wipeable_string std::string msg = tr("Enter device passphrase"); auto pwd_container = tools::password_container::prompt(false, msg.c_str()); THROW_WALLET_EXCEPTION_IF(!pwd_container, tools::error::password_entry_failed, tr("Failed to read device passphrase")); - passphrase = pwd_container->password(); + return pwd_container->password(); } //---------------------------------------------------------------------------------------------------- void simple_wallet::on_refresh_finished(uint64_t start_height, uint64_t fetched_blocks, bool is_init, bool received_money) { // Key image sync after the first refresh - if (!m_wallet->get_account().get_device().has_tx_cold_sign()) { + if (!m_wallet->get_account().get_device().has_tx_cold_sign() || m_wallet->get_account().get_device().has_ki_live_refresh()) { return; } @@ -4748,8 +4750,16 @@ bool simple_wallet::refresh_main(uint64_t start_height, enum ResetType reset, bo LOCK_IDLE_SCOPE(); + crypto::hash transfer_hash_pre{}; + uint64_t height_pre, height_post; + if (reset != ResetNone) - m_wallet->rescan_blockchain(reset == ResetHard, false); + { + if (reset == ResetSoftKeepKI) + height_pre = m_wallet->hash_m_transfers(-1, transfer_hash_pre); + + m_wallet->rescan_blockchain(reset == ResetHard, false, reset == ResetSoftKeepKI); + } #ifdef HAVE_READLINE rdln::suspend_readline pause_readline; @@ -4766,6 +4776,18 @@ bool simple_wallet::refresh_main(uint64_t start_height, enum ResetType reset, bo m_in_manual_refresh.store(true, std::memory_order_relaxed); epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){m_in_manual_refresh.store(false, std::memory_order_relaxed);}); m_wallet->refresh(m_wallet->is_trusted_daemon(), start_height, fetched_blocks, received_money); + + if (reset == ResetSoftKeepKI) + { + m_wallet->finish_rescan_bc_keep_key_images(height_pre, transfer_hash_pre); + + height_post = m_wallet->get_num_transfer_details(); + if (height_pre != height_post) + { + message_writer() << tr("New transfer received since rescan was started. Key images are incomplete."); + } + } + ok = true; // Clear line "Height xxx of xxx" std::cout << "\r \r"; @@ -4961,7 +4983,7 @@ bool simple_wallet::show_incoming_transfers(const std::vector<std::string>& args std::vector<uint64_t> heights; for (const auto &e: td.m_uses) heights.push_back(e.first); const std::pair<std::string, std::string> line = show_outputs_line(heights, blockchain_height, td.m_spent_height); - extra_string += tr("Heights: ") + line.first + "\n" + line.second; + extra_string += std::string("\n ") + tr("Used at heights: ") + line.first + "\n " + line.second; } message_writer(td.m_spent ? console_color_magenta : console_color_green, false) << boost::format("%21s%8s%12s%8s%16u%68s%16u%s") % @@ -5335,7 +5357,7 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri r = add_extra_nonce_to_tx_extra(extra, extra_nonce); local_args.pop_back(); payment_id_seen = true; - message_writer() << tr("Unencrypted payment IDs are bad for privacy: ask the recipient to use subaddresses instead"); + message_writer() << tr("Warning: Unencrypted payment IDs will harm your privacy: ask the recipient to use subaddresses instead"); } if(!r) { @@ -5445,7 +5467,7 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri { LONG_PAYMENT_ID_SUPPORT_CHECK(); set_payment_id_to_tx_extra_nonce(extra_nonce, payment_id); - message_writer() << tr("Unencrypted payment IDs are bad for privacy: ask the recipient to use subaddresses instead"); + message_writer() << tr("Warning: Unencrypted payment IDs will harm your privacy: ask the recipient to use subaddresses instead"); } else { @@ -6752,7 +6774,7 @@ bool simple_wallet::get_tx_key(const std::vector<std::string> &args_) { std::vector<std::string> local_args = args_; - if (m_wallet->key_on_device()) + if (m_wallet->key_on_device() && m_wallet->get_account().get_device().get_type() != hw::device::TREZOR) { fail_msg_writer() << tr("command not supported by HW wallet"); return true; @@ -6773,7 +6795,9 @@ bool simple_wallet::get_tx_key(const std::vector<std::string> &args_) crypto::secret_key tx_key; std::vector<crypto::secret_key> additional_tx_keys; - if (m_wallet->get_tx_key(txid, tx_key, additional_tx_keys)) + + bool found_tx_key = m_wallet->get_tx_key(txid, tx_key, additional_tx_keys); + if (found_tx_key) { ostringstream oss; oss << epee::string_tools::pod_to_hex(tx_key); @@ -6849,7 +6873,7 @@ bool simple_wallet::set_tx_key(const std::vector<std::string> &args_) //---------------------------------------------------------------------------------------------------- bool simple_wallet::get_tx_proof(const std::vector<std::string> &args) { - if (m_wallet->key_on_device()) + if (m_wallet->key_on_device() && m_wallet->get_account().get_device().get_type() != hw::device::TREZOR) { fail_msg_writer() << tr("command not supported by HW wallet"); return true; @@ -7773,18 +7797,43 @@ bool simple_wallet::unspent_outputs(const std::vector<std::string> &args_) //---------------------------------------------------------------------------------------------------- bool simple_wallet::rescan_blockchain(const std::vector<std::string> &args_) { - bool hard = false; + uint64_t start_height = 0; + ResetType reset_type = ResetSoft; + if (!args_.empty()) { - if (args_[0] != "hard") + if (args_[0] == "hard") + { + reset_type = ResetHard; + } + else if (args_[0] == "soft") + { + reset_type = ResetSoft; + } + else if (args_[0] == "keep_ki") + { + reset_type = ResetSoftKeepKI; + } + else { PRINT_USAGE(USAGE_RESCAN_BC); return true; } - hard = true; + + if (args_.size() > 1) + { + try + { + start_height = boost::lexical_cast<uint64_t>( args_[1] ); + } + catch(const boost::bad_lexical_cast &) + { + start_height = 0; + } + } } - if (hard) + if (reset_type == ResetHard) { message_writer() << tr("Warning: this will lose any information which can not be recovered from the blockchain."); message_writer() << tr("This includes destination addresses, tx secret keys, tx notes, etc"); @@ -7795,7 +7844,20 @@ bool simple_wallet::rescan_blockchain(const std::vector<std::string> &args_) return true; } } - return refresh_main(0, hard ? ResetHard : ResetSoft, true); + + const uint64_t wallet_from_height = m_wallet->get_refresh_from_block_height(); + if (start_height > wallet_from_height) + { + message_writer() << tr("Warning: your restore height is higher than wallet restore height: ") << wallet_from_height; + std::string confirm = input_line(tr("Rescan anyway ? (Y/Yes/N/No): ")); + if(!std::cin.eof()) + { + if (!command_line::is_yes(confirm)) + return true; + } + } + + return refresh_main(start_height, reset_type, true); } //---------------------------------------------------------------------------------------------------- void simple_wallet::check_for_messages() @@ -8597,13 +8659,8 @@ bool simple_wallet::import_key_images(const std::vector<std::string> &args) { uint64_t spent = 0, unspent = 0; uint64_t height = m_wallet->import_key_images(filename, spent, unspent); - if (height > 0) - { - success_msg_writer() << "Signed key images imported to height " << height << ", " - << print_money(spent) << " spent, " << print_money(unspent) << " unspent"; - } else { - fail_msg_writer() << "Failed to import key images"; - } + success_msg_writer() << "Signed key images imported to height " << height << ", " + << print_money(spent) << " spent, " << print_money(unspent) << " unspent"; } catch (const std::exception &e) { @@ -9954,6 +10011,16 @@ bool simple_wallet::mms(const std::vector<std::string> &args) { try { + m_wallet->get_multisig_wallet_state(); + } + catch(const std::exception &e) + { + fail_msg_writer() << tr("MMS not available in this wallet"); + return true; + } + + try + { mms::message_store& ms = m_wallet->get_message_store(); if (args.size() == 0) { diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h index 6655fda27..7bcb92190 100644 --- a/src/simplewallet/simplewallet.h +++ b/src/simplewallet/simplewallet.h @@ -84,7 +84,7 @@ namespace cryptonote std::string get_command_usage(const std::vector<std::string> &args); private: - enum ResetType { ResetNone, ResetSoft, ResetHard }; + enum ResetType { ResetNone, ResetSoft, ResetHard, ResetSoftKeepKI }; bool handle_command_line(const boost::program_options::variables_map& vm); @@ -300,9 +300,9 @@ namespace cryptonote virtual void on_money_spent(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& in_tx, uint64_t amount, const cryptonote::transaction& spend_tx, const cryptonote::subaddress_index& subaddr_index); virtual void on_skip_transaction(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx); virtual boost::optional<epee::wipeable_string> on_get_password(const char *reason); - virtual void on_button_request(); - virtual void on_pin_request(epee::wipeable_string & pin); - virtual void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase); + virtual void on_device_button_request(uint64_t code); + virtual boost::optional<epee::wipeable_string> on_device_pin_request(); + virtual boost::optional<epee::wipeable_string> on_device_passphrase_request(bool on_device); //---------------------------------------------------------- friend class refresh_progress_reporter_t; diff --git a/src/wallet/CMakeLists.txt b/src/wallet/CMakeLists.txt index efd61cb5a..def23aff0 100644 --- a/src/wallet/CMakeLists.txt +++ b/src/wallet/CMakeLists.txt @@ -63,6 +63,7 @@ target_link_libraries(wallet cryptonote_core mnemonics device_trezor + net ${LMDB_LIBRARY} ${Boost_CHRONO_LIBRARY} ${Boost_SERIALIZATION_LIBRARY} diff --git a/src/wallet/api/pending_transaction.cpp b/src/wallet/api/pending_transaction.cpp index e649f1f3a..52510164a 100644 --- a/src/wallet/api/pending_transaction.cpp +++ b/src/wallet/api/pending_transaction.cpp @@ -109,6 +109,23 @@ bool PendingTransactionImpl::commit(const std::string &filename, bool overwrite) } m_wallet.pauseRefresh(); + + const bool tx_cold_signed = m_wallet.m_wallet->get_account().get_device().has_tx_cold_sign(); + if (tx_cold_signed){ + std::unordered_set<size_t> selected_transfers; + for(const tools::wallet2::pending_tx & ptx : m_pending_tx){ + for(size_t s : ptx.selected_transfers){ + selected_transfers.insert(s); + } + } + + m_wallet.m_wallet->cold_tx_aux_import(m_pending_tx, m_tx_device_aux); + bool r = m_wallet.m_wallet->import_key_images(m_key_images, 0, selected_transfers); + if (!r){ + throw runtime_error("Cold sign transaction submit failed - key image sync fail"); + } + } + while (!m_pending_tx.empty()) { auto & ptx = m_pending_tx.back(); m_wallet.m_wallet->commit_tx(ptx); diff --git a/src/wallet/api/pending_transaction.h b/src/wallet/api/pending_transaction.h index 4ec7c656a..92801d77d 100644 --- a/src/wallet/api/pending_transaction.h +++ b/src/wallet/api/pending_transaction.h @@ -67,6 +67,8 @@ private: std::string m_errorString; std::vector<tools::wallet2::pending_tx> m_pending_tx; std::unordered_set<crypto::public_key> m_signers; + std::vector<std::string> m_tx_device_aux; + std::vector<crypto::key_image> m_key_images; }; diff --git a/src/wallet/api/wallet.cpp b/src/wallet/api/wallet.cpp index da6ddc8a3..82986ba2d 100644 --- a/src/wallet/api/wallet.cpp +++ b/src/wallet/api/wallet.cpp @@ -242,6 +242,42 @@ struct Wallet2CallbackImpl : public tools::i_wallet2_callback } } + virtual void on_device_button_request(uint64_t code) + { + if (m_listener) { + m_listener->onDeviceButtonRequest(code); + } + } + + virtual boost::optional<epee::wipeable_string> on_device_pin_request() + { + if (m_listener) { + auto pin = m_listener->onDevicePinRequest(); + if (pin){ + return boost::make_optional(epee::wipeable_string((*pin).data(), (*pin).size())); + } + } + return boost::none; + } + + virtual boost::optional<epee::wipeable_string> on_device_passphrase_request(bool on_device) + { + if (m_listener) { + auto passphrase = m_listener->onDevicePassphraseRequest(on_device); + if (!on_device && passphrase) { + return boost::make_optional(epee::wipeable_string((*passphrase).data(), (*passphrase).size())); + } + } + return boost::none; + } + + virtual void on_device_progress(const hw::device_progress & event) + { + if (m_listener) { + m_listener->onDeviceProgress(DeviceProgress(event.progress(), event.indeterminate())); + } + } + WalletListener * m_listener; WalletImpl * m_wallet; }; @@ -785,6 +821,28 @@ bool WalletImpl::setPassword(const std::string &password) return status() == Status_Ok; } +bool WalletImpl::setDevicePin(const std::string &pin) +{ + clearStatus(); + try { + m_wallet->get_account().get_device().set_pin(epee::wipeable_string(pin.data(), pin.size())); + } catch (const std::exception &e) { + setStatusError(e.what()); + } + return status() == Status_Ok; +} + +bool WalletImpl::setDevicePassphrase(const std::string &passphrase) +{ + clearStatus(); + try { + m_wallet->get_account().get_device().set_passphrase(epee::wipeable_string(passphrase.data(), passphrase.size())); + } catch (const std::exception &e) { + setStatusError(e.what()); + } + return status() == Status_Ok; +} + std::string WalletImpl::address(uint32_t accountIndex, uint32_t addressIndex) const { return m_wallet->get_subaddress_as_str({accountIndex, addressIndex}); @@ -1428,8 +1486,12 @@ PendingTransaction *WalletImpl::createTransaction(const string &dst_addr, const extra, subaddr_account, subaddr_indices); } + pendingTxPostProcess(transaction); + if (multisig().isMultisig) { - transaction->m_signers = m_wallet->make_multisig_tx_set(transaction->m_pending_tx).m_signers; + auto tx_set = m_wallet->make_multisig_tx_set(transaction->m_pending_tx); + transaction->m_pending_tx = tx_set.m_ptx; + transaction->m_signers = tx_set.m_signers; } } catch (const tools::error::daemon_busy&) { // TODO: make it translatable with "tr"? @@ -1511,6 +1573,7 @@ PendingTransaction *WalletImpl::createSweepUnmixableTransaction() do { try { transaction->m_pending_tx = m_wallet->create_unmixable_sweep_transactions(); + pendingTxPostProcess(transaction); } catch (const tools::error::daemon_busy&) { // TODO: make it translatable with "tr"? @@ -2093,10 +2156,24 @@ bool WalletImpl::isNewWallet() const return !(blockChainHeight() > 1 || m_recoveringFromSeed || m_recoveringFromDevice || m_rebuildWalletCache) && !watchOnly(); } +void WalletImpl::pendingTxPostProcess(PendingTransactionImpl * pending) +{ + // If the device being used is HW device with cold signing protocol, cold sign then. + if (!m_wallet->get_account().get_device().has_tx_cold_sign()){ + return; + } + + tools::wallet2::signed_tx_set exported_txs; + std::vector<cryptonote::address_parse_info> dsts_info; + + m_wallet->cold_sign_tx(pending->m_pending_tx, exported_txs, dsts_info, pending->m_tx_device_aux); + pending->m_key_images = exported_txs.key_images; + pending->m_pending_tx = exported_txs.ptx; +} + bool WalletImpl::doInit(const string &daemon_address, uint64_t upper_transaction_size_limit, bool ssl) { - // claim RPC so there's no in-memory encryption for now - if (!m_wallet->init(daemon_address, m_daemon_login, upper_transaction_size_limit, ssl)) + if (!m_wallet->init(daemon_address, m_daemon_login, tcp::endpoint{}, upper_transaction_size_limit)) return false; // in case new wallet, this will force fast-refresh (pulling hashes instead of blocks) @@ -2325,6 +2402,11 @@ bool WalletImpl::isKeysFileLocked() { return m_wallet->is_keys_file_locked(); } + +uint64_t WalletImpl::coldKeyImageSync(uint64_t &spent, uint64_t &unspent) +{ + return m_wallet->cold_key_image_sync(spent, unspent); +} } // namespace namespace Bitmonero = Monero; diff --git a/src/wallet/api/wallet.h b/src/wallet/api/wallet.h index bd33b773c..9e07b6e19 100644 --- a/src/wallet/api/wallet.h +++ b/src/wallet/api/wallet.h @@ -89,6 +89,8 @@ public: std::string errorString() const override; void statusWithErrorString(int& status, std::string& errorString) const override; bool setPassword(const std::string &password) override; + bool setDevicePin(const std::string &password) override; + bool setDevicePassphrase(const std::string &password) override; std::string address(uint32_t accountIndex = 0, uint32_t addressIndex = 0) const override; std::string integratedAddress(const std::string &payment_id) const override; std::string secretViewKey() const override; @@ -198,6 +200,7 @@ public: virtual bool lockKeysFile() override; virtual bool unlockKeysFile() override; virtual bool isKeysFileLocked() override; + virtual uint64_t coldKeyImageSync(uint64_t &spent, uint64_t &unspent) override; private: void clearStatus() const; @@ -209,6 +212,7 @@ private: bool daemonSynced() const; void stopRefresh(); bool isNewWallet() const; + void pendingTxPostProcess(PendingTransactionImpl * pending); bool doInit(const std::string &daemon_address, uint64_t upper_transaction_size_limit = 0, bool ssl = false); private: diff --git a/src/wallet/api/wallet2_api.h b/src/wallet/api/wallet2_api.h index c549c260b..ee1d6ae79 100644 --- a/src/wallet/api/wallet2_api.h +++ b/src/wallet/api/wallet2_api.h @@ -324,6 +324,19 @@ struct MultisigState { uint32_t total; }; + +struct DeviceProgress { + DeviceProgress(): m_progress(0), m_indeterminate(false) {} + DeviceProgress(double progress, bool indeterminate=false): m_progress(progress), m_indeterminate(indeterminate) {} + + virtual double progress() const { return m_progress; } + virtual bool indeterminate() const { return m_indeterminate; } + +protected: + double m_progress; + bool m_indeterminate; +}; + struct WalletListener { virtual ~WalletListener() = 0; @@ -364,6 +377,31 @@ struct WalletListener * @brief refreshed - called when wallet refreshed by background thread or explicitly refreshed by calling "refresh" synchronously */ virtual void refreshed() = 0; + + /** + * @brief called by device if the action is required + */ + virtual void onDeviceButtonRequest(uint64_t code) {} + + /** + * @brief called by device when PIN is needed + */ + virtual optional<std::string> onDevicePinRequest() { + throw std::runtime_error("Not supported"); + } + + /** + * @brief called by device when passphrase entry is needed + */ + virtual optional<std::string> onDevicePassphraseRequest(bool on_device) { + if (!on_device) throw std::runtime_error("Not supported"); + return optional<std::string>(); + } + + /** + * @brief Signalizes device operation progress + */ + virtual void onDeviceProgress(const DeviceProgress & event) {}; }; @@ -375,7 +413,8 @@ struct Wallet { enum Device { Device_Software = 0, - Device_Ledger = 1 + Device_Ledger = 1, + Device_Trezor = 2 }; enum Status { @@ -401,6 +440,8 @@ struct Wallet //! returns both error and error string atomically. suggested to use in instead of status() and errorString() virtual void statusWithErrorString(int& status, std::string& errorString) const = 0; virtual bool setPassword(const std::string &password) = 0; + virtual bool setDevicePin(const std::string &password) { return false; }; + virtual bool setDevicePassphrase(const std::string &password) { return false; }; virtual std::string address(uint32_t accountIndex = 0, uint32_t addressIndex = 0) const = 0; std::string mainAddress() const { return address(0, 0); } virtual std::string path() const = 0; @@ -947,6 +988,9 @@ struct Wallet * \return Device they are on */ virtual Device getDeviceType() const = 0; + + //! cold-device protocol key image sync + virtual uint64_t coldKeyImageSync(uint64_t &spent, uint64_t &unspent) = 0; }; /** diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 8bb886c6d..439873932 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -38,6 +38,7 @@ #include <boost/algorithm/string/trim.hpp> #include <boost/algorithm/string/split.hpp> #include <boost/algorithm/string/join.hpp> +#include <boost/asio/ip/address.hpp> #include <boost/range/adaptor/transformed.hpp> #include "include_base_utils.h" using namespace epee; @@ -75,6 +76,7 @@ using namespace epee; #include "ringdb.h" #include "device/device_cold.hpp" #include "device_trezor/device_trezor.hpp" +#include "net/socks_connect.h" extern "C" { @@ -231,6 +233,7 @@ namespace struct options { const command_line::arg_descriptor<std::string> daemon_address = {"daemon-address", tools::wallet2::tr("Use daemon instance at <host>:<port>"), ""}; const command_line::arg_descriptor<std::string> daemon_host = {"daemon-host", tools::wallet2::tr("Use daemon instance at host <arg> instead of localhost"), ""}; + const command_line::arg_descriptor<std::string> proxy = {"proxy", tools::wallet2::tr("[<ip>:]<port> socks proxy to use for daemon connections"), {}, true}; const command_line::arg_descriptor<bool> trusted_daemon = {"trusted-daemon", tools::wallet2::tr("Enable commands which rely on a trusted daemon"), false}; const command_line::arg_descriptor<bool> untrusted_daemon = {"untrusted-daemon", tools::wallet2::tr("Disable commands which rely on a trusted daemon"), false}; const command_line::arg_descriptor<std::string> password = {"password", tools::wallet2::tr("Wallet password (escape/quote as needed)"), "", true}; @@ -303,6 +306,8 @@ std::string get_weight_string(const cryptonote::transaction &tx, size_t blob_siz std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variables_map& vm, bool unattended, const options& opts, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter) { + namespace ip = boost::asio::ip; + const bool testnet = command_line::get_arg(vm, opts.testnet); const bool stagenet = command_line::get_arg(vm, opts.stagenet); const network_type nettype = testnet ? TESTNET : stagenet ? STAGENET : MAINNET; @@ -352,6 +357,44 @@ std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variabl if (daemon_address.empty()) daemon_address = std::string("http://") + daemon_host + ":" + std::to_string(daemon_port); + boost::asio::ip::tcp::endpoint proxy{}; + if (command_line::has_arg(vm, opts.proxy)) + { + namespace ip = boost::asio::ip; + const boost::string_ref real_daemon = boost::string_ref{daemon_address}.substr(0, daemon_address.rfind(':')); + + // onion and i2p addresses contain information about the server cert + // which both authenticates and encrypts + const bool unencrypted_proxy = + !real_daemon.ends_with(".onion") && !real_daemon.ends_with(".i2p") && + daemon_ssl_allowed_certificates.empty() && daemon_ssl_allowed_fingerprints.empty(); + THROW_WALLET_EXCEPTION_IF( + unencrypted_proxy, + tools::error::wallet_internal_error, + std::string{"Use of --"} + opts.proxy.name + " requires --" + opts.daemon_ssl_allowed_certificates.name + " or --" + opts.daemon_ssl_allowed_fingerprints.name + " or use of a .onion/.i2p domain" + ); + + const auto proxy_address = command_line::get_arg(vm, opts.proxy); + + boost::string_ref proxy_port{proxy_address}; + boost::string_ref proxy_host = proxy_port.substr(0, proxy_port.rfind(":")); + if (proxy_port.size() == proxy_host.size()) + proxy_host = "127.0.0.1"; + else + proxy_port = proxy_port.substr(proxy_host.size() + 1); + + uint16_t port_value = 0; + THROW_WALLET_EXCEPTION_IF( + !epee::string_tools::get_xtype_from_string(port_value, std::string{proxy_port}), + tools::error::wallet_internal_error, + std::string{"Invalid port specified for --"} + opts.proxy.name + ); + + boost::system::error_code error{}; + proxy = ip::tcp::endpoint{ip::address::from_string(std::string{proxy_host}, error), port_value}; + THROW_WALLET_EXCEPTION_IF(bool(error), tools::error::wallet_internal_error, std::string{"Invalid IP address specified for --"} + opts.proxy.name); + } + boost::optional<bool> trusted_daemon; if (!command_line::is_arg_defaulted(vm, opts.trusted_daemon) || !command_line::is_arg_defaulted(vm, opts.untrusted_daemon)) trusted_daemon = command_line::get_arg(vm, opts.trusted_daemon) && !command_line::get_arg(vm, opts.untrusted_daemon); @@ -388,8 +431,7 @@ std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variabl std::transform(daemon_ssl_allowed_fingerprints.begin(), daemon_ssl_allowed_fingerprints.end(), ssl_allowed_fingerprints.begin(), epee::from_hex::vector); std::unique_ptr<tools::wallet2> wallet(new tools::wallet2(nettype, kdf_rounds, unattended)); - wallet->init(std::move(daemon_address), std::move(login), 0, *trusted_daemon, ssl_support, std::make_pair(daemon_ssl_private_key, daemon_ssl_certificate), ssl_allowed_certificates, ssl_allowed_fingerprints, daemon_ssl_allow_any_cert); - + wallet->init(std::move(daemon_address), std::move(login), std::move(proxy), 0, *trusted_daemon, ssl_support, std::make_pair(daemon_ssl_private_key, daemon_ssl_certificate), ssl_allowed_certificates, ssl_allowed_fingerprints, daemon_ssl_allow_any_cert); boost::filesystem::path ringdb_path = command_line::get_arg(vm, opts.shared_ringdb_dir); wallet->set_ring_database(ringdb_path.string()); wallet->get_message_store().set_options(vm); @@ -470,7 +512,7 @@ std::pair<std::unique_ptr<tools::wallet2>, tools::password_container> generate_f GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, filename, std::string, String, true, std::string()); GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, scan_from_height, uint64_t, Uint64, false, 0); - const bool recover = field_scan_from_height_found; + const bool recover = true; GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, password, std::string, String, false, std::string()); @@ -580,6 +622,8 @@ std::pair<std::unique_ptr<tools::wallet2>, tools::password_container> generate_f wallet.reset(make_basic(vm, unattended, opts, password_prompter).release()); wallet->set_refresh_from_block_height(field_scan_from_height); wallet->explicit_refresh_from_block_height(field_scan_from_height_found); + if (!old_language.empty()) + wallet->set_seed_language(old_language); try { @@ -928,22 +972,30 @@ wallet_keys_unlocker::~wallet_keys_unlocker() } } -void wallet_device_callback::on_button_request() +void wallet_device_callback::on_button_request(uint64_t code) { if (wallet) - wallet->on_button_request(); + wallet->on_device_button_request(code); } -void wallet_device_callback::on_pin_request(epee::wipeable_string & pin) +boost::optional<epee::wipeable_string> wallet_device_callback::on_pin_request() { if (wallet) - wallet->on_pin_request(pin); + return wallet->on_device_pin_request(); + return boost::none; } -void wallet_device_callback::on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) +boost::optional<epee::wipeable_string> wallet_device_callback::on_passphrase_request(bool on_device) { if (wallet) - wallet->on_passphrase_request(on_device, passphrase); + return wallet->on_device_passphrase_request(on_device); + return boost::none; +} + +void wallet_device_callback::on_progress(const hw::device_progress& event) +{ + if (wallet) + wallet->on_device_progress(event); } wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended): @@ -1038,6 +1090,7 @@ void wallet2::init_options(boost::program_options::options_description& desc_par const options opts{}; command_line::add_arg(desc_params, opts.daemon_address); command_line::add_arg(desc_params, opts.daemon_host); + command_line::add_arg(desc_params, opts.proxy); command_line::add_arg(desc_params, opts.trusted_daemon); command_line::add_arg(desc_params, opts.untrusted_daemon); command_line::add_arg(desc_params, opts.password); @@ -1101,7 +1154,7 @@ std::unique_ptr<wallet2> wallet2::make_dummy(const boost::program_options::varia } //---------------------------------------------------------------------------------------------------- -bool wallet2::init(std::string daemon_address, boost::optional<epee::net_utils::http::login> daemon_login, uint64_t upper_transaction_weight_limit, bool trusted_daemon, epee::net_utils::ssl_support_t ssl_support, const std::pair<std::string, std::string> &private_key_and_certificate_path, const std::list<std::string> &allowed_certificates, const std::vector<std::vector<uint8_t>> &allowed_fingerprints, bool allow_any_cert) +bool wallet2::init(std::string daemon_address, boost::optional<epee::net_utils::http::login> daemon_login, boost::asio::ip::tcp::endpoint proxy, uint64_t upper_transaction_weight_limit, bool trusted_daemon, epee::net_utils::ssl_support_t ssl_support, const std::pair<std::string, std::string> &private_key_and_certificate_path, const std::list<std::string> &allowed_certificates, const std::vector<std::vector<uint8_t>> &allowed_fingerprints, bool allow_any_cert) { m_checkpoints.init_default_checkpoints(m_nettype); if(m_http_client.is_connected()) @@ -1111,6 +1164,10 @@ bool wallet2::init(std::string daemon_address, boost::optional<epee::net_utils:: m_daemon_address = std::move(daemon_address); m_daemon_login = std::move(daemon_login); m_trusted_daemon = trusted_daemon; + if (proxy != boost::asio::ip::tcp::endpoint{}) + m_http_client.set_connector(net::socks::connector{std::move(proxy)}); + + // When switching from light wallet to full wallet, we need to reset the height we got from lw node. return m_http_client.set_server(get_daemon_address(), get_daemon_login(), ssl_support, private_key_and_certificate_path, allowed_certificates, allowed_fingerprints, allow_any_cert); } //---------------------------------------------------------------------------------------------------- @@ -2398,7 +2455,7 @@ void wallet2::process_parsed_blocks(uint64_t start_height, const std::vector<cry " (height " + std::to_string(start_height) + "), local block id at this height: " + string_tools::pod_to_hex(m_blockchain[current_index])); - detach_blockchain(current_index); + detach_blockchain(current_index, output_tracker_cache); process_new_blockchain_entry(bl, blocks[i], parsed_blocks[i], bl_id, current_index, tx_cache_data, tx_cache_data_offset, output_tracker_cache); } else @@ -2842,7 +2899,7 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo // MyMonero get_address_info needs to be called occasionally to trigger wallet sync. // This call is not really needed for other purposes and can be removed if mymonero changes their backend. - cryptonote::COMMAND_RPC_GET_ADDRESS_INFO::response res; + tools::COMMAND_RPC_GET_ADDRESS_INFO::response res; // Get basic info if(light_wallet_get_address_info(res)) { @@ -2883,6 +2940,7 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo std::vector<parsed_block> parsed_blocks; bool refreshed = false; std::shared_ptr<std::map<std::pair<uint64_t, uint64_t>, size_t>> output_tracker_cache; + hw::device &hwdev = m_account.get_device(); // pull the first set of blocks get_short_chain_history(short_chain_history, (m_first_refresh_done || trusted_daemon) ? 1 : FIRST_REFRESH_GRANULARITY); @@ -2989,7 +3047,7 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo // if we've got at least 10 blocks to refresh, assume we're starting // a long refresh, and setup a tracking output cache if we need to - if (m_track_uses && !output_tracker_cache && next_blocks.size() >= 10) + if (m_track_uses && (!output_tracker_cache || output_tracker_cache->empty()) && next_blocks.size() >= 10) output_tracker_cache = create_output_tracker_cache(); // switch to the new blocks from the daemon @@ -3039,6 +3097,7 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo LOG_PRINT_L1("Failed to check pending transactions"); } + hwdev.computing_key_images(false); m_first_refresh_done = true; LOG_PRINT_L1("Refresh done, blocks received: " << blocks_fetched << ", balance (all accounts): " << print_money(balance_all()) << ", unlocked: " << print_money(unlocked_balance_all())); @@ -3129,7 +3188,7 @@ bool wallet2::get_rct_distribution(uint64_t &start_height, std::vector<uint64_t> return true; } //---------------------------------------------------------------------------------------------------- -void wallet2::detach_blockchain(uint64_t height) +void wallet2::detach_blockchain(uint64_t height, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache) { LOG_PRINT_L0("Detaching blockchain on height " << height); @@ -3151,6 +3210,15 @@ void wallet2::detach_blockchain(uint64_t height) } } + for (transfer_details &td: m_transfers) + { + while (!td.m_uses.empty() && td.m_uses.back().first >= height) + td.m_uses.pop_back(); + } + + if (output_tracker_cache) + output_tracker_cache->clear(); + auto it = std::find_if(m_transfers.begin(), m_transfers.end(), [&](const transfer_details& td){return td.m_block_height >= height;}); size_t i_start = it - m_transfers.begin(); @@ -3222,6 +3290,26 @@ bool wallet2::clear() m_device_last_key_image_sync = 0; return true; } +//---------------------------------------------------------------------------------------------------- +void wallet2::clear_soft(bool keep_key_images) +{ + m_blockchain.clear(); + m_transfers.clear(); + if (!keep_key_images) + m_key_images.clear(); + m_pub_keys.clear(); + m_unconfirmed_txs.clear(); + m_payments.clear(); + m_confirmed_txs.clear(); + m_unconfirmed_payments.clear(); + m_scanned_pool_txs[0].clear(); + m_scanned_pool_txs[1].clear(); + + cryptonote::block b; + generate_genesis(b); + m_blockchain.push_back(get_block_hash(b)); + m_last_block_reward = cryptonote::get_outs_money_amount(b.miner_tx); +} /*! * \brief Stores wallet information to wallet file. @@ -3460,7 +3548,8 @@ void wallet2::change_password(const std::string &filename, const epee::wipeable_ decrypt_keys(original_password); setup_keys(new_password); rewrite(filename, new_password); - store(); + if (!filename.empty()) + store(); } //---------------------------------------------------------------------------------------------------- /*! @@ -5090,7 +5179,14 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass MERROR("Failed to save rings, will try again next time"); } - m_message_store.read_from_file(get_multisig_wallet_state(), m_mms_file); + try + { + m_message_store.read_from_file(get_multisig_wallet_state(), m_mms_file); + } + catch (const std::exception &e) + { + MERROR("Failed to initialize MMS, it will be unusable"); + } } //---------------------------------------------------------------------------------------------------- void wallet2::trim_hashchain() @@ -5142,7 +5238,8 @@ std::string wallet2::path() const //---------------------------------------------------------------------------------------------------- void wallet2::store() { - store_to("", epee::wipeable_string()); + if (!m_wallet_file.empty()) + store_to("", epee::wipeable_string()); } //---------------------------------------------------------------------------------------------------- void wallet2::store_to(const std::string &path, const epee::wipeable_string &password) @@ -5468,8 +5565,12 @@ void wallet2::rescan_spent() } } //---------------------------------------------------------------------------------------------------- -void wallet2::rescan_blockchain(bool hard, bool refresh) +void wallet2::rescan_blockchain(bool hard, bool refresh, bool keep_key_images) { + CHECK_AND_ASSERT_THROW_MES(!hard || !keep_key_images, "Cannot preserve key images on hard rescan"); + const size_t transfers_cnt = m_transfers.size(); + crypto::hash transfers_hash{}; + if(hard) { clear(); @@ -5477,25 +5578,16 @@ void wallet2::rescan_blockchain(bool hard, bool refresh) } else { - m_blockchain.clear(); - m_transfers.clear(); - m_key_images.clear(); - m_pub_keys.clear(); - m_unconfirmed_txs.clear(); - m_payments.clear(); - m_confirmed_txs.clear(); - m_unconfirmed_payments.clear(); - m_scanned_pool_txs[0].clear(); - m_scanned_pool_txs[1].clear(); - - cryptonote::block b; - generate_genesis(b); - m_blockchain.push_back(get_block_hash(b)); - m_last_block_reward = cryptonote::get_outs_money_amount(b.miner_tx); + if (keep_key_images && refresh) + hash_m_transfers((int64_t) transfers_cnt, transfers_hash); + clear_soft(keep_key_images); } if (refresh) this->refresh(false); + + if (refresh && keep_key_images) + finish_rescan_bc_keep_key_images(transfers_cnt, transfers_hash); } //---------------------------------------------------------------------------------------------------- bool wallet2::is_transfer_unlocked(const transfer_details& td) const @@ -5980,12 +6072,7 @@ bool wallet2::sign_tx(unsigned_tx_set &exported_txs, std::vector<wallet2::pendin LOG_PRINT_L1(" " << (n+1) << ": " << sd.sources.size() << " inputs, ring size " << sd.sources[0].outputs.size()); signed_txes.ptx.push_back(pending_tx()); tools::wallet2::pending_tx &ptx = signed_txes.ptx.back(); - rct::RCTConfig rct_config = { rct::RangeProofBorromean, 0 }; - if (sd.use_bulletproofs) - { - rct_config.range_proof_type = rct::RangeProofPaddedBulletproof; - rct_config.bp_version = use_fork_rules(HF_VERSION_SMALLER_BP, -10) ? 2 : 1; - } + rct::RCTConfig rct_config = sd.rct_config; crypto::secret_key tx_key; std::vector<crypto::secret_key> additional_tx_keys; rct::multisig_out msout; @@ -6334,17 +6421,17 @@ bool wallet2::save_multisig_tx(const std::vector<pending_tx>& ptx_vector, const return epee::file_io_utils::save_string_to_file(filename, ciphertext); } //---------------------------------------------------------------------------------------------------- -bool wallet2::load_multisig_tx(cryptonote::blobdata s, multisig_tx_set &exported_txs, std::function<bool(const multisig_tx_set&)> accept_func) +bool wallet2::parse_multisig_tx_from_str(std::string multisig_tx_st, multisig_tx_set &exported_txs) const { const size_t magiclen = strlen(MULTISIG_UNSIGNED_TX_PREFIX); - if (strncmp(s.c_str(), MULTISIG_UNSIGNED_TX_PREFIX, magiclen)) + if (strncmp(multisig_tx_st.c_str(), MULTISIG_UNSIGNED_TX_PREFIX, magiclen)) { LOG_PRINT_L0("Bad magic from multisig tx data"); return false; } try { - s = decrypt_with_view_secret_key(std::string(s, magiclen)); + multisig_tx_st = decrypt_with_view_secret_key(std::string(multisig_tx_st, magiclen)); } catch (const std::exception &e) { @@ -6353,7 +6440,7 @@ bool wallet2::load_multisig_tx(cryptonote::blobdata s, multisig_tx_set &exported } try { - std::istringstream iss(s); + std::istringstream iss(multisig_tx_st); boost::archive::portable_binary_iarchive ar(iss); ar >> exported_txs; } @@ -6375,6 +6462,17 @@ bool wallet2::load_multisig_tx(cryptonote::blobdata s, multisig_tx_set &exported CHECK_AND_ASSERT_MES(ptx.construction_data.sources.size() == ptx.tx.vin.size(), false, "Mismatched sources/vin sizes"); } + return true; +} +//---------------------------------------------------------------------------------------------------- +bool wallet2::load_multisig_tx(cryptonote::blobdata s, multisig_tx_set &exported_txs, std::function<bool(const multisig_tx_set&)> accept_func) +{ + if(!parse_multisig_tx_from_str(s, exported_txs)) + { + LOG_PRINT_L0("Failed to parse multisig transaction from string"); + return false; + } + LOG_PRINT_L1("Loaded multisig tx unsigned data from binary: " << exported_txs.m_ptx.size() << " transactions"); for (auto &ptx: exported_txs.m_ptx) LOG_PRINT_L0(cryptonote::obj_to_json_str(ptx.tx)); @@ -6451,12 +6549,7 @@ bool wallet2::sign_multisig_tx(multisig_tx_set &exported_txs, std::vector<crypto cryptonote::transaction tx; rct::multisig_out msout = ptx.multisig_sigs.front().msout; auto sources = sd.sources; - rct::RCTConfig rct_config = { rct::RangeProofBorromean, 0 }; - if (sd.use_bulletproofs) - { - rct_config.range_proof_type = rct::RangeProofPaddedBulletproof; - rct_config.bp_version = use_fork_rules(HF_VERSION_SMALLER_BP, -10) ? 2 : 1; - } + rct::RCTConfig rct_config = sd.rct_config; bool r = cryptonote::construct_tx_with_tx_key(m_account.get_keys(), m_subaddresses, sources, sd.splitted_dsts, ptx.change_dts.addr, sd.extra, tx, sd.unlock_time, ptx.tx_key, ptx.additional_tx_keys, sd.use_rct, rct_config, &msout, false); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sd.sources, sd.splitted_dsts, sd.unlock_time, m_nettype); @@ -7024,6 +7117,17 @@ bool wallet2::tx_add_fake_output(std::vector<std::vector<tools::wallet2::get_out CHECK_AND_ASSERT_MES(!outs.empty(), false, "internal error: outs is empty"); if (std::find(outs.back().begin(), outs.back().end(), item) != outs.back().end()) // don't add duplicates return false; + // check the keys are valid + if (!rct::isInMainSubgroup(rct::pk2rct(output_public_key))) + { + MWARNING("Key " << output_public_key << " at index " << global_index << " is not in the main subgroup"); + return false; + } + if (!rct::isInMainSubgroup(mask)) + { + MWARNING("Commitment " << mask << " at index " << global_index << " is not in the main subgroup"); + return false; + } // if (is_output_blackballed(output_public_key)) // don't add blackballed outputs // return false; outs.back().push_back(item); @@ -7906,7 +8010,7 @@ void wallet2::transfer_selected(const std::vector<cryptonote::tx_destination_ent ptx.construction_data.extra = tx.extra; ptx.construction_data.unlock_time = unlock_time; ptx.construction_data.use_rct = false; - ptx.construction_data.use_bulletproofs = false; + ptx.construction_data.rct_config = { rct::RangeProofBorromean, 0 }; ptx.construction_data.dests = dsts; // record which subaddress indices are being used as inputs ptx.construction_data.subaddr_account = subaddr_account; @@ -8092,6 +8196,7 @@ void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry else { change_dts.addr = get_subaddress({subaddr_account, 0}); + change_dts.is_subaddress = subaddr_account != 0; splitted_dsts.push_back(change_dts); } @@ -8188,7 +8293,7 @@ void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry ptx.construction_data.extra = tx.extra; ptx.construction_data.unlock_time = unlock_time; ptx.construction_data.use_rct = true; - ptx.construction_data.use_bulletproofs = !tx.rct_signatures.p.bulletproofs.empty(); + ptx.construction_data.rct_config = { tx.rct_signatures.p.bulletproofs.empty() ? rct::RangeProofBorromean : rct::RangeProofPaddedBulletproof, use_fork_rules(HF_VERSION_SMALLER_BP, -10) ? 2 : 1}; ptx.construction_data.dests = dsts; // record which subaddress indices are being used as inputs ptx.construction_data.subaddr_account = subaddr_account; @@ -8316,8 +8421,8 @@ bool wallet2::light_wallet_login(bool &new_address) { MDEBUG("Light wallet login request"); m_light_wallet_connected = false; - cryptonote::COMMAND_RPC_LOGIN::request request; - cryptonote::COMMAND_RPC_LOGIN::response response; + tools::COMMAND_RPC_LOGIN::request request; + tools::COMMAND_RPC_LOGIN::response response; request.address = get_account().get_public_address_str(m_nettype); request.view_key = string_tools::pod_to_hex(get_account().get_keys().m_view_secret_key); // Always create account if it doesn't exist. @@ -8341,10 +8446,10 @@ bool wallet2::light_wallet_login(bool &new_address) return m_light_wallet_connected; } -bool wallet2::light_wallet_import_wallet_request(cryptonote::COMMAND_RPC_IMPORT_WALLET_REQUEST::response &response) +bool wallet2::light_wallet_import_wallet_request(tools::COMMAND_RPC_IMPORT_WALLET_REQUEST::response &response) { MDEBUG("Light wallet import wallet request"); - cryptonote::COMMAND_RPC_IMPORT_WALLET_REQUEST::request oreq; + tools::COMMAND_RPC_IMPORT_WALLET_REQUEST::request oreq; oreq.address = get_account().get_public_address_str(m_nettype); oreq.view_key = string_tools::pod_to_hex(get_account().get_keys().m_view_secret_key); m_daemon_rpc_mutex.lock(); @@ -8360,8 +8465,8 @@ void wallet2::light_wallet_get_unspent_outs() { MDEBUG("Getting unspent outs"); - cryptonote::COMMAND_RPC_GET_UNSPENT_OUTS::request oreq; - cryptonote::COMMAND_RPC_GET_UNSPENT_OUTS::response ores; + tools::COMMAND_RPC_GET_UNSPENT_OUTS::request oreq; + tools::COMMAND_RPC_GET_UNSPENT_OUTS::response ores; oreq.amount = "0"; oreq.address = get_account().get_public_address_str(m_nettype); @@ -8509,11 +8614,11 @@ void wallet2::light_wallet_get_unspent_outs() } } -bool wallet2::light_wallet_get_address_info(cryptonote::COMMAND_RPC_GET_ADDRESS_INFO::response &response) +bool wallet2::light_wallet_get_address_info(tools::COMMAND_RPC_GET_ADDRESS_INFO::response &response) { MTRACE(__FUNCTION__); - cryptonote::COMMAND_RPC_GET_ADDRESS_INFO::request request; + tools::COMMAND_RPC_GET_ADDRESS_INFO::request request; request.address = get_account().get_public_address_str(m_nettype); request.view_key = string_tools::pod_to_hex(get_account().get_keys().m_view_secret_key); @@ -8529,8 +8634,8 @@ void wallet2::light_wallet_get_address_txs() { MDEBUG("Refreshing light wallet"); - cryptonote::COMMAND_RPC_GET_ADDRESS_TXS::request ireq; - cryptonote::COMMAND_RPC_GET_ADDRESS_TXS::response ires; + tools::COMMAND_RPC_GET_ADDRESS_TXS::request ireq; + tools::COMMAND_RPC_GET_ADDRESS_TXS::response ires; ireq.address = get_account().get_public_address_str(m_nettype); ireq.view_key = string_tools::pod_to_hex(get_account().get_keys().m_view_secret_key); @@ -8803,6 +8908,8 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp boost::unique_lock<hw::device> hwdev_lock (hwdev); hw::reset_mode rst(hwdev); + auto original_dsts = dsts; + if(m_light_wallet) { // Populate m_transfers light_wallet_get_unspent_outs(); @@ -9323,10 +9430,77 @@ skip_tx: ptx_vector.push_back(tx.ptx); } + THROW_WALLET_EXCEPTION_IF(!sanity_check(ptx_vector, original_dsts), error::wallet_internal_error, "Created transaction(s) failed sanity check"); + // if we made it this far, we're OK to actually send the transactions return ptx_vector; } +bool wallet2::sanity_check(const std::vector<wallet2::pending_tx> &ptx_vector, std::vector<cryptonote::tx_destination_entry> dsts) const +{ + MDEBUG("sanity_check: " << ptx_vector.size() << " txes, " << dsts.size() << " destinations"); + + hw::device &hwdev = m_account.get_device(); + + THROW_WALLET_EXCEPTION_IF(ptx_vector.empty(), error::wallet_internal_error, "No transactions"); + + // check every party in there does receive at least the required amount + std::unordered_map<account_public_address, std::pair<uint64_t, bool>> required; + for (const auto &d: dsts) + { + required[d.addr].first += d.amount; + required[d.addr].second = d.is_subaddress; + } + + // add change + uint64_t change = 0; + for (const auto &ptx: ptx_vector) + { + for (size_t idx: ptx.selected_transfers) + change += m_transfers[idx].amount(); + change -= ptx.fee; + } + for (const auto &r: required) + change -= r.second.first; + MDEBUG("Adding " << cryptonote::print_money(change) << " expected change"); + + for (const pending_tx &ptx: ptx_vector) + THROW_WALLET_EXCEPTION_IF(ptx.change_dts.addr != ptx_vector[0].change_dts.addr, error::wallet_internal_error, + "Change goes to several different addresses"); + const auto it = m_subaddresses.find(ptx_vector[0].change_dts.addr.m_spend_public_key); + THROW_WALLET_EXCEPTION_IF(it == m_subaddresses.end(), error::wallet_internal_error, "Change address is not ours"); + + required[ptx_vector[0].change_dts.addr].first += change; + required[ptx_vector[0].change_dts.addr].second = ptx_vector[0].change_dts.is_subaddress; + + for (const auto &r: required) + { + const account_public_address &address = r.first; + const crypto::public_key &view_pkey = address.m_view_public_key; + + uint64_t total_received = 0; + for (const auto &ptx: ptx_vector) + { + uint64_t received = 0; + try + { + std::string proof = get_tx_proof(ptx.tx, ptx.tx_key, ptx.additional_tx_keys, address, r.second.second, "automatic-sanity-check"); + check_tx_proof(ptx.tx, address, r.second.second, "automatic-sanity-check", proof, received); + } + catch (const std::exception &e) { received = 0; } + total_received += received; + } + + std::stringstream ss; + ss << "Total received by " << cryptonote::get_account_address_as_str(m_nettype, r.second.second, address) << ": " + << cryptonote::print_money(total_received) << ", expected " << cryptonote::print_money(r.second.first); + MDEBUG(ss.str()); + THROW_WALLET_EXCEPTION_IF(total_received < r.second.first, error::wallet_internal_error, ss.str()); + } + + return true; +} + std::vector<wallet2::pending_tx> wallet2::create_transactions_all(uint64_t below, const cryptonote::account_public_address &address, bool is_subaddress, const size_t outputs, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices) { std::vector<size_t> unused_transfers_indices; @@ -9609,6 +9783,12 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton ptx_vector.push_back(tx.ptx); } + uint64_t a = 0; + for (size_t idx: unused_transfers_indices) a += m_transfers[idx].amount(); + for (size_t idx: unused_dust_indices) a += m_transfers[idx].amount(); + std::vector<cryptonote::tx_destination_entry> synthetic_dsts(1, cryptonote::tx_destination_entry("", a, address, is_subaddress)); + THROW_WALLET_EXCEPTION_IF(!sanity_check(ptx_vector, synthetic_dsts), error::wallet_internal_error, "Created transaction(s) failed sanity check"); + // if we made it this far, we're OK to actually send the transactions return ptx_vector; } @@ -9644,6 +9824,7 @@ void wallet2::cold_sign_tx(const std::vector<pending_tx>& ptx_vector, signed_tx_ hw::wallet_shim wallet_shim; setup_shim(&wallet_shim, this); aux_data.tx_recipients = dsts_info; + aux_data.bp_version = use_fork_rules(HF_VERSION_SMALLER_BP, -10) ? 2 : 1; dev_cold->tx_sign(&wallet_shim, txs, exported_txs, aux_data); tx_device_aux = aux_data.tx_device_aux; @@ -9688,7 +9869,7 @@ bool wallet2::use_fork_rules(uint8_t version, int64_t early_blocks) const result = m_node_rpc_proxy.get_earliest_height(version, earliest_height); throw_on_rpc_response_error(result, "get_hard_fork_info"); - bool close_enough = height >= earliest_height - early_blocks && earliest_height != std::numeric_limits<uint64_t>::max(); // start using the rules that many blocks beforehand + bool close_enough = (int64_t)height >= (int64_t)earliest_height - early_blocks && earliest_height != std::numeric_limits<uint64_t>::max(); // start using the rules that many blocks beforehand if (close_enough) LOG_PRINT_L2("Using v" << (unsigned)version << " rules"); else @@ -9860,7 +10041,7 @@ void wallet2::discard_unmixable_outputs() } } -bool wallet2::get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys) const +bool wallet2::get_tx_key_cached(const crypto::hash &txid, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys) const { additional_tx_keys.clear(); const std::unordered_map<crypto::hash, crypto::secret_key>::const_iterator i = m_tx_keys.find(txid); @@ -9873,6 +10054,82 @@ bool wallet2::get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, s return true; } //---------------------------------------------------------------------------------------------------- +bool wallet2::get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys) +{ + bool r = get_tx_key_cached(txid, tx_key, additional_tx_keys); + if (r) + { + return true; + } + + auto & hwdev = get_account().get_device(); + + // So far only Cold protocol devices are supported. + if (hwdev.device_protocol() != hw::device::PROTOCOL_COLD) + { + return false; + } + + const auto tx_data_it = m_tx_device.find(txid); + if (tx_data_it == m_tx_device.end()) + { + MDEBUG("Aux data not found for txid: " << txid); + return false; + } + + auto dev_cold = dynamic_cast<::hw::device_cold*>(&hwdev); + CHECK_AND_ASSERT_THROW_MES(dev_cold, "Device does not implement cold signing interface"); + if (!dev_cold->is_get_tx_key_supported()) + { + MDEBUG("get_tx_key not supported by the device"); + return false; + } + + hw::device_cold::tx_key_data_t tx_key_data; + dev_cold->load_tx_key_data(tx_key_data, tx_data_it->second); + + // Load missing tx prefix hash + if (tx_key_data.tx_prefix_hash.empty()) + { + COMMAND_RPC_GET_TRANSACTIONS::request req; + COMMAND_RPC_GET_TRANSACTIONS::response res; + req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); + req.decode_as_json = false; + req.prune = true; + m_daemon_rpc_mutex.lock(); + bool ok = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client); + m_daemon_rpc_mutex.unlock(); + THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1), + error::wallet_internal_error, "Failed to get transaction from daemon"); + + cryptonote::transaction tx; + crypto::hash tx_hash{}; + cryptonote::blobdata tx_data; + crypto::hash tx_prefix_hash{}; + ok = string_tools::parse_hexstr_to_binbuff(res.txs_as_hex.front(), tx_data); + THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx, tx_hash, tx_prefix_hash), + error::wallet_internal_error, "Failed to validate transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, + "Failed to get the right transaction from daemon"); + + tx_key_data.tx_prefix_hash = std::string(tx_prefix_hash.data, 32); + } + + std::vector<crypto::secret_key> tx_keys; + dev_cold->get_tx_key(tx_keys, tx_key_data, m_account.get_keys().m_view_secret_key); + if (tx_keys.empty()) + { + return false; + } + + tx_key = tx_keys[0]; + tx_keys.erase(tx_keys.begin()); + additional_tx_keys = tx_keys; + + return true; +} +//---------------------------------------------------------------------------------------------------- void wallet2::set_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys) { // fetch tx from daemon and check if secret keys agree with corresponding public keys @@ -10164,41 +10421,8 @@ void wallet2::check_tx_key(const crypto::hash &txid, const crypto::secret_key &t check_tx_key_helper(txid, derivation, additional_derivations, address, received, in_pool, confirmations); } -void wallet2::check_tx_key_helper(const crypto::hash &txid, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, const cryptonote::account_public_address &address, uint64_t &received, bool &in_pool, uint64_t &confirmations) +void wallet2::check_tx_key_helper(const cryptonote::transaction &tx, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, const cryptonote::account_public_address &address, uint64_t &received) const { - COMMAND_RPC_GET_TRANSACTIONS::request req; - COMMAND_RPC_GET_TRANSACTIONS::response res; - req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); - req.decode_as_json = false; - req.prune = true; - m_daemon_rpc_mutex.lock(); - bool ok = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client); - m_daemon_rpc_mutex.unlock(); - THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1), - error::wallet_internal_error, "Failed to get transaction from daemon"); - - cryptonote::transaction tx; - crypto::hash tx_hash; - if (res.txs.size() == 1) - { - ok = get_pruned_tx(res.txs.front(), tx, tx_hash); - THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); - } - else - { - cryptonote::blobdata tx_data; - ok = string_tools::parse_hexstr_to_binbuff(res.txs_as_hex.front(), tx_data); - THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); - THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx), - error::wallet_internal_error, "Failed to validate transaction from daemon"); - tx_hash = cryptonote::get_transaction_hash(tx); - } - - THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, - "Failed to get the right transaction from daemon"); - THROW_WALLET_EXCEPTION_IF(!additional_derivations.empty() && additional_derivations.size() != tx.vout.size(), error::wallet_internal_error, - "The size of additional derivations is wrong"); - received = 0; hw::device &hwdev = m_account.get_device(); for (size_t n = 0; n < tx.vout.size(); ++n) @@ -10246,6 +10470,44 @@ void wallet2::check_tx_key_helper(const crypto::hash &txid, const crypto::key_de received += amount; } } +} + +void wallet2::check_tx_key_helper(const crypto::hash &txid, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, const cryptonote::account_public_address &address, uint64_t &received, bool &in_pool, uint64_t &confirmations) +{ + COMMAND_RPC_GET_TRANSACTIONS::request req; + COMMAND_RPC_GET_TRANSACTIONS::response res; + req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); + req.decode_as_json = false; + req.prune = true; + m_daemon_rpc_mutex.lock(); + bool ok = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client); + m_daemon_rpc_mutex.unlock(); + THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1), + error::wallet_internal_error, "Failed to get transaction from daemon"); + + cryptonote::transaction tx; + crypto::hash tx_hash; + if (res.txs.size() == 1) + { + ok = get_pruned_tx(res.txs.front(), tx, tx_hash); + THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + } + else + { + cryptonote::blobdata tx_data; + ok = string_tools::parse_hexstr_to_binbuff(res.txs_as_hex.front(), tx_data); + THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx), + error::wallet_internal_error, "Failed to validate transaction from daemon"); + tx_hash = cryptonote::get_transaction_hash(tx); + } + + THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, + "Failed to get the right transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(!additional_derivations.empty() && additional_derivations.size() != tx.vout.size(), error::wallet_internal_error, + "The size of additional derivations is wrong"); + + check_tx_key_helper(tx, derivation, additional_derivations, address, received); in_pool = res.txs.front().in_pool; confirmations = (uint64_t)-1; @@ -10260,9 +10522,55 @@ void wallet2::check_tx_key_helper(const crypto::hash &txid, const crypto::key_de std::string wallet2::get_tx_proof(const crypto::hash &txid, const cryptonote::account_public_address &address, bool is_subaddress, const std::string &message) { + // fetch tx pubkey from the daemon + COMMAND_RPC_GET_TRANSACTIONS::request req; + COMMAND_RPC_GET_TRANSACTIONS::response res; + req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); + req.decode_as_json = false; + req.prune = true; + m_daemon_rpc_mutex.lock(); + bool ok = net_utils::invoke_http_json("/gettransactions", req, res, m_http_client); + m_daemon_rpc_mutex.unlock(); + THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1), + error::wallet_internal_error, "Failed to get transaction from daemon"); + + cryptonote::transaction tx; + crypto::hash tx_hash; + if (res.txs.size() == 1) + { + ok = get_pruned_tx(res.txs.front(), tx, tx_hash); + THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + } + else + { + cryptonote::blobdata tx_data; + ok = string_tools::parse_hexstr_to_binbuff(res.txs_as_hex.front(), tx_data); + THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx), + error::wallet_internal_error, "Failed to validate transaction from daemon"); + tx_hash = cryptonote::get_transaction_hash(tx); + } + + THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, "Failed to get the right transaction from daemon"); + + // determine if the address is found in the subaddress hash table (i.e. whether the proof is outbound or inbound) + crypto::secret_key tx_key = crypto::null_skey; + std::vector<crypto::secret_key> additional_tx_keys; + const bool is_out = m_subaddresses.count(address.m_spend_public_key) == 0; + if (is_out) + { + THROW_WALLET_EXCEPTION_IF(!get_tx_key(txid, tx_key, additional_tx_keys), error::wallet_internal_error, "Tx secret key wasn't found in the wallet file."); + } + + return get_tx_proof(tx, tx_key, additional_tx_keys, address, is_subaddress, message); +} + +std::string wallet2::get_tx_proof(const cryptonote::transaction &tx, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, const cryptonote::account_public_address &address, bool is_subaddress, const std::string &message) const +{ // determine if the address is found in the subaddress hash table (i.e. whether the proof is outbound or inbound) const bool is_out = m_subaddresses.count(address.m_spend_public_key) == 0; + const crypto::hash txid = cryptonote::get_transaction_hash(tx); std::string prefix_data((const char*)&txid, sizeof(crypto::hash)); prefix_data += message; crypto::hash prefix_hash; @@ -10273,10 +10581,6 @@ std::string wallet2::get_tx_proof(const crypto::hash &txid, const cryptonote::ac std::string sig_str; if (is_out) { - crypto::secret_key tx_key; - std::vector<crypto::secret_key> additional_tx_keys; - THROW_WALLET_EXCEPTION_IF(!get_tx_key(txid, tx_key, additional_tx_keys), error::wallet_internal_error, "Tx secret key wasn't found in the wallet file."); - const size_t num_sigs = 1 + additional_tx_keys.size(); shared_secret.resize(num_sigs); sig.resize(num_sigs); @@ -10311,37 +10615,6 @@ std::string wallet2::get_tx_proof(const crypto::hash &txid, const cryptonote::ac } else { - // fetch tx pubkey from the daemon - COMMAND_RPC_GET_TRANSACTIONS::request req; - COMMAND_RPC_GET_TRANSACTIONS::response res; - req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); - req.decode_as_json = false; - req.prune = true; - m_daemon_rpc_mutex.lock(); - bool ok = net_utils::invoke_http_json("/gettransactions", req, res, m_http_client); - m_daemon_rpc_mutex.unlock(); - THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1), - error::wallet_internal_error, "Failed to get transaction from daemon"); - - cryptonote::transaction tx; - crypto::hash tx_hash; - if (res.txs.size() == 1) - { - ok = get_pruned_tx(res.txs.front(), tx, tx_hash); - THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); - } - else - { - cryptonote::blobdata tx_data; - ok = string_tools::parse_hexstr_to_binbuff(res.txs_as_hex.front(), tx_data); - THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); - THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx), - error::wallet_internal_error, "Failed to validate transaction from daemon"); - tx_hash = cryptonote::get_transaction_hash(tx); - } - - THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, "Failed to get the right transaction from daemon"); - crypto::public_key tx_pub_key = get_tx_pub_key_from_extra(tx); THROW_WALLET_EXCEPTION_IF(tx_pub_key == null_pkey, error::wallet_internal_error, "Tx pubkey was not found"); @@ -10383,9 +10656,7 @@ std::string wallet2::get_tx_proof(const crypto::hash &txid, const cryptonote::ac for (size_t i = 1; i < num_sigs; ++i) THROW_WALLET_EXCEPTION_IF(!crypto::generate_key_derivation(shared_secret[i], rct::rct2sk(rct::I), additional_derivations[i - 1]), error::wallet_internal_error, "Failed to generate key derivation"); uint64_t received; - bool in_pool; - uint64_t confirmations; - check_tx_key_helper(txid, derivation, additional_derivations, address, received, in_pool, confirmations); + check_tx_key_helper(tx, derivation, additional_derivations, address, received); THROW_WALLET_EXCEPTION_IF(!received, error::wallet_internal_error, tr("No funds received in this tx.")); // concatenate all signature strings @@ -10398,37 +10669,6 @@ std::string wallet2::get_tx_proof(const crypto::hash &txid, const cryptonote::ac bool wallet2::check_tx_proof(const crypto::hash &txid, const cryptonote::account_public_address &address, bool is_subaddress, const std::string &message, const std::string &sig_str, uint64_t &received, bool &in_pool, uint64_t &confirmations) { - const bool is_out = sig_str.substr(0, 3) == "Out"; - const std::string header = is_out ? "OutProofV1" : "InProofV1"; - const size_t header_len = header.size(); - THROW_WALLET_EXCEPTION_IF(sig_str.size() < header_len || sig_str.substr(0, header_len) != header, error::wallet_internal_error, - "Signature header check error"); - - // decode base58 - std::vector<crypto::public_key> shared_secret(1); - std::vector<crypto::signature> sig(1); - const size_t pk_len = tools::base58::encode(std::string((const char *)&shared_secret[0], sizeof(crypto::public_key))).size(); - const size_t sig_len = tools::base58::encode(std::string((const char *)&sig[0], sizeof(crypto::signature))).size(); - const size_t num_sigs = (sig_str.size() - header_len) / (pk_len + sig_len); - THROW_WALLET_EXCEPTION_IF(sig_str.size() != header_len + num_sigs * (pk_len + sig_len), error::wallet_internal_error, - "Wrong signature size"); - shared_secret.resize(num_sigs); - sig.resize(num_sigs); - for (size_t i = 0; i < num_sigs; ++i) - { - std::string pk_decoded; - std::string sig_decoded; - const size_t offset = header_len + i * (pk_len + sig_len); - THROW_WALLET_EXCEPTION_IF(!tools::base58::decode(sig_str.substr(offset, pk_len), pk_decoded), error::wallet_internal_error, - "Signature decoding error"); - THROW_WALLET_EXCEPTION_IF(!tools::base58::decode(sig_str.substr(offset + pk_len, sig_len), sig_decoded), error::wallet_internal_error, - "Signature decoding error"); - THROW_WALLET_EXCEPTION_IF(sizeof(crypto::public_key) != pk_decoded.size() || sizeof(crypto::signature) != sig_decoded.size(), error::wallet_internal_error, - "Signature decoding error"); - memcpy(&shared_secret[i], pk_decoded.data(), sizeof(crypto::public_key)); - memcpy(&sig[i], sig_decoded.data(), sizeof(crypto::signature)); - } - // fetch tx pubkey from the daemon COMMAND_RPC_GET_TRANSACTIONS::request req; COMMAND_RPC_GET_TRANSACTIONS::response res; @@ -10460,12 +10700,62 @@ bool wallet2::check_tx_proof(const crypto::hash &txid, const cryptonote::account THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, "Failed to get the right transaction from daemon"); + if (!check_tx_proof(tx, address, is_subaddress, message, sig_str, received)) + return false; + + in_pool = res.txs.front().in_pool; + confirmations = (uint64_t)-1; + if (!in_pool) + { + std::string err; + uint64_t bc_height = get_daemon_blockchain_height(err); + if (err.empty()) + confirmations = bc_height - (res.txs.front().block_height + 1); + } + + return true; +} + +bool wallet2::check_tx_proof(const cryptonote::transaction &tx, const cryptonote::account_public_address &address, bool is_subaddress, const std::string &message, const std::string &sig_str, uint64_t &received) const +{ + const bool is_out = sig_str.substr(0, 3) == "Out"; + const std::string header = is_out ? "OutProofV1" : "InProofV1"; + const size_t header_len = header.size(); + THROW_WALLET_EXCEPTION_IF(sig_str.size() < header_len || sig_str.substr(0, header_len) != header, error::wallet_internal_error, + "Signature header check error"); + + // decode base58 + std::vector<crypto::public_key> shared_secret(1); + std::vector<crypto::signature> sig(1); + const size_t pk_len = tools::base58::encode(std::string((const char *)&shared_secret[0], sizeof(crypto::public_key))).size(); + const size_t sig_len = tools::base58::encode(std::string((const char *)&sig[0], sizeof(crypto::signature))).size(); + const size_t num_sigs = (sig_str.size() - header_len) / (pk_len + sig_len); + THROW_WALLET_EXCEPTION_IF(sig_str.size() != header_len + num_sigs * (pk_len + sig_len), error::wallet_internal_error, + "Wrong signature size"); + shared_secret.resize(num_sigs); + sig.resize(num_sigs); + for (size_t i = 0; i < num_sigs; ++i) + { + std::string pk_decoded; + std::string sig_decoded; + const size_t offset = header_len + i * (pk_len + sig_len); + THROW_WALLET_EXCEPTION_IF(!tools::base58::decode(sig_str.substr(offset, pk_len), pk_decoded), error::wallet_internal_error, + "Signature decoding error"); + THROW_WALLET_EXCEPTION_IF(!tools::base58::decode(sig_str.substr(offset + pk_len, sig_len), sig_decoded), error::wallet_internal_error, + "Signature decoding error"); + THROW_WALLET_EXCEPTION_IF(sizeof(crypto::public_key) != pk_decoded.size() || sizeof(crypto::signature) != sig_decoded.size(), error::wallet_internal_error, + "Signature decoding error"); + memcpy(&shared_secret[i], pk_decoded.data(), sizeof(crypto::public_key)); + memcpy(&sig[i], sig_decoded.data(), sizeof(crypto::signature)); + } + crypto::public_key tx_pub_key = get_tx_pub_key_from_extra(tx); THROW_WALLET_EXCEPTION_IF(tx_pub_key == null_pkey, error::wallet_internal_error, "Tx pubkey was not found"); std::vector<crypto::public_key> additional_tx_pub_keys = get_additional_tx_pub_keys_from_extra(tx); THROW_WALLET_EXCEPTION_IF(additional_tx_pub_keys.size() + 1 != num_sigs, error::wallet_internal_error, "Signature size mismatch with additional tx pubkeys"); + const crypto::hash txid = cryptonote::get_transaction_hash(tx); std::string prefix_data((const char*)&txid, sizeof(crypto::hash)); prefix_data += message; crypto::hash prefix_hash; @@ -10512,7 +10802,7 @@ bool wallet2::check_tx_proof(const crypto::hash &txid, const cryptonote::account if (good_signature[i]) THROW_WALLET_EXCEPTION_IF(!crypto::generate_key_derivation(shared_secret[i], rct::rct2sk(rct::I), additional_derivations[i - 1]), error::wallet_internal_error, "Failed to generate key derivation"); - check_tx_key_helper(txid, derivation, additional_derivations, address, received, in_pool, confirmations); + check_tx_key_helper(tx, derivation, additional_derivations, address, received); return true; } return false; @@ -11014,15 +11304,6 @@ crypto::public_key wallet2::get_tx_pub_key_from_received_outs(const tools::walle size_t pk_index = 0; hw::device &hwdev = m_account.get_device(); - const std::vector<crypto::public_key> additional_tx_pub_keys = get_additional_tx_pub_keys_from_extra(td.m_tx); - std::vector<crypto::key_derivation> additional_derivations; - for (size_t i = 0; i < additional_tx_pub_keys.size(); ++i) - { - additional_derivations.push_back({}); - bool r = hwdev.generate_key_derivation(additional_tx_pub_keys[i], keys.m_view_secret_key, additional_derivations.back()); - THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "Failed to generate key derivation"); - } - while (find_tx_extra_field_by_type(tx_extra_fields, pub_key_field, pk_index++)) { const crypto::public_key tx_pub_key = pub_key_field.pub_key; crypto::key_derivation derivation; @@ -11032,16 +11313,15 @@ crypto::public_key wallet2::get_tx_pub_key_from_received_outs(const tools::walle for (size_t i = 0; i < td.m_tx.vout.size(); ++i) { tx_scan_info_t tx_scan_info; - check_acc_out_precomp(td.m_tx.vout[i], derivation, additional_derivations, i, tx_scan_info); + check_acc_out_precomp(td.m_tx.vout[i], derivation, {}, i, tx_scan_info); if (!tx_scan_info.error && tx_scan_info.received) return tx_pub_key; } } - // we found no key yielding an output - THROW_WALLET_EXCEPTION_IF(true, error::wallet_internal_error, - "Public key yielding at least one output wasn't found in the transaction extra"); - return crypto::null_pkey; + // we found no key yielding an output, but it might be in the additional + // tx pub keys only, which we do not need to check, so return the first one + return tx_pub_key; } bool wallet2::export_key_images(const std::string &filename) const @@ -11283,6 +11563,17 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag } PERF_TIMER_STOP(import_key_images_C); + // accumulate outputs before the updated data + for(size_t i = 0; i < offset; ++i) + { + const transfer_details &td = m_transfers[i]; + uint64_t amount = td.amount(); + if (td.m_spent) + spent += amount; + else + unspent += amount; + } + PERF_TIMER_START(import_key_images_D); for(size_t i = 0; i < signed_key_images.size(); ++i) { @@ -11392,6 +11683,7 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag auto it = m_key_images.find(boost::get<cryptonote::txin_to_key>(in).k_image); if (it != m_key_images.end()) { + THROW_WALLET_EXCEPTION_IF(it->second >= m_transfers.size(), error::wallet_internal_error, std::string("Key images cache contains illegal transfer offset: ") + std::to_string(it->second) + std::string(" m_transfers.size() = ") + std::to_string(m_transfers.size())); const transfer_details& td = m_transfers[it->second]; uint64_t amount = boost::get<cryptonote::txin_to_key>(in).amount; if (amount > 0) @@ -11445,32 +11737,52 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag PERF_TIMER_STOP(import_key_images_G); } - return m_transfers[signed_key_images.size() - 1].m_block_height; + // this can be 0 if we do not know the height + return m_transfers[signed_key_images.size() + offset - 1].m_block_height; } -bool wallet2::import_key_images(std::vector<crypto::key_image> key_images) +bool wallet2::import_key_images(std::vector<crypto::key_image> key_images, size_t offset, boost::optional<std::unordered_set<size_t>> selected_transfers) { - if (key_images.size() > m_transfers.size()) + if (key_images.size() + offset > m_transfers.size()) { LOG_PRINT_L1("More key images returned that we know outputs for"); return false; } - for (size_t i = 0; i < key_images.size(); ++i) + for (size_t ki_idx = 0; ki_idx < key_images.size(); ++ki_idx) { - transfer_details &td = m_transfers[i]; - if (td.m_key_image_known && !td.m_key_image_partial && td.m_key_image != key_images[i]) - LOG_PRINT_L0("WARNING: imported key image differs from previously known key image at index " << i << ": trusting imported one"); - td.m_key_image = key_images[i]; - m_key_images[m_transfers[i].m_key_image] = i; + const size_t transfer_idx = ki_idx + offset; + if (selected_transfers && selected_transfers.get().find(transfer_idx) == selected_transfers.get().end()) + continue; + + transfer_details &td = m_transfers[transfer_idx]; + if (td.m_key_image_known && !td.m_key_image_partial && td.m_key_image != key_images[ki_idx]) + LOG_PRINT_L0("WARNING: imported key image differs from previously known key image at index " << ki_idx << ": trusting imported one"); + td.m_key_image = key_images[ki_idx]; + m_key_images[td.m_key_image] = transfer_idx; td.m_key_image_known = true; td.m_key_image_request = false; td.m_key_image_partial = false; - m_pub_keys[m_transfers[i].get_public_key()] = i; + m_pub_keys[td.get_public_key()] = transfer_idx; } return true; } +bool wallet2::import_key_images(signed_tx_set & signed_tx, size_t offset, bool only_selected_transfers) +{ + std::unordered_set<size_t> selected_transfers; + if (only_selected_transfers) + { + for (const pending_tx & ptx : signed_tx.ptx) + { + for (const size_t s: ptx.selected_transfers) + selected_transfers.insert(s); + } + } + + return import_key_images(signed_tx.key_images, offset, only_selected_transfers ? boost::make_optional(selected_transfers) : boost::none); +} + wallet2::payment_container wallet2::export_payments() const { payment_container payments; @@ -12451,22 +12763,30 @@ wallet_device_callback * wallet2::get_device_callback() } return m_device_callback.get(); }//---------------------------------------------------------------------------------------------------- -void wallet2::on_button_request() +void wallet2::on_device_button_request(uint64_t code) { - if (0 != m_callback) - m_callback->on_button_request(); + if (nullptr != m_callback) + m_callback->on_device_button_request(code); } //---------------------------------------------------------------------------------------------------- -void wallet2::on_pin_request(epee::wipeable_string & pin) +boost::optional<epee::wipeable_string> wallet2::on_device_pin_request() { - if (0 != m_callback) - m_callback->on_pin_request(pin); + if (nullptr != m_callback) + return m_callback->on_device_pin_request(); + return boost::none; } //---------------------------------------------------------------------------------------------------- -void wallet2::on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) +boost::optional<epee::wipeable_string> wallet2::on_device_passphrase_request(bool on_device) { - if (0 != m_callback) - m_callback->on_passphrase_request(on_device, passphrase); + if (nullptr != m_callback) + return m_callback->on_device_passphrase_request(on_device); + return boost::none; +} +//---------------------------------------------------------------------------------------------------- +void wallet2::on_device_progress(const hw::device_progress& event) +{ + if (nullptr != m_callback) + m_callback->on_device_progress(event); } //---------------------------------------------------------------------------------------------------- std::string wallet2::get_rpc_status(const std::string &s) const @@ -12490,5 +12810,61 @@ void wallet2::throw_on_rpc_response_error(const boost::optional<std::string> &st THROW_WALLET_EXCEPTION_IF(*status == CORE_RPC_STATUS_BUSY, tools::error::daemon_busy, method); THROW_WALLET_EXCEPTION_IF(*status != CORE_RPC_STATUS_OK, tools::error::wallet_generic_rpc_error, method, m_trusted_daemon ? *status : "daemon error"); } +//---------------------------------------------------------------------------------------------------- +void wallet2::hash_m_transfer(const transfer_details & transfer, crypto::hash &hash) const +{ + KECCAK_CTX state; + keccak_init(&state); + keccak_update(&state, (const uint8_t *) transfer.m_txid.data, sizeof(transfer.m_txid.data)); + keccak_update(&state, (const uint8_t *) transfer.m_internal_output_index, sizeof(transfer.m_internal_output_index)); + keccak_update(&state, (const uint8_t *) transfer.m_global_output_index, sizeof(transfer.m_global_output_index)); + keccak_update(&state, (const uint8_t *) transfer.m_amount, sizeof(transfer.m_amount)); + keccak_finish(&state, (uint8_t *) hash.data); +} +//---------------------------------------------------------------------------------------------------- +uint64_t wallet2::hash_m_transfers(int64_t transfer_height, crypto::hash &hash) const +{ + CHECK_AND_ASSERT_THROW_MES(transfer_height > (int64_t)m_transfers.size(), "Hash height is greater than number of transfers"); + + KECCAK_CTX state; + crypto::hash tmp_hash{}; + uint64_t current_height = 0; + + keccak_init(&state); + for(const transfer_details & transfer : m_transfers){ + if (transfer_height >= 0 && current_height >= (uint64_t)transfer_height){ + break; + } + + hash_m_transfer(transfer, tmp_hash); + keccak_update(&state, (const uint8_t *) transfer.m_block_height, sizeof(transfer.m_block_height)); + keccak_update(&state, (const uint8_t *) tmp_hash.data, sizeof(tmp_hash.data)); + current_height += 1; + } + + keccak_finish(&state, (uint8_t *) hash.data); + return current_height; +} +//---------------------------------------------------------------------------------------------------- +void wallet2::finish_rescan_bc_keep_key_images(uint64_t transfer_height, const crypto::hash &hash) +{ + // Compute hash of m_transfers, if differs there had to be BC reorg. + crypto::hash new_transfers_hash{}; + hash_m_transfers((int64_t) transfer_height, new_transfers_hash); + if (new_transfers_hash != hash) + { + // Soft-Reset to avoid inconsistency in case of BC reorg. + clear_soft(false); // keep_key_images works only with soft reset. + THROW_WALLET_EXCEPTION_IF(true, error::wallet_internal_error, "Transfers changed during rescan, soft or hard rescan is needed"); + } + + // Restore key images in m_transfers from m_key_images + for(auto it = m_key_images.begin(); it != m_key_images.end(); it++) + { + THROW_WALLET_EXCEPTION_IF(it->second >= m_transfers.size(), error::wallet_internal_error, "Key images cache contains illegal transfer offset"); + m_transfers[it->second].m_key_image = it->first; + m_transfers[it->second].m_key_image_known = true; + } +} } diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 36271c82a..28ebd6704 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -62,6 +62,7 @@ #include "common/password.h" #include "node_rpc_proxy.h" #include "message_store.h" +#include "wallet_light_rpc.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "wallet.wallet2" @@ -103,9 +104,10 @@ namespace tools virtual void on_lw_unconfirmed_money_received(uint64_t height, const crypto::hash &txid, uint64_t amount) {} virtual void on_lw_money_spent(uint64_t height, const crypto::hash &txid, uint64_t amount) {} // Device callbacks - virtual void on_button_request() {} - virtual void on_pin_request(epee::wipeable_string & pin) {} - virtual void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) {} + virtual void on_device_button_request(uint64_t code) {} + virtual boost::optional<epee::wipeable_string> on_device_pin_request() { return boost::none; } + virtual boost::optional<epee::wipeable_string> on_device_passphrase_request(bool on_device) { return boost::none; } + virtual void on_device_progress(const hw::device_progress& event) {}; // Common callbacks virtual void on_pool_tx_removed(const crypto::hash &txid) {} virtual ~i_wallet2_callback() {} @@ -115,9 +117,10 @@ namespace tools { public: wallet_device_callback(wallet2 * wallet): wallet(wallet) {}; - void on_button_request() override; - void on_pin_request(epee::wipeable_string & pin) override; - void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) override; + void on_button_request(uint64_t code=0) override; + boost::optional<epee::wipeable_string> on_pin_request() override; + boost::optional<epee::wipeable_string> on_passphrase_request(bool on_device) override; + void on_progress(const hw::device_progress& event) override; private: wallet2 * wallet; }; @@ -372,7 +375,7 @@ namespace tools std::vector<uint8_t> extra; uint64_t unlock_time; bool use_rct; - bool use_bulletproofs; + rct::RCTConfig rct_config; std::vector<cryptonote::tx_destination_entry> dests; // original setup, does not include change uint32_t subaddr_account; // subaddress account of your wallet to be used in this transfer std::set<uint32_t> subaddr_indices; // set of address indices used as inputs in this transfer @@ -385,7 +388,7 @@ namespace tools FIELD(extra) FIELD(unlock_time) FIELD(use_rct) - FIELD(use_bulletproofs) + FIELD(rct_config) FIELD(dests) FIELD(subaddr_account) FIELD(subaddr_indices) @@ -678,7 +681,9 @@ namespace tools bool deinit(); bool init(std::string daemon_address = "http://localhost:8080", - boost::optional<epee::net_utils::http::login> daemon_login = boost::none, uint64_t upper_transaction_weight_limit = 0, + boost::optional<epee::net_utils::http::login> daemon_login = boost::none, + boost::asio::ip::tcp::endpoint proxy = {}, + uint64_t upper_transaction_weight_limit = 0, bool trusted_daemon = true, epee::net_utils::ssl_support_t ssl_support = epee::net_utils::ssl_support_t::e_ssl_support_autodetect, const std::pair<std::string, std::string> &private_key_and_certificate_path = {}, @@ -797,9 +802,11 @@ namespace tools std::vector<wallet2::pending_tx> create_transactions_all(uint64_t below, const cryptonote::account_public_address &address, bool is_subaddress, const size_t outputs, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices); std::vector<wallet2::pending_tx> create_transactions_single(const crypto::key_image &ki, const cryptonote::account_public_address &address, bool is_subaddress, const size_t outputs, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra); std::vector<wallet2::pending_tx> create_transactions_from(const cryptonote::account_public_address &address, bool is_subaddress, const size_t outputs, std::vector<size_t> unused_transfers_indices, std::vector<size_t> unused_dust_indices, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra); + bool sanity_check(const std::vector<wallet2::pending_tx> &ptx_vector, std::vector<cryptonote::tx_destination_entry> dsts) const; void cold_tx_aux_import(const std::vector<pending_tx>& ptx, const std::vector<std::string>& tx_device_aux); void cold_sign_tx(const std::vector<pending_tx>& ptx_vector, signed_tx_set &exported_txs, std::vector<cryptonote::address_parse_info> &dsts_info, std::vector<std::string> & tx_device_aux); uint64_t cold_key_image_sync(uint64_t &spent, uint64_t &unspent); + bool parse_multisig_tx_from_str(std::string multisig_tx_st, multisig_tx_set &exported_txs) const; bool load_multisig_tx(cryptonote::blobdata blob, multisig_tx_set &exported_txs, std::function<bool(const multisig_tx_set&)> accept_func = NULL); bool load_multisig_tx_from_file(const std::string &filename, multisig_tx_set &exported_txs, std::function<bool(const multisig_tx_set&)> accept_func = NULL); bool sign_multisig_tx_from_file(const std::string &filename, std::vector<crypto::hash> &txids, std::function<bool(const multisig_tx_set&)> accept_func); @@ -818,7 +825,7 @@ namespace tools uint64_t get_blockchain_current_height() const { return m_light_wallet_blockchain_height ? m_light_wallet_blockchain_height : m_blockchain.size(); } void rescan_spent(); - void rescan_blockchain(bool hard, bool refresh = true); + void rescan_blockchain(bool hard, bool refresh = true, bool keep_key_images = false); bool is_transfer_unlocked(const transfer_details& td) const; bool is_transfer_unlocked(uint64_t unlock_time, uint64_t block_height) const; @@ -1004,12 +1011,16 @@ namespace tools const std::string & device_derivation_path() const { return m_device_derivation_path; } void device_derivation_path(const std::string &device_derivation_path) { m_device_derivation_path = device_derivation_path; } - bool get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys) const; + bool get_tx_key_cached(const crypto::hash &txid, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys) const; void set_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys); + bool get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys); void check_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, const cryptonote::account_public_address &address, uint64_t &received, bool &in_pool, uint64_t &confirmations); void check_tx_key_helper(const crypto::hash &txid, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, const cryptonote::account_public_address &address, uint64_t &received, bool &in_pool, uint64_t &confirmations); + void check_tx_key_helper(const cryptonote::transaction &tx, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, const cryptonote::account_public_address &address, uint64_t &received) const; std::string get_tx_proof(const crypto::hash &txid, const cryptonote::account_public_address &address, bool is_subaddress, const std::string &message); + std::string get_tx_proof(const cryptonote::transaction &tx, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, const cryptonote::account_public_address &address, bool is_subaddress, const std::string &message) const; bool check_tx_proof(const crypto::hash &txid, const cryptonote::account_public_address &address, bool is_subaddress, const std::string &message, const std::string &sig_str, uint64_t &received, bool &in_pool, uint64_t &confirmations); + bool check_tx_proof(const cryptonote::transaction &tx, const cryptonote::account_public_address &address, bool is_subaddress, const std::string &message, const std::string &sig_str, uint64_t &received) const; std::string get_spend_proof(const crypto::hash &txid, const std::string &message); bool check_spend_proof(const crypto::hash &txid, const std::string &message, const std::string &sig_str); @@ -1127,7 +1138,8 @@ namespace tools std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> export_key_images(bool all = false) const; uint64_t import_key_images(const std::vector<std::pair<crypto::key_image, crypto::signature>> &signed_key_images, size_t offset, uint64_t &spent, uint64_t &unspent, bool check_spent = true); uint64_t import_key_images(const std::string &filename, uint64_t &spent, uint64_t &unspent); - bool import_key_images(std::vector<crypto::key_image> key_images); + bool import_key_images(std::vector<crypto::key_image> key_images, size_t offset=0, boost::optional<std::unordered_set<size_t>> selected_transfers=boost::none); + bool import_key_images(signed_tx_set & signed_tx, size_t offset=0, bool only_selected_transfers=false); crypto::public_key get_tx_pub_key_from_received_outs(const tools::wallet2::transfer_details &td) const; void update_pool_state(bool refreshed = false); @@ -1167,11 +1179,11 @@ namespace tools // fetch txs and store in m_payments void light_wallet_get_address_txs(); // get_address_info - bool light_wallet_get_address_info(cryptonote::COMMAND_RPC_GET_ADDRESS_INFO::response &response); + bool light_wallet_get_address_info(tools::COMMAND_RPC_GET_ADDRESS_INFO::response &response); // Login. new_address is true if address hasn't been used on lw node before. bool light_wallet_login(bool &new_address); // Send an import request to lw node. returns info about import fee, address and payment_id - bool light_wallet_import_wallet_request(cryptonote::COMMAND_RPC_IMPORT_WALLET_REQUEST::response &response); + bool light_wallet_import_wallet_request(tools::COMMAND_RPC_IMPORT_WALLET_REQUEST::response &response); // get random outputs from light wallet server void light_wallet_get_outs(std::vector<std::vector<get_outs_entry>> &outs, const std::vector<size_t> &selected_transfers, size_t fake_outputs_count); // Parse rct string @@ -1248,6 +1260,9 @@ namespace tools void set_tx_notify(const std::shared_ptr<tools::Notify> ¬ify) { m_tx_notify = notify; } bool is_tx_spendtime_unlocked(uint64_t unlock_time, uint64_t block_height) const; + void hash_m_transfer(const transfer_details & transfer, crypto::hash &hash) const; + uint64_t hash_m_transfers(int64_t transfer_height, crypto::hash &hash) const; + void finish_rescan_bc_keep_key_images(uint64_t transfer_height, const crypto::hash &hash); private: /*! @@ -1266,9 +1281,10 @@ namespace tools bool load_keys(const std::string& keys_file_name, const epee::wipeable_string& password); void process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector<uint64_t> &o_indices, uint64_t height, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen, const tx_cache_data &tx_cache_data, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL); void process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const parsed_block &parsed_block, const crypto::hash& bl_id, uint64_t height, const std::vector<tx_cache_data> &tx_cache_data, size_t tx_cache_data_offset, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL); - void detach_blockchain(uint64_t height); + void detach_blockchain(uint64_t height, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL); void get_short_chain_history(std::list<crypto::hash>& ids, uint64_t granularity = 1) const; bool clear(); + void clear_soft(bool keep_key_images=false); void pull_blocks(uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices); void pull_hashes(uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<crypto::hash> &hashes); void fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, bool force = false); @@ -1333,9 +1349,10 @@ namespace tools void create_keys_file(const std::string &wallet_, bool watch_only, const epee::wipeable_string &password, bool create_address_file); wallet_device_callback * get_device_callback(); - void on_button_request(); - void on_pin_request(epee::wipeable_string & pin); - void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase); + void on_device_button_request(uint64_t code); + boost::optional<epee::wipeable_string> on_device_pin_request(); + boost::optional<epee::wipeable_string> on_device_passphrase_request(bool on_device); + void on_device_progress(const hw::device_progress& event); std::string get_rpc_status(const std::string &s) const; void throw_on_rpc_response_error(const boost::optional<std::string> &status, const char *method) const; @@ -1477,7 +1494,7 @@ BOOST_CLASS_VERSION(tools::wallet2::address_book_row, 17) BOOST_CLASS_VERSION(tools::wallet2::reserve_proof_entry, 0) BOOST_CLASS_VERSION(tools::wallet2::unsigned_tx_set, 0) BOOST_CLASS_VERSION(tools::wallet2::signed_tx_set, 1) -BOOST_CLASS_VERSION(tools::wallet2::tx_construction_data, 3) +BOOST_CLASS_VERSION(tools::wallet2::tx_construction_data, 4) BOOST_CLASS_VERSION(tools::wallet2::pending_tx, 3) BOOST_CLASS_VERSION(tools::wallet2::multisig_sig, 0) @@ -1846,11 +1863,27 @@ namespace boost a & x.subaddr_account; a & x.subaddr_indices; if (ver < 2) + { + if (!typename Archive::is_saving()) + x.rct_config = { rct::RangeProofBorromean, 0 }; return; + } a & x.selected_transfers; if (ver < 3) + { + if (!typename Archive::is_saving()) + x.rct_config = { rct::RangeProofBorromean, 0 }; return; - a & x.use_bulletproofs; + } + if (ver < 4) + { + bool use_bulletproofs = x.rct_config.range_proof_type != rct::RangeProofBorromean; + a & use_bulletproofs; + if (!typename Archive::is_saving()) + x.rct_config = { use_bulletproofs ? rct::RangeProofBulletproof : rct::RangeProofBorromean, 0 }; + return; + } + a & x.rct_config; } template <class Archive> diff --git a/src/wallet/wallet_light_rpc.h b/src/wallet/wallet_light_rpc.h new file mode 100644 index 000000000..1d35cec33 --- /dev/null +++ b/src/wallet/wallet_light_rpc.h @@ -0,0 +1,320 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +#pragma once +#include "cryptonote_basic/cryptonote_basic.h" +#include "crypto/hash.h" + +namespace tools +{ + //----------------------------------------------- + struct COMMAND_RPC_GET_ADDRESS_TXS + { + struct request_t + { + std::string address; + std::string view_key; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(address) + KV_SERIALIZE(view_key) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<request_t> request; + + struct spent_output { + uint64_t amount; + std::string key_image; + std::string tx_pub_key; + uint64_t out_index; + uint32_t mixin; + + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(amount) + KV_SERIALIZE(key_image) + KV_SERIALIZE(tx_pub_key) + KV_SERIALIZE(out_index) + KV_SERIALIZE(mixin) + END_KV_SERIALIZE_MAP() + }; + + struct transaction + { + uint64_t id; + std::string hash; + uint64_t timestamp; + uint64_t total_received; + uint64_t total_sent; + uint64_t unlock_time; + uint64_t height; + std::list<spent_output> spent_outputs; + std::string payment_id; + bool coinbase; + bool mempool; + uint32_t mixin; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(id) + KV_SERIALIZE(hash) + KV_SERIALIZE(timestamp) + KV_SERIALIZE(total_received) + KV_SERIALIZE(total_sent) + KV_SERIALIZE(unlock_time) + KV_SERIALIZE(height) + KV_SERIALIZE(spent_outputs) + KV_SERIALIZE(payment_id) + KV_SERIALIZE(coinbase) + KV_SERIALIZE(mempool) + KV_SERIALIZE(mixin) + END_KV_SERIALIZE_MAP() + }; + + + struct response_t + { + //std::list<std::string> txs_as_json; + uint64_t total_received; + uint64_t total_received_unlocked = 0; // OpenMonero only + uint64_t scanned_height; + std::vector<transaction> transactions; + uint64_t blockchain_height; + uint64_t scanned_block_height; + std::string status; + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(total_received) + KV_SERIALIZE(total_received_unlocked) + KV_SERIALIZE(scanned_height) + KV_SERIALIZE(transactions) + KV_SERIALIZE(blockchain_height) + KV_SERIALIZE(scanned_block_height) + KV_SERIALIZE(status) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<response_t> response; + }; + + //----------------------------------------------- + struct COMMAND_RPC_GET_ADDRESS_INFO + { + struct request_t + { + std::string address; + std::string view_key; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(address) + KV_SERIALIZE(view_key) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<request_t> request; + + struct spent_output + { + uint64_t amount; + std::string key_image; + std::string tx_pub_key; + uint64_t out_index; + uint32_t mixin; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(amount) + KV_SERIALIZE(key_image) + KV_SERIALIZE(tx_pub_key) + KV_SERIALIZE(out_index) + KV_SERIALIZE(mixin) + END_KV_SERIALIZE_MAP() + }; + + struct response_t + { + uint64_t locked_funds; + uint64_t total_received; + uint64_t total_sent; + uint64_t scanned_height; + uint64_t scanned_block_height; + uint64_t start_height; + uint64_t transaction_height; + uint64_t blockchain_height; + std::list<spent_output> spent_outputs; + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(locked_funds) + KV_SERIALIZE(total_received) + KV_SERIALIZE(total_sent) + KV_SERIALIZE(scanned_height) + KV_SERIALIZE(scanned_block_height) + KV_SERIALIZE(start_height) + KV_SERIALIZE(transaction_height) + KV_SERIALIZE(blockchain_height) + KV_SERIALIZE(spent_outputs) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<response_t> response; + }; + + //----------------------------------------------- + struct COMMAND_RPC_GET_UNSPENT_OUTS + { + struct request_t + { + std::string amount; + std::string address; + std::string view_key; + // OpenMonero specific + uint64_t mixin; + bool use_dust; + std::string dust_threshold; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(amount) + KV_SERIALIZE(address) + KV_SERIALIZE(view_key) + KV_SERIALIZE(mixin) + KV_SERIALIZE(use_dust) + KV_SERIALIZE(dust_threshold) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<request_t> request; + + + struct output { + uint64_t amount; + std::string public_key; + uint64_t index; + uint64_t global_index; + std::string rct; + std::string tx_hash; + std::string tx_pub_key; + std::string tx_prefix_hash; + std::vector<std::string> spend_key_images; + uint64_t timestamp; + uint64_t height; + + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(amount) + KV_SERIALIZE(public_key) + KV_SERIALIZE(index) + KV_SERIALIZE(global_index) + KV_SERIALIZE(rct) + KV_SERIALIZE(tx_hash) + KV_SERIALIZE(tx_pub_key) + KV_SERIALIZE(tx_prefix_hash) + KV_SERIALIZE(spend_key_images) + KV_SERIALIZE(timestamp) + KV_SERIALIZE(height) + END_KV_SERIALIZE_MAP() + }; + + struct response_t + { + uint64_t amount; + std::list<output> outputs; + uint64_t per_kb_fee; + std::string status; + std::string reason; + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(amount) + KV_SERIALIZE(outputs) + KV_SERIALIZE(per_kb_fee) + KV_SERIALIZE(status) + KV_SERIALIZE(reason) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<response_t> response; + }; + //----------------------------------------------- + struct COMMAND_RPC_LOGIN + { + struct request_t + { + std::string address; + std::string view_key; + bool create_account; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(address) + KV_SERIALIZE(view_key) + KV_SERIALIZE(create_account) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<request_t> request; + + struct response_t + { + std::string status; + std::string reason; + bool new_address; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + KV_SERIALIZE(reason) + KV_SERIALIZE(new_address) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<response_t> response; + }; + //----------------------------------------------- + struct COMMAND_RPC_IMPORT_WALLET_REQUEST + { + struct request_t + { + std::string address; + std::string view_key; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(address) + KV_SERIALIZE(view_key) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<request_t> request; + + struct response_t + { + std::string payment_id; + uint64_t import_fee; + bool new_request; + bool request_fulfilled; + std::string payment_address; + std::string status; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(payment_id) + KV_SERIALIZE(import_fee) + KV_SERIALIZE(new_request) + KV_SERIALIZE(request_fulfilled) + KV_SERIALIZE(payment_address) + KV_SERIALIZE(status) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<response_t> response; + }; + //----------------------------------------------- +} diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index 5b7cf90d0..18b2de33f 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -56,6 +56,8 @@ using namespace epee; #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "wallet.rpc" +#define DEFAULT_AUTO_REFRESH_PERIOD 20 // seconds + namespace { const command_line::arg_descriptor<std::string, true> arg_rpc_bind_port = {"rpc-bind-port", "Sets bind port for server"}; @@ -124,13 +126,18 @@ namespace tools { m_stop = false; m_net_server.add_idle_handler([this](){ + if (m_auto_refresh_period == 0) // disabled + return true; + if (boost::posix_time::microsec_clock::universal_time() < m_last_auto_refresh_time + boost::posix_time::seconds(m_auto_refresh_period)) + return true; try { if (m_wallet) m_wallet->refresh(m_wallet->is_trusted_daemon()); } catch (const std::exception& ex) { LOG_ERROR("Exception at while refreshing, what=" << ex.what()); } + m_last_auto_refresh_time = boost::posix_time::microsec_clock::universal_time(); return true; - }, 20000); + }, 1000); m_net_server.add_idle_handler([this](){ if (m_stop.load(std::memory_order_relaxed)) { @@ -263,6 +270,9 @@ namespace tools std::vector<std::vector<uint8_t>> allowed_fingerprints{ rpc_ssl_allowed_fingerprints.size() }; std::transform(rpc_ssl_allowed_fingerprints.begin(), rpc_ssl_allowed_fingerprints.end(), allowed_fingerprints.begin(), epee::from_hex::vector); + m_auto_refresh_period = DEFAULT_AUTO_REFRESH_PERIOD; + m_last_auto_refresh_time = boost::posix_time::min_date_time; + m_net_server.set_threads_prefix("RPC"); auto rng = [](size_t len, uint8_t *ptr) { return crypto::rand(len, ptr); }; return epee::http_server_impl_base<wallet_rpc_server, connection_context>::init( @@ -319,6 +329,8 @@ namespace tools entry.type = "out"; entry.subaddr_index = { pd.m_subaddr_account, 0 }; + for (uint32_t i: pd.m_subaddr_indices) + entry.subaddr_indices.push_back({pd.m_subaddr_account, i}); entry.address = m_wallet->get_subaddress_as_str({pd.m_subaddr_account, 0}); set_confirmations(entry, m_wallet->get_blockchain_current_height(), m_wallet->get_last_block_reward()); } @@ -339,6 +351,8 @@ namespace tools entry.note = m_wallet->get_tx_note(txid); entry.type = is_failed ? "failed" : "pending"; entry.subaddr_index = { pd.m_subaddr_account, 0 }; + for (uint32_t i: pd.m_subaddr_indices) + entry.subaddr_indices.push_back({pd.m_subaddr_account, i}); entry.address = m_wallet->get_subaddress_as_str({pd.m_subaddr_account, 0}); set_confirmations(entry, m_wallet->get_blockchain_current_height(), m_wallet->get_last_block_reward()); } @@ -1074,29 +1088,59 @@ namespace tools er.message = "command not supported by watch-only wallet"; return false; } - - tools::wallet2::unsigned_tx_set exported_txs; - try + if(req.unsigned_txset.empty() && req.multisig_txset.empty()) { - cryptonote::blobdata blob; - if (!epee::string_tools::parse_hexstr_to_binbuff(req.unsigned_txset, blob)) - { - er.code = WALLET_RPC_ERROR_CODE_BAD_HEX; - er.message = "Failed to parse hex."; - return false; + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "no txset provided"; + return false; + } + + std::vector <wallet2::tx_construction_data> tx_constructions; + if (!req.unsigned_txset.empty()) { + try { + tools::wallet2::unsigned_tx_set exported_txs; + cryptonote::blobdata blob; + if (!epee::string_tools::parse_hexstr_to_binbuff(req.unsigned_txset, blob)) { + er.code = WALLET_RPC_ERROR_CODE_BAD_HEX; + er.message = "Failed to parse hex."; + return false; + } + if (!m_wallet->parse_unsigned_tx_from_str(blob, exported_txs)) { + er.code = WALLET_RPC_ERROR_CODE_BAD_UNSIGNED_TX_DATA; + er.message = "cannot load unsigned_txset"; + return false; + } + tx_constructions = exported_txs.txes; } - if(!m_wallet->parse_unsigned_tx_from_str(blob, exported_txs)) - { + catch (const std::exception &e) { er.code = WALLET_RPC_ERROR_CODE_BAD_UNSIGNED_TX_DATA; - er.message = "cannot load unsigned_txset"; + er.message = "failed to parse unsigned transfers: " + std::string(e.what()); + return false; + } + } else if (!req.multisig_txset.empty()) { + try { + tools::wallet2::multisig_tx_set exported_txs; + cryptonote::blobdata blob; + if (!epee::string_tools::parse_hexstr_to_binbuff(req.multisig_txset, blob)) { + er.code = WALLET_RPC_ERROR_CODE_BAD_HEX; + er.message = "Failed to parse hex."; + return false; + } + if (!m_wallet->parse_multisig_tx_from_str(blob, exported_txs)) { + er.code = WALLET_RPC_ERROR_CODE_BAD_MULTISIG_TX_DATA; + er.message = "cannot load multisig_txset"; + return false; + } + + for (size_t n = 0; n < exported_txs.m_ptx.size(); ++n) { + tx_constructions.push_back(exported_txs.m_ptx[n].construction_data); + } + } + catch (const std::exception &e) { + er.code = WALLET_RPC_ERROR_CODE_BAD_MULTISIG_TX_DATA; + er.message = "failed to parse multisig transfers: " + std::string(e.what()); return false; } - } - catch (const std::exception &e) - { - er.code = WALLET_RPC_ERROR_CODE_BAD_UNSIGNED_TX_DATA; - er.message = "failed to parse unsigned transfers: " + std::string(e.what()); - return false; } std::vector<tools::wallet2::pending_tx> ptx; @@ -1105,9 +1149,9 @@ namespace tools // gather info to ask the user std::unordered_map<cryptonote::account_public_address, std::pair<std::string, uint64_t>> dests; int first_known_non_zero_change_index = -1; - for (size_t n = 0; n < exported_txs.txes.size(); ++n) + for (size_t n = 0; n < tx_constructions.size(); ++n) { - const tools::wallet2::tx_construction_data &cd = exported_txs.txes[n]; + const tools::wallet2::tx_construction_data &cd = tx_constructions[n]; res.desc.push_back({0, 0, std::numeric_limits<uint32_t>::max(), 0, {}, "", 0, "", 0, 0, ""}); wallet_rpc::COMMAND_RPC_DESCRIBE_TRANSFER::transfer_description &desc = res.desc.back(); @@ -1171,7 +1215,7 @@ namespace tools { if (first_known_non_zero_change_index == -1) first_known_non_zero_change_index = n; - const tools::wallet2::tx_construction_data &cdn = exported_txs.txes[first_known_non_zero_change_index]; + const tools::wallet2::tx_construction_data &cdn = tx_constructions[first_known_non_zero_change_index]; if (memcmp(&cd.change_dts.addr, &cdn.change_dts.addr, sizeof(cd.change_dts.addr))) { er.code = WALLET_RPC_ERROR_CODE_BAD_UNSIGNED_TX_DATA; @@ -1199,7 +1243,7 @@ namespace tools if (desc.change_amount > 0) { - const tools::wallet2::tx_construction_data &cd0 = exported_txs.txes[0]; + const tools::wallet2::tx_construction_data &cd0 = tx_constructions[0]; desc.change_address = get_account_address_as_str(m_wallet->nettype(), cd0.subaddr_account > 0, cd0.change_dts.addr); } @@ -2800,6 +2844,28 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_auto_refresh(const wallet_rpc::COMMAND_RPC_AUTO_REFRESH::request& req, wallet_rpc::COMMAND_RPC_AUTO_REFRESH::response& res, epee::json_rpc::error& er, const connection_context *ctx) + { + if (m_restricted) + { + er.code = WALLET_RPC_ERROR_CODE_DENIED; + er.message = "Command unavailable in restricted mode."; + return false; + } + try + { + m_auto_refresh_period = req.enable ? req.period ? req.period : DEFAULT_AUTO_REFRESH_PERIOD : 0; + MINFO("Auto refresh now " << (m_auto_refresh_period ? std::to_string(m_auto_refresh_period) + " seconds" : std::string("disabled"))); + return true; + } + catch (const std::exception& e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_rescan_spent(const wallet_rpc::COMMAND_RPC_RESCAN_SPENT::request& req, wallet_rpc::COMMAND_RPC_RESCAN_SPENT::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); @@ -2902,7 +2968,7 @@ namespace tools er.message = "Invalid filename"; return false; } - std::string wallet_file = m_wallet_dir + "/" + req.filename; + std::string wallet_file = req.filename.empty() ? "" : (m_wallet_dir + "/" + req.filename); { std::vector<std::string> languages; crypto::ElectrumWords::get_language_list(languages); @@ -3187,12 +3253,6 @@ namespace tools } // early check for mandatory fields - if (req.filename.empty()) - { - er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; - er.message = "field 'filename' is mandatory. Please provide a filename to save the restored wallet to."; - return false; - } if (req.viewkey.empty()) { er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; @@ -3221,7 +3281,7 @@ namespace tools er.message = "Invalid filename"; return false; } - std::string wallet_file = m_wallet_dir + "/" + req.filename; + std::string wallet_file = req.filename.empty() ? "" : (m_wallet_dir + "/" + req.filename); // check if wallet file already exists if (!wallet_file.empty()) { @@ -3331,7 +3391,8 @@ namespace tools { try { - m_wallet->store(); + if (!wallet_file.empty()) + m_wallet->store(); } catch (const std::exception &e) { @@ -3355,12 +3416,6 @@ namespace tools } // early check for mandatory fields - if (req.filename.empty()) - { - er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; - er.message = "field 'filename' is mandatory. Please provide a filename to save the restored wallet to."; - return false; - } if (req.seed.empty()) { er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; @@ -3383,7 +3438,7 @@ namespace tools er.message = "Invalid filename"; return false; } - std::string wallet_file = m_wallet_dir + "/" + req.filename; + std::string wallet_file = req.filename.empty() ? "" : (m_wallet_dir + "/" + req.filename); // check if wallet file already exists if (!wallet_file.empty()) { @@ -3499,7 +3554,7 @@ namespace tools er.message = "Failed to encode seed"; return false; } - res.seed = electrum_words.data(); + res.seed = std::string(electrum_words.data(), electrum_words.size()); if (!wal) { diff --git a/src/wallet/wallet_rpc_server.h b/src/wallet/wallet_rpc_server.h index affaf10f7..2b52275b8 100644 --- a/src/wallet/wallet_rpc_server.h +++ b/src/wallet/wallet_rpc_server.h @@ -129,6 +129,7 @@ namespace tools MAP_JON_RPC_WE("add_address_book", on_add_address_book, wallet_rpc::COMMAND_RPC_ADD_ADDRESS_BOOK_ENTRY) MAP_JON_RPC_WE("delete_address_book",on_delete_address_book,wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY) MAP_JON_RPC_WE("refresh", on_refresh, wallet_rpc::COMMAND_RPC_REFRESH) + MAP_JON_RPC_WE("auto_refresh", on_auto_refresh, wallet_rpc::COMMAND_RPC_AUTO_REFRESH) MAP_JON_RPC_WE("rescan_spent", on_rescan_spent, wallet_rpc::COMMAND_RPC_RESCAN_SPENT) MAP_JON_RPC_WE("start_mining", on_start_mining, wallet_rpc::COMMAND_RPC_START_MINING) MAP_JON_RPC_WE("stop_mining", on_stop_mining, wallet_rpc::COMMAND_RPC_STOP_MINING) @@ -210,6 +211,7 @@ namespace tools bool on_add_address_book(const wallet_rpc::COMMAND_RPC_ADD_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_ADD_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_delete_address_book(const wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_refresh(const wallet_rpc::COMMAND_RPC_REFRESH::request& req, wallet_rpc::COMMAND_RPC_REFRESH::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_auto_refresh(const wallet_rpc::COMMAND_RPC_AUTO_REFRESH::request& req, wallet_rpc::COMMAND_RPC_AUTO_REFRESH::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_rescan_spent(const wallet_rpc::COMMAND_RPC_RESCAN_SPENT::request& req, wallet_rpc::COMMAND_RPC_RESCAN_SPENT::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_start_mining(const wallet_rpc::COMMAND_RPC_START_MINING::request& req, wallet_rpc::COMMAND_RPC_START_MINING::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_stop_mining(const wallet_rpc::COMMAND_RPC_STOP_MINING::request& req, wallet_rpc::COMMAND_RPC_STOP_MINING::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); @@ -256,5 +258,7 @@ namespace tools std::atomic<bool> m_stop; bool m_restricted; const boost::program_options::variables_map *m_vm; + uint32_t m_auto_refresh_period; + boost::posix_time::ptime m_last_auto_refresh_time; }; } diff --git a/src/wallet/wallet_rpc_server_commands_defs.h b/src/wallet/wallet_rpc_server_commands_defs.h index 0879502b9..df4370949 100644 --- a/src/wallet/wallet_rpc_server_commands_defs.h +++ b/src/wallet/wallet_rpc_server_commands_defs.h @@ -610,9 +610,11 @@ namespace wallet_rpc struct request_t { std::string unsigned_txset; + std::string multisig_txset; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(unsigned_txset) + KV_SERIALIZE(multisig_txset) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<request_t> request; @@ -1357,6 +1359,7 @@ namespace wallet_rpc std::string type; uint64_t unlock_time; cryptonote::subaddress_index subaddr_index; + std::vector<cryptonote::subaddress_index> subaddr_indices; std::string address; bool double_spend_seen; uint64_t confirmations; @@ -1374,6 +1377,7 @@ namespace wallet_rpc KV_SERIALIZE(type); KV_SERIALIZE(unlock_time) KV_SERIALIZE(subaddr_index); + KV_SERIALIZE(subaddr_indices); KV_SERIALIZE(address); KV_SERIALIZE(double_spend_seen) KV_SERIALIZE_OPT(confirmations, (uint64_t)0) @@ -1928,6 +1932,28 @@ namespace wallet_rpc typedef epee::misc_utils::struct_init<response_t> response; }; + struct COMMAND_RPC_AUTO_REFRESH + { + struct request_t + { + bool enable; + uint32_t period; // seconds + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_OPT(enable, true) + KV_SERIALIZE_OPT(period, (uint32_t)0) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<request_t> request; + + struct response_t + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<response_t> response; + }; + struct COMMAND_RPC_START_MINING { struct request_t |