diff options
Diffstat (limited to 'src')
101 files changed, 2893 insertions, 1663 deletions
diff --git a/src/blockchain_db/berkeleydb/db_bdb.cpp b/src/blockchain_db/berkeleydb/db_bdb.cpp index e1b76ec1e..f827ab7c3 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.cpp +++ b/src/blockchain_db/berkeleydb/db_bdb.cpp @@ -1213,6 +1213,11 @@ std::vector<std::string> BlockchainBDB::get_filenames() const return full_paths; } +bool BlockchainBDB::remove_data_file(const std::string& folder) +{ + return true; +} + std::string BlockchainBDB::get_db_name() const { LOG_PRINT_L3("BlockchainBDB::" << __func__); diff --git a/src/blockchain_db/berkeleydb/db_bdb.h b/src/blockchain_db/berkeleydb/db_bdb.h index cecbba28f..c90d030a2 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.h +++ b/src/blockchain_db/berkeleydb/db_bdb.h @@ -244,6 +244,8 @@ public: virtual std::vector<std::string> get_filenames() const; + virtual bool remove_data_file(const std::string& folder); + virtual std::string get_db_name() const; virtual bool lock(); diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp index 88ac34255..8544cc3f0 100644 --- a/src/blockchain_db/blockchain_db.cpp +++ b/src/blockchain_db/blockchain_db.cpp @@ -218,13 +218,22 @@ uint64_t BlockchainDB::add_block( const block& blk // call out to add the transactions time1 = epee::misc_utils::get_tick_count(); + + uint64_t num_rct_outs = 0; add_transaction(blk_hash, blk.miner_tx); + if (blk.miner_tx.version == 2) + num_rct_outs += blk.miner_tx.vout.size(); int tx_i = 0; crypto::hash tx_hash = crypto::null_hash; for (const transaction& tx : txs) { tx_hash = blk.tx_hashes[tx_i]; add_transaction(blk_hash, tx, &tx_hash); + for (const auto &vout: tx.vout) + { + if (vout.amount == 0) + ++num_rct_outs; + } ++tx_i; } TIME_MEASURE_FINISH(time1); @@ -232,7 +241,7 @@ uint64_t BlockchainDB::add_block( const block& blk // call out to subclass implementation to add the block & metadata time1 = epee::misc_utils::get_tick_count(); - add_block(blk, block_size, cumulative_difficulty, coins_generated, blk_hash); + add_block(blk, block_size, cumulative_difficulty, coins_generated, num_rct_outs, blk_hash); TIME_MEASURE_FINISH(time1); time_add_block1 += time1; diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 19ba32340..6851e2404 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -148,6 +148,7 @@ struct txpool_tx_meta_t uint8_t relayed; uint8_t do_not_relay; uint8_t double_spend_seen: 1; + uint8_t bf_padding: 7; uint8_t padding[76]; // till 192 bytes }; @@ -366,6 +367,7 @@ private: , const size_t& block_size , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated + , uint64_t num_rct_outs , const crypto::hash& blk_hash ) = 0; @@ -654,6 +656,20 @@ public: */ virtual std::vector<std::string> get_filenames() const = 0; + /** + * @brief remove file(s) storing the database + * + * This function is for resetting the database (for core tests, functional tests, etc). + * The function reset() is not usable because it needs to open the database file first + * which can fail if the existing database file is in an incompatible format. + * As such, this function needs to be called before calling open(). + * + * @param folder The path of the folder containing the database file(s) which must not end with slash '/'. + * + * @return true if the operation is succesfull + */ + virtual bool remove_data_file(const std::string& folder) const = 0; + // return the name of the folder the db's file(s) should reside in /** * @brief gets the name of the folder the BlockchainDB's file(s) should be in @@ -891,6 +907,20 @@ public: virtual uint64_t get_block_timestamp(const uint64_t& height) const = 0; /** + * @brief fetch a block's cumulative number of rct outputs + * + * The subclass should return the numer of rct outputs in the blockchain + * up to the block with the given height (inclusive). + * + * If the block does not exist, the subclass should throw BLOCK_DNE + * + * @param height the height requested + * + * @return the cumulative number of rct outputs + */ + virtual std::vector<uint64_t> get_block_cumulative_rct_outputs(const std::vector<uint64_t> &heights) const = 0; + + /** * @brief fetch the top block's timestamp * * The subclass should return the timestamp of the most recent block. @@ -1535,6 +1565,13 @@ public: */ virtual bool is_read_only() const = 0; + /** + * @brief get disk space requirements + * + * @return the size required + */ + virtual uint64_t get_database_size() const = 0; + // TODO: this should perhaps be (or call) a series of functions which // progressively update through version updates /** diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index fe31321f3..a6eb3d880 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -35,6 +35,7 @@ #include <random> #include "string_tools.h" +#include "file_io_utils.h" #include "common/util.h" #include "cryptonote_basic/cryptonote_format_utils.h" #include "crypto/crypto.h" @@ -53,7 +54,7 @@ using epee::string_tools::pod_to_hex; using namespace crypto; // Increase when the DB structure changes -#define VERSION 2 +#define VERSION 3 namespace { @@ -250,6 +251,16 @@ inline void lmdb_db_open(MDB_txn* txn, const char* name, int flags, MDB_dbi& dbi namespace cryptonote { +typedef struct mdb_block_info_old +{ + uint64_t bi_height; + uint64_t bi_timestamp; + uint64_t bi_coins; + uint64_t bi_size; // a size_t really but we need 32-bit compat + difficulty_type bi_diff; + crypto::hash bi_hash; +} mdb_block_info_old; + typedef struct mdb_block_info { uint64_t bi_height; @@ -258,6 +269,7 @@ typedef struct mdb_block_info uint64_t bi_size; // a size_t really but we need 32-bit compat difficulty_type bi_diff; crypto::hash bi_hash; + uint64_t bi_cum_rct; } mdb_block_info; typedef struct blk_height { @@ -667,7 +679,7 @@ estim: } void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, - const crypto::hash& blk_hash) + uint64_t num_rct_outs, const crypto::hash& blk_hash) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -715,6 +727,16 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const bi.bi_size = block_size; bi.bi_diff = cumulative_difficulty; bi.bi_hash = blk_hash; + bi.bi_cum_rct = num_rct_outs; + if (blk.major_version >= 4) + { + uint64_t last_height = m_height-1; + MDB_val_set(h, last_height); + if ((result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &h, MDB_GET_BOTH))) + throw1(BLOCK_DNE(lmdb_error("Failed to get block info: ", result).c_str())); + const mdb_block_info *bi_prev = (const mdb_block_info*)h.mv_data; + bi.bi_cum_rct += bi_prev->bi_cum_rct; + } MDB_val_set(val, bi); result = mdb_cursor_put(m_cur_block_info, (MDB_val *)&zerokval, &val, MDB_APPENDDUP); @@ -759,8 +781,6 @@ void BlockchainLMDB::remove_block() if ((result = mdb_cursor_del(m_cur_block_heights, 0))) throw1(DB_ERROR(lmdb_error("Failed to add removal of block height by hash to db transaction: ", result).c_str())); - if ((result = mdb_cursor_get(m_cur_blocks, &k, NULL, MDB_SET))) - throw1(DB_ERROR(lmdb_error("Failed to locate block for removal: ", result).c_str())); if ((result = mdb_cursor_del(m_cur_blocks, 0))) throw1(DB_ERROR(lmdb_error("Failed to add removal of block to db transaction: ", result).c_str())); @@ -1143,6 +1163,8 @@ BlockchainLMDB::BlockchainLMDB(bool batch_transactions): BlockchainDB() m_cum_size = 0; m_cum_count = 0; + // reset may also need changing when initialize things here + m_hardfork = nullptr; } @@ -1306,20 +1328,21 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) auto get_result = mdb_get(txn, m_properties, &k, &v); if(get_result == MDB_SUCCESS) { - if (*(const uint32_t*)v.mv_data > VERSION) + const uint32_t db_version = *(const uint32_t*)v.mv_data; + if (db_version > VERSION) { - MWARNING("Existing lmdb database was made by a later version. We don't know how it will change yet."); + MWARNING("Existing lmdb database was made by a later version (" << db_version << "). We don't know how it will change yet."); compatible = false; } #if VERSION > 0 - else if (*(const uint32_t*)v.mv_data < VERSION) + else if (db_version < VERSION) { // Note that there was a schema change within version 0 as well. // See commit e5d2680094ee15889934fe28901e4e133cda56f2 2015/07/10 // We don't handle the old format previous to that commit. txn.commit(); m_open = true; - migrate(*(const uint32_t *)v.mv_data); + migrate(db_version); return; } #endif @@ -1390,6 +1413,9 @@ void BlockchainLMDB::sync() LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); + if (is_read_only()) + return; + // Does nothing unless LMDB environment was opened with MDB_NOSYNC or in part // MDB_NOMETASYNC. Force flush to be synchronous. if (auto result = mdb_env_sync(m_env, true)) @@ -1468,6 +1494,21 @@ std::vector<std::string> BlockchainLMDB::get_filenames() const return filenames; } +bool BlockchainLMDB::remove_data_file(const std::string& folder) const +{ + const std::string filename = folder + "/data.mdb"; + try + { + boost::filesystem::remove(filename); + } + catch (const std::exception &e) + { + MERROR("Failed to remove " << filename << ": " << e.what()); + return false; + } + return true; +} + std::string BlockchainLMDB::get_db_name() const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -1907,6 +1948,43 @@ uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const return ret; } +std::vector<uint64_t> BlockchainLMDB::get_block_cumulative_rct_outputs(const std::vector<uint64_t> &heights) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + std::vector<uint64_t> res; + int result; + + if (heights.empty()) + return {}; + res.reserve(heights.size()); + + TXN_PREFIX_RDONLY(); + RCURSOR(block_info); + + MDB_stat db_stats; + if ((result = mdb_stat(m_txn, m_blocks, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str())); + for (size_t i = 0; i < heights.size(); ++i) + if (heights[i] >= db_stats.ms_entries) + throw0(BLOCK_DNE(std::string("Attempt to get rct distribution from height " + std::to_string(heights[i]) + " failed -- block size not in db").c_str())); + + MDB_val v; + + for (uint64_t height: heights) + { + MDB_val_set(v, height); + result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); + if (result) + throw0(DB_ERROR(lmdb_error("Error attempting to retrieve rct distribution from the db: ", result).c_str())); + const mdb_block_info *bi = (const mdb_block_info *)v.mv_data; + res.push_back(bi->bi_cum_rct); + } + + TXN_POSTFIX_RDONLY(); + return res; +} + uint64_t BlockchainLMDB::get_top_block_timestamp() const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -3327,6 +3405,7 @@ bool BlockchainLMDB::get_output_distribution(uint64_t amount, uint64_t from_heig MDB_val_set(k, amount); MDB_val v; MDB_cursor_op op = MDB_SET; + base = 0; while (1) { int ret = mdb_cursor_get(m_cur_output_amounts, &k, &v, op); @@ -3345,6 +3424,9 @@ bool BlockchainLMDB::get_output_distribution(uint64_t amount, uint64_t from_heig break; } + for (size_t n = 1; n < distribution.size(); ++n) + distribution[n] += distribution[n - 1]; + TXN_POSTFIX_RDONLY(); return true; @@ -3422,6 +3504,16 @@ bool BlockchainLMDB::is_read_only() const return false; } +uint64_t BlockchainLMDB::get_database_size() const +{ + uint64_t size = 0; + boost::filesystem::path datafile(m_folder); + datafile /= CRYPTONOTE_BLOCKCHAINDATA_FILENAME; + if (!epee::file_io_utils::get_file_size(datafile.string(), size)) + size = 0; + return size; +} + void BlockchainLMDB::fixup() { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -3439,7 +3531,7 @@ void BlockchainLMDB::fixup() if (result) \ throw0(DB_ERROR(lmdb_error("Failed to get DB record for " name ": ", result).c_str())); \ ptr = (char *)k.mv_data; \ - ptr[sizeof(name)-2] = 's' + ptr[sizeof(name)-2]++ #define LOGIF(y) if (ELPP->vRegistry()->allowed(y, "global")) @@ -3579,7 +3671,7 @@ void BlockchainLMDB::migrate_0_1() break; } MDB_dbi diffs, hashes, sizes, timestamps; - mdb_block_info bi; + mdb_block_info_old bi; MDB_val_set(nv, bi); lmdb_db_open(txn, "block_diffs", 0, diffs, "Failed to open db handle for block_diffs"); @@ -4119,6 +4211,141 @@ void BlockchainLMDB::migrate_1_2() txn.commit(); } +void BlockchainLMDB::migrate_2_3() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + uint64_t i; + int result; + mdb_txn_safe txn(false); + MDB_val k, v; + char *ptr; + + MGINFO_YELLOW("Migrating blockchain from DB version 2 to 3 - this may take a while:"); + + do { + LOG_PRINT_L1("migrating block info:"); + + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + + MDB_stat db_stats; + if ((result = mdb_stat(txn, m_blocks, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str())); + const uint64_t blockchain_height = db_stats.ms_entries; + + MDEBUG("enumerating rct outputs..."); + std::vector<uint64_t> distribution(blockchain_height, 0); + bool r = for_all_outputs(0, [&](uint64_t height) { + if (height >= blockchain_height) + { + MERROR("Output found claiming height >= blockchain height"); + return false; + } + distribution[height]++; + return true; + }); + if (!r) + throw0(DB_ERROR("Failed to build rct output distribution")); + for (size_t i = 1; i < distribution.size(); ++i) + distribution[i] += distribution[i - 1]; + + /* the block_info table name is the same but the old version and new version + * have incompatible data. Create a new table. We want the name to be similar + * to the old name so that it will occupy the same location in the DB. + */ + MDB_dbi o_block_info = m_block_info; + lmdb_db_open(txn, "block_infn", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn"); + mdb_set_dupsort(txn, m_block_info, compare_uint64); + + MDB_cursor *c_old, *c_cur; + i = 0; + while(1) { + if (!(i % 1000)) { + if (i) { + LOGIF(el::Level::Info) { + std::cout << i << " / " << blockchain_height << " \r" << std::flush; + } + txn.commit(); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + } + result = mdb_cursor_open(txn, m_block_info, &c_cur); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_infn: ", result).c_str())); + result = mdb_cursor_open(txn, o_block_info, &c_old); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_info: ", result).c_str())); + if (!i) { + MDB_stat db_stat; + result = mdb_stat(txn, m_block_info, &db_stats); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to query m_block_info: ", result).c_str())); + i = db_stats.ms_entries; + } + } + result = mdb_cursor_get(c_old, &k, &v, MDB_NEXT); + if (result == MDB_NOTFOUND) { + txn.commit(); + break; + } + else if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from block_info: ", result).c_str())); + const mdb_block_info_old *bi_old = (const mdb_block_info_old*)v.mv_data; + mdb_block_info bi; + bi.bi_height = bi_old->bi_height; + bi.bi_timestamp = bi_old->bi_timestamp; + bi.bi_coins = bi_old->bi_coins; + bi.bi_size = bi_old->bi_size; + bi.bi_diff = bi_old->bi_diff; + bi.bi_hash = bi_old->bi_hash; + if (bi_old->bi_height >= distribution.size()) + throw0(DB_ERROR("Bad height in block_info record")); + bi.bi_cum_rct = distribution[bi_old->bi_height]; + MDB_val_set(nv, bi); + result = mdb_cursor_put(c_cur, (MDB_val *)&zerokval, &nv, MDB_APPENDDUP); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to put a record into block_infn: ", result).c_str())); + /* we delete the old records immediately, so the overall DB and mapsize should not grow. + * This is a little slower than just letting mdb_drop() delete it all at the end, but + * it saves a significant amount of disk space. + */ + result = mdb_cursor_del(c_old, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_info: ", result).c_str())); + i++; + } + + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + /* Delete the old table */ + result = mdb_drop(txn, o_block_info, 1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete old block_info table: ", result).c_str())); + + RENAME_DB("block_infn"); + + lmdb_db_open(txn, "block_info", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn"); + mdb_set_dupsort(txn, m_block_info, compare_uint64); + + txn.commit(); + } while(0); + + uint32_t version = 3; + v.mv_data = (void *)&version; + v.mv_size = sizeof(version); + MDB_val_copy<const char *> vk("version"); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + result = mdb_put(txn, m_properties, &vk, &v, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to update version for the db: ", result).c_str())); + txn.commit(); +} + void BlockchainLMDB::migrate(const uint32_t oldversion) { switch(oldversion) { @@ -4126,6 +4353,8 @@ void BlockchainLMDB::migrate(const uint32_t oldversion) migrate_0_1(); /* FALLTHRU */ case 1: migrate_1_2(); /* FALLTHRU */ + case 2: + migrate_2_3(); /* FALLTHRU */ default: ; } diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index cc1b06ca0..8b214d2df 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -166,7 +166,7 @@ struct mdb_txn_safe class BlockchainLMDB : public BlockchainDB { public: - BlockchainLMDB(bool batch_transactions=false); + BlockchainLMDB(bool batch_transactions=true); ~BlockchainLMDB(); virtual void open(const std::string& filename, const int mdb_flags=0); @@ -181,6 +181,8 @@ public: virtual std::vector<std::string> get_filenames() const; + virtual bool remove_data_file(const std::string& folder) const; + virtual std::string get_db_name() const; virtual bool lock(); @@ -197,6 +199,8 @@ public: virtual cryptonote::blobdata get_block_blob_from_height(const uint64_t& height) const; + virtual std::vector<uint64_t> get_block_cumulative_rct_outputs(const std::vector<uint64_t> &heights) const; + virtual uint64_t get_block_timestamp(const uint64_t& height) const; virtual uint64_t get_top_block_timestamp() const; @@ -317,6 +321,7 @@ private: , const size_t& block_size , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated + , uint64_t num_rct_outs , const crypto::hash& block_hash ); @@ -375,6 +380,8 @@ private: virtual bool is_read_only() const; + virtual uint64_t get_database_size() const; + // fix up anything that may be wrong due to past bugs virtual void fixup(); @@ -387,6 +394,9 @@ private: // migrate from DB version 1 to 2 void migrate_1_2(); + // migrate from DB version 2 to 3 + void migrate_2_3(); + void cleanup_batch(); private: diff --git a/src/blockchain_utilities/blockchain_blackball.cpp b/src/blockchain_utilities/blockchain_blackball.cpp index 95eb2f73d..52d2938cd 100644 --- a/src/blockchain_utilities/blockchain_blackball.cpp +++ b/src/blockchain_utilities/blockchain_blackball.cpp @@ -235,7 +235,7 @@ int main(int argc, char* argv[]) "blackball-db-dir", "Specify blackball database directory", get_default_db_path(), {{ &arg_testnet_on, &arg_stagenet_on }}, - [](std::array<bool, 2> testnet_stagenet, bool defaulted, std::string val) { + [](std::array<bool, 2> testnet_stagenet, bool defaulted, std::string val)->std::string { if (testnet_stagenet[0]) return (boost::filesystem::path(val) / "testnet").string(); else if (testnet_stagenet[1]) diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp index caa549c13..7291dbd68 100644 --- a/src/blockchain_utilities/blockchain_import.cpp +++ b/src/blockchain_utilities/blockchain_import.cpp @@ -33,6 +33,7 @@ #include <boost/filesystem.hpp> #include <boost/algorithm/string.hpp> +#include <unistd.h> #include "misc_log_ex.h" #include "bootstrap_file.h" #include "bootstrap_serialization.h" @@ -164,7 +165,7 @@ int pop_blocks(cryptonote::core& core, int num_blocks) return num_blocks; } -int check_flush(cryptonote::core &core, std::list<block_complete_entry> &blocks, bool force) +int check_flush(cryptonote::core &core, std::vector<block_complete_entry> &blocks, bool force) { if (blocks.empty()) return 0; @@ -176,7 +177,7 @@ int check_flush(cryptonote::core &core, std::list<block_complete_entry> &blocks, if (!force && new_height % HASH_OF_HASHES_STEP) return 0; - std::list<crypto::hash> hashes; + std::vector<crypto::hash> hashes; for (const auto &b: blocks) { cryptonote::block block; @@ -312,7 +313,7 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path MINFO("Reading blockchain from bootstrap file..."); std::cout << ENDL; - std::list<block_complete_entry> blocks; + std::vector<block_complete_entry> blocks; // Skip to start_height before we start adding. { @@ -437,7 +438,7 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path { cryptonote::blobdata block; cryptonote::block_to_blob(bp.block, block); - std::list<cryptonote::blobdata> txs; + std::vector<cryptonote::blobdata> txs; for (const auto &tx: bp.txs) { txs.push_back(cryptonote::blobdata()); @@ -593,8 +594,8 @@ int main(int argc, char* argv[]) const command_line::arg_descriptor<std::string> arg_database = { "database", available_dbs.c_str(), default_db_type }; - const command_line::arg_descriptor<bool> arg_verify = {"guard-against-pwnage", - "Verify blocks and transactions during import (only disable if you exported the file yourself)", true}; + const command_line::arg_descriptor<bool> arg_noverify = {"dangerous-unverified-import", + "Blindly trust the import file and use potentially malicious blocks and transactions during import (only enable if you exported the file yourself)", false}; const command_line::arg_descriptor<bool> arg_batch = {"batch", "Batch transactions for faster import", true}; const command_line::arg_descriptor<bool> arg_resume = {"resume", @@ -614,7 +615,7 @@ int main(int argc, char* argv[]) // call add_options() directly for these arguments since // command_line helpers support only boolean switch, not boolean argument desc_cmd_sett.add_options() - (arg_verify.name, make_semantic(arg_verify), arg_verify.description) + (arg_noverify.name, make_semantic(arg_noverify), arg_noverify.description) (arg_batch.name, make_semantic(arg_batch), arg_batch.description) (arg_resume.name, make_semantic(arg_resume), arg_resume.description) ; @@ -633,7 +634,7 @@ int main(int argc, char* argv[]) if (! r) return 1; - opt_verify = command_line::get_arg(vm, arg_verify); + opt_verify = !command_line::get_arg(vm, arg_noverify); opt_batch = command_line::get_arg(vm, arg_batch); opt_resume = command_line::get_arg(vm, arg_resume); block_stop = command_line::get_arg(vm, arg_block_stop); @@ -738,6 +739,18 @@ int main(int argc, char* argv[]) MINFO("bootstrap file path: " << import_file_path); MINFO("database path: " << m_config_folder); + if (!opt_verify) + { + MCLOG_RED(el::Level::Warning, "global", "\n" + "Import is set to proceed WITHOUT VERIFICATION.\n" + "This is a DANGEROUS operation: if the file was tampered with in transit, or obtained from a malicious source,\n" + "you could end up with a compromised database. It is recommended to NOT use " << arg_noverify.name << ".\n" + "*****************************************************************************************\n" + "You have 90 seconds to press ^C or terminate this program before unverified import starts\n" + "*****************************************************************************************"); + sleep(90); + } + cryptonote::cryptonote_protocol_stub pr; //TODO: stub only for this kind of test, make real validation of relayed objects cryptonote::core core(&pr); diff --git a/src/blockchain_utilities/bootstrap_file.cpp b/src/blockchain_utilities/bootstrap_file.cpp index aa85e5e53..ba2697226 100644 --- a/src/blockchain_utilities/bootstrap_file.cpp +++ b/src/blockchain_utilities/bootstrap_file.cpp @@ -400,18 +400,18 @@ uint64_t BootstrapFile::count_bytes(std::ifstream& import_file, uint64_t blocks, { std::cout << refresh_string; MWARNING("WARNING: chunk_size " << chunk_size << " > BUFFER_SIZE " << BUFFER_SIZE - << " height: " << h-1); + << " height: " << h-1 << ", offset " << bytes_read); throw std::runtime_error("Aborting: chunk size exceeds buffer size"); } if (chunk_size > CHUNK_SIZE_WARNING_THRESHOLD) { std::cout << refresh_string; MDEBUG("NOTE: chunk_size " << chunk_size << " > " << CHUNK_SIZE_WARNING_THRESHOLD << " << height: " - << h-1); + << h-1 << ", offset " << bytes_read); } else if (chunk_size <= 0) { std::cout << refresh_string; - MDEBUG("ERROR: chunk_size " << chunk_size << " <= 0" << " height: " << h-1); + MDEBUG("ERROR: chunk_size " << chunk_size << " <= 0" << " height: " << h-1 << ", offset " << bytes_read); throw std::runtime_error("Aborting"); } // skip to next expected block size value diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 808ef7630..f0df05b0d 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -78,7 +78,6 @@ monero_add_library(common DEPENDS generate_translations_header) target_link_libraries(common PUBLIC - epee cncrypto ${UNBOUND_LIBRARY} ${LIBUNWIND_LIBRARIES} diff --git a/src/common/password.cpp b/src/common/password.cpp index 9336a14fc..3ce2ba42a 100644 --- a/src/common/password.cpp +++ b/src/common/password.cpp @@ -164,7 +164,7 @@ namespace while (true) { if (message) - std::cout << message <<": "; + std::cout << message <<": " << std::flush; if (!read_from_tty(pass1)) return false; if (verify) diff --git a/src/common/scoped_message_writer.h b/src/common/scoped_message_writer.h index d7517babb..d887a13c9 100644 --- a/src/common/scoped_message_writer.h +++ b/src/common/scoped_message_writer.h @@ -73,7 +73,7 @@ public: #if defined(_MSC_VER) , m_oss(std::move(rhs.m_oss)) #else - // GCC bug: http://gcc.gnu.org/bugzilla/show_bug.cgi?id=54316 + // GCC bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=54316 , m_oss(rhs.m_oss.str(), std::ios_base::out | std::ios_base::ate) #endif , m_color(std::move(rhs.m_color)) diff --git a/src/common/stack_trace.cpp b/src/common/stack_trace.cpp index 9c2bf4b53..d6dc4d7cc 100644 --- a/src/common/stack_trace.cpp +++ b/src/common/stack_trace.cpp @@ -51,7 +51,7 @@ #define ST_LOG(x) CINFO(el::base::Writer,el::base::DispatchAction::FileOnlyLog,MONERO_DEFAULT_LOG_CATEGORY) << x -// from http://stackoverflow.com/questions/11665829/how-can-i-print-stack-trace-for-caught-exceptions-in-c-code-injection-in-c +// from https://stackoverflow.com/questions/11665829/how-can-i-print-stack-trace-for-caught-exceptions-in-c-code-injection-in-c // The decl of __cxa_throw in /usr/include/.../cxxabi.h uses // 'std::type_info *', but GCC's built-in protype uses 'void *'. diff --git a/src/common/threadpool.cpp b/src/common/threadpool.cpp index 51e071577..6b69e2a12 100644 --- a/src/common/threadpool.cpp +++ b/src/common/threadpool.cpp @@ -36,16 +36,17 @@ #include "common/util.h" static __thread int depth = 0; +static __thread bool is_leaf = false; namespace tools { -threadpool::threadpool() : running(true), active(0) { +threadpool::threadpool(unsigned int max_threads) : running(true), active(0) { boost::thread::attributes attrs; attrs.set_stack_size(THREAD_STACK_SIZE); - max = tools::get_max_concurrency(); - size_t i = max; + max = max_threads ? max_threads : tools::get_max_concurrency(); + size_t i = max ? max - 1 : 0; while(i--) { - threads.push_back(boost::thread(attrs, boost::bind(&threadpool::run, this))); + threads.push_back(boost::thread(attrs, boost::bind(&threadpool::run, this, false))); } } @@ -60,25 +61,30 @@ threadpool::~threadpool() { } } -void threadpool::submit(waiter *obj, std::function<void()> f) { - entry e = {obj, f}; +void threadpool::submit(waiter *obj, std::function<void()> f, bool leaf) { + CHECK_AND_ASSERT_THROW_MES(!is_leaf, "A leaf routine is using a thread pool"); boost::unique_lock<boost::mutex> lock(mutex); - if ((active == max && !queue.empty()) || depth > 0) { + if (!leaf && ((active == max && !queue.empty()) || depth > 0)) { // if all available threads are already running // and there's work waiting, just run in current thread lock.unlock(); ++depth; + is_leaf = leaf; f(); --depth; + is_leaf = false; } else { if (obj) obj->inc(); - queue.push_back(e); + if (leaf) + queue.push_front({obj, f, leaf}); + else + queue.push_back({obj, f, leaf}); has_work.notify_one(); } } -int threadpool::get_max_concurrency() { +unsigned int threadpool::get_max_concurrency() const { return max; } @@ -91,7 +97,7 @@ threadpool::waiter::~waiter() } try { - wait(); + wait(NULL); } catch (const std::exception &e) { @@ -99,9 +105,12 @@ threadpool::waiter::~waiter() } } -void threadpool::waiter::wait() { +void threadpool::waiter::wait(threadpool *tpool) { + if (tpool) + tpool->run(true); boost::unique_lock<boost::mutex> lock(mt); - while(num) cv.wait(lock); + while(num) + cv.wait(lock); } void threadpool::waiter::inc() { @@ -113,15 +122,19 @@ void threadpool::waiter::dec() { const boost::unique_lock<boost::mutex> lock(mt); num--; if (!num) - cv.notify_one(); + cv.notify_all(); } -void threadpool::run() { +void threadpool::run(bool flush) { boost::unique_lock<boost::mutex> lock(mutex); while (running) { entry e; while(queue.empty() && running) + { + if (flush) + return; has_work.wait(lock); + } if (!running) break; active++; @@ -129,8 +142,10 @@ void threadpool::run() { queue.pop_front(); lock.unlock(); ++depth; + is_leaf = e.leaf; e.f(); --depth; + is_leaf = false; if (e.wo) e.wo->dec(); diff --git a/src/common/threadpool.h b/src/common/threadpool.h index 34152541c..a43e38a76 100644 --- a/src/common/threadpool.h +++ b/src/common/threadpool.h @@ -46,6 +46,9 @@ public: static threadpool instance; return instance; } + static threadpool *getNewForUnitTests(unsigned max_threads = 0) { + return new threadpool(max_threads); + } // The waiter lets the caller know when all of its // tasks are completed. @@ -56,7 +59,7 @@ public: public: void inc(); void dec(); - void wait(); //! Wait for a set of tasks to finish. + void wait(threadpool *tpool); //! Wait for a set of tasks to finish. waiter() : num(0){} ~waiter(); }; @@ -64,25 +67,27 @@ public: // Submit a task to the pool. The waiter pointer may be // NULL if the caller doesn't care to wait for the // task to finish. - void submit(waiter *waiter, std::function<void()> f); + void submit(waiter *waiter, std::function<void()> f, bool leaf = false); + + unsigned int get_max_concurrency() const; - int get_max_concurrency(); + ~threadpool(); private: - threadpool(); - ~threadpool(); + threadpool(unsigned int max_threads = 0); typedef struct entry { waiter *wo; std::function<void()> f; + bool leaf; } entry; std::deque<entry> queue; boost::condition_variable has_work; boost::mutex mutex; std::vector<boost::thread> threads; - int active; - int max; + unsigned int active; + unsigned int max; bool running; - void run(); + void run(bool flush = false); }; } diff --git a/src/common/util.cpp b/src/common/util.cpp index 008610117..f644c573c 100644 --- a/src/common/util.cpp +++ b/src/common/util.cpp @@ -195,6 +195,73 @@ namespace tools catch (...) {} } + file_locker::file_locker(const std::string &filename) + { +#ifdef WIN32 + m_fd = INVALID_HANDLE_VALUE; + std::wstring filename_wide; + try + { + filename_wide = string_tools::utf8_to_utf16(filename); + } + catch (const std::exception &e) + { + MERROR("Failed to convert path \"" << filename << "\" to UTF-16: " << e.what()); + return; + } + m_fd = CreateFileW(filename_wide.c_str(), GENERIC_READ, 0, NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + if (m_fd != INVALID_HANDLE_VALUE) + { + OVERLAPPED ov; + memset(&ov, 0, sizeof(ov)); + if (!LockFileEx(m_fd, LOCKFILE_FAIL_IMMEDIATELY | LOCKFILE_EXCLUSIVE_LOCK, 0, 1, 0, &ov)) + { + MERROR("Failed to lock " << filename << ": " << std::error_code(GetLastError(), std::system_category())); + CloseHandle(m_fd); + m_fd = INVALID_HANDLE_VALUE; + } + } + else + { + MERROR("Failed to open " << filename << ": " << std::error_code(GetLastError(), std::system_category())); + } +#else + m_fd = open(filename.c_str(), O_RDONLY | O_CREAT, 0666); + if (m_fd != -1) + { + if (flock(m_fd, LOCK_EX | LOCK_NB) == -1) + { + MERROR("Failed to lock " << filename << ": " << std::strerror(errno)); + close(m_fd); + m_fd = -1; + } + } + else + { + MERROR("Failed to open " << filename << ": " << std::strerror(errno)); + } +#endif + } + file_locker::~file_locker() + { + if (locked()) + { +#ifdef WIN32 + CloseHandle(m_fd); +#else + close(m_fd); +#endif + } + } + bool file_locker::locked() const + { +#ifdef WIN32 + return m_fd != INVALID_HANDLE_VALUE; +#else + return m_fd != -1; +#endif + } + #ifdef WIN32 std::string get_windows_version_display_string() { @@ -451,10 +518,15 @@ std::string get_nix_version_display_string() if (SHGetSpecialFolderPathW(NULL, psz_path, nfolder, iscreate)) { - int size_needed = WideCharToMultiByte(CP_UTF8, 0, psz_path, wcslen(psz_path), NULL, 0, NULL, NULL); - std::string folder_name(size_needed, 0); - WideCharToMultiByte(CP_UTF8, 0, psz_path, wcslen(psz_path), &folder_name[0], size_needed, NULL, NULL); - return folder_name; + try + { + return string_tools::utf16_to_utf8(psz_path); + } + catch (const std::exception &e) + { + MERROR("utf16_to_utf8 failed: " << e.what()); + return ""; + } } LOG_ERROR("SHGetSpecialFolderPathW() failed, could not obtain requested path."); @@ -515,18 +587,20 @@ std::string get_nix_version_display_string() int code; #if defined(WIN32) // Maximizing chances for success - WCHAR wide_replacement_name[1000]; - MultiByteToWideChar(CP_UTF8, 0, replacement_name.c_str(), replacement_name.size() + 1, wide_replacement_name, 1000); - WCHAR wide_replaced_name[1000]; - MultiByteToWideChar(CP_UTF8, 0, replaced_name.c_str(), replaced_name.size() + 1, wide_replaced_name, 1000); - - DWORD attributes = ::GetFileAttributesW(wide_replaced_name); + std::wstring wide_replacement_name; + try { wide_replacement_name = string_tools::utf8_to_utf16(replacement_name); } + catch (...) { return std::error_code(GetLastError(), std::system_category()); } + std::wstring wide_replaced_name; + try { wide_replaced_name = string_tools::utf8_to_utf16(replaced_name); } + catch (...) { return std::error_code(GetLastError(), std::system_category()); } + + DWORD attributes = ::GetFileAttributesW(wide_replaced_name.c_str()); if (INVALID_FILE_ATTRIBUTES != attributes) { - ::SetFileAttributesW(wide_replaced_name, attributes & (~FILE_ATTRIBUTE_READONLY)); + ::SetFileAttributesW(wide_replaced_name.c_str(), attributes & (~FILE_ATTRIBUTE_READONLY)); } - bool ok = 0 != ::MoveFileExW(wide_replacement_name, wide_replaced_name, MOVEFILE_REPLACE_EXISTING); + bool ok = 0 != ::MoveFileExW(wide_replacement_name.c_str(), wide_replaced_name.c_str(), MOVEFILE_REPLACE_EXISTING); code = ok ? 0 : static_cast<int>(::GetLastError()); #else bool ok = 0 == std::rename(replacement_name.c_str(), replaced_name.c_str()); @@ -727,6 +801,13 @@ std::string get_nix_version_display_string() bool is_local_address(const std::string &address) { + // always assume Tor/I2P addresses to be untrusted by default + if (boost::ends_with(address, ".onion") || boost::ends_with(address, ".i2p")) + { + MDEBUG("Address '" << address << "' is Tor/I2P, non local"); + return false; + } + // extract host epee::net_utils::http::url_content u_c; if (!epee::net_utils::parse_url(address, u_c)) @@ -820,4 +901,41 @@ std::string get_nix_version_display_string() return false; return true; } + + boost::optional<std::pair<uint32_t, uint32_t>> parse_subaddress_lookahead(const std::string& str) + { + auto pos = str.find(":"); + bool r = pos != std::string::npos; + uint32_t major; + r = r && epee::string_tools::get_xtype_from_string(major, str.substr(0, pos)); + uint32_t minor; + r = r && epee::string_tools::get_xtype_from_string(minor, str.substr(pos + 1)); + if (r) + { + return std::make_pair(major, minor); + } + else + { + return {}; + } + } + + std::string glob_to_regex(const std::string &val) + { + std::string newval; + + bool escape = false; + for (char c: val) + { + if (c == '*') + newval += escape ? "*" : ".*"; + else if (c == '?') + newval += escape ? "?" : "."; + else if (c == '\\') + newval += '\\', escape = !escape; + else + newval += c; + } + return newval; + } } diff --git a/src/common/util.h b/src/common/util.h index 7caf0e3c5..6ec901e7f 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -32,6 +32,7 @@ #include <boost/thread/locks.hpp> #include <boost/thread/mutex.hpp> +#include <boost/optional.hpp> #include <system_error> #include <csignal> #include <cstdio> @@ -90,6 +91,20 @@ namespace tools const std::string& filename() const noexcept { return m_filename; } }; + class file_locker + { + public: + file_locker(const std::string &filename); + ~file_locker(); + bool locked() const; + private: +#ifdef WIN32 + HANDLE m_fd; +#else + int m_fd; +#endif + }; + /*! \brief Returns the default data directory. * * \details Windows < Vista: C:\\Documents and Settings\\Username\\Application Data\\CRYPTONOTE_NAME @@ -214,4 +229,8 @@ namespace tools bool sha256sum(const std::string &filename, crypto::hash &hash); bool is_hdd(const char *path); + + boost::optional<std::pair<uint32_t, uint32_t>> parse_subaddress_lookahead(const std::string& str); + + std::string glob_to_regex(const std::string &val); } diff --git a/src/crypto/blake256.c b/src/crypto/blake256.c index d503c47e0..6ef7d4207 100644 --- a/src/crypto/blake256.c +++ b/src/crypto/blake256.c @@ -31,7 +31,7 @@ * The blake256_* and blake224_* functions are largely copied from * blake256_light.c and blake224_light.c from the BLAKE website: * - * http://131002.net/blake/ + * https://131002.net/blake/ * * The hmac_* functions implement HMAC-BLAKE-256 and HMAC-BLAKE-224. * HMAC is specified by RFC 2104. diff --git a/src/crypto/crypto.cpp b/src/crypto/crypto.cpp index f4ef751d3..0c019938d 100644 --- a/src/crypto/crypto.cpp +++ b/src/crypto/crypto.cpp @@ -93,18 +93,32 @@ namespace crypto { generate_random_bytes_not_thread_safe(N, bytes); } - /* generate a random 32-byte (256-bit) integer and copy it to res */ - static inline void random_scalar_not_thread_safe(ec_scalar &res) { - unsigned char tmp[64]; - generate_random_bytes_not_thread_safe(64, tmp); - sc_reduce(tmp); - memcpy(&res, tmp, 32); + static inline bool less32(const unsigned char *k0, const unsigned char *k1) + { + for (int n = 31; n >= 0; --n) + { + if (k0[n] < k1[n]) + return true; + if (k0[n] > k1[n]) + return false; + } + return false; } + + void random32_unbiased(unsigned char *bytes) + { + // l = 2^252 + 27742317777372353535851937790883648493. + // it fits 15 in 32 bytes + static const unsigned char limit[32] = { 0xe3, 0x6a, 0x67, 0x72, 0x8b, 0xce, 0x13, 0x29, 0x8f, 0x30, 0x82, 0x8c, 0x0b, 0xa4, 0x10, 0x39, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf0 }; + do + { + generate_random_bytes_thread_safe(32, bytes); + } while (!less32(bytes, limit)); // should be good about 15/16 of the time + sc_reduce32(bytes); + } + /* generate a random 32-byte (256-bit) integer and copy it to res */ static inline void random_scalar(ec_scalar &res) { - unsigned char tmp[64]; - generate_random_bytes_thread_safe(64, tmp); - sc_reduce(tmp); - memcpy(&res, tmp, 32); + random32_unbiased((unsigned char*)res.data); } void hash_to_scalar(const void *data, size_t length, ec_scalar &res) { diff --git a/src/crypto/crypto.h b/src/crypto/crypto.h index 9ea0f2ec0..449af8f6d 100644 --- a/src/crypto/crypto.h +++ b/src/crypto/crypto.h @@ -99,6 +99,7 @@ namespace crypto { #pragma pack(pop) void hash_to_scalar(const void *data, size_t length, ec_scalar &res); + void random32_unbiased(unsigned char *bytes); static_assert(sizeof(ec_point) == 32 && sizeof(ec_scalar) == 32 && sizeof(public_key) == 32 && sizeof(secret_key) == 32 && diff --git a/src/crypto/crypto_ops_builder/ref10/README.md b/src/crypto/crypto_ops_builder/ref10/README.md new file mode 100644 index 000000000..59193305b --- /dev/null +++ b/src/crypto/crypto_ops_builder/ref10/README.md @@ -0,0 +1,4 @@ +This code comes from Daniel J. Bernstein's SUPERCOP source, +released in the public domain. + +[http://ed25519.cr.yp.to/software.html](http://ed25519.cr.yp.to/software.html) diff --git a/src/crypto/crypto_ops_builder/ref10/description b/src/crypto/crypto_ops_builder/ref10/description index cbfcb2cba..99f747747 100644 --- a/src/crypto/crypto_ops_builder/ref10/description +++ b/src/crypto/crypto_ops_builder/ref10/description @@ -1,2 +1,2 @@ EdDSA signatures using Curve25519 -from http://hyperelliptic.org/ebats/supercop-20141124.tar.bz2 +from https://hyperelliptic.org/ebats/supercop-20141124.tar.bz2 diff --git a/src/crypto/crypto_ops_builder/ref10CommentedCombined/MakeCryptoOps.py b/src/crypto/crypto_ops_builder/ref10CommentedCombined/MakeCryptoOps.py index 9b55d260d..0ed97d5f4 100644 --- a/src/crypto/crypto_ops_builder/ref10CommentedCombined/MakeCryptoOps.py +++ b/src/crypto/crypto_ops_builder/ref10CommentedCombined/MakeCryptoOps.py @@ -1,5 +1,5 @@ #assumes you have gnu sed, osx sed might need slight syntax changeo -#c.f. http://unix.stackexchange.com/questions/112023/how-can-i-replace-a-string-in-a-files +#c.f. https://unix.stackexchange.com/questions/112023/how-can-i-replace-a-string-in-a-files #written by shen-noether monero research labs @@ -8,7 +8,7 @@ import glob #for copy files import textwrap #for comments etc print("make sure you have cat and grep installed") -print("also assumes gnu sed syntax, c.f. :http://unix.stackexchange.com/questions/112023/how-can-i-replace-a-string-in-a-files") +print("also assumes gnu sed syntax, c.f. :https://unix.stackexchange.com/questions/112023/how-can-i-replace-a-string-in-a-files") print("I believe osx may have slightly different version of sed") print("maybe someone smart can replace the sed with perl..") diff --git a/src/crypto/crypto_ops_builder/ref10CommentedCombined/description b/src/crypto/crypto_ops_builder/ref10CommentedCombined/description index fadc9f9af..9407b400a 100644 --- a/src/crypto/crypto_ops_builder/ref10CommentedCombined/description +++ b/src/crypto/crypto_ops_builder/ref10CommentedCombined/description @@ -2,6 +2,6 @@ shen_ed25519_ref10 MakeCryptoOps.py makes crypto-ops.c in the Monero source from the ref10 implementation EdDSA signatures using Curve25519 -from http://hyperelliptic.org/ebats/supercop-20141124.tar.bz2 +from https://hyperelliptic.org/ebats/supercop-20141124.tar.bz2 Commented / combined by Shen Noether, Monero Research Lab diff --git a/src/crypto/initializer.h b/src/crypto/initializer.h index afbace726..107988d2b 100644 --- a/src/crypto/initializer.h +++ b/src/crypto/initializer.h @@ -43,8 +43,8 @@ #elif defined(_MSC_VER) #include <assert.h> #include <stdlib.h> -// http://stackoverflow.com/questions/1113409/attribute-constructor-equivalent-in-vc -// http://msdn.microsoft.com/en-us/library/bb918180.aspx +// https://stackoverflow.com/questions/1113409/attribute-constructor-equivalent-in-vc +// https://msdn.microsoft.com/en-us/library/bb918180.aspx #pragma section(".CRT$XCT", read) #define INITIALIZER(name) \ static void __cdecl name(void); \ diff --git a/src/crypto/slow-hash.c b/src/crypto/slow-hash.c index 35e98f2f5..9d4fc0dfa 100644 --- a/src/crypto/slow-hash.c +++ b/src/crypto/slow-hash.c @@ -309,7 +309,7 @@ STATIC INLINE void aes_256_assist2(__m128i* t1, __m128i * t3) * CPU AES support. * For more information about these functions, see page 19 of Intel's AES instructions * white paper: - * http://www.intel.com/content/dam/www/public/us/en/documents/white-papers/aes-instructions-set-white-paper.pdf + * https://www.intel.com/content/dam/doc/white-paper/advanced-encryption-standard-new-instructions-set-paper.pdf * * @param key the input 128 bit key * @param expandedKey An output buffer to hold the generated key schedule @@ -558,7 +558,7 @@ void slow_hash_free_state(void) * AES support on x86 CPUs. * * A diagram of the inner loop of this function can be found at - * http://www.cs.cmu.edu/~dga/crypto/xmr/cryptonight.png + * https://www.cs.cmu.edu/~dga/crypto/xmr/cryptonight.png * * @param data the data to hash * @param length the length in bytes of the data diff --git a/src/crypto/tree-hash.c b/src/crypto/tree-hash.c index e6d6a267c..57c38b86b 100644 --- a/src/crypto/tree-hash.c +++ b/src/crypto/tree-hash.c @@ -67,7 +67,7 @@ size_t tree_hash_cnt(size_t count) { } void tree_hash(const char (*hashes)[HASH_SIZE], size_t count, char *root_hash) { -// The blockchain block at height 202612 http://monerochain.info/block/bbd604d2ba11ba27935e006ed39c9bfdd99b76bf4a50654bc1e1e61217962698 +// The blockchain block at height 202612 https://moneroblocks.info/block/202612 // contained 514 transactions, that triggered bad calculation of variable "cnt" in the original version of this function // as from CryptoNote code. // diff --git a/src/cryptonote_basic/account.cpp b/src/cryptonote_basic/account.cpp index bab991d19..aac6ec22b 100644 --- a/src/cryptonote_basic/account.cpp +++ b/src/cryptonote_basic/account.cpp @@ -157,7 +157,7 @@ DISABLE_VS_WARNINGS(4244 4345) void account_base::create_from_viewkey(const cryptonote::account_public_address& address, const crypto::secret_key& viewkey) { crypto::secret_key fake; - memset(&fake, 0, sizeof(fake)); + memset(&unwrap(fake), 0, sizeof(fake)); create_from_keys(address, fake, viewkey); } //----------------------------------------------------------------- diff --git a/src/cryptonote_basic/connection_context.h b/src/cryptonote_basic/connection_context.h index 3f4651565..eb73ab0ea 100644 --- a/src/cryptonote_basic/connection_context.h +++ b/src/cryptonote_basic/connection_context.h @@ -52,7 +52,7 @@ namespace cryptonote }; state m_state; - std::list<crypto::hash> m_needed_objects; + std::vector<crypto::hash> m_needed_objects; std::unordered_set<crypto::hash> m_requested_objects; uint64_t m_remote_blockchain_height; uint64_t m_last_response_height; diff --git a/src/cryptonote_basic/cryptonote_basic_impl.cpp b/src/cryptonote_basic/cryptonote_basic_impl.cpp index 08a95d2e9..cff23695f 100644 --- a/src/cryptonote_basic/cryptonote_basic_impl.cpp +++ b/src/cryptonote_basic/cryptonote_basic_impl.cpp @@ -162,10 +162,7 @@ namespace cryptonote { , account_public_address const & adr ) { - uint64_t address_prefix = nettype == TESTNET ? - (subaddress ? config::testnet::CRYPTONOTE_PUBLIC_SUBADDRESS_BASE58_PREFIX : config::testnet::CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX) : nettype == STAGENET ? - (subaddress ? config::stagenet::CRYPTONOTE_PUBLIC_SUBADDRESS_BASE58_PREFIX : config::stagenet::CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX) : - (subaddress ? config::CRYPTONOTE_PUBLIC_SUBADDRESS_BASE58_PREFIX : config::CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX); + uint64_t address_prefix = subaddress ? get_config(nettype).CRYPTONOTE_PUBLIC_SUBADDRESS_BASE58_PREFIX : get_config(nettype).CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX; return tools::base58::encode_addr(address_prefix, t_serializable_object_to_blob(adr)); } @@ -176,7 +173,7 @@ namespace cryptonote { , crypto::hash8 const & payment_id ) { - uint64_t integrated_address_prefix = nettype == TESTNET ? config::testnet::CRYPTONOTE_PUBLIC_INTEGRATED_ADDRESS_BASE58_PREFIX : nettype == STAGENET ? config::stagenet::CRYPTONOTE_PUBLIC_INTEGRATED_ADDRESS_BASE58_PREFIX : config::CRYPTONOTE_PUBLIC_INTEGRATED_ADDRESS_BASE58_PREFIX; + uint64_t integrated_address_prefix = get_config(nettype).CRYPTONOTE_PUBLIC_INTEGRATED_ADDRESS_BASE58_PREFIX; integrated_address iadr = { adr, payment_id @@ -201,15 +198,9 @@ namespace cryptonote { , std::string const & str ) { - uint64_t address_prefix = nettype == TESTNET ? - config::testnet::CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX : nettype == STAGENET ? - config::stagenet::CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX : config::CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX; - uint64_t integrated_address_prefix = nettype == TESTNET ? - config::testnet::CRYPTONOTE_PUBLIC_INTEGRATED_ADDRESS_BASE58_PREFIX : nettype == STAGENET ? - config::stagenet::CRYPTONOTE_PUBLIC_INTEGRATED_ADDRESS_BASE58_PREFIX : config::CRYPTONOTE_PUBLIC_INTEGRATED_ADDRESS_BASE58_PREFIX; - uint64_t subaddress_prefix = nettype == TESTNET ? - config::testnet::CRYPTONOTE_PUBLIC_SUBADDRESS_BASE58_PREFIX : nettype == STAGENET ? - config::stagenet::CRYPTONOTE_PUBLIC_SUBADDRESS_BASE58_PREFIX : config::CRYPTONOTE_PUBLIC_SUBADDRESS_BASE58_PREFIX; + uint64_t address_prefix = get_config(nettype).CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX; + uint64_t integrated_address_prefix = get_config(nettype).CRYPTONOTE_PUBLIC_INTEGRATED_ADDRESS_BASE58_PREFIX; + uint64_t subaddress_prefix = get_config(nettype).CRYPTONOTE_PUBLIC_SUBADDRESS_BASE58_PREFIX; if (2 * sizeof(public_address_outer_blob) != str.size()) { diff --git a/src/cryptonote_basic/hardfork.h b/src/cryptonote_basic/hardfork.h index ee5ec0596..a63a66976 100644 --- a/src/cryptonote_basic/hardfork.h +++ b/src/cryptonote_basic/hardfork.h @@ -220,6 +220,14 @@ namespace cryptonote */ uint64_t get_window_size() const { return window_size; } + struct Params { + uint8_t version; + uint8_t threshold; + uint64_t height; + time_t time; + Params(uint8_t version, uint64_t height, uint8_t threshold, time_t time): version(version), threshold(threshold), height(height), time(time) {} + }; + private: uint8_t get_block_version(uint64_t height) const; @@ -244,13 +252,6 @@ namespace cryptonote uint8_t original_version; uint64_t original_version_till_height; - struct Params { - uint8_t version; - uint8_t threshold; - uint64_t height; - time_t time; - Params(uint8_t version, uint64_t height, uint8_t threshold, time_t time): version(version), threshold(threshold), height(height), time(time) {} - }; std::vector<Params> heights; std::deque<uint8_t> versions; /* rolling window of the last N blocks' versions */ diff --git a/src/cryptonote_basic/miner.cpp b/src/cryptonote_basic/miner.cpp index 2c777f5a2..dfe456ef4 100644 --- a/src/cryptonote_basic/miner.cpp +++ b/src/cryptonote_basic/miner.cpp @@ -328,6 +328,11 @@ namespace cryptonote LOG_PRINT_L0("Background mining controller thread started" ); } + if(get_ignore_battery()) + { + MINFO("Ignoring battery"); + } + return true; } //----------------------------------------------------------------------------------------------------- @@ -485,7 +490,7 @@ namespace cryptonote { //we lucky! ++m_config.current_extra_message_index; - MGINFO_GREEN("Found block for difficulty: " << local_diff); + MGINFO_GREEN("Found block " << get_block_hash(b) << " at height " << height << " for difficulty: " << local_diff); if(!m_phandler->handle_block_found(b)) { --m_config.current_extra_message_index; diff --git a/src/cryptonote_config.h b/src/cryptonote_config.h index 3e64073dd..a0dcf2df1 100644 --- a/src/cryptonote_config.h +++ b/src/cryptonote_config.h @@ -203,4 +203,60 @@ namespace cryptonote FAKECHAIN, UNDEFINED = 255 }; + struct config_t + { + uint64_t const CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX; + uint64_t const CRYPTONOTE_PUBLIC_INTEGRATED_ADDRESS_BASE58_PREFIX; + uint64_t const CRYPTONOTE_PUBLIC_SUBADDRESS_BASE58_PREFIX; + uint16_t const P2P_DEFAULT_PORT; + uint16_t const RPC_DEFAULT_PORT; + uint16_t const ZMQ_RPC_DEFAULT_PORT; + boost::uuids::uuid const NETWORK_ID; + std::string const GENESIS_TX; + uint32_t const GENESIS_NONCE; + }; + inline const config_t& get_config(network_type nettype) + { + static const config_t mainnet = { + ::config::CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX, + ::config::CRYPTONOTE_PUBLIC_INTEGRATED_ADDRESS_BASE58_PREFIX, + ::config::CRYPTONOTE_PUBLIC_SUBADDRESS_BASE58_PREFIX, + ::config::P2P_DEFAULT_PORT, + ::config::RPC_DEFAULT_PORT, + ::config::ZMQ_RPC_DEFAULT_PORT, + ::config::NETWORK_ID, + ::config::GENESIS_TX, + ::config::GENESIS_NONCE + }; + static const config_t testnet = { + ::config::testnet::CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX, + ::config::testnet::CRYPTONOTE_PUBLIC_INTEGRATED_ADDRESS_BASE58_PREFIX, + ::config::testnet::CRYPTONOTE_PUBLIC_SUBADDRESS_BASE58_PREFIX, + ::config::testnet::P2P_DEFAULT_PORT, + ::config::testnet::RPC_DEFAULT_PORT, + ::config::testnet::ZMQ_RPC_DEFAULT_PORT, + ::config::testnet::NETWORK_ID, + ::config::testnet::GENESIS_TX, + ::config::testnet::GENESIS_NONCE + }; + static const config_t stagenet = { + ::config::stagenet::CRYPTONOTE_PUBLIC_ADDRESS_BASE58_PREFIX, + ::config::stagenet::CRYPTONOTE_PUBLIC_INTEGRATED_ADDRESS_BASE58_PREFIX, + ::config::stagenet::CRYPTONOTE_PUBLIC_SUBADDRESS_BASE58_PREFIX, + ::config::stagenet::P2P_DEFAULT_PORT, + ::config::stagenet::RPC_DEFAULT_PORT, + ::config::stagenet::ZMQ_RPC_DEFAULT_PORT, + ::config::stagenet::NETWORK_ID, + ::config::stagenet::GENESIS_TX, + ::config::stagenet::GENESIS_NONCE + }; + switch (nettype) + { + case MAINNET: return mainnet; + case TESTNET: return testnet; + case STAGENET: return stagenet; + case FAKECHAIN: return mainnet; + default: throw std::runtime_error("Invalid network type"); + } + }; } diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 38f7f94e1..dad30906e 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -242,6 +242,7 @@ bool Blockchain::scan_outputkeys_for_indexes(size_t tx_version, const txin_to_ke MDEBUG("Additional outputs needed: " << absolute_offsets.size() - outputs.size()); std::vector < uint64_t > add_offsets; std::vector<output_data_t> add_outputs; + add_outputs.reserve(absolute_offsets.size() - outputs.size()); for (size_t i = outputs.size(); i < absolute_offsets.size(); i++) add_offsets.push_back(absolute_offsets[i]); try @@ -329,7 +330,7 @@ uint64_t Blockchain::get_current_blockchain_height() const //------------------------------------------------------------------ //FIXME: possibly move this into the constructor, to avoid accidentally // dereferencing a null BlockchainDB pointer -bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline, const cryptonote::test_options *test_options) +bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline, const cryptonote::test_options *test_options, difficulty_type fixed_difficulty) { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_tx_pool); @@ -351,6 +352,7 @@ bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline m_nettype = test_options != NULL ? FAKECHAIN : nettype; m_offline = offline; + m_fixed_difficulty = fixed_difficulty; if (m_hardfork == nullptr) { if (m_nettype == FAKECHAIN || m_nettype == STAGENET) @@ -393,18 +395,7 @@ bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline MINFO("Blockchain not loaded, generating genesis block."); block bl = boost::value_initialized<block>(); block_verification_context bvc = boost::value_initialized<block_verification_context>(); - if (m_nettype == TESTNET) - { - generate_genesis_block(bl, config::testnet::GENESIS_TX, config::testnet::GENESIS_NONCE); - } - else if (m_nettype == STAGENET) - { - generate_genesis_block(bl, config::stagenet::GENESIS_TX, config::stagenet::GENESIS_NONCE); - } - else - { - generate_genesis_block(bl, config::GENESIS_TX, config::GENESIS_NONCE); - } + generate_genesis_block(bl, get_config(m_nettype).GENESIS_TX, get_config(m_nettype).GENESIS_NONCE); add_new_block(bl, bvc); CHECK_AND_ASSERT_MES(!bvc.m_verifivation_failed, false, "Failed to add genesis block to blockchain"); } @@ -444,7 +435,7 @@ bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline m_db->block_txn_stop(); uint64_t num_popped_blocks = 0; - while (true) + while (!m_db->is_read_only()) { const uint64_t top_height = m_db->height() - 1; const crypto::hash top_id = m_db->top_block_hash(); @@ -805,6 +796,11 @@ bool Blockchain::get_block_by_hash(const crypto::hash &h, block &blk, bool *orph // less blocks than desired if there aren't enough. difficulty_type Blockchain::get_difficulty_for_next_block() { + if (m_fixed_difficulty) + { + return m_db->height() ? m_fixed_difficulty : 1; + } + LOG_PRINT_L3("Blockchain::" << __func__); crypto::hash top_hash = get_tail_id(); @@ -850,6 +846,11 @@ difficulty_type Blockchain::get_difficulty_for_next_block() timestamps.clear(); difficulties.clear(); + if (height > offset) + { + timestamps.reserve(height - offset); + difficulties.reserve(height - offset); + } for (; offset < height; offset++) { timestamps.push_back(m_db->get_block_timestamp(offset)); @@ -1011,6 +1012,11 @@ bool Blockchain::switch_to_alternative_blockchain(std::list<blocks_ext_by_hash:: // an alternate chain. difficulty_type Blockchain::get_next_difficulty_for_alternative_chain(const std::list<blocks_ext_by_hash::iterator>& alt_chain, block_extended_info& bei) const { + if (m_fixed_difficulty) + { + return m_db->height() ? m_fixed_difficulty : 1; + } + LOG_PRINT_L3("Blockchain::" << __func__); std::vector<uint64_t> timestamps; std::vector<difficulty_type> cumulative_difficulties; @@ -1170,6 +1176,7 @@ void Blockchain::get_last_n_blocks_sizes(std::vector<size_t>& sz, size_t count) m_db->block_txn_start(true); // add size of last <count> blocks to vector <sz> (or less, if blockchain size < count) size_t start_offset = h - std::min<size_t>(h, count); + sz.reserve(sz.size() + h - start_offset); for(size_t i = start_offset; i < h; i++) { sz.push_back(m_db->get_block_size(i)); @@ -1367,6 +1374,7 @@ bool Blockchain::complete_timestamps_vector(uint64_t start_top_height, std::vect size_t need_elements = BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW - timestamps.size(); CHECK_AND_ASSERT_MES(start_top_height < m_db->height(), false, "internal error: passed start_height not < " << " m_db->height() -- " << start_top_height << " >= " << m_db->height()); size_t stop_offset = start_top_height > need_elements ? start_top_height - need_elements : 0; + timestamps.reserve(timestamps.size() + start_top_height - stop_offset); while (start_top_height != stop_offset) { timestamps.push_back(m_db->get_block_timestamp(start_top_height)); @@ -1566,7 +1574,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id return true; } //------------------------------------------------------------------ -bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::list<std::pair<cryptonote::blobdata,block>>& blocks, std::list<cryptonote::blobdata>& txs) const +bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::vector<std::pair<cryptonote::blobdata,block>>& blocks, std::vector<cryptonote::blobdata>& txs) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -1580,7 +1588,7 @@ bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::list<std:: for(const auto& blk : blocks) { - std::list<crypto::hash> missed_ids; + std::vector<crypto::hash> missed_ids; get_transactions_blobs(blk.second.tx_hashes, txs, missed_ids); CHECK_AND_ASSERT_MES(!missed_ids.size(), false, "has missed transactions in own block in main blockchain"); } @@ -1588,14 +1596,16 @@ bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::list<std:: return true; } //------------------------------------------------------------------ -bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::list<std::pair<cryptonote::blobdata,block>>& blocks) const +bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::vector<std::pair<cryptonote::blobdata,block>>& blocks) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); - if(start_offset >= m_db->height()) + const uint64_t height = m_db->height(); + if(start_offset >= height) return false; - for(size_t i = start_offset; i < start_offset + count && i < m_db->height();i++) + blocks.reserve(blocks.size() + height - start_offset); + for(size_t i = start_offset; i < start_offset + count && i < height;i++) { blocks.push_back(std::make_pair(m_db->get_block_blob_from_height(i), block())); if (!parse_and_validate_block_from_blob(blocks.back().first, blocks.back().second)) @@ -1620,17 +1630,20 @@ bool Blockchain::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NO CRITICAL_REGION_LOCAL(m_blockchain_lock); m_db->block_txn_start(true); rsp.current_blockchain_height = get_current_blockchain_height(); - std::list<std::pair<cryptonote::blobdata,block>> blocks; + std::vector<std::pair<cryptonote::blobdata,block>> blocks; get_blocks(arg.blocks, blocks, rsp.missed_ids); - for (const auto& bl: blocks) + for (auto& bl: blocks) { - std::list<crypto::hash> missed_tx_ids; - std::list<cryptonote::blobdata> txs; + std::vector<crypto::hash> missed_tx_ids; + std::vector<cryptonote::blobdata> txs; + + rsp.blocks.push_back(block_complete_entry()); + block_complete_entry& e = rsp.blocks.back(); // FIXME: s/rsp.missed_ids/missed_tx_id/ ? Seems like rsp.missed_ids // is for missed blocks, not missed transactions as well. - get_transactions_blobs(bl.second.tx_hashes, txs, missed_tx_ids); + get_transactions_blobs(bl.second.tx_hashes, e.txs, missed_tx_ids); if (missed_tx_ids.size() != 0) { @@ -1642,35 +1655,28 @@ bool Blockchain::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NO // append missed transaction hashes to response missed_ids field, // as done below if any standalone transactions were requested // and missed. - rsp.missed_ids.splice(rsp.missed_ids.end(), missed_tx_ids); - m_db->block_txn_stop(); + rsp.missed_ids.insert(rsp.missed_ids.end(), missed_tx_ids.begin(), missed_tx_ids.end()); + m_db->block_txn_stop(); return false; } - rsp.blocks.push_back(block_complete_entry()); - block_complete_entry& e = rsp.blocks.back(); //pack block - e.block = bl.first; - //pack transactions - for (const cryptonote::blobdata& tx: txs) - e.txs.push_back(tx); - } - //get another transactions, if need - std::list<cryptonote::blobdata> txs; - get_transactions_blobs(arg.txs, txs, rsp.missed_ids); - //pack aside transactions - for (const auto& tx: txs) - rsp.txs.push_back(tx); + e.block = std::move(bl.first); + } + //get and pack other transactions, if needed + std::vector<cryptonote::blobdata> txs; + get_transactions_blobs(arg.txs, rsp.txs, rsp.missed_ids); m_db->block_txn_stop(); return true; } //------------------------------------------------------------------ -bool Blockchain::get_alternative_blocks(std::list<block>& blocks) const +bool Blockchain::get_alternative_blocks(std::vector<block>& blocks) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); + blocks.reserve(m_alternative_chains.size()); for (const auto& alt_bl: m_alternative_chains) { blocks.push_back(alt_bl.second.bl); @@ -1960,14 +1966,21 @@ bool Blockchain::get_outs(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMA res.outs.clear(); res.outs.reserve(req.outputs.size()); - for (const auto &i: req.outputs) + try { - // get tx_hash, tx_out_index from DB - const output_data_t od = m_db->get_output_key(i.amount, i.index); - tx_out_index toi = m_db->get_output_tx_and_index(i.amount, i.index); - bool unlocked = is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first)); + for (const auto &i: req.outputs) + { + // get tx_hash, tx_out_index from DB + const output_data_t od = m_db->get_output_key(i.amount, i.index); + tx_out_index toi = m_db->get_output_tx_and_index(i.amount, i.index); + bool unlocked = is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first)); - res.outs.push_back({od.pubkey, od.commitment, unlocked, od.height, toi.first}); + res.outs.push_back({od.pubkey, od.commitment, unlocked, od.height, toi.first}); + } + } + catch (const std::exception &e) + { + return false; } return true; } @@ -1983,14 +1996,14 @@ void Blockchain::get_output_key_mask_unlocked(const uint64_t& amount, const uint //------------------------------------------------------------------ bool Blockchain::get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t to_height, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) const { - // rct outputs don't exist before v3 + // rct outputs don't exist before v4 if (amount == 0) { switch (m_nettype) { - case STAGENET: start_height = stagenet_hard_forks[2].height; break; - case TESTNET: start_height = testnet_hard_forks[2].height; break; - case MAINNET: start_height = mainnet_hard_forks[2].height; break; + case STAGENET: start_height = stagenet_hard_forks[3].height; break; + case TESTNET: start_height = testnet_hard_forks[3].height; break; + case MAINNET: start_height = mainnet_hard_forks[3].height; break; default: return false; } } @@ -1998,11 +2011,40 @@ bool Blockchain::get_output_distribution(uint64_t amount, uint64_t from_height, start_height = 0; base = 0; + if (to_height > 0 && to_height < from_height) + return false; + const uint64_t real_start_height = start_height; if (from_height > start_height) start_height = from_height; - return m_db->get_output_distribution(amount, start_height, to_height, distribution, base); + distribution.clear(); + uint64_t db_height = m_db->height(); + if (db_height == 0) + return false; + if (to_height == 0) + to_height = db_height - 1; + if (start_height >= db_height || to_height >= db_height) + return false; + if (amount == 0) + { + std::vector<uint64_t> heights; + heights.reserve(to_height + 1 - start_height); + uint64_t real_start_height = start_height > 0 ? start_height-1 : start_height; + for (uint64_t h = real_start_height; h <= to_height; ++h) + heights.push_back(h); + distribution = m_db->get_block_cumulative_rct_outputs(heights); + if (start_height > 0) + { + base = distribution[0]; + distribution.erase(distribution.begin()); + } + return true; + } + else + { + return m_db->get_output_distribution(amount, start_height, to_height, distribution, base); + } } //------------------------------------------------------------------ // This function takes a list of block hashes from another node @@ -2015,9 +2057,9 @@ bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qbloc // make sure the request includes at least the genesis block, otherwise // how can we expect to sync from the client that the block list came from? - if(!qblock_ids.size() /*|| !req.m_total_height*/) + if(!qblock_ids.size()) { - MCERROR("net.p2p", "Client sent wrong NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << qblock_ids.size() << /*", m_height=" << req.m_total_height <<*/ ", dropping connection"); + MCERROR("net.p2p", "Client sent wrong NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << qblock_ids.size() << ", dropping connection"); return false; } @@ -2083,6 +2125,9 @@ uint64_t Blockchain::block_difficulty(uint64_t i) const return 0; } //------------------------------------------------------------------ +template<typename T> void reserve_container(std::vector<T> &v, size_t N) { v.reserve(N); } +template<typename T> void reserve_container(std::list<T> &v, size_t N) { } +//------------------------------------------------------------------ //TODO: return type should be void, throw on exception // alternatively, return true only if no blocks missed template<class t_ids_container, class t_blocks_container, class t_missed_container> @@ -2091,6 +2136,7 @@ bool Blockchain::get_blocks(const t_ids_container& block_ids, t_blocks_container LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); + reserve_container(blocks, block_ids.size()); for (const auto& block_hash : block_ids) { try @@ -2125,6 +2171,7 @@ bool Blockchain::get_transactions_blobs(const t_ids_container& txs_ids, t_tx_con LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); + reserve_container(txs, txs_ids.size()); for (const auto& tx_hash : txs_ids) { try @@ -2151,6 +2198,7 @@ bool Blockchain::get_transactions(const t_ids_container& txs_ids, t_tx_container LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); + reserve_container(txs, txs_ids.size()); for (const auto& tx_hash : txs_ids) { try @@ -2179,7 +2227,7 @@ bool Blockchain::get_transactions(const t_ids_container& txs_ids, t_tx_container // Find the split point between us and foreign blockchain and return // (by reference) the most recent common block hash along with up to // BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT additional (more recent) hashes. -bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, std::list<crypto::hash>& hashes, uint64_t& start_height, uint64_t& current_height) const +bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, std::vector<crypto::hash>& hashes, uint64_t& start_height, uint64_t& current_height) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -2193,6 +2241,7 @@ bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qbloc m_db->block_txn_start(true); current_height = get_current_blockchain_height(); size_t count = 0; + hashes.reserve(std::max((size_t)(current_height - start_height), (size_t)BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT)); for(size_t i = start_height; i < current_height && count < BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT; i++, count++) { hashes.push_back(m_db->get_block_hash_from_height(i)); @@ -2208,7 +2257,8 @@ bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qbloc CRITICAL_REGION_LOCAL(m_blockchain_lock); bool result = find_blockchain_supplement(qblock_ids, resp.m_block_ids, resp.start_height, resp.total_height); - resp.cumulative_difficulty = m_db->get_block_cumulative_difficulty(m_db->height() - 1); + if (result) + resp.cumulative_difficulty = m_db->get_block_cumulative_difficulty(resp.total_height - 1); return result; } @@ -2217,7 +2267,7 @@ bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qbloc // find split point between ours and foreign blockchain (or start at // blockchain height <req_start_block>), and return up to max_count FULL // blocks by reference. -bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::list<std::pair<cryptonote::blobdata, std::list<cryptonote::blobdata> > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, size_t max_count) const +bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_count) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -2243,18 +2293,28 @@ bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, cons m_db->block_txn_start(true); total_height = get_current_blockchain_height(); size_t count = 0, size = 0; + blocks.reserve(std::min(std::min(max_count, (size_t)10000), (size_t)(total_height - start_height))); for(size_t i = start_height; i < total_height && count < max_count && (size < FIND_BLOCKCHAIN_SUPPLEMENT_MAX_SIZE || count < 3); i++, count++) { blocks.resize(blocks.size()+1); - blocks.back().first = m_db->get_block_blob_from_height(i); + blocks.back().first.first = m_db->get_block_blob_from_height(i); block b; - CHECK_AND_ASSERT_MES(parse_and_validate_block_from_blob(blocks.back().first, b), false, "internal error, invalid block"); - std::list<crypto::hash> mis; - get_transactions_blobs(b.tx_hashes, blocks.back().second, mis, pruned); + CHECK_AND_ASSERT_MES(parse_and_validate_block_from_blob(blocks.back().first.first, b), false, "internal error, invalid block"); + blocks.back().first.second = get_miner_tx_hash ? cryptonote::get_transaction_hash(b.miner_tx) : crypto::null_hash; + std::vector<crypto::hash> mis; + std::vector<cryptonote::blobdata> txs; + get_transactions_blobs(b.tx_hashes, txs, mis, pruned); CHECK_AND_ASSERT_MES(!mis.size(), false, "internal error, transaction from block not found"); - size += blocks.back().first.size(); - for (const auto &t: blocks.back().second) + size += blocks.back().first.first.size(); + for (const auto &t: txs) size += t.size(); + + CHECK_AND_ASSERT_MES(txs.size() == b.tx_hashes.size(), false, "mismatched sizes of b.tx_hashes and txs"); + blocks.back().second.reserve(txs.size()); + for (size_t i = 0; i < txs.size(); ++i) + { + blocks.back().second.push_back(std::make_pair(b.tx_hashes[i], std::move(txs[i]))); + } } m_db->block_txn_stop(); return true; @@ -2726,7 +2786,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, tools::threadpool& tpool = tools::threadpool::getInstance(); tools::threadpool::waiter waiter; - const auto waiter_guard = epee::misc_utils::create_scope_leave_handler([&]() { waiter.wait(); }); + const auto waiter_guard = epee::misc_utils::create_scope_leave_handler([&]() { waiter.wait(&tpool); }); int threads = tpool.get_max_concurrency(); for (const auto& txin : tx.vin) @@ -2788,7 +2848,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, { // ND: Speedup // 1. Thread ring signature verification if possible. - tpool.submit(&waiter, boost::bind(&Blockchain::check_ring_signature, this, std::cref(tx_prefix_hash), std::cref(in_to_key.k_image), std::cref(pubkeys[sig_index]), std::cref(tx.signatures[sig_index]), std::ref(results[sig_index]))); + tpool.submit(&waiter, boost::bind(&Blockchain::check_ring_signature, this, std::cref(tx_prefix_hash), std::cref(in_to_key.k_image), std::cref(pubkeys[sig_index]), std::cref(tx.signatures[sig_index]), std::ref(results[sig_index])), true); } else { @@ -2812,7 +2872,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, sig_index++; } if (tx.version == 1 && threads > 1) - waiter.wait(); + waiter.wait(&tpool); if (tx.version == 1) { @@ -2984,6 +3044,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, void Blockchain::check_ring_signature(const crypto::hash &tx_prefix_hash, const crypto::key_image &key_image, const std::vector<rct::ctkey> &pubkeys, const std::vector<crypto::signature>& sig, uint64_t &result) { std::vector<const crypto::public_key *> p_output_keys; + p_output_keys.reserve(pubkeys.size()); for (auto &key : pubkeys) { // rct::key and crypto::public_key have the same structure, avoid object ctor/memcpy @@ -3080,6 +3141,7 @@ uint64_t Blockchain::get_dynamic_per_kb_fee_estimate(uint64_t grace_blocks) cons const uint64_t min_block_size = get_min_block_size(version); std::vector<size_t> sz; get_last_n_blocks_sizes(sz, CRYPTONOTE_REWARD_BLOCKS_WINDOW - grace_blocks); + sz.reserve(grace_blocks); for (size_t i = 0; i < grace_blocks; ++i) sz.push_back(min_block_size); @@ -3237,6 +3299,7 @@ bool Blockchain::check_block_timestamp(const block& b, uint64_t& median_ts) cons // need most recent 60 blocks, get index of first of those size_t offset = h - BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW; + timestamps.reserve(h - offset); for(;offset < h; ++offset) { timestamps.push_back(m_db->get_block_timestamp(offset)); @@ -3263,7 +3326,7 @@ void Blockchain::return_tx_to_pool(std::vector<transaction> &txs) } } //------------------------------------------------------------------ -bool Blockchain::flush_txes_from_pool(const std::list<crypto::hash> &txids) +bool Blockchain::flush_txes_from_pool(const std::vector<crypto::hash> &txids) { CRITICAL_REGION_LOCAL(m_tx_pool); @@ -3301,6 +3364,7 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& if(bl.prev_id != get_tail_id()) { MERROR_VER("Block with id: " << id << std::endl << "has wrong prev_id: " << bl.prev_id << std::endl << "expected: " << get_tail_id()); + bvc.m_verifivation_failed = true; leave: m_db->block_txn_stop(); return false; @@ -3453,6 +3517,7 @@ leave: // Iterate over the block's transaction hashes, grabbing each // from the tx_pool and validating them. Each is then added // to txs. Keys spent in each are added to <keys> by the double spend check. + txs.reserve(bl.tx_hashes.size()); for (const crypto::hash& tx_id : bl.tx_hashes) { transaction tx; @@ -3601,6 +3666,7 @@ leave: { //TODO: figure out the best way to deal with this failure LOG_ERROR("Error adding block with hash: " << id << " to blockchain, what = " << e.what()); + bvc.m_verifivation_failed = true; return_tx_to_pool(txs); return false; } @@ -3630,6 +3696,7 @@ leave: // appears to be a NOP *and* is called elsewhere. wat? m_tx_pool.on_blockchain_inc(new_height, id); + get_difficulty_for_next_block(); // just to cache it return true; } @@ -3866,7 +3933,7 @@ void Blockchain::output_scan_worker(const uint64_t amount, const std::vector<uin } } -uint64_t Blockchain::prevalidate_block_hashes(uint64_t height, const std::list<crypto::hash> &hashes) +uint64_t Blockchain::prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes) { // new: . . . . . X X X X X . . . . . . // pre: A A A A B B B B C C C C D D D D @@ -3969,7 +4036,7 @@ uint64_t Blockchain::prevalidate_block_hashes(uint64_t height, const std::list<c // vs [k_image, output_keys] (m_scan_table). This is faster because it takes advantage of bulk queries // and is threaded if possible. The table (m_scan_table) will be used later when querying output // keys. -bool Blockchain::prepare_handle_incoming_blocks(const std::list<block_complete_entry> &blocks_entry) +bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete_entry> &blocks_entry) { MTRACE("Blockchain::" << __func__); TIME_MEASURE_START(prepare); @@ -4035,6 +4102,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::list<block_complete_e for (uint64_t i = 0; i < threads; i++) { + blocks[i].reserve(batches + 1); for (int j = 0; j < batches; j++) { block block; @@ -4093,11 +4161,11 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::list<block_complete_e tools::threadpool::waiter waiter; for (uint64_t i = 0; i < threads; i++) { - tpool.submit(&waiter, boost::bind(&Blockchain::block_longhash_worker, this, thread_height, std::cref(blocks[i]), std::ref(maps[i]))); + tpool.submit(&waiter, boost::bind(&Blockchain::block_longhash_worker, this, thread_height, std::cref(blocks[i]), std::ref(maps[i])), true); thread_height += blocks[i].size(); } - waiter.wait(); + waiter.wait(&tpool); if (m_cancel) return false; @@ -4232,9 +4300,9 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::list<block_complete_e for (size_t i = 0; i < amounts.size(); i++) { uint64_t amount = amounts[i]; - tpool.submit(&waiter, boost::bind(&Blockchain::output_scan_worker, this, amount, std::cref(offset_map[amount]), std::ref(tx_map[amount]), std::ref(transactions[i]))); + tpool.submit(&waiter, boost::bind(&Blockchain::output_scan_worker, this, amount, std::cref(offset_map[amount]), std::ref(tx_map[amount]), std::ref(transactions[i])), true); } - waiter.wait(); + waiter.wait(&tpool); } else { @@ -4381,6 +4449,39 @@ HardFork::State Blockchain::get_hard_fork_state() const return m_hardfork->get_state(); } +const std::vector<HardFork::Params>& Blockchain::get_hard_fork_heights(network_type nettype) +{ + static const std::vector<HardFork::Params> mainnet_heights = []() + { + std::vector<HardFork::Params> heights; + for (const auto& i : mainnet_hard_forks) + heights.emplace_back(i.version, i.height, i.threshold, i.time); + return heights; + }(); + static const std::vector<HardFork::Params> testnet_heights = []() + { + std::vector<HardFork::Params> heights; + for (const auto& i : testnet_hard_forks) + heights.emplace_back(i.version, i.height, i.threshold, i.time); + return heights; + }(); + static const std::vector<HardFork::Params> stagenet_heights = []() + { + std::vector<HardFork::Params> heights; + for (const auto& i : stagenet_hard_forks) + heights.emplace_back(i.version, i.height, i.threshold, i.time); + return heights; + }(); + static const std::vector<HardFork::Params> dummy; + switch (nettype) + { + case MAINNET: return mainnet_heights; + case TESTNET: return testnet_heights; + case STAGENET: return stagenet_heights; + default: return dummy; + } +} + bool Blockchain::get_hard_fork_voting_info(uint8_t version, uint32_t &window, uint32_t &votes, uint32_t &threshold, uint64_t &earliest_height, uint8_t &voting) const { return m_hardfork->get_voting_info(version, window, votes, threshold, earliest_height, voting); @@ -4396,9 +4497,9 @@ std::map<uint64_t, std::tuple<uint64_t, uint64_t, uint64_t>> Blockchain:: get_ou return m_db->get_output_histogram(amounts, unlocked, recent_cutoff, min_count); } -std::list<std::pair<Blockchain::block_extended_info,uint64_t>> Blockchain::get_alternative_chains() const +std::list<std::pair<Blockchain::block_extended_info,std::vector<crypto::hash>>> Blockchain::get_alternative_chains() const { - std::list<std::pair<Blockchain::block_extended_info,uint64_t>> chains; + std::list<std::pair<Blockchain::block_extended_info,std::vector<crypto::hash>>> chains; for (const auto &i: m_alternative_chains) { @@ -4414,15 +4515,16 @@ std::list<std::pair<Blockchain::block_extended_info,uint64_t>> Blockchain::get_a } if (!found) { - uint64_t length = 1; + std::vector<crypto::hash> chain; auto h = i.second.bl.prev_id; + chain.push_back(top); blocks_ext_by_hash::const_iterator prev; while ((prev = m_alternative_chains.find(h)) != m_alternative_chains.end()) { + chain.push_back(h); h = prev->second.bl.prev_id; - ++length; } - chains.push_back(std::make_pair(i.second, length)); + chains.push_back(std::make_pair(i.second, chain)); } } return chains; @@ -4498,7 +4600,7 @@ void Blockchain::load_compiled_in_block_hashes() // for tx hashes will fail in handle_block_to_main_chain(..) CRITICAL_REGION_LOCAL(m_tx_pool); - std::list<transaction> txs; + std::vector<transaction> txs; m_tx_pool.get_transactions(txs); size_t blob_size; @@ -4561,6 +4663,6 @@ bool Blockchain::for_all_outputs(uint64_t amount, std::function<bool(uint64_t he } namespace cryptonote { -template bool Blockchain::get_transactions(const std::vector<crypto::hash>&, std::list<transaction>&, std::list<crypto::hash>&) const; -template bool Blockchain::get_transactions_blobs(const std::vector<crypto::hash>&, std::list<cryptonote::blobdata>&, std::list<crypto::hash>&, bool) const; +template bool Blockchain::get_transactions(const std::vector<crypto::hash>&, std::vector<transaction>&, std::vector<crypto::hash>&) const; +template bool Blockchain::get_transactions_blobs(const std::vector<crypto::hash>&, std::vector<cryptonote::blobdata>&, std::vector<crypto::hash>&, bool) const; } diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index ef736d1e7..d95c8ed15 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -114,10 +114,11 @@ namespace cryptonote * @param nettype network type * @param offline true if running offline, else false * @param test_options test parameters + * @param fixed_difficulty fixed difficulty for testing purposes; 0 means disabled * * @return true on success, false if any initialization steps fail */ - bool init(BlockchainDB* db, const network_type nettype = MAINNET, bool offline = false, const cryptonote::test_options *test_options = NULL); + bool init(BlockchainDB* db, const network_type nettype = MAINNET, bool offline = false, const cryptonote::test_options *test_options = NULL, difficulty_type fixed_difficulty = 0); /** * @brief Initialize the Blockchain state @@ -157,7 +158,7 @@ namespace cryptonote * * @return false if start_offset > blockchain height, else true */ - bool get_blocks(uint64_t start_offset, size_t count, std::list<std::pair<cryptonote::blobdata,block>>& blocks, std::list<cryptonote::blobdata>& txs) const; + bool get_blocks(uint64_t start_offset, size_t count, std::vector<std::pair<cryptonote::blobdata,block>>& blocks, std::vector<cryptonote::blobdata>& txs) const; /** * @brief get blocks from blocks based on start height and count @@ -168,7 +169,7 @@ namespace cryptonote * * @return false if start_offset > blockchain height, else true */ - bool get_blocks(uint64_t start_offset, size_t count, std::list<std::pair<cryptonote::blobdata,block>>& blocks) const; + bool get_blocks(uint64_t start_offset, size_t count, std::vector<std::pair<cryptonote::blobdata,block>>& blocks) const; /** * @brief compiles a list of all blocks stored as alternative chains @@ -177,7 +178,7 @@ namespace cryptonote * * @return true */ - bool get_alternative_blocks(std::list<block>& blocks) const; + bool get_alternative_blocks(std::vector<block>& blocks) const; /** * @brief returns the number of alternative blocks stored @@ -213,7 +214,7 @@ namespace cryptonote * * @return false on erroneous blocks, else true */ - bool prepare_handle_incoming_blocks(const std::list<block_complete_entry> &blocks); + bool prepare_handle_incoming_blocks(const std::vector<block_complete_entry> &blocks); /** * @brief incoming blocks post-processing, cleanup, and disk sync @@ -373,7 +374,7 @@ namespace cryptonote * * @return true if a block found in common, else false */ - bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, std::list<crypto::hash>& hashes, uint64_t& start_height, uint64_t& current_height) const; + bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, std::vector<crypto::hash>& hashes, uint64_t& start_height, uint64_t& current_height) const; /** * @brief get recent block hashes for a foreign chain @@ -420,7 +421,7 @@ namespace cryptonote * * @return true if a block found in common or req_start_block specified, else false */ - bool find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::list<std::pair<cryptonote::blobdata, std::list<cryptonote::blobdata> > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, size_t max_count) const; + bool find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_count) const; /** * @brief retrieves a set of blocks and their transactions, and possibly other transactions @@ -755,6 +756,13 @@ namespace cryptonote HardFork::State get_hard_fork_state() const; /** + * @brief gets the hardfork heights of given network + * + * @return the HardFork object + */ + static const std::vector<HardFork::Params>& get_hard_fork_heights(network_type nettype); + + /** * @brief gets the current hardfork version in use/voted for * * @return the version @@ -829,7 +837,7 @@ namespace cryptonote * * @return false if any removals fail, otherwise true */ - bool flush_txes_from_pool(const std::list<crypto::hash> &txids); + bool flush_txes_from_pool(const std::vector<crypto::hash> &txids); /** * @brief return a histogram of outputs on the blockchain @@ -939,7 +947,7 @@ namespace cryptonote * * @return a list of chains */ - std::list<std::pair<block_extended_info,uint64_t>> get_alternative_chains() const; + std::list<std::pair<block_extended_info,std::vector<crypto::hash>>> get_alternative_chains() const; void add_txpool_tx(transaction &tx, const txpool_tx_meta_t &meta); void update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t &meta); @@ -952,7 +960,7 @@ namespace cryptonote bool is_within_compiled_block_hash_area(uint64_t height) const; bool is_within_compiled_block_hash_area() const { return is_within_compiled_block_hash_area(m_db->height()); } - uint64_t prevalidate_block_hashes(uint64_t height, const std::list<crypto::hash> &hashes); + uint64_t prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes); void lock(); void unlock(); @@ -1040,6 +1048,7 @@ namespace cryptonote network_type m_nettype; bool m_offline; + difficulty_type m_fixed_difficulty; std::atomic<bool> m_cancel; diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 7d34415c4..18490c65e 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -76,6 +76,16 @@ namespace cryptonote , "Run on stagenet. The wallet must be launched with --stagenet flag." , false }; + const command_line::arg_descriptor<bool> arg_regtest_on = { + "regtest" + , "Run in a regression testing mode." + , false + }; + const command_line::arg_descriptor<difficulty_type> arg_fixed_difficulty = { + "fixed-difficulty" + , "Fixed difficulty used for testing." + , 0 + }; const command_line::arg_descriptor<std::string, false, true, 2> arg_data_dir = { "data-dir" , "Specify data directory" @@ -170,7 +180,6 @@ namespace cryptonote m_last_dns_checkpoints_update(0), m_last_json_checkpoints_update(0), m_disable_dns_checkpoints(false), - m_threadpool(tools::threadpool::getInstance()), m_update_download(0), m_nettype(UNDEFINED) { @@ -252,6 +261,8 @@ namespace cryptonote command_line::add_arg(desc, arg_testnet_on); command_line::add_arg(desc, arg_stagenet_on); + command_line::add_arg(desc, arg_regtest_on); + command_line::add_arg(desc, arg_fixed_difficulty); command_line::add_arg(desc, arg_dns_checkpoints); command_line::add_arg(desc, arg_prep_blocks_threads); command_line::add_arg(desc, arg_fast_block_sync); @@ -324,19 +335,19 @@ namespace cryptonote top_id = m_blockchain_storage.get_tail_id(height); } //----------------------------------------------------------------------------------------------- - bool core::get_blocks(uint64_t start_offset, size_t count, std::list<std::pair<cryptonote::blobdata,block>>& blocks, std::list<cryptonote::blobdata>& txs) const + bool core::get_blocks(uint64_t start_offset, size_t count, std::vector<std::pair<cryptonote::blobdata,block>>& blocks, std::vector<cryptonote::blobdata>& txs) const { return m_blockchain_storage.get_blocks(start_offset, count, blocks, txs); } //----------------------------------------------------------------------------------------------- - bool core::get_blocks(uint64_t start_offset, size_t count, std::list<std::pair<cryptonote::blobdata,block>>& blocks) const + bool core::get_blocks(uint64_t start_offset, size_t count, std::vector<std::pair<cryptonote::blobdata,block>>& blocks) const { return m_blockchain_storage.get_blocks(start_offset, count, blocks); } //----------------------------------------------------------------------------------------------- - bool core::get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks) const + bool core::get_blocks(uint64_t start_offset, size_t count, std::vector<block>& blocks) const { - std::list<std::pair<cryptonote::blobdata, cryptonote::block>> bs; + std::vector<std::pair<cryptonote::blobdata, cryptonote::block>> bs; if (!m_blockchain_storage.get_blocks(start_offset, count, bs)) return false; for (const auto &b: bs) @@ -344,7 +355,7 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- - bool core::get_transactions(const std::vector<crypto::hash>& txs_ids, std::list<cryptonote::blobdata>& txs, std::list<crypto::hash>& missed_txs) const + bool core::get_transactions(const std::vector<crypto::hash>& txs_ids, std::vector<cryptonote::blobdata>& txs, std::vector<crypto::hash>& missed_txs) const { return m_blockchain_storage.get_transactions_blobs(txs_ids, txs, missed_txs); } @@ -355,12 +366,12 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- - bool core::get_transactions(const std::vector<crypto::hash>& txs_ids, std::list<transaction>& txs, std::list<crypto::hash>& missed_txs) const + bool core::get_transactions(const std::vector<crypto::hash>& txs_ids, std::vector<transaction>& txs, std::vector<crypto::hash>& missed_txs) const { return m_blockchain_storage.get_transactions(txs_ids, txs, missed_txs); } //----------------------------------------------------------------------------------------------- - bool core::get_alternative_blocks(std::list<block>& blocks) const + bool core::get_alternative_blocks(std::vector<block>& blocks) const { return m_blockchain_storage.get_alternative_blocks(blocks); } @@ -374,7 +385,8 @@ namespace cryptonote { start_time = std::time(nullptr); - if (test_options != NULL) + const bool regtest = command_line::get_arg(vm, arg_regtest_on); + if (test_options != NULL || regtest) { m_nettype = FAKECHAIN; } @@ -431,6 +443,16 @@ namespace cryptonote blockchain_db_sync_mode sync_mode = db_defaultsync; uint64_t blocks_per_sync = 1; + if (m_nettype == FAKECHAIN) + { + // reset the db by removing the database file before opening it + if (!db->remove_data_file(filename)) + { + MERROR("Failed to remove data file in " << filename); + return false; + } + } + try { uint64_t db_flags = 0; @@ -508,7 +530,12 @@ namespace cryptonote m_blockchain_storage.set_user_options(blocks_threads, blocks_per_sync, sync_mode, fast_sync); - r = m_blockchain_storage.init(db.release(), m_nettype, m_offline, test_options); + const std::pair<uint8_t, uint64_t> regtest_hard_forks[3] = {std::make_pair(1, 0), std::make_pair(Blockchain::get_hard_fork_heights(MAINNET).back().version, 1), std::make_pair(0, 0)}; + const cryptonote::test_options regtest_test_options = { + regtest_hard_forks + }; + const difficulty_type fixed_difficulty = command_line::get_arg(vm, arg_fixed_difficulty); + r = m_blockchain_storage.init(db.release(), m_nettype, m_offline, regtest ? ®test_test_options : test_options, fixed_difficulty); r = m_mempool.init(max_txpool_size); CHECK_AND_ASSERT_MES(r, false, "Failed to initialize memory pool"); @@ -673,18 +700,20 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- - bool core::handle_incoming_txs(const std::list<blobdata>& tx_blobs, std::vector<tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) + bool core::handle_incoming_txs(const std::vector<blobdata>& tx_blobs, std::vector<tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { TRY_ENTRY(); + CRITICAL_REGION_LOCAL(m_incoming_tx_lock); struct result { bool res; cryptonote::transaction tx; crypto::hash hash; crypto::hash prefix_hash; bool in_txpool; bool in_blockchain; }; std::vector<result> results(tx_blobs.size()); tvc.resize(tx_blobs.size()); + tools::threadpool& tpool = tools::threadpool::getInstance(); tools::threadpool::waiter waiter; - std::list<blobdata>::const_iterator it = tx_blobs.begin(); + std::vector<blobdata>::const_iterator it = tx_blobs.begin(); for (size_t i = 0; i < tx_blobs.size(); i++, ++it) { - m_threadpool.submit(&waiter, [&, i, it] { + tpool.submit(&waiter, [&, i, it] { try { results[i].res = handle_incoming_tx_pre(*it, tvc[i], results[i].tx, results[i].hash, results[i].prefix_hash, keeped_by_block, relayed, do_not_relay); @@ -696,7 +725,7 @@ namespace cryptonote } }); } - waiter.wait(); + waiter.wait(&tpool); it = tx_blobs.begin(); for (size_t i = 0; i < tx_blobs.size(); i++, ++it) { if (!results[i].res) @@ -711,7 +740,7 @@ namespace cryptonote } else { - m_threadpool.submit(&waiter, [&, i, it] { + tpool.submit(&waiter, [&, i, it] { try { results[i].res = handle_incoming_tx_post(*it, tvc[i], results[i].tx, results[i].hash, results[i].prefix_hash, keeped_by_block, relayed, do_not_relay); @@ -724,7 +753,7 @@ namespace cryptonote }); } } - waiter.wait(); + waiter.wait(&tpool); bool ok = true; it = tx_blobs.begin(); @@ -751,7 +780,7 @@ namespace cryptonote //----------------------------------------------------------------------------------------------- bool core::handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { - std::list<cryptonote::blobdata> tx_blobs; + std::vector<cryptonote::blobdata> tx_blobs; tx_blobs.push_back(tx_blob); std::vector<tx_verification_context> tvcv(1); bool r = handle_incoming_txs(tx_blobs, tvcv, keeped_by_block, relayed, do_not_relay); @@ -917,8 +946,8 @@ namespace cryptonote const uint64_t end = start_offset + count - 1; m_blockchain_storage.for_blocks_range(start_offset, end, [this, &emission_amount, &total_fee_amount](uint64_t, const crypto::hash& hash, const block& b){ - std::list<transaction> txs; - std::list<crypto::hash> missed_txs; + std::vector<transaction> txs; + std::vector<crypto::hash> missed_txs; uint64_t coinbase_amount = get_outs_money_amount(b.miner_tx); this->get_transactions(b.tx_hashes, txs, missed_txs); uint64_t tx_fee_amount = 0; @@ -1014,7 +1043,7 @@ namespace cryptonote bool core::relay_txpool_transactions() { // we attempt to relay txes that should be relayed, but were not - std::list<std::pair<crypto::hash, cryptonote::blobdata>> txs; + std::vector<std::pair<crypto::hash, cryptonote::blobdata>> txs; if (m_mempool.get_relayable_transactions(txs) && !txs.empty()) { cryptonote_connection_context fake_context = AUTO_VAL_INIT(fake_context); @@ -1032,7 +1061,7 @@ namespace cryptonote //----------------------------------------------------------------------------------------------- void core::on_transaction_relayed(const cryptonote::blobdata& tx_blob) { - std::list<std::pair<crypto::hash, cryptonote::blobdata>> txs; + std::vector<std::pair<crypto::hash, cryptonote::blobdata>> txs; cryptonote::transaction tx; crypto::hash tx_hash, tx_prefix_hash; if (!parse_and_validate_tx_from_blob(tx_blob, tx, tx_hash, tx_prefix_hash)) @@ -1054,9 +1083,9 @@ namespace cryptonote return m_blockchain_storage.find_blockchain_supplement(qblock_ids, resp); } //----------------------------------------------------------------------------------------------- - bool core::find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::list<std::pair<cryptonote::blobdata, std::list<cryptonote::blobdata> > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, size_t max_count) const + bool core::find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_count) const { - return m_blockchain_storage.find_blockchain_supplement(req_start_block, qblock_ids, blocks, total_height, start_height, pruned, max_count); + return m_blockchain_storage.find_blockchain_supplement(req_start_block, qblock_ids, blocks, total_height, start_height, pruned, get_miner_tx_hash, max_count); } //----------------------------------------------------------------------------------------------- bool core::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) const @@ -1111,7 +1140,7 @@ namespace cryptonote { block_verification_context bvc = boost::value_initialized<block_verification_context>(); m_miner.pause(); - std::list<block_complete_entry> blocks; + std::vector<block_complete_entry> blocks; try { blocks.push_back(get_block_complete_entry(b, m_mempool)); @@ -1135,8 +1164,8 @@ namespace cryptonote cryptonote_connection_context exclude_context = boost::value_initialized<cryptonote_connection_context>(); NOTIFY_NEW_BLOCK::request arg = AUTO_VAL_INIT(arg); arg.current_blockchain_height = m_blockchain_storage.get_current_blockchain_height(); - std::list<crypto::hash> missed_txs; - std::list<cryptonote::blobdata> txs; + std::vector<crypto::hash> missed_txs; + std::vector<cryptonote::blobdata> txs; m_blockchain_storage.get_transactions_blobs(b.tx_hashes, txs, missed_txs); if(missed_txs.size() && m_blockchain_storage.get_block_id_by_height(get_block_height(b)) != get_block_hash(b)) { @@ -1172,7 +1201,7 @@ namespace cryptonote } //----------------------------------------------------------------------------------------------- - bool core::prepare_handle_incoming_blocks(const std::list<block_complete_entry> &blocks) + bool core::prepare_handle_incoming_blocks(const std::vector<block_complete_entry> &blocks) { m_incoming_tx_lock.lock(); m_blockchain_storage.prepare_handle_incoming_blocks(blocks); @@ -1265,7 +1294,7 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- - bool core::get_pool_transactions(std::list<transaction>& txs, bool include_sensitive_data) const + bool core::get_pool_transactions(std::vector<transaction>& txs, bool include_sensitive_data) const { m_mempool.get_transactions(txs, include_sensitive_data); return true; @@ -1553,7 +1582,7 @@ namespace cryptonote return m_target_blockchain_height; } //----------------------------------------------------------------------------------------------- - uint64_t core::prevalidate_block_hashes(uint64_t height, const std::list<crypto::hash> &hashes) + uint64_t core::prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes) { return get_blockchain_storage().prevalidate_block_hashes(height, hashes); } diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 567966d48..84e1bb918 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -39,7 +39,6 @@ #include "cryptonote_protocol/cryptonote_protocol_handler_common.h" #include "storages/portable_storage_template_helper.h" #include "common/download.h" -#include "common/threadpool.h" #include "common/command_line.h" #include "tx_pool.h" #include "blockchain.h" @@ -61,6 +60,8 @@ namespace cryptonote extern const command_line::arg_descriptor<std::string, false, true, 2> arg_data_dir; extern const command_line::arg_descriptor<bool, false> arg_testnet_on; extern const command_line::arg_descriptor<bool, false> arg_stagenet_on; + extern const command_line::arg_descriptor<bool, false> arg_regtest_on; + extern const command_line::arg_descriptor<difficulty_type> arg_fixed_difficulty; extern const command_line::arg_descriptor<bool> arg_offline; /************************************************************************/ @@ -134,7 +135,7 @@ namespace cryptonote * * @return true if the transactions made it to the transaction pool, otherwise false */ - bool handle_incoming_txs(const std::list<blobdata>& tx_blobs, std::vector<tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); + bool handle_incoming_txs(const std::vector<blobdata>& tx_blobs, std::vector<tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); /** * @brief handles an incoming block @@ -157,7 +158,7 @@ namespace cryptonote * * @note see Blockchain::prepare_handle_incoming_blocks */ - bool prepare_handle_incoming_blocks(const std::list<block_complete_entry> &blocks); + bool prepare_handle_incoming_blocks(const std::vector<block_complete_entry> &blocks); /** * @copydoc Blockchain::cleanup_handle_incoming_blocks @@ -309,25 +310,25 @@ namespace cryptonote void get_blockchain_top(uint64_t& height, crypto::hash& top_id) const; /** - * @copydoc Blockchain::get_blocks(uint64_t, size_t, std::list<std::pair<cryptonote::blobdata,block>>&, std::list<transaction>&) const + * @copydoc Blockchain::get_blocks(uint64_t, size_t, std::vector<std::pair<cryptonote::blobdata,block>>&, std::vector<transaction>&) const * - * @note see Blockchain::get_blocks(uint64_t, size_t, std::list<std::pair<cryptonote::blobdata,block>>&, std::list<transaction>&) const + * @note see Blockchain::get_blocks(uint64_t, size_t, std::vector<std::pair<cryptonote::blobdata,block>>&, std::vector<transaction>&) const */ - bool get_blocks(uint64_t start_offset, size_t count, std::list<std::pair<cryptonote::blobdata,block>>& blocks, std::list<cryptonote::blobdata>& txs) const; + bool get_blocks(uint64_t start_offset, size_t count, std::vector<std::pair<cryptonote::blobdata,block>>& blocks, std::vector<cryptonote::blobdata>& txs) const; /** - * @copydoc Blockchain::get_blocks(uint64_t, size_t, std::list<std::pair<cryptonote::blobdata,block>>&) const + * @copydoc Blockchain::get_blocks(uint64_t, size_t, std::vector<std::pair<cryptonote::blobdata,block>>&) const * - * @note see Blockchain::get_blocks(uint64_t, size_t, std::list<std::pair<cryptonote::blobdata,block>>&) const + * @note see Blockchain::get_blocks(uint64_t, size_t, std::vector<std::pair<cryptonote::blobdata,block>>&) const */ - bool get_blocks(uint64_t start_offset, size_t count, std::list<std::pair<cryptonote::blobdata,block>>& blocks) const; + bool get_blocks(uint64_t start_offset, size_t count, std::vector<std::pair<cryptonote::blobdata,block>>& blocks) const; /** - * @copydoc Blockchain::get_blocks(uint64_t, size_t, std::list<std::pair<cryptonote::blobdata,block>>&) const + * @copydoc Blockchain::get_blocks(uint64_t, size_t, std::vector<std::pair<cryptonote::blobdata,block>>&) const * - * @note see Blockchain::get_blocks(uint64_t, size_t, std::list<std::pair<cryptonote::blobdata,block>>&) const + * @note see Blockchain::get_blocks(uint64_t, size_t, std::vector<std::pair<cryptonote::blobdata,block>>&) const */ - bool get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks) const; + bool get_blocks(uint64_t start_offset, size_t count, std::vector<block>& blocks) const; /** * @copydoc Blockchain::get_blocks(const t_ids_container&, t_blocks_container&, t_missed_container&) const @@ -352,14 +353,14 @@ namespace cryptonote * * @note see Blockchain::get_transactions */ - bool get_transactions(const std::vector<crypto::hash>& txs_ids, std::list<cryptonote::blobdata>& txs, std::list<crypto::hash>& missed_txs) const; + bool get_transactions(const std::vector<crypto::hash>& txs_ids, std::vector<cryptonote::blobdata>& txs, std::vector<crypto::hash>& missed_txs) const; /** * @copydoc Blockchain::get_transactions * * @note see Blockchain::get_transactions */ - bool get_transactions(const std::vector<crypto::hash>& txs_ids, std::list<transaction>& txs, std::list<crypto::hash>& missed_txs) const; + bool get_transactions(const std::vector<crypto::hash>& txs_ids, std::vector<transaction>& txs, std::vector<crypto::hash>& missed_txs) const; /** * @copydoc Blockchain::get_block_by_hash @@ -371,9 +372,9 @@ namespace cryptonote /** * @copydoc Blockchain::get_alternative_blocks * - * @note see Blockchain::get_alternative_blocks(std::list<block>&) const + * @note see Blockchain::get_alternative_blocks(std::vector<block>&) const */ - bool get_alternative_blocks(std::list<block>& blocks) const; + bool get_alternative_blocks(std::vector<block>& blocks) const; /** * @copydoc Blockchain::get_alternative_blocks_count @@ -430,7 +431,7 @@ namespace cryptonote * * @note see tx_memory_pool::get_transactions */ - bool get_pool_transactions(std::list<transaction>& txs, bool include_unrelayed_txes = true) const; + bool get_pool_transactions(std::vector<transaction>& txs, bool include_unrelayed_txes = true) const; /** * @copydoc tx_memory_pool::get_txpool_backlog @@ -513,11 +514,11 @@ namespace cryptonote bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const; /** - * @copydoc Blockchain::find_blockchain_supplement(const uint64_t, const std::list<crypto::hash>&, std::list<std::pair<cryptonote::blobdata, std::list<cryptonote::blobdata> > >&, uint64_t&, uint64_t&, size_t) const + * @copydoc Blockchain::find_blockchain_supplement(const uint64_t, const std::list<crypto::hash>&, std::vector<std::pair<cryptonote::blobdata, std::vector<cryptonote::blobdata> > >&, uint64_t&, uint64_t&, size_t) const * - * @note see Blockchain::find_blockchain_supplement(const uint64_t, const std::list<crypto::hash>&, std::list<std::pair<cryptonote::blobdata, std::list<transaction> > >&, uint64_t&, uint64_t&, size_t) const + * @note see Blockchain::find_blockchain_supplement(const uint64_t, const std::list<crypto::hash>&, std::vector<std::pair<cryptonote::blobdata, std::vector<transaction> > >&, uint64_t&, uint64_t&, size_t) const */ - bool find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::list<std::pair<cryptonote::blobdata, std::list<cryptonote::blobdata> > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, size_t max_count) const; + bool find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_count) const; /** * @brief gets some stats about the daemon @@ -764,7 +765,7 @@ namespace cryptonote * * @return number of usable blocks */ - uint64_t prevalidate_block_hashes(uint64_t height, const std::list<crypto::hash> &hashes); + uint64_t prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes); /** * @brief get free disk space on the blockchain partition @@ -991,8 +992,6 @@ namespace cryptonote std::unordered_set<crypto::hash> bad_semantics_txes[2]; boost::mutex bad_semantics_txes_lock; - tools::threadpool& m_threadpool; - enum { UPDATES_DISABLED, UPDATES_NOTIFY, diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index bf1fe476e..eac0f1f57 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -220,7 +220,7 @@ namespace cryptonote crypto::hash max_used_block_id = null_hash; uint64_t max_used_block_height = 0; cryptonote::txpool_tx_meta_t meta; - bool ch_inp_res = m_blockchain.check_tx_inputs(tx, max_used_block_height, max_used_block_id, tvc, kept_by_block); + bool ch_inp_res = check_tx_inputs([&tx]()->cryptonote::transaction&{ return tx; }, id, max_used_block_height, max_used_block_id, tvc, kept_by_block); if(!ch_inp_res) { // if the transaction was valid before (kept_by_block), then it @@ -239,6 +239,7 @@ namespace cryptonote meta.relayed = relayed; meta.do_not_relay = do_not_relay; meta.double_spend_seen = have_tx_keyimges_as_spent(tx); + meta.bf_padding = 0; memset(meta.padding, 0, sizeof(meta.padding)); try { @@ -278,6 +279,7 @@ namespace cryptonote meta.relayed = relayed; meta.do_not_relay = do_not_relay; meta.double_spend_seen = false; + meta.bf_padding = 0; memset(meta.padding, 0, sizeof(meta.padding)); try @@ -556,11 +558,12 @@ namespace cryptonote } //--------------------------------------------------------------------------------- //TODO: investigate whether boolean return is appropriate - bool tx_memory_pool::get_relayable_transactions(std::list<std::pair<crypto::hash, cryptonote::blobdata>> &txs) const + bool tx_memory_pool::get_relayable_transactions(std::vector<std::pair<crypto::hash, cryptonote::blobdata>> &txs) const { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); const uint64_t now = time(NULL); + txs.reserve(m_blockchain.get_txpool_tx_count()); m_blockchain.for_all_txpool_txes([this, now, &txs](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *){ // 0 fee transactions are never relayed if(meta.fee > 0 && !meta.do_not_relay && now - meta.last_relayed_time > get_relay_delay(now, meta.receive_time)) @@ -588,7 +591,7 @@ namespace cryptonote return true; } //--------------------------------------------------------------------------------- - void tx_memory_pool::set_relayed(const std::list<std::pair<crypto::hash, cryptonote::blobdata>> &txs) + void tx_memory_pool::set_relayed(const std::vector<std::pair<crypto::hash, cryptonote::blobdata>> &txs) { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); @@ -621,10 +624,11 @@ namespace cryptonote return m_blockchain.get_txpool_tx_count(include_unrelayed_txes); } //--------------------------------------------------------------------------------- - void tx_memory_pool::get_transactions(std::list<transaction>& txs, bool include_unrelayed_txes) const + void tx_memory_pool::get_transactions(std::vector<transaction>& txs, bool include_unrelayed_txes) const { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); + txs.reserve(m_blockchain.get_txpool_tx_count(include_unrelayed_txes)); m_blockchain.for_all_txpool_txes([&txs](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ transaction tx; if (!parse_and_validate_tx_from_blob(*bd, tx)) @@ -642,6 +646,7 @@ namespace cryptonote { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); + txs.reserve(m_blockchain.get_txpool_tx_count(include_unrelayed_txes)); m_blockchain.for_all_txpool_txes([&txs](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ txs.push_back(txid); return true; @@ -653,6 +658,7 @@ namespace cryptonote CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); const uint64_t now = time(NULL); + backlog.reserve(m_blockchain.get_txpool_tx_count(include_unrelayed_txes)); m_blockchain.for_all_txpool_txes([&backlog, now](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ backlog.push_back({meta.blob_size, meta.fee, meta.receive_time - now}); return true; @@ -741,6 +747,8 @@ namespace cryptonote { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); + tx_infos.reserve(m_blockchain.get_txpool_tx_count()); + key_image_infos.reserve(m_blockchain.get_txpool_tx_count()); m_blockchain.for_all_txpool_txes([&tx_infos, key_image_infos, include_sensitive_data](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ tx_info txi; txi.id_hash = epee::string_tools::pod_to_hex(txid); @@ -811,6 +819,8 @@ namespace cryptonote { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); + tx_infos.reserve(m_blockchain.get_txpool_tx_count()); + key_image_infos.reserve(m_blockchain.get_txpool_tx_count()); m_blockchain.for_all_txpool_txes([&tx_infos, key_image_infos](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ cryptonote::rpc::tx_in_pool txi; txi.tx_hash = txid; @@ -883,11 +893,15 @@ namespace cryptonote //--------------------------------------------------------------------------------- bool tx_memory_pool::on_blockchain_inc(uint64_t new_block_height, const crypto::hash& top_block_id) { + CRITICAL_REGION_LOCAL(m_transactions_lock); + m_input_cache.clear(); return true; } //--------------------------------------------------------------------------------- bool tx_memory_pool::on_blockchain_dec(uint64_t new_block_height, const crypto::hash& top_block_id) { + CRITICAL_REGION_LOCAL(m_transactions_lock); + m_input_cache.clear(); return true; } //--------------------------------------------------------------------------------- @@ -927,8 +941,45 @@ namespace cryptonote m_transactions_lock.unlock(); } //--------------------------------------------------------------------------------- - bool tx_memory_pool::is_transaction_ready_to_go(txpool_tx_meta_t& txd, transaction &tx) const + bool tx_memory_pool::check_tx_inputs(const std::function<cryptonote::transaction&(void)> &get_tx, const crypto::hash &txid, uint64_t &max_used_block_height, crypto::hash &max_used_block_id, tx_verification_context &tvc, bool kept_by_block) const + { + if (!kept_by_block) + { + const std::unordered_map<crypto::hash, std::tuple<bool, tx_verification_context, uint64_t, crypto::hash>>::const_iterator i = m_input_cache.find(txid); + if (i != m_input_cache.end()) + { + max_used_block_height = std::get<2>(i->second); + max_used_block_id = std::get<3>(i->second); + tvc = std::get<1>(i->second); + return std::get<0>(i->second); + } + } + bool ret = m_blockchain.check_tx_inputs(get_tx(), max_used_block_height, max_used_block_id, tvc, kept_by_block); + if (!kept_by_block) + m_input_cache.insert(std::make_pair(txid, std::make_tuple(ret, tvc, max_used_block_height, max_used_block_id))); + return ret; + } + //--------------------------------------------------------------------------------- + bool tx_memory_pool::is_transaction_ready_to_go(txpool_tx_meta_t& txd, const crypto::hash &txid, const cryptonote::blobdata &txblob, transaction &tx) const { + struct transction_parser + { + transction_parser(const cryptonote::blobdata &txblob, transaction &tx): txblob(txblob), tx(tx), parsed(false) {} + cryptonote::transaction &operator()() + { + if (!parsed) + { + if (!parse_and_validate_tx_from_blob(txblob, tx)) + throw std::runtime_error("failed to parse transaction blob"); + parsed = true; + } + return tx; + } + const cryptonote::blobdata &txblob; + transaction &tx; + bool parsed; + } lazy_tx(txblob, tx); + //not the best implementation at this time, sorry :( //check is ring_signature already checked ? if(txd.max_used_block_id == null_hash) @@ -938,7 +989,7 @@ namespace cryptonote return false;//we already sure that this tx is broken for this height tx_verification_context tvc; - if(!m_blockchain.check_tx_inputs(tx, txd.max_used_block_height, txd.max_used_block_id, tvc)) + if(!check_tx_inputs([&lazy_tx]()->cryptonote::transaction&{ return lazy_tx(); }, txid, txd.max_used_block_height, txd.max_used_block_id, tvc)) { txd.last_failed_height = m_blockchain.get_current_blockchain_height()-1; txd.last_failed_id = m_blockchain.get_block_id_by_height(txd.last_failed_height); @@ -955,7 +1006,7 @@ namespace cryptonote return false; //check ring signature again, it is possible (with very small chance) that this transaction become again valid tx_verification_context tvc; - if(!m_blockchain.check_tx_inputs(tx, txd.max_used_block_height, txd.max_used_block_id, tvc)) + if(!check_tx_inputs([&lazy_tx]()->cryptonote::transaction&{ return lazy_tx(); }, txid, txd.max_used_block_height, txd.max_used_block_id, tvc)) { txd.last_failed_height = m_blockchain.get_current_blockchain_height()-1; txd.last_failed_id = m_blockchain.get_block_id_by_height(txd.last_failed_height); @@ -964,7 +1015,7 @@ namespace cryptonote } } //if we here, transaction seems valid, but, anyway, check for key_images collisions with blockchain, just to be sure - if(m_blockchain.have_tx_keyimges_as_spent(tx)) + if(m_blockchain.have_tx_keyimges_as_spent(lazy_tx())) { txd.double_spend_seen = true; return false; @@ -1140,18 +1191,21 @@ namespace cryptonote cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(sorted_it->second); cryptonote::transaction tx; - if (!parse_and_validate_tx_from_blob(txblob, tx)) - { - MERROR("Failed to parse tx from txpool"); - sorted_it++; - continue; - } // Skip transactions that are not ready to be // included into the blockchain or that are // missing key images const cryptonote::txpool_tx_meta_t original_meta = meta; - bool ready = is_transaction_ready_to_go(meta, tx); + bool ready = false; + try + { + ready = is_transaction_ready_to_go(meta, sorted_it->second, txblob, tx); + } + catch (const std::exception &e) + { + MERROR("Failed to check transaction readiness: " << e.what()); + // continue, not fatal + } if (memcmp(&original_meta, &meta, sizeof(meta))) { try diff --git a/src/cryptonote_core/tx_pool.h b/src/cryptonote_core/tx_pool.h index 19cd83ed9..4ade7ddbe 100644 --- a/src/cryptonote_core/tx_pool.h +++ b/src/cryptonote_core/tx_pool.h @@ -237,7 +237,7 @@ namespace cryptonote * @param include_unrelayed_txes include unrelayed txes in the result * */ - void get_transactions(std::list<transaction>& txs, bool include_unrelayed_txes = true) const; + void get_transactions(std::vector<transaction>& txs, bool include_unrelayed_txes = true) const; /** * @brief get a list of all transaction hashes in the pool @@ -324,14 +324,14 @@ namespace cryptonote * * @return true */ - bool get_relayable_transactions(std::list<std::pair<crypto::hash, cryptonote::blobdata>>& txs) const; + bool get_relayable_transactions(std::vector<std::pair<crypto::hash, cryptonote::blobdata>>& txs) const; /** * @brief tell the pool that certain transactions were just relayed * * @param txs the list of transactions (and their hashes) */ - void set_relayed(const std::list<std::pair<crypto::hash, cryptonote::blobdata>>& txs); + void set_relayed(const std::vector<std::pair<crypto::hash, cryptonote::blobdata>>& txs); /** * @brief get the total number of transactions in the pool @@ -499,10 +499,13 @@ namespace cryptonote * @brief check if a transaction is a valid candidate for inclusion in a block * * @param txd the transaction to check (and info about it) + * @param txid the txid of the transaction to check + * @param txblob the transaction blob to check + * @param tx the parsed transaction, if successful * * @return true if the transaction is good to go, otherwise false */ - bool is_transaction_ready_to_go(txpool_tx_meta_t& txd, transaction &tx) const; + bool is_transaction_ready_to_go(txpool_tx_meta_t& txd, const crypto::hash &txid, const cryptonote::blobdata &txblob, transaction &tx) const; /** * @brief mark all transactions double spending the one passed @@ -555,6 +558,9 @@ private: */ sorted_tx_container::iterator find_tx_in_sorted_container(const crypto::hash& id) const; + //! cache/call Blockchain::check_tx_inputs results + bool check_tx_inputs(const std::function<cryptonote::transaction&(void)> &get_tx, const crypto::hash &txid, uint64_t &max_used_block_height, crypto::hash &max_used_block_id, tx_verification_context &tvc, bool kept_by_block = false) const; + //! transactions which are unlikely to be included in blocks /*! These transactions are kept in RAM in case they *are* included * in a block eventually, but this container is not saved to disk. @@ -565,6 +571,8 @@ private: size_t m_txpool_max_size; size_t m_txpool_size; + + mutable std::unordered_map<crypto::hash, std::tuple<bool, tx_verification_context, uint64_t, crypto::hash>> m_input_cache; }; } diff --git a/src/cryptonote_protocol/block_queue.cpp b/src/cryptonote_protocol/block_queue.cpp index 9ae33d540..c39d67ceb 100644 --- a/src/cryptonote_protocol/block_queue.cpp +++ b/src/cryptonote_protocol/block_queue.cpp @@ -50,10 +50,10 @@ namespace std { namespace cryptonote { -void block_queue::add_blocks(uint64_t height, std::list<cryptonote::block_complete_entry> bcel, const boost::uuids::uuid &connection_id, float rate, size_t size) +void block_queue::add_blocks(uint64_t height, std::vector<cryptonote::block_complete_entry> bcel, const boost::uuids::uuid &connection_id, float rate, size_t size) { boost::unique_lock<boost::recursive_mutex> lock(mutex); - std::list<crypto::hash> hashes; + std::vector<crypto::hash> hashes; bool has_hashes = remove_span(height, &hashes); blocks.insert(span(height, std::move(bcel), connection_id, rate, size)); if (has_hashes) @@ -97,7 +97,7 @@ void block_queue::flush_stale_spans(const std::set<boost::uuids::uuid> &live_con } } -bool block_queue::remove_span(uint64_t start_block_height, std::list<crypto::hash> *hashes) +bool block_queue::remove_span(uint64_t start_block_height, std::vector<crypto::hash> *hashes) { boost::unique_lock<boost::recursive_mutex> lock(mutex); for (block_map::iterator i = blocks.begin(); i != blocks.end(); ++i) @@ -172,7 +172,7 @@ bool block_queue::requested(const crypto::hash &hash) const return false; } -std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, const std::list<crypto::hash> &block_hashes, boost::posix_time::ptime time) +std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, const std::vector<crypto::hash> &block_hashes, boost::posix_time::ptime time) { boost::unique_lock<boost::recursive_mutex> lock(mutex); @@ -183,14 +183,14 @@ std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_hei } uint64_t span_start_height = last_block_height - block_hashes.size() + 1; - std::list<crypto::hash>::const_iterator i = block_hashes.begin(); + std::vector<crypto::hash>::const_iterator i = block_hashes.begin(); while (i != block_hashes.end() && requested(*i)) { ++i; ++span_start_height; } uint64_t span_length = 0; - std::list<crypto::hash> hashes; + std::vector<crypto::hash> hashes; while (i != block_hashes.end() && span_length < max_blocks) { hashes.push_back(*i); @@ -230,7 +230,7 @@ std::pair<uint64_t, uint64_t> block_queue::get_start_gap_span() const return std::make_pair(current_height + 1, first_span_height - current_height - 1); } -std::pair<uint64_t, uint64_t> block_queue::get_next_span_if_scheduled(std::list<crypto::hash> &hashes, boost::uuids::uuid &connection_id, boost::posix_time::ptime &time) const +std::pair<uint64_t, uint64_t> block_queue::get_next_span_if_scheduled(std::vector<crypto::hash> &hashes, boost::uuids::uuid &connection_id, boost::posix_time::ptime &time) const { boost::unique_lock<boost::recursive_mutex> lock(mutex); if (blocks.empty()) @@ -248,7 +248,7 @@ std::pair<uint64_t, uint64_t> block_queue::get_next_span_if_scheduled(std::list< return std::make_pair(i->start_block_height, i->nblocks); } -void block_queue::set_span_hashes(uint64_t start_height, const boost::uuids::uuid &connection_id, std::list<crypto::hash> hashes) +void block_queue::set_span_hashes(uint64_t start_height, const boost::uuids::uuid &connection_id, std::vector<crypto::hash> hashes) { boost::unique_lock<boost::recursive_mutex> lock(mutex); for (block_map::iterator i = blocks.begin(); i != blocks.end(); ++i) @@ -264,7 +264,7 @@ void block_queue::set_span_hashes(uint64_t start_height, const boost::uuids::uui } } -bool block_queue::get_next_span(uint64_t &height, std::list<cryptonote::block_complete_entry> &bcel, boost::uuids::uuid &connection_id, bool filled) const +bool block_queue::get_next_span(uint64_t &height, std::vector<cryptonote::block_complete_entry> &bcel, boost::uuids::uuid &connection_id, bool filled) const { boost::unique_lock<boost::recursive_mutex> lock(mutex); if (blocks.empty()) diff --git a/src/cryptonote_protocol/block_queue.h b/src/cryptonote_protocol/block_queue.h index 69ddaa435..9059e89ac 100644 --- a/src/cryptonote_protocol/block_queue.h +++ b/src/cryptonote_protocol/block_queue.h @@ -31,7 +31,7 @@ #pragma once #include <string> -#include <list> +#include <vector> #include <set> #include <boost/thread/recursive_mutex.hpp> #include <boost/uuid/uuid.hpp> @@ -49,15 +49,15 @@ namespace cryptonote struct span { uint64_t start_block_height; - std::list<crypto::hash> hashes; - std::list<cryptonote::block_complete_entry> blocks; + std::vector<crypto::hash> hashes; + std::vector<cryptonote::block_complete_entry> blocks; boost::uuids::uuid connection_id; uint64_t nblocks; float rate; size_t size; boost::posix_time::ptime time; - span(uint64_t start_block_height, std::list<cryptonote::block_complete_entry> blocks, const boost::uuids::uuid &connection_id, float rate, size_t size): + span(uint64_t start_block_height, std::vector<cryptonote::block_complete_entry> blocks, const boost::uuids::uuid &connection_id, float rate, size_t size): start_block_height(start_block_height), blocks(std::move(blocks)), connection_id(connection_id), nblocks(this->blocks.size()), rate(rate), size(size), time() {} span(uint64_t start_block_height, uint64_t nblocks, const boost::uuids::uuid &connection_id, boost::posix_time::ptime time): start_block_height(start_block_height), connection_id(connection_id), nblocks(nblocks), rate(0.0f), size(0), time(time) {} @@ -67,21 +67,21 @@ namespace cryptonote typedef std::set<span> block_map; public: - void add_blocks(uint64_t height, std::list<cryptonote::block_complete_entry> bcel, const boost::uuids::uuid &connection_id, float rate, size_t size); + void add_blocks(uint64_t height, std::vector<cryptonote::block_complete_entry> bcel, const boost::uuids::uuid &connection_id, float rate, size_t size); void add_blocks(uint64_t height, uint64_t nblocks, const boost::uuids::uuid &connection_id, boost::posix_time::ptime time = boost::date_time::min_date_time); void flush_spans(const boost::uuids::uuid &connection_id, bool all = false); void flush_stale_spans(const std::set<boost::uuids::uuid> &live_connections); - bool remove_span(uint64_t start_block_height, std::list<crypto::hash> *hashes = NULL); + bool remove_span(uint64_t start_block_height, std::vector<crypto::hash> *hashes = NULL); void remove_spans(const boost::uuids::uuid &connection_id, uint64_t start_block_height); uint64_t get_max_block_height() const; void print() const; std::string get_overview() const; - std::pair<uint64_t, uint64_t> reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, const std::list<crypto::hash> &block_hashes, boost::posix_time::ptime time = boost::posix_time::microsec_clock::universal_time()); + std::pair<uint64_t, uint64_t> reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, const std::vector<crypto::hash> &block_hashes, boost::posix_time::ptime time = boost::posix_time::microsec_clock::universal_time()); bool is_blockchain_placeholder(const span &span) const; std::pair<uint64_t, uint64_t> get_start_gap_span() const; - std::pair<uint64_t, uint64_t> get_next_span_if_scheduled(std::list<crypto::hash> &hashes, boost::uuids::uuid &connection_id, boost::posix_time::ptime &time) const; - void set_span_hashes(uint64_t start_height, const boost::uuids::uuid &connection_id, std::list<crypto::hash> hashes); - bool get_next_span(uint64_t &height, std::list<cryptonote::block_complete_entry> &bcel, boost::uuids::uuid &connection_id, bool filled = true) const; + std::pair<uint64_t, uint64_t> get_next_span_if_scheduled(std::vector<crypto::hash> &hashes, boost::uuids::uuid &connection_id, boost::posix_time::ptime &time) const; + void set_span_hashes(uint64_t start_height, const boost::uuids::uuid &connection_id, std::vector<crypto::hash> hashes); + bool get_next_span(uint64_t &height, std::vector<cryptonote::block_complete_entry> &bcel, boost::uuids::uuid &connection_id, bool filled = true) const; bool has_next_span(const boost::uuids::uuid &connection_id, bool &filled) const; size_t get_data_size() const; size_t get_num_filled_spans_prefix() const; diff --git a/src/cryptonote_protocol/cryptonote_protocol_defs.h b/src/cryptonote_protocol/cryptonote_protocol_defs.h index cf0043287..db159f0f4 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_defs.h +++ b/src/cryptonote_protocol/cryptonote_protocol_defs.h @@ -109,7 +109,7 @@ namespace cryptonote struct block_complete_entry { blobdata block; - std::list<blobdata> txs; + std::vector<blobdata> txs; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(block) KV_SERIALIZE(txs) @@ -145,7 +145,7 @@ namespace cryptonote struct request { - std::list<blobdata> txs; + std::vector<blobdata> txs; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(txs) @@ -161,8 +161,8 @@ namespace cryptonote struct request { - std::list<crypto::hash> txs; - std::list<crypto::hash> blocks; + std::vector<crypto::hash> txs; + std::vector<crypto::hash> blocks; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE_CONTAINER_POD_AS_BLOB(txs) @@ -177,9 +177,9 @@ namespace cryptonote struct request { - std::list<blobdata> txs; - std::list<block_complete_entry> blocks; - std::list<crypto::hash> missed_ids; + std::vector<blobdata> txs; + std::vector<block_complete_entry> blocks; + std::vector<crypto::hash> missed_ids; uint64_t current_blockchain_height; BEGIN_KV_SERIALIZE_MAP() @@ -230,7 +230,7 @@ namespace cryptonote uint64_t start_height; uint64_t total_height; uint64_t cumulative_difficulty; - std::list<crypto::hash> m_block_ids; + std::vector<crypto::hash> m_block_ids; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(start_height) diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index 2d4bc1299..a931d3b57 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -351,7 +351,7 @@ namespace cryptonote return 1; } m_core.pause_mine(); - std::list<block_complete_entry> blocks; + std::vector<block_complete_entry> blocks; blocks.push_back(arg.b); m_core.prepare_handle_incoming_blocks(blocks); for(auto tx_blob_it = arg.b.txs.begin(); tx_blob_it!=arg.b.txs.end();tx_blob_it++) @@ -438,7 +438,7 @@ namespace cryptonote } } - std::list<blobdata> have_tx; + std::vector<blobdata> have_tx; // Instead of requesting missing transactions by hash like BTC, // we do it by index (thanks to a suggestion from moneromooo) because @@ -578,8 +578,8 @@ namespace cryptonote else { std::vector<crypto::hash> tx_ids; - std::list<transaction> txes; - std::list<crypto::hash> missing; + std::vector<transaction> txes; + std::vector<crypto::hash> missing; tx_ids.push_back(tx_hash); if (m_core.get_transactions(tx_ids, txes, missing) && missing.empty()) { @@ -626,7 +626,7 @@ namespace cryptonote b.block = arg.b.block; b.txs = have_tx; - std::list<block_complete_entry> blocks; + std::vector<block_complete_entry> blocks; blocks.push_back(b); m_core.prepare_handle_incoming_blocks(blocks); @@ -687,8 +687,8 @@ namespace cryptonote { MLOG_P2P_MESSAGE("Received NOTIFY_REQUEST_FLUFFY_MISSING_TX (" << arg.missing_tx_indices.size() << " txes), block hash " << arg.block_hash); - std::list<std::pair<cryptonote::blobdata, block>> local_blocks; - std::list<cryptonote::blobdata> local_txs; + std::vector<std::pair<cryptonote::blobdata, block>> local_blocks; + std::vector<cryptonote::blobdata> local_txs; block b; if (!m_core.get_block_by_hash(arg.block_hash, b)) @@ -725,8 +725,8 @@ namespace cryptonote } } - std::list<cryptonote::transaction> txs; - std::list<crypto::hash> missed; + std::vector<cryptonote::transaction> txs; + std::vector<crypto::hash> missed; if (!m_core.get_transactions(txids, txs, missed)) { LOG_ERROR_CCONTEXT("Failed to handle request NOTIFY_REQUEST_FLUFFY_MISSING_TX, " @@ -774,10 +774,12 @@ namespace cryptonote return 1; } - for(auto tx_blob_it = arg.txs.begin(); tx_blob_it!=arg.txs.end();) + std::vector<cryptonote::blobdata> newtxs; + newtxs.reserve(arg.txs.size()); + for (size_t i = 0; i < arg.txs.size(); ++i) { cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc); - m_core.handle_incoming_tx(*tx_blob_it, tvc, false, true, false); + m_core.handle_incoming_tx(arg.txs[i], tvc, false, true, false); if(tvc.m_verifivation_failed) { LOG_PRINT_CCONTEXT_L1("Tx verification failed, dropping connection"); @@ -785,10 +787,9 @@ namespace cryptonote return 1; } if(tvc.m_should_be_relayed) - ++tx_blob_it; - else - arg.txs.erase(tx_blob_it++); + newtxs.push_back(std::move(arg.txs[i])); } + arg.txs = std::move(newtxs); if(arg.txs.size()) { @@ -996,7 +997,7 @@ skip: { const uint64_t previous_height = m_core.get_current_blockchain_height(); uint64_t start_height; - std::list<cryptonote::block_complete_entry> blocks; + std::vector<cryptonote::block_complete_entry> blocks; boost::uuids::uuid span_connection_id; if (!m_block_queue.get_next_span(start_height, blocks, span_connection_id)) { @@ -1070,7 +1071,7 @@ skip: LOG_ERROR_CCONTEXT("Internal error: tvc.size() != block_entry.txs.size()"); return 1; } - std::list<blobdata>::const_iterator it = block_entry.txs.begin(); + std::vector<blobdata>::const_iterator it = block_entry.txs.begin(); for (size_t i = 0; i < tvc.size(); ++i, ++it) { if(tvc[i].m_verifivation_failed) @@ -1260,7 +1261,7 @@ skip: template<class t_core> bool t_cryptonote_protocol_handler<t_core>::should_download_next_span(cryptonote_connection_context& context) const { - std::list<crypto::hash> hashes; + std::vector<crypto::hash> hashes; boost::uuids::uuid span_connection_id; boost::posix_time::ptime request_time; std::pair<uint64_t, uint64_t> span; @@ -1279,7 +1280,7 @@ skip: // we might be in a weird case where there is a filled next span, // but it starts higher than the current height uint64_t height; - std::list<cryptonote::block_complete_entry> bcel; + std::vector<cryptonote::block_complete_entry> bcel; if (!m_block_queue.get_next_span(height, bcel, span_connection_id, true)) return false; if (height > m_core.get_current_blockchain_height()) @@ -1427,7 +1428,7 @@ skip: { if (span.second == 0) { - std::list<crypto::hash> hashes; + std::vector<crypto::hash> hashes; boost::uuids::uuid span_connection_id; boost::posix_time::ptime time; span = m_block_queue.get_next_span_if_scheduled(hashes, span_connection_id, time); @@ -1453,14 +1454,18 @@ skip: goto skip; } // take out blocks we already have - while (!context.m_needed_objects.empty() && m_core.have_block(context.m_needed_objects.front())) + size_t skip = 0; + while (skip < context.m_needed_objects.size() && m_core.have_block(context.m_needed_objects[skip])) { // if we're popping the last hash, record it so we can ask again from that hash, // this prevents never being able to progress on peers we get old hash lists from - if (context.m_needed_objects.size() == 1) - context.m_last_known_hash = context.m_needed_objects.front(); - context.m_needed_objects.pop_front(); + if (skip + 1 == context.m_needed_objects.size()) + context.m_last_known_hash = context.m_needed_objects[skip]; + ++skip; } + if (skip > 0) + context.m_needed_objects = std::vector<crypto::hash>(context.m_needed_objects.begin() + skip, context.m_needed_objects.end()); + const uint64_t first_block_height = context.m_last_response_height - context.m_needed_objects.size() + 1; span = m_block_queue.reserve_span(first_block_height, context.m_last_response_height, count_limit, context.m_connection_id, context.m_needed_objects); MDEBUG(context << " span from " << first_block_height << ": " << span.first << "/" << span.second); @@ -1468,7 +1473,7 @@ skip: if (span.second == 0 && !force_next_span) { MDEBUG(context << " still no span reserved, we may be in the corner case of next span scheduled and everything else scheduled/filled"); - std::list<crypto::hash> hashes; + std::vector<crypto::hash> hashes; boost::uuids::uuid span_connection_id; boost::posix_time::ptime time; span = m_block_queue.get_next_span_if_scheduled(hashes, span_connection_id, time); @@ -1499,23 +1504,21 @@ skip: MERROR("ERROR: skip " << skip << ", m_needed_objects " << context.m_needed_objects.size() << ", first_context_block_height" << first_context_block_height); return false; } - while (skip--) - context.m_needed_objects.pop_front(); + if (skip > 0) + context.m_needed_objects = std::vector<crypto::hash>(context.m_needed_objects.begin() + skip, context.m_needed_objects.end()); if (context.m_needed_objects.size() < span.second) { MERROR("ERROR: span " << span.first << "/" << span.second << ", m_needed_objects " << context.m_needed_objects.size()); return false; } - auto it = context.m_needed_objects.begin(); for (size_t n = 0; n < span.second; ++n) { - req.blocks.push_back(*it); + req.blocks.push_back(context.m_needed_objects[n]); ++count; - context.m_requested_objects.insert(*it); - auto j = it++; - context.m_needed_objects.erase(j); + context.m_requested_objects.insert(context.m_needed_objects[n]); } + context.m_needed_objects = std::vector<crypto::hash>(context.m_needed_objects.begin() + span.second, context.m_needed_objects.end()); } context.m_last_request_time = boost::posix_time::microsec_clock::universal_time(); @@ -1676,7 +1679,7 @@ skip: { NOTIFY_NEW_FLUFFY_BLOCK::request fluffy_arg = AUTO_VAL_INIT(fluffy_arg); fluffy_arg.current_blockchain_height = arg.current_blockchain_height; - std::list<blobdata> fluffy_txs; + std::vector<blobdata> fluffy_txs; fluffy_arg.b = arg.b; fluffy_arg.b.txs = fluffy_txs; @@ -1724,6 +1727,9 @@ skip: template<class t_core> void t_cryptonote_protocol_handler<t_core>::drop_connection(cryptonote_connection_context &context, bool add_fail, bool flush_all_spans) { + LOG_DEBUG_CC(context, "dropping connection id " << context.m_connection_id << + ", add_fail " << add_fail << ", flush_all_spans " << flush_all_spans); + if (add_fail) m_p2p->add_host_fail(context.m_remote_address); diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt index 3b1d0d826..84004c3c6 100644 --- a/src/daemon/CMakeLists.txt +++ b/src/daemon/CMakeLists.txt @@ -92,7 +92,6 @@ target_link_libraries(daemon daemonizer serialization daemon_rpc_server - epee ${EPEE_READLINE} version ${Boost_CHRONO_LIBRARY} diff --git a/src/daemon/command_line_args.h b/src/daemon/command_line_args.h index 4673590aa..cba71bf3b 100644 --- a/src/daemon/command_line_args.h +++ b/src/daemon/command_line_args.h @@ -72,6 +72,11 @@ namespace daemon_args , "Specify maximum log file size [B]" , MAX_LOG_FILE_SIZE }; + const command_line::arg_descriptor<std::size_t> arg_max_log_files = { + "max-log-files" + , "Specify maximum number of rotated log files to be saved (no limit by setting to 0)" + , MAX_LOG_FILES + }; const command_line::arg_descriptor<std::string> arg_log_level = { "log-level" , "" diff --git a/src/daemon/command_parser_executor.cpp b/src/daemon/command_parser_executor.cpp index 34d9fb4c8..1638cf505 100644 --- a/src/daemon/command_parser_executor.cpp +++ b/src/daemon/command_parser_executor.cpp @@ -27,6 +27,7 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "common/dns_utils.h" +#include "common/command_line.h" #include "version.h" #include "daemon/command_parser_executor.h" @@ -326,12 +327,26 @@ bool t_command_parser_executor::start_mining(const std::vector<std::string>& arg if(args.size() == 4) { - ignore_battery = args[3] == "true"; + if(args[3] == "true" || command_line::is_yes(args[3]) || args[3] == "1") + { + ignore_battery = true; + } + else if(args[3] != "false" && !command_line::is_no(args[3]) && args[3] != "0") + { + return false; + } } if(args.size() >= 3) { - do_background_mining = args[2] == "true"; + if(args[2] == "true" || command_line::is_yes(args[2]) || args[2] == "1") + { + do_background_mining = true; + } + else if(args[2] != "false" && !command_line::is_no(args[2]) && args[2] != "0") + { + return false; + } } if(args.size() >= 2) @@ -599,13 +614,13 @@ bool t_command_parser_executor::print_coinbase_tx_sum(const std::vector<std::str bool t_command_parser_executor::alt_chain_info(const std::vector<std::string>& args) { - if(args.size()) + if(args.size() > 1) { - std::cout << "No parameters allowed" << std::endl; + std::cout << "usage: alt_chain_info [block_hash]" << std::endl; return false; } - return m_executor.alt_chain_info(); + return m_executor.alt_chain_info(args.size() == 1 ? args[0] : ""); } bool t_command_parser_executor::print_blockchain_dynamic_stats(const std::vector<std::string>& args) diff --git a/src/daemon/command_server.cpp b/src/daemon/command_server.cpp index 144603597..35504f2c9 100644 --- a/src/daemon/command_server.cpp +++ b/src/daemon/command_server.cpp @@ -255,6 +255,7 @@ t_command_server::t_command_server( m_command_lookup.set_handler( "alt_chain_info" , std::bind(&t_command_parser_executor::alt_chain_info, &m_parser, p::_1) + , "alt_chain_info [blockhash]" , "Print the information about alternative chains." ); m_command_lookup.set_handler( diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 48671f190..ea24e32eb 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -77,9 +77,10 @@ public: const auto testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); const auto stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); + const auto regtest = command_line::get_arg(vm, cryptonote::arg_regtest_on); const auto restricted = command_line::get_arg(vm, cryptonote::core_rpc_server::arg_restricted_rpc); const auto main_rpc_port = command_line::get_arg(vm, cryptonote::core_rpc_server::arg_rpc_bind_port); - rpcs.emplace_back(new t_rpc{vm, core, p2p, restricted, testnet ? cryptonote::TESTNET : stagenet ? cryptonote::STAGENET : cryptonote::MAINNET, main_rpc_port, "core"}); + rpcs.emplace_back(new t_rpc{vm, core, p2p, restricted, testnet ? cryptonote::TESTNET : stagenet ? cryptonote::STAGENET : regtest ? cryptonote::FAKECHAIN : cryptonote::MAINNET, main_rpc_port, "core"}); auto restricted_rpc_port_arg = cryptonote::core_rpc_server::arg_rpc_restricted_bind_port; if(!command_line::is_arg_defaulted(vm, restricted_rpc_port_arg)) diff --git a/src/daemon/main.cpp b/src/daemon/main.cpp index 49494e889..82ece62a9 100644 --- a/src/daemon/main.cpp +++ b/src/daemon/main.cpp @@ -84,6 +84,7 @@ int main(int argc, char const * argv[]) command_line::add_arg(core_settings, daemon_args::arg_log_file); command_line::add_arg(core_settings, daemon_args::arg_log_level); command_line::add_arg(core_settings, daemon_args::arg_max_log_file_size); + command_line::add_arg(core_settings, daemon_args::arg_max_log_files); command_line::add_arg(core_settings, daemon_args::arg_max_concurrency); command_line::add_arg(core_settings, daemon_args::arg_zmq_rpc_bind_ip); command_line::add_arg(core_settings, daemon_args::arg_zmq_rpc_bind_port); @@ -162,9 +163,10 @@ int main(int argc, char const * argv[]) const bool testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); const bool stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); - if (testnet && stagenet) + const bool regtest = command_line::get_arg(vm, cryptonote::arg_regtest_on); + if (testnet + stagenet + regtest > 1) { - std::cerr << "Can't specify more than one of --tesnet and --stagenet" << ENDL; + std::cerr << "Can't specify more than one of --tesnet and --stagenet and --regtest" << ENDL; return 1; } @@ -203,7 +205,7 @@ int main(int argc, char const * argv[]) if (!command_line::is_arg_defaulted(vm, daemon_args::arg_log_file)) log_file_path = command_line::get_arg(vm, daemon_args::arg_log_file); log_file_path = bf::absolute(log_file_path, relative_path_base); - mlog_configure(log_file_path.string(), true, command_line::get_arg(vm, daemon_args::arg_max_log_file_size)); + mlog_configure(log_file_path.string(), true, command_line::get_arg(vm, daemon_args::arg_max_log_file_size), command_line::get_arg(vm, daemon_args::arg_max_log_files)); // Set log level if (!command_line::is_arg_defaulted(vm, daemon_args::arg_log_level)) @@ -262,6 +264,9 @@ int main(int argc, char const * argv[]) } else { +#ifdef HAVE_READLINE + rdln::suspend_readline pause_readline; +#endif std::cerr << "Unknown command: " << command.front() << std::endl; return 1; } diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp index 8735f5463..e2b42c806 100644 --- a/src/daemon/rpc_command_executor.cpp +++ b/src/daemon/rpc_command_executor.cpp @@ -75,7 +75,9 @@ namespace { << "hash: " << header.hash << std::endl << "difficulty: " << boost::lexical_cast<std::string>(header.difficulty) << std::endl << "POW hash: " << header.pow_hash << std::endl - << "reward: " << boost::lexical_cast<std::string>(header.reward); + << "block size: " << header.block_size << std::endl + << "num txes: " << header.num_txes << std::endl + << "reward: " << cryptonote::print_money(header.reward); } std::string get_human_time_ago(time_t t, time_t now) @@ -976,7 +978,7 @@ bool t_rpc_command_executor::print_transaction_pool_stats() { } else { - memset(&res.pool_stats, 0, sizeof(res.pool_stats)); + res.pool_stats = {}; if (!m_rpc_server->on_get_transaction_pool_stats(req, res, false) || res.status != CORE_RPC_STATUS_OK) { tools::fail_msg_writer() << make_error(fail_message, res.status); @@ -1628,7 +1630,7 @@ bool t_rpc_command_executor::print_coinbase_tx_sum(uint64_t height, uint64_t cou return true; } -bool t_rpc_command_executor::alt_chain_info() +bool t_rpc_command_executor::alt_chain_info(const std::string &tip) { cryptonote::COMMAND_RPC_GET_INFO::request ireq; cryptonote::COMMAND_RPC_GET_INFO::response ires; @@ -1663,12 +1665,32 @@ bool t_rpc_command_executor::alt_chain_info() } } - tools::msg_writer() << boost::lexical_cast<std::string>(res.chains.size()) << " alternate chains found:"; - for (const auto &chain: res.chains) + if (tip.empty()) { - uint64_t start_height = (chain.height - chain.length + 1); - tools::msg_writer() << chain.length << " blocks long, from height " << start_height << " (" << (ires.height - start_height - 1) - << " deep), diff " << chain.difficulty << ": " << chain.block_hash; + tools::msg_writer() << boost::lexical_cast<std::string>(res.chains.size()) << " alternate chains found:"; + for (const auto &chain: res.chains) + { + uint64_t start_height = (chain.height - chain.length + 1); + tools::msg_writer() << chain.length << " blocks long, from height " << start_height << " (" << (ires.height - start_height - 1) + << " deep), diff " << chain.difficulty << ": " << chain.block_hash; + } + } + else + { + const auto i = std::find_if(res.chains.begin(), res.chains.end(), [&tip](cryptonote::COMMAND_RPC_GET_ALTERNATE_CHAINS::chain_info &info){ return info.block_hash == tip; }); + if (i != res.chains.end()) + { + const auto &chain = *i; + tools::success_msg_writer() << "Found alternate chain with tip " << tip; + uint64_t start_height = (chain.height - chain.length + 1); + tools::msg_writer() << chain.length << " blocks long, from height " << start_height << " (" << (ires.height - start_height - 1) + << " deep), diff " << chain.difficulty << ":"; + for (const std::string &block_id: chain.block_hashes) + tools::msg_writer() << " " << block_id; + tools::msg_writer() << "Chain parent on main chain: " << chain.main_chain_parent_block; + } + else + tools::fail_msg_writer() << "Block hash " << tip << " is not the tip of any known alternate chain"; } return true; } diff --git a/src/daemon/rpc_command_executor.h b/src/daemon/rpc_command_executor.h index 46168c93b..9e6010c5b 100644 --- a/src/daemon/rpc_command_executor.h +++ b/src/daemon/rpc_command_executor.h @@ -143,7 +143,7 @@ public: bool print_coinbase_tx_sum(uint64_t height, uint64_t count); - bool alt_chain_info(); + bool alt_chain_info(const std::string &tip); bool print_blockchain_dynamic_stats(uint64_t nblocks); diff --git a/src/debug_utilities/object_sizes.cpp b/src/debug_utilities/object_sizes.cpp index a3d037a59..ab8839636 100644 --- a/src/debug_utilities/object_sizes.cpp +++ b/src/debug_utilities/object_sizes.cpp @@ -94,6 +94,11 @@ int main(int argc, char* argv[]) SL(nodetool::node_server<cryptonote::t_cryptonote_protocol_handler<cryptonote::core>>); SL(nodetool::p2p_connection_context_t<cryptonote::t_cryptonote_protocol_handler<cryptonote::core>::connection_context>); SL(nodetool::network_address_old); + SL(nodetool::peerlist_entry_base<nodetool::network_address_old>); + + SL(nodetool::network_config); + SL(nodetool::basic_node_data); + SL(cryptonote::CORE_SYNC_DATA); SL(tools::wallet2::transfer_details); SL(tools::wallet2::payment_details); diff --git a/src/device/device_ledger.cpp b/src/device/device_ledger.cpp index 3b9ab6744..f7bf58531 100644 --- a/src/device/device_ledger.cpp +++ b/src/device/device_ledger.cpp @@ -48,6 +48,15 @@ namespace hw { /* ===================================================================== */ /* === Debug ==== */ /* ===================================================================== */ + #ifdef WIN32 + static char *pcsc_stringify_error(LONG rv) { + static __thread char out[20]; + sprintf_s(out, sizeof(out), "0x%08lX", rv); + + return out; + } + #endif + void set_apdu_verbose(bool verbose) { apdu_verbose = verbose; } @@ -178,14 +187,15 @@ namespace hw { void device_ledger::logCMD() { if (apdu_verbose) { char strbuffer[1024]; - sprintf(strbuffer, "%.02x %.02x %.02x %.02x %.02x ", + snprintf(strbuffer, sizeof(strbuffer), "%.02x %.02x %.02x %.02x %.02x ", this->buffer_send[0], this->buffer_send[1], this->buffer_send[2], this->buffer_send[3], this->buffer_send[4] ); - buffer_to_str(strbuffer+strlen(strbuffer), sizeof(strbuffer), (char*)(this->buffer_send+5), this->length_send-5); + const size_t len = strlen(strbuffer); + buffer_to_str(strbuffer+len, sizeof(strbuffer)-len, (char*)(this->buffer_send+5), this->length_send-5); MDEBUG( "CMD :" << strbuffer); } } @@ -193,11 +203,12 @@ namespace hw { void device_ledger::logRESP() { if (apdu_verbose) { char strbuffer[1024]; - sprintf(strbuffer, "%.02x%.02x ", + snprintf(strbuffer, sizeof(strbuffer), "%.02x%.02x ", this->buffer_recv[this->length_recv-2], this->buffer_recv[this->length_recv-1] ); - buffer_to_str(strbuffer+strlen(strbuffer), sizeof(strbuffer), (char*)(this->buffer_recv), this->length_recv-2); + const size_t len = strlen(strbuffer); + buffer_to_str(strbuffer+len, sizeof(strbuffer)-len, (char*)(this->buffer_recv), this->length_recv-2); MDEBUG( "RESP :" << strbuffer); } @@ -263,28 +274,38 @@ namespace hw { /* ======================================================================= */ /* MISC */ /* ======================================================================= */ - bool device_ledger::reset() { - int offset; - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_RESET; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; + int device_ledger::set_command_header(BYTE ins, BYTE p1, BYTE p2) { + reset_buffer(); + int offset = 0; + this->buffer_send[0] = 0x00; + this->buffer_send[1] = ins; + this->buffer_send[2] = p1; + this->buffer_send[3] = p2; + this->buffer_send[4] = 0x00; + return 5; + } - this->buffer_send[4] = offset-5; - this->length_send = offset; - this->exchange(); + int device_ledger::set_command_header_noopt(BYTE ins, BYTE p1, BYTE p2) { + int offset = set_command_header(ins, p1, p2); + //options + this->buffer_send[offset++] = 0; + this->buffer_send[4] = offset - 5; + return offset; + } + + void device_ledger::send_simple(BYTE ins, BYTE p1) { + this->length_send = set_command_header_noopt(ins, p1); + this->exchange(); + } + + bool device_ledger::reset() { + send_simple(INS_RESET); return true; } unsigned int device_ledger::exchange(unsigned int ok, unsigned int mask) { LONG rv; - int sw; + unsigned int sw; ASSERT_T0(this->length_send <= BUFFER_SEND_SIZE); logCMD(); @@ -293,6 +314,7 @@ namespace hw { SCARD_PCI_T0, this->buffer_send, this->length_send, NULL, this->buffer_recv, &this->length_recv); ASSERT_RV(rv); + ASSERT_T0(this->length_recv >= 2); ASSERT_T0(this->length_recv <= BUFFER_RECV_SIZE); logRESP(); @@ -396,7 +418,7 @@ namespace hw { } } - if (mszReaders) { + if (rv == SCARD_S_SUCCESS && mszReaders) { #ifdef SCARD_AUTOALLOCATE SCardFreeMemory(this->hContext, mszReaders); #else @@ -439,20 +461,10 @@ namespace hw { int offset; - reset_buffer(); - switch(mode) { case TRANSACTION_CREATE_REAL: case TRANSACTION_CREATE_FAKE: - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_SET_SIGNATURE_MODE; - this->buffer_send[2] = 0x01; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; + offset = set_command_header_noopt(INS_SET_SIGNATURE_MODE, 1); //account this->buffer_send[offset] = mode; offset += 1; @@ -483,22 +495,7 @@ namespace hw { bool device_ledger::get_public_address(cryptonote::account_public_address &pubkey){ AUTO_LOCK_CMD(); - int offset; - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_GET_KEY; - this->buffer_send[2] = 0x01; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; - - this->buffer_send[4] = offset-5; - this->length_send = offset; - this->exchange(); + send_simple(INS_GET_KEY, 1); memmove(pubkey.m_view_public_key.data, this->buffer_recv, 32); memmove(pubkey.m_spend_public_key.data, this->buffer_recv+32, 32); @@ -514,22 +511,7 @@ namespace hw { memset(skey.data, 0xFF, 32); //spcialkey, normal conf handled in decrypt - int offset; - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_GET_KEY; - this->buffer_send[2] = 0x02; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; - - this->buffer_send[4] = offset-5; - this->length_send = offset; - this->exchange(); + send_simple(INS_GET_KEY, 0x02); //View key is retrievied, if allowed, to speed up blockchain parsing memmove(this->viewkey.data, this->buffer_recv+0, 32); @@ -551,7 +533,6 @@ namespace hw { bool device_ledger::generate_chacha_key(const cryptonote::account_keys &keys, crypto::chacha_key &key) { AUTO_LOCK_CMD(); - int offset; #ifdef DEBUG_HWDEVICE crypto::chacha_key key_x; @@ -559,20 +540,7 @@ namespace hw { this->controle_device->generate_chacha_key(keys_x, key_x); #endif - reset_buffer(); - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_GET_CHACHA8_PREKEY; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; - - this->buffer_send[4] = offset-5; - this->length_send = offset; - this->exchange(); + send_simple(INS_GET_CHACHA8_PREKEY); char prekey[200]; memmove(prekey, &this->buffer_recv[0], 200); @@ -615,19 +583,7 @@ namespace hw { crypto::derive_subaddress_public_key(pub, derivation, output_index,derived_pub); } else { - int offset =0; - - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_DERIVE_SUBADDRESS_PUBLIC_KEY; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0; - offset += 1; + int offset = set_command_header_noopt(INS_DERIVE_SUBADDRESS_PUBLIC_KEY); //pub memmove(this->buffer_send+offset, pub.data, 32); offset += 32; @@ -658,7 +614,6 @@ namespace hw { crypto::public_key device_ledger::get_subaddress_spend_public_key(const cryptonote::account_keys& keys, const cryptonote::subaddress_index &index) { AUTO_LOCK_CMD(); crypto::public_key D; - int offset; #ifdef DEBUG_HWDEVICE const cryptonote::account_keys keys_x = hw::ledger::decrypt(keys); @@ -675,17 +630,7 @@ namespace hw { D = keys.m_account_address.m_spend_public_key; } else { - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_GET_SUBADDRESS_SPEND_PUBLIC_KEY; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; + int offset = set_command_header_noopt(INS_GET_SUBADDRESS_SPEND_PUBLIC_KEY); //index static_assert(sizeof(cryptonote::subaddress_index) == 8, "cryptonote::subaddress_index shall be 8 bytes length"); memmove(this->buffer_send+offset, &index, sizeof(cryptonote::subaddress_index)); @@ -720,7 +665,6 @@ namespace hw { cryptonote::account_public_address device_ledger::get_subaddress(const cryptonote::account_keys& keys, const cryptonote::subaddress_index &index) { AUTO_LOCK_CMD(); cryptonote::account_public_address address; - int offset; #ifdef DEBUG_HWDEVICE const cryptonote::account_keys keys_x = hw::ledger::decrypt(keys); @@ -739,17 +683,7 @@ namespace hw { if (index.is_zero()) { address = keys.m_account_address; } else { - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_GET_SUBADDRESS; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; + int offset = set_command_header_noopt(INS_GET_SUBADDRESS); //index static_assert(sizeof(cryptonote::subaddress_index) == 8, "cryptonote::subaddress_index shall be 8 bytes length"); memmove(this->buffer_send+offset, &index, sizeof(cryptonote::subaddress_index)); @@ -774,7 +708,6 @@ namespace hw { crypto::secret_key device_ledger::get_subaddress_secret_key(const crypto::secret_key &sec, const cryptonote::subaddress_index &index) { AUTO_LOCK_CMD(); crypto::secret_key sub_sec; - int offset; #ifdef DEBUG_HWDEVICE const crypto::secret_key sec_x = hw::ledger::decrypt(sec); @@ -786,17 +719,7 @@ namespace hw { hw::ledger::log_hexbuffer("get_subaddress_secret_key: [[OUT]] sub_sec", sub_sec_x.data, 32); #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_GET_SUBADDRESS_SECRET_KEY; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0; - offset += 1; + int offset = set_command_header_noopt(INS_GET_SUBADDRESS_SECRET_KEY); //sec memmove(this->buffer_send+offset, sec.data, 32); offset += 32; @@ -827,16 +750,7 @@ namespace hw { AUTO_LOCK_CMD(); int offset, sw; - reset_buffer(); - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_VERIFY_KEY; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; + offset = set_command_header_noopt(INS_VERIFY_KEY); //sec memmove(this->buffer_send+offset, secret_key.data, 32); offset += 32; @@ -859,7 +773,6 @@ namespace hw { bool device_ledger::scalarmultKey(rct::key & aP, const rct::key &P, const rct::key &a) { AUTO_LOCK_CMD(); - int offset; #ifdef DEBUG_HWDEVICE const rct::key P_x = P; @@ -871,17 +784,7 @@ namespace hw { hw::ledger::log_hexbuffer("scalarmultKey: [[OUT]] aP", (char*)aP_x.bytes, 32); #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_SECRET_SCAL_MUL_KEY; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0; - offset += 1; + int offset = set_command_header_noopt(INS_SECRET_SCAL_MUL_KEY); //pub memmove(this->buffer_send+offset, P.bytes, 32); offset += 32; @@ -906,7 +809,6 @@ namespace hw { bool device_ledger::scalarmultBase(rct::key &aG, const rct::key &a) { AUTO_LOCK_CMD(); - int offset; #ifdef DEBUG_HWDEVICE const rct::key a_x = hw::ledger::decrypt(a); @@ -916,17 +818,7 @@ namespace hw { hw::ledger::log_hexbuffer("scalarmultKey: [[OUT]] aG", (char*)aG_x.bytes, 32); #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_SECRET_SCAL_MUL_BASE; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0; - offset += 1; + int offset = set_command_header_noopt(INS_SECRET_SCAL_MUL_BASE); //sec memmove(this->buffer_send+offset, a.bytes, 32); offset += 32; @@ -947,7 +839,6 @@ namespace hw { bool device_ledger::sc_secret_add( crypto::secret_key &r, const crypto::secret_key &a, const crypto::secret_key &b) { AUTO_LOCK_CMD(); - int offset; #ifdef DEBUG_HWDEVICE const crypto::secret_key a_x = hw::ledger::decrypt(a); @@ -956,17 +847,7 @@ namespace hw { this->controle_device->sc_secret_add(r_x, a_x, b_x); #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_SECRET_KEY_ADD; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0; - offset += 1; + int offset = set_command_header_noopt(INS_SECRET_KEY_ADD); //sec key memmove(this->buffer_send+offset, a.data, 32); offset += 32; @@ -995,8 +876,6 @@ namespace hw { throw std::runtime_error("device generate key does not support recover"); } - int offset; - #ifdef DEBUG_HWDEVICE bool recover_x = recover; const crypto::secret_key recovery_key_x = recovery_key; @@ -1004,21 +883,7 @@ namespace hw { crypto::secret_key sec_x; #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_GENERATE_KEYPAIR; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0; - offset += 1; - - this->buffer_send[4] = offset-5; - this->length_send = offset; - this->exchange(); + send_simple(INS_GENERATE_KEYPAIR); //pub key memmove(pub.data, &this->buffer_recv[0], 32); @@ -1058,19 +923,7 @@ namespace hw { r = crypto::generate_key_derivation(pub, this->viewkey, derivation); } else { - int offset; - - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_GEN_KEY_DERIVATION; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0; - offset += 1; + int offset = set_command_header_noopt(INS_GEN_KEY_DERIVATION); //pub memmove(this->buffer_send+offset, pub.data, 32); offset += 32; @@ -1119,7 +972,6 @@ namespace hw { bool device_ledger::derivation_to_scalar(const crypto::key_derivation &derivation, const size_t output_index, crypto::ec_scalar &res) { AUTO_LOCK_CMD(); - int offset; #ifdef DEBUG_HWDEVICE const crypto::key_derivation derivation_x = hw::ledger::decrypt(derivation); @@ -1131,17 +983,7 @@ namespace hw { hw::ledger::log_hexbuffer("derivation_to_scalar: [[OUT]] res ", res_x.data, 32); #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_DERIVATION_TO_SCALAR; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0; - offset += 1; + int offset = set_command_header_noopt(INS_DERIVATION_TO_SCALAR); //derivattion memmove(this->buffer_send+offset, derivation.data, 32); offset += 32; @@ -1169,7 +1011,6 @@ namespace hw { bool device_ledger::derive_secret_key(const crypto::key_derivation &derivation, const std::size_t output_index, const crypto::secret_key &sec, crypto::secret_key &derived_sec) { AUTO_LOCK_CMD(); - int offset; #ifdef DEBUG_HWDEVICE const crypto::key_derivation derivation_x = hw::ledger::decrypt(derivation); @@ -1183,17 +1024,7 @@ namespace hw { hw::ledger::log_hexbuffer("derive_secret_key: [[OUT]] derived_sec", derived_sec_x.data, 32); #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_DERIVE_SECRET_KEY; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0; - offset += 1; + int offset = set_command_header_noopt(INS_DERIVE_SECRET_KEY); //derivation memmove(this->buffer_send+offset, derivation.data, 32); offset += 32; @@ -1224,7 +1055,6 @@ namespace hw { bool device_ledger::derive_public_key(const crypto::key_derivation &derivation, const std::size_t output_index, const crypto::public_key &pub, crypto::public_key &derived_pub){ AUTO_LOCK_CMD(); - int offset; #ifdef DEBUG_HWDEVICE const crypto::key_derivation derivation_x = hw::ledger::decrypt(derivation); @@ -1238,17 +1068,7 @@ namespace hw { hw::ledger::log_hexbuffer("derive_public_key: [[OUT]] derived_pub ", derived_pub_x.data, 32); #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_DERIVE_PUBLIC_KEY; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0; - offset += 1; + int offset = set_command_header_noopt(INS_DERIVE_PUBLIC_KEY); //derivation memmove(this->buffer_send+offset, derivation.data, 32); offset += 32; @@ -1278,7 +1098,6 @@ namespace hw { bool device_ledger::secret_key_to_public_key(const crypto::secret_key &sec, crypto::public_key &pub) { AUTO_LOCK_CMD(); - int offset; #ifdef DEBUG_HWDEVICE const crypto::secret_key sec_x = hw::ledger::decrypt(sec); @@ -1291,17 +1110,7 @@ namespace hw { } #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_SECRET_KEY_TO_PUBLIC_KEY; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0; - offset += 1; + int offset = set_command_header_noopt(INS_SECRET_KEY_TO_PUBLIC_KEY); //sec key memmove(this->buffer_send+offset, sec.data, 32); offset += 32; @@ -1322,7 +1131,6 @@ namespace hw { bool device_ledger::generate_key_image(const crypto::public_key &pub, const crypto::secret_key &sec, crypto::key_image &image){ AUTO_LOCK_CMD(); - int offset; #ifdef DEBUG_HWDEVICE const crypto::public_key pub_x = pub; @@ -1334,17 +1142,7 @@ namespace hw { hw::ledger::log_hexbuffer("generate_key_image: [[OUT]] image ", image_x.data, 32); #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_GEN_KEY_IMAGE; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0; - offset += 1; + int offset = set_command_header_noopt(INS_GEN_KEY_IMAGE); //pub memmove(this->buffer_send+offset, pub.data, 32); offset += 32; @@ -1372,20 +1170,10 @@ namespace hw { bool device_ledger::open_tx(crypto::secret_key &tx_key) { AUTO_LOCK_CMD(); - int offset; - reset_buffer(); key_map.clear(); + int offset = set_command_header_noopt(INS_OPEN_TX, 0x01); - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_OPEN_TX; - this->buffer_send[2] = 0x01; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; //account this->buffer_send[offset+0] = 0x00; this->buffer_send[offset+1] = 0x00; @@ -1404,7 +1192,6 @@ namespace hw { bool device_ledger::encrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key) { AUTO_LOCK_CMD(); - int offset; #ifdef DEBUG_HWDEVICE const crypto::public_key public_key_x = public_key; @@ -1413,17 +1200,7 @@ namespace hw { this->controle_device->encrypt_payment_id(payment_id_x, public_key_x, secret_key_x); #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_STEALTH; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; + int offset = set_command_header_noopt(INS_STEALTH); //pub memmove(&this->buffer_send[offset], public_key.data, 32); offset += 32; @@ -1455,7 +1232,6 @@ namespace hw { bool device_ledger::ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & AKout) { AUTO_LOCK_CMD(); - int offset; #ifdef DEBUG_HWDEVICE const rct::key AKout_x = hw::ledger::decrypt(AKout); @@ -1463,17 +1239,7 @@ namespace hw { this->controle_device->ecdhEncode(unmasked_x, AKout_x); #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_BLIND; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; + int offset = set_command_header_noopt(INS_BLIND); // AKout memmove(this->buffer_send+offset, AKout.bytes, 32); offset += 32; @@ -1503,7 +1269,6 @@ namespace hw { bool device_ledger::ecdhDecode(rct::ecdhTuple & masked, const rct::key & AKout) { AUTO_LOCK_CMD(); - int offset; #ifdef DEBUG_HWDEVICE const rct::key AKout_x = hw::ledger::decrypt(AKout); @@ -1511,17 +1276,8 @@ namespace hw { this->controle_device->ecdhDecode(masked_x, AKout_x); #endif - reset_buffer(); + int offset = set_command_header_noopt(INS_UNBLIND); - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_UNBLIND; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; // AKout memmove(this->buffer_send+offset, AKout.bytes, 32); offset += 32; @@ -1571,16 +1327,7 @@ namespace hw { data = blob.data(); // ====== u8 type, varint txnfee ====== - int offset; - - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_VALIDATE; - this->buffer_send[2] = 0x01; - this->buffer_send[3] = 0x01; - this->buffer_send[4] = 0x00; - offset = 5; + int offset = set_command_header(INS_VALIDATE, 0x01, 0x01); //options this->buffer_send[offset] = (inputs_size == 0)?0x00:0x80; offset += 1; @@ -1608,13 +1355,7 @@ namespace hw { //pseudoOuts if ((type == rct::RCTTypeSimple) || (type == rct::RCTTypeSimpleBulletproof)) { for ( i = 0; i < inputs_size; i++) { - reset_buffer(); - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_VALIDATE; - this->buffer_send[2] = 0x01; - this->buffer_send[3] = i+2; - this->buffer_send[4] = 0x00; - offset = 5; + offset = set_command_header(INS_VALIDATE, 0x01, i+2); //options this->buffer_send[offset] = (i==inputs_size-1)? 0x00:0x80; offset += 1; @@ -1641,14 +1382,7 @@ namespace hw { log_hexbuffer("Pout not found", (char*)outPk[i].dest.bytes, 32); CHECK_AND_ASSERT_THROW_MES(found, "Pout not found"); } - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_VALIDATE; - this->buffer_send[2] = 0x02; - this->buffer_send[3] = i+1; - this->buffer_send[4] = 0x00; - offset = 5; + offset = set_command_header(INS_VALIDATE, 0x02, i+1); //options this->buffer_send[offset] = (i==outputs_size-1)? 0x00:0x80 ; offset += 1; @@ -1693,14 +1427,7 @@ namespace hw { // ====== C[], message, proof====== C_offset = kv_offset; for (i = 0; i < outputs_size; i++) { - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_VALIDATE; - this->buffer_send[2] = 0x03; - this->buffer_send[3] = i+1; - this->buffer_send[4] = 0x00; - offset = 5; + offset = set_command_header(INS_VALIDATE, 0x03, i+1); //options this->buffer_send[offset] = 0x80 ; offset += 1; @@ -1715,17 +1442,7 @@ namespace hw { } - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_VALIDATE; - this->buffer_send[2] = 0x03; - this->buffer_send[3] = i+1; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; + offset = set_command_header_noopt(INS_VALIDATE, 0x03, i+1); //message memmove(this->buffer_send+offset, hashes[0].bytes,32); offset += 32; @@ -1750,7 +1467,6 @@ namespace hw { bool device_ledger::mlsag_prepare(const rct::key &H, const rct::key &xx, rct::key &a, rct::key &aG, rct::key &aHP, rct::key &II) { AUTO_LOCK_CMD(); - int offset; unsigned char options; #ifdef DEBUG_HWDEVICE @@ -1762,17 +1478,7 @@ namespace hw { rct::key II_x; #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_MLSAG; - this->buffer_send[2] = 0x01; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; + int offset = set_command_header_noopt(INS_MLSAG, 0x01); //value H memmove(this->buffer_send+offset, H.bytes, 32); offset += 32; @@ -1805,7 +1511,6 @@ namespace hw { bool device_ledger::mlsag_prepare(rct::key &a, rct::key &aG) { AUTO_LOCK_CMD(); - int offset; unsigned char options; #ifdef DEBUG_HWDEVICE @@ -1813,21 +1518,7 @@ namespace hw { rct::key aG_x; #endif - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_MLSAG; - this->buffer_send[2] = 0x01; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; - - this->buffer_send[4] = offset-5; - this->length_send = offset; - this->exchange(); + send_simple(INS_MLSAG, 0x01); memmove(a.bytes, &this->buffer_recv[32*0], 32); memmove(aG.bytes, &this->buffer_recv[32*1], 32); @@ -1843,7 +1534,6 @@ namespace hw { bool device_ledger::mlsag_hash(const rct::keyV &long_message, rct::key &c) { AUTO_LOCK_CMD(); - int offset; unsigned char options; size_t cnt; @@ -1855,14 +1545,7 @@ namespace hw { cnt = long_message.size(); for (size_t i = 0; i<cnt; i++) { - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_MLSAG; - this->buffer_send[2] = 0x02; - this->buffer_send[3] = i+1; - this->buffer_send[4] = 0x00; - offset = 5; + int offset = set_command_header(INS_MLSAG, 0x02, i+1); //options this->buffer_send[offset] = (i==(cnt-1))?0x00:0x80; //last @@ -1887,7 +1570,6 @@ namespace hw { bool device_ledger::mlsag_sign(const rct::key &c, const rct::keyV &xx, const rct::keyV &alpha, const size_t rows, const size_t dsRows, rct::keyV &ss) { AUTO_LOCK_CMD(); - int offset; CHECK_AND_ASSERT_THROW_MES(dsRows<=rows, "dsRows greater than rows"); CHECK_AND_ASSERT_THROW_MES(xx.size() == rows, "xx size does not match rows"); @@ -1905,14 +1587,7 @@ namespace hw { #endif for (size_t j = 0; j < dsRows; j++) { - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_MLSAG; - this->buffer_send[2] = 0x03; - this->buffer_send[3] = j+1; - this->buffer_send[4] = 0x00; - offset = 5; + int offset = set_command_header(INS_MLSAG, 0x03, j+1); //options this->buffer_send[offset] = 0x00; if (j==(dsRows-1)) { @@ -1949,24 +1624,7 @@ namespace hw { bool device_ledger::close_tx() { AUTO_LOCK_CMD(); - int offset; - - reset_buffer(); - - this->buffer_send[0] = 0x00; - this->buffer_send[1] = INS_CLOSE_TX; - this->buffer_send[2] = 0x00; - this->buffer_send[3] = 0x00; - this->buffer_send[4] = 0x00; - offset = 5; - //options - this->buffer_send[offset] = 0x00; - offset += 1; - - this->buffer_send[4] = offset-5; - this->length_send = offset; - this->exchange(); - + send_simple(INS_CLOSE_TX); return true; } diff --git a/src/device/device_ledger.hpp b/src/device/device_ledger.hpp index f1fcaab87..c30a38aca 100644 --- a/src/device/device_ledger.hpp +++ b/src/device/device_ledger.hpp @@ -33,8 +33,13 @@ #include <cstddef> #include <string> #include "device.hpp" +#ifdef WIN32 +#include <winscard.h> +#define MAX_ATR_SIZE 33 +#else #include <PCSC/winscard.h> #include <PCSC/wintypes.h> +#endif #include <boost/thread/mutex.hpp> #include <boost/thread/recursive_mutex.hpp> @@ -97,6 +102,9 @@ namespace hw { void logRESP(void); unsigned int exchange(unsigned int ok=0x9000, unsigned int mask=0xFFFF); void reset_buffer(void); + int set_command_header(BYTE ins, BYTE p1 = 0x00, BYTE p2 = 0x00); + int set_command_header_noopt(BYTE ins, BYTE p1 = 0x00, BYTE p2 = 0x00); + void send_simple(BYTE ins, BYTE p1 = 0x00); // hw running mode device_mode mode; diff --git a/src/device/log.cpp b/src/device/log.cpp index cbbcfc953..1707524fb 100644 --- a/src/device/log.cpp +++ b/src/device/log.cpp @@ -45,13 +45,13 @@ namespace hw { } } - void log_hexbuffer(std::string msg, const char* buff, size_t len) { + void log_hexbuffer(const std::string &msg, const char* buff, size_t len) { char logstr[1025]; buffer_to_str(logstr, sizeof(logstr), buff, len); MDEBUG(msg<< ": " << logstr); } - void log_message(std::string msg, std::string info ) { + void log_message(const std::string &msg, const std::string &info ) { MDEBUG(msg << ": " << info); } @@ -122,16 +122,18 @@ namespace hw { rct::keyV decrypt(const rct::keyV &keys) { rct::keyV x ; + x.reserve(keys.size()); for (unsigned int j = 0; j<keys.size(); j++) { x.push_back(decrypt(keys[j])); } return x; } - static void check(std::string msg, std::string info, const char *h, const char *d, int len, bool crypted) { + static void check(const std::string &msg, const std::string &info, const char *h, const char *d, size_t len, bool crypted) { char dd[32]; char logstr[128]; + CHECK_AND_ASSERT_THROW_MES(len <= sizeof(dd), "invalid len"); memmove(dd,d,len); if (crypted) { CHECK_AND_ASSERT_THROW_MES(len<=32, "encrypted data greater than 32"); @@ -149,11 +151,11 @@ namespace hw { } } - void check32(std::string msg, std::string info, const char *h, const char *d, bool crypted) { + void check32(const std::string &msg, const std::string &info, const char *h, const char *d, bool crypted) { check(msg, info, h, d, 32, crypted); } - void check8(std::string msg, std::string info, const char *h, const char *d, bool crypted) { + void check8(const std::string &msg, const std::string &info, const char *h, const char *d, bool crypted) { check(msg, info, h, d, 8, crypted); } #endif diff --git a/src/device/log.hpp b/src/device/log.hpp index 1ab572c2c..1d1635dc1 100644 --- a/src/device/log.hpp +++ b/src/device/log.hpp @@ -44,8 +44,8 @@ namespace hw { namespace ledger { void buffer_to_str(char *to_buff, size_t to_len, const char *buff, size_t len) ; - void log_hexbuffer(std::string msg, const char* buff, size_t len); - void log_message(std::string msg, std::string info ); + void log_hexbuffer(const std::string &msg, const char* buff, size_t len); + void log_message(const std::string &msg, const std::string &info ); #ifdef DEBUG_HWDEVICE #define TRACK printf("file %s:%d\n",__FILE__, __LINE__) //#define TRACK MCDEBUG("ledger"," At file " << __FILE__ << ":" << __LINE__) @@ -59,8 +59,8 @@ namespace hw { crypto::ec_scalar decrypt(const crypto::ec_scalar &res); rct::keyV decrypt(const rct::keyV &keys); - void check32(std::string msg, std::string info, const char *h, const char *d, bool crypted=false); - void check8(std::string msg, std::string info, const char *h, const char *d, bool crypted=false); + void check32(const std::string &msg, const std::string &info, const char *h, const char *d, bool crypted=false); + void check8(const std::string &msg, const std::string &info, const char *h, const char *d, bool crypted=false); void set_check_verbose(bool verbose); #endif diff --git a/src/mnemonics/esperanto.h b/src/mnemonics/esperanto.h index cb377a58e..a1d1a3f30 100644 --- a/src/mnemonics/esperanto.h +++ b/src/mnemonics/esperanto.h @@ -37,7 +37,7 @@ * Sources:
* Baza Radikaro Oficiala
* Reta Vortaro (http://www.reta-vortaro.de/revo/)
- * Esperanto Panorama - Esperanto-English Dictionary (http://www.esperanto-panorama.net/vortaro/eoen.htm)
+ * Esperanto Panorama - Esperanto-English Dictionary (https://www.esperanto-panorama.net/vortaro/eoen.htm)
* ESPDIC - Paul Denisowski (http://www.denisowski.org/Esperanto/ESPDIC/espdic.txt)
*/
diff --git a/src/mnemonics/lojban.h b/src/mnemonics/lojban.h index 412531a23..0966a1169 100644 --- a/src/mnemonics/lojban.h +++ b/src/mnemonics/lojban.h @@ -35,7 +35,7 @@ /*
* Word list authored by: sorpaas
* Sources:
- * lo gimste jo'u lo ma'oste (http://guskant.github.io/lojbo/gismu-cmavo.html)
+ * lo gimste jo'u lo ma'oste (https://guskant.github.io/lojbo/gismu-cmavo.html)
* N-grams of Lojban corpus (https://mw.lojban.org/papri/N-grams_of_Lojban_corpus)
*/
diff --git a/src/p2p/CMakeLists.txt b/src/p2p/CMakeLists.txt index 83bdffab5..9b924907e 100644 --- a/src/p2p/CMakeLists.txt +++ b/src/p2p/CMakeLists.txt @@ -38,7 +38,6 @@ source_group(p2p FILES ${P2P}) monero_add_library(p2p ${P2P}) target_link_libraries(p2p PUBLIC - epee version cryptonote_core ${UPNP_LIBRARIES} diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index d5346afb5..90e2f78b1 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -244,6 +244,7 @@ namespace nodetool bool check_connection_and_handshake_with_peer(const epee::net_utils::network_address& na, uint64_t last_seen_stamp); bool gray_peerlist_housekeeping(); + bool check_incoming_connections(); void kill() { ///< will be called e.g. from deinit() _info("Killing the net_node"); @@ -307,6 +308,7 @@ namespace nodetool epee::math_helper::once_a_time_seconds<1> m_connections_maker_interval; epee::math_helper::once_a_time_seconds<60*30, false> m_peerlist_store_interval; epee::math_helper::once_a_time_seconds<60> m_gray_peerlist_housekeeping_interval; + epee::math_helper::once_a_time_seconds<900, false> m_incoming_connections_interval; std::string m_bind_ip; std::string m_port; diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index 07f369d40..9390626a8 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -62,6 +62,7 @@ namespace nodetool { + inline bool append_net_address(std::vector<epee::net_utils::network_address> & seed_nodes, std::string const & addr, uint16_t default_port); //----------------------------------------------------------------------------------- template<class t_payload_net_handler> void node_server<t_payload_net_handler>::init_options(boost::program_options::options_description& desc) @@ -273,10 +274,22 @@ namespace nodetool { nodetool::peerlist_entry pe = AUTO_VAL_INIT(pe); pe.id = crypto::rand<uint64_t>(); - const uint16_t default_port = testnet ? ::config::testnet::P2P_DEFAULT_PORT : stagenet ? ::config::stagenet::P2P_DEFAULT_PORT : ::config::P2P_DEFAULT_PORT; + const uint16_t default_port = cryptonote::get_config(m_nettype).P2P_DEFAULT_PORT; bool r = parse_peer_from_string(pe.adr, pr_str, default_port); - CHECK_AND_ASSERT_MES(r, false, "Failed to parse address from string: " << pr_str); - m_command_line_peers.push_back(pe); + if (r) + { + m_command_line_peers.push_back(pe); + continue; + } + std::vector<epee::net_utils::network_address> resolved_addrs; + r = append_net_address(resolved_addrs, pr_str, default_port); + CHECK_AND_ASSERT_MES(r, false, "Failed to parse or resolve address from string: " << pr_str); + for (const epee::net_utils::network_address& addr : resolved_addrs) + { + pe.id = crypto::rand<uint64_t>(); + pe.adr = addr; + m_command_line_peers.push_back(pe); + } } } @@ -327,24 +340,31 @@ namespace nodetool return true; } //----------------------------------------------------------------------------------- - inline void append_net_address( + inline bool append_net_address( std::vector<epee::net_utils::network_address> & seed_nodes , std::string const & addr + , uint16_t default_port ) { using namespace boost::asio; + std::string host = addr; + std::string port = std::to_string(default_port); size_t pos = addr.find_last_of(':'); - CHECK_AND_ASSERT_MES_NO_RET(std::string::npos != pos && addr.length() - 1 != pos && 0 != pos, "Failed to parse seed address from string: '" << addr << '\''); - std::string host = addr.substr(0, pos); - std::string port = addr.substr(pos + 1); + if (std::string::npos != pos) + { + CHECK_AND_ASSERT_MES(addr.length() - 1 != pos && 0 != pos, false, "Failed to parse seed address from string: '" << addr << '\''); + host = addr.substr(0, pos); + port = addr.substr(pos + 1); + } + MINFO("Resolving node address: host=" << host << ", port=" << port); io_service io_srv; ip::tcp::resolver resolver(io_srv); ip::tcp::resolver::query query(host, port, boost::asio::ip::tcp::resolver::query::canonical_name); boost::system::error_code ec; ip::tcp::resolver::iterator i = resolver.resolve(query, ec); - CHECK_AND_ASSERT_MES_NO_RET(!ec, "Failed to resolve host name '" << host << "': " << ec.message() << ':' << ec.value()); + CHECK_AND_ASSERT_MES(!ec, false, "Failed to resolve host name '" << host << "': " << ec.message() << ':' << ec.value()); ip::tcp::resolver::iterator iend; for (; i != iend; ++i) @@ -354,14 +374,14 @@ namespace nodetool { epee::net_utils::network_address na{epee::net_utils::ipv4_network_address{boost::asio::detail::socket_ops::host_to_network_long(endpoint.address().to_v4().to_ulong()), endpoint.port()}}; seed_nodes.push_back(na); - MINFO("Added seed node: " << na.str()); + MINFO("Added node: " << na.str()); } else { MWARNING("IPv6 unsupported, skip '" << host << "' -> " << endpoint.address().to_v6().to_string(ec)); - throw std::runtime_error("IPv6 unsupported"); } } + return true; } //----------------------------------------------------------------------------------- @@ -382,6 +402,9 @@ namespace nodetool full_addrs.insert("162.210.173.150:38080"); full_addrs.insert("162.210.173.151:38080"); } + else if (nettype == cryptonote::FAKECHAIN) + { + } else { full_addrs.insert("107.152.130.98:18080"); @@ -484,7 +507,7 @@ namespace nodetool if (result.size()) { for (const auto& addr_string : result) - full_addrs.insert(addr_string + ":" + std::to_string(m_nettype == cryptonote::TESTNET ? ::config::testnet::P2P_DEFAULT_PORT : m_nettype == cryptonote::STAGENET ? ::config::stagenet::P2P_DEFAULT_PORT : ::config::P2P_DEFAULT_PORT)); + full_addrs.insert(addr_string + ":" + std::to_string(cryptonote::get_config(m_nettype).P2P_DEFAULT_PORT)); } ++i; } @@ -507,7 +530,7 @@ namespace nodetool for (const auto& full_addr : full_addrs) { MDEBUG("Seed node: " << full_addr); - append_net_address(m_seed_nodes, full_addr); + append_net_address(m_seed_nodes, full_addr, cryptonote::get_config(m_nettype).P2P_DEFAULT_PORT); } MDEBUG("Number of seed nodes: " << m_seed_nodes.size()); @@ -1152,7 +1175,7 @@ namespace nodetool for (const auto &peer: get_seed_nodes(m_nettype)) { MDEBUG("Fallback seed node: " << peer); - append_net_address(m_seed_nodes, peer); + append_net_address(m_seed_nodes, peer, cryptonote::get_config(m_nettype).P2P_DEFAULT_PORT); } m_fallback_seed_nodes_added = true; if (current_index == m_seed_nodes.size()) @@ -1300,6 +1323,20 @@ namespace nodetool m_connections_maker_interval.do_call(boost::bind(&node_server<t_payload_net_handler>::connections_maker, this)); m_gray_peerlist_housekeeping_interval.do_call(boost::bind(&node_server<t_payload_net_handler>::gray_peerlist_housekeeping, this)); m_peerlist_store_interval.do_call(boost::bind(&node_server<t_payload_net_handler>::store_config, this)); + m_incoming_connections_interval.do_call(boost::bind(&node_server<t_payload_net_handler>::check_incoming_connections, this)); + return true; + } + //----------------------------------------------------------------------------------- + template<class t_payload_net_handler> + bool node_server<t_payload_net_handler>::check_incoming_connections() + { + if (m_offline || m_hide_my_port) + return true; + if (get_incoming_connections_count() == 0) + { + const el::Level level = el::Level::Warning; + MCLOG_RED(level, "global", "No incoming connections - check firewalls/routers allow port " << get_this_peer_port()); + } return true; } //----------------------------------------------------------------------------------- @@ -1814,10 +1851,20 @@ namespace nodetool for(const std::string& pr_str: perrs) { epee::net_utils::network_address na = AUTO_VAL_INIT(na); - const uint16_t default_port = m_nettype == cryptonote::TESTNET ? ::config::testnet::P2P_DEFAULT_PORT : m_nettype == cryptonote::STAGENET ? ::config::stagenet::P2P_DEFAULT_PORT : ::config::P2P_DEFAULT_PORT; + const uint16_t default_port = cryptonote::get_config(m_nettype).P2P_DEFAULT_PORT; bool r = parse_peer_from_string(na, pr_str, default_port); - CHECK_AND_ASSERT_MES(r, false, "Failed to parse address from string: " << pr_str); - container.push_back(na); + if (r) + { + container.push_back(na); + continue; + } + std::vector<epee::net_utils::network_address> resolved_addrs; + r = append_net_address(resolved_addrs, pr_str, default_port); + CHECK_AND_ASSERT_MES(r, false, "Failed to parse or resolve address from string: " << pr_str); + for (const epee::net_utils::network_address& addr : resolved_addrs) + { + container.push_back(addr); + } } return true; diff --git a/src/ringct/rctOps.cpp b/src/ringct/rctOps.cpp index cc46d0aa7..50693bad7 100644 --- a/src/ringct/rctOps.cpp +++ b/src/ringct/rctOps.cpp @@ -62,14 +62,13 @@ namespace rct { //generates a random scalar which can be used as a secret key or mask void skGen(key &sk) { - sk = crypto::rand<key>(); - sc_reduce32(sk.bytes); + random32_unbiased(sk.bytes); } //generates a random scalar which can be used as a secret key or mask key skGen() { - key sk = crypto::rand<key>(); - sc_reduce32(sk.bytes); + key sk; + skGen(sk); return sk; } @@ -79,9 +78,8 @@ namespace rct { CHECK_AND_ASSERT_THROW_MES(rows > 0, "0 keys requested"); keyV rv(rows); size_t i = 0; - crypto::rand(rows * sizeof(key), (uint8_t*)&rv[0]); for (i = 0 ; i < rows ; i++) { - sc_reduce32(rv[i].bytes); + skGen(rv[i]); } return rv; } @@ -134,12 +132,9 @@ namespace rct { } key zeroCommit(xmr_amount amount) { - key mask = identity(); - mask = scalarmultBase(mask); key am = d2h(amount); key bH = scalarmultH(am); - addKeys(mask, mask, bH); - return mask; + return addKeys(G, bH); } key commit(xmr_amount amount, const key &mask) { diff --git a/src/ringct/rctOps.h b/src/ringct/rctOps.h index 3f8f6955c..f8889af5c 100644 --- a/src/ringct/rctOps.h +++ b/src/ringct/rctOps.h @@ -62,6 +62,7 @@ namespace rct { static const key Z = { {0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 } }; static const key I = { {0x01, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 } }; static const key L = { {0xed, 0xd3, 0xf5, 0x5c, 0x1a, 0x63, 0x12, 0x58, 0xd6, 0x9c, 0xf7, 0xa2, 0xde, 0xf9, 0xde, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10 } }; + static const key G = { {0x58, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66 } }; //Creates a zero scalar inline key zero() { return Z; } diff --git a/src/ringct/rctSigs.cpp b/src/ringct/rctSigs.cpp index 777b4d13a..f74216ed4 100644 --- a/src/ringct/rctSigs.cpp +++ b/src/ringct/rctSigs.cpp @@ -147,7 +147,7 @@ namespace rct { //This is a just slghtly more efficient version than the ones described below //(will be explained in more detail in Ring Multisig paper //These are aka MG signatutes in earlier drafts of the ring ct paper - // c.f. http://eprint.iacr.org/2015/1098 section 2. + // c.f. https://eprint.iacr.org/2015/1098 section 2. // Gen creates a signature which proves that for some column in the keymatrix "pk" // the signer knows a secret key for each row in that column // Ver verifies that the MG sig was created correctly @@ -244,7 +244,7 @@ namespace rct { //This is a just slghtly more efficient version than the ones described below //(will be explained in more detail in Ring Multisig paper //These are aka MG signatutes in earlier drafts of the ring ct paper - // c.f. http://eprint.iacr.org/2015/1098 section 2. + // c.f. https://eprint.iacr.org/2015/1098 section 2. // Gen creates a signature which proves that for some column in the keymatrix "pk" // the signer knows a secret key for each row in that column // Ver verifies that the MG sig was created correctly @@ -307,7 +307,7 @@ namespace rct { //proveRange and verRange //proveRange gives C, and mask such that \sumCi = C - // c.f. http://eprint.iacr.org/2015/1098 section 5.1 + // c.f. https://eprint.iacr.org/2015/1098 section 5.1 // and Ci is a commitment to either 0 or 2^i, i=0,...,63 // thus this proves that "amount" is in [0, 2^64] // mask is a such that C = aG + bH, and b = amount @@ -339,7 +339,7 @@ namespace rct { //proveRange and verRange //proveRange gives C, and mask such that \sumCi = C - // c.f. http://eprint.iacr.org/2015/1098 section 5.1 + // c.f. https://eprint.iacr.org/2015/1098 section 5.1 // and Ci is a commitment to either 0 or 2^i, i=0,...,63 // thus this proves that "amount" is in [0, 2^64] // mask is a such that C = aG + bH, and b = amount @@ -441,7 +441,7 @@ namespace rct { //Ring-ct MG sigs //Prove: - // c.f. http://eprint.iacr.org/2015/1098 section 4. definition 10. + // c.f. https://eprint.iacr.org/2015/1098 section 4. definition 10. // This does the MG sig on the "dest" part of the given key matrix, and // the last row is the sum of input commitments from that column - sum output commitments // this shows that sum inputs = sum outputs @@ -527,7 +527,7 @@ namespace rct { //Ring-ct MG sigs //Prove: - // c.f. http://eprint.iacr.org/2015/1098 section 4. definition 10. + // c.f. https://eprint.iacr.org/2015/1098 section 4. definition 10. // This does the MG sig on the "dest" part of the given key matrix, and // the last row is the sum of input commitments from that column - sum output commitments // this shows that sum inputs = sum outputs @@ -650,7 +650,7 @@ namespace rct { // Also contains masked "amount" and "mask" so the receiver can see how much they received //verRct: // verifies that all signatures (rangeProogs, MG sig, sum inputs = outputs) are correct - //decodeRct: (c.f. http://eprint.iacr.org/2015/1098 section 5.1.1) + //decodeRct: (c.f. https://eprint.iacr.org/2015/1098 section 5.1.1) // uses the attached ecdh info to find the amounts represented by each output commitment // must know the destination private key to find the correct amount, else will return a random number // Note: For txn fees, the last index in the amounts vector should contain that @@ -828,7 +828,7 @@ namespace rct { // Also contains masked "amount" and "mask" so the receiver can see how much they received //verRct: // verifies that all signatures (rangeProogs, MG sig, sum inputs = outputs) are correct - //decodeRct: (c.f. http://eprint.iacr.org/2015/1098 section 5.1.1) + //decodeRct: (c.f. https://eprint.iacr.org/2015/1098 section 5.1.1) // uses the attached ecdh info to find the amounts represented by each output commitment // must know the destination private key to find the correct amount, else will return a random number bool verRct(const rctSig & rv, bool semantics) { @@ -862,9 +862,9 @@ namespace rct { results[i] = verBulletproof(rv.p.bulletproofs[i]); else results[i] = verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]); - }); + }, true); } - waiter.wait(); + waiter.wait(&tpool); for (size_t i = 0; i < rv.outPk.size(); ++i) { if (!results[i]) { @@ -970,9 +970,9 @@ namespace rct { results[i] = verBulletproof(rv.p.bulletproofs[i]); else results[i] = verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]); - }); + }, true); } - waiter.wait(); + waiter.wait(&tpool); for (size_t i = 0; i < results.size(); ++i) { if (!results[i]) { @@ -989,9 +989,9 @@ namespace rct { for (size_t i = 0 ; i < rv.mixRing.size() ; i++) { tpool.submit(&waiter, [&, i] { results[i] = verRctMGSimple(message, rv.p.MGs[i], rv.mixRing[i], pseudoOuts[i]); - }); + }, true); } - waiter.wait(); + waiter.wait(&tpool); for (size_t i = 0; i < results.size(); ++i) { if (!results[i]) { @@ -1023,7 +1023,7 @@ namespace rct { // Also contains masked "amount" and "mask" so the receiver can see how much they received //verRct: // verifies that all signatures (rangeProogs, MG sig, sum inputs = outputs) are correct - //decodeRct: (c.f. http://eprint.iacr.org/2015/1098 section 5.1.1) + //decodeRct: (c.f. https://eprint.iacr.org/2015/1098 section 5.1.1) // uses the attached ecdh info to find the amounts represented by each output commitment // must know the destination private key to find the correct amount, else will return a random number xmr_amount decodeRct(const rctSig & rv, const key & sk, unsigned int i, key & mask, hw::device &hwdev) { diff --git a/src/ringct/rctSigs.h b/src/ringct/rctSigs.h index b8aab0f11..5a9b2dd44 100644 --- a/src/ringct/rctSigs.h +++ b/src/ringct/rctSigs.h @@ -70,7 +70,7 @@ namespace rct { //Multilayered Spontaneous Anonymous Group Signatures (MLSAG signatures) //These are aka MG signatutes in earlier drafts of the ring ct paper - // c.f. http://eprint.iacr.org/2015/1098 section 2. + // c.f. https://eprint.iacr.org/2015/1098 section 2. // Gen creates a signature which proves that for some column in the keymatrix "pk" // the signer knows a secret key for each row in that column // Ver verifies that the MG sig was created correctly @@ -80,7 +80,7 @@ namespace rct { //proveRange and verRange //proveRange gives C, and mask such that \sumCi = C - // c.f. http://eprint.iacr.org/2015/1098 section 5.1 + // c.f. https://eprint.iacr.org/2015/1098 section 5.1 // and Ci is a commitment to either 0 or 2^i, i=0,...,63 // thus this proves that "amount" is in [0, 2^64] // mask is a such that C = aG + bH, and b = amount @@ -90,7 +90,7 @@ namespace rct { //Ring-ct MG sigs //Prove: - // c.f. http://eprint.iacr.org/2015/1098 section 4. definition 10. + // c.f. https://eprint.iacr.org/2015/1098 section 4. definition 10. // This does the MG sig on the "dest" part of the given key matrix, and // the last row is the sum of input commitments from that column - sum output commitments // this shows that sum inputs = sum outputs @@ -116,7 +116,7 @@ namespace rct { // Also contains masked "amount" and "mask" so the receiver can see how much they received //verRct: // verifies that all signatures (rangeProogs, MG sig, sum inputs = outputs) are correct - //decodeRct: (c.f. http://eprint.iacr.org/2015/1098 section 5.1.1) + //decodeRct: (c.f. https://eprint.iacr.org/2015/1098 section 5.1.1) // uses the attached ecdh info to find the amounts represented by each output commitment // must know the destination private key to find the correct amount, else will return a random number rctSig genRct(const key &message, const ctkeyV & inSk, const keyV & destinations, const std::vector<xmr_amount> & amounts, const ctkeyM &mixRing, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, unsigned int index, ctkeyV &outSk, bool bulletproof, hw::device &hwdev); diff --git a/src/ringct/rctTypes.h b/src/ringct/rctTypes.h index eba1e3d93..844291d0c 100644 --- a/src/ringct/rctTypes.h +++ b/src/ringct/rctTypes.h @@ -150,7 +150,7 @@ namespace rct { }; //just contains the necessary keys to represent MLSAG sigs - //c.f. http://eprint.iacr.org/2015/1098 + //c.f. https://eprint.iacr.org/2015/1098 struct mgSig { keyM ss; key cc; diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt index 7162317ed..4f8f96524 100644 --- a/src/rpc/CMakeLists.txt +++ b/src/rpc/CMakeLists.txt @@ -111,7 +111,6 @@ target_link_libraries(rpc common cryptonote_core cryptonote_protocol - epee ${Boost_REGEX_LIBRARY} ${Boost_THREAD_LIBRARY} PRIVATE diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index dc7b6b30f..0a6daf8f0 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -47,7 +47,6 @@ using namespace epee; #include "rpc/rpc_args.h" #include "core_rpc_server_error_codes.h" #include "p2p/net_node.h" -#include "get_output_distribution_cache.h" #include "version.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -194,6 +193,7 @@ namespace cryptonote res.mainnet = m_nettype == MAINNET; res.testnet = m_nettype == TESTNET; res.stagenet = m_nettype == STAGENET; + res.nettype = m_nettype == MAINNET ? "mainnet" : m_nettype == TESTNET ? "testnet" : m_nettype == STAGENET ? "stagenet" : "fakechain"; res.cumulative_difficulty = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.height - 1); res.block_size_limit = m_core.get_blockchain_storage().get_current_cumulative_blocksize_limit(); res.block_size_median = m_core.get_blockchain_storage().get_current_cumulative_blocksize_median(); @@ -207,6 +207,7 @@ namespace cryptonote boost::shared_lock<boost::shared_mutex> lock(m_bootstrap_daemon_mutex); res.was_bootstrap_ever_used = m_was_bootstrap_ever_used; } + res.database_size = m_core.get_blockchain_storage().get_db().get_database_size(); return true; } //------------------------------------------------------------------------------------------------------------------------------ @@ -226,47 +227,47 @@ namespace cryptonote if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_BLOCKS_FAST>(invoke_http_mode::BIN, "/getblocks.bin", req, res, r)) return r; - std::list<std::pair<cryptonote::blobdata, std::list<cryptonote::blobdata> > > bs; + std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > > bs; - if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, bs, res.current_height, res.start_height, req.prune, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT)) + if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, bs, res.current_height, res.start_height, req.prune, !req.no_miner_tx, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT)) { res.status = "Failed"; return false; } size_t pruned_size = 0, unpruned_size = 0, ntxes = 0; + res.blocks.reserve(bs.size()); + res.output_indices.reserve(bs.size()); for(auto& bd: bs) { res.blocks.resize(res.blocks.size()+1); - res.blocks.back().block = bd.first; - pruned_size += bd.first.size(); - unpruned_size += bd.first.size(); + res.blocks.back().block = bd.first.first; + pruned_size += bd.first.first.size(); + unpruned_size += bd.first.first.size(); res.output_indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices()); res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices()); - block b; - if (!parse_and_validate_block_from_blob(bd.first, b)) + if (!req.no_miner_tx) { - res.status = "Invalid block"; - return false; - } - bool r = m_core.get_tx_outputs_gindexs(get_transaction_hash(b.miner_tx), res.output_indices.back().indices.back().indices); - if (!r) - { - res.status = "Failed"; - return false; + bool r = m_core.get_tx_outputs_gindexs(bd.first.second, res.output_indices.back().indices.back().indices); + if (!r) + { + res.status = "Failed"; + return false; + } } - size_t txidx = 0; ntxes += bd.second.size(); - for (std::list<cryptonote::blobdata>::iterator i = bd.second.begin(); i != bd.second.end(); ++i) + res.blocks.back().txs.reserve(bd.second.size()); + res.output_indices.back().indices.reserve(bd.second.size()); + for (std::vector<std::pair<crypto::hash, cryptonote::blobdata>>::iterator i = bd.second.begin(); i != bd.second.end(); ++i) { - unpruned_size += i->size(); - res.blocks.back().txs.push_back(std::move(*i)); - i->clear(); - i->shrink_to_fit(); + unpruned_size += i->second.size(); + res.blocks.back().txs.push_back(std::move(i->second)); + i->second.clear(); + i->second.shrink_to_fit(); pruned_size += res.blocks.back().txs.back().size(); res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices()); - bool r = m_core.get_tx_outputs_gindexs(b.tx_hashes[txidx++], res.output_indices.back().indices.back().indices); + bool r = m_core.get_tx_outputs_gindexs(i->first, res.output_indices.back().indices.back().indices); if (!r) { res.status = "Failed"; @@ -286,7 +287,7 @@ namespace cryptonote if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_ALT_BLOCKS_HASHES>(invoke_http_mode::JON, "/get_alt_blocks_hashes", req, res, r)) return r; - std::list<block> blks; + std::vector<block> blks; if(!m_core.get_alternative_blocks(blks)) { @@ -328,8 +329,8 @@ namespace cryptonote res.status = "Error retrieving block at height " + std::to_string(height); return true; } - std::list<transaction> txs; - std::list<crypto::hash> missed_txs; + std::vector<transaction> txs; + std::vector<crypto::hash> missed_txs; m_core.get_transactions(blk.tx_hashes, txs, missed_txs); res.blocks.resize(res.blocks.size() + 1); res.blocks.back().block = block_to_blob(blk); @@ -544,8 +545,8 @@ namespace cryptonote } vh.push_back(*reinterpret_cast<const crypto::hash*>(b.data())); } - std::list<crypto::hash> missed_txs; - std::list<transaction> txs; + std::vector<crypto::hash> missed_txs; + std::vector<transaction> txs; bool r = m_core.get_transactions(vh, txs, missed_txs); if(!r) { @@ -566,25 +567,26 @@ namespace cryptonote if(r) { // sort to match original request - std::list<transaction> sorted_txs; + std::vector<transaction> sorted_txs; std::vector<tx_info>::const_iterator i; + unsigned txs_processed = 0; for (const crypto::hash &h: vh) { if (std::find(missed_txs.begin(), missed_txs.end(), h) == missed_txs.end()) { - if (txs.empty()) + if (txs.size() == txs_processed) { res.status = "Failed: internal error - txs is empty"; return true; } // core returns the ones it finds in the right order - if (get_transaction_hash(txs.front()) != h) + if (get_transaction_hash(txs[txs_processed]) != h) { res.status = "Failed: tx hash mismatch"; return true; } - sorted_txs.push_back(std::move(txs.front())); - txs.pop_front(); + sorted_txs.push_back(std::move(txs[txs_processed])); + ++txs_processed; } else if ((i = std::find_if(pool_tx_info.begin(), pool_tx_info.end(), [h](const tx_info &txi) { return epee::string_tools::pod_to_hex(h) == txi.id_hash; })) != pool_tx_info.end()) { @@ -595,7 +597,7 @@ namespace cryptonote return true; } sorted_txs.push_back(tx); - missed_txs.remove(h); + missed_txs.erase(std::find(missed_txs.begin(), missed_txs.end(), h)); pool_tx_hashes.insert(h); const std::string hash_string = epee::string_tools::pod_to_hex(h); for (const auto &ti: pool_tx_info) @@ -614,7 +616,7 @@ namespace cryptonote LOG_PRINT_L2("Found " << found_in_pool << "/" << vh.size() << " transactions in the pool"); } - std::list<std::string>::const_iterator txhi = req.txs_hashes.begin(); + std::vector<std::string>::const_iterator txhi = req.txs_hashes.begin(); std::vector<crypto::hash>::const_iterator vhi = vh.begin(); for(auto& tx: txs) { @@ -981,11 +983,11 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::on_get_transaction_pool_hashes(const COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::response& res, bool request_has_rpc_origin) + bool core_rpc_server::on_get_transaction_pool_hashes_bin(const COMMAND_RPC_GET_TRANSACTION_POOL_HASHES_BIN::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES_BIN::response& res, bool request_has_rpc_origin) { PERF_TIMER(on_get_transaction_pool_hashes); bool r; - if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_TRANSACTION_POOL_HASHES>(invoke_http_mode::JON, "/get_transaction_pool_hashes.bin", req, res, r)) + if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_TRANSACTION_POOL_HASHES_BIN>(invoke_http_mode::JON, "/get_transaction_pool_hashes.bin", req, res, r)) return r; m_core.get_pool_transaction_hashes(res.tx_hashes, !request_has_rpc_origin || !m_restricted); @@ -993,6 +995,22 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_get_transaction_pool_hashes(const COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::response& res, bool request_has_rpc_origin) + { + PERF_TIMER(on_get_transaction_pool_hashes); + bool r; + if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_TRANSACTION_POOL_HASHES>(invoke_http_mode::JON, "/get_transaction_pool_hashes", req, res, r)) + return r; + + std::vector<crypto::hash> tx_hashes; + m_core.get_pool_transaction_hashes(tx_hashes, !request_has_rpc_origin || !m_restricted); + res.tx_hashes.reserve(tx_hashes.size()); + for (const crypto::hash &tx_hash: tx_hashes) + res.tx_hashes.push_back(epee::string_tools::pod_to_hex(tx_hash)); + res.status = CORE_RPC_STATUS_OK; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_get_transaction_pool_stats(const COMMAND_RPC_GET_TRANSACTION_POOL_STATS::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_STATS::response& res, bool request_has_rpc_origin) { PERF_TIMER(on_get_transaction_pool_stats); @@ -1209,6 +1227,68 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_generateblocks(const COMMAND_RPC_GENERATEBLOCKS::request& req, COMMAND_RPC_GENERATEBLOCKS::response& res, epee::json_rpc::error& error_resp) + { + PERF_TIMER(on_generateblocks); + + CHECK_CORE_READY(); + + res.status = CORE_RPC_STATUS_OK; + + if(m_core.get_nettype() != FAKECHAIN) + { + error_resp.code = CORE_RPC_ERROR_CODE_REGTEST_REQUIRED; + error_resp.message = "Regtest required when generating blocks"; + return false; + } + + COMMAND_RPC_GETBLOCKTEMPLATE::request template_req; + COMMAND_RPC_GETBLOCKTEMPLATE::response template_res; + COMMAND_RPC_SUBMITBLOCK::request submit_req; + COMMAND_RPC_SUBMITBLOCK::response submit_res; + + template_req.reserve_size = 1; + template_req.wallet_address = req.wallet_address; + submit_req.push_back(boost::value_initialized<std::string>()); + res.height = m_core.get_blockchain_storage().get_current_blockchain_height(); + + bool r; + + for(size_t i = 0; i < req.amount_of_blocks; i++) + { + r = on_getblocktemplate(template_req, template_res, error_resp); + res.status = template_res.status; + + if (!r) return false; + + blobdata blockblob; + if(!string_tools::parse_hexstr_to_binbuff(template_res.blocktemplate_blob, blockblob)) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_BLOCKBLOB; + error_resp.message = "Wrong block blob"; + return false; + } + block b = AUTO_VAL_INIT(b); + if(!parse_and_validate_block_from_blob(blockblob, b)) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_BLOCKBLOB; + error_resp.message = "Wrong block blob"; + return false; + } + miner::find_nonce_for_given_block(b, template_res.difficulty, template_res.height); + + submit_req.front() = string_tools::buff_to_hex_nodelimer(block_to_blob(b)); + r = on_submitblock(submit_req, submit_res, error_resp); + res.status = submit_res.status; + + if (!r) return false; + + res.height = template_res.height; + } + + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ uint64_t core_rpc_server::get_block_reward(const block& blk) { uint64_t reward = 0; @@ -1564,6 +1644,7 @@ namespace cryptonote res.mainnet = m_nettype == MAINNET; res.testnet = m_nettype == TESTNET; res.stagenet = m_nettype == STAGENET; + res.nettype = m_nettype == MAINNET ? "mainnet" : m_nettype == TESTNET ? "testnet" : m_nettype == STAGENET ? "stagenet" : "fakechain"; res.cumulative_difficulty = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.height - 1); res.block_size_limit = m_core.get_blockchain_storage().get_current_cumulative_blocksize_limit(); res.block_size_median = m_core.get_blockchain_storage().get_current_cumulative_blocksize_median(); @@ -1577,6 +1658,7 @@ namespace cryptonote boost::shared_lock<boost::shared_mutex> lock(m_bootstrap_daemon_mutex); res.was_bootstrap_ever_used = m_was_bootstrap_ever_used; } + res.database_size = m_core.get_blockchain_storage().get_db().get_database_size(); return true; } //------------------------------------------------------------------------------------------------------------------------------ @@ -1655,10 +1737,10 @@ namespace cryptonote PERF_TIMER(on_flush_txpool); bool failed = false; - std::list<crypto::hash> txids; + std::vector<crypto::hash> txids; if (req.txids.empty()) { - std::list<transaction> pool_txs; + std::vector<transaction> pool_txs; bool r = m_core.get_pool_transactions(pool_txs); if (!r) { @@ -1774,10 +1856,22 @@ namespace cryptonote PERF_TIMER(on_get_alternate_chains); try { - std::list<std::pair<Blockchain::block_extended_info, uint64_t>> chains = m_core.get_blockchain_storage().get_alternative_chains(); + std::list<std::pair<Blockchain::block_extended_info, std::vector<crypto::hash>>> chains = m_core.get_blockchain_storage().get_alternative_chains(); for (const auto &i: chains) { - res.chains.push_back(COMMAND_RPC_GET_ALTERNATE_CHAINS::chain_info{epee::string_tools::pod_to_hex(get_block_hash(i.first.bl)), i.first.height, i.second, i.first.cumulative_difficulty}); + res.chains.push_back(COMMAND_RPC_GET_ALTERNATE_CHAINS::chain_info{epee::string_tools::pod_to_hex(get_block_hash(i.first.bl)), i.first.height, i.second.size(), i.first.cumulative_difficulty, {}, std::string()}); + res.chains.back().block_hashes.reserve(i.second.size()); + for (const crypto::hash &block_id: i.second) + res.chains.back().block_hashes.push_back(epee::string_tools::pod_to_hex(block_id)); + if (i.first.height < i.second.size()) + { + res.status = "Error finding alternate chain attachment point"; + return true; + } + cryptonote::block main_chain_parent_block; + try { main_chain_parent_block = m_core.get_blockchain_storage().get_db().get_block_from_height(i.first.height - i.second.size()); } + catch (const std::exception &e) { res.status = "Error finding alternate chain attachment point"; return true; } + res.chains.back().main_chain_parent_block = epee::string_tools::pod_to_hex(get_block_hash(main_chain_parent_block)); } res.status = CORE_RPC_STATUS_OK; } @@ -2070,6 +2164,10 @@ namespace cryptonote bool core_rpc_server::on_get_output_distribution(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res, epee::json_rpc::error& error_resp) { PERF_TIMER(on_get_output_distribution); + bool r; + if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_OUTPUT_DISTRIBUTION>(invoke_http_mode::JON_RPC, "get_output_distribution", req, res, r)) + return r; + try { for (uint64_t amount: req.amounts) @@ -2086,38 +2184,17 @@ namespace cryptonote if (d.cached && amount == 0 && d.cached_from == req.from_height && d.cached_to == req.to_height) { - res.distributions.push_back({amount, d.cached_start_height, d.cached_distribution, d.cached_base}); - if (req.cumulative) + res.distributions.push_back({amount, d.cached_start_height, req.binary, d.cached_distribution, d.cached_base}); + if (!req.cumulative) { auto &distribution = res.distributions.back().distribution; - distribution[0] += d.cached_base; - for (size_t n = 1; n < distribution.size(); ++n) - distribution[n] += distribution[n-1]; + for (size_t n = distribution.size() - 1; n > 0; --n) + distribution[n] -= distribution[n-1]; + distribution[0] -= d.cached_base; } continue; } - // this is a slow operation, so we have precomputed caches of common cases - bool found = false; - for (const auto &slot: get_output_distribution_cache) - { - if (slot.amount == amount && slot.from_height == req.from_height && slot.to_height == req.to_height) - { - res.distributions.push_back({amount, slot.start_height, slot.distribution, slot.base}); - found = true; - if (req.cumulative) - { - auto &distribution = res.distributions.back().distribution; - distribution[0] += slot.base; - for (size_t n = 1; n < distribution.size(); ++n) - distribution[n] += distribution[n-1]; - } - break; - } - } - if (found) - continue; - std::vector<uint64_t> distribution; uint64_t start_height, base; if (!m_core.get_output_distribution(amount, req.from_height, req.to_height, start_height, distribution, base)) @@ -2143,14 +2220,14 @@ namespace cryptonote d.cached = true; } - if (req.cumulative) + if (!req.cumulative) { - distribution[0] += base; - for (size_t n = 1; n < distribution.size(); ++n) - distribution[n] += distribution[n-1]; + for (size_t n = distribution.size() - 1; n > 0; --n) + distribution[n] -= distribution[n-1]; + distribution[0] -= base; } - res.distributions.push_back({amount, start_height, std::move(distribution), base}); + res.distributions.push_back({amount, start_height, req.binary, std::move(distribution), base}); } } catch (const std::exception &e) diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index 324f219f8..166020d01 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -107,7 +107,8 @@ namespace cryptonote MAP_URI_AUTO_JON2_IF("/set_log_level", on_set_log_level, COMMAND_RPC_SET_LOG_LEVEL, !m_restricted) MAP_URI_AUTO_JON2_IF("/set_log_categories", on_set_log_categories, COMMAND_RPC_SET_LOG_CATEGORIES, !m_restricted) MAP_URI_AUTO_JON2("/get_transaction_pool", on_get_transaction_pool, COMMAND_RPC_GET_TRANSACTION_POOL) - MAP_URI_AUTO_JON2("/get_transaction_pool_hashes.bin", on_get_transaction_pool_hashes, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES) + MAP_URI_AUTO_JON2("/get_transaction_pool_hashes.bin", on_get_transaction_pool_hashes_bin, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES_BIN) + MAP_URI_AUTO_JON2("/get_transaction_pool_hashes", on_get_transaction_pool_hashes, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES) MAP_URI_AUTO_JON2("/get_transaction_pool_stats", on_get_transaction_pool_stats, COMMAND_RPC_GET_TRANSACTION_POOL_STATS) MAP_URI_AUTO_JON2_IF("/stop_daemon", on_stop_daemon, COMMAND_RPC_STOP_DAEMON, !m_restricted) MAP_URI_AUTO_JON2("/get_info", on_get_info, COMMAND_RPC_GET_INFO) @@ -129,6 +130,7 @@ namespace cryptonote MAP_JON_RPC_WE("getblocktemplate", on_getblocktemplate, COMMAND_RPC_GETBLOCKTEMPLATE) MAP_JON_RPC_WE("submit_block", on_submitblock, COMMAND_RPC_SUBMITBLOCK) MAP_JON_RPC_WE("submitblock", on_submitblock, COMMAND_RPC_SUBMITBLOCK) + MAP_JON_RPC_WE_IF("generateblocks", on_generateblocks, COMMAND_RPC_GENERATEBLOCKS, !m_restricted) MAP_JON_RPC_WE("get_last_block_header", on_get_last_block_header, COMMAND_RPC_GET_LAST_BLOCK_HEADER) MAP_JON_RPC_WE("getlastblockheader", on_get_last_block_header, COMMAND_RPC_GET_LAST_BLOCK_HEADER) MAP_JON_RPC_WE("get_block_header_by_hash", on_get_block_header_by_hash, COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH) @@ -180,6 +182,7 @@ namespace cryptonote bool on_set_log_level(const COMMAND_RPC_SET_LOG_LEVEL::request& req, COMMAND_RPC_SET_LOG_LEVEL::response& res); bool on_set_log_categories(const COMMAND_RPC_SET_LOG_CATEGORIES::request& req, COMMAND_RPC_SET_LOG_CATEGORIES::response& res); bool on_get_transaction_pool(const COMMAND_RPC_GET_TRANSACTION_POOL::request& req, COMMAND_RPC_GET_TRANSACTION_POOL::response& res, bool request_has_rpc_origin = true); + bool on_get_transaction_pool_hashes_bin(const COMMAND_RPC_GET_TRANSACTION_POOL_HASHES_BIN::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES_BIN::response& res, bool request_has_rpc_origin = true); bool on_get_transaction_pool_hashes(const COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::response& res, bool request_has_rpc_origin = true); bool on_get_transaction_pool_stats(const COMMAND_RPC_GET_TRANSACTION_POOL_STATS::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_STATS::response& res, bool request_has_rpc_origin = true); bool on_stop_daemon(const COMMAND_RPC_STOP_DAEMON::request& req, COMMAND_RPC_STOP_DAEMON::response& res); @@ -196,6 +199,7 @@ namespace cryptonote bool on_getblockhash(const COMMAND_RPC_GETBLOCKHASH::request& req, COMMAND_RPC_GETBLOCKHASH::response& res, epee::json_rpc::error& error_resp); bool on_getblocktemplate(const COMMAND_RPC_GETBLOCKTEMPLATE::request& req, COMMAND_RPC_GETBLOCKTEMPLATE::response& res, epee::json_rpc::error& error_resp); bool on_submitblock(const COMMAND_RPC_SUBMITBLOCK::request& req, COMMAND_RPC_SUBMITBLOCK::response& res, epee::json_rpc::error& error_resp); + bool on_generateblocks(const COMMAND_RPC_GENERATEBLOCKS::request& req, COMMAND_RPC_GENERATEBLOCKS::response& res, epee::json_rpc::error& error_resp); bool on_get_last_block_header(const COMMAND_RPC_GET_LAST_BLOCK_HEADER::request& req, COMMAND_RPC_GET_LAST_BLOCK_HEADER::response& res, epee::json_rpc::error& error_resp); bool on_get_block_header_by_hash(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HASH::response& res, epee::json_rpc::error& error_resp); bool on_get_block_header_by_height(const COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::request& req, COMMAND_RPC_GET_BLOCK_HEADER_BY_HEIGHT::response& res, epee::json_rpc::error& error_resp); diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index 70e186848..e2d120cad 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -49,7 +49,7 @@ namespace cryptonote // advance which version they will stop working with // Don't go over 32767 for any of these #define CORE_RPC_VERSION_MAJOR 1 -#define CORE_RPC_VERSION_MINOR 20 +#define CORE_RPC_VERSION_MINOR 21 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR) @@ -83,10 +83,12 @@ namespace cryptonote std::list<crypto::hash> block_ids; //*first 10 blocks id goes sequential, next goes in pow(2,n) offset, like 2, 4, 8, 16, 32, 64 and so on, and the last one is always genesis block */ uint64_t start_height; bool prune; + bool no_miner_tx; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE_CONTAINER_POD_AS_BLOB(block_ids) KV_SERIALIZE(start_height) KV_SERIALIZE(prune) + KV_SERIALIZE_OPT(no_miner_tx, false) END_KV_SERIALIZE_MAP() }; @@ -110,7 +112,7 @@ namespace cryptonote struct response { - std::list<block_complete_entry> blocks; + std::vector<block_complete_entry> blocks; uint64_t start_height; uint64_t current_height; std::string status; @@ -188,7 +190,7 @@ namespace cryptonote struct response { - std::list<crypto::hash> m_block_ids; + std::vector<crypto::hash> m_block_ids; uint64_t start_height; uint64_t current_height; std::string status; @@ -273,7 +275,7 @@ namespace cryptonote uint64_t total_received; uint64_t total_received_unlocked = 0; // OpenMonero only uint64_t scanned_height; - std::list<transaction> transactions; + std::vector<transaction> transactions; uint64_t blockchain_height; uint64_t scanned_block_height; std::string status; @@ -561,7 +563,7 @@ namespace cryptonote { struct request { - std::list<std::string> txs_hashes; + std::vector<std::string> txs_hashes; bool decode_as_json; bool prune; @@ -598,11 +600,11 @@ namespace cryptonote struct response { // older compatibility stuff - std::list<std::string> txs_as_hex; //transactions blobs as hex (old compat) - std::list<std::string> txs_as_json; //transactions decoded as json (old compat) + std::vector<std::string> txs_as_hex; //transactions blobs as hex (old compat) + std::vector<std::string> txs_as_json; //transactions decoded as json (old compat) // in both old and new - std::list<std::string> missed_tx; //not found transactions + std::vector<std::string> missed_tx; //not found transactions // new style std::vector<entry> txs; @@ -953,6 +955,7 @@ namespace cryptonote bool mainnet; bool testnet; bool stagenet; + std::string nettype; std::string top_block_hash; uint64_t cumulative_difficulty; uint64_t block_size_limit; @@ -964,6 +967,7 @@ namespace cryptonote std::string bootstrap_daemon_address; uint64_t height_without_bootstrap; bool was_bootstrap_ever_used; + uint64_t database_size; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(status) @@ -982,6 +986,7 @@ namespace cryptonote KV_SERIALIZE(mainnet) KV_SERIALIZE(testnet) KV_SERIALIZE(stagenet) + KV_SERIALIZE(nettype) KV_SERIALIZE(top_block_hash) KV_SERIALIZE(cumulative_difficulty) KV_SERIALIZE(block_size_limit) @@ -993,6 +998,7 @@ namespace cryptonote KV_SERIALIZE(bootstrap_daemon_address) KV_SERIALIZE(height_without_bootstrap) KV_SERIALIZE(was_bootstrap_ever_used) + KV_SERIALIZE(database_size) END_KV_SERIALIZE_MAP() }; }; @@ -1149,6 +1155,31 @@ namespace cryptonote END_KV_SERIALIZE_MAP() }; }; + + struct COMMAND_RPC_GENERATEBLOCKS + { + struct request + { + uint64_t amount_of_blocks; + std::string wallet_address; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(amount_of_blocks) + KV_SERIALIZE(wallet_address) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + uint64_t height; + std::string status; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(height) + KV_SERIALIZE(status) + END_KV_SERIALIZE_MAP() + }; + }; struct block_header_response { @@ -1487,7 +1518,7 @@ namespace cryptonote }; }; - struct COMMAND_RPC_GET_TRANSACTION_POOL_HASHES + struct COMMAND_RPC_GET_TRANSACTION_POOL_HASHES_BIN { struct request { @@ -1509,6 +1540,28 @@ namespace cryptonote }; }; + struct COMMAND_RPC_GET_TRANSACTION_POOL_HASHES + { + struct request + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string status; + std::vector<std::string> tx_hashes; + bool untrusted; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + KV_SERIALIZE(tx_hashes) + KV_SERIALIZE(untrusted) + END_KV_SERIALIZE_MAP() + }; + }; + struct tx_backlog_entry { uint64_t blob_size; @@ -1565,6 +1618,8 @@ namespace cryptonote std::vector<txpool_histo> histo; uint32_t num_double_spends; + txpool_stats(): bytes_total(0), bytes_min(0), bytes_max(0), bytes_med(0), fee_total(0), oldest(0), txs_total(0), num_failing(0), num_10m(0), num_not_relayed(0), histo_98pc(0), num_double_spends(0) {} + BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(bytes_total) KV_SERIALIZE(bytes_min) @@ -1929,7 +1984,7 @@ namespace cryptonote { struct request { - std::list<std::string> txids; + std::vector<std::string> txids; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(txids) @@ -2086,12 +2141,16 @@ namespace cryptonote uint64_t height; uint64_t length; uint64_t difficulty; + std::vector<std::string> block_hashes; + std::string main_chain_parent_block; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(block_hash) KV_SERIALIZE(height) KV_SERIALIZE(length) KV_SERIALIZE(difficulty) + KV_SERIALIZE(block_hashes) + KV_SERIALIZE(main_chain_parent_block) END_KV_SERIALIZE_MAP() }; @@ -2146,7 +2205,7 @@ namespace cryptonote { struct request { - std::list<std::string> txids; + std::vector<std::string> txids; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(txids) @@ -2227,12 +2286,14 @@ namespace cryptonote uint64_t from_height; uint64_t to_height; bool cumulative; + bool binary; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(amounts) KV_SERIALIZE_OPT(from_height, (uint64_t)0) KV_SERIALIZE_OPT(to_height, (uint64_t)0) KV_SERIALIZE_OPT(cumulative, false) + KV_SERIALIZE_OPT(binary, true) END_KV_SERIALIZE_MAP() }; @@ -2240,13 +2301,18 @@ namespace cryptonote { uint64_t amount; uint64_t start_height; + bool binary; std::vector<uint64_t> distribution; uint64_t base; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(amount) KV_SERIALIZE(start_height) - KV_SERIALIZE_CONTAINER_POD_AS_BLOB(distribution) + KV_SERIALIZE(binary) + if (this_ref.binary) + KV_SERIALIZE_CONTAINER_POD_AS_BLOB(distribution) + else + KV_SERIALIZE(distribution) KV_SERIALIZE(base) END_KV_SERIALIZE_MAP() }; @@ -2255,10 +2321,12 @@ namespace cryptonote { std::string status; std::vector<distribution> distributions; + bool untrusted; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(status) KV_SERIALIZE(distributions) + KV_SERIALIZE(untrusted) END_KV_SERIALIZE_MAP() }; }; diff --git a/src/rpc/core_rpc_server_error_codes.h b/src/rpc/core_rpc_server_error_codes.h index 69caaa6a6..5a754749f 100644 --- a/src/rpc/core_rpc_server_error_codes.h +++ b/src/rpc/core_rpc_server_error_codes.h @@ -42,5 +42,6 @@ #define CORE_RPC_ERROR_CODE_WRONG_BLOCKBLOB_SIZE -10 #define CORE_RPC_ERROR_CODE_UNSUPPORTED_RPC -11 #define CORE_RPC_ERROR_CODE_MINING_TO_SUBADDRESS -12 +#define CORE_RPC_ERROR_CODE_REGTEST_REQUIRED -13 diff --git a/src/rpc/daemon_handler.cpp b/src/rpc/daemon_handler.cpp index 39f169cdf..55858cc2a 100644 --- a/src/rpc/daemon_handler.cpp +++ b/src/rpc/daemon_handler.cpp @@ -50,9 +50,9 @@ namespace rpc void DaemonHandler::handle(const GetBlocksFast::Request& req, GetBlocksFast::Response& res) { - std::list<std::pair<blobdata, std::list<blobdata> > > blocks; + std::vector<std::pair<std::pair<blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, blobdata> > > > blocks; - if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, blocks, res.current_height, res.start_height, req.prune, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT)) + if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, blocks, res.current_height, res.start_height, req.prune, true, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT)) { res.status = Message::STATUS_FAILED; res.error_details = "core::find_blockchain_supplement() returned false"; @@ -62,9 +62,6 @@ namespace rpc res.blocks.resize(blocks.size()); res.output_indices.resize(blocks.size()); - //TODO: really need to switch uses of std::list to std::vector unless - // it's a huge performance concern - auto it = blocks.begin(); uint64_t block_count = 0; @@ -72,7 +69,7 @@ namespace rpc { cryptonote::rpc::block_with_transactions& bwt = res.blocks[block_count]; - if (!parse_and_validate_block_from_blob(it->first, bwt.block)) + if (!parse_and_validate_block_from_blob(it->first.first, bwt.block)) { res.blocks.clear(); res.output_indices.clear(); @@ -89,11 +86,11 @@ namespace rpc res.error_details = "incorrect number of transactions retrieved for block"; return; } - std::list<transaction> txs; - for (const auto& blob : it->second) + std::vector<transaction> txs; + for (const auto& p : it->second) { txs.resize(txs.size() + 1); - if (!parse_and_validate_tx_from_blob(blob, txs.back())) + if (!parse_and_validate_tx_from_blob(p.second, txs.back())) { res.blocks.clear(); res.output_indices.clear(); @@ -163,10 +160,10 @@ namespace rpc void DaemonHandler::handle(const GetTransactions::Request& req, GetTransactions::Response& res) { - std::list<cryptonote::transaction> found_txs; - std::list<crypto::hash> missed_hashes; + std::vector<cryptonote::transaction> found_txs_vec; + std::vector<crypto::hash> missed_vec; - bool r = m_core.get_transactions(req.tx_hashes, found_txs, missed_hashes); + bool r = m_core.get_transactions(req.tx_hashes, found_txs_vec, missed_vec); // TODO: consider fixing core::get_transactions to not hide exceptions if (!r) @@ -176,20 +173,7 @@ namespace rpc return; } - size_t num_found = found_txs.size(); - - // std::list is annoying - std::vector<cryptonote::transaction> found_txs_vec - { - std::make_move_iterator(std::begin(found_txs)), - std::make_move_iterator(std::end(found_txs)) - }; - - std::vector<crypto::hash> missed_vec - { - std::make_move_iterator(std::begin(missed_hashes)), - std::make_move_iterator(std::end(missed_hashes)) - }; + size_t num_found = found_txs_vec.size(); std::vector<uint64_t> heights(num_found); std::vector<bool> in_pool(num_found, false); @@ -204,7 +188,7 @@ namespace rpc // if any missing from blockchain, check in tx pool if (!missed_vec.empty()) { - std::list<cryptonote::transaction> pool_txs; + std::vector<cryptonote::transaction> pool_txs; m_core.get_pool_transactions(pool_txs); diff --git a/src/rpc/daemon_messages.h b/src/rpc/daemon_messages.h index 1495c845f..8fff369df 100644 --- a/src/rpc/daemon_messages.h +++ b/src/rpc/daemon_messages.h @@ -106,7 +106,7 @@ BEGIN_RPC_MESSAGE_CLASS(GetHashesFast); RPC_MESSAGE_MEMBER(uint64_t, start_height); END_RPC_MESSAGE_REQUEST; BEGIN_RPC_MESSAGE_RESPONSE; - RPC_MESSAGE_MEMBER(std::list<crypto::hash>, hashes); + RPC_MESSAGE_MEMBER(std::vector<crypto::hash>, hashes); RPC_MESSAGE_MEMBER(uint64_t, start_height); RPC_MESSAGE_MEMBER(uint64_t, current_height); END_RPC_MESSAGE_RESPONSE; diff --git a/src/rpc/get_output_distribution_cache.h b/src/rpc/get_output_distribution_cache.h deleted file mode 100644 index 6495e7d4c..000000000 --- a/src/rpc/get_output_distribution_cache.h +++ /dev/null @@ -1,113 +0,0 @@ -static const struct -{ - uint64_t amount; - uint64_t from_height; - uint64_t to_height; - uint64_t start_height; - uint64_t base; - std::vector<uint64_t> distribution; -} -get_output_distribution_cache[] = -{ - { - 0, - 1544704, - 1546001, - 1544704, - 5143500, - { - 5, 38, 37, 33, 39, 7, 1, 1, 5, 9, 7, 5, 17, 5, 3, 9, 3, 17, 5, 17, 1, 1, 15, 13, 3, 10, 5, 3, 34, 1, 45, 7, - 5, 17, 5, 22, 3, 1, 17, 16, 5, 1, 3, 43, 5, 13, 3, 23, 9, 7, 9, 13, 1, 11, 1, 17, 1, 3, 16, 11, 5, 11, 7, 7, - 33, 11, 7, 1, 5, 1, 21, 19, 1, 17, 1, 49, 17, 3, 3, 9, 35, 46, 46, 39, 26, 33, 21, 3, 23, 3, 9, 37, 1, 33, 11, 32, - 1, 13, 16, 12, 3, 21, 1, 18, 3, 19, 1, 25, 5, 3, 18, 7, 17, 5, 9, 15, 7, 7, 11, 9, 9, 17, 5, 16, 1, 3, 13, 3, - 5, 5, 5, 13, 5, 9, 5, 13, 3, 17, 15, 36, 13, 3, 20, 12, 6, 23, 17, 10, 22, 23, 1, 7, 21, 6, 23, 1, 3, 19, 13, 1, - 3, 43, 35, 13, 1, 31, 7, 3, 17, 1, 15, 5, 11, 15, 24, 1, 18, 13, 5, 15, 1, 29, 3, 3, 13, 3, 15, 7, 17, 3, 1, 1, - 17, 1, 1, 45, 39, 27, 45, 46, 34, 7, 3, 3, 9, 3, 3, 11, 7, 5, 9, 25, 19, 3, 33, 1, 5, 17, 1, 45, 4, 1, 45, 11, - 44, 32, 3, 1, 3, 7, 17, 15, 5, 45, 35, 41, 1, 35, 3, 3, 19, 1, 9, 17, 29, 29, 3, 1, 13, 1, 3, 47, 21, 13, 7, 1, - 7, 5, 1, 11, 1, 40, 9, 7, 3, 3, 13, 25, 1, 47, 5, 7, 3, 7, 31, 40, 34, 6, 3, 15, 3, 31, 5, 13, 27, 9, 12, 21, - 3, 1, 19, 1, 19, 5, 47, 49, 47, 42, 50, 34, 29, 23, 1, 5, 9, 16, 11, 7, 1, 19, 7, 5, 1, 15, 1, 1, 9, 13, 9, 5, - 27, 3, 3, 29, 1, 33, 3, 9, 5, 35, 5, 1, 17, 7, 3, 39, 3, 28, 19, 1, 1, 9, 1, 3, 27, 1, 37, 3, 1, 1, 16, 3, - 25, 11, 5, 3, 33, 45, 17, 11, 7, 22, 9, 1, 5, 5, 5, 15, 1, 15, 9, 7, 11, 13, 37, 49, 46, 38, 11, 1, 25, 1, 13, 18, - 3, 7, 39, 3, 37, 19, 35, 3, 1, 3, 19, 1, 3, 15, 21, 3, 27, 1, 45, 48, 1, 13, 29, 9, 1, 1, 46, 43, 5, 15, 3, 7, - 29, 26, 5, 5, 21, 37, 17, 21, 3, 13, 1, 5, 1, 17, 5, 31, 13, 1, 11, 3, 46, 9, 3, 7, 1, 1, 41, 1, 21, 1, 5, 12, - 7, 13, 9, 25, 1, 47, 47, 48, 48, 48, 48, 48, 47, 48, 45, 45, 33, 52, 50, 46, 45, 47, 35, 41, 38, 35, 42, 38, 34, 41, 39, 35, - 51, 51, 45, 43, 49, 52, 53, 45, 42, 46, 37, 53, 49, 41, 46, 49, 46, 47, 48, 37, 41, 33, 43, 38, 15, 3, 3, 27, 11, 5, 23, 13, - 1, 1, 37, 3, 15, 3, 30, 13, 3, 45, 12, 3, 5, 11, 1, 1, 21, 9, 11, 19, 1, 1, 1, 25, 5, 21, 3, 1, 32, 44, 3, 33, - 11, 7, 5, 23, 1, 37, 47, 48, 48, 48, 48, 48, 48, 46, 47, 47, 50, 45, 49, 50, 46, 47, 49, 45, 51, 49, 50, 49, 49, 46, 47, 48, - 46, 48, 46, 50, 46, 43, 46, 46, 48, 47, 46, 47, 45, 49, 46, 43, 50, 45, 45, 49, 45, 48, 45, 49, 48, 45, 45, 51, 45, 51, 45, 46, - 52, 45, 45, 51, 51, 52, 44, 45, 52, 50, 50, 46, 47, 51, 51, 46, 47, 47, 47, 50, 47, 51, 48, 49, 51, 50, 48, 48, 48, 50, 49, 49, - 52, 52, 49, 50, 49, 49, 49, 51, 52, 49, 52, 50, 49, 47, 29, 15, 39, 17, 31, 5, 40, 5, 18, 23, 25, 7, 35, 26, 5, 31, 49, 22, - 3, 17, 7, 49, 7, 49, 47, 12, 44, 46, 36, 15, 3, 1, 47, 13, 35, 40, 5, 21, 19, 39, 21, 33, 31, 29, 1, 1, 37, 1, 15, 47, - 7, 7, 47, 41, 13, 3, 47, 31, 9, 33, 13, 43, 29, 5, 1, 9, 33, 7, 27, 15, 15, 25, 5, 43, 22, 31, 7, 1, 47, 1, 15, 27, - 3, 27, 45, 15, 1, 36, 17, 1, 23, 39, 38, 45, 7, 7, 19, 7, 11, 47, 33, 16, 3, 45, 45, 45, 9, 27, 3, 3, 21, 3, 7, 21, - 7, 3, 43, 1, 17, 1, 45, 37, 46, 5, 5, 13, 46, 40, 48, 48, 45, 34, 1, 46, 19, 25, 9, 7, 47, 23, 37, 31, 3, 25, 13, 46, - 31, 25, 5, 46, 35, 52, 11, 23, 27, 4, 15, 11, 11, 11, 9, 34, 7, 9, 15, 34, 9, 27, 37, 28, 25, 45, 13, 30, 5, 25, 15, 7, - 3, 19, 27, 1, 7, 11, 1, 32, 3, 45, 11, 9, 21, 25, 9, 13, 13, 1, 7, 1, 33, 11, 5, 3, 3, 27, 27, 5, 3, 37, 17, 17, - 3, 7, 5, 13, 1, 3, 44, 45, 26, 25, 1, 13, 3, 13, 3, 11, 1, 11, 7, 45, 3, 3, 1, 43, 1, 19, 3, 1, 15, 5, 39, 7, - 7, 1, 9, 1, 11, 19, 3, 35, 29, 7, 15, 11, 40, 7, 44, 38, 34, 7, 9, 7, 1, 27, 1, 9, 5, 45, 1, 21, 3, 1, 5, 9, - 3, 21, 23, 33, 3, 1, 7, 3, 3, 7, 41, 9, 7, 1, 5, 31, 9, 7, 1, 1, 11, 41, 51, 20, 9, 47, 39, 17, 9, 35, 1, 41, - 1, 19, 1, 19, 15, 1, 13, 5, 23, 15, 9, 15, 17, 1, 15, 27, 33, 31, 29, 7, 13, 1, 5, 45, 5, 1, 1, 11, 1, 13, 3, 7, - 9, 1, 13, 39, 3, 33, 5, 3, 7, 7, 5, 29, 11, 1, 7, 1, 15, 3, 13, 3, 15, 3, 3, 1, 5, 1, 9, 1, 44, 49, 24, 25, - 1, 1, 34, 22, 7, 5, 5, 5, 10, 9, 13, 3, 9, 1, 9, 19, 7, 43, 48, 7, 11, 7, 3, 3, 7, 21, 1, 1, 3, 3, 11, 31, - 1, 1, 13, 22, 23, 7, 27, 9, 3, 3, 21, 1, 35, 21, 9, 11, 13, 39, 1, 3, 7, 23, 3, 28, 3, 45, 47, 38, 32, 37, 34, 1, - 23, 3, 3, 1, 19, 19, 1, 5, 13, 1, 5, 11, 38, 3, 1, 36, 13, 1, 1, 23, 5, 17, 11, 1, 13, 1, 3, 7, 11, 3, 33, 7, - 19, 5, 5, 1, 1, 3, 5, 41, 1, 3, 25, 1, 7, 7, 9, 3, 11, 3, 13, 5, 7, 1, 3, 9, 1, 1, 43, 47, 47, 47, 17, 7, - 17, 3, 19, 1, 9, 9, 33, 22, 1, 25, 1, 3, 3, 32, 5, 1, 23, 9, 5, 1, 31, 5, 9, 1, 3, 7, 19, 1, 12, 11, 5, 1, - 1, 9, 25, 15, 15, 13, 5, 3, 15, 1, 17, 1, 1, 5, 5, 41, 11, 15, 7, 3, 21, 21, 35, 22, 46, 35, 3, 27, 5, 3, 45, 22, - 27, 1, 19, 9, 1, 25, 1, 29, 3, 5, 25, 17, 27, 5, 19, 5, 25, 7, 19, 1, 9, 21, 3, 7, 29, 27, 17, 3, 3, 15, 7, 19, - 5, 25, 1, 23, 45, 4, 31, 1, 37, 14, 29, 3, 29, 1, 23, 29, 19, 11, 1, 13, 13, 9, 1, 25, 1, 33, 1, 37, 37, 23, 7, 21, - 7, 3, 13, 7, 3, 7, 21, 11, 9, 1, 31, 3, 1, 7, 39, 46, 3, 30, - }, - }, - { - 0, - 1562704, - 1564001, - 1562704, - 5521986, - { - 35, 45, 23, 3, 44, 47, 44, 3, 17, 35, 7, 11, 7, 29, 43, 27, 11, 7, 5, 31, 13, 9, 45, 45, 7, 42, 17, 15, 19, 11, 45, 19, - 45, 46, 45, 46, 32, 34, 43, 34, 46, 47, 45, 30, 17, 45, 46, 36, 35, 38, 19, 9, 23, 17, 3, 19, 31, 41, 35, 24, 15, 45, 15, 5, - 11, 5, 19, 11, 11, 7, 15, 19, 45, 34, 7, 7, 29, 1, 9, 36, 7, 44, 45, 33, 25, 8, 17, 7, 44, 43, 48, 45, 42, 46, 40, 44, - 1, 43, 45, 46, 46, 35, 19, 19, 23, 5, 13, 19, 7, 16, 9, 3, 25, 34, 3, 27, 9, 39, 3, 43, 21, 1, 45, 45, 39, 25, 23, 13, - 39, 39, 3, 45, 43, 46, 44, 40, 39, 33, 45, 47, 38, 45, 45, 39, 47, 47, 45, 46, 35, 45, 43, 47, 45, 40, 34, 42, 42, 48, 49, 47, - 47, 48, 47, 45, 43, 48, 37, 48, 41, 45, 48, 34, 42, 44, 9, 19, 27, 1, 47, 47, 43, 25, 29, 5, 5, 21, 39, 35, 43, 37, 13, 45, - 25, 31, 26, 47, 45, 23, 23, 39, 32, 25, 44, 47, 35, 47, 15, 17, 7, 9, 5, 35, 31, 3, 45, 47, 46, 13, 17, 48, 45, 9, 13, 45, - 45, 31, 1, 53, 44, 46, 39, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 47, 47, 47, 47, 47, 47, 47, 47, 53, 49, 45, 45, 50, - 50, 27, 43, 46, 47, 46, 45, 48, 36, 42, 42, 46, 45, 47, 41, 45, 43, 47, 38, 48, 47, 33, 11, 45, 46, 34, 42, 32, 3, 45, 37, 45, - 15, 3, 45, 29, 31, 9, 5, 3, 27, 5, 21, 25, 7, 15, 46, 34, 5, 3, 17, 3, 9, 13, 7, 11, 3, 1, 34, 13, 7, 45, 33, 26, - 9, 5, 9, 41, 43, 34, 3, 35, 3, 17, 37, 11, 17, 3, 15, 27, 15, 45, 46, 13, 26, 16, 11, 7, 5, 45, 38, 45, 45, 22, 44, 44, - 43, 6, 11, 35, 15, 44, 17, 27, 13, 3, 40, 5, 9, 7, 35, 19, 5, 5, 1, 28, 33, 15, 45, 5, 29, 3, 31, 12, 5, 32, 24, 3, - 23, 13, 47, 45, 42, 46, 39, 21, 21, 1, 44, 44, 47, 41, 5, 1, 11, 36, 20, 5, 45, 39, 45, 45, 44, 45, 32, 22, 40, 11, 38, 1, - 45, 46, 37, 9, 23, 9, 15, 44, 7, 16, 38, 46, 11, 14, 24, 19, 19, 7, 26, 25, 45, 37, 17, 1, 35, 1, 3, 28, 3, 11, 22, 13, - 3, 1, 7, 38, 5, 3, 1, 26, 1, 3, 43, 44, 46, 45, 21, 11, 1, 44, 27, 1, 11, 26, 10, 44, 45, 45, 45, 47, 47, 45, 48, 45, - 38, 9, 5, 44, 46, 27, 3, 12, 1, 11, 3, 44, 43, 1, 5, 2, 46, 17, 13, 19, 1, 12, 7, 23, 1, 17, 6, 13, 3, 5, 27, 7, - 46, 36, 19, 25, 1, 1, 3, 8, 15, 3, 45, 45, 45, 37, 6, 15, 3, 5, 38, 14, 41, 1, 13, 45, 45, 39, 44, 29, 43, 48, 51, 50, - 37, 5, 17, 46, 47, 31, 5, 42, 49, 38, 39, 24, 7, 11, 44, 35, 21, 6, 15, 5, 47, 13, 28, 45, 34, 27, 24, 15, 35, 13, 7, 25, - 43, 13, 14, 5, 3, 5, 46, 45, 45, 35, 5, 21, 28, 3, 13, 4, 30, 19, 45, 45, 40, 37, 5, 40, 17, 9, 3, 9, 13, 4, 17, 33, - 44, 39, 17, 45, 28, 5, 11, 45, 19, 45, 21, 44, 31, 43, 49, 48, 15, 3, 1, 44, 45, 43, 11, 1, 1, 27, 43, 45, 39, 3, 1, 3, - 5, 31, 1, 43, 23, 19, 7, 5, 45, 31, 11, 7, 3, 9, 5, 7, 13, 43, 47, 45, 46, 47, 14, 3, 25, 45, 7, 17, 32, 21, 3, 17, - 5, 11, 31, 40, 45, 20, 45, 45, 32, 38, 47, 38, 45, 41, 49, 30, 45, 5, 36, 31, 22, 3, 46, 45, 13, 21, 23, 5, 46, 45, 33, 19, - 25, 1, 7, 13, 5, 44, 23, 29, 23, 24, 7, 5, 37, 13, 29, 13, 7, 17, 7, 43, 3, 21, 7, 44, 1, 19, 15, 9, 34, 43, 26, 3, - 17, 5, 6, 5, 7, 3, 33, 35, 43, 41, 48, 47, 30, 45, 19, 7, 5, 33, 11, 34, 25, 1, 21, 11, 29, 7, 1, 4, 5, 10, 14, 3, - 44, 11, 47, 45, 33, 3, 9, 7, 40, 23, 9, 1, 3, 1, 7, 5, 9, 9, 6, 11, 45, 41, 45, 19, 5, 11, 10, 39, 9, 19, 3, 11, - 43, 42, 1, 13, 35, 5, 32, 7, 5, 5, 43, 37, 3, 32, 17, 3, 23, 1, 13, 45, 17, 1, 21, 45, 43, 46, 49, 47, 45, 30, 9, 31, - 19, 42, 19, 44, 17, 14, 19, 25, 1, 7, 5, 45, 19, 13, 7, 3, 1, 1, 9, 21, 37, 9, 11, 1, 3, 37, 27, 13, 5, 21, 33, 3, - 27, 9, 27, 1, 39, 1, 46, 21, 10, 13, 21, 40, 22, 35, 41, 9, 33, 3, 17, 8, 45, 46, 42, 46, 47, 47, 47, 48, 48, 47, 43, 48, - 37, 39, 50, 35, 3, 40, 45, 40, 46, 36, 34, 28, 9, 9, 37, 9, 5, 7, 13, 31, 1, 7, 3, 3, 44, 45, 25, 15, 1, 21, 43, 25, - 1, 38, 34, 42, 31, 23, 33, 35, 37, 20, 7, 15, 3, 7, 7, 27, 45, 45, 48, 47, 45, 44, 47, 23, 25, 36, 11, 3, 18, 24, 27, 13, - 41, 13, 5, 5, 7, 19, 15, 7, 5, 14, 45, 45, 37, 1, 5, 17, 21, 41, 17, 37, 53, 41, 45, 45, 45, 45, 45, 45, 45, 45, 43, 47, - 47, 48, 53, 47, 47, 47, 49, 27, 45, 47, 47, 47, 47, 45, 45, 45, 47, 43, 48, 34, 34, 43, 46, 15, 37, 21, 5, 27, 11, 1, 9, 7, - 19, 15, 1, 1, 19, 3, 36, 27, 29, 13, 21, 9, 17, 5, 16, 45, 23, 34, 3, 1, 7, 25, 28, 13, 29, 15, 11, 19, 17, 1, 27, 23, - 31, 19, 41, 41, 40, 47, 28, 31, 26, 26, 36, 17, 5, 1, 23, 1, 45, 34, 49, 51, 34, 43, 37, 5, 41, 15, 5, 21, 1, 7, 9, 19, - 5, 11, 39, 19, 45, 45, 38, 17, 9, 1, 15, 11, 5, 13, 47, 46, 48, 45, 19, 32, 7, 19, 5, 7, 23, 29, 5, 45, 41, 37, 1, 5, - 27, 5, 5, 7, 19, 9, 1, 35, 48, 38, 38, 39, 42, 43, 21, 23, 43, 41, 7, 3, 7, 13, 1, 46, 47, 46, 23, 46, 45, 25, 7, 9, - 21, 7, 41, 13, 20, 1, 21, 15, 37, 5, 40, 45, 45, 5, 45, 46, 15, 33, 46, 12, 13, 7, 24, 7, 5, 30, 7, 46, 13, 8, 44, 35, - 45, 33, 40, 36, 47, 47, 29, 43, 36, 43, 45, 42, 36, 19, 7, 7, 43, 3, 44, 25, 48, 29, 11, 45, 30, 1, 17, 13, 25, 1, 48, 45, - 45, 45, 44, 49, 37, 9, 21, 17, 15, 7, 15, 25, 1, 1, 9, 43, 33, 11, 3, 29, 45, 45, 9, 7, 7, 27, 47, 45, 47, 48, 45, 47, - 26, 1, 43, 15, 45, 17, 1, 5, 35, 31, 9, 3, 9, 19, 9, 21, 43, 5, 27, 1, 5, 9, 4, 34, 19, 3, 7, 11, 45, 46, 45, 45, - 46, 47, 47, 44, 45, 43, 27, 9, 17, 15, 19, 44, 45, 46, 47, 47, 45, 45, - } - } -}; - diff --git a/src/rpc/message_data_structs.h b/src/rpc/message_data_structs.h index 17ae9629f..fc1b2329d 100644 --- a/src/rpc/message_data_structs.h +++ b/src/rpc/message_data_structs.h @@ -181,6 +181,7 @@ namespace rpc bool mainnet; bool testnet; bool stagenet; + std::string nettype; crypto::hash top_block_hash; uint64_t cumulative_difficulty; uint64_t block_size_limit; diff --git a/src/rpc/rpc_args.cpp b/src/rpc/rpc_args.cpp index d4a6138ba..d4044d11b 100644 --- a/src/rpc/rpc_args.cpp +++ b/src/rpc/rpc_args.cpp @@ -102,7 +102,7 @@ namespace cryptonote { if (!config.login) { - LOG_ERROR(arg.rpc_access_control_origins.name << tr(" requires RFC server password --") << arg.rpc_login.name << tr(" cannot be empty")); + LOG_ERROR(arg.rpc_access_control_origins.name << tr(" requires RPC server password --") << arg.rpc_login.name << tr(" cannot be empty")); return boost::none; } diff --git a/src/rpc/zmq_server.cpp b/src/rpc/zmq_server.cpp index 3aee8c4c7..edd3e6669 100644 --- a/src/rpc/zmq_server.cpp +++ b/src/rpc/zmq_server.cpp @@ -104,6 +104,10 @@ bool ZmqServer::addTCPSocket(std::string address, std::string port) rep_socket->setsockopt(ZMQ_RCVTIMEO, &DEFAULT_RPC_RECV_TIMEOUT_MS, sizeof(DEFAULT_RPC_RECV_TIMEOUT_MS)); + if (address.empty()) + address = "*"; + if (port.empty()) + port = "*"; std::string bind_address = addr_prefix + address + std::string(":") + port; rep_socket->bind(bind_address.c_str()); } diff --git a/src/serialization/json_object.cpp b/src/serialization/json_object.cpp index a7fb58ee4..c2467b863 100644 --- a/src/serialization/json_object.cpp +++ b/src/serialization/json_object.cpp @@ -1175,6 +1175,7 @@ void toJsonValue(rapidjson::Document& doc, const cryptonote::rpc::DaemonInfo& in INSERT_INTO_JSON_OBJECT(val, doc, white_peerlist_size, info.white_peerlist_size); INSERT_INTO_JSON_OBJECT(val, doc, grey_peerlist_size, info.grey_peerlist_size); INSERT_INTO_JSON_OBJECT(val, doc, testnet, info.testnet); + INSERT_INTO_JSON_OBJECT(val, doc, nettype, info.nettype); INSERT_INTO_JSON_OBJECT(val, doc, top_block_hash, info.top_block_hash); INSERT_INTO_JSON_OBJECT(val, doc, cumulative_difficulty, info.cumulative_difficulty); INSERT_INTO_JSON_OBJECT(val, doc, block_size_limit, info.block_size_limit); @@ -1200,6 +1201,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::DaemonInfo& inf GET_FROM_JSON_OBJECT(val, info.white_peerlist_size, white_peerlist_size); GET_FROM_JSON_OBJECT(val, info.grey_peerlist_size, grey_peerlist_size); GET_FROM_JSON_OBJECT(val, info.testnet, testnet); + GET_FROM_JSON_OBJECT(val, info.nettype, nettype); GET_FROM_JSON_OBJECT(val, info.top_block_hash, top_block_hash); GET_FROM_JSON_OBJECT(val, info.cumulative_difficulty, cumulative_difficulty); GET_FROM_JSON_OBJECT(val, info.block_size_limit, block_size_limit); diff --git a/src/simplewallet/CMakeLists.txt b/src/simplewallet/CMakeLists.txt index 8bb295931..c31cdebde 100644 --- a/src/simplewallet/CMakeLists.txt +++ b/src/simplewallet/CMakeLists.txt @@ -48,7 +48,6 @@ target_link_libraries(simplewallet cncrypto common mnemonics - epee ${EPEE_READLINE} version ${Boost_CHRONO_LIBRARY} diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index 196a2eb74..775b7c359 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -379,21 +379,10 @@ namespace boost::optional<std::pair<uint32_t, uint32_t>> parse_subaddress_lookahead(const std::string& str) { - auto pos = str.find(":"); - bool r = pos != std::string::npos; - uint32_t major; - r = r && epee::string_tools::get_xtype_from_string(major, str.substr(0, pos)); - uint32_t minor; - r = r && epee::string_tools::get_xtype_from_string(minor, str.substr(pos + 1)); - if (r) - { - return std::make_pair(major, minor); - } - else - { + auto r = tools::parse_subaddress_lookahead(str); + if (!r) fail_msg_writer() << tr("invalid format for subaddress lookahead; must be <major>:<minor>"); - return {}; - } + return r; } void handle_transfer_exception(const std::exception_ptr &e, bool trusted_daemon) @@ -1434,7 +1423,6 @@ bool simple_wallet::set_ring(const std::vector<std::string> &args) } ring.push_back(offset); ptr += read; - MGINFO("read offset: " << offset); } if (!valid) continue; @@ -2103,6 +2091,19 @@ bool simple_wallet::set_segregation_height(const std::vector<std::string> &args/ return true; } +bool simple_wallet::set_ignore_fractional_outputs(const std::vector<std::string> &args/* = std::vector<std::string>()*/) +{ + const auto pwd_container = get_and_verify_password(); + if (pwd_container) + { + parse_bool_and_use(args[1], [&](bool r) { + m_wallet->ignore_fractional_outputs(r); + m_wallet->rewrite(m_wallet_file, pwd_container->password()); + }); + } + return true; +} + bool simple_wallet::help(const std::vector<std::string> &args/* = std::vector<std::string>()*/) { if(args.empty()) @@ -2134,7 +2135,7 @@ simple_wallet::simple_wallet() tr("Stop mining in the daemon.")); m_cmd_binder.set_handler("set_daemon", boost::bind(&simple_wallet::set_daemon, this, _1), - tr("set_daemon <host>[:<port>]"), + tr("set_daemon <host>[:<port>] [trusted|untrusted]"), tr("Set another daemon to connect to.")); m_cmd_binder.set_handler("save_bc", boost::bind(&simple_wallet::save_bc, this, _1), @@ -2168,6 +2169,10 @@ simple_wallet::simple_wallet() boost::bind(&simple_wallet::locked_transfer, this, _1), tr("locked_transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <addr> <amount> <lockblocks> [<payment_id>]"), tr("Transfer <amount> to <address> and lock it for <lockblocks> (max. 1000000). If the parameter \"index=<N1>[,<N2>,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. <priority> is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability. Multiple payments can be made at once by adding <address_2> <amount_2> etcetera (before the payment ID, if it's included)")); + m_cmd_binder.set_handler("locked_sweep_all", + boost::bind(&simple_wallet::locked_sweep_all, this, _1), + tr("locked_sweep_all [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> <lockblocks> [<payment_id>]"), + tr("Send all unlocked balance to an address and lock it for <lockblocks> (max. 1000000). If the parameter \"index<N1>[,<N2>,...]\" is specified, the wallet sweeps outputs received by those address indices. If omitted, the wallet randomly chooses an address index to be used. <priority> is the priority of the sweep. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability.")); m_cmd_binder.set_handler("sweep_unmixable", boost::bind(&simple_wallet::sweep_unmixable, this, _1), tr("Send all unmixable outputs to yourself with ring_size 1")); @@ -2188,8 +2193,8 @@ simple_wallet::simple_wallet() tr("Donate <amount> to the development team (donate.getmonero.org).")); m_cmd_binder.set_handler("sign_transfer", boost::bind(&simple_wallet::sign_transfer, this, _1), - tr("sign_transfer [export]"), - tr("Sign a transaction from a file.")); + tr("sign_transfer [export_raw]"), + tr("Sign a transaction from a file. If the parameter \"export_raw\" is specified, transaction raw hex data suitable for the daemon RPC /sendrawtransaction is exported.")); m_cmd_binder.set_handler("submit_transfer", boost::bind(&simple_wallet::submit_transfer, this, _1), tr("Submit a signed transaction from a file.")); @@ -2259,7 +2264,7 @@ simple_wallet::simple_wallet() "refresh-type <full|optimize-coinbase|no-coinbase|default>\n " " Set the wallet's refresh behaviour.\n " "priority [0|1|2|3|4]\n " - " Set the fee too default/unimportant/normal/elevated/priority.\n " + " Set the fee to default/unimportant/normal/elevated/priority.\n " "confirm-missing-payment-id <1|0>\n " "ask-password <1|0>\n " "unit <monero|millinero|micronero|nanonero|piconero>\n " @@ -2490,6 +2495,7 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args) const std::pair<size_t, size_t> lookahead = m_wallet->get_subaddress_lookahead(); success_msg_writer() << "subaddress-lookahead = " << lookahead.first << ":" << lookahead.second; success_msg_writer() << "segregation-height = " << m_wallet->segregation_height(); + success_msg_writer() << "ignore-fractional-outputs = " << m_wallet->ignore_fractional_outputs(); return true; } else @@ -2544,6 +2550,7 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args) CHECK_SIMPLE_VARIABLE("key-reuse-mitigation2", set_key_reuse_mitigation2, tr("0 or 1")); CHECK_SIMPLE_VARIABLE("subaddress-lookahead", set_subaddress_lookahead, tr("<major>:<minor>")); CHECK_SIMPLE_VARIABLE("segregation-height", set_segregation_height, tr("unsigned integer")); + CHECK_SIMPLE_VARIABLE("ignore-fractional-outputs", set_ignore_fractional_outputs, tr("0 or 1")); } fail_msg_writer() << tr("set: unrecognized argument(s)"); return true; @@ -3245,6 +3252,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) { try { + m_trusted_daemon = false; if (tools::is_local_address(m_wallet->get_daemon_address())) { MINFO(tr("Daemon is local, assuming trusted")); @@ -3807,7 +3815,6 @@ bool simple_wallet::start_mining(const std::vector<std::string>& args) req.miner_address = m_wallet->get_account().get_public_address_str(m_wallet->nettype()); bool ok = true; - size_t max_mining_threads_count = (std::max)(tools::get_max_concurrency(), static_cast<unsigned>(2)); size_t arg_size = args.size(); if(arg_size >= 3) { @@ -3823,7 +3830,7 @@ bool simple_wallet::start_mining(const std::vector<std::string>& args) { uint16_t num = 1; ok = string_tools::get_xtype_from_string(num, args[0]); - ok = ok && (1 <= num && num <= max_mining_threads_count); + ok = ok && 1 <= num; req.threads_count = num; } else @@ -3833,8 +3840,7 @@ bool simple_wallet::start_mining(const std::vector<std::string>& args) if (!ok) { - fail_msg_writer() << tr("invalid arguments. Please use start_mining [<number_of_threads>] [do_bg_mining] [ignore_battery], " - "<number_of_threads> should be from 1 to ") << max_mining_threads_count; + fail_msg_writer() << tr("invalid arguments. Please use start_mining [<number_of_threads>] [do_bg_mining] [ignore_battery]"); return true; } @@ -3893,13 +3899,40 @@ bool simple_wallet::set_daemon(const std::vector<std::string>& args) // If no port has been provided, use the default from config if (!match[3].length()) { - int daemon_port = m_wallet->nettype() == cryptonote::TESTNET ? config::testnet::RPC_DEFAULT_PORT : m_wallet->nettype() == cryptonote::STAGENET ? config::stagenet::RPC_DEFAULT_PORT : config::RPC_DEFAULT_PORT; + int daemon_port = get_config(m_wallet->nettype()).RPC_DEFAULT_PORT; daemon_url = match[1] + match[2] + std::string(":") + std::to_string(daemon_port); } else { daemon_url = args[0]; } LOCK_IDLE_SCOPE(); m_wallet->init(daemon_url); + + if (args.size() == 2) + { + if (args[1] == "trusted") + m_trusted_daemon = true; + else if (args[1] == "untrusted") + m_trusted_daemon = false; + else + { + fail_msg_writer() << tr("Expected trusted or untrusted, got ") << args[1] << ": assuming untrusted"; + m_trusted_daemon = false; + } + } + else + { + m_trusted_daemon = false; + try + { + if (tools::is_local_address(m_wallet->get_daemon_address())) + { + MINFO(tr("Daemon is local, assuming trusted")); + m_trusted_daemon = true; + } + } + catch (const std::exception &e) { } + } + success_msg_writer() << boost::format("Daemon set to %s, %s") % daemon_url % (*m_trusted_daemon ? tr("trusted") : tr("untrusted")); } else { fail_msg_writer() << tr("This does not seem to be a valid daemon URL."); } @@ -3940,6 +3973,24 @@ void simple_wallet::on_money_received(uint64_t height, const crypto::hash &txid, tr("txid ") << txid << ", " << print_money(amount) << ", " << tr("idx ") << subaddr_index; + + const uint64_t warn_height = m_wallet->nettype() == TESTNET ? 1000000 : m_wallet->nettype() == STAGENET ? 50000 : 1650000; + if (height >= warn_height) + { + std::vector<tx_extra_field> tx_extra_fields; + parse_tx_extra(tx.extra, tx_extra_fields); // failure ok + tx_extra_nonce extra_nonce; + if (find_tx_extra_field_by_type(tx_extra_fields, extra_nonce)) + { + crypto::hash8 payment_id8 = crypto::null_hash8; + if (get_encrypted_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id8)) + message_writer() << + tr("NOTE: this transaction uses an encrypted payment ID: consider using subaddresses instead"); + else + message_writer(console_color_red, false) << + tr("WARNING: this transaction uses an unencrypted payment ID: consider using subaddresses instead"); + } + } if (m_auto_refresh_refreshing) m_cmd_binder.print_prompt(); else @@ -3991,7 +4042,7 @@ bool simple_wallet::refresh_main(uint64_t start_height, bool reset, bool is_init { m_in_manual_refresh.store(true, std::memory_order_relaxed); epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){m_in_manual_refresh.store(false, std::memory_order_relaxed);}); - m_wallet->refresh(start_height, fetched_blocks); + m_wallet->refresh(is_daemon_trusted(), start_height, fetched_blocks); ok = true; // Clear line "Height xxx of xxx" std::cout << "\r \r"; @@ -4263,12 +4314,7 @@ uint64_t simple_wallet::get_daemon_blockchain_height(std::string& err) { throw std::runtime_error("simple_wallet null wallet"); } - - COMMAND_RPC_GET_HEIGHT::request req; - COMMAND_RPC_GET_HEIGHT::response res = boost::value_initialized<COMMAND_RPC_GET_HEIGHT::response>(); - bool r = m_wallet->invoke_http_json("/getheight", req, res); - err = interpret_rpc_response(r, res.status); - return res.height; + return m_wallet->get_daemon_blockchain_height(err); } //---------------------------------------------------------------------------------------------------- bool simple_wallet::show_blockchain_height(const std::vector<std::string>& args) @@ -4538,6 +4584,7 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri return true; } payment_id_seen = true; + message_writer() << tr("Unencrypted payment IDs are bad for privacy: ask the recipient to use subaddresses instead"); } uint64_t locked_blocks = 0; @@ -4561,6 +4608,7 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri } vector<cryptonote::tx_destination_entry> dsts; + size_t num_subaddresses = 0; for (size_t i = 0; i < local_args.size(); i += 2) { cryptonote::address_parse_info info; @@ -4572,6 +4620,7 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri } de.addr = info.address; de.is_subaddress = info.is_subaddress; + num_subaddresses += info.is_subaddress; if (info.has_payment_id) { @@ -4604,7 +4653,7 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri } // prompt is there is no payment id and confirmation is required - if (!payment_id_seen && m_wallet->confirm_missing_payment_id()) + if (!payment_id_seen && m_wallet->confirm_missing_payment_id() && dsts.size() > num_subaddresses) { std::string accepted = input_line(tr("No payment id is included with this transaction. Is this okay? (Y/Yes/N/No): ")); if (std::cin.eof()) @@ -4844,6 +4893,11 @@ bool simple_wallet::locked_transfer(const std::vector<std::string> &args_) return transfer_main(TransferLocked, args_); } //---------------------------------------------------------------------------------------------------- +bool simple_wallet::locked_sweep_all(const std::vector<std::string> &args_) +{ + return sweep_main(0, true, args_); +} +//---------------------------------------------------------------------------------------------------- bool simple_wallet::sweep_unmixable(const std::vector<std::string> &args_) { @@ -4924,6 +4978,20 @@ bool simple_wallet::sweep_unmixable(const std::vector<std::string> &args_) commit_or_save(ptx_vector, m_do_not_relay); } } + catch (const tools::error::not_enough_unlocked_money& e) + { + fail_msg_writer() << tr("Not enough money in unlocked balance"); + std::string accepted = input_line((boost::format(tr("Discarding %s of unmixable outputs that cannot be spent, which can be undone by \"rescan_spent\". Is this okay? (Y/Yes/N/No): ")) % print_money(e.available())).str()); + if (std::cin.eof()) + return true; + if (command_line::is_yes(accepted)) + { + try + { + m_wallet->discard_unmixable_outputs(is_daemon_trusted()); + } catch (...) {} + } + } catch (const std::exception &e) { handle_transfer_exception(std::current_exception(), is_daemon_trusted()); @@ -4937,12 +5005,16 @@ bool simple_wallet::sweep_unmixable(const std::vector<std::string> &args_) return true; } //---------------------------------------------------------------------------------------------------- -bool simple_wallet::sweep_main(uint64_t below, const std::vector<std::string> &args_) +bool simple_wallet::sweep_main(uint64_t below, bool locked, const std::vector<std::string> &args_) { - // sweep_all [index=<N1>[,<N2>,...]] [<ring_size>] <address> [<payment_id>] + auto print_usage = [below]() + { + fail_msg_writer() << boost::format(tr("usage: %s [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> [<payment_id>]")) % (below ? "sweep_below" : "sweep_all"); + }; if (args_.size() == 0) { fail_msg_writer() << tr("No address given"); + print_usage(); return true; } @@ -4956,7 +5028,10 @@ bool simple_wallet::sweep_main(uint64_t below, const std::vector<std::string> &a if (local_args.size() > 0 && local_args[0].substr(0, 6) == "index=") { if (!parse_subaddress_indices(local_args[0], subaddr_indices)) + { + print_usage(); return true; + } local_args.erase(local_args.begin()); } @@ -4993,9 +5068,44 @@ bool simple_wallet::sweep_main(uint64_t below, const std::vector<std::string> &a return true; } + uint64_t unlock_block = 0; + if (locked) { + uint64_t locked_blocks = 0; + + if (local_args.size() < 2) { + fail_msg_writer() << tr("missing lockedblocks parameter"); + return true; + } + + try + { + locked_blocks = boost::lexical_cast<uint64_t>(local_args[1]); + } + catch (const std::exception &e) + { + fail_msg_writer() << tr("bad locked_blocks parameter"); + return true; + } + if (locked_blocks > 1000000) + { + fail_msg_writer() << tr("Locked blocks too high, max 1000000 (Ëœ4 yrs)"); + return true; + } + std::string err; + uint64_t bc_height = get_daemon_blockchain_height(err); + if (!err.empty()) + { + fail_msg_writer() << tr("failed to get blockchain height: ") << err; + return true; + } + unlock_block = bc_height + locked_blocks; + + local_args.erase(local_args.begin() + 1); + } + std::vector<uint8_t> extra; bool payment_id_seen = false; - if (2 >= local_args.size()) + if (local_args.size() >= 2) { std::string payment_id_str = local_args.back(); @@ -5024,6 +5134,7 @@ bool simple_wallet::sweep_main(uint64_t below, const std::vector<std::string> &a if(!r && local_args.size() == 3) { fail_msg_writer() << tr("payment id has invalid format, expected 16 or 64 character hex string: ") << payment_id_str; + print_usage(); return true; } if (payment_id_seen) @@ -5034,6 +5145,7 @@ bool simple_wallet::sweep_main(uint64_t below, const std::vector<std::string> &a if (!cryptonote::get_account_address_from_str_or_url(info, m_wallet->nettype(), local_args[0], oa_prompter)) { fail_msg_writer() << tr("failed to parse address"); + print_usage(); return true; } @@ -5057,7 +5169,7 @@ bool simple_wallet::sweep_main(uint64_t below, const std::vector<std::string> &a } // prompt is there is no payment id and confirmation is required - if (!payment_id_seen && m_wallet->confirm_missing_payment_id()) + if (!payment_id_seen && m_wallet->confirm_missing_payment_id() && !info.is_subaddress) { std::string accepted = input_line(tr("No payment id is included with this transaction. Is this okay? (Y/Yes/N/No): ")); if (std::cin.eof()) @@ -5075,7 +5187,7 @@ bool simple_wallet::sweep_main(uint64_t below, const std::vector<std::string> &a try { // figure out what tx will be necessary - auto ptx_vector = m_wallet->create_transactions_all(below, info.address, info.is_subaddress, fake_outs_count, 0 /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices, is_daemon_trusted()); + auto ptx_vector = m_wallet->create_transactions_all(below, info.address, info.is_subaddress, fake_outs_count, unlock_block /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices, is_daemon_trusted()); if (ptx_vector.empty()) { @@ -5270,7 +5382,7 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) } // prompt if there is no payment id and confirmation is required - if (!payment_id_seen && m_wallet->confirm_missing_payment_id()) + if (!payment_id_seen && m_wallet->confirm_missing_payment_id() && !info.is_subaddress) { std::string accepted = input_line(tr("No payment id is included with this transaction. Is this okay? (Y/Yes/N/No): ")); if (std::cin.eof()) @@ -5371,7 +5483,7 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) //---------------------------------------------------------------------------------------------------- bool simple_wallet::sweep_all(const std::vector<std::string> &args_) { - return sweep_main(0, args_); + return sweep_main(0, false, args_); } //---------------------------------------------------------------------------------------------------- bool simple_wallet::sweep_below(const std::vector<std::string> &args_) @@ -5387,7 +5499,7 @@ bool simple_wallet::sweep_below(const std::vector<std::string> &args_) fail_msg_writer() << tr("invalid amount threshold"); return true; } - return sweep_main(below, std::vector<std::string>(++args_.begin(), args_.end())); + return sweep_main(below, false, std::vector<std::string>(++args_.begin(), args_.end())); } //---------------------------------------------------------------------------------------------------- bool simple_wallet::donate(const std::vector<std::string> &args_) @@ -5423,7 +5535,7 @@ bool simple_wallet::donate(const std::vector<std::string> &args_) local_args.push_back(amount_str); if (!payment_id_str.empty()) local_args.push_back(payment_id_str); - message_writer() << tr("Donating ") << amount_str << " to The Monero Project (donate.getmonero.org or "<< MONERO_DONATION_ADDR <<")."; + message_writer() << (boost::format(tr("Donating %s %s to The Monero Project (donate.getmonero.org or %s).")) % amount_str % cryptonote::get_unit(cryptonote::get_default_decimal_point()) % MONERO_DONATION_ADDR).str(); transfer_new(local_args); return true; } @@ -5593,9 +5705,9 @@ bool simple_wallet::sign_transfer(const std::vector<std::string> &args_) fail_msg_writer() << tr("This is a watch only wallet"); return true; } - if (args_.size() > 1 || (args_.size() == 1 && args_[0] != "export")) + if (args_.size() > 1 || (args_.size() == 1 && args_[0] != "export_raw")) { - fail_msg_writer() << tr("usage: sign_transfer [export]"); + fail_msg_writer() << tr("usage: sign_transfer [export_raw]"); return true; } if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } @@ -6501,7 +6613,7 @@ void simple_wallet::wallet_idle_thread() { uint64_t fetched_blocks; if (try_connect_to_daemon(true)) - m_wallet->refresh(0, fetched_blocks); + m_wallet->refresh(is_daemon_trusted(), 0, fetched_blocks); } catch(...) {} m_auto_refresh_refreshing = false; @@ -7394,8 +7506,12 @@ bool simple_wallet::show_transfer(const std::vector<std::string> &args) if (pd.m_unlock_time < CRYPTONOTE_MAX_BLOCK_NUMBER) { uint64_t bh = std::max(pd.m_unlock_time, pd.m_block_height + CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE); + uint64_t last_block_reward = m_wallet->get_last_block_reward(); + uint64_t suggested_threshold = last_block_reward ? (pd.m_amount + last_block_reward - 1) / last_block_reward : 0; if (bh >= last_block_height) success_msg_writer() << "Locked: " << (bh - last_block_height) << " blocks to unlock"; + else if (suggested_threshold > 0) + success_msg_writer() << std::to_string(last_block_height - bh) << " confirmations (" << suggested_threshold << " suggested threshold)"; else success_msg_writer() << std::to_string(last_block_height - bh) << " confirmations"; } @@ -7592,7 +7708,7 @@ int main(int argc, char* argv[]) std::tie(vm, should_terminate) = wallet_args::main( argc, argv, "monero-wallet-cli [--wallet-file=<file>|--generate-new-wallet=<file>] [<COMMAND>]", - sw::tr("This is the command line monero wallet. It needs to connect to a monero\ndaemon to work correctly.\nWARNING: Do not reuse your Monero keys on an another fork, UNLESS this fork has key reuse mitigations built in. Doing so will harm your privacy."), + sw::tr("This is the command line monero wallet. It needs to connect to a monero\ndaemon to work correctly.\nWARNING: Do not reuse your Monero keys on another fork, UNLESS this fork has key reuse mitigations built in. Doing so will harm your privacy."), desc_params, positional_options, [](const std::string &s, bool emphasis){ tools::scoped_message_writer(emphasis ? epee::console_color_white : epee::console_color_default, true) << s; }, diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h index 7a788d432..b8f7bb84b 100644 --- a/src/simplewallet/simplewallet.h +++ b/src/simplewallet/simplewallet.h @@ -137,6 +137,7 @@ namespace cryptonote bool set_key_reuse_mitigation2(const std::vector<std::string> &args = std::vector<std::string>()); bool set_subaddress_lookahead(const std::vector<std::string> &args = std::vector<std::string>()); bool set_segregation_height(const std::vector<std::string> &args = std::vector<std::string>()); + bool set_ignore_fractional_outputs(const std::vector<std::string> &args = std::vector<std::string>()); bool help(const std::vector<std::string> &args = std::vector<std::string>()); bool start_mining(const std::vector<std::string> &args); bool stop_mining(const std::vector<std::string> &args); @@ -152,7 +153,8 @@ namespace cryptonote bool transfer(const std::vector<std::string> &args); bool transfer_new(const std::vector<std::string> &args); bool locked_transfer(const std::vector<std::string> &args); - bool sweep_main(uint64_t below, const std::vector<std::string> &args); + bool locked_sweep_all(const std::vector<std::string> &args); + bool sweep_main(uint64_t below, bool locked, const std::vector<std::string> &args); bool sweep_all(const std::vector<std::string> &args); bool sweep_below(const std::vector<std::string> &args); bool sweep_single(const std::vector<std::string> &args); diff --git a/src/version.cpp.in b/src/version.cpp.in index a03da7889..55ba51f50 100644 --- a/src/version.cpp.in +++ b/src/version.cpp.in @@ -1,5 +1,5 @@ #define DEF_MONERO_VERSION_TAG "@VERSIONTAG@" -#define DEF_MONERO_VERSION "0.12.2.0-master" +#define DEF_MONERO_VERSION "0.12.3.0-master" #define DEF_MONERO_RELEASE_NAME "Lithium Luna" #define DEF_MONERO_VERSION_FULL DEF_MONERO_VERSION "-" DEF_MONERO_VERSION_TAG diff --git a/src/wallet/CMakeLists.txt b/src/wallet/CMakeLists.txt index a5a4c7f56..a16f4fe19 100644 --- a/src/wallet/CMakeLists.txt +++ b/src/wallet/CMakeLists.txt @@ -85,7 +85,6 @@ monero_add_executable(wallet_rpc_server target_link_libraries(wallet_rpc_server PRIVATE wallet - epee rpc_base cryptonote_core cncrypto diff --git a/src/wallet/api/address_book.h b/src/wallet/api/address_book.h index 7d9200550..f4ca68efd 100644 --- a/src/wallet/api/address_book.h +++ b/src/wallet/api/address_book.h @@ -42,16 +42,16 @@ public: ~AddressBookImpl(); // Fetches addresses from Wallet2 - void refresh(); - std::vector<AddressBookRow*> getAll() const; - bool addRow(const std::string &dst_addr , const std::string &payment_id, const std::string &description); - bool deleteRow(std::size_t rowId); + void refresh() override; + std::vector<AddressBookRow*> getAll() const override; + bool addRow(const std::string &dst_addr , const std::string &payment_id, const std::string &description) override; + bool deleteRow(std::size_t rowId) override; // Error codes. See AddressBook:ErrorCode enum in wallet2_api.h - std::string errorString() const {return m_errorString;} - int errorCode() const {return m_errorCode;} + std::string errorString() const override {return m_errorString;} + int errorCode() const override {return m_errorCode;} - int lookupPaymentID(const std::string &payment_id) const; + int lookupPaymentID(const std::string &payment_id) const override; private: void clearRows(); diff --git a/src/wallet/api/pending_transaction.h b/src/wallet/api/pending_transaction.h index 4f963c134..50b9f07ef 100644 --- a/src/wallet/api/pending_transaction.h +++ b/src/wallet/api/pending_transaction.h @@ -43,21 +43,21 @@ class PendingTransactionImpl : public PendingTransaction public: PendingTransactionImpl(WalletImpl &wallet); ~PendingTransactionImpl(); - int status() const; - std::string errorString() const; - bool commit(const std::string &filename = "", bool overwrite = false); - uint64_t amount() const; - uint64_t dust() const; - uint64_t fee() const; - std::vector<std::string> txid() const; - uint64_t txCount() const; - std::vector<uint32_t> subaddrAccount() const; - std::vector<std::set<uint32_t>> subaddrIndices() const; + int status() const override; + std::string errorString() const override; + bool commit(const std::string &filename = "", bool overwrite = false) override; + uint64_t amount() const override; + uint64_t dust() const override; + uint64_t fee() const override; + std::vector<std::string> txid() const override; + uint64_t txCount() const override; + std::vector<uint32_t> subaddrAccount() const override; + std::vector<std::set<uint32_t>> subaddrIndices() const override; // TODO: continue with interface; - std::string multisigSignData(); - void signMultisigTx(); - std::vector<std::string> signersKeys() const; + std::string multisigSignData() override; + void signMultisigTx() override; + std::vector<std::string> signersKeys() const override; private: friend class WalletImpl; diff --git a/src/wallet/api/subaddress.h b/src/wallet/api/subaddress.h index 3f9e37ac8..f3db7d97b 100644 --- a/src/wallet/api/subaddress.h +++ b/src/wallet/api/subaddress.h @@ -40,10 +40,10 @@ public: ~SubaddressImpl(); // Fetches addresses from Wallet2 - void refresh(uint32_t accountIndex); - std::vector<SubaddressRow*> getAll() const; - void addRow(uint32_t accountIndex, const std::string &label); - void setLabel(uint32_t accountIndex, uint32_t addressIndex, const std::string &label); + void refresh(uint32_t accountIndex) override; + std::vector<SubaddressRow*> getAll() const override; + void addRow(uint32_t accountIndex, const std::string &label) override; + void setLabel(uint32_t accountIndex, uint32_t addressIndex, const std::string &label) override; private: void clearRows(); diff --git a/src/wallet/api/transaction_info.h b/src/wallet/api/transaction_info.h index 5df9a44ef..37e0445d9 100644 --- a/src/wallet/api/transaction_info.h +++ b/src/wallet/api/transaction_info.h @@ -42,24 +42,24 @@ public: TransactionInfoImpl(); ~TransactionInfoImpl(); //! in/out - virtual int direction() const; + virtual int direction() const override; //! true if hold - virtual bool isPending() const; - virtual bool isFailed() const; - virtual uint64_t amount() const; + virtual bool isPending() const override; + virtual bool isFailed() const override; + virtual uint64_t amount() const override; //! always 0 for incoming txes - virtual uint64_t fee() const; - virtual uint64_t blockHeight() const; - virtual std::set<uint32_t> subaddrIndex() const; - virtual uint32_t subaddrAccount() const; - virtual std::string label() const; + virtual uint64_t fee() const override; + virtual uint64_t blockHeight() const override; + virtual std::set<uint32_t> subaddrIndex() const override; + virtual uint32_t subaddrAccount() const override; + virtual std::string label() const override; - virtual std::string hash() const; - virtual std::time_t timestamp() const; - virtual std::string paymentId() const; - virtual const std::vector<Transfer> &transfers() const; - virtual uint64_t confirmations() const; - virtual uint64_t unlockTime() const; + virtual std::string hash() const override; + virtual std::time_t timestamp() const override; + virtual std::string paymentId() const override; + virtual const std::vector<Transfer> &transfers() const override; + virtual uint64_t confirmations() const override; + virtual uint64_t unlockTime() const override; private: int m_direction; diff --git a/src/wallet/api/unsigned_transaction.h b/src/wallet/api/unsigned_transaction.h index a35630535..8a3330014 100644 --- a/src/wallet/api/unsigned_transaction.h +++ b/src/wallet/api/unsigned_transaction.h @@ -43,19 +43,18 @@ class UnsignedTransactionImpl : public UnsignedTransaction public: UnsignedTransactionImpl(WalletImpl &wallet); ~UnsignedTransactionImpl(); - int status() const; - std::string errorString() const; - std::vector<uint64_t> amount() const; - std::vector<uint64_t> dust() const; - std::vector<uint64_t> fee() const; - std::vector<uint64_t> mixin() const; - std::vector<std::string> paymentId() const; - std::vector<std::string> recipientAddress() const; - uint64_t txCount() const; + int status() const override; + std::string errorString() const override; + std::vector<uint64_t> amount() const override; + std::vector<uint64_t> fee() const override; + std::vector<uint64_t> mixin() const override; + std::vector<std::string> paymentId() const override; + std::vector<std::string> recipientAddress() const override; + uint64_t txCount() const override; // sign txs and save to file - bool sign(const std::string &signedFileName); - std::string confirmationMessage() const {return m_confirmationMessage;} - uint64_t minMixinCount() const; + bool sign(const std::string &signedFileName) override; + std::string confirmationMessage() const override {return m_confirmationMessage;} + uint64_t minMixinCount() const override; private: // Callback function to check all loaded tx's and generate confirmationMessage diff --git a/src/wallet/api/wallet.cpp b/src/wallet/api/wallet.cpp index cb9122134..680da26ce 100644 --- a/src/wallet/api/wallet.cpp +++ b/src/wallet/api/wallet.cpp @@ -372,6 +372,7 @@ WalletImpl::WalletImpl(NetworkType nettype) , m_trustedDaemon(false) , m_wallet2Callback(nullptr) , m_recoveringFromSeed(false) + , m_recoveringFromDevice(false) , m_synchronized(false) , m_rebuildWalletCache(false) , m_is_connected(false) @@ -419,6 +420,7 @@ bool WalletImpl::create(const std::string &path, const std::string &password, co clearStatus(); m_recoveringFromSeed = false; + m_recoveringFromDevice = false; bool keys_file_exists; bool wallet_file_exists; tools::wallet2::wallet_exists(path, keys_file_exists, wallet_file_exists); @@ -554,18 +556,26 @@ bool WalletImpl::recoverFromKeysWithPassword(const std::string &path, } // parse view secret key + bool has_viewkey = true; + crypto::secret_key viewkey; if (viewkey_string.empty()) { - setStatusError(tr("No view key supplied, cancelled")); - return false; + if(has_spendkey) { + has_viewkey = false; + } + else { + setStatusError(tr("Neither view key nor spend key supplied, cancelled")); + return false; + } } - cryptonote::blobdata viewkey_data; - if(!epee::string_tools::parse_hexstr_to_binbuff(viewkey_string, viewkey_data) || viewkey_data.size() != sizeof(crypto::secret_key)) - { - setStatusError(tr("failed to parse secret view key")); - return false; + if(has_viewkey) { + cryptonote::blobdata viewkey_data; + if(!epee::string_tools::parse_hexstr_to_binbuff(viewkey_string, viewkey_data) || viewkey_data.size() != sizeof(crypto::secret_key)) + { + setStatusError(tr("failed to parse secret view key")); + return false; + } + viewkey = *reinterpret_cast<const crypto::secret_key*>(viewkey_data.data()); } - crypto::secret_key viewkey = *reinterpret_cast<const crypto::secret_key*>(viewkey_data.data()); - // check the spend and view keys match the given address crypto::public_key pkey; if(has_spendkey) { @@ -578,26 +588,32 @@ bool WalletImpl::recoverFromKeysWithPassword(const std::string &path, return false; } } - if (!crypto::secret_key_to_public_key(viewkey, pkey)) { - setStatusError(tr("failed to verify secret view key")); - return false; - } - if (info.address.m_view_public_key != pkey) { - setStatusError(tr("view key does not match address")); - return false; + if(has_viewkey) { + if (!crypto::secret_key_to_public_key(viewkey, pkey)) { + setStatusError(tr("failed to verify secret view key")); + return false; + } + if (info.address.m_view_public_key != pkey) { + setStatusError(tr("view key does not match address")); + return false; + } } try { - if (has_spendkey) { + if (has_spendkey && has_viewkey) { m_wallet->generate(path, password, info.address, spendkey, viewkey); - setSeedLanguage(language); - LOG_PRINT_L1("Generated new wallet from keys with seed language: " + language); + LOG_PRINT_L1("Generated new wallet from spend key and view key"); } - else { + if(!has_spendkey && has_viewkey) { m_wallet->generate(path, password, info.address, viewkey); LOG_PRINT_L1("Generated new view only wallet from keys"); } + if(has_spendkey && !has_viewkey) { + m_wallet->generate(path, password, spendkey, true, false, false); + setSeedLanguage(language); + LOG_PRINT_L1("Generated deterministic wallet from spend key with seed language: " + language); + } } catch (const std::exception& e) { @@ -607,11 +623,28 @@ bool WalletImpl::recoverFromKeysWithPassword(const std::string &path, return true; } +bool WalletImpl::recoverFromDevice(const std::string &path, const std::string &password, const std::string &device_name) +{ + clearStatus(); + m_recoveringFromSeed = false; + m_recoveringFromDevice = true; + try + { + m_wallet->restore(path, password, device_name); + LOG_PRINT_L1("Generated new wallet from device: " + device_name); + } + catch (const std::exception& e) { + setStatusError(string(tr("failed to generate new wallet: ")) + e.what()); + return false; + } + return true; +} bool WalletImpl::open(const std::string &path, const std::string &password) { clearStatus(); m_recoveringFromSeed = false; + m_recoveringFromDevice = false; try { // TODO: handle "deprecated" // Check if wallet cache exists @@ -649,6 +682,7 @@ bool WalletImpl::recover(const std::string &path, const std::string &password, c } m_recoveringFromSeed = true; + m_recoveringFromDevice = false; crypto::secret_key recovery_key; std::string old_language; if (!crypto::ElectrumWords::words_to_bytes(seed, recovery_key, old_language)) { @@ -687,6 +721,7 @@ bool WalletImpl::close(bool store) LOG_PRINT_L1("Calling wallet::stop..."); m_wallet->stop(); LOG_PRINT_L1("wallet::stop done"); + m_wallet->deinit(); result = true; clearStatus(); } catch (const std::exception &e) { @@ -870,6 +905,16 @@ void WalletImpl::setRecoveringFromSeed(bool recoveringFromSeed) m_recoveringFromSeed = recoveringFromSeed; } +void WalletImpl::setRecoveringFromDevice(bool recoveringFromDevice) +{ + m_recoveringFromDevice = recoveringFromDevice; +} + +void WalletImpl::setSubaddressLookahead(uint32_t major, uint32_t minor) +{ + m_wallet->set_subaddress_lookahead(major, minor); +} + uint64_t WalletImpl::balance(uint32_t accountIndex) const { return m_wallet->balance(accountIndex); @@ -1921,7 +1966,7 @@ void WalletImpl::doRefresh() // Syncing daemon and refreshing wallet simultaneously is very resource intensive. // Disable refresh if wallet is disconnected or daemon isn't synced. if (m_wallet->light_wallet() || daemonSynced()) { - m_wallet->refresh(); + m_wallet->refresh(trustedDaemon()); if (!m_synchronized) { m_synchronized = true; } @@ -1982,7 +2027,7 @@ bool WalletImpl::isNewWallet() const // with the daemon (pull hashes instead of pull blocks). // If wallet cache is rebuilt, creation height stored in .keys is used. // Watch only wallet is a copy of an existing wallet. - return !(blockChainHeight() > 1 || m_recoveringFromSeed || m_rebuildWalletCache) && !watchOnly(); + return !(blockChainHeight() > 1 || m_recoveringFromSeed || m_recoveringFromDevice || m_rebuildWalletCache) && !watchOnly(); } bool WalletImpl::doInit(const string &daemon_address, uint64_t upper_transaction_size_limit, bool ssl) @@ -2160,6 +2205,20 @@ void WalletImpl::keyReuseMitigation2(bool mitigation) m_wallet->key_reuse_mitigation2(mitigation); } +bool WalletImpl::lockKeysFile() +{ + return m_wallet->lock_keys_file(); +} + +bool WalletImpl::unlockKeysFile() +{ + return m_wallet->unlock_keys_file(); +} + +bool WalletImpl::isKeysFileLocked() +{ + return m_wallet->is_keys_file_locked(); +} } // namespace namespace Bitmonero = Monero; diff --git a/src/wallet/api/wallet.h b/src/wallet/api/wallet.h index 813ca4b30..58be686fc 100644 --- a/src/wallet/api/wallet.h +++ b/src/wallet/api/wallet.h @@ -57,7 +57,7 @@ public: bool create(const std::string &path, const std::string &password, const std::string &language); bool createWatchOnly(const std::string &path, const std::string &password, - const std::string &language) const; + const std::string &language) const override; bool open(const std::string &path, const std::string &password); bool recover(const std::string &path,const std::string &password, const std::string &seed); @@ -76,57 +76,62 @@ public: const std::string &address_string, const std::string &viewkey_string, const std::string &spendkey_string = ""); + bool recoverFromDevice(const std::string &path, + const std::string &password, + const std::string &device_name); bool close(bool store = true); - std::string seed() const; - std::string getSeedLanguage() const; - void setSeedLanguage(const std::string &arg); + std::string seed() const override; + std::string getSeedLanguage() const override; + void setSeedLanguage(const std::string &arg) override; // void setListener(Listener *) {} - int status() const; - std::string errorString() const; + int status() const override; + std::string errorString() const override; void statusWithErrorString(int& status, std::string& errorString) const override; - bool setPassword(const std::string &password); - std::string address(uint32_t accountIndex = 0, uint32_t addressIndex = 0) const; - std::string integratedAddress(const std::string &payment_id) const; - std::string secretViewKey() const; - std::string publicViewKey() const; - std::string secretSpendKey() const; - std::string publicSpendKey() const; - std::string publicMultisigSignerKey() const; - std::string path() const; - bool store(const std::string &path); - std::string filename() const; - std::string keysFilename() const; - bool init(const std::string &daemon_address, uint64_t upper_transaction_size_limit = 0, const std::string &daemon_username = "", const std::string &daemon_password = "", bool use_ssl = false, bool lightWallet = false); - bool connectToDaemon(); - ConnectionStatus connected() const; - void setTrustedDaemon(bool arg); - bool trustedDaemon() const; - uint64_t balance(uint32_t accountIndex = 0) const; - uint64_t unlockedBalance(uint32_t accountIndex = 0) const; - uint64_t blockChainHeight() const; - uint64_t approximateBlockChainHeight() const; - uint64_t daemonBlockChainHeight() const; - uint64_t daemonBlockChainTargetHeight() const; - bool synchronized() const; - bool refresh(); - void refreshAsync(); - void setAutoRefreshInterval(int millis); - int autoRefreshInterval() const; - void setRefreshFromBlockHeight(uint64_t refresh_from_block_height); - uint64_t getRefreshFromBlockHeight() const { return m_wallet->get_refresh_from_block_height(); }; - void setRecoveringFromSeed(bool recoveringFromSeed); - bool watchOnly() const; - bool rescanSpent(); - NetworkType nettype() const {return static_cast<NetworkType>(m_wallet->nettype());} - void hardForkInfo(uint8_t &version, uint64_t &earliest_height) const; - bool useForkRules(uint8_t version, int64_t early_blocks) const; - - void addSubaddressAccount(const std::string& label); - size_t numSubaddressAccounts() const; - size_t numSubaddresses(uint32_t accountIndex) const; - void addSubaddress(uint32_t accountIndex, const std::string& label); - std::string getSubaddressLabel(uint32_t accountIndex, uint32_t addressIndex) const; - void setSubaddressLabel(uint32_t accountIndex, uint32_t addressIndex, const std::string &label); + bool setPassword(const std::string &password) override; + std::string address(uint32_t accountIndex = 0, uint32_t addressIndex = 0) const override; + std::string integratedAddress(const std::string &payment_id) const override; + std::string secretViewKey() const override; + std::string publicViewKey() const override; + std::string secretSpendKey() const override; + std::string publicSpendKey() const override; + std::string publicMultisigSignerKey() const override; + std::string path() const override; + bool store(const std::string &path) override; + std::string filename() const override; + std::string keysFilename() const override; + bool init(const std::string &daemon_address, uint64_t upper_transaction_size_limit = 0, const std::string &daemon_username = "", const std::string &daemon_password = "", bool use_ssl = false, bool lightWallet = false) override; + bool connectToDaemon() override; + ConnectionStatus connected() const override; + void setTrustedDaemon(bool arg) override; + bool trustedDaemon() const override; + uint64_t balance(uint32_t accountIndex = 0) const override; + uint64_t unlockedBalance(uint32_t accountIndex = 0) const override; + uint64_t blockChainHeight() const override; + uint64_t approximateBlockChainHeight() const override; + uint64_t daemonBlockChainHeight() const override; + uint64_t daemonBlockChainTargetHeight() const override; + bool synchronized() const override; + bool refresh() override; + void refreshAsync() override; + void setAutoRefreshInterval(int millis) override; + int autoRefreshInterval() const override; + void setRefreshFromBlockHeight(uint64_t refresh_from_block_height) override; + uint64_t getRefreshFromBlockHeight() const override { return m_wallet->get_refresh_from_block_height(); }; + void setRecoveringFromSeed(bool recoveringFromSeed) override; + void setRecoveringFromDevice(bool recoveringFromDevice) override; + void setSubaddressLookahead(uint32_t major, uint32_t minor) override; + bool watchOnly() const override; + bool rescanSpent() override; + NetworkType nettype() const override {return static_cast<NetworkType>(m_wallet->nettype());} + void hardForkInfo(uint8_t &version, uint64_t &earliest_height) const override; + bool useForkRules(uint8_t version, int64_t early_blocks) const override; + + void addSubaddressAccount(const std::string& label) override; + size_t numSubaddressAccounts() const override; + size_t numSubaddresses(uint32_t accountIndex) const override; + void addSubaddress(uint32_t accountIndex, const std::string& label) override; + std::string getSubaddressLabel(uint32_t accountIndex, uint32_t addressIndex) const override; + void setSubaddressLabel(uint32_t accountIndex, uint32_t addressIndex, const std::string &label) override; MultisigState multisig() const override; std::string getMultisigInfo() const override; @@ -140,49 +145,52 @@ public: optional<uint64_t> amount, uint32_t mixin_count, PendingTransaction::Priority priority = PendingTransaction::Priority_Low, uint32_t subaddr_account = 0, - std::set<uint32_t> subaddr_indices = {}); - virtual PendingTransaction * createSweepUnmixableTransaction(); - bool submitTransaction(const std::string &fileName); - virtual UnsignedTransaction * loadUnsignedTx(const std::string &unsigned_filename); - bool exportKeyImages(const std::string &filename); - bool importKeyImages(const std::string &filename); - - virtual void disposeTransaction(PendingTransaction * t); - virtual TransactionHistory * history(); - virtual AddressBook * addressBook(); - virtual Subaddress * subaddress(); - virtual SubaddressAccount * subaddressAccount(); - virtual void setListener(WalletListener * l); - virtual uint32_t defaultMixin() const; - virtual void setDefaultMixin(uint32_t arg); - virtual bool setUserNote(const std::string &txid, const std::string ¬e); - virtual std::string getUserNote(const std::string &txid) const; - virtual std::string getTxKey(const std::string &txid) const; - virtual bool checkTxKey(const std::string &txid, std::string tx_key, const std::string &address, uint64_t &received, bool &in_pool, uint64_t &confirmations); - virtual std::string getTxProof(const std::string &txid, const std::string &address, const std::string &message) const; - virtual bool checkTxProof(const std::string &txid, const std::string &address, const std::string &message, const std::string &signature, bool &good, uint64_t &received, bool &in_pool, uint64_t &confirmations); - virtual std::string getSpendProof(const std::string &txid, const std::string &message) const; - virtual bool checkSpendProof(const std::string &txid, const std::string &message, const std::string &signature, bool &good) const; - virtual std::string getReserveProof(bool all, uint32_t account_index, uint64_t amount, const std::string &message) const; - virtual bool checkReserveProof(const std::string &address, const std::string &message, const std::string &signature, bool &good, uint64_t &total, uint64_t &spent) const; - virtual std::string signMessage(const std::string &message); - virtual bool verifySignedMessage(const std::string &message, const std::string &address, const std::string &signature) const; - virtual std::string signMultisigParticipant(const std::string &message) const; - virtual bool verifyMessageWithPublicKey(const std::string &message, const std::string &publicKey, const std::string &signature) const; - virtual void startRefresh(); - virtual void pauseRefresh(); - virtual bool parse_uri(const std::string &uri, std::string &address, std::string &payment_id, uint64_t &amount, std::string &tx_description, std::string &recipient_name, std::vector<std::string> &unknown_parameters, std::string &error); - virtual std::string getDefaultDataDir() const; - virtual bool lightWalletLogin(bool &isNewWallet) const; - virtual bool lightWalletImportWalletRequest(std::string &payment_id, uint64_t &fee, bool &new_request, bool &request_fulfilled, std::string &payment_address, std::string &status); - virtual bool blackballOutputs(const std::vector<std::string> &pubkeys, bool add); - virtual bool unblackballOutput(const std::string &pubkey); - virtual bool getRing(const std::string &key_image, std::vector<uint64_t> &ring) const; - virtual bool getRings(const std::string &txid, std::vector<std::pair<std::string, std::vector<uint64_t>>> &rings) const; - virtual bool setRing(const std::string &key_image, const std::vector<uint64_t> &ring, bool relative); - virtual void segregatePreForkOutputs(bool segregate); - virtual void segregationHeight(uint64_t height); - virtual void keyReuseMitigation2(bool mitigation); + std::set<uint32_t> subaddr_indices = {}) override; + virtual PendingTransaction * createSweepUnmixableTransaction() override; + bool submitTransaction(const std::string &fileName) override; + virtual UnsignedTransaction * loadUnsignedTx(const std::string &unsigned_filename) override; + bool exportKeyImages(const std::string &filename) override; + bool importKeyImages(const std::string &filename) override; + + virtual void disposeTransaction(PendingTransaction * t) override; + virtual TransactionHistory * history() override; + virtual AddressBook * addressBook() override; + virtual Subaddress * subaddress() override; + virtual SubaddressAccount * subaddressAccount() override; + virtual void setListener(WalletListener * l) override; + virtual uint32_t defaultMixin() const override; + virtual void setDefaultMixin(uint32_t arg) override; + virtual bool setUserNote(const std::string &txid, const std::string ¬e) override; + virtual std::string getUserNote(const std::string &txid) const override; + virtual std::string getTxKey(const std::string &txid) const override; + virtual bool checkTxKey(const std::string &txid, std::string tx_key, const std::string &address, uint64_t &received, bool &in_pool, uint64_t &confirmations) override; + virtual std::string getTxProof(const std::string &txid, const std::string &address, const std::string &message) const override; + virtual bool checkTxProof(const std::string &txid, const std::string &address, const std::string &message, const std::string &signature, bool &good, uint64_t &received, bool &in_pool, uint64_t &confirmations) override; + virtual std::string getSpendProof(const std::string &txid, const std::string &message) const override; + virtual bool checkSpendProof(const std::string &txid, const std::string &message, const std::string &signature, bool &good) const override; + virtual std::string getReserveProof(bool all, uint32_t account_index, uint64_t amount, const std::string &message) const override; + virtual bool checkReserveProof(const std::string &address, const std::string &message, const std::string &signature, bool &good, uint64_t &total, uint64_t &spent) const override; + virtual std::string signMessage(const std::string &message) override; + virtual bool verifySignedMessage(const std::string &message, const std::string &address, const std::string &signature) const override; + virtual std::string signMultisigParticipant(const std::string &message) const override; + virtual bool verifyMessageWithPublicKey(const std::string &message, const std::string &publicKey, const std::string &signature) const override; + virtual void startRefresh() override; + virtual void pauseRefresh() override; + virtual bool parse_uri(const std::string &uri, std::string &address, std::string &payment_id, uint64_t &amount, std::string &tx_description, std::string &recipient_name, std::vector<std::string> &unknown_parameters, std::string &error) override; + virtual std::string getDefaultDataDir() const override; + virtual bool lightWalletLogin(bool &isNewWallet) const override; + virtual bool lightWalletImportWalletRequest(std::string &payment_id, uint64_t &fee, bool &new_request, bool &request_fulfilled, std::string &payment_address, std::string &status) override; + virtual bool blackballOutputs(const std::vector<std::string> &pubkeys, bool add) override; + virtual bool unblackballOutput(const std::string &pubkey) override; + virtual bool getRing(const std::string &key_image, std::vector<uint64_t> &ring) const override; + virtual bool getRings(const std::string &txid, std::vector<std::pair<std::string, std::vector<uint64_t>>> &rings) const override; + virtual bool setRing(const std::string &key_image, const std::vector<uint64_t> &ring, bool relative) override; + virtual void segregatePreForkOutputs(bool segregate) override; + virtual void segregationHeight(uint64_t height) override; + virtual void keyReuseMitigation2(bool mitigation) override; + virtual bool lockKeysFile() override; + virtual bool unlockKeysFile() override; + virtual bool isKeysFileLocked() override; private: void clearStatus() const; @@ -232,6 +240,7 @@ private: // so it shouldn't be considered as new and pull blocks (slow-refresh) // instead of pulling hashes (fast-refresh) std::atomic<bool> m_recoveringFromSeed; + std::atomic<bool> m_recoveringFromDevice; std::atomic<bool> m_synchronized; std::atomic<bool> m_rebuildWalletCache; // cache connection status to avoid unnecessary RPC calls diff --git a/src/wallet/api/wallet2_api.h b/src/wallet/api/wallet2_api.h index 5b99bd975..0cd0ff5cf 100644 --- a/src/wallet/api/wallet2_api.h +++ b/src/wallet/api/wallet2_api.h @@ -509,6 +509,21 @@ struct Wallet */ virtual void setRecoveringFromSeed(bool recoveringFromSeed) = 0; + /*! + * \brief setRecoveringFromDevice - set state to recovering from device + * + * \param recoveringFromDevice - true/false + */ + virtual void setRecoveringFromDevice(bool recoveringFromDevice) = 0; + + /*! + * \brief setSubaddressLookahead - set size of subaddress lookahead + * + * \param major - size fot the major index + * \param minor - size fot the minor index + */ + virtual void setSubaddressLookahead(uint32_t major, uint32_t minor) = 0; + /** * @brief connectToDaemon - connects to the daemon. TODO: check if it can be removed * @return @@ -885,6 +900,12 @@ struct Wallet //! Initiates a light wallet import wallet request virtual bool lightWalletImportWalletRequest(std::string &payment_id, uint64_t &fee, bool &new_request, bool &request_fulfilled, std::string &payment_address, std::string &status) = 0; + + //! locks/unlocks the keys file; returns true on success + virtual bool lockKeysFile() = 0; + virtual bool unlockKeysFile() = 0; + //! returns true if the keys file is locked + virtual bool isKeysFileLocked() = 0; }; /** @@ -1015,6 +1036,23 @@ struct WalletManager } /*! + * \brief creates wallet using hardware device. + * \param path Name of wallet file to be created + * \param password Password of wallet file + * \param nettype Network type + * \param deviceName Device name + * \param restoreHeight restore from start height (0 sets to current height) + * \param subaddressLookahead Size of subaddress lookahead (empty sets to some default low value) + * \return Wallet instance (Wallet::status() needs to be called to check if recovered successfully) + */ + virtual Wallet * createWalletFromDevice(const std::string &path, + const std::string &password, + NetworkType nettype, + const std::string &deviceName, + uint64_t restoreHeight = 0, + const std::string &subaddressLookahead = "") = 0; + + /*! * \brief Closes wallet. In case operation succeeded, wallet object deleted. in case operation failed, wallet object not deleted * \param wallet previously opened / created wallet instance * \return None @@ -1038,6 +1076,10 @@ struct WalletManager * @param password - password to verify * @param no_spend_key - verify only view keys? * @return - true if password is correct + * + * @note + * This function will fail when the wallet keys file is opened because the wallet program locks the keys file. + * In this case, Wallet::unlockKeysFile() and Wallet::lockKeysFile() need to be called before and after the call to this function, respectively. */ virtual bool verifyWalletPassword(const std::string &keys_file_name, const std::string &password, bool no_spend_key) const = 0; diff --git a/src/wallet/api/wallet_manager.cpp b/src/wallet/api/wallet_manager.cpp index a63716576..99eadc82f 100644 --- a/src/wallet/api/wallet_manager.cpp +++ b/src/wallet/api/wallet_manager.cpp @@ -114,6 +114,26 @@ Wallet *WalletManagerImpl::createWalletFromKeys(const std::string &path, return wallet; } +Wallet *WalletManagerImpl::createWalletFromDevice(const std::string &path, + const std::string &password, + NetworkType nettype, + const std::string &deviceName, + uint64_t restoreHeight, + const std::string &subaddressLookahead) +{ + WalletImpl * wallet = new WalletImpl(nettype); + if(restoreHeight > 0){ + wallet->setRefreshFromBlockHeight(restoreHeight); + } + auto lookahead = tools::parse_subaddress_lookahead(subaddressLookahead); + if (lookahead) + { + wallet->setSubaddressLookahead(lookahead->first, lookahead->second); + } + wallet->recoverFromDevice(path, password, deviceName); + return wallet; +} + bool WalletManagerImpl::closeWallet(Wallet *wallet, bool store) { WalletImpl * wallet_ = dynamic_cast<WalletImpl*>(wallet); diff --git a/src/wallet/api/wallet_manager.h b/src/wallet/api/wallet_manager.h index 26238b658..656a7142c 100644 --- a/src/wallet/api/wallet_manager.h +++ b/src/wallet/api/wallet_manager.h @@ -39,13 +39,13 @@ class WalletManagerImpl : public WalletManager { public: Wallet * createWallet(const std::string &path, const std::string &password, - const std::string &language, NetworkType nettype); - Wallet * openWallet(const std::string &path, const std::string &password, NetworkType nettype); + const std::string &language, NetworkType nettype) override; + Wallet * openWallet(const std::string &path, const std::string &password, NetworkType nettype) override; virtual Wallet * recoveryWallet(const std::string &path, const std::string &password, const std::string &mnemonic, NetworkType nettype, - uint64_t restoreHeight); + uint64_t restoreHeight) override; virtual Wallet * createWalletFromKeys(const std::string &path, const std::string &password, const std::string &language, @@ -53,9 +53,9 @@ public: uint64_t restoreHeight, const std::string &addressString, const std::string &viewKeyString, - const std::string &spendKeyString = ""); + const std::string &spendKeyString = "") override; // next two methods are deprecated - use the above version which allow setting of a password - virtual Wallet * recoveryWallet(const std::string &path, const std::string &mnemonic, NetworkType nettype, uint64_t restoreHeight); + virtual Wallet * recoveryWallet(const std::string &path, const std::string &mnemonic, NetworkType nettype, uint64_t restoreHeight) override; // deprecated: use createWalletFromKeys(..., password, ...) instead virtual Wallet * createWalletFromKeys(const std::string &path, const std::string &language, @@ -63,23 +63,29 @@ public: uint64_t restoreHeight, const std::string &addressString, const std::string &viewKeyString, - const std::string &spendKeyString = ""); - virtual bool closeWallet(Wallet *wallet, bool store = true); - bool walletExists(const std::string &path); - bool verifyWalletPassword(const std::string &keys_file_name, const std::string &password, bool no_spend_key) const; - std::vector<std::string> findWallets(const std::string &path); - std::string errorString() const; - void setDaemonAddress(const std::string &address); - bool connected(uint32_t *version = NULL); - uint64_t blockchainHeight(); - uint64_t blockchainTargetHeight(); - uint64_t networkDifficulty(); - double miningHashRate(); - uint64_t blockTarget(); - bool isMining(); - bool startMining(const std::string &address, uint32_t threads = 1, bool background_mining = false, bool ignore_battery = true); - bool stopMining(); - std::string resolveOpenAlias(const std::string &address, bool &dnssec_valid) const; + const std::string &spendKeyString = "") override; + virtual Wallet * createWalletFromDevice(const std::string &path, + const std::string &password, + NetworkType nettype, + const std::string &deviceName, + uint64_t restoreHeight = 0, + const std::string &subaddressLookahead = "") override; + virtual bool closeWallet(Wallet *wallet, bool store = true) override; + bool walletExists(const std::string &path) override; + bool verifyWalletPassword(const std::string &keys_file_name, const std::string &password, bool no_spend_key) const override; + std::vector<std::string> findWallets(const std::string &path) override; + std::string errorString() const override; + void setDaemonAddress(const std::string &address) override; + bool connected(uint32_t *version = NULL) override; + uint64_t blockchainHeight() override; + uint64_t blockchainTargetHeight() override; + uint64_t networkDifficulty() override; + double miningHashRate() override; + uint64_t blockTarget() override; + bool isMining() override; + bool startMining(const std::string &address, uint32_t threads = 1, bool background_mining = false, bool ignore_battery = true) override; + bool stopMining() override; + std::string resolveOpenAlias(const std::string &address, bool &dnssec_valid) const override; private: WalletManagerImpl() {} diff --git a/src/wallet/node_rpc_proxy.cpp b/src/wallet/node_rpc_proxy.cpp index c5d869354..401ada61b 100644 --- a/src/wallet/node_rpc_proxy.cpp +++ b/src/wallet/node_rpc_proxy.cpp @@ -41,21 +41,13 @@ static const std::chrono::seconds rpc_timeout = std::chrono::minutes(3) + std::c NodeRPCProxy::NodeRPCProxy(epee::net_utils::http::http_simple_client &http_client, boost::mutex &mutex) : m_http_client(http_client) , m_daemon_rpc_mutex(mutex) - , m_height(0) - , m_height_time(0) - , m_earliest_height() - , m_dynamic_per_kb_fee_estimate(0) - , m_dynamic_per_kb_fee_estimate_cached_height(0) - , m_dynamic_per_kb_fee_estimate_grace_blocks(0) - , m_rpc_version(0) - , m_target_height(0) - , m_target_height_time(0) -{} +{ + invalidate(); +} void NodeRPCProxy::invalidate() { m_height = 0; - m_height_time = 0; for (size_t n = 0; n < 256; ++n) m_earliest_height[n] = 0; m_dynamic_per_kb_fee_estimate = 0; @@ -63,7 +55,8 @@ void NodeRPCProxy::invalidate() m_dynamic_per_kb_fee_estimate_grace_blocks = 0; m_rpc_version = 0; m_target_height = 0; - m_target_height_time = 0; + m_block_size_limit = 0; + m_get_info_time = 0; } boost::optional<std::string> NodeRPCProxy::get_rpc_version(uint32_t &rpc_version) const @@ -84,36 +77,15 @@ boost::optional<std::string> NodeRPCProxy::get_rpc_version(uint32_t &rpc_version return boost::optional<std::string>(); } -boost::optional<std::string> NodeRPCProxy::get_height(uint64_t &height) const -{ - const time_t now = time(NULL); - if (m_height == 0 || now >= m_height_time + 30) // re-cache every 30 seconds - { - cryptonote::COMMAND_RPC_GET_HEIGHT::request req = AUTO_VAL_INIT(req); - cryptonote::COMMAND_RPC_GET_HEIGHT::response res = AUTO_VAL_INIT(res); - - m_daemon_rpc_mutex.lock(); - bool r = net_utils::invoke_http_json("/getheight", req, res, m_http_client, rpc_timeout); - m_daemon_rpc_mutex.unlock(); - CHECK_AND_ASSERT_MES(r, std::string(), "Failed to connect to daemon"); - CHECK_AND_ASSERT_MES(res.status != CORE_RPC_STATUS_BUSY, res.status, "Failed to connect to daemon"); - CHECK_AND_ASSERT_MES(res.status == CORE_RPC_STATUS_OK, res.status, "Failed to get current blockchain height"); - m_height = res.height; - m_height_time = now; - } - height = m_height; - return boost::optional<std::string>(); -} - void NodeRPCProxy::set_height(uint64_t h) { m_height = h; } -boost::optional<std::string> NodeRPCProxy::get_target_height(uint64_t &height) const +boost::optional<std::string> NodeRPCProxy::get_info() const { const time_t now = time(NULL); - if (m_target_height == 0 || now >= m_target_height_time + 30) // re-cache every 30 seconds + if (now >= m_get_info_time + 30) // re-cache every 30 seconds { cryptonote::COMMAND_RPC_GET_INFO::request req_t = AUTO_VAL_INIT(req_t); cryptonote::COMMAND_RPC_GET_INFO::response resp_t = AUTO_VAL_INIT(resp_t); @@ -125,13 +97,41 @@ boost::optional<std::string> NodeRPCProxy::get_target_height(uint64_t &height) c CHECK_AND_ASSERT_MES(r, std::string(), "Failed to connect to daemon"); CHECK_AND_ASSERT_MES(resp_t.status != CORE_RPC_STATUS_BUSY, resp_t.status, "Failed to connect to daemon"); CHECK_AND_ASSERT_MES(resp_t.status == CORE_RPC_STATUS_OK, resp_t.status, "Failed to get target blockchain height"); + m_height = resp_t.height; m_target_height = resp_t.target_height; - m_target_height_time = now; + m_block_size_limit = resp_t.block_size_limit; + m_get_info_time = now; } + return boost::optional<std::string>(); +} + +boost::optional<std::string> NodeRPCProxy::get_height(uint64_t &height) const +{ + auto res = get_info(); + if (res) + return res; + height = m_height; + return boost::optional<std::string>(); +} + +boost::optional<std::string> NodeRPCProxy::get_target_height(uint64_t &height) const +{ + auto res = get_info(); + if (res) + return res; height = m_target_height; return boost::optional<std::string>(); } +boost::optional<std::string> NodeRPCProxy::get_block_size_limit(uint64_t &block_size_limit) const +{ + auto res = get_info(); + if (res) + return res; + block_size_limit = m_block_size_limit; + return boost::optional<std::string>(); +} + boost::optional<std::string> NodeRPCProxy::get_earliest_height(uint8_t version, uint64_t &earliest_height) const { if (m_earliest_height[version] == 0) diff --git a/src/wallet/node_rpc_proxy.h b/src/wallet/node_rpc_proxy.h index 1b183212d..8a65884f7 100644 --- a/src/wallet/node_rpc_proxy.h +++ b/src/wallet/node_rpc_proxy.h @@ -47,22 +47,25 @@ public: boost::optional<std::string> get_height(uint64_t &height) const; void set_height(uint64_t h); boost::optional<std::string> get_target_height(uint64_t &height) const; + boost::optional<std::string> get_block_size_limit(uint64_t &block_size_limit) const; boost::optional<std::string> get_earliest_height(uint8_t version, uint64_t &earliest_height) const; boost::optional<std::string> get_dynamic_per_kb_fee_estimate(uint64_t grace_blocks, uint64_t &fee) const; private: + boost::optional<std::string> get_info() const; + epee::net_utils::http::http_simple_client &m_http_client; boost::mutex &m_daemon_rpc_mutex; mutable uint64_t m_height; - mutable time_t m_height_time; mutable uint64_t m_earliest_height[256]; mutable uint64_t m_dynamic_per_kb_fee_estimate; mutable uint64_t m_dynamic_per_kb_fee_estimate_cached_height; mutable uint64_t m_dynamic_per_kb_fee_estimate_grace_blocks; mutable uint32_t m_rpc_version; mutable uint64_t m_target_height; - mutable time_t m_target_height_time; + mutable uint64_t m_block_size_limit; + mutable time_t m_get_info_time; }; } diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 617fb9a87..66f1fecbc 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -117,8 +117,11 @@ using namespace cryptonote; #define STAGENET_SEGREGATION_FORK_HEIGHT 1000000 #define SEGREGATION_FORK_VICINITY 1500 /* blocks */ +#define FIRST_REFRESH_GRANULARITY 1024 + static const std::string MULTISIG_SIGNATURE_MAGIC = "SigMultisigPkV1"; +std::atomic<unsigned int> tools::wallet2::key_ref::refs(0); namespace { @@ -198,6 +201,7 @@ std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variabl { const bool testnet = command_line::get_arg(vm, opts.testnet); const bool stagenet = command_line::get_arg(vm, opts.stagenet); + const network_type nettype = testnet ? TESTNET : stagenet ? STAGENET : MAINNET; const bool restricted = command_line::get_arg(vm, opts.restricted); auto daemon_address = command_line::get_arg(vm, opts.daemon_address); @@ -226,13 +230,13 @@ std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variabl if (!daemon_port) { - daemon_port = testnet ? config::testnet::RPC_DEFAULT_PORT : stagenet ? config::stagenet::RPC_DEFAULT_PORT : config::RPC_DEFAULT_PORT; + daemon_port = get_config(nettype).RPC_DEFAULT_PORT; } if (daemon_address.empty()) daemon_address = std::string("http://") + daemon_host + ":" + std::to_string(daemon_port); - std::unique_ptr<tools::wallet2> wallet(new tools::wallet2(testnet ? TESTNET : stagenet ? STAGENET : MAINNET, restricted)); + std::unique_ptr<tools::wallet2> wallet(new tools::wallet2(nettype, restricted)); wallet->init(std::move(daemon_address), std::move(login)); boost::filesystem::path ringdb_path = command_line::get_arg(vm, opts.shared_ringdb_dir); wallet->set_ring_database(ringdb_path.string()); @@ -656,6 +660,7 @@ wallet2::wallet2(network_type nettype, bool restricted): m_default_priority(0), m_refresh_type(RefreshOptimizeCoinbase), m_auto_refresh(true), + m_first_refresh_done(false), m_refresh_from_block_height(0), m_explicit_refresh_from_block_height(true), m_confirm_missing_payment_id(true), @@ -671,6 +676,7 @@ wallet2::wallet2(network_type nettype, bool restricted): m_segregate_pre_fork_outputs(true), m_key_reuse_mitigation2(true), m_segregation_height(0), + m_ignore_fractional_outputs(true), m_is_initialized(false), m_restricted(restricted), is_old_file_format(false), @@ -685,7 +691,8 @@ wallet2::wallet2(network_type nettype, bool restricted): m_light_wallet_unlocked_balance(0), m_key_on_device(false), m_ring_history_saved(false), - m_ringdb() + m_ringdb(), + m_last_block_reward(0) { } @@ -769,8 +776,6 @@ bool wallet2::init(std::string daemon_address, boost::optional<epee::net_utils:: m_daemon_address = std::move(daemon_address); m_daemon_login = std::move(daemon_login); // When switching from light wallet to full wallet, we need to reset the height we got from lw node. - if(m_light_wallet) - m_local_bc_height = m_blockchain.size(); return m_http_client.set_server(get_daemon_address(), get_daemon_login(), ssl); } //---------------------------------------------------------------------------------------------------- @@ -946,6 +951,7 @@ void wallet2::expand_subaddresses(const cryptonote::subaddress_index& index) } m_subaddress_labels.resize(index.major + 1, {"Untitled account"}); m_subaddress_labels[index.major].resize(index.minor + 1); + get_account_tags(); } else if (m_subaddress_labels[index.major].size() <= index.minor) { @@ -1035,6 +1041,33 @@ void wallet2::check_acc_out_precomp(const tx_out &o, const crypto::key_derivatio tx_scan_info.error = false; } //---------------------------------------------------------------------------------------------------- +void wallet2::check_acc_out_precomp(const tx_out &o, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, size_t i, const is_out_data *is_out_data, tx_scan_info_t &tx_scan_info) const +{ + if (!is_out_data || i >= is_out_data->received.size()) + return check_acc_out_precomp(o, derivation, additional_derivations, i, tx_scan_info); + + tx_scan_info.received = is_out_data->received[i]; + if(tx_scan_info.received) + { + tx_scan_info.money_transfered = o.amount; // may be 0 for ringct outputs + } + else + { + tx_scan_info.money_transfered = 0; + } + tx_scan_info.error = false; +} +//---------------------------------------------------------------------------------------------------- +void wallet2::check_acc_out_precomp_once(const tx_out &o, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, size_t i, const is_out_data *is_out_data, tx_scan_info_t &tx_scan_info, bool &already_seen) const +{ + tx_scan_info.received = boost::none; + if (already_seen) + return; + check_acc_out_precomp(o, derivation, additional_derivations, i, is_out_data, tx_scan_info); + if (tx_scan_info.received) + already_seen = true; +} +//---------------------------------------------------------------------------------------------------- static uint64_t decodeRct(const rct::rctSig & rv, const crypto::key_derivation &derivation, unsigned int i, rct::key & mask, hw::device &hwdev) { crypto::secret_key scalar1; @@ -1088,16 +1121,48 @@ void wallet2::scan_output(const cryptonote::transaction &tx, const crypto::publi ++num_vouts_received; } //---------------------------------------------------------------------------------------------------- -void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector<uint64_t> &o_indices, uint64_t height, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen) +void wallet2::cache_tx_data(const cryptonote::transaction& tx, const crypto::hash &txid, tx_cache_data &tx_cache_data) const { - //ensure device is let in NONE mode in any case - hw::device &hwdev = m_account.get_device(); - - boost::unique_lock<hw::device> hwdev_lock (hwdev); - hw::reset_mode rst(hwdev); - hwdev_lock.unlock(); + const cryptonote::account_keys& keys = m_account.get_keys(); + + if(!parse_tx_extra(tx.extra, tx_cache_data.tx_extra_fields)) + { + // Extra may only be partially parsed, it's OK if tx_extra_fields contains public key + LOG_PRINT_L0("Transaction extra has unsupported format: " << txid); + tx_cache_data.tx_extra_fields.clear(); + return; + } + + // Don't try to extract tx public key if tx has no ouputs + const bool is_miner = tx.vin.size() == 1 && tx.vin[0].type() == typeid(cryptonote::txin_gen); + if (!is_miner || m_refresh_type != RefreshType::RefreshNoCoinbase) + { + const size_t rec_size = is_miner && m_refresh_type == RefreshType::RefreshOptimizeCoinbase ? 1 : tx.vout.size(); + if (!tx.vout.empty()) + { + // if tx.vout is not empty, we loop through all tx pubkeys + const std::vector<boost::optional<cryptonote::subaddress_receive_info>> rec(rec_size, boost::none); + + tx_extra_pub_key pub_key_field; + size_t pk_index = 0; + while (find_tx_extra_field_by_type(tx_cache_data.tx_extra_fields, pub_key_field, pk_index++)) + tx_cache_data.primary.push_back({pub_key_field.pub_key, {}, rec}); - // In this function, tx (probably) only contains the base information + // additional tx pubkeys and derivations for multi-destination transfers involving one or more subaddresses + tx_extra_additional_pub_keys additional_tx_pub_keys; + std::vector<crypto::key_derivation> additional_derivations; + if (find_tx_extra_field_by_type(tx_cache_data.tx_extra_fields, additional_tx_pub_keys)) + { + for (size_t i = 0; i < additional_tx_pub_keys.data.size(); ++i) + tx_cache_data.additional.push_back({additional_tx_pub_keys.data[i], {}, {}}); + } + } + } +} +//---------------------------------------------------------------------------------------------------- +void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector<uint64_t> &o_indices, uint64_t height, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen, const tx_cache_data &tx_cache_data) +{ + // In this function, tx (probably) only contains the base information // (that is, the prunable stuff may or may not be included) if (!miner_tx && !pool) process_unconfirmed(txid, tx, height); @@ -1105,16 +1170,21 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote std::unordered_map<cryptonote::subaddress_index, uint64_t> tx_money_got_in_outs; // per receiving subaddress index crypto::public_key tx_pub_key = null_pkey; - std::vector<tx_extra_field> tx_extra_fields; - if(!parse_tx_extra(tx.extra, tx_extra_fields)) + std::vector<tx_extra_field> local_tx_extra_fields; + if (tx_cache_data.tx_extra_fields.empty()) { - // Extra may only be partially parsed, it's OK if tx_extra_fields contains public key - LOG_PRINT_L0("Transaction extra has unsupported format: " << txid); + if(!parse_tx_extra(tx.extra, local_tx_extra_fields)) + { + // Extra may only be partially parsed, it's OK if tx_extra_fields contains public key + LOG_PRINT_L0("Transaction extra has unsupported format: " << txid); + } } + const std::vector<tx_extra_field> &tx_extra_fields = tx_cache_data.tx_extra_fields.empty() ? local_tx_extra_fields : tx_cache_data.tx_extra_fields; // Don't try to extract tx public key if tx has no ouputs size_t pk_index = 0; std::vector<tx_scan_info_t> tx_scan_info(tx.vout.size()); + std::deque<bool> output_found(tx.vout.size(), false); while (!tx.vout.empty()) { // if tx.vout is not empty, we loop through all tx pubkeys @@ -1129,6 +1199,11 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote m_callback->on_skip_transaction(height, txid, tx); break; } + if (!tx_cache_data.primary.empty()) + { + THROW_WALLET_EXCEPTION_IF(tx_cache_data.primary.size() < pk_index || pub_key_field.pub_key != tx_cache_data.primary[pk_index - 1].pkey, + error::wallet_internal_error, "tx_cache_data is out of sync"); + } int num_vouts_received = 0; tx_pub_key = pub_key_field.pub_key; @@ -1137,28 +1212,55 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote const cryptonote::account_keys& keys = m_account.get_keys(); crypto::key_derivation derivation; - hwdev_lock.lock(); - hwdev.set_mode(hw::device::TRANSACTION_PARSE); - if (!hwdev.generate_key_derivation(tx_pub_key, keys.m_view_secret_key, derivation)) + std::vector<crypto::key_derivation> additional_derivations; + tx_extra_additional_pub_keys additional_tx_pub_keys; + const wallet2::is_out_data *is_out_data_ptr = NULL; + if (tx_cache_data.primary.empty()) { - MWARNING("Failed to generate key derivation from tx pubkey, skipping"); - static_assert(sizeof(derivation) == sizeof(rct::key), "Mismatched sizes of key_derivation and rct::key"); - memcpy(&derivation, rct::identity().bytes, sizeof(derivation)); - } + hw::device &hwdev = m_account.get_device(); + boost::unique_lock<hw::device> hwdev_lock (hwdev); + hw::reset_mode rst(hwdev); - // additional tx pubkeys and derivations for multi-destination transfers involving one or more subaddresses - std::vector<crypto::public_key> additional_tx_pub_keys = get_additional_tx_pub_keys_from_extra(tx); - std::vector<crypto::key_derivation> additional_derivations; - for (size_t i = 0; i < additional_tx_pub_keys.size(); ++i) + hwdev.set_mode(hw::device::TRANSACTION_PARSE); + if (!hwdev.generate_key_derivation(tx_pub_key, keys.m_view_secret_key, derivation)) + { + MWARNING("Failed to generate key derivation from tx pubkey in " << txid << ", skipping"); + static_assert(sizeof(derivation) == sizeof(rct::key), "Mismatched sizes of key_derivation and rct::key"); + memcpy(&derivation, rct::identity().bytes, sizeof(derivation)); + } + + if (pk_index == 1) + { + // additional tx pubkeys and derivations for multi-destination transfers involving one or more subaddresses + if (find_tx_extra_field_by_type(tx_extra_fields, additional_tx_pub_keys)) + { + for (size_t i = 0; i < additional_tx_pub_keys.data.size(); ++i) + { + additional_derivations.push_back({}); + if (!hwdev.generate_key_derivation(additional_tx_pub_keys.data[i], keys.m_view_secret_key, additional_derivations.back())) + { + MWARNING("Failed to generate key derivation from additional tx pubkey in " << txid << ", skipping"); + memcpy(&additional_derivations.back(), rct::identity().bytes, sizeof(crypto::key_derivation)); + } + } + } + } + } + else { - additional_derivations.push_back({}); - if (!hwdev.generate_key_derivation(additional_tx_pub_keys[i], keys.m_view_secret_key, additional_derivations.back())) + THROW_WALLET_EXCEPTION_IF(pk_index - 1 >= tx_cache_data.primary.size(), + error::wallet_internal_error, "pk_index out of range of tx_cache_data"); + is_out_data_ptr = &tx_cache_data.primary[pk_index - 1]; + derivation = tx_cache_data.primary[pk_index - 1].derivation; + if (pk_index == 1) { - MWARNING("Failed to generate key derivation from tx pubkey, skipping"); - additional_derivations.pop_back(); + for (size_t n = 0; n < tx_cache_data.additional.size(); ++n) + { + additional_tx_pub_keys.data.push_back(tx_cache_data.additional[n].pkey); + additional_derivations.push_back(tx_cache_data.additional[n].derivation); + } } } - hwdev_lock.unlock(); if (miner_tx && m_refresh_type == RefreshNoCoinbase) { @@ -1166,7 +1268,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote } else if (miner_tx && m_refresh_type == RefreshOptimizeCoinbase) { - check_acc_out_precomp(tx.vout[0], derivation, additional_derivations, 0, tx_scan_info[0]); + check_acc_out_precomp_once(tx.vout[0], derivation, additional_derivations, 0, is_out_data_ptr, tx_scan_info[0], output_found[0]); THROW_WALLET_EXCEPTION_IF(tx_scan_info[0].error, error::acc_outs_lookup_error, tx, tx_pub_key, m_account.get_keys()); // this assumes that the miner tx pays a single address @@ -1176,60 +1278,60 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote // the first one was already checked for (size_t i = 1; i < tx.vout.size(); ++i) { - tpool.submit(&waiter, boost::bind(&wallet2::check_acc_out_precomp, this, std::cref(tx.vout[i]), std::cref(derivation), std::cref(additional_derivations), i, - std::ref(tx_scan_info[i]))); + tpool.submit(&waiter, boost::bind(&wallet2::check_acc_out_precomp_once, this, std::cref(tx.vout[i]), std::cref(derivation), std::cref(additional_derivations), i, + std::cref(is_out_data_ptr), std::ref(tx_scan_info[i]), std::ref(output_found[i])), true); } - waiter.wait(); + waiter.wait(&tpool); // then scan all outputs from 0 - hwdev_lock.lock(); + hw::device &hwdev = m_account.get_device(); + boost::unique_lock<hw::device> hwdev_lock (hwdev); hwdev.set_mode(hw::device::NONE); for (size_t i = 0; i < tx.vout.size(); ++i) { THROW_WALLET_EXCEPTION_IF(tx_scan_info[i].error, error::acc_outs_lookup_error, tx, tx_pub_key, m_account.get_keys()); if (tx_scan_info[i].received) { - hwdev.conceal_derivation(tx_scan_info[i].received->derivation, tx_pub_key, additional_tx_pub_keys, derivation, additional_derivations); + hwdev.conceal_derivation(tx_scan_info[i].received->derivation, tx_pub_key, additional_tx_pub_keys.data, derivation, additional_derivations); scan_output(tx, tx_pub_key, i, tx_scan_info[i], num_vouts_received, tx_money_got_in_outs, outs); } } - hwdev_lock.unlock(); } } - else if (tx.vout.size() > 1 && tools::threadpool::getInstance().get_max_concurrency() > 1) + else if (tx.vout.size() > 1 && tools::threadpool::getInstance().get_max_concurrency() > 1 && !is_out_data_ptr) { for (size_t i = 0; i < tx.vout.size(); ++i) { - tpool.submit(&waiter, boost::bind(&wallet2::check_acc_out_precomp, this, std::cref(tx.vout[i]), std::cref(derivation), std::cref(additional_derivations), i, - std::ref(tx_scan_info[i]))); + tpool.submit(&waiter, boost::bind(&wallet2::check_acc_out_precomp_once, this, std::cref(tx.vout[i]), std::cref(derivation), std::cref(additional_derivations), i, + std::cref(is_out_data_ptr), std::ref(tx_scan_info[i]), std::ref(output_found[i])), true); } - waiter.wait(); + waiter.wait(&tpool); - hwdev_lock.lock(); + hw::device &hwdev = m_account.get_device(); + boost::unique_lock<hw::device> hwdev_lock (hwdev); hwdev.set_mode(hw::device::NONE); for (size_t i = 0; i < tx.vout.size(); ++i) { THROW_WALLET_EXCEPTION_IF(tx_scan_info[i].error, error::acc_outs_lookup_error, tx, tx_pub_key, m_account.get_keys()); if (tx_scan_info[i].received) { - hwdev.conceal_derivation(tx_scan_info[i].received->derivation, tx_pub_key, additional_tx_pub_keys, derivation, additional_derivations); + hwdev.conceal_derivation(tx_scan_info[i].received->derivation, tx_pub_key, additional_tx_pub_keys.data, derivation, additional_derivations); scan_output(tx, tx_pub_key, i, tx_scan_info[i], num_vouts_received, tx_money_got_in_outs, outs); } } - hwdev_lock.unlock(); } else { for (size_t i = 0; i < tx.vout.size(); ++i) { - check_acc_out_precomp(tx.vout[i], derivation, additional_derivations, i, tx_scan_info[i]); + check_acc_out_precomp_once(tx.vout[i], derivation, additional_derivations, i, is_out_data_ptr, tx_scan_info[i], output_found[i]); THROW_WALLET_EXCEPTION_IF(tx_scan_info[i].error, error::acc_outs_lookup_error, tx, tx_pub_key, m_account.get_keys()); if (tx_scan_info[i].received) { - hwdev_lock.lock(); + hw::device &hwdev = m_account.get_device(); + boost::unique_lock<hw::device> hwdev_lock (hwdev); hwdev.set_mode(hw::device::NONE); - hwdev.conceal_derivation(tx_scan_info[i].received->derivation, tx_pub_key, additional_tx_pub_keys, derivation, additional_derivations); + hwdev.conceal_derivation(tx_scan_info[i].received->derivation, tx_pub_key, additional_tx_pub_keys.data, derivation, additional_derivations); scan_output(tx, tx_pub_key, i, tx_scan_info[i], num_vouts_received, tx_money_got_in_outs, outs); - hwdev_lock.unlock(); } } } @@ -1305,20 +1407,20 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote m_callback->on_money_received(height, txid, tx, td.m_amount, td.m_subaddr_index); } } - else if (m_transfers[kit->second].m_spent || m_transfers[kit->second].amount() >= tx.vout[o].amount) + else if (m_transfers[kit->second].m_spent || m_transfers[kit->second].amount() >= tx_scan_info[o].amount) { LOG_ERROR("Public key " << epee::string_tools::pod_to_hex(kit->first) - << " from received " << print_money(tx.vout[o].amount) << " output already exists with " + << " from received " << print_money(tx_scan_info[o].amount) << " output already exists with " << (m_transfers[kit->second].m_spent ? "spent" : "unspent") << " " - << print_money(m_transfers[kit->second].amount()) << ", received output ignored"); + << print_money(m_transfers[kit->second].amount()) << " in tx " << m_transfers[kit->second].m_txid << ", received output ignored"); } else { LOG_ERROR("Public key " << epee::string_tools::pod_to_hex(kit->first) - << " from received " << print_money(tx.vout[o].amount) << " output already exists with " + << " from received " << print_money(tx_scan_info[o].amount) << " output already exists with " << print_money(m_transfers[kit->second].amount()) << ", replacing with new output"); // The new larger output replaced a previous smaller one - tx_money_got_in_outs[tx_scan_info[o].received->index] -= tx.vout[o].amount; + tx_money_got_in_outs[tx_scan_info[o].received->index] -= tx_scan_info[o].amount; if (!pool) { @@ -1455,6 +1557,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote { // We got a payment ID to go with this tx LOG_PRINT_L2("Found encrypted payment ID: " << payment_id8); + MINFO("Consider using subaddresses instead of encrypted payment IDs"); if (tx_pub_key != null_pkey) { if (!m_account.get_device().decrypt_payment_id(payment_id8, tx_pub_key, m_account.get_keys().m_view_secret_key)) @@ -1478,12 +1581,9 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote else if (get_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id)) { LOG_PRINT_L2("Found unencrypted payment ID: " << payment_id); + MWARNING("Found unencrypted payment ID: these are bad for privacy, consider using subaddresses instead"); } } - else if (get_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id)) - { - LOG_PRINT_L2("Found unencrypted payment ID: " << payment_id); - } for (const auto& i : tx_money_got_in_outs) { @@ -1569,12 +1669,11 @@ void wallet2::process_outgoing(const crypto::hash &txid, const cryptonote::trans add_rings(tx); } //---------------------------------------------------------------------------------------------------- -void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const crypto::hash& bl_id, uint64_t height, const cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices &o_indices) +void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const parsed_block &parsed_block, const crypto::hash& bl_id, uint64_t height, const std::vector<tx_cache_data> &tx_cache_data, size_t tx_cache_data_offset) { - size_t txidx = 0; - THROW_WALLET_EXCEPTION_IF(bche.txs.size() + 1 != o_indices.indices.size(), error::wallet_internal_error, + THROW_WALLET_EXCEPTION_IF(bche.txs.size() + 1 != parsed_block.o_indices.indices.size(), error::wallet_internal_error, "block transactions=" + std::to_string(bche.txs.size()) + - " not match with daemon response size=" + std::to_string(o_indices.indices.size())); + " not match with daemon response size=" + std::to_string(parsed_block.o_indices.indices.size())); //handle transactions from new block @@ -1582,39 +1681,38 @@ void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cry if(b.timestamp + 60*60*24 > m_account.get_createtime() && height >= m_refresh_from_block_height) { TIME_MEASURE_START(miner_tx_handle_time); - process_new_transaction(get_transaction_hash(b.miner_tx), b.miner_tx, o_indices.indices[txidx++].indices, height, b.timestamp, true, false, false); + if (m_refresh_type != RefreshNoCoinbase) + process_new_transaction(get_transaction_hash(b.miner_tx), b.miner_tx, parsed_block.o_indices.indices[0].indices, height, b.timestamp, true, false, false, tx_cache_data[tx_cache_data_offset]); + ++tx_cache_data_offset; TIME_MEASURE_FINISH(miner_tx_handle_time); TIME_MEASURE_START(txs_handle_time); THROW_WALLET_EXCEPTION_IF(bche.txs.size() != b.tx_hashes.size(), error::wallet_internal_error, "Wrong amount of transactions for block"); - size_t idx = 0; - for (const auto& txblob: bche.txs) + THROW_WALLET_EXCEPTION_IF(bche.txs.size() != parsed_block.txes.size(), error::wallet_internal_error, "Wrong amount of transactions for block"); + for (size_t idx = 0; idx < b.tx_hashes.size(); ++idx) { - cryptonote::transaction tx; - bool r = parse_and_validate_tx_base_from_blob(txblob, tx); - THROW_WALLET_EXCEPTION_IF(!r, error::tx_parse_error, txblob); - process_new_transaction(b.tx_hashes[idx], tx, o_indices.indices[txidx++].indices, height, b.timestamp, false, false, false); - ++idx; + process_new_transaction(b.tx_hashes[idx], parsed_block.txes[idx], parsed_block.o_indices.indices[idx+1].indices, height, b.timestamp, false, false, false, tx_cache_data[tx_cache_data_offset++]); } TIME_MEASURE_FINISH(txs_handle_time); + m_last_block_reward = cryptonote::get_outs_money_amount(b.miner_tx); LOG_PRINT_L2("Processed block: " << bl_id << ", height " << height << ", " << miner_tx_handle_time + txs_handle_time << "(" << miner_tx_handle_time << "/" << txs_handle_time <<")ms"); }else { - if (!(height % 100)) + if (!(height % 128)) LOG_PRINT_L2( "Skipped block by timestamp, height: " << height << ", block time " << b.timestamp << ", account time " << m_account.get_createtime()); } m_blockchain.push_back(bl_id); - ++m_local_bc_height; if (0 != m_callback) m_callback->on_new_block(height, b); } //---------------------------------------------------------------------------------------------------- -void wallet2::get_short_chain_history(std::list<crypto::hash>& ids) const +void wallet2::get_short_chain_history(std::list<crypto::hash>& ids, uint64_t granularity) const { size_t i = 0; size_t current_multiplier = 1; - size_t sz = m_blockchain.size() - m_blockchain.offset(); + size_t blockchain_size = std::max((size_t)(m_blockchain.size() / granularity * granularity), m_blockchain.offset()); + size_t sz = blockchain_size - m_blockchain.offset(); if(!sz) { ids.push_back(m_blockchain.genesis()); @@ -1649,7 +1747,7 @@ void wallet2::parse_block_round(const cryptonote::blobdata &blob, cryptonote::bl bl_id = get_block_hash(bl); } //---------------------------------------------------------------------------------------------------- -void wallet2::pull_blocks(uint64_t start_height, uint64_t &blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::list<cryptonote::block_complete_entry> &blocks, std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices) +void wallet2::pull_blocks(uint64_t start_height, uint64_t &blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices) { cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::request req = AUTO_VAL_INIT(req); cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::response res = AUTO_VAL_INIT(res); @@ -1684,6 +1782,7 @@ void wallet2::pull_blocks(uint64_t start_height, uint64_t &blocks_start_height, } req.start_height = start_height; + req.no_miner_tx = m_refresh_type == RefreshNoCoinbase; m_daemon_rpc_mutex.lock(); bool r = net_utils::invoke_http_bin("/getblocks.bin", req, res, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); @@ -1695,11 +1794,11 @@ void wallet2::pull_blocks(uint64_t start_height, uint64_t &blocks_start_height, boost::lexical_cast<std::string>(res.output_indices.size()) + ") sizes from daemon"); blocks_start_height = res.start_height; - blocks = res.blocks; - o_indices = res.output_indices; + blocks = std::move(res.blocks); + o_indices = std::move(res.output_indices); } //---------------------------------------------------------------------------------------------------- -void wallet2::pull_hashes(uint64_t start_height, uint64_t &blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::list<crypto::hash> &hashes) +void wallet2::pull_hashes(uint64_t start_height, uint64_t &blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<crypto::hash> &hashes) { cryptonote::COMMAND_RPC_GET_HASHES_FAST::request req = AUTO_VAL_INIT(req); cryptonote::COMMAND_RPC_GET_HASHES_FAST::response res = AUTO_VAL_INIT(res); @@ -1714,88 +1813,117 @@ void wallet2::pull_hashes(uint64_t start_height, uint64_t &blocks_start_height, THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::get_hashes_error, res.status); blocks_start_height = res.start_height; - hashes = res.m_block_ids; + hashes = std::move(res.m_block_ids); } //---------------------------------------------------------------------------------------------------- -void wallet2::process_blocks(uint64_t start_height, const std::list<cryptonote::block_complete_entry> &blocks, const std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices, uint64_t& blocks_added) +void wallet2::process_parsed_blocks(uint64_t start_height, const std::vector<cryptonote::block_complete_entry> &blocks, const std::vector<parsed_block> &parsed_blocks, uint64_t& blocks_added) { size_t current_index = start_height; blocks_added = 0; - size_t tx_o_indices_idx = 0; - THROW_WALLET_EXCEPTION_IF(blocks.size() != o_indices.size(), error::wallet_internal_error, "size mismatch"); - THROW_WALLET_EXCEPTION_IF(!m_blockchain.is_in_bounds(current_index), error::wallet_internal_error, "Index out of bounds of hashchain"); + THROW_WALLET_EXCEPTION_IF(blocks.size() != parsed_blocks.size(), error::wallet_internal_error, "size mismatch"); + THROW_WALLET_EXCEPTION_IF(!m_blockchain.is_in_bounds(current_index), error::out_of_hashchain_bounds_error); tools::threadpool& tpool = tools::threadpool::getInstance(); - int threads = tpool.get_max_concurrency(); - if (threads > 1) + tools::threadpool::waiter waiter; + + size_t num_txes = 0; + std::vector<tx_cache_data> tx_cache_data; + for (size_t i = 0; i < blocks.size(); ++i) + num_txes += 1 + parsed_blocks[i].txes.size(); + tx_cache_data.resize(num_txes); + size_t txidx = 0; + for (size_t i = 0; i < blocks.size(); ++i) { - std::vector<crypto::hash> round_block_hashes(threads); - std::vector<cryptonote::block> round_blocks(threads); - std::deque<bool> error(threads); - size_t blocks_size = blocks.size(); - std::list<block_complete_entry>::const_iterator blocki = blocks.begin(); - for (size_t b = 0; b < blocks_size; b += threads) + THROW_WALLET_EXCEPTION_IF(parsed_blocks[i].txes.size() != parsed_blocks[i].block.tx_hashes.size(), + error::wallet_internal_error, "Mismatched parsed_blocks[i].txes.size() and parsed_blocks[i].block.tx_hashes.size()"); + if (m_refresh_type != RefreshNoCoinbase) + tpool.submit(&waiter, [&, i, txidx](){ cache_tx_data(parsed_blocks[i].block.miner_tx, get_transaction_hash(parsed_blocks[i].block.miner_tx), tx_cache_data[txidx]); }); + ++txidx; + for (size_t idx = 0; idx < parsed_blocks[i].txes.size(); ++idx) { - size_t round_size = std::min((size_t)threads, blocks_size - b); - tools::threadpool::waiter waiter; + tpool.submit(&waiter, [&, i, idx, txidx](){ cache_tx_data(parsed_blocks[i].txes[idx], parsed_blocks[i].block.tx_hashes[idx], tx_cache_data[txidx]); }); + ++txidx; + } + } + THROW_WALLET_EXCEPTION_IF(txidx != num_txes, error::wallet_internal_error, "txidx does not match tx_cache_data size"); + waiter.wait(&tpool); - std::list<block_complete_entry>::const_iterator tmpblocki = blocki; - for (size_t i = 0; i < round_size; ++i) - { - tpool.submit(&waiter, boost::bind(&wallet2::parse_block_round, this, std::cref(tmpblocki->block), - std::ref(round_blocks[i]), std::ref(round_block_hashes[i]), std::ref(error[i]))); - ++tmpblocki; - } - waiter.wait(); - tmpblocki = blocki; - for (size_t i = 0; i < round_size; ++i) - { - THROW_WALLET_EXCEPTION_IF(error[i], error::block_parse_error, tmpblocki->block); - ++tmpblocki; - } - for (size_t i = 0; i < round_size; ++i) - { - const crypto::hash &bl_id = round_block_hashes[i]; - cryptonote::block &bl = round_blocks[i]; + hw::device &hwdev = m_account.get_device(); + hw::reset_mode rst(hwdev); + hwdev.set_mode(hw::device::TRANSACTION_PARSE); + const cryptonote::account_keys &keys = m_account.get_keys(); - if(current_index >= m_blockchain.size()) - { - process_new_blockchain_entry(bl, *blocki, bl_id, current_index, o_indices[b+i]); - ++blocks_added; - } - else if(bl_id != m_blockchain[current_index]) - { - //split detected here !!! - THROW_WALLET_EXCEPTION_IF(current_index == start_height, error::wallet_internal_error, - "wrong daemon response: split starts from the first block in response " + string_tools::pod_to_hex(bl_id) + - " (height " + std::to_string(start_height) + "), local block id at this height: " + - string_tools::pod_to_hex(m_blockchain[current_index])); - - detach_blockchain(current_index); - process_new_blockchain_entry(bl, *blocki, bl_id, current_index, o_indices[b+i]); - } - else + auto gender = [&](wallet2::is_out_data &iod) { + boost::unique_lock<hw::device> hwdev_lock(hwdev); + if (!hwdev.generate_key_derivation(iod.pkey, keys.m_view_secret_key, iod.derivation)) + { + MWARNING("Failed to generate key derivation from tx pubkey, skipping"); + static_assert(sizeof(iod.derivation) == sizeof(rct::key), "Mismatched sizes of key_derivation and rct::key"); + memcpy(&iod.derivation, rct::identity().bytes, sizeof(iod.derivation)); + } + }; + + for (auto &slot: tx_cache_data) + { + for (auto &iod: slot.primary) + tpool.submit(&waiter, [&gender, &iod]() { gender(iod); }, true); + for (auto &iod: slot.additional) + tpool.submit(&waiter, [&gender, &iod]() { gender(iod); }, true); + } + waiter.wait(&tpool); + + auto geniod = [&](const cryptonote::transaction &tx, size_t n_vouts, size_t txidx) { + for (size_t k = 0; k < n_vouts; ++k) + { + const auto &o = tx.vout[k]; + if (o.target.type() == typeid(cryptonote::txout_to_key)) + { + std::vector<crypto::key_derivation> additional_derivations; + for (const auto &iod: tx_cache_data[txidx].additional) + additional_derivations.push_back(iod.derivation); + const auto &key = boost::get<txout_to_key>(o.target).key; + for (size_t l = 0; l < tx_cache_data[txidx].primary.size(); ++l) { - LOG_PRINT_L2("Block is already in blockchain: " << string_tools::pod_to_hex(bl_id)); + THROW_WALLET_EXCEPTION_IF(tx_cache_data[txidx].primary[l].received.size() != n_vouts, + error::wallet_internal_error, "Unexpected received array size"); + tx_cache_data[txidx].primary[l].received[k] = is_out_to_acc_precomp(m_subaddresses, key, tx_cache_data[txidx].primary[l].derivation, additional_derivations, k, hwdev); + additional_derivations.clear(); } - ++current_index; - ++blocki; } } - } - else + }; + + txidx = 0; + for (size_t i = 0; i < blocks.size(); ++i) { - for(auto& bl_entry: blocks) + if (m_refresh_type != RefreshType::RefreshNoCoinbase) + { + THROW_WALLET_EXCEPTION_IF(txidx >= tx_cache_data.size(), error::wallet_internal_error, "txidx out of range"); + const size_t n_vouts = m_refresh_type == RefreshType::RefreshOptimizeCoinbase ? 1 : parsed_blocks[i].block.miner_tx.vout.size(); + tpool.submit(&waiter, [&, i, txidx](){ geniod(parsed_blocks[i].block.miner_tx, n_vouts, txidx); }, true); + } + ++txidx; + for (size_t j = 0; j < parsed_blocks[i].txes.size(); ++j) + { + THROW_WALLET_EXCEPTION_IF(txidx >= tx_cache_data.size(), error::wallet_internal_error, "txidx out of range"); + tpool.submit(&waiter, [&, i, j, txidx](){ geniod(parsed_blocks[i].txes[j], parsed_blocks[i].txes[j].vout.size(), txidx); }, true); + ++txidx; + } + } + THROW_WALLET_EXCEPTION_IF(txidx != tx_cache_data.size(), error::wallet_internal_error, "txidx did not reach expected value"); + waiter.wait(&tpool); + hwdev.set_mode(hw::device::NONE); + + size_t tx_cache_data_offset = 0; + for (size_t i = 0; i < blocks.size(); ++i) { - cryptonote::block bl; - bool r = cryptonote::parse_and_validate_block_from_blob(bl_entry.block, bl); - THROW_WALLET_EXCEPTION_IF(!r, error::block_parse_error, bl_entry.block); + const crypto::hash &bl_id = parsed_blocks[i].hash; + const cryptonote::block &bl = parsed_blocks[i].block; - crypto::hash bl_id = get_block_hash(bl); if(current_index >= m_blockchain.size()) { - process_new_blockchain_entry(bl, bl_entry, bl_id, current_index, o_indices[tx_o_indices_idx]); + process_new_blockchain_entry(bl, blocks[i], parsed_blocks[i], bl_id, current_index, tx_cache_data, tx_cache_data_offset); ++blocks_added; } else if(bl_id != m_blockchain[current_index]) @@ -1807,32 +1935,30 @@ void wallet2::process_blocks(uint64_t start_height, const std::list<cryptonote:: string_tools::pod_to_hex(m_blockchain[current_index])); detach_blockchain(current_index); - process_new_blockchain_entry(bl, bl_entry, bl_id, current_index, o_indices[tx_o_indices_idx]); + process_new_blockchain_entry(bl, blocks[i], parsed_blocks[i], bl_id, current_index, tx_cache_data, tx_cache_data_offset); } else { LOG_PRINT_L2("Block is already in blockchain: " << string_tools::pod_to_hex(bl_id)); } - ++current_index; - ++tx_o_indices_idx; - } + tx_cache_data_offset += 1 + parsed_blocks[i].txes.size(); } } //---------------------------------------------------------------------------------------------------- -void wallet2::refresh() +void wallet2::refresh(bool trusted_daemon) { uint64_t blocks_fetched = 0; - refresh(0, blocks_fetched); + refresh(trusted_daemon, 0, blocks_fetched); } //---------------------------------------------------------------------------------------------------- -void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched) +void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched) { bool received_money = false; - refresh(start_height, blocks_fetched, received_money); + refresh(trusted_daemon, start_height, blocks_fetched, received_money); } //---------------------------------------------------------------------------------------------------- -void wallet2::pull_next_blocks(uint64_t start_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, const std::list<cryptonote::block_complete_entry> &prev_blocks, std::list<cryptonote::block_complete_entry> &blocks, std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices, bool &error) +void wallet2::pull_and_parse_next_blocks(uint64_t start_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, const std::vector<cryptonote::block_complete_entry> &prev_blocks, const std::vector<parsed_block> &prev_parsed_blocks, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<parsed_block> &parsed_blocks, bool &error) { error = false; @@ -1841,18 +1967,53 @@ void wallet2::pull_next_blocks(uint64_t start_height, uint64_t &blocks_start_hei drop_from_short_history(short_chain_history, 3); // prepend the last 3 blocks, should be enough to guard against a block or two's reorg - cryptonote::block bl; - std::list<cryptonote::block_complete_entry>::const_reverse_iterator i = prev_blocks.rbegin(); + std::vector<parsed_block>::const_reverse_iterator i = prev_parsed_blocks.rbegin(); for (size_t n = 0; n < std::min((size_t)3, prev_blocks.size()); ++n) { - bool ok = cryptonote::parse_and_validate_block_from_blob(i->block, bl); - THROW_WALLET_EXCEPTION_IF(!ok, error::block_parse_error, i->block); - short_chain_history.push_front(cryptonote::get_block_hash(bl)); + short_chain_history.push_front(i->hash); ++i; } // pull the new blocks + std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> o_indices; pull_blocks(start_height, blocks_start_height, short_chain_history, blocks, o_indices); + THROW_WALLET_EXCEPTION_IF(blocks.size() != o_indices.size(), error::wallet_internal_error, "Mismatched sizes of blocks and o_indices"); + + tools::threadpool& tpool = tools::threadpool::getInstance(); + tools::threadpool::waiter waiter; + parsed_blocks.resize(blocks.size()); + for (size_t i = 0; i < blocks.size(); ++i) + { + tpool.submit(&waiter, boost::bind(&wallet2::parse_block_round, this, std::cref(blocks[i].block), + std::ref(parsed_blocks[i].block), std::ref(parsed_blocks[i].hash), std::ref(parsed_blocks[i].error)), true); + } + waiter.wait(&tpool); + for (size_t i = 0; i < blocks.size(); ++i) + { + if (parsed_blocks[i].error) + { + error = true; + break; + } + parsed_blocks[i].o_indices = std::move(o_indices[i]); + } + + boost::mutex error_lock; + for (size_t i = 0; i < blocks.size(); ++i) + { + parsed_blocks[i].txes.resize(blocks[i].txs.size()); + for (size_t j = 0; j < blocks[i].txs.size(); ++j) + { + tpool.submit(&waiter, [&, i, j](){ + if (!parse_and_validate_tx_base_from_blob(blocks[i].txs[j], parsed_blocks[i].txes[j])) + { + boost::unique_lock<boost::mutex> lock(error_lock); + error = true; + } + }, true); + } + } + waiter.wait(&tpool); } catch(...) { @@ -1893,8 +2054,8 @@ void wallet2::update_pool_state(bool refreshed) MDEBUG("update_pool_state start"); // get the pool state - cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::request req; - cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL_HASHES::response res; + cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL_HASHES_BIN::request req; + cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL_HASHES_BIN::response res; m_daemon_rpc_mutex.lock(); bool r = epee::net_utils::invoke_http_json("/get_transaction_pool_hashes.bin", req, res, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); @@ -2065,7 +2226,7 @@ void wallet2::update_pool_state(bool refreshed) [tx_hash](const std::pair<crypto::hash, bool> &e) { return e.first == tx_hash; }); if (i != txids.end()) { - process_new_transaction(tx_hash, tx, std::vector<uint64_t>(), 0, time(NULL), false, true, tx_entry.double_spend_seen); + process_new_transaction(tx_hash, tx, std::vector<uint64_t>(), 0, time(NULL), false, true, tx_entry.double_spend_seen, {}); m_scanned_pool_txs[0].insert(tx_hash); if (m_scanned_pool_txs[0].size() > 5000) { @@ -2107,19 +2268,18 @@ void wallet2::update_pool_state(bool refreshed) MDEBUG("update_pool_state end"); } //---------------------------------------------------------------------------------------------------- -void wallet2::fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history) +void wallet2::fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, bool force) { - std::list<crypto::hash> hashes; + std::vector<crypto::hash> hashes; const uint64_t checkpoint_height = m_checkpoints.get_max_height(); - if (stop_height > checkpoint_height && m_blockchain.size()-1 < checkpoint_height) + if ((stop_height > checkpoint_height && m_blockchain.size()-1 < checkpoint_height) && !force) { // we will drop all these, so don't bother getting them uint64_t missing_blocks = m_checkpoints.get_max_height() - m_blockchain.size(); while (missing_blocks-- > 0) m_blockchain.push_back(crypto::null_hash); // maybe a bit suboptimal, but deque won't do huge reallocs like vector m_blockchain.push_back(m_checkpoints.get_points().at(checkpoint_height)); - m_local_bc_height = m_blockchain.size(); short_chain_history.clear(); get_short_chain_history(short_chain_history); } @@ -2137,7 +2297,7 @@ void wallet2::fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, } if (hashes.size() + current_index < stop_height) { drop_from_short_history(short_chain_history, 3); - std::list<crypto::hash>::iterator right = hashes.end(); + std::vector<crypto::hash>::iterator right = hashes.end(); // prepend 3 more for (int i = 0; i<3; i++) { right--; @@ -2149,10 +2309,9 @@ void wallet2::fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, { if(current_index >= m_blockchain.size()) { - if (!(current_index % 1000)) + if (!(current_index % 1024)) LOG_PRINT_L2( "Skipped block by height: " << current_index); m_blockchain.push_back(bl_id); - ++m_local_bc_height; if (0 != m_callback) { // FIXME: this isn't right, but simplewallet just logs that we got a block. @@ -2198,8 +2357,10 @@ bool wallet2::delete_address_book_row(std::size_t row_id) { } //---------------------------------------------------------------------------------------------------- -void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& received_money) +void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched, bool& received_money) { + key_ref kref(*this); + if(m_light_wallet) { // MyMonero get_address_info needs to be called occasionally to trigger wallet sync. @@ -2213,7 +2374,6 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& re // Update lw heights m_light_wallet_scanned_block_height = res.scanned_block_height; m_light_wallet_blockchain_height = res.blockchain_height; - m_local_bc_height = res.blockchain_height; // If new height - call new_block callback if(m_light_wallet_blockchain_height != prev_height) { @@ -2242,12 +2402,12 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& re tools::threadpool& tpool = tools::threadpool::getInstance(); tools::threadpool::waiter waiter; uint64_t blocks_start_height; - std::list<cryptonote::block_complete_entry> blocks; - std::vector<COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> o_indices; + std::vector<cryptonote::block_complete_entry> blocks; + std::vector<parsed_block> parsed_blocks; bool refreshed = false; // pull the first set of blocks - get_short_chain_history(short_chain_history); + get_short_chain_history(short_chain_history, (m_first_refresh_done || trusted_daemon) ? 1 : FIRST_REFRESH_GRANULARITY); m_run.store(true, std::memory_order_relaxed); if (start_height > m_blockchain.size() || m_refresh_from_block_height > m_blockchain.size()) { if (!start_height) @@ -2256,7 +2416,7 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& re fast_refresh(start_height, blocks_start_height, short_chain_history); // regenerate the history now that we've got a full set of hashes short_chain_history.clear(); - get_short_chain_history(short_chain_history); + get_short_chain_history(short_chain_history, (m_first_refresh_done || trusted_daemon) ? 1 : FIRST_REFRESH_GRANULARITY); start_height = 0; // and then fall through to regular refresh processing } @@ -2264,31 +2424,61 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& re // If stop() is called during fast refresh we don't need to continue if(!m_run.load(std::memory_order_relaxed)) return; - pull_blocks(start_height, blocks_start_height, short_chain_history, blocks, o_indices); // always reset start_height to 0 to force short_chain_ history to be used on // subsequent pulls in this refresh. start_height = 0; + bool first = true; while(m_run.load(std::memory_order_relaxed)) { try { // pull the next set of blocks while we're processing the current one uint64_t next_blocks_start_height; - std::list<cryptonote::block_complete_entry> next_blocks; - std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> next_o_indices; + std::vector<cryptonote::block_complete_entry> next_blocks; + std::vector<parsed_block> next_parsed_blocks; bool error = false; - if (blocks.empty()) + added_blocks = 0; + if (!first && blocks.empty()) { refreshed = false; break; } - tpool.submit(&waiter, [&]{pull_next_blocks(start_height, next_blocks_start_height, short_chain_history, blocks, next_blocks, next_o_indices, error);}); + tpool.submit(&waiter, [&]{pull_and_parse_next_blocks(start_height, next_blocks_start_height, short_chain_history, blocks, parsed_blocks, next_blocks, next_parsed_blocks, error);}); - process_blocks(blocks_start_height, blocks, o_indices, added_blocks); - blocks_fetched += added_blocks; - waiter.wait(); - if(blocks_start_height == next_blocks_start_height) + if (!first) + { + try + { + process_parsed_blocks(blocks_start_height, blocks, parsed_blocks, added_blocks); + } + catch (const tools::error::out_of_hashchain_bounds_error&) + { + MINFO("Daemon claims next refresh block is out of hash chain bounds, resetting hash chain"); + uint64_t stop_height = m_blockchain.offset(); + std::vector<crypto::hash> tip(m_blockchain.size() - m_blockchain.offset()); + for (size_t i = m_blockchain.offset(); i < m_blockchain.size(); ++i) + tip[i - m_blockchain.offset()] = m_blockchain[i]; + cryptonote::block b; + generate_genesis(b); + m_blockchain.clear(); + m_blockchain.push_back(get_block_hash(b)); + short_chain_history.clear(); + get_short_chain_history(short_chain_history); + fast_refresh(stop_height, blocks_start_height, short_chain_history, true); + THROW_WALLET_EXCEPTION_IF(m_blockchain.size() != stop_height, error::wallet_internal_error, "Unexpected hashchain size"); + THROW_WALLET_EXCEPTION_IF(m_blockchain.offset() != 0, error::wallet_internal_error, "Unexpected hashchain offset"); + for (const auto &h: tip) + m_blockchain.push_back(h); + short_chain_history.clear(); + get_short_chain_history(short_chain_history); + start_height = stop_height; + throw std::runtime_error(""); // loop again + } + blocks_fetched += added_blocks; + } + waiter.wait(&tpool); + if(!first && blocks_start_height == next_blocks_start_height) { m_node_rpc_proxy.set_height(m_blockchain.size()); refreshed = true; @@ -2297,8 +2487,9 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& re // switch to the new blocks from the daemon blocks_start_height = next_blocks_start_height; - blocks = next_blocks; - o_indices = next_o_indices; + blocks = std::move(next_blocks); + parsed_blocks = std::move(next_parsed_blocks); + first = false; // handle error from async fetching thread if (error) @@ -2309,10 +2500,11 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& re catch (const std::exception&) { blocks_fetched += added_blocks; - waiter.wait(); + waiter.wait(&tpool); if(try_count < 3) { LOG_PRINT_L1("Another try pull_blocks (try_count=" << try_count << ")..."); + first = true; ++try_count; } else @@ -2336,14 +2528,16 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& re LOG_PRINT_L1("Failed to check pending transactions"); } + m_first_refresh_done = true; + LOG_PRINT_L1("Refresh done, blocks received: " << blocks_fetched << ", balance (all accounts): " << print_money(balance_all()) << ", unlocked: " << print_money(unlocked_balance_all())); } //---------------------------------------------------------------------------------------------------- -bool wallet2::refresh(uint64_t & blocks_fetched, bool& received_money, bool& ok) +bool wallet2::refresh(bool trusted_daemon, uint64_t & blocks_fetched, bool& received_money, bool& ok) { try { - refresh(0, blocks_fetched, received_money); + refresh(trusted_daemon, 0, blocks_fetched, received_money); ok = true; } catch (...) @@ -2353,7 +2547,7 @@ bool wallet2::refresh(uint64_t & blocks_fetched, bool& received_money, bool& ok) return ok; } //---------------------------------------------------------------------------------------------------- -bool wallet2::get_output_distribution(uint64_t &start_height, std::vector<uint64_t> &distribution) +bool wallet2::get_rct_distribution(uint64_t &start_height, std::vector<uint64_t> &distribution) { uint32_t rpc_version; boost::optional<std::string> result = m_node_rpc_proxy.get_rpc_version(rpc_version); @@ -2464,7 +2658,6 @@ void wallet2::detach_blockchain(uint64_t height) size_t blocks_detached = m_blockchain.size() - height; m_blockchain.crop(height); - m_local_bc_height -= blocks_detached; for (auto it = m_payments.begin(); it != m_payments.end(); ) { @@ -2488,6 +2681,7 @@ void wallet2::detach_blockchain(uint64_t height) bool wallet2::deinit() { m_is_initialized=false; + unlock_keys_file(); return true; } //---------------------------------------------------------------------------------------------------- @@ -2506,7 +2700,6 @@ bool wallet2::clear() m_scanned_pool_txs[0].clear(); m_scanned_pool_txs[1].clear(); m_address_book.clear(); - m_local_bc_height = 1; m_subaddresses.clear(); m_subaddress_labels.clear(); return true; @@ -2634,6 +2827,9 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable value2.SetUint(m_segregation_height); json.AddMember("segregation_height", value2, json.GetAllocator()); + value2.SetInt(m_ignore_fractional_outputs ? 1 : 0); + json.AddMember("ignore_fractional_outputs", value2, json.GetAllocator()); + value2.SetUint(m_subaddress_lookahead_major); json.AddMember("subaddress_lookahead_major", value2, json.GetAllocator()); @@ -2655,10 +2851,12 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable crypto::chacha20(account_data.data(), account_data.size(), key, keys_file_data.iv, &cipher[0]); keys_file_data.account_data = cipher; + unlock_keys_file(); std::string buf; r = ::serialization::dump_binary(keys_file_data, buf); r = r && epee::file_io_utils::save_string_to_file(keys_file_name, buf); //and never touch wallet_keys_file again, only read CHECK_AND_ASSERT_MES(r, false, "failed to generate wallet keys file " << keys_file_name); + lock_keys_file(); return true; } @@ -2714,6 +2912,7 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ m_segregate_pre_fork_outputs = true; m_key_reuse_mitigation2 = true; m_segregation_height = 0; + m_ignore_fractional_outputs = true; m_subaddress_lookahead_major = SUBADDRESS_LOOKAHEAD_MAJOR; m_subaddress_lookahead_minor = SUBADDRESS_LOOKAHEAD_MINOR; m_key_on_device = false; @@ -2840,6 +3039,8 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ m_key_reuse_mitigation2 = field_key_reuse_mitigation2; GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, segregation_height, int, Uint, false, 0); m_segregation_height = field_segregation_height; + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, ignore_fractional_outputs, int, Int, false, true); + m_ignore_fractional_outputs = field_ignore_fractional_outputs; GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, subaddress_lookahead_major, uint32_t, Uint, false, SUBADDRESS_LOOKAHEAD_MAJOR); m_subaddress_lookahead_major = field_subaddress_lookahead_major; GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, subaddress_lookahead_minor, uint32_t, Uint, false, SUBADDRESS_LOOKAHEAD_MINOR); @@ -2879,9 +3080,13 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ * can be used prior to rewriting wallet keys file, to ensure user has entered the correct password * */ -bool wallet2::verify_password(const epee::wipeable_string& password) const +bool wallet2::verify_password(const epee::wipeable_string& password) { - return verify_password(m_keys_file, password, m_watch_only || m_multisig, m_account.get_device()); + // this temporary unlocking is necessary for Windows (otherwise the file couldn't be loaded). + unlock_keys_file(); + bool r = verify_password(m_keys_file, password, m_watch_only || m_multisig, m_account.get_device()); + lock_keys_file(); + return r; } /*! @@ -3028,6 +3233,7 @@ void wallet2::generate(const std::string& wallet_, const epee::wipeable_string& cryptonote::block b; generate_genesis(b); m_blockchain.push_back(get_block_hash(b)); + m_last_block_reward = cryptonote::get_outs_money_amount(b.miner_tx); add_subaddress_account(tr("Primary account")); if (!wallet_.empty()) @@ -3086,6 +3292,7 @@ crypto::secret_key wallet2::generate(const std::string& wallet_, const epee::wip cryptonote::block b; generate_genesis(b); m_blockchain.push_back(get_block_hash(b)); + m_last_block_reward = cryptonote::get_outs_money_amount(b.miner_tx); add_subaddress_account(tr("Primary account")); if (!wallet_.empty()) @@ -3181,6 +3388,7 @@ void wallet2::generate(const std::string& wallet_, const epee::wipeable_string& cryptonote::block b; generate_genesis(b); m_blockchain.push_back(get_block_hash(b)); + m_last_block_reward = cryptonote::get_outs_money_amount(b.miner_tx); add_subaddress_account(tr("Primary account")); if (!wallet_.empty()) @@ -3233,6 +3441,7 @@ void wallet2::generate(const std::string& wallet_, const epee::wipeable_string& cryptonote::block b; generate_genesis(b); m_blockchain.push_back(get_block_hash(b)); + m_last_block_reward = cryptonote::get_outs_money_amount(b.miner_tx); add_subaddress_account(tr("Primary account")); if (!wallet_.empty()) @@ -3273,6 +3482,13 @@ void wallet2::restore(const std::string& wallet_, const epee::wipeable_string& p cryptonote::block b; generate_genesis(b); m_blockchain.push_back(get_block_hash(b)); + if (m_subaddress_lookahead_major == SUBADDRESS_LOOKAHEAD_MAJOR && m_subaddress_lookahead_minor == SUBADDRESS_LOOKAHEAD_MINOR) + { + // the default lookahead setting (50:200) is clearly too much for hardware wallet + m_subaddress_lookahead_major = 5; + m_subaddress_lookahead_minor = 20; + } + m_last_block_reward = cryptonote::get_outs_money_amount(b.miner_tx); add_subaddress_account(tr("Primary account")); if (!wallet_.empty()) { store(); @@ -3369,6 +3585,7 @@ std::string wallet2::make_multisig(const epee::wipeable_string &password, cryptonote::block b; generate_genesis(b); m_blockchain.push_back(get_block_hash(b)); + m_last_block_reward = cryptonote::get_outs_money_amount(b.miner_tx); add_subaddress_account(tr("Primary account")); if (!m_wallet_file.empty()) @@ -3776,12 +3993,17 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass boost::system::error_code e; bool exists = boost::filesystem::exists(m_keys_file, e); THROW_WALLET_EXCEPTION_IF(e || !exists, error::file_not_found, m_keys_file); + lock_keys_file(); + THROW_WALLET_EXCEPTION_IF(!is_keys_file_locked(), error::wallet_internal_error, "internal error: \"" + m_keys_file + "\" is opened by another wallet program"); + // this temporary unlocking is necessary for Windows (otherwise the file couldn't be loaded). + unlock_keys_file(); if (!load_keys(m_keys_file, password)) { THROW_WALLET_EXCEPTION_IF(true, error::file_read_error, m_keys_file); } LOG_PRINT_L0("Loaded wallet keys file, with public address: " << m_account.get_public_address_str(m_nettype)); + lock_keys_file(); //keys loaded ok! //try to load wallet file. but even if we failed, it is not big problem @@ -3794,7 +4016,7 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass { wallet2::cache_file_data cache_file_data; std::string buf; - bool r = epee::file_io_utils::load_file_to_string(m_wallet_file, buf); + bool r = epee::file_io_utils::load_file_to_string(m_wallet_file, buf, std::numeric_limits<size_t>::max()); THROW_WALLET_EXCEPTION_IF(!r, error::file_read_error, m_wallet_file); // try to read it as an encrypted cache @@ -3871,6 +4093,7 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass if (m_blockchain.empty()) { m_blockchain.push_back(genesis_hash); + m_last_block_reward = cryptonote::get_outs_money_amount(genesis.miner_tx); } else { @@ -3882,8 +4105,6 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass if (get_num_subaddress_accounts() == 0) add_subaddress_account(tr("Primary account")); - m_local_bc_height = m_blockchain.size(); - try { find_and_save_rings(false); @@ -4262,11 +4483,11 @@ void wallet2::rescan_blockchain(bool refresh) generate_genesis(genesis); crypto::hash genesis_hash = get_block_hash(genesis); m_blockchain.push_back(genesis_hash); + m_last_block_reward = cryptonote::get_outs_money_amount(genesis.miner_tx); add_subaddress_account(tr("Primary account")); - m_local_bc_height = 1; if (refresh) - this->refresh(); + this->refresh(false); } //---------------------------------------------------------------------------------------------------- bool wallet2::is_transfer_unlocked(const transfer_details& td) const @@ -4279,7 +4500,7 @@ bool wallet2::is_transfer_unlocked(uint64_t unlock_time, uint64_t block_height) if(!is_tx_spendtime_unlocked(unlock_time, block_height)) return false; - if(block_height + CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE > m_local_bc_height) + if(block_height + CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE > get_blockchain_current_height()) return false; return true; @@ -4290,7 +4511,7 @@ bool wallet2::is_tx_spendtime_unlocked(uint64_t unlock_time, uint64_t block_heig if(unlock_time < CRYPTONOTE_MAX_BLOCK_NUMBER) { //interpret as block index - if(m_local_bc_height-1 + CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_BLOCKS >= unlock_time) + if(get_blockchain_current_height()-1 + CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_BLOCKS >= unlock_time) return true; else return false; @@ -4814,11 +5035,10 @@ bool wallet2::sign_tx(unsigned_tx_set &exported_txs, std::vector<wallet2::pendin LOG_PRINT_L1(" " << (n+1) << ": " << sd.sources.size() << " inputs, ring size " << sd.sources[0].outputs.size()); signed_txes.ptx.push_back(pending_tx()); tools::wallet2::pending_tx &ptx = signed_txes.ptx.back(); - bool bulletproof = sd.use_rct && !ptx.tx.rct_signatures.p.bulletproofs.empty(); crypto::secret_key tx_key; std::vector<crypto::secret_key> additional_tx_keys; rct::multisig_out msout; - bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sd.sources, sd.splitted_dsts, sd.change_dts.addr, sd.extra, ptx.tx, sd.unlock_time, tx_key, additional_tx_keys, sd.use_rct, bulletproof, m_multisig ? &msout : NULL); + bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sd.sources, sd.splitted_dsts, sd.change_dts.addr, sd.extra, ptx.tx, sd.unlock_time, tx_key, additional_tx_keys, sd.use_rct, sd.use_bulletproofs, m_multisig ? &msout : NULL); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sd.sources, sd.splitted_dsts, sd.unlock_time, m_nettype); // we don't test tx size, because we don't know the current limit, due to not having a blockchain, // and it's a bit pointless to fail there anyway, since it'd be a (good) guess only. We sign anyway, @@ -4882,7 +5102,7 @@ bool wallet2::sign_tx(unsigned_tx_set &exported_txs, const std::string &signed_f return false; } - if (!epee::file_io_utils::save_string_to_file(signed_filename, std::string(SIGNED_TX_PREFIX) + ciphertext)) + if (!epee::file_io_utils::save_string_to_file(signed_filename, ciphertext)) { LOG_PRINT_L0("Failed to save file to " << signed_filename); return false; @@ -5231,8 +5451,7 @@ bool wallet2::sign_multisig_tx(multisig_tx_set &exported_txs, std::vector<crypto cryptonote::transaction tx; rct::multisig_out msout = ptx.multisig_sigs.front().msout; auto sources = sd.sources; - const bool bulletproof = sd.use_rct && (ptx.tx.rct_signatures.type == rct::RCTTypeFullBulletproof || ptx.tx.rct_signatures.type == rct::RCTTypeSimpleBulletproof); - bool r = cryptonote::construct_tx_with_tx_key(m_account.get_keys(), m_subaddresses, sources, sd.splitted_dsts, ptx.change_dts.addr, sd.extra, tx, sd.unlock_time, ptx.tx_key, ptx.additional_tx_keys, sd.use_rct, bulletproof, &msout, false); + bool r = cryptonote::construct_tx_with_tx_key(m_account.get_keys(), m_subaddresses, sources, sd.splitted_dsts, ptx.change_dts.addr, sd.extra, tx, sd.unlock_time, ptx.tx_key, ptx.additional_tx_keys, sd.use_rct, sd.use_bulletproofs, &msout, false); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sd.sources, sd.splitted_dsts, sd.unlock_time, m_nettype); THROW_WALLET_EXCEPTION_IF(get_transaction_prefix_hash (tx) != get_transaction_prefix_hash(ptx.tx), @@ -5436,15 +5655,10 @@ uint32_t wallet2::adjust_priority(uint32_t priority) } // get the current full reward zone - cryptonote::COMMAND_RPC_GET_INFO::request getinfo_req = AUTO_VAL_INIT(getinfo_req); - cryptonote::COMMAND_RPC_GET_INFO::response getinfo_res = AUTO_VAL_INIT(getinfo_res); - m_daemon_rpc_mutex.lock(); - bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_info", getinfo_req, getinfo_res, m_http_client); - m_daemon_rpc_mutex.unlock(); - THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "get_info"); - THROW_WALLET_EXCEPTION_IF(getinfo_res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_info"); - THROW_WALLET_EXCEPTION_IF(getinfo_res.status != CORE_RPC_STATUS_OK, error::get_tx_pool_error); - const uint64_t full_reward_zone = getinfo_res.block_size_limit / 2; + uint64_t block_size_limit = 0; + const auto result = m_node_rpc_proxy.get_block_size_limit(block_size_limit); + throw_on_rpc_response_error(result, "get_info"); + const uint64_t full_reward_zone = block_size_limit / 2; // get the last N block headers and sum the block sizes const size_t N = 10; @@ -5458,7 +5672,7 @@ uint32_t wallet2::adjust_priority(uint32_t priority) m_daemon_rpc_mutex.lock(); getbh_req.start_height = m_blockchain.size() - N; getbh_req.end_height = m_blockchain.size() - 1; - r = net_utils::invoke_http_json_rpc("/json_rpc", "getblockheadersrange", getbh_req, getbh_res, m_http_client, rpc_timeout); + bool r = net_utils::invoke_http_json_rpc("/json_rpc", "getblockheadersrange", getbh_req, getbh_res, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "getblockheadersrange"); THROW_WALLET_EXCEPTION_IF(getbh_res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "getblockheadersrange"); @@ -5625,6 +5839,24 @@ bool wallet2::set_ring_database(const std::string &filename) return true; } +crypto::chacha_key wallet2::get_ringdb_key() +{ + if (!m_ringdb_key) + { + MINFO("caching ringdb key"); + crypto::chacha_key key; + generate_chacha_key_from_secret_keys(key); + m_ringdb_key = key; + } + return *m_ringdb_key; +} + +void wallet2::clear_ringdb_key() +{ + MINFO("clearing ringdb key"); + m_ringdb_key = boost::none; +} + bool wallet2::add_rings(const crypto::chacha_key &key, const cryptonote::transaction_prefix &tx) { if (!m_ringdb) @@ -5635,9 +5867,8 @@ bool wallet2::add_rings(const crypto::chacha_key &key, const cryptonote::transac bool wallet2::add_rings(const cryptonote::transaction_prefix &tx) { - crypto::chacha_key key; - generate_chacha_key_from_secret_keys(key); - try { return add_rings(key, tx); } + key_ref kref(*this); + try { return add_rings(get_ringdb_key(), tx); } catch (const std::exception &e) { return false; } } @@ -5645,9 +5876,8 @@ bool wallet2::remove_rings(const cryptonote::transaction_prefix &tx) { if (!m_ringdb) return false; - crypto::chacha_key key; - generate_chacha_key_from_secret_keys(key); - try { return m_ringdb->remove_rings(key, tx); } + key_ref kref(*this); + try { return m_ringdb->remove_rings(get_ringdb_key(), tx); } catch (const std::exception &e) { return false; } } @@ -5684,10 +5914,8 @@ bool wallet2::get_rings(const crypto::hash &txid, std::vector<std::pair<crypto:: bool wallet2::get_ring(const crypto::key_image &key_image, std::vector<uint64_t> &outs) { - crypto::chacha_key key; - generate_chacha_key_from_secret_keys(key); - - try { return get_ring(key, key_image, outs); } + key_ref kref(*this); + try { return get_ring(get_ringdb_key(), key_image, outs); } catch (const std::exception &e) { return false; } } @@ -5696,10 +5924,8 @@ bool wallet2::set_ring(const crypto::key_image &key_image, const std::vector<uin if (!m_ringdb) return false; - crypto::chacha_key key; - generate_chacha_key_from_secret_keys(key); - - try { return m_ringdb->set_ring(key, key_image, outs, relative); } + key_ref kref(*this); + try { return m_ringdb->set_ring(get_ringdb_key(), key_image, outs, relative); } catch (const std::exception &e) { return false; } } @@ -5710,6 +5936,7 @@ bool wallet2::find_and_save_rings(bool force) if (!m_ringdb) return false; + key_ref kref(*this); COMMAND_RPC_GET_TRANSACTIONS::request req = AUTO_VAL_INIT(req); COMMAND_RPC_GET_TRANSACTIONS::response res = AUTO_VAL_INIT(res); @@ -5727,9 +5954,6 @@ bool wallet2::find_and_save_rings(bool force) MDEBUG("Found " << std::to_string(txs_hashes.size()) << " transactions"); - crypto::chacha_key key; - generate_chacha_key_from_secret_keys(key); - // get those transactions from the daemon static const size_t SLICE_SIZE = 200; for (size_t slice = 0; slice < txs_hashes.size(); slice += SLICE_SIZE) @@ -5766,7 +5990,7 @@ bool wallet2::find_and_save_rings(bool force) crypto::hash tx_hash, tx_prefix_hash; THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(bd, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, "failed to parse tx from blob"); THROW_WALLET_EXCEPTION_IF(epee::string_tools::pod_to_hex(tx_hash) != tx_info.tx_hash, error::wallet_internal_error, "txid mismatch"); - THROW_WALLET_EXCEPTION_IF(!add_rings(key, tx), error::wallet_internal_error, "Failed to save ring"); + THROW_WALLET_EXCEPTION_IF(!add_rings(get_ringdb_key(), tx), error::wallet_internal_error, "Failed to save ring"); } } @@ -5815,6 +6039,33 @@ bool wallet2::is_output_blackballed(const crypto::public_key &output) const catch (const std::exception &e) { return false; } } +bool wallet2::lock_keys_file() +{ + if (m_keys_file_locker) + { + MDEBUG(m_keys_file << " is already locked."); + return false; + } + m_keys_file_locker.reset(new tools::file_locker(m_keys_file)); + return true; +} + +bool wallet2::unlock_keys_file() +{ + if (!m_keys_file_locker) + { + MDEBUG(m_keys_file << " is already unlocked."); + return false; + } + m_keys_file_locker.reset(); + return true; +} + +bool wallet2::is_keys_file_locked() const +{ + return m_keys_file_locker->locked(); +} + bool wallet2::tx_add_fake_output(std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, uint64_t global_index, const crypto::public_key& output_public_key, const rct::key& mask, uint64_t real_index, bool unlocked) const { if (!unlocked) // don't add locked outs @@ -5945,9 +6196,6 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> return; } - crypto::chacha_key key; - generate_chacha_key_from_secret_keys(key); - if (fake_outputs_count > 0) { uint64_t segregation_fork_height = get_segregation_fork_height(); @@ -5958,22 +6206,42 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> bool is_shortly_after_segregation_fork = height >= segregation_fork_height && height < segregation_fork_height + SEGREGATION_FORK_VICINITY; bool is_after_segregation_fork = height >= segregation_fork_height; + // if we have at least one rct out, get the distribution, or fall back to the previous system + uint64_t rct_start_height; + std::vector<uint64_t> rct_offsets; + bool has_rct = false; + for (size_t idx: selected_transfers) + if (m_transfers[idx].is_rct()) + { has_rct = true; break; } + const bool has_rct_distribution = has_rct && get_rct_distribution(rct_start_height, rct_offsets); + if (has_rct_distribution) + { + // check we're clear enough of rct start, to avoid corner cases below + THROW_WALLET_EXCEPTION_IF(rct_offsets.size() <= CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE, + error::get_output_distribution, "Not enough rct outputs"); + } + // get histogram for the amounts we need cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request req_t = AUTO_VAL_INIT(req_t); cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response resp_t = AUTO_VAL_INIT(resp_t); - m_daemon_rpc_mutex.lock(); + // request histogram for all outputs, except 0 if we have the rct distribution for(size_t idx: selected_transfers) - req_t.amounts.push_back(m_transfers[idx].is_rct() ? 0 : m_transfers[idx].amount()); - std::sort(req_t.amounts.begin(), req_t.amounts.end()); - auto end = std::unique(req_t.amounts.begin(), req_t.amounts.end()); - req_t.amounts.resize(std::distance(req_t.amounts.begin(), end)); - req_t.unlocked = true; - req_t.recent_cutoff = time(NULL) - RECENT_OUTPUT_ZONE; - bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_histogram", req_t, resp_t, m_http_client, rpc_timeout); - m_daemon_rpc_mutex.unlock(); - THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "transfer_selected"); - THROW_WALLET_EXCEPTION_IF(resp_t.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_output_histogram"); - THROW_WALLET_EXCEPTION_IF(resp_t.status != CORE_RPC_STATUS_OK, error::get_histogram_error, resp_t.status); + if (!m_transfers[idx].is_rct() || !has_rct_distribution) + req_t.amounts.push_back(m_transfers[idx].is_rct() ? 0 : m_transfers[idx].amount()); + if (!req_t.amounts.empty()) + { + std::sort(req_t.amounts.begin(), req_t.amounts.end()); + auto end = std::unique(req_t.amounts.begin(), req_t.amounts.end()); + req_t.amounts.resize(std::distance(req_t.amounts.begin(), end)); + req_t.unlocked = true; + req_t.recent_cutoff = time(NULL) - RECENT_OUTPUT_ZONE; + m_daemon_rpc_mutex.lock(); + bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_histogram", req_t, resp_t, m_http_client, rpc_timeout); + m_daemon_rpc_mutex.unlock(); + THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "transfer_selected"); + THROW_WALLET_EXCEPTION_IF(resp_t.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_output_histogram"); + THROW_WALLET_EXCEPTION_IF(resp_t.status != CORE_RPC_STATUS_OK, error::get_histogram_error, resp_t.status); + } // if we want to segregate fake outs pre or post fork, get distribution std::unordered_map<uint64_t, std::pair<uint64_t, uint64_t>> segregation_limit; @@ -6029,6 +6297,36 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> COMMAND_RPC_GET_OUTPUTS_BIN::request req = AUTO_VAL_INIT(req); COMMAND_RPC_GET_OUTPUTS_BIN::response daemon_resp = AUTO_VAL_INIT(daemon_resp); + struct gamma_engine + { + typedef uint64_t result_type; + static constexpr result_type min() { return 0; } + static constexpr result_type max() { return std::numeric_limits<result_type>::max(); } + result_type operator()() { return crypto::rand<result_type>(); } + } engine; + static const double shape = 19.28/*16.94*/; + //static const double shape = m_testnet ? 17.02 : 17.28; + static const double scale = 1/1.61; + std::gamma_distribution<double> gamma(shape, scale); + auto pick_gamma = [&]() + { + double x = gamma(engine); + x = exp(x); + uint64_t block_offset = x / DIFFICULTY_TARGET_V2; // this assumes constant target over the whole rct range + if (block_offset >= rct_offsets.size() - 1) + return std::numeric_limits<uint64_t>::max(); // bad pick + block_offset = rct_offsets.size() - 2 - block_offset; + THROW_WALLET_EXCEPTION_IF(block_offset >= rct_offsets.size() - 1, error::wallet_internal_error, "Bad offset calculation"); + THROW_WALLET_EXCEPTION_IF(rct_offsets[block_offset + 1] < rct_offsets[block_offset], + error::get_output_distribution, "Decreasing offsets in rct distribution: " + + std::to_string(block_offset) + ": " + std::to_string(rct_offsets[block_offset]) + ", " + + std::to_string(block_offset + 1) + ": " + std::to_string(rct_offsets[block_offset + 1])); + uint64_t n_rct = rct_offsets[block_offset + 1] - rct_offsets[block_offset]; + if (n_rct == 0) + return rct_offsets[block_offset] ? rct_offsets[block_offset] - 1 : 0; + return rct_offsets[block_offset] + crypto::rand<uint64_t>() % n_rct; + }; + size_t num_selected_transfers = 0; for(size_t idx: selected_transfers) { @@ -6039,6 +6337,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> // request more for rct in base recent (locked) coinbases are picked, since they're locked for longer size_t requested_outputs_count = base_requested_outputs_count + (td.is_rct() ? CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW - CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE : 0); size_t start = req.outputs.size(); + bool use_histogram = amount != 0 || !has_rct_distribution; const bool output_is_pre_fork = td.m_block_height < segregation_fork_height; uint64_t num_outs = 0, num_recent_outs = 0; @@ -6094,26 +6393,41 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> num_post_fork_outs = num_outs - segregation_limit[amount].first; } - LOG_PRINT_L1("" << num_outs << " unlocked outputs of size " << print_money(amount)); - THROW_WALLET_EXCEPTION_IF(num_outs == 0, error::wallet_internal_error, - "histogram reports no unlocked outputs for " + boost::lexical_cast<std::string>(amount) + ", not even ours"); - THROW_WALLET_EXCEPTION_IF(num_recent_outs > num_outs, error::wallet_internal_error, - "histogram reports more recent outs than outs for " + boost::lexical_cast<std::string>(amount)); + if (use_histogram) + { + LOG_PRINT_L1("" << num_outs << " unlocked outputs of size " << print_money(amount)); + THROW_WALLET_EXCEPTION_IF(num_outs == 0, error::wallet_internal_error, + "histogram reports no unlocked outputs for " + boost::lexical_cast<std::string>(amount) + ", not even ours"); + THROW_WALLET_EXCEPTION_IF(num_recent_outs > num_outs, error::wallet_internal_error, + "histogram reports more recent outs than outs for " + boost::lexical_cast<std::string>(amount)); + } + else + { + // the base offset of the first rct output in the first unlocked block (or the one to be if there's none) + num_outs = rct_offsets[rct_offsets.size() - CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE]; + LOG_PRINT_L1("" << num_outs << " unlocked rct outputs"); + THROW_WALLET_EXCEPTION_IF(num_outs == 0, error::wallet_internal_error, + "histogram reports no unlocked rct outputs, not even ours"); + } - // how many fake outs to draw on a pre-fork triangular distribution + // how many fake outs to draw on a pre-fork distribution size_t pre_fork_outputs_count = requested_outputs_count * pre_fork_num_out_ratio; size_t post_fork_outputs_count = requested_outputs_count * post_fork_num_out_ratio; // how many fake outs to draw otherwise size_t normal_output_count = requested_outputs_count - pre_fork_outputs_count - post_fork_outputs_count; - // X% of those outs are to be taken from recent outputs - size_t recent_outputs_count = normal_output_count * RECENT_OUTPUT_RATIO; - if (recent_outputs_count == 0) - recent_outputs_count = 1; // ensure we have at least one, if possible - if (recent_outputs_count > num_recent_outs) - recent_outputs_count = num_recent_outs; - if (td.m_global_output_index >= num_outs - num_recent_outs && recent_outputs_count > 0) - --recent_outputs_count; // if the real out is recent, pick one less recent fake out + size_t recent_outputs_count = 0; + if (use_histogram) + { + // X% of those outs are to be taken from recent outputs + recent_outputs_count = normal_output_count * RECENT_OUTPUT_RATIO; + if (recent_outputs_count == 0) + recent_outputs_count = 1; // ensure we have at least one, if possible + if (recent_outputs_count > num_recent_outs) + recent_outputs_count = num_recent_outs; + if (td.m_global_output_index >= num_outs - num_recent_outs && recent_outputs_count > 0) + --recent_outputs_count; // if the real out is recent, pick one less recent fake out + } LOG_PRINT_L1("Fake output makeup: " << requested_outputs_count << " requested: " << recent_outputs_count << " recent, " << pre_fork_outputs_count << " pre-fork, " << post_fork_outputs_count << " post-fork, " << (requested_outputs_count - recent_outputs_count - pre_fork_outputs_count - post_fork_outputs_count) << " full-chain"); @@ -6125,7 +6439,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> if (td.m_key_image_known && !td.m_key_image_partial) { std::vector<uint64_t> ring; - if (get_ring(key, td.m_key_image, ring)) + if (get_ring(get_ringdb_key(), td.m_key_image, ring)) { MINFO("This output has a known ring, reusing (size " << ring.size() << ")"); THROW_WALLET_EXCEPTION_IF(ring.size() > fake_outputs_count + 1, error::wallet_internal_error, @@ -6193,7 +6507,26 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> uint64_t i; const char *type = ""; - if (num_found - 1 < recent_outputs_count) // -1 to account for the real one we seeded with + if (amount == 0 && has_rct_distribution) + { + // gamma distribution + if (num_found -1 < recent_outputs_count + pre_fork_outputs_count) + { + do i = pick_gamma(); while (i >= segregation_limit[amount].first); + type = "pre-fork gamma"; + } + else if (num_found -1 < recent_outputs_count + pre_fork_outputs_count + post_fork_outputs_count) + { + do i = pick_gamma(); while (i < segregation_limit[amount].first || i >= num_outs); + type = "post-fork gamma"; + } + else + { + do i = pick_gamma(); while (i >= num_outs); + type = "gamma"; + } + } + else if (num_found - 1 < recent_outputs_count) // -1 to account for the real one we seeded with { // triangular distribution over [a,b) with a=0, mode c=b=up_index_limit uint64_t r = crypto::rand<uint64_t>() % ((uint64_t)1 << 53); @@ -6258,7 +6591,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> // get the keys for those m_daemon_rpc_mutex.lock(); - r = epee::net_utils::invoke_http_bin("/get_outs.bin", req, daemon_resp, m_http_client, rpc_timeout); + bool r = epee::net_utils::invoke_http_bin("/get_outs.bin", req, daemon_resp, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "get_outs.bin"); THROW_WALLET_EXCEPTION_IF(daemon_resp.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_outs.bin"); @@ -6317,7 +6650,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> if (td.m_key_image_known && !td.m_key_image_partial) { std::vector<uint64_t> ring; - if (get_ring(key, td.m_key_image, ring)) + if (get_ring(get_ringdb_key(), td.m_key_image, ring)) { for (uint64_t out: ring) { @@ -6535,6 +6868,7 @@ void wallet2::transfer_selected(const std::vector<cryptonote::tx_destination_ent ptx.construction_data.extra = tx.extra; ptx.construction_data.unlock_time = unlock_time; ptx.construction_data.use_rct = false; + ptx.construction_data.use_bulletproofs = false; ptx.construction_data.dests = dsts; // record which subaddress indices are being used as inputs ptx.construction_data.subaddr_account = subaddr_account; @@ -6790,6 +7124,7 @@ void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry ptx.construction_data.extra = tx.extra; ptx.construction_data.unlock_time = unlock_time; ptx.construction_data.use_rct = true; + ptx.construction_data.use_bulletproofs = !tx.rct_signatures.p.bulletproofs.empty(); ptx.construction_data.dests = dsts; // record which subaddress indices are being used as inputs ptx.construction_data.subaddr_account = subaddr_account; @@ -7417,8 +7752,11 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp cryptonote::transaction tx; pending_tx ptx; size_t bytes; + uint64_t needed_fee; std::vector<std::vector<tools::wallet2::get_outs_entry>> outs; + TX() : bytes(0), needed_fee(0) {} + void add(const account_public_address &addr, bool is_subaddress, uint64_t amount, unsigned int original_output_index, bool merge_destinations) { if (merge_destinations) { @@ -7497,12 +7835,24 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp for (uint32_t i : subaddr_indices) LOG_PRINT_L2("Candidate subaddress index for spending: " << i); + // determine threshold for fractional amount + const size_t tx_size_one_ring = estimate_tx_size(use_rct, 1, fake_outs_count, 2, 0, bulletproof); + const size_t tx_size_two_rings = estimate_tx_size(use_rct, 2, fake_outs_count, 2, 0, bulletproof); + THROW_WALLET_EXCEPTION_IF(tx_size_one_ring > tx_size_two_rings, error::wallet_internal_error, "Estimated tx size with 1 input is larger than with 2 inputs!"); + const size_t tx_size_per_ring = tx_size_two_rings - tx_size_one_ring; + const uint64_t fractional_threshold = (fee_multiplier * fee_per_kb * tx_size_per_ring) / 1024; + // gather all dust and non-dust outputs belonging to specified subaddresses size_t num_nondust_outputs = 0; size_t num_dust_outputs = 0; for (size_t i = 0; i < m_transfers.size(); ++i) { const transfer_details& td = m_transfers[i]; + if (m_ignore_fractional_outputs && td.amount() < fractional_threshold) + { + MDEBUG("Ignoring output " << i << " of amount " << print_money(td.amount()) << " which is below threshold " << print_money(fractional_threshold)); + continue; + } if (!td.m_spent && !td.m_key_image_partial && (use_rct ? true : !td.is_rct()) && is_transfer_unlocked(td) && td.m_subaddr_index.major == subaddr_account && subaddr_indices.count(td.m_subaddr_index.minor) == 1) { const uint32_t index_minor = td.m_subaddr_index.minor; @@ -7811,6 +8161,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp tx.ptx = test_ptx; tx.bytes = txBlob.size(); tx.outs = outs; + tx.needed_fee = needed_fee; accumulated_fee += test_ptx.fee; accumulated_change += test_ptx.change_dts.amount; adding_fee = false; @@ -7862,7 +8213,7 @@ skip_tx: fake_outs_count, /* CONST size_t fake_outputs_count, */ tx.outs, /* MOD std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, */ unlock_time, /* CONST uint64_t unlock_time, */ - needed_fee, /* CONST uint64_t fee, */ + tx.needed_fee, /* CONST uint64_t fee, */ extra, /* const std::vector<uint8_t>& extra, */ test_tx, /* OUT cryptonote::transaction& tx, */ test_ptx, /* OUT cryptonote::transaction& tx, */ @@ -7873,7 +8224,7 @@ skip_tx: fake_outs_count, tx.outs, unlock_time, - needed_fee, + tx.needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), @@ -7894,7 +8245,7 @@ skip_tx: for (size_t idx: tx.selected_transfers) tx_money += m_transfers[idx].amount(); LOG_PRINT_L1(" Transaction " << (1+std::distance(txes.begin(), i)) << "/" << txes.size() << - ": " << get_size_string(tx.bytes) << ", sending " << print_money(tx_money) << " in " << tx.selected_transfers.size() << + " " << get_transaction_hash(tx.ptx.tx) << ": " << get_size_string(tx.bytes) << ", sending " << print_money(tx_money) << " in " << tx.selected_transfers.size() << " outputs to " << tx.dsts.size() << " destination(s), including " << print_money(tx.ptx.fee) << " fee, " << print_money(tx.ptx.change_dts.amount) << " change"); ptx_vector.push_back(tx.ptx); @@ -7993,7 +8344,10 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton cryptonote::transaction tx; pending_tx ptx; size_t bytes; + uint64_t needed_fee; std::vector<std::vector<get_outs_entry>> outs; + + TX() : bytes(0), needed_fee(0) {} }; std::vector<TX> txes; uint64_t needed_fee, available_for_fee = 0; @@ -8091,6 +8445,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton tx.ptx = test_ptx; tx.bytes = txBlob.size(); tx.outs = outs; + tx.needed_fee = needed_fee; accumulated_fee += test_ptx.fee; accumulated_change += test_ptx.change_dts.amount; if (!unused_transfers_indices.empty() || !unused_dust_indices.empty()) @@ -8111,10 +8466,10 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton cryptonote::transaction test_tx; pending_tx test_ptx; if (use_rct) { - transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, tx.outs, unlock_time, needed_fee, extra, + transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, tx.outs, unlock_time, tx.needed_fee, extra, test_tx, test_ptx, bulletproof); } else { - transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, tx.outs, unlock_time, needed_fee, extra, + transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, tx.outs, unlock_time, tx.needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); } auto txBlob = t_serializable_object_to_blob(test_ptx.tx); @@ -8131,7 +8486,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton for (size_t idx: tx.selected_transfers) tx_money += m_transfers[idx].amount(); LOG_PRINT_L1(" Transaction " << (1+std::distance(txes.begin(), i)) << "/" << txes.size() << - ": " << get_size_string(tx.bytes) << ", sending " << print_money(tx_money) << " in " << tx.selected_transfers.size() << + " " << get_transaction_hash(tx.ptx.tx) << ": " << get_size_string(tx.bytes) << ", sending " << print_money(tx_money) << " in " << tx.selected_transfers.size() << " outputs to " << tx.dsts.size() << " destination(s), including " << print_money(tx.ptx.fee) << " fee, " << print_money(tx.ptx.change_dts.amount) << " change"); ptx_vector.push_back(tx.ptx); @@ -8318,6 +8673,16 @@ std::vector<wallet2::pending_tx> wallet2::create_unmixable_sweep_transactions(bo return create_transactions_from(m_account_public_address, false, unmixable_transfer_outputs, unmixable_dust_outputs, 0 /*fake_outs_count */, 0 /* unlock_time */, 1 /*priority */, std::vector<uint8_t>(), trusted_daemon); } +//---------------------------------------------------------------------------------------------------- +void wallet2::discard_unmixable_outputs(bool trusted_daemon) +{ + // may throw + std::vector<size_t> unmixable_outputs = select_available_unmixable_outputs(trusted_daemon); + for (size_t idx : unmixable_outputs) + { + m_transfers[idx].m_spent = true; + } +} bool wallet2::get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys) const { @@ -9193,39 +9558,23 @@ uint64_t wallet2::get_daemon_blockchain_height(string &err) const uint64_t wallet2::get_daemon_blockchain_target_height(string &err) { - cryptonote::COMMAND_RPC_GET_INFO::request req_t = AUTO_VAL_INIT(req_t); - cryptonote::COMMAND_RPC_GET_INFO::response resp_t = AUTO_VAL_INIT(resp_t); - m_daemon_rpc_mutex.lock(); - bool ok = net_utils::invoke_http_json_rpc("/json_rpc", "get_info", req_t, resp_t, m_http_client); - m_daemon_rpc_mutex.unlock(); - if (ok) - { - if (resp_t.status == CORE_RPC_STATUS_BUSY) - { - err = "daemon is busy. Please try again later."; - } - else if (resp_t.status != CORE_RPC_STATUS_OK) - { - err = resp_t.status; - } - else // success, cleaning up error message - { - err = ""; - } - } - else + err = ""; + uint64_t target_height = 0; + const auto result = m_node_rpc_proxy.get_target_height(target_height); + if (result && *result != CORE_RPC_STATUS_OK) { - err = "possibly lost connection to daemon"; + err= *result; + return 0; } - return resp_t.target_height; + return target_height; } uint64_t wallet2::get_approximate_blockchain_height() const { // time of v2 fork - const time_t fork_time = m_nettype == TESTNET ? 1448285909 : m_nettype == STAGENET ? (time_t)-1/*TODO*/ : 1458748658; + const time_t fork_time = m_nettype == TESTNET ? 1448285909 : m_nettype == STAGENET ? 1520937818 : 1458748658; // v2 fork block - const uint64_t fork_block = m_nettype == TESTNET ? 624634 : m_nettype == STAGENET ? (uint64_t)-1/*TODO*/ : 1009827; + const uint64_t fork_block = m_nettype == TESTNET ? 624634 : m_nettype == STAGENET ? 32000 : 1009827; // avg seconds per block const int seconds_per_block = DIFFICULTY_TARGET_V2; // Calculated blockchain height @@ -9635,7 +9984,7 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag std::unordered_set<crypto::hash> spent_txids; // For each spent key image, search for a tx in m_transfers that uses it as input. std::vector<size_t> swept_transfers; // If such a spending tx wasn't found in m_transfers, this means the spending tx // was created by sweep_all, so we can't know the spent height and other detailed info. - for(size_t i = 0; i < m_transfers.size(); ++i) + for(size_t i = 0; i < signed_key_images.size(); ++i) { transfer_details &td = m_transfers[i]; uint64_t amount = td.amount(); @@ -9858,7 +10207,7 @@ void wallet2::import_blockchain(const std::tuple<size_t, crypto::hash, std::vect generate_genesis(genesis); crypto::hash genesis_hash = get_block_hash(genesis); check_genesis(genesis_hash); - m_local_bc_height = m_blockchain.size(); + m_last_block_reward = cryptonote::get_outs_money_amount(genesis.miner_tx); } //---------------------------------------------------------------------------------------------------- std::vector<tools::wallet2::transfer_details> wallet2::export_outputs() const @@ -10258,7 +10607,7 @@ size_t wallet2::import_multisig(std::vector<cryptonote::blobdata> blobs) m_multisig_rescan_info = &info; try { - refresh(); + refresh(false); } catch (...) {} m_multisig_rescan_info = NULL; @@ -10575,15 +10924,10 @@ std::vector<std::pair<uint64_t, uint64_t>> wallet2::estimate_backlog(const std:: THROW_WALLET_EXCEPTION_IF(res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_txpool_backlog"); THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::get_tx_pool_error); - cryptonote::COMMAND_RPC_GET_INFO::request req_t = AUTO_VAL_INIT(req_t); - cryptonote::COMMAND_RPC_GET_INFO::response resp_t = AUTO_VAL_INIT(resp_t); - m_daemon_rpc_mutex.lock(); - r = net_utils::invoke_http_json_rpc("/json_rpc", "get_info", req_t, resp_t, m_http_client); - m_daemon_rpc_mutex.unlock(); - THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "get_info"); - THROW_WALLET_EXCEPTION_IF(resp_t.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_info"); - THROW_WALLET_EXCEPTION_IF(resp_t.status != CORE_RPC_STATUS_OK, error::get_tx_pool_error); - uint64_t full_reward_zone = resp_t.block_size_limit / 2; + uint64_t block_size_limit = 0; + const auto result = m_node_rpc_proxy.get_block_size_limit(block_size_limit); + throw_on_rpc_response_error(result, "get_info"); + uint64_t full_reward_zone = block_size_limit / 2; std::vector<std::pair<uint64_t, uint64_t>> blocks; for (const auto &fee_level: fee_levels) @@ -10685,17 +11029,6 @@ uint64_t wallet2::get_segregation_fork_height() const } //---------------------------------------------------------------------------------------------------- void wallet2::generate_genesis(cryptonote::block& b) const { - if (m_nettype == TESTNET) - { - cryptonote::generate_genesis_block(b, config::testnet::GENESIS_TX, config::testnet::GENESIS_NONCE); - } - else if (m_nettype == STAGENET) - { - cryptonote::generate_genesis_block(b, config::stagenet::GENESIS_TX, config::stagenet::GENESIS_NONCE); - } - else - { - cryptonote::generate_genesis_block(b, config::GENESIS_TX, config::GENESIS_NONCE); - } + cryptonote::generate_genesis_block(b, get_config(m_nettype).GENESIS_TX, get_config(m_nettype).GENESIS_NONCE); } } diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index a10cef561..0a9049c7b 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -320,6 +320,7 @@ namespace tools std::vector<uint8_t> extra; uint64_t unlock_time; bool use_rct; + bool use_bulletproofs; std::vector<cryptonote::tx_destination_entry> dests; // original setup, does not include change uint32_t subaddr_account; // subaddress account of your wallet to be used in this transfer std::set<uint32_t> subaddr_indices; // set of address indices used as inputs in this transfer @@ -332,6 +333,7 @@ namespace tools FIELD(extra) FIELD(unlock_time) FIELD(use_rct) + FIELD(use_bulletproofs) FIELD(dests) FIELD(subaddr_account) FIELD(subaddr_indices) @@ -452,6 +454,39 @@ namespace tools typedef std::tuple<uint64_t, crypto::public_key, rct::key> get_outs_entry; + struct parsed_block + { + crypto::hash hash; + cryptonote::block block; + std::vector<cryptonote::transaction> txes; + cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices o_indices; + bool error; + }; + + struct is_out_data + { + crypto::public_key pkey; + crypto::key_derivation derivation; + std::vector<boost::optional<cryptonote::subaddress_receive_info>> received; + }; + + struct tx_cache_data + { + std::vector<cryptonote::tx_extra_field> tx_extra_fields; + std::vector<is_out_data> primary; + std::vector<is_out_data> additional; + }; + + struct key_ref + { + key_ref(tools::wallet2 &w): wallet(w) { ++refs; } + ~key_ref() { if (!--refs) wallet.clear_ringdb_key(); } + + private: + tools::wallet2 &wallet; + static std::atomic<unsigned int> refs; + }; + /*! * \brief Generates a wallet or restores one. * \param wallet_ Name of wallet file @@ -574,7 +609,7 @@ namespace tools /*! * \brief verifies given password is correct for default wallet keys file */ - bool verify_password(const epee::wipeable_string& password) const; + bool verify_password(const epee::wipeable_string& password); cryptonote::account_base& get_account(){return m_account;} const cryptonote::account_base& get_account()const{return m_account;} @@ -642,10 +677,10 @@ namespace tools * \brief Tells if the wallet file is deprecated. */ bool is_deprecated() const; - void refresh(); - void refresh(uint64_t start_height, uint64_t & blocks_fetched); - void refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& received_money); - bool refresh(uint64_t & blocks_fetched, bool& received_money, bool& ok); + void refresh(bool trusted_daemon); + void refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched); + void refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched, bool& received_money); + bool refresh(bool trusted_daemon, uint64_t & blocks_fetched, bool& received_money, bool& ok); void set_refresh_type(RefreshType refresh_type) { m_refresh_type = refresh_type; } RefreshType get_refresh_type() const { return m_refresh_type; } @@ -713,6 +748,7 @@ namespace tools bool sign_multisig_tx(multisig_tx_set &exported_txs, std::vector<crypto::hash> &txids); bool sign_multisig_tx_to_file(multisig_tx_set &exported_txs, const std::string &filename, std::vector<crypto::hash> &txids); std::vector<pending_tx> create_unmixable_sweep_transactions(bool trusted_daemon); + void discard_unmixable_outputs(bool trusted_daemon); bool check_connection(uint32_t *version = NULL, uint32_t timeout = 200000); void get_transfers(wallet2::transfer_container& incoming_transfers) const; void get_payments(const crypto::hash& payment_id, std::list<wallet2::payment_details>& payments, uint64_t min_height = 0, const boost::optional<uint32_t>& subaddr_account = boost::none, const std::set<uint32_t>& subaddr_indices = {}) const; @@ -722,11 +758,14 @@ namespace tools void get_unconfirmed_payments_out(std::list<std::pair<crypto::hash,wallet2::unconfirmed_transfer_details>>& unconfirmed_payments, const boost::optional<uint32_t>& subaddr_account = boost::none, const std::set<uint32_t>& subaddr_indices = {}) const; void get_unconfirmed_payments(std::list<std::pair<crypto::hash,wallet2::pool_payment_details>>& unconfirmed_payments, const boost::optional<uint32_t>& subaddr_account = boost::none, const std::set<uint32_t>& subaddr_indices = {}) const; - uint64_t get_blockchain_current_height() const { return m_local_bc_height; } + uint64_t get_blockchain_current_height() const { return m_light_wallet_blockchain_height ? m_light_wallet_blockchain_height : m_blockchain.size(); } void rescan_spent(); void rescan_blockchain(bool refresh = true); bool is_transfer_unlocked(const transfer_details& td) const; bool is_transfer_unlocked(uint64_t unlock_time, uint64_t block_height) const; + + uint64_t get_last_block_reward() const { return m_last_block_reward; } + template <class t_archive> inline void serialize(t_archive &a, const unsigned int ver) { @@ -828,6 +867,9 @@ namespace tools if(ver < 24) return; a & m_ring_history_saved; + if(ver < 25) + return; + a & m_last_block_reward; } /*! @@ -883,6 +925,8 @@ namespace tools void key_reuse_mitigation2(bool value) { m_key_reuse_mitigation2 = value; } uint64_t segregation_height() const { return m_segregation_height; } void segregation_height(uint64_t height) { m_segregation_height = height; } + bool ignore_fractional_outputs() const { return m_ignore_fractional_outputs; } + void ignore_fractional_outputs(bool value) { m_ignore_fractional_outputs = value; } bool confirm_non_default_ring_size() const { return m_confirm_non_default_ring_size; } void confirm_non_default_ring_size(bool always) { m_confirm_non_default_ring_size = always; } @@ -1102,6 +1146,9 @@ namespace tools bool unblackball_output(const crypto::public_key &output); bool is_output_blackballed(const crypto::public_key &output) const; + bool lock_keys_file(); + bool unlock_keys_file(); + bool is_keys_file_locked() const; private: /*! * \brief Stores wallet information to wallet file. @@ -1117,17 +1164,17 @@ namespace tools * \param password Password of wallet file */ bool load_keys(const std::string& keys_file_name, const epee::wipeable_string& password); - void process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector<uint64_t> &o_indices, uint64_t height, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen); - void process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const crypto::hash& bl_id, uint64_t height, const cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices &o_indices); + void process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector<uint64_t> &o_indices, uint64_t height, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen, const tx_cache_data &tx_cache_data); + void process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const parsed_block &parsed_block, const crypto::hash& bl_id, uint64_t height, const std::vector<tx_cache_data> &tx_cache_data, size_t tx_cache_data_offset); void detach_blockchain(uint64_t height); - void get_short_chain_history(std::list<crypto::hash>& ids) const; + void get_short_chain_history(std::list<crypto::hash>& ids, uint64_t granularity = 1) const; bool is_tx_spendtime_unlocked(uint64_t unlock_time, uint64_t block_height) const; bool clear(); - void pull_blocks(uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::list<cryptonote::block_complete_entry> &blocks, std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices); - void pull_hashes(uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::list<crypto::hash> &hashes); - void fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history); - void pull_next_blocks(uint64_t start_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, const std::list<cryptonote::block_complete_entry> &prev_blocks, std::list<cryptonote::block_complete_entry> &blocks, std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices, bool &error); - void process_blocks(uint64_t start_height, const std::list<cryptonote::block_complete_entry> &blocks, const std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices, uint64_t& blocks_added); + void pull_blocks(uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices); + void pull_hashes(uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<crypto::hash> &hashes); + void fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, bool force = false); + void pull_and_parse_next_blocks(uint64_t start_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, const std::vector<cryptonote::block_complete_entry> &prev_blocks, const std::vector<parsed_block> &prev_parsed_blocks, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<parsed_block> &parsed_blocks, bool &error); + void process_parsed_blocks(uint64_t start_height, const std::vector<cryptonote::block_complete_entry> &blocks, const std::vector<parsed_block> &parsed_blocks, uint64_t& blocks_added); uint64_t select_transfers(uint64_t needed_money, std::vector<size_t> unused_transfers_indices, std::vector<size_t>& selected_transfers, bool trusted_daemon) const; bool prepare_file_names(const std::string& file_path); void process_unconfirmed(const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t height); @@ -1138,6 +1185,8 @@ namespace tools bool generate_chacha_key_from_secret_keys(crypto::chacha_key &key) const; crypto::hash get_payment_id(const pending_tx &ptx) const; void check_acc_out_precomp(const cryptonote::tx_out &o, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, size_t i, tx_scan_info_t &tx_scan_info) const; + void check_acc_out_precomp(const cryptonote::tx_out &o, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, size_t i, const is_out_data *is_out_data, tx_scan_info_t &tx_scan_info) const; + void check_acc_out_precomp_once(const cryptonote::tx_out &o, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, size_t i, const is_out_data *is_out_data, tx_scan_info_t &tx_scan_info, bool &already_seen) const; void parse_block_round(const cryptonote::blobdata &blob, cryptonote::block &bl, crypto::hash &bl_id, bool &error) const; uint64_t get_upper_transaction_size_limit() const; std::vector<uint64_t> get_unspent_amounts_vector() const; @@ -1162,11 +1211,16 @@ namespace tools bool add_rings(const cryptonote::transaction_prefix &tx); bool remove_rings(const cryptonote::transaction_prefix &tx); bool get_ring(const crypto::chacha_key &key, const crypto::key_image &key_image, std::vector<uint64_t> &outs); + crypto::chacha_key get_ringdb_key(); + void cache_ringdb_key(); + void clear_ringdb_key(); - bool get_output_distribution(uint64_t &start_height, std::vector<uint64_t> &distribution); + bool get_rct_distribution(uint64_t &start_height, std::vector<uint64_t> &distribution); uint64_t get_segregation_fork_height() const; + void cache_tx_data(const cryptonote::transaction& tx, const crypto::hash &txid, tx_cache_data &tx_cache_data) const; + cryptonote::account_base m_account; boost::optional<epee::net_utils::http::login> m_daemon_login; std::string m_daemon_address; @@ -1174,7 +1228,6 @@ namespace tools std::string m_keys_file; epee::net_utils::http::http_simple_client m_http_client; hashchain m_blockchain; - std::atomic<uint64_t> m_local_bc_height; //temporary workaround std::unordered_map<crypto::hash, unconfirmed_transfer_details> m_unconfirmed_txs; std::unordered_map<crypto::hash, confirmed_transfer_details> m_confirmed_txs; std::unordered_multimap<crypto::hash, pool_payment_details> m_unconfirmed_payments; @@ -1218,6 +1271,7 @@ namespace tools uint32_t m_default_priority; RefreshType m_refresh_type; bool m_auto_refresh; + bool m_first_refresh_done; uint64_t m_refresh_from_block_height; // If m_refresh_from_block_height is explicitly set to zero we need this to differentiate it from the case that // m_refresh_from_block_height was defaulted to zero.*/ @@ -1235,6 +1289,7 @@ namespace tools bool m_segregate_pre_fork_outputs; bool m_key_reuse_mitigation2; uint64_t m_segregation_height; + bool m_ignore_fractional_outputs; bool m_is_initialized; NodeRPCProxy m_node_rpc_proxy; std::unordered_set<crypto::hash> m_scanned_pool_txs[2]; @@ -1257,9 +1312,13 @@ namespace tools std::string m_ring_database; bool m_ring_history_saved; std::unique_ptr<ringdb> m_ringdb; + boost::optional<crypto::chacha_key> m_ringdb_key; + + uint64_t m_last_block_reward; + std::unique_ptr<tools::file_locker> m_keys_file_locker; }; } -BOOST_CLASS_VERSION(tools::wallet2, 24) +BOOST_CLASS_VERSION(tools::wallet2, 25) BOOST_CLASS_VERSION(tools::wallet2::transfer_details, 9) BOOST_CLASS_VERSION(tools::wallet2::multisig_info, 1) BOOST_CLASS_VERSION(tools::wallet2::multisig_info::LR, 0) @@ -1272,7 +1331,7 @@ BOOST_CLASS_VERSION(tools::wallet2::address_book_row, 17) BOOST_CLASS_VERSION(tools::wallet2::reserve_proof_entry, 0) BOOST_CLASS_VERSION(tools::wallet2::unsigned_tx_set, 0) BOOST_CLASS_VERSION(tools::wallet2::signed_tx_set, 0) -BOOST_CLASS_VERSION(tools::wallet2::tx_construction_data, 2) +BOOST_CLASS_VERSION(tools::wallet2::tx_construction_data, 3) BOOST_CLASS_VERSION(tools::wallet2::pending_tx, 3) BOOST_CLASS_VERSION(tools::wallet2::multisig_sig, 0) @@ -1619,6 +1678,9 @@ namespace boost if (ver < 2) return; a & x.selected_transfers; + if (ver < 3) + return; + a & x.use_bulletproofs; } template <class Archive> @@ -1904,6 +1966,7 @@ namespace tools ptx.construction_data.extra = tx.extra; ptx.construction_data.unlock_time = unlock_time; ptx.construction_data.use_rct = false; + ptx.construction_data.use_bulletproofs = false; ptx.construction_data.dests = dsts; // record which subaddress indices are being used as inputs ptx.construction_data.subaddr_account = subaddr_account; diff --git a/src/wallet/wallet_args.cpp b/src/wallet/wallet_args.cpp index 6311e7700..a629eb506 100644 --- a/src/wallet/wallet_args.cpp +++ b/src/wallet/wallet_args.cpp @@ -101,6 +101,7 @@ namespace wallet_args const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""}; const command_line::arg_descriptor<std::size_t> arg_max_log_file_size = {"max-log-file-size", "Specify maximum log file size [B]", MAX_LOG_FILE_SIZE}; + const command_line::arg_descriptor<std::size_t> arg_max_log_files = {"max-log-files", "Specify maximum number of rotated log files to be saved (no limit by setting to 0)", MAX_LOG_FILES}; const command_line::arg_descriptor<uint32_t> arg_max_concurrency = {"max-concurrency", wallet_args::tr("Max number of threads to use for a parallel job"), DEFAULT_MAX_CONCURRENCY}; const command_line::arg_descriptor<std::string> arg_log_file = {"log-file", wallet_args::tr("Specify log file"), ""}; const command_line::arg_descriptor<std::string> arg_config_file = {"config-file", wallet_args::tr("Config file"), "", true}; @@ -119,6 +120,7 @@ namespace wallet_args command_line::add_arg(desc_params, arg_log_file); command_line::add_arg(desc_params, arg_log_level); command_line::add_arg(desc_params, arg_max_log_file_size); + command_line::add_arg(desc_params, arg_max_log_files); command_line::add_arg(desc_params, arg_max_concurrency); command_line::add_arg(desc_params, arg_config_file); @@ -180,11 +182,15 @@ namespace wallet_args log_path = command_line::get_arg(vm, arg_log_file); else log_path = mlog_get_default_log_path(default_log_name); - mlog_configure(log_path, log_to_console, command_line::get_arg(vm, arg_max_log_file_size)); + mlog_configure(log_path, log_to_console, command_line::get_arg(vm, arg_max_log_file_size), command_line::get_arg(vm, arg_max_log_files)); if (!command_line::is_arg_defaulted(vm, arg_log_level)) { mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); } + else if (!log_to_console) + { + mlog_set_categories(""); + } if (notice) Print(print) << notice << ENDL; diff --git a/src/wallet/wallet_errors.h b/src/wallet/wallet_errors.h index 214d51cde..e80652750 100644 --- a/src/wallet/wallet_errors.h +++ b/src/wallet/wallet_errors.h @@ -70,6 +70,7 @@ namespace tools // get_out_indexes_error // tx_parse_error // get_tx_pool_error + // out_of_hashchain_bounds_error // transfer_error * // get_random_outs_general_error // not_enough_unlocked_money @@ -398,6 +399,16 @@ namespace tools std::string to_string() const { return refresh_error::to_string(); } }; //---------------------------------------------------------------------------------------------------- + struct out_of_hashchain_bounds_error : public refresh_error + { + explicit out_of_hashchain_bounds_error(std::string&& loc) + : refresh_error(std::move(loc), "Index out of bounds of of hashchain") + { + } + + std::string to_string() const { return refresh_error::to_string(); } + }; + //---------------------------------------------------------------------------------------------------- struct transfer_error : public wallet_logic_error { protected: diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index 7f7d33642..b9cf99635 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -74,6 +74,21 @@ namespace } return pwd_container; } + //------------------------------------------------------------------------------------------------------------------------------ + void set_confirmations(tools::wallet_rpc::transfer_entry &entry, uint64_t blockchain_height, uint64_t block_reward) + { + if (entry.height >= blockchain_height) + { + entry.confirmations = 0; + entry.suggested_confirmations_threshold = 0; + return; + } + entry.confirmations = blockchain_height - entry.height; + if (block_reward == 0) + entry.suggested_confirmations_threshold = 0; + else + entry.suggested_confirmations_threshold = (entry.amount + block_reward - 1) / block_reward; + } } namespace tools @@ -104,7 +119,7 @@ namespace tools m_stop = false; m_net_server.add_idle_handler([this](){ try { - if (m_wallet) m_wallet->refresh(); + if (m_wallet) m_wallet->refresh(m_trusted_daemon); } catch (const std::exception& ex) { LOG_ERROR("Exception at while refreshing, what=" << ex.what()); } @@ -258,6 +273,7 @@ namespace tools entry.type = "in"; entry.subaddr_index = pd.m_subaddr_index; entry.address = m_wallet->get_subaddress_as_str(pd.m_subaddr_index); + set_confirmations(entry, m_wallet->get_blockchain_current_height(), m_wallet->get_last_block_reward()); } //------------------------------------------------------------------------------------------------------------------------------ void wallet_rpc_server::fill_transfer_entry(tools::wallet_rpc::transfer_entry &entry, const crypto::hash &txid, const tools::wallet2::confirmed_transfer_details &pd) @@ -284,6 +300,7 @@ namespace tools entry.type = "out"; entry.subaddr_index = { pd.m_subaddr_account, 0 }; entry.address = m_wallet->get_subaddress_as_str({pd.m_subaddr_account, 0}); + set_confirmations(entry, m_wallet->get_blockchain_current_height(), m_wallet->get_last_block_reward()); } //------------------------------------------------------------------------------------------------------------------------------ void wallet_rpc_server::fill_transfer_entry(tools::wallet_rpc::transfer_entry &entry, const crypto::hash &txid, const tools::wallet2::unconfirmed_transfer_details &pd) @@ -303,6 +320,7 @@ namespace tools entry.type = is_failed ? "failed" : "pending"; entry.subaddr_index = { pd.m_subaddr_account, 0 }; entry.address = m_wallet->get_subaddress_as_str({pd.m_subaddr_account, 0}); + set_confirmations(entry, m_wallet->get_blockchain_current_height(), m_wallet->get_last_block_reward()); } //------------------------------------------------------------------------------------------------------------------------------ void wallet_rpc_server::fill_transfer_entry(tools::wallet_rpc::transfer_entry &entry, const crypto::hash &payment_id, const tools::wallet2::pool_payment_details &ppd) @@ -322,6 +340,7 @@ namespace tools entry.type = "pool"; entry.subaddr_index = pd.m_subaddr_index; entry.address = m_wallet->get_subaddress_as_str(pd.m_subaddr_index); + set_confirmations(entry, m_wallet->get_blockchain_current_height(), m_wallet->get_last_block_reward()); } //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_getbalance(const wallet_rpc::COMMAND_RPC_GET_BALANCE::request& req, wallet_rpc::COMMAND_RPC_GET_BALANCE::response& res, epee::json_rpc::error& er) @@ -362,6 +381,7 @@ namespace tools if (!m_wallet) return not_open(er); try { + THROW_WALLET_EXCEPTION_IF(req.account_index >= m_wallet->get_num_subaddress_accounts(), error::account_index_outofbound); res.addresses.clear(); std::vector<uint32_t> req_address_index; if (req.address_index.empty()) @@ -377,6 +397,7 @@ namespace tools m_wallet->get_transfers(transfers); for (uint32_t i : req_address_index) { + THROW_WALLET_EXCEPTION_IF(i >= m_wallet->get_num_subaddresses(req.account_index), error::address_index_outofbound); res.addresses.resize(res.addresses.size() + 1); auto& info = res.addresses.back(); const cryptonote::subaddress_index index = {req.account_index, i}; @@ -500,6 +521,7 @@ namespace tools //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_get_account_tags(const wallet_rpc::COMMAND_RPC_GET_ACCOUNT_TAGS::request& req, wallet_rpc::COMMAND_RPC_GET_ACCOUNT_TAGS::response& res, epee::json_rpc::error& er) { + if (!m_wallet) return not_open(er); const std::pair<std::map<std::string, std::string>, std::vector<std::string>> account_tags = m_wallet->get_account_tags(); for (const std::pair<std::string, std::string>& p : account_tags.first) { @@ -518,6 +540,7 @@ namespace tools //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_tag_accounts(const wallet_rpc::COMMAND_RPC_TAG_ACCOUNTS::request& req, wallet_rpc::COMMAND_RPC_TAG_ACCOUNTS::response& res, epee::json_rpc::error& er) { + if (!m_wallet) return not_open(er); try { m_wallet->set_account_tag(req.accounts, req.tag); @@ -532,6 +555,7 @@ namespace tools //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_untag_accounts(const wallet_rpc::COMMAND_RPC_UNTAG_ACCOUNTS::request& req, wallet_rpc::COMMAND_RPC_UNTAG_ACCOUNTS::response& res, epee::json_rpc::error& er) { + if (!m_wallet) return not_open(er); try { m_wallet->set_account_tag(req.accounts, ""); @@ -546,6 +570,7 @@ namespace tools //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_set_account_tag_description(const wallet_rpc::COMMAND_RPC_SET_ACCOUNT_TAG_DESCRIPTION::request& req, wallet_rpc::COMMAND_RPC_SET_ACCOUNT_TAG_DESCRIPTION::response& res, epee::json_rpc::error& er) { + if (!m_wallet) return not_open(er); try { m_wallet->set_account_tag_description(req.tag, req.description); @@ -2264,6 +2289,7 @@ namespace tools //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_make_uri(const wallet_rpc::COMMAND_RPC_MAKE_URI::request& req, wallet_rpc::COMMAND_RPC_MAKE_URI::response& res, epee::json_rpc::error& er) { + if (!m_wallet) return not_open(er); std::string error; std::string uri = m_wallet->make_uri(req.address, req.payment_id, req.amount, req.tx_description, req.recipient_name, error); if (uri.empty()) @@ -2477,6 +2503,7 @@ namespace tools //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_stop_mining(const wallet_rpc::COMMAND_RPC_STOP_MINING::request& req, wallet_rpc::COMMAND_RPC_STOP_MINING::response& res, epee::json_rpc::error& er) { + if (!m_wallet) return not_open(er); cryptonote::COMMAND_RPC_STOP_MINING::request daemon_req; cryptonote::COMMAND_RPC_STOP_MINING::response daemon_res; bool r = m_wallet->invoke_http_json("/stop_mining", daemon_req, daemon_res); @@ -3196,7 +3223,7 @@ int main(int argc, char** argv) { wal->stop(); }); - wal->refresh(); + wal->refresh(command_line::get_arg(*vm, arg_trusted_daemon)); // if we ^C during potentially length load/refresh, there's no server loop yet if (quit) { diff --git a/src/wallet/wallet_rpc_server_commands_defs.h b/src/wallet/wallet_rpc_server_commands_defs.h index 96e135f01..1bd572add 100644 --- a/src/wallet/wallet_rpc_server_commands_defs.h +++ b/src/wallet/wallet_rpc_server_commands_defs.h @@ -47,7 +47,7 @@ // advance which version they will stop working with // Don't go over 32767 for any of these #define WALLET_RPC_VERSION_MAJOR 1 -#define WALLET_RPC_VERSION_MINOR 0 +#define WALLET_RPC_VERSION_MINOR 1 #define MAKE_WALLET_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define WALLET_RPC_VERSION MAKE_WALLET_RPC_VERSION(WALLET_RPC_VERSION_MAJOR, WALLET_RPC_VERSION_MINOR) namespace tools @@ -1194,6 +1194,8 @@ namespace wallet_rpc cryptonote::subaddress_index subaddr_index; std::string address; bool double_spend_seen; + uint64_t confirmations; + uint64_t suggested_confirmations_threshold; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(txid); @@ -1209,6 +1211,8 @@ namespace wallet_rpc KV_SERIALIZE(subaddr_index); KV_SERIALIZE(address); KV_SERIALIZE(double_spend_seen) + KV_SERIALIZE_OPT(confirmations, (uint64_t)0) + KV_SERIALIZE_OPT(suggested_confirmations_threshold, (uint64_t)0) END_KV_SERIALIZE_MAP() }; |