diff options
Diffstat (limited to 'src')
191 files changed, 19042 insertions, 3626 deletions
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3b71c38cd..a0b62da77 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -34,11 +34,6 @@ if (WIN32 OR STATIC) add_definitions(-DMINIUPNP_STATICLIB) endif () -# warnings are cleared only for GCC on Linux -if (NOT (MINGW OR APPLE OR FREEBSD OR OPENBSD OR DRAGONFLY)) - add_compile_options("${WARNINGS_AS_ERRORS_FLAG}") # applies only to targets that follow -endif() - function (monero_private_headers group) source_group("${group}\\Private" FILES @@ -145,3 +140,4 @@ if(PER_BLOCK_CHECKPOINT) endif() add_subdirectory(device) +add_subdirectory(device_trezor) diff --git a/src/blockchain_db/berkeleydb/db_bdb.cpp b/src/blockchain_db/berkeleydb/db_bdb.cpp index 6c79120e8..60a7326f8 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.cpp +++ b/src/blockchain_db/berkeleydb/db_bdb.cpp @@ -714,29 +714,6 @@ bool BlockchainBDB::for_all_outputs(std::function<bool(uint64_t amount, const cr return ret; } -blobdata BlockchainBDB::output_to_blob(const tx_out& output) const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - blobdata b; - if (!t_serializable_object_to_blob(output, b)) - throw1(DB_ERROR("Error serializing output to blob")); - return b; -} - -tx_out BlockchainBDB::output_from_blob(const blobdata& blob) const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - std::stringstream ss; - ss << blob; - binary_archive<false> ba(ss); - tx_out o; - - if (!(::serialization::serialize(ba, o))) - throw1(DB_ERROR("Error deserializing tx output blob")); - - return o; -} - uint64_t BlockchainBDB::get_output_global_index(const uint64_t& amount, const uint64_t& index) { LOG_PRINT_L3("BlockchainBDB::" << __func__); @@ -1655,7 +1632,7 @@ output_data_t BlockchainBDB::get_output_key(const uint64_t& global_index) const return v; } -output_data_t BlockchainBDB::get_output_key(const uint64_t& amount, const uint64_t& index) +output_data_t BlockchainBDB::get_output_key(const uint64_t& amount, const uint64_t& index) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); @@ -1664,7 +1641,7 @@ output_data_t BlockchainBDB::get_output_key(const uint64_t& amount, const uint64 return get_output_key(glob_index); } -tx_out_index BlockchainBDB::get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) +tx_out_index BlockchainBDB::get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); std::vector < uint64_t > offsets; diff --git a/src/blockchain_db/berkeleydb/db_bdb.h b/src/blockchain_db/berkeleydb/db_bdb.h index 76d0a0517..e80adae9e 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.h +++ b/src/blockchain_db/berkeleydb/db_bdb.h @@ -392,24 +392,6 @@ private: virtual void drop_hard_fork_info(); /** - * @brief convert a tx output to a blob for storage - * - * @param output the output to convert - * - * @return the resultant blob - */ - blobdata output_to_blob(const tx_out& output) const; - - /** - * @brief convert a tx output blob to a tx output - * - * @param blob the blob to convert - * - * @return the resultant tx output - */ - tx_out output_from_blob(const blobdata& blob) const; - - /** * @brief get the global index of the index-th output of the given amount * * @param amount the output amount diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp index be0ffeac3..d3eefef6e 100644 --- a/src/blockchain_db/blockchain_db.cpp +++ b/src/blockchain_db/blockchain_db.cpp @@ -170,7 +170,7 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti uint64_t tx_id = add_transaction_data(blk_hash, tx, tx_hash, tx_prunable_hash); - std::vector<uint64_t> amount_output_indices; + std::vector<uint64_t> amount_output_indices(tx.vout.size()); // iterate tx.vout using indices instead of C++11 foreach syntax because // we need the index @@ -183,13 +183,13 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti cryptonote::tx_out vout = tx.vout[i]; rct::key commitment = rct::zeroCommit(vout.amount); vout.amount = 0; - amount_output_indices.push_back(add_output(tx_hash, vout, i, tx.unlock_time, - &commitment)); + amount_output_indices[i] = add_output(tx_hash, vout, i, tx.unlock_time, + &commitment); } else { - amount_output_indices.push_back(add_output(tx_hash, tx.vout[i], i, tx.unlock_time, - tx.version > 1 ? &tx.rct_signatures.outPk[i].mask : NULL)); + amount_output_indices[i] = add_output(tx_hash, tx.vout[i], i, tx.unlock_time, + tx.version > 1 ? &tx.rct_signatures.outPk[i].mask : NULL); } } add_tx_amount_output_indices(tx_id, amount_output_indices); @@ -267,7 +267,10 @@ void BlockchainDB::pop_block(block& blk, std::vector<transaction>& txs) for (const auto& h : boost::adaptors::reverse(blk.tx_hashes)) { - txs.push_back(get_tx(h)); + cryptonote::transaction tx; + if (!get_tx(h, tx) && !get_pruned_tx(h, tx)) + throw DB_ERROR("Failed to get pruned or unpruned transaction from the db"); + txs.push_back(std::move(tx)); remove_transaction(h); } remove_transaction(get_transaction_hash(blk.miner_tx)); @@ -280,7 +283,7 @@ bool BlockchainDB::is_open() const void BlockchainDB::remove_transaction(const crypto::hash& tx_hash) { - transaction tx = get_tx(tx_hash); + transaction tx = get_pruned_tx(tx_hash); for (const txin_v& tx_input : tx.vin) { @@ -325,6 +328,17 @@ bool BlockchainDB::get_tx(const crypto::hash& h, cryptonote::transaction &tx) co return true; } +bool BlockchainDB::get_pruned_tx(const crypto::hash& h, cryptonote::transaction &tx) const +{ + blobdata bd; + if (!get_pruned_tx_blob(h, bd)) + return false; + if (!parse_and_validate_tx_base_from_blob(bd, tx)) + throw DB_ERROR("Failed to parse transaction base from blob retrieved from the db"); + + return true; +} + transaction BlockchainDB::get_tx(const crypto::hash& h) const { transaction tx; @@ -333,6 +347,14 @@ transaction BlockchainDB::get_tx(const crypto::hash& h) const return tx; } +transaction BlockchainDB::get_pruned_tx(const crypto::hash& h) const +{ + transaction tx; + if (!get_pruned_tx(h, tx)) + throw TX_DNE(std::string("pruned tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str()); + return tx; +} + void BlockchainDB::reset_stats() { num_calls = 0; diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 396ae7544..137d5958a 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -30,7 +30,6 @@ #pragma once -#include <list> #include <string> #include <exception> #include <boost/program_options.hpp> @@ -544,7 +543,7 @@ public: /** * @brief An empty constructor. */ - BlockchainDB(): m_open(false) { } + BlockchainDB(): m_hardfork(NULL), m_open(false) { } /** * @brief An empty destructor. @@ -1127,6 +1126,17 @@ public: virtual transaction get_tx(const crypto::hash& h) const; /** + * @brief fetches the transaction base with the given hash + * + * If the transaction does not exist, the subclass should throw TX_DNE. + * + * @param h the hash to look for + * + * @return the transaction with the given hash + */ + virtual transaction get_pruned_tx(const crypto::hash& h) const; + + /** * @brief fetches the transaction with the given hash * * If the transaction does not exist, the subclass should return false. @@ -1138,6 +1148,17 @@ public: virtual bool get_tx(const crypto::hash& h, transaction &tx) const; /** + * @brief fetches the transaction base with the given hash + * + * If the transaction does not exist, the subclass should return false. + * + * @param h the hash to look for + * + * @return true iff the transaction was found + */ + virtual bool get_pruned_tx(const crypto::hash& h, transaction &tx) const; + + /** * @brief fetches the transaction blob with the given hash * * The subclass should return the transaction stored which has the given @@ -1166,6 +1187,21 @@ public: virtual bool get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const = 0; /** + * @brief fetches the prunable transaction blob with the given hash + * + * The subclass should return the prunable transaction stored which has the given + * hash. + * + * If the transaction does not exist, or if we do not have that prunable data, + * the subclass should return false. + * + * @param h the hash to look for + * + * @return true iff the transaction was found and we have its prunable data + */ + virtual bool get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const = 0; + + /** * @brief fetches the prunable transaction hash * * The subclass should return the hash of the prunable transaction data. @@ -1259,7 +1295,7 @@ public: * * @return the requested output data */ - virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index) = 0; + virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index, bool include_commitmemt = true) const = 0; /** * @brief gets an output's tx hash and index @@ -1307,11 +1343,11 @@ public: * get_output_data(const uint64_t& amount, const uint64_t& index) * but for a list of outputs rather than just one. * - * @param amount an output amount + * @param amounts an output amount, or as many as offsets * @param offsets a list of amount-specific output indices * @param outputs return-by-reference a list of outputs' metadata */ - virtual void get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs, bool allow_partial = false) = 0; + virtual void get_output_key(const epee::span<const uint64_t> &amounts, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs, bool allow_partial = false) const = 0; /* * FIXME: Need to check with git blame and ask what this does to @@ -1330,10 +1366,11 @@ public: * If an output cannot be found, the subclass should throw OUTPUT_DNE. * * @param tx_id a transaction ID + * @param n_txes how many txes to get data for, starting with tx_id * * @return a list of amount-specific output indices */ - virtual std::vector<uint64_t> get_tx_amount_output_indices(const uint64_t tx_id) const = 0; + virtual std::vector<std::vector<uint64_t>> get_tx_amount_output_indices(const uint64_t tx_id, size_t n_txes = 1) const = 0; /** * @brief check if a key image is stored as spent @@ -1349,7 +1386,7 @@ public: * * @param details the details of the transaction to add */ - virtual void add_txpool_tx(const transaction &tx, const txpool_tx_meta_t& details) = 0; + virtual void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const txpool_tx_meta_t& details) = 0; /** * @brief update a txpool transaction's metadata @@ -1406,6 +1443,38 @@ public: virtual cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid) const = 0; /** + * @brief prune output data for the given amount + * + * @param amount the amount for which to prune data + */ + virtual void prune_outputs(uint64_t amount) = 0; + + /** + * @brief get the blockchain pruning seed + * @return the blockchain pruning seed + */ + virtual uint32_t get_blockchain_pruning_seed() const = 0; + + /** + * @brief prunes the blockchain + * @param pruning_seed the seed to use, 0 for default (highly recommended) + * @return success iff true + */ + virtual bool prune_blockchain(uint32_t pruning_seed = 0) = 0; + + /** + * @brief prunes recent blockchain changes as needed, iff pruning is enabled + * @return success iff true + */ + virtual bool update_pruning() = 0; + + /** + * @brief checks pruning was done correctly, iff enabled + * @return success iff true + */ + virtual bool check_pruning() = 0; + + /** * @brief runs a function over all txpool transactions * * The subclass should run the passed function for each txpool tx it has diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index bd91f308a..8a1303be9 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -29,14 +29,13 @@ #include <boost/filesystem.hpp> #include <boost/format.hpp> -#include <boost/current_function.hpp> #include <memory> // std::unique_ptr #include <cstring> // memcpy -#include <random> #include "string_tools.h" #include "file_io_utils.h" #include "common/util.h" +#include "common/pruning.h" #include "cryptonote_basic/cryptonote_format_utils.h" #include "crypto/crypto.h" #include "profile_tools.h" @@ -85,6 +84,10 @@ inline void throw1(const T &e) #define MDB_val_set(var, val) MDB_val var = {sizeof(val), (void *)&val} +#define MDB_val_sized(var, val) MDB_val var = {val.size(), (void *)val.data()} + +#define MDB_val_str(var, val) MDB_val var = {strlen(val) + 1, (void *)val} + template<typename T> struct MDB_val_copy: public MDB_val { @@ -128,14 +131,20 @@ private: std::unique_ptr<char[]> data; }; -int compare_uint64(const MDB_val *a, const MDB_val *b) +} + +namespace cryptonote { - const uint64_t va = *(const uint64_t *)a->mv_data; - const uint64_t vb = *(const uint64_t *)b->mv_data; + +int BlockchainLMDB::compare_uint64(const MDB_val *a, const MDB_val *b) +{ + uint64_t va, vb; + memcpy(&va, a->mv_data, sizeof(va)); + memcpy(&vb, b->mv_data, sizeof(vb)); return (va < vb) ? -1 : va > vb; } -int compare_hash32(const MDB_val *a, const MDB_val *b) +int BlockchainLMDB::compare_hash32(const MDB_val *a, const MDB_val *b) { uint32_t *va = (uint32_t*) a->mv_data; uint32_t *vb = (uint32_t*) b->mv_data; @@ -149,13 +158,18 @@ int compare_hash32(const MDB_val *a, const MDB_val *b) return 0; } -int compare_string(const MDB_val *a, const MDB_val *b) +int BlockchainLMDB::compare_string(const MDB_val *a, const MDB_val *b) { const char *va = (const char*) a->mv_data; const char *vb = (const char*) b->mv_data; return strcmp(va, vb); } +} + +namespace +{ + /* DB schema: * * Table Key Data @@ -167,6 +181,7 @@ int compare_string(const MDB_val *a, const MDB_val *b) * txs_pruned txn ID pruned txn blob * txs_prunable txn ID prunable txn blob * txs_prunable_hash txn ID prunable txn hash + * txs_prunable_tip txn ID height * tx_indices txn hash {txn ID, metadata} * tx_outputs txn ID [txn amount output indices] * @@ -194,6 +209,7 @@ const char* const LMDB_TXS = "txs"; const char* const LMDB_TXS_PRUNED = "txs_pruned"; const char* const LMDB_TXS_PRUNABLE = "txs_prunable"; const char* const LMDB_TXS_PRUNABLE_HASH = "txs_prunable_hash"; +const char* const LMDB_TXS_PRUNABLE_TIP = "txs_prunable_tip"; const char* const LMDB_TX_INDICES = "tx_indices"; const char* const LMDB_TX_OUTPUTS = "tx_outputs"; @@ -277,11 +293,6 @@ typedef struct blk_height { uint64_t bh_height; } blk_height; -typedef struct txindex { - crypto::hash key; - tx_data_t data; -} txindex; - typedef struct pre_rct_outkey { uint64_t amount_index; uint64_t output_id; @@ -454,6 +465,12 @@ inline int lmdb_txn_renew(MDB_txn *txn) return res; } +inline void BlockchainLMDB::check_open() const +{ + if (!m_open) + throw0(DB_ERROR("DB operation attempted on a not-open DB instance")); +} + void BlockchainLMDB::do_resize(uint64_t increase_size) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -541,18 +558,18 @@ bool BlockchainLMDB::need_resize(uint64_t threshold_size) const // additional size needed. uint64_t size_used = mst.ms_psize * mei.me_last_pgno; - LOG_PRINT_L1("DB map size: " << mei.me_mapsize); - LOG_PRINT_L1("Space used: " << size_used); - LOG_PRINT_L1("Space remaining: " << mei.me_mapsize - size_used); - LOG_PRINT_L1("Size threshold: " << threshold_size); + MDEBUG("DB map size: " << mei.me_mapsize); + MDEBUG("Space used: " << size_used); + MDEBUG("Space remaining: " << mei.me_mapsize - size_used); + MDEBUG("Size threshold: " << threshold_size); float resize_percent = RESIZE_PERCENT; - LOG_PRINT_L1(boost::format("Percent used: %.04f Percent threshold: %.04f") % ((double)size_used/mei.me_mapsize) % resize_percent); + MDEBUG(boost::format("Percent used: %.04f Percent threshold: %.04f") % ((double)size_used/mei.me_mapsize) % resize_percent); if (threshold_size > 0) { if (mei.me_mapsize - size_used < threshold_size) { - LOG_PRINT_L1("Threshold met (size-based)"); + MINFO("Threshold met (size-based)"); return true; } else @@ -561,7 +578,7 @@ bool BlockchainLMDB::need_resize(uint64_t threshold_size) const if ((double)size_used / mei.me_mapsize > resize_percent) { - LOG_PRINT_L1("Threshold met (percent-based)"); + MINFO("Threshold met (percent-based)"); return true; } return false; @@ -573,7 +590,7 @@ bool BlockchainLMDB::need_resize(uint64_t threshold_size) const void BlockchainLMDB::check_and_resize_for_batch(uint64_t batch_num_blocks, uint64_t batch_bytes) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); - LOG_PRINT_L1("[" << __func__ << "] " << "checking DB size"); + MTRACE("[" << __func__ << "] " << "checking DB size"); const uint64_t min_increase_size = 512 * (1 << 20); uint64_t threshold_size = 0; uint64_t increase_size = 0; @@ -714,7 +731,8 @@ void BlockchainLMDB::add_block(const block& blk, size_t block_weight, const diff CURSOR(block_info) // this call to mdb_cursor_put will change height() - MDB_val_copy<blobdata> blob(block_to_blob(blk)); + cryptonote::blobdata block_blob(block_to_blob(blk)); + MDB_val_sized(blob, block_blob); result = mdb_cursor_put(m_cur_blocks, &key, &blob, MDB_APPEND); if (result) throw0(DB_ERROR(lmdb_error("Failed to add block blob to db transaction: ", result).c_str())); @@ -802,6 +820,7 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons CURSOR(txs_pruned) CURSOR(txs_prunable) CURSOR(txs_prunable_hash) + CURSOR(txs_prunable_tip) CURSOR(tx_indices) MDB_val_set(val_tx_id, tx_id); @@ -828,7 +847,7 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons throw0(DB_ERROR(lmdb_error("Failed to add tx data to db transaction: ", result).c_str())); cryptonote::blobdata blob = tx_to_blob(tx); - MDB_val_copy<blobdata> blobval(blob); + MDB_val_sized(blobval, blob); std::stringstream ss; binary_archive<true> ba(ss); @@ -836,7 +855,7 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons if (!r) throw0(DB_ERROR("Failed to serialize pruned tx")); std::string pruned = ss.str(); - MDB_val_copy<blobdata> pruned_blob(pruned); + MDB_val_sized(pruned_blob, pruned); result = mdb_cursor_put(m_cur_txs_pruned, &val_tx_id, &pruned_blob, MDB_APPEND); if (result) throw0(DB_ERROR(lmdb_error("Failed to add pruned tx blob to db transaction: ", result).c_str())); @@ -844,11 +863,19 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons if (pruned.size() > blob.size()) throw0(DB_ERROR("pruned tx size is larger than tx size")); cryptonote::blobdata prunable(blob.data() + pruned.size(), blob.size() - pruned.size()); - MDB_val_copy<blobdata> prunable_blob(prunable); + MDB_val_sized(prunable_blob, prunable); result = mdb_cursor_put(m_cur_txs_prunable, &val_tx_id, &prunable_blob, MDB_APPEND); if (result) throw0(DB_ERROR(lmdb_error("Failed to add prunable tx blob to db transaction: ", result).c_str())); + if (get_blockchain_pruning_seed()) + { + MDB_val_set(val_height, m_height); + result = mdb_cursor_put(m_cur_txs_prunable_tip, &val_tx_id, &val_height, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to add prunable tx id to db transaction: ", result).c_str())); + } + if (tx.version > 1) { MDB_val_set(val_prunable_hash, tx_prunable_hash); @@ -874,6 +901,7 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const CURSOR(txs_pruned) CURSOR(txs_prunable) CURSOR(txs_prunable_hash) + CURSOR(txs_prunable_tip) CURSOR(tx_outputs) MDB_val_set(val_h, tx_hash); @@ -889,11 +917,25 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const if (result) throw1(DB_ERROR(lmdb_error("Failed to add removal of pruned tx to db transaction: ", result).c_str())); - if ((result = mdb_cursor_get(m_cur_txs_prunable, &val_tx_id, NULL, MDB_SET))) + result = mdb_cursor_get(m_cur_txs_prunable, &val_tx_id, NULL, MDB_SET); + if (result == 0) + { + result = mdb_cursor_del(m_cur_txs_prunable, 0); + if (result) + throw1(DB_ERROR(lmdb_error("Failed to add removal of prunable tx to db transaction: ", result).c_str())); + } + else if (result != MDB_NOTFOUND) throw1(DB_ERROR(lmdb_error("Failed to locate prunable tx for removal: ", result).c_str())); - result = mdb_cursor_del(m_cur_txs_prunable, 0); - if (result) - throw1(DB_ERROR(lmdb_error("Failed to add removal of prunable tx to db transaction: ", result).c_str())); + + result = mdb_cursor_get(m_cur_txs_prunable_tip, &val_tx_id, NULL, MDB_SET); + if (result && result != MDB_NOTFOUND) + throw1(DB_ERROR(lmdb_error("Failed to locate tx id for removal: ", result).c_str())); + if (result == 0) + { + result = mdb_cursor_del(m_cur_txs_prunable_tip, 0); + if (result) + throw1(DB_ERROR(lmdb_error("Error adding removal of tx id to db transaction", result).c_str())); + } if (tx.version > 1) { @@ -1016,7 +1058,8 @@ void BlockchainLMDB::remove_tx_outputs(const uint64_t tx_id, const transaction& { LOG_PRINT_L3("BlockchainLMDB::" << __func__); - std::vector<uint64_t> amount_output_indices = get_tx_amount_output_indices(tx_id); + std::vector<std::vector<uint64_t>> amount_output_indices_set = get_tx_amount_output_indices(tx_id, 1); + const std::vector<uint64_t> &amount_output_indices = amount_output_indices_set.front(); if (amount_output_indices.empty()) { @@ -1072,6 +1115,60 @@ void BlockchainLMDB::remove_output(const uint64_t amount, const uint64_t& out_in throw0(DB_ERROR(lmdb_error(std::string("Error deleting amount for output index ").append(boost::lexical_cast<std::string>(out_index).append(": ")).c_str(), result).c_str())); } +void BlockchainLMDB::prune_outputs(uint64_t amount) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + mdb_txn_cursors *m_cursors = &m_wcursors; + CURSOR(output_amounts); + CURSOR(output_txs); + + MINFO("Pruning outputs for amount " << amount); + + MDB_val v; + MDB_val_set(k, amount); + int result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + return; + if (result) + throw0(DB_ERROR(lmdb_error("Error looking up outputs: ", result).c_str())); + + // gather output ids + mdb_size_t num_elems; + mdb_cursor_count(m_cur_output_amounts, &num_elems); + MINFO(num_elems << " outputs found"); + std::vector<uint64_t> output_ids; + output_ids.reserve(num_elems); + while (1) + { + const pre_rct_outkey *okp = (const pre_rct_outkey *)v.mv_data; + output_ids.push_back(okp->output_id); + MDEBUG("output id " << okp->output_id); + result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_NEXT_DUP); + if (result == MDB_NOTFOUND) + break; + if (result) + throw0(DB_ERROR(lmdb_error("Error counting outputs: ", result).c_str())); + } + if (output_ids.size() != num_elems) + throw0(DB_ERROR("Unexpected number of outputs")); + + result = mdb_cursor_del(m_cur_output_amounts, MDB_NODUPDATA); + if (result) + throw0(DB_ERROR(lmdb_error("Error deleting outputs: ", result).c_str())); + + for (uint64_t output_id: output_ids) + { + MDB_val_set(v, output_id); + result = mdb_cursor_get(m_cur_output_txs, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); + if (result) + throw0(DB_ERROR(lmdb_error("Error looking up output: ", result).c_str())); + result = mdb_cursor_del(m_cur_output_txs, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Error deleting output: ", result).c_str())); + } +} + void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -1109,36 +1206,6 @@ void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image) } } -blobdata BlockchainLMDB::output_to_blob(const tx_out& output) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - blobdata b; - if (!t_serializable_object_to_blob(output, b)) - throw1(DB_ERROR("Error serializing output to blob")); - return b; -} - -tx_out BlockchainLMDB::output_from_blob(const blobdata& blob) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - std::stringstream ss; - ss << blob; - binary_archive<false> ba(ss); - tx_out o; - - if (!(::serialization::serialize(ba, o))) - throw1(DB_ERROR("Error deserializing tx output blob")); - - return o; -} - -void BlockchainLMDB::check_open() const -{ -// LOG_PRINT_L3("BlockchainLMDB::" << __func__); - if (!m_open) - throw0(DB_ERROR("DB operation attempted on a not-open DB instance")); -} - BlockchainLMDB::~BlockchainLMDB() { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -1208,7 +1275,7 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) if (is_hdd_result) { if (is_hdd_result.value()) - MCLOG_RED(el::Level::Warning, "global", "The blockchain is on a rotating drive: this will be very slow, use a SSD if possible"); + MCLOG_RED(el::Level::Warning, "global", "The blockchain is on a rotating drive: this will be very slow, use an SSD if possible"); } m_folder = filename; @@ -1274,6 +1341,7 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) // open necessary databases, and set properties as needed // uses macros to avoid having to change things too many places + // also change blockchain_prune.cpp to match lmdb_db_open(txn, LMDB_BLOCKS, MDB_INTEGERKEY | MDB_CREATE, m_blocks, "Failed to open db handle for m_blocks"); lmdb_db_open(txn, LMDB_BLOCK_INFO, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for m_block_info"); @@ -1282,7 +1350,9 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) lmdb_db_open(txn, LMDB_TXS, MDB_INTEGERKEY | MDB_CREATE, m_txs, "Failed to open db handle for m_txs"); lmdb_db_open(txn, LMDB_TXS_PRUNED, MDB_INTEGERKEY | MDB_CREATE, m_txs_pruned, "Failed to open db handle for m_txs_pruned"); lmdb_db_open(txn, LMDB_TXS_PRUNABLE, MDB_INTEGERKEY | MDB_CREATE, m_txs_prunable, "Failed to open db handle for m_txs_prunable"); - lmdb_db_open(txn, LMDB_TXS_PRUNABLE_HASH, MDB_INTEGERKEY | MDB_CREATE, m_txs_prunable_hash, "Failed to open db handle for m_txs_prunable_hash"); + lmdb_db_open(txn, LMDB_TXS_PRUNABLE_HASH, MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_txs_prunable_hash, "Failed to open db handle for m_txs_prunable_hash"); + if (!(mdb_flags & MDB_RDONLY)) + lmdb_db_open(txn, LMDB_TXS_PRUNABLE_TIP, MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_txs_prunable_tip, "Failed to open db handle for m_txs_prunable_tip"); lmdb_db_open(txn, LMDB_TX_INDICES, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_tx_indices, "Failed to open db handle for m_tx_indices"); lmdb_db_open(txn, LMDB_TX_OUTPUTS, MDB_INTEGERKEY | MDB_CREATE, m_tx_outputs, "Failed to open db handle for m_tx_outputs"); @@ -1310,6 +1380,10 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) mdb_set_dupsort(txn, m_output_amounts, compare_uint64); mdb_set_dupsort(txn, m_output_txs, compare_uint64); mdb_set_dupsort(txn, m_block_info, compare_uint64); + if (!(mdb_flags & MDB_RDONLY)) + mdb_set_dupsort(txn, m_txs_prunable_tip, compare_uint64); + mdb_set_compare(txn, m_txs_prunable, compare_uint64); + mdb_set_dupsort(txn, m_txs_prunable_hash, compare_uint64); mdb_set_compare(txn, m_txpool_meta, compare_hash32); mdb_set_compare(txn, m_txpool_blob, compare_hash32); @@ -1331,7 +1405,7 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) bool compatible = true; - MDB_val_copy<const char*> k("version"); + MDB_val_str(k, "version"); MDB_val v; auto get_result = mdb_get(txn, m_properties, &k, &v); if(get_result == MDB_SUCCESS) @@ -1345,6 +1419,15 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) #if VERSION > 0 else if (db_version < VERSION) { + if (mdb_flags & MDB_RDONLY) + { + txn.abort(); + mdb_env_close(m_env); + m_open = false; + MFATAL("Existing lmdb database needs to be converted, which cannot be done on a read-only database."); + MFATAL("Please run monerod once to convert the database."); + return; + } // Note that there was a schema change within version 0 as well. // See commit e5d2680094ee15889934fe28901e4e133cda56f2 2015/07/10 // We don't handle the old format previous to that commit. @@ -1379,7 +1462,7 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) // only write version on an empty DB if (m_height == 0) { - MDB_val_copy<const char*> k("version"); + MDB_val_str(k, "version"); MDB_val_copy<uint32_t> v(VERSION); auto put_result = mdb_put(txn, m_properties, &k, &v, 0); if (put_result != MDB_SUCCESS) @@ -1459,6 +1542,8 @@ void BlockchainLMDB::reset() throw0(DB_ERROR(lmdb_error("Failed to drop m_txs_prunable: ", result).c_str())); if (auto result = mdb_drop(txn, m_txs_prunable_hash, 0)) throw0(DB_ERROR(lmdb_error("Failed to drop m_txs_prunable_hash: ", result).c_str())); + if (auto result = mdb_drop(txn, m_txs_prunable_tip, 0)) + throw0(DB_ERROR(lmdb_error("Failed to drop m_txs_prunable_tip: ", result).c_str())); if (auto result = mdb_drop(txn, m_tx_indices, 0)) throw0(DB_ERROR(lmdb_error("Failed to drop m_tx_indices: ", result).c_str())); if (auto result = mdb_drop(txn, m_tx_outputs, 0)) @@ -1476,7 +1561,7 @@ void BlockchainLMDB::reset() throw0(DB_ERROR(lmdb_error("Failed to drop m_properties: ", result).c_str())); // init with current version - MDB_val_copy<const char*> k("version"); + MDB_val_str(k, "version"); MDB_val_copy<uint32_t> v(VERSION); if (auto result = mdb_put(txn, m_properties, &k, &v, 0)) throw0(DB_ERROR(lmdb_error("Failed to write version to database: ", result).c_str())); @@ -1591,7 +1676,7 @@ void BlockchainLMDB::unlock() auto_txn.commit(); \ } while(0) -void BlockchainLMDB::add_txpool_tx(const transaction &tx, const txpool_tx_meta_t &meta) +void BlockchainLMDB::add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const txpool_tx_meta_t &meta) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -1600,8 +1685,6 @@ void BlockchainLMDB::add_txpool_tx(const transaction &tx, const txpool_tx_meta_t CURSOR(txpool_meta) CURSOR(txpool_blob) - const crypto::hash txid = get_transaction_hash(tx); - MDB_val k = {sizeof(txid), (void *)&txid}; MDB_val v = {sizeof(meta), (void *)&meta}; if (auto result = mdb_cursor_put(m_cur_txpool_meta, &k, &v, MDB_NODUPDATA)) { @@ -1610,7 +1693,7 @@ void BlockchainLMDB::add_txpool_tx(const transaction &tx, const txpool_tx_meta_t else throw1(DB_ERROR(lmdb_error("Error adding txpool tx metadata to db transaction: ", result).c_str())); } - MDB_val_copy<cryptonote::blobdata> blob_val(tx_to_blob(tx)); + MDB_val_sized(blob_val, blob); if (auto result = mdb_cursor_put(m_cur_txpool_blob, &k, &blob_val, MDB_NODUPDATA)) { if (result == MDB_KEYEXIST) throw1(DB_ERROR("Attempting to add txpool tx blob that's already in the db")); @@ -1786,6 +1869,290 @@ cryptonote::blobdata BlockchainLMDB::get_txpool_tx_blob(const crypto::hash& txid return bd; } +uint32_t BlockchainLMDB::get_blockchain_pruning_seed() const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + RCURSOR(properties) + MDB_val_str(k, "pruning_seed"); + MDB_val v; + int result = mdb_cursor_get(m_cur_properties, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + return 0; + if (result) + throw0(DB_ERROR(lmdb_error("Failed to retrieve pruning seed: ", result).c_str())); + if (v.mv_size != sizeof(uint32_t)) + throw0(DB_ERROR("Failed to retrieve or create pruning seed: unexpected value size")); + uint32_t pruning_seed; + memcpy(&pruning_seed, v.mv_data, sizeof(pruning_seed)); + TXN_POSTFIX_RDONLY(); + return pruning_seed; +} + +static bool is_v1_tx(MDB_cursor *c_txs_pruned, MDB_val *tx_id) +{ + MDB_val v; + int ret = mdb_cursor_get(c_txs_pruned, tx_id, &v, MDB_SET); + if (ret) + throw0(DB_ERROR(lmdb_error("Failed to find transaction pruned data: ", ret).c_str())); + if (v.mv_size == 0) + throw0(DB_ERROR("Invalid transaction pruned data")); + return cryptonote::is_v1_tx(cryptonote::blobdata_ref{(const char*)v.mv_data, v.mv_size}); +} + +enum { prune_mode_prune, prune_mode_update, prune_mode_check }; + +bool BlockchainLMDB::prune_worker(int mode, uint32_t pruning_seed) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + const uint32_t log_stripes = tools::get_pruning_log_stripes(pruning_seed); + if (log_stripes && log_stripes != CRYPTONOTE_PRUNING_LOG_STRIPES) + throw0(DB_ERROR("Pruning seed not in range")); + pruning_seed = tools::get_pruning_stripe(pruning_seed);; + if (pruning_seed > (1ul << CRYPTONOTE_PRUNING_LOG_STRIPES)) + throw0(DB_ERROR("Pruning seed not in range")); + check_open(); + + TIME_MEASURE_START(t); + + size_t n_total_records = 0, n_prunable_records = 0, n_pruned_records = 0; + uint64_t n_bytes = 0; + + mdb_txn_safe txn; + auto result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + + MDB_stat db_stats; + if ((result = mdb_stat(txn, m_txs_prunable, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_txs_prunable: ", result).c_str())); + const size_t pages0 = db_stats.ms_branch_pages + db_stats.ms_leaf_pages + db_stats.ms_overflow_pages; + + MDB_val_str(k, "pruning_seed"); + MDB_val v; + result = mdb_get(txn, m_properties, &k, &v); + bool prune_tip_table = false; + if (result == MDB_NOTFOUND) + { + // not pruned yet + if (mode != prune_mode_prune) + { + txn.abort(); + TIME_MEASURE_FINISH(t); + MDEBUG("Pruning not enabled, nothing to do"); + return true; + } + if (pruning_seed == 0) + pruning_seed = tools::get_random_stripe(); + pruning_seed = tools::make_pruning_seed(pruning_seed, CRYPTONOTE_PRUNING_LOG_STRIPES); + v.mv_data = &pruning_seed; + v.mv_size = sizeof(pruning_seed); + result = mdb_put(txn, m_properties, &k, &v, 0); + if (result) + throw0(DB_ERROR("Failed to save pruning seed")); + prune_tip_table = false; + } + else if (result == 0) + { + // pruned already + if (v.mv_size != sizeof(uint32_t)) + throw0(DB_ERROR("Failed to retrieve or create pruning seed: unexpected value size")); + const uint32_t data = *(const uint32_t*)v.mv_data; + if (pruning_seed == 0) + pruning_seed = tools::get_pruning_stripe(data); + if (tools::get_pruning_stripe(data) != pruning_seed) + throw0(DB_ERROR("Blockchain already pruned with different seed")); + if (tools::get_pruning_log_stripes(data) != CRYPTONOTE_PRUNING_LOG_STRIPES) + throw0(DB_ERROR("Blockchain already pruned with different base")); + pruning_seed = tools::make_pruning_seed(pruning_seed, CRYPTONOTE_PRUNING_LOG_STRIPES); + prune_tip_table = (mode == prune_mode_update); + } + else + { + throw0(DB_ERROR(lmdb_error("Failed to retrieve or create pruning seed: ", result).c_str())); + } + + if (mode == prune_mode_check) + MINFO("Checking blockchain pruning..."); + else + MINFO("Pruning blockchain..."); + + MDB_cursor *c_txs_pruned, *c_txs_prunable, *c_txs_prunable_tip; + result = mdb_cursor_open(txn, m_txs_pruned, &c_txs_pruned); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_pruned: ", result).c_str())); + result = mdb_cursor_open(txn, m_txs_prunable, &c_txs_prunable); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_prunable: ", result).c_str())); + result = mdb_cursor_open(txn, m_txs_prunable_tip, &c_txs_prunable_tip); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_prunable_tip: ", result).c_str())); + const uint64_t blockchain_height = height(); + + if (prune_tip_table) + { + MDB_cursor_op op = MDB_FIRST; + while (1) + { + int ret = mdb_cursor_get(c_txs_prunable_tip, &k, &v, op); + op = MDB_NEXT; + if (ret == MDB_NOTFOUND) + break; + if (ret) + throw0(DB_ERROR(lmdb_error("Failed to enumerate transactions: ", ret).c_str())); + + uint64_t block_height; + memcpy(&block_height, v.mv_data, sizeof(block_height)); + if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS < blockchain_height) + { + ++n_total_records; + if (!tools::has_unpruned_block(block_height, blockchain_height, pruning_seed) && !is_v1_tx(c_txs_pruned, &k)) + { + ++n_prunable_records; + result = mdb_cursor_get(c_txs_prunable, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + MWARNING("Already pruned at height " << block_height << "/" << blockchain_height); + else if (result) + throw0(DB_ERROR(lmdb_error("Failed to find transaction prunable data: ", result).c_str())); + else + { + MDEBUG("Pruning at height " << block_height << "/" << blockchain_height); + ++n_pruned_records; + n_bytes += k.mv_size + v.mv_size; + result = mdb_cursor_del(c_txs_prunable, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete transaction prunable data: ", result).c_str())); + } + } + result = mdb_cursor_del(c_txs_prunable_tip, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete transaction tip data: ", result).c_str())); + } + } + } + else + { + MDB_cursor *c_tx_indices; + result = mdb_cursor_open(txn, m_tx_indices, &c_tx_indices); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for tx_indices: ", result).c_str())); + MDB_cursor_op op = MDB_FIRST; + while (1) + { + int ret = mdb_cursor_get(c_tx_indices, &k, &v, op); + op = MDB_NEXT; + if (ret == MDB_NOTFOUND) + break; + if (ret) + throw0(DB_ERROR(lmdb_error("Failed to enumerate transactions: ", ret).c_str())); + + ++n_total_records; + //const txindex *ti = (const txindex *)v.mv_data; + txindex ti; + memcpy(&ti, v.mv_data, sizeof(ti)); + const uint64_t block_height = ti.data.block_id; + if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS >= blockchain_height) + { + MDB_val_set(kp, ti.data.tx_id); + MDB_val_set(vp, block_height); + if (mode == prune_mode_check) + { + result = mdb_cursor_get(c_txs_prunable_tip, &kp, &vp, MDB_SET); + if (result && result != MDB_NOTFOUND) + throw0(DB_ERROR(lmdb_error("Error looking for transaction prunable data: ", result).c_str())); + if (result == MDB_NOTFOUND) + MERROR("Transaction not found in prunable tip table for height " << block_height << "/" << blockchain_height << + ", seed " << epee::string_tools::to_string_hex(pruning_seed)); + } + else + { + result = mdb_cursor_put(c_txs_prunable_tip, &kp, &vp, 0); + if (result && result != MDB_NOTFOUND) + throw0(DB_ERROR(lmdb_error("Error looking for transaction prunable data: ", result).c_str())); + } + } + MDB_val_set(kp, ti.data.tx_id); + if (!tools::has_unpruned_block(block_height, blockchain_height, pruning_seed) && !is_v1_tx(c_txs_pruned, &kp)) + { + result = mdb_cursor_get(c_txs_prunable, &kp, &v, MDB_SET); + if (result && result != MDB_NOTFOUND) + throw0(DB_ERROR(lmdb_error("Error looking for transaction prunable data: ", result).c_str())); + if (mode == prune_mode_check) + { + if (result != MDB_NOTFOUND) + MERROR("Prunable data found for pruned height " << block_height << "/" << blockchain_height << + ", seed " << epee::string_tools::to_string_hex(pruning_seed)); + } + else + { + ++n_prunable_records; + if (result == MDB_NOTFOUND) + MWARNING("Already pruned at height " << block_height << "/" << blockchain_height); + else + { + MDEBUG("Pruning at height " << block_height << "/" << blockchain_height); + ++n_pruned_records; + n_bytes += kp.mv_size + v.mv_size; + result = mdb_cursor_del(c_txs_prunable, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete transaction prunable data: ", result).c_str())); + } + } + } + else + { + if (mode == prune_mode_check) + { + MDB_val_set(kp, ti.data.tx_id); + result = mdb_cursor_get(c_txs_prunable, &kp, &v, MDB_SET); + if (result && result != MDB_NOTFOUND) + throw0(DB_ERROR(lmdb_error("Error looking for transaction prunable data: ", result).c_str())); + if (result == MDB_NOTFOUND) + MERROR("Prunable data not found for unpruned height " << block_height << "/" << blockchain_height << + ", seed " << epee::string_tools::to_string_hex(pruning_seed)); + } + } + } + mdb_cursor_close(c_tx_indices); + } + + if ((result = mdb_stat(txn, m_txs_prunable, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_txs_prunable: ", result).c_str())); + const size_t pages1 = db_stats.ms_branch_pages + db_stats.ms_leaf_pages + db_stats.ms_overflow_pages; + const size_t db_bytes = (pages0 - pages1) * db_stats.ms_psize; + + mdb_cursor_close(c_txs_prunable_tip); + mdb_cursor_close(c_txs_prunable); + mdb_cursor_close(c_txs_pruned); + + txn.commit(); + + TIME_MEASURE_FINISH(t); + + MINFO((mode == prune_mode_check ? "Checked" : "Pruned") << " blockchain in " << + t << " ms: " << (n_bytes/1024.0f/1024.0f) << " MB (" << db_bytes/1024.0f/1024.0f << " MB) pruned in " << + n_pruned_records << " records (" << pages0 - pages1 << "/" << pages0 << " " << db_stats.ms_psize << " byte pages), " << + n_prunable_records << "/" << n_total_records << " pruned records"); + return true; +} + +bool BlockchainLMDB::prune_blockchain(uint32_t pruning_seed) +{ + return prune_worker(prune_mode_prune, pruning_seed); +} + +bool BlockchainLMDB::update_pruning() +{ + return prune_worker(prune_mode_update, 0); +} + +bool BlockchainLMDB::check_pruning() +{ + return prune_worker(prune_mode_check, 0); +} + bool BlockchainLMDB::for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)> f, bool include_blob, bool include_unrelayed_txes) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -1980,22 +2347,36 @@ std::vector<uint64_t> BlockchainLMDB::get_block_cumulative_rct_outputs(const std MDB_val v; uint64_t prev_height = heights[0]; + uint64_t range_begin = 0, range_end = 0; for (uint64_t height: heights) { - if (height == prev_height + 1) + if (height >= range_begin && height < range_end) { - MDB_val k2; - result = mdb_cursor_get(m_cur_block_info, &k2, &v, MDB_NEXT); + // nohting to do } else { - v.mv_size = sizeof(uint64_t); - v.mv_data = (void*)&height; - result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); + if (height == prev_height + 1) + { + MDB_val k2; + result = mdb_cursor_get(m_cur_block_info, &k2, &v, MDB_NEXT_MULTIPLE); + range_begin = ((const mdb_block_info*)v.mv_data)->bi_height; + range_end = range_begin + v.mv_size / sizeof(mdb_block_info); // whole records please + if (height < range_begin || height >= range_end) + throw0(DB_ERROR(("Height " + std::to_string(height) + " not included in multuple record range: " + std::to_string(range_begin) + "-" + std::to_string(range_end)).c_str())); + } + else + { + v.mv_size = sizeof(uint64_t); + v.mv_data = (void*)&height; + result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); + range_begin = height; + range_end = range_begin + 1; + } + if (result) + throw0(DB_ERROR(lmdb_error("Error attempting to retrieve rct distribution from the db: ", result).c_str())); } - if (result) - throw0(DB_ERROR(lmdb_error("Error attempting to retrieve rct distribution from the db: ", result).c_str())); - const mdb_block_info *bi = (const mdb_block_info *)v.mv_data; + const mdb_block_info *bi = ((const mdb_block_info *)v.mv_data) + (height - range_begin); res.push_back(bi->bi_cum_rct); prev_height = height; } @@ -2205,11 +2586,19 @@ uint64_t BlockchainLMDB::num_outputs() const TXN_PREFIX_RDONLY(); int result; - // get current height - MDB_stat db_stats; - if ((result = mdb_stat(m_txn, m_output_txs, &db_stats))) + RCURSOR(output_txs) + + uint64_t num = 0; + MDB_val k, v; + result = mdb_cursor_get(m_cur_output_txs, &k, &v, MDB_LAST); + if (result == MDB_NOTFOUND) + num = 0; + else if (result == 0) + num = 1 + ((const outtx*)v.mv_data)->output_id; + else throw0(DB_ERROR(lmdb_error("Failed to query m_output_txs: ", result).c_str())); - return db_stats.ms_entries; + + return num; } bool BlockchainLMDB::tx_exists(const crypto::hash& h) const @@ -2365,6 +2754,36 @@ bool BlockchainLMDB::get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobd return true; } +bool BlockchainLMDB::get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &bd) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + RCURSOR(tx_indices); + RCURSOR(txs_prunable); + + MDB_val_set(v, h); + MDB_val result; + auto get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); + if (get_result == 0) + { + const txindex *tip = (const txindex *)v.mv_data; + MDB_val_set(val_tx_id, tip->data.tx_id); + get_result = mdb_cursor_get(m_cur_txs_prunable, &val_tx_id, &result, MDB_SET); + } + if (get_result == MDB_NOTFOUND) + return false; + else if (get_result) + throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx from hash", get_result).c_str())); + + bd.assign(reinterpret_cast<char*>(result.mv_data), result.mv_size); + + TXN_POSTFIX_RDONLY(); + + return true; +} + bool BlockchainLMDB::get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -2473,7 +2892,7 @@ uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) const return num_elems; } -output_data_t BlockchainLMDB::get_output_key(const uint64_t& amount, const uint64_t& index) +output_data_t BlockchainLMDB::get_output_key(const uint64_t& amount, const uint64_t& index, bool include_commitmemt) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -2485,7 +2904,8 @@ output_data_t BlockchainLMDB::get_output_key(const uint64_t& amount, const uint6 MDB_val_set(v, index); auto get_result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("Attempting to get output pubkey by index, but key does not exist")); + throw1(OUTPUT_DNE(std::string("Attempting to get output pubkey by index, but key does not exist: amount " + + std::to_string(amount) + ", index " + std::to_string(index)).c_str())); else if (get_result) throw0(DB_ERROR("Error attempting to retrieve an output pubkey from the db")); @@ -2499,7 +2919,8 @@ output_data_t BlockchainLMDB::get_output_key(const uint64_t& amount, const uint6 { const pre_rct_outkey *okp = (const pre_rct_outkey *)v.mv_data; memcpy(&ret, &okp->data, sizeof(pre_rct_output_data_t));; - ret.commitment = rct::zeroCommit(amount); + if (include_commitmemt) + ret.commitment = rct::zeroCommit(amount); } TXN_POSTFIX_RDONLY(); return ret; @@ -2541,7 +2962,7 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con return indices[0]; } -std::vector<uint64_t> BlockchainLMDB::get_tx_amount_output_indices(const uint64_t tx_id) const +std::vector<std::vector<uint64_t>> BlockchainLMDB::get_tx_amount_output_indices(uint64_t tx_id, size_t n_txes) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -2550,35 +2971,40 @@ std::vector<uint64_t> BlockchainLMDB::get_tx_amount_output_indices(const uint64_ TXN_PREFIX_RDONLY(); RCURSOR(tx_outputs); - int result = 0; MDB_val_set(k_tx_id, tx_id); MDB_val v; - std::vector<uint64_t> amount_output_indices; + std::vector<std::vector<uint64_t>> amount_output_indices_set; + amount_output_indices_set.reserve(n_txes); - result = mdb_cursor_get(m_cur_tx_outputs, &k_tx_id, &v, MDB_SET); - if (result == MDB_NOTFOUND) - LOG_PRINT_L0("WARNING: Unexpected: tx has no amount indices stored in " - "tx_outputs, but it should have an empty entry even if it's a tx without " - "outputs"); - else if (result) - throw0(DB_ERROR(lmdb_error("DB error attempting to get data for tx_outputs[tx_index]", result).c_str())); + MDB_cursor_op op = MDB_SET; + while (n_txes-- > 0) + { + int result = mdb_cursor_get(m_cur_tx_outputs, &k_tx_id, &v, op); + if (result == MDB_NOTFOUND) + LOG_PRINT_L0("WARNING: Unexpected: tx has no amount indices stored in " + "tx_outputs, but it should have an empty entry even if it's a tx without " + "outputs"); + else if (result) + throw0(DB_ERROR(lmdb_error("DB error attempting to get data for tx_outputs[tx_index]", result).c_str())); - const uint64_t* indices = (const uint64_t*)v.mv_data; - int num_outputs = v.mv_size / sizeof(uint64_t); + op = MDB_NEXT; - amount_output_indices.reserve(num_outputs); - for (int i = 0; i < num_outputs; ++i) - { - // LOG_PRINT_L0("amount output index[" << 2*i << "]" << ": " << paired_indices[2*i] << " global output index: " << paired_indices[2*i+1]); - amount_output_indices.push_back(indices[i]); + const uint64_t* indices = (const uint64_t*)v.mv_data; + size_t num_outputs = v.mv_size / sizeof(uint64_t); + + amount_output_indices_set.resize(amount_output_indices_set.size() + 1); + std::vector<uint64_t> &amount_output_indices = amount_output_indices_set.back(); + amount_output_indices.reserve(num_outputs); + for (size_t i = 0; i < num_outputs; ++i) + { + amount_output_indices.push_back(indices[i]); + } } - indices = nullptr; TXN_POSTFIX_RDONLY(); - return amount_output_indices; + return amount_output_indices_set; } - bool BlockchainLMDB::has_key_image(const crypto::key_image& img) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -3105,7 +3531,7 @@ uint64_t BlockchainLMDB::add_block(const block& blk, size_t block_weight, const check_open(); uint64_t m_height = height(); - if (m_height % 1000 == 0) + if (m_height % 1024 == 0) { // for batch mode, DB resize check is done at start of batch transaction if (! m_batch_active && need_resize()) @@ -3157,6 +3583,7 @@ void BlockchainLMDB::get_output_tx_and_index_from_global(const std::vector<uint6 LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); tx_out_indices.clear(); + tx_out_indices.reserve(global_indices.size()); TXN_PREFIX_RDONLY(); RCURSOR(output_txs); @@ -3171,29 +3598,33 @@ void BlockchainLMDB::get_output_tx_and_index_from_global(const std::vector<uint6 else if (get_result) throw0(DB_ERROR("DB error attempting to fetch output tx hash")); - outtx *ot = (outtx *)v.mv_data; - auto result = tx_out_index(ot->tx_hash, ot->local_index); - tx_out_indices.push_back(result); + const outtx *ot = (const outtx *)v.mv_data; + tx_out_indices.push_back(tx_out_index(ot->tx_hash, ot->local_index)); } TXN_POSTFIX_RDONLY(); } -void BlockchainLMDB::get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs, bool allow_partial) +void BlockchainLMDB::get_output_key(const epee::span<const uint64_t> &amounts, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs, bool allow_partial) const { + if (amounts.size() != 1 && amounts.size() != offsets.size()) + throw0(DB_ERROR("Invalid sizes of amounts and offets")); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); TIME_MEASURE_START(db3); check_open(); outputs.clear(); + outputs.reserve(offsets.size()); TXN_PREFIX_RDONLY(); RCURSOR(output_amounts); - MDB_val_set(k, amount); - for (const uint64_t &index : offsets) + for (size_t i = 0; i < offsets.size(); ++i) { - MDB_val_set(v, index); + const uint64_t amount = amounts.size() == 1 ? amounts[0] : amounts[i]; + MDB_val_set(k, amount); + MDB_val_set(v, offsets[i]); auto get_result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_BOTH); if (get_result == MDB_NOTFOUND) @@ -3203,24 +3634,24 @@ void BlockchainLMDB::get_output_key(const uint64_t &amount, const std::vector<ui MDEBUG("Partial result: " << outputs.size() << "/" << offsets.size()); break; } - throw1(OUTPUT_DNE((std::string("Attempting to get output pubkey by global index (amount ") + boost::lexical_cast<std::string>(amount) + ", index " + boost::lexical_cast<std::string>(index) + ", count " + boost::lexical_cast<std::string>(get_num_outputs(amount)) + "), but key does not exist (current height " + boost::lexical_cast<std::string>(height()) + ")").c_str())); + throw1(OUTPUT_DNE((std::string("Attempting to get output pubkey by global index (amount ") + boost::lexical_cast<std::string>(amount) + ", index " + boost::lexical_cast<std::string>(offsets[i]) + ", count " + boost::lexical_cast<std::string>(get_num_outputs(amount)) + "), but key does not exist (current height " + boost::lexical_cast<std::string>(height()) + ")").c_str())); } else if (get_result) throw0(DB_ERROR(lmdb_error("Error attempting to retrieve an output pubkey from the db", get_result).c_str())); - output_data_t data; if (amount == 0) { const outkey *okp = (const outkey *)v.mv_data; - data = okp->data; + outputs.push_back(okp->data); } else { const pre_rct_outkey *okp = (const pre_rct_outkey *)v.mv_data; + outputs.resize(outputs.size() + 1); + output_data_t &data = outputs.back(); memcpy(&data, &okp->data, sizeof(pre_rct_output_data_t)); data.commitment = rct::zeroCommit(amount); } - outputs.push_back(data); } TXN_POSTFIX_RDONLY(); @@ -3236,6 +3667,7 @@ void BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, const std:: indices.clear(); std::vector <uint64_t> tx_indices; + tx_indices.reserve(offsets.size()); TXN_PREFIX_RDONLY(); RCURSOR(output_amounts); @@ -4049,7 +4481,7 @@ void BlockchainLMDB::migrate_0_1() uint32_t version = 1; v.mv_data = (void *)&version; v.mv_size = sizeof(version); - MDB_val_copy<const char *> vk("version"); + MDB_val_str(vk, "version"); result = mdb_txn_begin(m_env, NULL, 0, txn); if (result) throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); @@ -4191,7 +4623,7 @@ void BlockchainLMDB::migrate_1_2() uint32_t version = 2; v.mv_data = (void *)&version; v.mv_size = sizeof(version); - MDB_val_copy<const char *> vk("version"); + MDB_val_str(vk, "version"); result = mdb_txn_begin(m_env, NULL, 0, txn); if (result) throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); @@ -4326,7 +4758,7 @@ void BlockchainLMDB::migrate_2_3() uint32_t version = 3; v.mv_data = (void *)&version; v.mv_size = sizeof(version); - MDB_val_copy<const char *> vk("version"); + MDB_val_str(vk, "version"); result = mdb_txn_begin(m_env, NULL, 0, txn); if (result) throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); @@ -4338,16 +4770,12 @@ void BlockchainLMDB::migrate_2_3() void BlockchainLMDB::migrate(const uint32_t oldversion) { - switch(oldversion) { - case 0: - migrate_0_1(); /* FALLTHRU */ - case 1: - migrate_1_2(); /* FALLTHRU */ - case 2: - migrate_2_3(); /* FALLTHRU */ - default: - ; - } + if (oldversion < 1) + migrate_0_1(); + if (oldversion < 2) + migrate_1_2(); + if (oldversion < 3) + migrate_2_3(); } } // namespace cryptonote diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index e1f748ed8..c07ab8da5 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -40,6 +40,11 @@ namespace cryptonote { +typedef struct txindex { + crypto::hash key; + tx_data_t data; +} txindex; + typedef struct mdb_txn_cursors { MDB_cursor *m_txc_blocks; @@ -53,6 +58,7 @@ typedef struct mdb_txn_cursors MDB_cursor *m_txc_txs_pruned; MDB_cursor *m_txc_txs_prunable; MDB_cursor *m_txc_txs_prunable_hash; + MDB_cursor *m_txc_txs_prunable_tip; MDB_cursor *m_txc_tx_indices; MDB_cursor *m_txc_tx_outputs; @@ -62,6 +68,8 @@ typedef struct mdb_txn_cursors MDB_cursor *m_txc_txpool_blob; MDB_cursor *m_txc_hf_versions; + + MDB_cursor *m_txc_properties; } mdb_txn_cursors; #define m_cur_blocks m_cursors->m_txc_blocks @@ -73,12 +81,14 @@ typedef struct mdb_txn_cursors #define m_cur_txs_pruned m_cursors->m_txc_txs_pruned #define m_cur_txs_prunable m_cursors->m_txc_txs_prunable #define m_cur_txs_prunable_hash m_cursors->m_txc_txs_prunable_hash +#define m_cur_txs_prunable_tip m_cursors->m_txc_txs_prunable_tip #define m_cur_tx_indices m_cursors->m_txc_tx_indices #define m_cur_tx_outputs m_cursors->m_txc_tx_outputs #define m_cur_spent_keys m_cursors->m_txc_spent_keys #define m_cur_txpool_meta m_cursors->m_txc_txpool_meta #define m_cur_txpool_blob m_cursors->m_txc_txpool_blob #define m_cur_hf_versions m_cursors->m_txc_hf_versions +#define m_cur_properties m_cursors->m_txc_properties typedef struct mdb_rflags { @@ -92,12 +102,14 @@ typedef struct mdb_rflags bool m_rf_txs_pruned; bool m_rf_txs_prunable; bool m_rf_txs_prunable_hash; + bool m_rf_txs_prunable_tip; bool m_rf_tx_indices; bool m_rf_tx_outputs; bool m_rf_spent_keys; bool m_rf_txpool_meta; bool m_rf_txpool_blob; bool m_rf_hf_versions; + bool m_rf_properties; } mdb_rflags; typedef struct mdb_threadinfo @@ -232,6 +244,7 @@ public: virtual bool get_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const; virtual bool get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const; + virtual bool get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const; virtual bool get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const; virtual uint64_t get_tx_count() const; @@ -242,8 +255,8 @@ public: virtual uint64_t get_num_outputs(const uint64_t& amount) const; - virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index); - virtual void get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs, bool allow_partial = false); + virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index, bool include_commitmemt) const; + virtual void get_output_key(const epee::span<const uint64_t> &amounts, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs, bool allow_partial = false) const; virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const; virtual void get_output_tx_and_index_from_global(const std::vector<uint64_t> &global_indices, @@ -252,11 +265,11 @@ public: virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const; virtual void get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices) const; - virtual std::vector<uint64_t> get_tx_amount_output_indices(const uint64_t tx_id) const; + virtual std::vector<std::vector<uint64_t>> get_tx_amount_output_indices(const uint64_t tx_id, size_t n_txes) const; virtual bool has_key_image(const crypto::key_image& img) const; - virtual void add_txpool_tx(const transaction &tx, const txpool_tx_meta_t& meta); + virtual void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const txpool_tx_meta_t& meta); virtual void update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t& meta); virtual uint64_t get_txpool_tx_count(bool include_unrelayed_txes = true) const; virtual bool txpool_has_tx(const crypto::hash &txid) const; @@ -264,6 +277,11 @@ public: virtual bool get_txpool_tx_meta(const crypto::hash& txid, txpool_tx_meta_t &meta) const; virtual bool get_txpool_tx_blob(const crypto::hash& txid, cryptonote::blobdata &bd) const; virtual cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid) const; + virtual uint32_t get_blockchain_pruning_seed() const; + virtual bool prune_blockchain(uint32_t pruning_seed = 0); + virtual bool update_pruning(); + virtual bool check_pruning(); + virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)> f, bool include_blob = false, bool include_unrelayed_txes = true) const; virtual bool for_all_key_images(std::function<bool(const crypto::key_image&)>) const; @@ -309,6 +327,11 @@ public: bool get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t to_height, std::vector<uint64_t> &distribution, uint64_t &base) const; + // helper functions + static int compare_uint64(const MDB_val *a, const MDB_val *b); + static int compare_hash32(const MDB_val *a, const MDB_val *b); + static int compare_string(const MDB_val *a, const MDB_val *b); + private: void do_resize(uint64_t size_increase=0); @@ -345,6 +368,8 @@ private: void remove_output(const uint64_t amount, const uint64_t& out_index); + virtual void prune_outputs(uint64_t amount); + virtual void add_spent_key(const crypto::key_image& k_image); virtual void remove_spent_key(const crypto::key_image& k_image); @@ -357,25 +382,9 @@ private: virtual void check_hard_fork_info(); virtual void drop_hard_fork_info(); - /** - * @brief convert a tx output to a blob for storage - * - * @param output the output to convert - * - * @return the resultant blob - */ - blobdata output_to_blob(const tx_out& output) const; - - /** - * @brief convert a tx output blob to a tx output - * - * @param blob the blob to convert - * - * @return the resultant tx output - */ - tx_out output_from_blob(const blobdata& blob) const; + inline void check_open() const; - void check_open() const; + bool prune_worker(int mode, uint32_t pruning_seed); virtual bool is_read_only() const; @@ -409,6 +418,7 @@ private: MDB_dbi m_txs_pruned; MDB_dbi m_txs_prunable; MDB_dbi m_txs_prunable_hash; + MDB_dbi m_txs_prunable_tip; MDB_dbi m_tx_indices; MDB_dbi m_tx_outputs; diff --git a/src/blockchain_utilities/CMakeLists.txt b/src/blockchain_utilities/CMakeLists.txt index 24a750eb0..df74eb695 100644 --- a/src/blockchain_utilities/CMakeLists.txt +++ b/src/blockchain_utilities/CMakeLists.txt @@ -26,20 +26,6 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -set(blocksdat "") -if(PER_BLOCK_CHECKPOINT) - if(APPLE AND DEPENDS) - add_custom_command(OUTPUT blocksdat.o MAIN_DEPENDENCY ../blocks/checkpoints.dat COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && touch stub.c && ${CMAKE_C_COMPILER} --target=x86_64-apple-darwin11 -o stub.o -c stub.c COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${CMAKE_LINKER} -r -sectcreate __DATA __blocks_dat ../blocks/checkpoints.dat -o ${CMAKE_CURRENT_BINARY_DIR}/blocksdat.o stub.o && rm -f stub.*) - elseif(APPLE AND NOT DEPENDS) - add_custom_command(OUTPUT blocksdat.o MAIN_DEPENDENCY ../blocks/checkpoints.dat COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && touch stub.c && ${CMAKE_C_COMPILER} -o stub.o -c stub.c COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${CMAKE_LINKER} ${LD_RAW_FLAGS} -r -sectcreate __DATA __blocks_dat ../blocks/checkpoints.dat -o ${CMAKE_CURRENT_BINARY_DIR}/blocksdat.o stub.o && rm -f stub.*) - elseif(LINUX_32) - add_custom_command(OUTPUT blocksdat.o MAIN_DEPENDENCY ../blocks/checkpoints.dat COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && cp ../blocks/checkpoints.dat blocks.dat && ${CMAKE_LINKER} -m elf_i386 ${LD_RAW_FLAGS} -r -b binary -o ${CMAKE_CURRENT_BINARY_DIR}/blocksdat.o blocks.dat && rm -f blocks.dat) - else() - add_custom_command(OUTPUT blocksdat.o MAIN_DEPENDENCY ../blocks/checkpoints.dat COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && cp ../blocks/checkpoints.dat blocks.dat && ${CMAKE_LINKER} ${LD_RAW_FLAGS} -r -b binary -o ${CMAKE_CURRENT_BINARY_DIR}/blocksdat.o blocks.dat && rm -f blocks.dat) - endif() - set(blocksdat "blocksdat.o") -endif() - set(blockchain_import_sources blockchain_import.cpp bootstrap_file.cpp @@ -95,6 +81,28 @@ monero_private_headers(blockchain_usage +set(blockchain_prune_known_spent_data_sources + blockchain_prune_known_spent_data.cpp + ) + +set(blockchain_prune_known_spent_data_private_headers) + +monero_private_headers(blockchain_prune_known_spent_data + ${blockchain_prune_known_spent_data_private_headers}) + + + +set(blockchain_prune_sources + blockchain_prune.cpp + ) + +set(blockchain_prune_private_headers) + +monero_private_headers(blockchain_prune + ${blockchain_prune_private_headers}) + + + set(blockchain_ancestry_sources blockchain_ancestry.cpp ) @@ -115,12 +123,19 @@ set(blockchain_depth_private_headers) monero_private_headers(blockchain_depth ${blockchain_depth_private_headers}) +set(blockchain_stats_sources + blockchain_stats.cpp + ) + +set(blockchain_stats_private_headers) + +monero_private_headers(blockchain_stats + ${blockchain_stats_private_headers}) monero_add_executable(blockchain_import ${blockchain_import_sources} - ${blockchain_import_private_headers} - ${blocksdat}) + ${blockchain_import_private_headers}) target_link_libraries(blockchain_import PRIVATE @@ -132,7 +147,8 @@ target_link_libraries(blockchain_import ${Boost_SYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} - ${EXTRA_LIBRARIES}) + ${EXTRA_LIBRARIES} + ${Blocks}) if(ARCH_WIDTH) target_compile_definitions(blockchain_import @@ -251,3 +267,67 @@ set_property(TARGET blockchain_depth OUTPUT_NAME "monero-blockchain-depth") install(TARGETS blockchain_depth DESTINATION bin) +monero_add_executable(blockchain_stats + ${blockchain_stats_sources} + ${blockchain_stats_private_headers}) + +target_link_libraries(blockchain_stats + PRIVATE + cryptonote_core + blockchain_db + version + epee + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${Boost_THREAD_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES}) + +set_property(TARGET blockchain_stats + PROPERTY + OUTPUT_NAME "monero-blockchain-stats") +install(TARGETS blockchain_stats DESTINATION bin) + +monero_add_executable(blockchain_prune_known_spent_data + ${blockchain_prune_known_spent_data_sources} + ${blockchain_prune_known_spent_data_private_headers}) + +target_link_libraries(blockchain_prune_known_spent_data + PRIVATE + cryptonote_core + blockchain_db + p2p + version + epee + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${Boost_THREAD_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES}) + +set_property(TARGET blockchain_prune_known_spent_data + PROPERTY + OUTPUT_NAME "monero-blockchain-prune-known-spent-data") +install(TARGETS blockchain_prune_known_spent_data DESTINATION bin) + +monero_add_executable(blockchain_prune + ${blockchain_prune_sources} + ${blockchain_prune_private_headers}) + +set_property(TARGET blockchain_prune + PROPERTY + OUTPUT_NAME "monero-blockchain-prune") +install(TARGETS blockchain_prune DESTINATION bin) + +target_link_libraries(blockchain_prune + PRIVATE + cryptonote_core + blockchain_db + p2p + version + epee + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${Boost_THREAD_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES}) diff --git a/src/blockchain_utilities/blockchain_ancestry.cpp b/src/blockchain_utilities/blockchain_ancestry.cpp index 2f0bbffd6..a64ce160a 100644 --- a/src/blockchain_utilities/blockchain_ancestry.cpp +++ b/src/blockchain_utilities/blockchain_ancestry.cpp @@ -51,6 +51,8 @@ using namespace epee; using namespace cryptonote; static bool stop_requested = false; +static uint64_t cached_txes = 0, cached_blocks = 0, cached_outputs = 0, total_txes = 0, total_blocks = 0, total_outputs = 0; +static bool opt_cache_outputs = false, opt_cache_txes = false, opt_cache_blocks = false; struct ancestor { @@ -137,6 +139,8 @@ struct ancestry_state_t std::unordered_map<crypto::hash, ::tx_data_t> tx_cache; std::vector<cryptonote::block> block_cache; + ancestry_state_t(): height(0) {} + template <typename t_archive> void serialize(t_archive &a, const unsigned int ver) { a & height; @@ -219,6 +223,113 @@ static std::unordered_set<ancestor> get_ancestry(const std::unordered_map<crypto return i->second; } +static bool get_block_from_height(ancestry_state_t &state, BlockchainDB *db, uint64_t height, cryptonote::block &b) +{ + ++total_blocks; + if (state.block_cache.size() > height && !state.block_cache[height].miner_tx.vin.empty()) + { + ++cached_blocks; + b = state.block_cache[height]; + return true; + } + cryptonote::blobdata bd = db->get_block_blob_from_height(height); + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return false; + } + if (opt_cache_blocks) + { + state.block_cache.resize(height + 1); + state.block_cache[height] = b; + } + return true; +} + +static bool get_transaction(ancestry_state_t &state, BlockchainDB *db, const crypto::hash &txid, ::tx_data_t &tx_data) +{ + std::unordered_map<crypto::hash, ::tx_data_t>::const_iterator i = state.tx_cache.find(txid); + ++total_txes; + if (i != state.tx_cache.end()) + { + ++cached_txes; + tx_data = i->second; + return true; + } + + cryptonote::blobdata bd; + if (!db->get_pruned_tx_blob(txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << txid << " from db"); + return false; + } + cryptonote::transaction tx; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) + { + LOG_PRINT_L0("Bad tx: " << txid); + return false; + } + tx_data = ::tx_data_t(tx); + if (opt_cache_txes) + state.tx_cache.insert(std::make_pair(txid, tx_data)); + return true; +} + +static bool get_output_txid(ancestry_state_t &state, BlockchainDB *db, uint64_t amount, uint64_t offset, crypto::hash &txid) +{ + ++total_outputs; + std::unordered_map<ancestor, crypto::hash>::const_iterator i = state.output_cache.find({amount, offset}); + if (i != state.output_cache.end()) + { + ++cached_outputs; + txid = i->second; + return true; + } + + const output_data_t od = db->get_output_key(amount, offset, false); + cryptonote::block b; + if (!get_block_from_height(state, db, od.height, b)) + return false; + + for (size_t out = 0; out < b.miner_tx.vout.size(); ++out) + { + if (b.miner_tx.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get<cryptonote::txout_to_key>(b.miner_tx.vout[out].target); + if (txout.key == od.pubkey) + { + txid = cryptonote::get_transaction_hash(b.miner_tx); + if (opt_cache_outputs) + state.output_cache.insert(std::make_pair(ancestor{amount, offset}, txid)); + return true; + } + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << cryptonote::get_transaction_hash(b.miner_tx)); + return false; + } + } + for (const crypto::hash &block_txid: b.tx_hashes) + { + ::tx_data_t tx_data3; + if (!get_transaction(state, db, block_txid, tx_data3)) + return false; + + for (size_t out = 0; out < tx_data3.vout.size(); ++out) + { + if (tx_data3.vout[out] == od.pubkey) + { + txid = block_txid; + if (opt_cache_outputs) + state.output_cache.insert(std::make_pair(ancestor{amount, offset}, txid)); + return true; + } + } + } + return false; +} + int main(int argc, char* argv[]) { TRY_ENTRY(); @@ -243,12 +354,13 @@ int main(int argc, char* argv[]) "database", available_dbs.c_str(), default_db_type }; const command_line::arg_descriptor<std::string> arg_txid = {"txid", "Get ancestry for this txid", ""}; + const command_line::arg_descriptor<std::string> arg_output = {"output", "Get ancestry for this output (amount/offset format)", ""}; const command_line::arg_descriptor<uint64_t> arg_height = {"height", "Get ancestry for all txes at this height", 0}; - const command_line::arg_descriptor<bool> arg_all = {"all", "Include the whole chain", false}; + const command_line::arg_descriptor<bool> arg_refresh = {"refresh", "Refresh the whole chain first", false}; const command_line::arg_descriptor<bool> arg_cache_outputs = {"cache-outputs", "Cache outputs (memory hungry)", false}; const command_line::arg_descriptor<bool> arg_cache_txes = {"cache-txes", "Cache txes (memory hungry)", false}; const command_line::arg_descriptor<bool> arg_cache_blocks = {"cache-blocks", "Cache blocks (memory hungry)", false}; - const command_line::arg_descriptor<bool> arg_include_coinbase = {"include-coinbase", "Including coinbase tx", false}; + const command_line::arg_descriptor<bool> arg_include_coinbase = {"include-coinbase", "Including coinbase tx in per height average", false}; const command_line::arg_descriptor<bool> arg_show_cache_stats = {"show-cache-stats", "Show cache statistics", false}; command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir); @@ -257,8 +369,9 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_cmd_sett, arg_log_level); command_line::add_arg(desc_cmd_sett, arg_database); command_line::add_arg(desc_cmd_sett, arg_txid); + command_line::add_arg(desc_cmd_sett, arg_output); command_line::add_arg(desc_cmd_sett, arg_height); - command_line::add_arg(desc_cmd_sett, arg_all); + command_line::add_arg(desc_cmd_sett, arg_refresh); command_line::add_arg(desc_cmd_sett, arg_cache_outputs); command_line::add_arg(desc_cmd_sett, arg_cache_txes); command_line::add_arg(desc_cmd_sett, arg_cache_blocks); @@ -300,20 +413,22 @@ int main(int argc, char* argv[]) bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; std::string opt_txid_string = command_line::get_arg(vm, arg_txid); + std::string opt_output_string = command_line::get_arg(vm, arg_output); uint64_t opt_height = command_line::get_arg(vm, arg_height); - bool opt_all = command_line::get_arg(vm, arg_all); - bool opt_cache_outputs = command_line::get_arg(vm, arg_cache_outputs); - bool opt_cache_txes = command_line::get_arg(vm, arg_cache_txes); - bool opt_cache_blocks = command_line::get_arg(vm, arg_cache_blocks); + bool opt_refresh = command_line::get_arg(vm, arg_refresh); + opt_cache_outputs = command_line::get_arg(vm, arg_cache_outputs); + opt_cache_txes = command_line::get_arg(vm, arg_cache_txes); + opt_cache_blocks = command_line::get_arg(vm, arg_cache_blocks); bool opt_include_coinbase = command_line::get_arg(vm, arg_include_coinbase); bool opt_show_cache_stats = command_line::get_arg(vm, arg_show_cache_stats); - if ((!opt_txid_string.empty()) + !!opt_height + !!opt_all > 1) + if ((!opt_txid_string.empty()) + !!opt_height + !opt_output_string.empty() > 1) { - std::cerr << "Only one of --txid, --height and --all can be given" << std::endl; + std::cerr << "Only one of --txid, --height, --output can be given" << std::endl; return 1; } crypto::hash opt_txid = crypto::null_hash; + uint64_t output_amount = 0, output_offset = 0; if (!opt_txid_string.empty()) { if (!epee::string_tools::hex_to_pod(opt_txid_string, opt_txid)) @@ -322,6 +437,14 @@ int main(int argc, char* argv[]) return 1; } } + else if (!opt_output_string.empty()) + { + if (sscanf(opt_output_string.c_str(), "%" SCNu64 "/%" SCNu64, &output_amount, &output_offset) != 2) + { + std::cerr << "Invalid output" << std::endl; + return 1; + } + } std::string db_type = command_line::get_arg(vm, arg_database); if (!cryptonote::blockchain_valid_db_type(db_type)) @@ -372,43 +495,41 @@ int main(int argc, char* argv[]) std::vector<crypto::hash> start_txids; - // forward method - if (opt_all) - { - uint64_t cached_txes = 0, cached_blocks = 0, cached_outputs = 0, total_txes = 0, total_blocks = 0, total_outputs = 0; - ancestry_state_t state; + ancestry_state_t state; - const std::string state_file_path = (boost::filesystem::path(opt_data_dir) / "ancestry-state.bin").string(); - LOG_PRINT_L0("Loading state data from " << state_file_path); - std::ifstream state_data_in; - state_data_in.open(state_file_path, std::ios_base::binary | std::ios_base::in); - if (!state_data_in.fail()) + const std::string state_file_path = (boost::filesystem::path(opt_data_dir) / "ancestry-state.bin").string(); + LOG_PRINT_L0("Loading state data from " << state_file_path); + std::ifstream state_data_in; + state_data_in.open(state_file_path, std::ios_base::binary | std::ios_base::in); + if (!state_data_in.fail()) + { + try { - try - { - boost::archive::portable_binary_iarchive a(state_data_in); - a >> state; - } - catch (const std::exception &e) - { - MERROR("Failed to load state data from " << state_file_path << ", restarting from scratch"); - state = ancestry_state_t(); - } - state_data_in.close(); + boost::archive::portable_binary_iarchive a(state_data_in); + a >> state; + } + catch (const std::exception &e) + { + MERROR("Failed to load state data from " << state_file_path << ", restarting from scratch"); + state = ancestry_state_t(); } + state_data_in.close(); + } - tools::signal_handler::install([](int type) { - stop_requested = true; - }); + tools::signal_handler::install([](int type) { + stop_requested = true; + }); + // forward method + const uint64_t db_height = db->height(); + if (opt_refresh) + { MINFO("Starting from height " << state.height); - const uint64_t db_height = db->height(); state.block_cache.reserve(db_height); for (uint64_t h = state.height; h < db_height; ++h) { size_t block_ancestry_size = 0; - const crypto::hash block_hash = db->get_block_hash_from_height(h); - const cryptonote::blobdata bd = db->get_block_blob(block_hash); + const cryptonote::blobdata bd = db->get_block_blob_from_height(h); ++total_blocks; cryptonote::block b; if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) @@ -465,114 +586,20 @@ int main(int argc, char* argv[]) { for (size_t ring = 0; ring < tx_data.vin.size(); ++ring) { - if (1) + const uint64_t amount = tx_data.vin[ring].first; + const std::vector<uint64_t> &absolute_offsets = tx_data.vin[ring].second; + for (uint64_t offset: absolute_offsets) { - const uint64_t amount = tx_data.vin[ring].first; - const std::vector<uint64_t> &absolute_offsets = tx_data.vin[ring].second; - for (uint64_t offset: absolute_offsets) + add_ancestry(state.ancestry, txid, ancestor{amount, offset}); + // find the tx which created this output + bool found = false; + crypto::hash output_txid; + if (!get_output_txid(state, db, amount, offset, output_txid)) { - const output_data_t od = db->get_output_key(amount, offset); - add_ancestry(state.ancestry, txid, ancestor{amount, offset}); - cryptonote::block b; - ++total_blocks; - if (state.block_cache.size() > od.height && !state.block_cache[od.height].miner_tx.vin.empty()) - { - ++cached_blocks; - b = state.block_cache[od.height]; - } - else - { - const crypto::hash block_hash = db->get_block_hash_from_height(od.height); - cryptonote::blobdata bd = db->get_block_blob(block_hash); - if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) - { - LOG_PRINT_L0("Bad block from db"); - return 1; - } - if (opt_cache_blocks) - { - state.block_cache.resize(od.height + 1); - state.block_cache[od.height] = b; - } - } - // find the tx which created this output - bool found = false; - std::unordered_map<ancestor, crypto::hash>::const_iterator i = state.output_cache.find({amount, offset}); - ++total_outputs; - if (i != state.output_cache.end()) - { - ++cached_outputs; - add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, i->second)); - found = true; - } - else for (size_t out = 0; out < b.miner_tx.vout.size(); ++out) - { - if (b.miner_tx.vout[out].target.type() == typeid(cryptonote::txout_to_key)) - { - const auto &txout = boost::get<cryptonote::txout_to_key>(b.miner_tx.vout[out].target); - if (txout.key == od.pubkey) - { - found = true; - add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, cryptonote::get_transaction_hash(b.miner_tx))); - if (opt_cache_outputs) - state.output_cache.insert(std::make_pair(ancestor{amount, offset}, cryptonote::get_transaction_hash(b.miner_tx))); - break; - } - } - else - { - LOG_PRINT_L0("Bad vout type in txid " << cryptonote::get_transaction_hash(b.miner_tx)); - return 1; - } - } - for (const crypto::hash &block_txid: b.tx_hashes) - { - if (found) - break; - ::tx_data_t tx_data2; - std::unordered_map<crypto::hash, ::tx_data_t>::const_iterator i = state.tx_cache.find(block_txid); - ++total_txes; - if (i != state.tx_cache.end()) - { - ++cached_txes; - tx_data2 = i->second; - } - else - { - cryptonote::blobdata bd; - if (!db->get_pruned_tx_blob(block_txid, bd)) - { - LOG_PRINT_L0("Failed to get txid " << block_txid << " from db"); - return 1; - } - cryptonote::transaction tx; - if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) - { - LOG_PRINT_L0("Bad tx: " << block_txid); - return 1; - } - tx_data2 = ::tx_data_t(tx); - if (opt_cache_txes) - state.tx_cache.insert(std::make_pair(block_txid, tx_data2)); - } - for (size_t out = 0; out < tx_data2.vout.size(); ++out) - { - if (tx_data2.vout[out] == od.pubkey) - { - found = true; - add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, block_txid)); - if (opt_cache_outputs) - state.output_cache.insert(std::make_pair(ancestor{amount, offset}, block_txid)); - break; - } - } - } - if (!found) - { - LOG_PRINT_L0("Output originating transaction not found"); - return 1; - } + LOG_PRINT_L0("Output originating transaction not found"); + return 1; } + add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, output_txid)); } } } @@ -583,10 +610,6 @@ int main(int argc, char* argv[]) if (!txids.empty()) { std::string stats_msg; - if (opt_show_cache_stats) - stats_msg = std::string(", cache: txes ") + std::to_string(cached_txes*100./total_txes) - + ", blocks " + std::to_string(cached_blocks*100./total_blocks) + ", outputs " - + std::to_string(cached_outputs*100./total_outputs); MINFO("Height " << h << ": " << (block_ancestry_size / txids.size()) << " average over " << txids.size() << stats_msg); } state.height = h; @@ -610,18 +633,33 @@ int main(int argc, char* argv[]) } state_data_out.close(); } - - goto done; + } + else + { + if (state.height < db_height) + { + MWARNING("The state file is only built up to height " << state.height << ", but the blockchain reached height " << db_height); + MWARNING("You may want to run with --refresh if you want to get ancestry for newer data"); + } } if (!opt_txid_string.empty()) { start_txids.push_back(opt_txid); } + else if (!opt_output_string.empty()) + { + crypto::hash txid; + if (!get_output_txid(state, db, output_amount, output_offset, txid)) + { + LOG_PRINT_L0("Output not found in db"); + return 1; + } + start_txids.push_back(txid); + } else { - const crypto::hash block_hash = db->get_block_hash_from_height(opt_height); - const cryptonote::blobdata bd = db->get_block_blob(block_hash); + const cryptonote::blobdata bd = db->get_block_blob_from_height(opt_height); cryptonote::block b; if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) { @@ -651,109 +689,40 @@ int main(int argc, char* argv[]) const crypto::hash txid = txids.front(); txids.pop_front(); - cryptonote::blobdata bd; - if (!db->get_pruned_tx_blob(txid, bd)) - { - LOG_PRINT_L0("Failed to get txid " << txid << " from db"); - return 1; - } - cryptonote::transaction tx; - if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) - { - LOG_PRINT_L0("Bad tx: " << txid); + if (stop_requested) + goto done; + + ::tx_data_t tx_data2; + if (!get_transaction(state, db, txid, tx_data2)) return 1; - } - const bool coinbase = tx.vin.size() == 1 && tx.vin[0].type() == typeid(cryptonote::txin_gen); + + const bool coinbase = tx_data2.coinbase; if (coinbase) continue; - for (size_t ring = 0; ring < tx.vin.size(); ++ring) + for (size_t ring = 0; ring < tx_data2.vin.size(); ++ring) { - if (tx.vin[ring].type() == typeid(cryptonote::txin_to_key)) { - const cryptonote::txin_to_key &txin = boost::get<cryptonote::txin_to_key>(tx.vin[ring]); - const uint64_t amount = txin.amount; - auto absolute_offsets = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets); + const uint64_t amount = tx_data2.vin[ring].first; + auto absolute_offsets = tx_data2.vin[ring].second; for (uint64_t offset: absolute_offsets) { add_ancestor(ancestry, amount, offset); - const output_data_t od = db->get_output_key(amount, offset); - const crypto::hash block_hash = db->get_block_hash_from_height(od.height); - bd = db->get_block_blob(block_hash); - cryptonote::block b; - if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) - { - LOG_PRINT_L0("Bad block from db"); - return 1; - } + // find the tx which created this output bool found = false; - for (size_t out = 0; out < b.miner_tx.vout.size(); ++out) - { - if (b.miner_tx.vout[out].target.type() == typeid(cryptonote::txout_to_key)) - { - const auto &txout = boost::get<cryptonote::txout_to_key>(b.miner_tx.vout[out].target); - if (txout.key == od.pubkey) - { - found = true; - txids.push_back(cryptonote::get_transaction_hash(b.miner_tx)); - MDEBUG("adding txid: " << cryptonote::get_transaction_hash(b.miner_tx)); - break; - } - } - else - { - LOG_PRINT_L0("Bad vout type in txid " << cryptonote::get_transaction_hash(b.miner_tx)); - return 1; - } - } - for (const crypto::hash &block_txid: b.tx_hashes) - { - if (found) - break; - if (!db->get_pruned_tx_blob(block_txid, bd)) - { - LOG_PRINT_L0("Failed to get txid " << block_txid << " from db"); - return 1; - } - cryptonote::transaction tx2; - if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx2)) - { - LOG_PRINT_L0("Bad tx: " << block_txid); - return 1; - } - for (size_t out = 0; out < tx2.vout.size(); ++out) - { - if (tx2.vout[out].target.type() == typeid(cryptonote::txout_to_key)) - { - const auto &txout = boost::get<cryptonote::txout_to_key>(tx2.vout[out].target); - if (txout.key == od.pubkey) - { - found = true; - txids.push_back(block_txid); - MDEBUG("adding txid: " << block_txid); - break; - } - } - else - { - LOG_PRINT_L0("Bad vout type in txid " << block_txid); - return 1; - } - } - } - if (!found) + crypto::hash output_txid; + if (!get_output_txid(state, db, amount, offset, output_txid)) { LOG_PRINT_L0("Output originating transaction not found"); return 1; } + + add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, output_txid)); + txids.push_back(output_txid); + MDEBUG("adding txid: " << output_txid); } } - else - { - LOG_PRINT_L0("Bad vin type in txid " << txid); - return 1; - } } } @@ -766,6 +735,13 @@ int main(int argc, char* argv[]) done: core_storage->deinit(); + + if (opt_show_cache_stats) + MINFO("cache: txes " << std::to_string(cached_txes*100./total_txes) + << "%, blocks " << std::to_string(cached_blocks*100./total_blocks) + << "%, outputs " << std::to_string(cached_outputs*100./total_outputs) + << "%"); + return 0; CATCH_ENTRY("Depth query error", 1); diff --git a/src/blockchain_utilities/blockchain_blackball.cpp b/src/blockchain_utilities/blockchain_blackball.cpp index d2ce5cf76..8b007e901 100644 --- a/src/blockchain_utilities/blockchain_blackball.cpp +++ b/src/blockchain_utilities/blockchain_blackball.cpp @@ -59,6 +59,7 @@ static MDB_dbi dbi_relative_rings; static MDB_dbi dbi_outputs; static MDB_dbi dbi_processed_txidx; static MDB_dbi dbi_spent; +static MDB_dbi dbi_per_amount; static MDB_dbi dbi_ring_instances; static MDB_dbi dbi_stats; static MDB_env *env = NULL; @@ -238,7 +239,7 @@ static void init(std::string cache_filename) dbr = mdb_env_create(&env); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); - dbr = mdb_env_set_maxdbs(env, 6); + dbr = mdb_env_set_maxdbs(env, 7); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); const std::string actual_filename = get_cache_filename(cache_filename); dbr = mdb_env_open(env, actual_filename.c_str(), flags, 0664); @@ -265,6 +266,10 @@ static void init(std::string cache_filename) CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); mdb_set_dupsort(txn, dbi_spent, compare_uint64); + dbr = mdb_dbi_open(txn, "per_amount", MDB_CREATE | MDB_INTEGERKEY, &dbi_per_amount); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_compare(txn, dbi_per_amount, compare_uint64); + dbr = mdb_dbi_open(txn, "ring_instances", MDB_CREATE, &dbi_ring_instances); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); @@ -283,6 +288,7 @@ static void close() mdb_dbi_close(env, dbi_relative_rings); mdb_dbi_close(env, dbi_outputs); mdb_dbi_close(env, dbi_processed_txidx); + mdb_dbi_close(env, dbi_per_amount); mdb_dbi_close(env, dbi_spent); mdb_dbi_close(env, dbi_ring_instances); mdb_dbi_close(env, dbi_stats); @@ -585,6 +591,55 @@ static std::vector<output_data> get_spent_outputs(MDB_txn *txn) return outs; } +static void get_per_amount_outputs(MDB_txn *txn, uint64_t amount, uint64_t &total, uint64_t &spent) +{ + MDB_cursor *cur; + int dbr = mdb_cursor_open(txn, dbi_per_amount, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for per amount outputs: " + std::string(mdb_strerror(dbr))); + MDB_val k, v; + mdb_size_t count = 0; + k.mv_size = sizeof(uint64_t); + k.mv_data = (void*)&amount; + dbr = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (dbr == MDB_NOTFOUND) + { + total = spent = 0; + } + else + { + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get per amount outputs: " + std::string(mdb_strerror(dbr))); + total = ((const uint64_t*)v.mv_data)[0]; + spent = ((const uint64_t*)v.mv_data)[1]; + } + mdb_cursor_close(cur); +} + +static void inc_per_amount_outputs(MDB_txn *txn, uint64_t amount, uint64_t total, uint64_t spent) +{ + MDB_cursor *cur; + int dbr = mdb_cursor_open(txn, dbi_per_amount, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for per amount outputs: " + std::string(mdb_strerror(dbr))); + MDB_val k, v; + mdb_size_t count = 0; + k.mv_size = sizeof(uint64_t); + k.mv_data = (void*)&amount; + dbr = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (dbr == 0) + { + total += ((const uint64_t*)v.mv_data)[0]; + spent += ((const uint64_t*)v.mv_data)[1]; + } + else + { + CHECK_AND_ASSERT_THROW_MES(dbr == MDB_NOTFOUND, "Failed to get per amount outputs: " + std::string(mdb_strerror(dbr))); + } + uint64_t data[2] = {total, spent}; + v.mv_size = 2 * sizeof(uint64_t); + v.mv_data = (void*)data; + dbr = mdb_cursor_put(cur, &k, &v, 0); + mdb_cursor_close(cur); +} + static uint64_t get_processed_txidx(const std::string &name) { MDB_txn *txn; @@ -1076,7 +1131,7 @@ int main(int argc, char* argv[]) return 1; } - mlog_configure(mlog_get_default_log_path("monero-blockchain-find-spent-outputs.log"), true); + mlog_configure(mlog_get_default_log_path("monero-blockchain-mark-spent-outputs.log"), true); if (!command_line::is_arg_defaulted(vm, arg_log_level)) mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); else @@ -1193,6 +1248,7 @@ int main(int argc, char* argv[]) for_all_transactions(filename, start_idx, n_txes, [&](const cryptonote::transaction_prefix &tx)->bool { std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + const bool miner_tx = tx.vin.size() == 1 && tx.vin[0].type() == typeid(txin_gen); for (const auto &in: tx.vin) { if (in.type() != typeid(txin_to_key)) @@ -1210,6 +1266,9 @@ int main(int argc, char* argv[]) std::vector<uint64_t> new_ring = canonicalize(txin.key_offsets); const uint32_t ring_size = txin.key_offsets.size(); const uint64_t instances = inc_ring_instances(txn, txin.amount, new_ring); + uint64_t pa_total = 0, pa_spent = 0; + if (!opt_rct_only) + get_per_amount_outputs(txn, txin.amount, pa_total, pa_spent); if (n == 0 && ring_size == 1) { const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, absolute[0]); @@ -1237,6 +1296,21 @@ int main(int argc, char* argv[]) inc_stat(txn, txin.amount ? "pre-rct-duplicate-rings" : "rct-duplicate-rings"); } } + else if (n == 0 && !opt_rct_only && pa_spent + 1 == pa_total) + { + for (size_t o = 0; o < pa_total; ++o) + { + const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, o); + if (opt_verbose) + { + MINFO("Marking output " << output.first << "/" << output.second << " as spent, due to as many outputs of that amount being spent as exist so far"); + std::cout << "\r" << start_idx << "/" << n_txes << " \r" << std::flush; + } + blackballs.push_back(output); + if (add_spent_output(cur, output_data(txin.amount, o))) + inc_stat(txn, txin.amount ? "pre-rct-full-count" : "rct-full-count"); + } + } else if (n == 0 && opt_check_subsets && get_ring_subset_instances(txn, txin.amount, new_ring) >= new_ring.size()) { for (size_t o = 0; o < new_ring.size(); ++o) @@ -1299,9 +1373,28 @@ int main(int argc, char* argv[]) } } if (n == 0) + { set_relative_ring(txn, txin.k_image, new_ring); + if (!opt_rct_only) + inc_per_amount_outputs(txn, txin.amount, 0, 1); + } } set_processed_txidx(txn, canonical, start_idx+1); + if (!opt_rct_only) + { + for (const auto &out: tx.vout) + { + uint64_t amount = out.amount; + if (miner_tx && tx.version >= 2) + amount = 0; + + if (opt_rct_only && amount != 0) + continue; + if (out.target.type() != typeid(txout_to_key)) + continue; + inc_per_amount_outputs(txn, amount, 1, 0); + } + } ++records; if (records >= records_per_sync) @@ -1433,6 +1526,7 @@ skip_secondary_passes: { "pre-rct-ring-size-1", pre_rct }, { "rct-ring-size-1", rct }, { "pre-rct-duplicate-rings", pre_rct }, { "rct-duplicate-rings", rct }, { "pre-rct-subset-rings", pre_rct }, { "rct-subset-rings", rct }, + { "pre-rct-full-count", pre_rct }, { "rct-full-count", rct }, { "pre-rct-key-image-attack", pre_rct }, { "rct-key-image-attack", rct }, { "pre-rct-extra", pre_rct }, { "rct-ring-extra", rct }, { "pre-rct-chain-reaction", pre_rct }, { "rct-chain-reaction", rct }, diff --git a/src/blockchain_utilities/blockchain_depth.cpp b/src/blockchain_utilities/blockchain_depth.cpp index dd2387e5b..8060b0de4 100644 --- a/src/blockchain_utilities/blockchain_depth.cpp +++ b/src/blockchain_utilities/blockchain_depth.cpp @@ -187,8 +187,7 @@ int main(int argc, char* argv[]) } else { - const crypto::hash block_hash = db->get_block_hash_from_height(opt_height); - const cryptonote::blobdata bd = db->get_block_blob(block_hash); + const cryptonote::blobdata bd = db->get_block_blob_from_height(opt_height); cryptonote::block b; if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) { diff --git a/src/blockchain_utilities/blockchain_export.cpp b/src/blockchain_utilities/blockchain_export.cpp index 5a49f3478..5c5bc7f69 100644 --- a/src/blockchain_utilities/blockchain_export.cpp +++ b/src/blockchain_utilities/blockchain_export.cpp @@ -177,6 +177,12 @@ int main(int argc, char* argv[]) } r = core_storage->init(db, opt_testnet ? cryptonote::TESTNET : opt_stagenet ? cryptonote::STAGENET : cryptonote::MAINNET); + if (core_storage->get_blockchain_pruning_seed()) + { + LOG_PRINT_L0("Blockchain is pruned, cannot export"); + return 1; + } + CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage"); LOG_PRINT_L0("Source blockchain storage initialized OK"); LOG_PRINT_L0("Exporting blockchain raw data..."); diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp index 9ec768d26..2a8a6f494 100644 --- a/src/blockchain_utilities/blockchain_import.cpp +++ b/src/blockchain_utilities/blockchain_import.cpp @@ -37,6 +37,7 @@ #include "misc_log_ex.h" #include "bootstrap_file.h" #include "bootstrap_serialization.h" +#include "blocks/blocks.h" #include "cryptonote_basic/cryptonote_format_utils.h" #include "serialization/binary_utils.h" // dump_binary(), parse_binary() #include "serialization/json_utils.h" // dump_json() @@ -395,7 +396,7 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path { std::cout << refresh_string << "block " << h-1 << " / " << block_stop - << std::flush; + << "\r" << std::flush; std::cout << ENDL << ENDL; MINFO("Specified block number reached - stopping. block: " << h-1 << " total blocks: " << h); quit = 1; @@ -431,7 +432,7 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path { std::cout << refresh_string << "block " << h-1 << " / " << block_stop - << std::flush; + << "\r" << std::flush; } if (opt_verify) @@ -758,7 +759,12 @@ int main(int argc, char* argv[]) { core.disable_dns_checkpoints(true); - if (!core.init(vm, NULL)) +#if defined(PER_BLOCK_CHECKPOINT) + const GetCheckpointsCallback& get_checkpoints = blocks::GetCheckpointsData; +#else + const GetCheckpointsCallback& get_checkpoints = nullptr; +#endif + if (!core.init(vm, nullptr, get_checkpoints)) { std::cerr << "Failed to initialize core" << ENDL; return 1; diff --git a/src/blockchain_utilities/blockchain_prune.cpp b/src/blockchain_utilities/blockchain_prune.cpp new file mode 100644 index 000000000..8e13f2c04 --- /dev/null +++ b/src/blockchain_utilities/blockchain_prune.cpp @@ -0,0 +1,663 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <array> +#include <lmdb.h> +#include <boost/algorithm/string.hpp> +#include "common/command_line.h" +#include "common/pruning.h" +#include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_core/blockchain.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/lmdb/db_lmdb.h" +#include "blockchain_db/db_types.h" +#include "version.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "bcutil" + +#define MDB_val_set(var, val) MDB_val var = {sizeof(val), (void *)&val} + +namespace po = boost::program_options; +using namespace epee; +using namespace cryptonote; + +static std::string db_path; + +// default to fast:1 +static uint64_t records_per_sync = 128; +static const size_t slack = 512 * 1024 * 1024; + +static std::error_code replace_file(const boost::filesystem::path& replacement_name, const boost::filesystem::path& replaced_name) +{ + std::error_code ec = tools::replace_file(replacement_name.string(), replaced_name.string()); + if (ec) + MERROR("Error renaming " << replacement_name << " to " << replaced_name << ": " << ec.message()); + return ec; +} + +static void open(MDB_env *&env, const boost::filesystem::path &path, uint64_t db_flags, bool readonly) +{ + int dbr; + int flags = 0; + + if (db_flags & DBF_FAST) + flags |= MDB_NOSYNC; + if (db_flags & DBF_FASTEST) + flags |= MDB_NOSYNC | MDB_WRITEMAP | MDB_MAPASYNC; + if (readonly) + flags |= MDB_RDONLY; + + dbr = mdb_env_create(&env); + if (dbr) throw std::runtime_error("Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); + dbr = mdb_env_set_maxdbs(env, 32); + if (dbr) throw std::runtime_error("Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); + dbr = mdb_env_open(env, path.string().c_str(), flags, 0664); + if (dbr) throw std::runtime_error("Failed to open database file '" + + path.string() + "': " + std::string(mdb_strerror(dbr))); +} + +static void close(MDB_env *env) +{ + mdb_env_close(env); +} + +static void add_size(MDB_env *env, uint64_t bytes) +{ + try + { + boost::filesystem::path path(db_path); + boost::filesystem::space_info si = boost::filesystem::space(path); + if(si.available < bytes) + { + MERROR("!! WARNING: Insufficient free space to extend database !!: " << + (si.available >> 20L) << " MB available, " << (bytes >> 20L) << " MB needed"); + return; + } + } + catch(...) + { + // print something but proceed. + MWARNING("Unable to query free disk space."); + } + + MDB_envinfo mei; + mdb_env_info(env, &mei); + MDB_stat mst; + mdb_env_stat(env, &mst); + + uint64_t new_mapsize = (uint64_t)mei.me_mapsize + bytes; + new_mapsize += (new_mapsize % mst.ms_psize); + + int result = mdb_env_set_mapsize(env, new_mapsize); + if (result) + throw std::runtime_error("Failed to set new mapsize to " + std::to_string(new_mapsize) + ": " + std::string(mdb_strerror(result))); + + MGINFO("LMDB Mapsize increased." << " Old: " << mei.me_mapsize / (1024 * 1024) << "MiB" << ", New: " << new_mapsize / (1024 * 1024) << "MiB"); +} + +static void check_resize(MDB_env *env, size_t bytes) +{ + MDB_envinfo mei; + MDB_stat mst; + + mdb_env_info(env, &mei); + mdb_env_stat(env, &mst); + + uint64_t size_used = mst.ms_psize * mei.me_last_pgno; + if (size_used + bytes + slack >= mei.me_mapsize) + add_size(env, size_used + bytes + 2 * slack - mei.me_mapsize); +} + +static bool resize_point(size_t nrecords, MDB_env *env, MDB_txn **txn, size_t &bytes) +{ + if (nrecords % records_per_sync && bytes <= slack / 2) + return false; + int dbr = mdb_txn_commit(*txn); + if (dbr) throw std::runtime_error("Failed to commit txn: " + std::string(mdb_strerror(dbr))); + check_resize(env, bytes); + dbr = mdb_txn_begin(env, NULL, 0, txn); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + bytes = 0; + return true; +} + +static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned int flags, unsigned int putflags, int (*cmp)(const MDB_val*, const MDB_val*)=0) +{ + MDB_dbi dbi0, dbi1; + MDB_txn *txn0, *txn1; + MDB_cursor *cur0, *cur1; + bool tx_active0 = false, tx_active1 = false; + int dbr; + + MINFO("Copying " << table); + + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){ + if (tx_active1) mdb_txn_abort(txn1); + if (tx_active0) mdb_txn_abort(txn0); + }); + + dbr = mdb_txn_begin(env0, NULL, MDB_RDONLY, &txn0); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + tx_active0 = true; + dbr = mdb_txn_begin(env1, NULL, 0, &txn1); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + tx_active1 = true; + + dbr = mdb_dbi_open(txn0, table, flags, &dbi0); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + if (cmp) + ((flags & MDB_DUPSORT) ? mdb_set_dupsort : mdb_set_compare)(txn0, dbi0, cmp); + + dbr = mdb_dbi_open(txn1, table, flags, &dbi1); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + if (cmp) + ((flags & MDB_DUPSORT) ? mdb_set_dupsort : mdb_set_compare)(txn1, dbi1, cmp); + + dbr = mdb_txn_commit(txn1); + if (dbr) throw std::runtime_error("Failed to commit txn: " + std::string(mdb_strerror(dbr))); + tx_active1 = false; + MDB_stat stats; + dbr = mdb_env_stat(env0, &stats); + if (dbr) throw std::runtime_error("Failed to stat " + std::string(table) + " LMDB table: " + std::string(mdb_strerror(dbr))); + check_resize(env1, (stats.ms_branch_pages + stats.ms_overflow_pages + stats.ms_leaf_pages) * stats.ms_psize); + dbr = mdb_txn_begin(env1, NULL, 0, &txn1); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + tx_active1 = true; + + dbr = mdb_drop(txn1, dbi1, 0); + if (dbr) throw std::runtime_error("Failed to empty " + std::string(table) + " LMDB table: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_cursor_open(txn0, dbi0, &cur0); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_open(txn1, dbi1, &cur1); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + + MDB_val k; + MDB_val v; + MDB_cursor_op op = MDB_FIRST; + size_t nrecords = 0, bytes = 0; + while (1) + { + int ret = mdb_cursor_get(cur0, &k, &v, op); + op = MDB_NEXT; + if (ret == MDB_NOTFOUND) + break; + if (ret) + throw std::runtime_error("Failed to enumerate " + std::string(table) + " records: " + std::string(mdb_strerror(ret))); + + bytes += k.mv_size + v.mv_size; + if (resize_point(++nrecords, env1, &txn1, bytes)) + { + dbr = mdb_cursor_open(txn1, dbi1, &cur1); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + } + + ret = mdb_cursor_put(cur1, &k, &v, putflags); + if (ret) + throw std::runtime_error("Failed to write " + std::string(table) + " record: " + std::string(mdb_strerror(ret))); + } + + mdb_cursor_close(cur1); + mdb_cursor_close(cur0); + mdb_txn_commit(txn1); + tx_active1 = false; + mdb_txn_commit(txn0); + tx_active0 = false; + mdb_dbi_close(env1, dbi1); + mdb_dbi_close(env0, dbi0); +} + +static bool is_v1_tx(MDB_cursor *c_txs_pruned, MDB_val *tx_id) +{ + MDB_val v; + int ret = mdb_cursor_get(c_txs_pruned, tx_id, &v, MDB_SET); + if (ret) + throw std::runtime_error("Failed to find transaction pruned data: " + std::string(mdb_strerror(ret))); + if (v.mv_size == 0) + throw std::runtime_error("Invalid transaction pruned data"); + return cryptonote::is_v1_tx(cryptonote::blobdata_ref{(const char*)v.mv_data, v.mv_size}); +} + +static void prune(MDB_env *env0, MDB_env *env1) +{ + MDB_dbi dbi0_blocks, dbi0_txs_pruned, dbi0_txs_prunable, dbi0_tx_indices, dbi1_txs_prunable, dbi1_txs_prunable_tip, dbi1_properties; + MDB_txn *txn0, *txn1; + MDB_cursor *cur0_txs_pruned, *cur0_txs_prunable, *cur0_tx_indices, *cur1_txs_prunable, *cur1_txs_prunable_tip; + bool tx_active0 = false, tx_active1 = false; + int dbr; + + MGINFO("Creating pruned txs_prunable"); + + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){ + if (tx_active1) mdb_txn_abort(txn1); + if (tx_active0) mdb_txn_abort(txn0); + }); + + dbr = mdb_txn_begin(env0, NULL, MDB_RDONLY, &txn0); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + tx_active0 = true; + dbr = mdb_txn_begin(env1, NULL, 0, &txn1); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + tx_active1 = true; + + dbr = mdb_dbi_open(txn0, "txs_pruned", MDB_INTEGERKEY, &dbi0_txs_pruned); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_compare(txn0, dbi0_txs_pruned, BlockchainLMDB::compare_uint64); + dbr = mdb_cursor_open(txn0, dbi0_txs_pruned, &cur0_txs_pruned); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_dbi_open(txn0, "txs_prunable", MDB_INTEGERKEY, &dbi0_txs_prunable); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_compare(txn0, dbi0_txs_prunable, BlockchainLMDB::compare_uint64); + dbr = mdb_cursor_open(txn0, dbi0_txs_prunable, &cur0_txs_prunable); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_dbi_open(txn0, "tx_indices", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi0_tx_indices); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_dupsort(txn0, dbi0_tx_indices, BlockchainLMDB::compare_hash32); + dbr = mdb_cursor_open(txn0, dbi0_tx_indices, &cur0_tx_indices); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_dbi_open(txn1, "txs_prunable", MDB_INTEGERKEY, &dbi1_txs_prunable); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_compare(txn1, dbi1_txs_prunable, BlockchainLMDB::compare_uint64); + dbr = mdb_cursor_open(txn1, dbi1_txs_prunable, &cur1_txs_prunable); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_dbi_open(txn1, "txs_prunable_tip", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi1_txs_prunable_tip); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_dupsort(txn1, dbi1_txs_prunable_tip, BlockchainLMDB::compare_uint64); + dbr = mdb_cursor_open(txn1, dbi1_txs_prunable_tip, &cur1_txs_prunable_tip); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_drop(txn1, dbi1_txs_prunable, 0); + if (dbr) throw std::runtime_error("Failed to empty LMDB table: " + std::string(mdb_strerror(dbr))); + dbr = mdb_drop(txn1, dbi1_txs_prunable_tip, 0); + if (dbr) throw std::runtime_error("Failed to empty LMDB table: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_dbi_open(txn1, "properties", 0, &dbi1_properties); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + + MDB_val k, v; + uint32_t pruning_seed = tools::make_pruning_seed(tools::get_random_stripe(), CRYPTONOTE_PRUNING_LOG_STRIPES); + static char pruning_seed_key[] = "pruning_seed"; + k.mv_data = pruning_seed_key; + k.mv_size = strlen("pruning_seed") + 1; + v.mv_data = (void*)&pruning_seed; + v.mv_size = sizeof(pruning_seed); + dbr = mdb_put(txn1, dbi1_properties, &k, &v, 0); + if (dbr) throw std::runtime_error("Failed to save pruning seed: " + std::string(mdb_strerror(dbr))); + + MDB_stat stats; + dbr = mdb_dbi_open(txn0, "blocks", 0, &dbi0_blocks); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + dbr = mdb_stat(txn0, dbi0_blocks, &stats); + if (dbr) throw std::runtime_error("Failed to query size of blocks: " + std::string(mdb_strerror(dbr))); + mdb_dbi_close(env0, dbi0_blocks); + const uint64_t blockchain_height = stats.ms_entries; + size_t nrecords = 0, bytes = 0; + + MDB_cursor_op op = MDB_FIRST; + while (1) + { + int ret = mdb_cursor_get(cur0_tx_indices, &k, &v, op); + op = MDB_NEXT; + if (ret == MDB_NOTFOUND) + break; + if (ret) throw std::runtime_error("Failed to enumerate records: " + std::string(mdb_strerror(ret))); + + const txindex *ti = (const txindex*)v.mv_data; + const uint64_t block_height = ti->data.block_id; + MDB_val_set(kk, ti->data.tx_id); + if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS >= blockchain_height) + { + MDEBUG(block_height << "/" << blockchain_height << " is in tip"); + MDB_val_set(vv, block_height); + dbr = mdb_cursor_put(cur1_txs_prunable_tip, &kk, &vv, 0); + if (dbr) throw std::runtime_error("Failed to write prunable tx tip data: " + std::string(mdb_strerror(dbr))); + bytes += kk.mv_size + vv.mv_size; + } + if (tools::has_unpruned_block(block_height, blockchain_height, pruning_seed) || is_v1_tx(cur0_txs_pruned, &kk)) + { + MDB_val vv; + dbr = mdb_cursor_get(cur0_txs_prunable, &kk, &vv, MDB_SET); + if (dbr) throw std::runtime_error("Failed to read prunable tx data: " + std::string(mdb_strerror(dbr))); + bytes += kk.mv_size + vv.mv_size; + if (resize_point(++nrecords, env1, &txn1, bytes)) + { + dbr = mdb_cursor_open(txn1, dbi1_txs_prunable, &cur1_txs_prunable); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_open(txn1, dbi1_txs_prunable_tip, &cur1_txs_prunable_tip); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + } + dbr = mdb_cursor_put(cur1_txs_prunable, &kk, &vv, 0); + if (dbr) throw std::runtime_error("Failed to write prunable tx data: " + std::string(mdb_strerror(dbr))); + } + else + { + MDEBUG("" << block_height << "/" << blockchain_height << " should be pruned, dropping"); + } + } + + mdb_cursor_close(cur1_txs_prunable_tip); + mdb_cursor_close(cur1_txs_prunable); + mdb_cursor_close(cur0_txs_prunable); + mdb_cursor_close(cur0_txs_pruned); + mdb_cursor_close(cur0_tx_indices); + mdb_txn_commit(txn1); + tx_active1 = false; + mdb_txn_commit(txn0); + tx_active0 = false; + mdb_dbi_close(env1, dbi1_properties); + mdb_dbi_close(env1, dbi1_txs_prunable_tip); + mdb_dbi_close(env1, dbi1_txs_prunable); + mdb_dbi_close(env0, dbi0_txs_prunable); + mdb_dbi_close(env0, dbi0_txs_pruned); + mdb_dbi_close(env0, dbi0_tx_indices); +} + +static bool parse_db_sync_mode(std::string db_sync_mode, uint64_t &db_flags) +{ + std::vector<std::string> options; + boost::trim(db_sync_mode); + boost::split(options, db_sync_mode, boost::is_any_of(" :")); + + for(const auto &option : options) + MDEBUG("option: " << option); + + // default to fast:async:1 + uint64_t DEFAULT_FLAGS = DBF_FAST; + + db_flags = 0; + + if(options.size() == 0) + { + // default to fast:async:1 + db_flags = DEFAULT_FLAGS; + } + + bool safemode = false; + if(options.size() >= 1) + { + if(options[0] == "safe") + { + safemode = true; + db_flags = DBF_SAFE; + } + else if(options[0] == "fast") + { + db_flags = DBF_FAST; + } + else if(options[0] == "fastest") + { + db_flags = DBF_FASTEST; + records_per_sync = 1000; // default to fastest:async:1000 + } + else + return false; + } + + if(options.size() >= 2 && !safemode) + { + char *endptr; + uint64_t bps = strtoull(options[1].c_str(), &endptr, 0); + if (*endptr != '\0') + return false; + records_per_sync = bps; + } + + return true; +} + +int main(int argc, char* argv[]) +{ + TRY_ENTRY(); + + epee::string_tools::set_module_name_and_folder(argv[0]); + + std::string default_db_type = "lmdb"; + + std::string available_dbs = cryptonote::blockchain_db_types(", "); + available_dbs = "available: " + available_dbs; + + uint32_t log_level = 0; + + tools::on_startup(); + + boost::filesystem::path output_file_path; + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""}; + const command_line::arg_descriptor<std::string> arg_database = { + "database", available_dbs.c_str(), default_db_type + }; + const command_line::arg_descriptor<std::string> arg_db_sync_mode = { + "db-sync-mode" + , "Specify sync option, using format [safe|fast|fastest]:[nrecords_per_sync]." + , "fast:1000" + }; + const command_line::arg_descriptor<bool> arg_copy_pruned_database = {"copy-pruned-database", "Copy database anyway if already pruned"}; + + command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_database); + command_line::add_arg(desc_cmd_sett, arg_db_sync_mode); + command_line::add_arg(desc_cmd_sett, arg_copy_pruned_database); + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + auto parser = po::command_line_parser(argc, argv).options(desc_options); + po::store(parser.run(), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + mlog_configure(mlog_get_default_log_path("monero-blockchain-prune.log"), true); + if (!command_line::is_arg_defaulted(vm, arg_log_level)) + mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); + else + mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str()); + + MINFO("Starting..."); + + bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); + bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); + network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; + bool opt_copy_pruned_database = command_line::get_arg(vm, arg_copy_pruned_database); + std::string data_dir = command_line::get_arg(vm, cryptonote::arg_data_dir); + while (boost::ends_with(data_dir, "/") || boost::ends_with(data_dir, "\\")) + data_dir.pop_back(); + + std::string db_type = command_line::get_arg(vm, arg_database); + if (!cryptonote::blockchain_valid_db_type(db_type)) + { + MERROR("Invalid database type: " << db_type); + return 1; + } + if (db_type != "lmdb") + { + MERROR("Unsupported database type: " << db_type << ". Only lmdb is supported"); + return 1; + } + + std::string db_sync_mode = command_line::get_arg(vm, arg_db_sync_mode); + uint64_t db_flags = 0; + if (!parse_db_sync_mode(db_sync_mode, db_flags)) + { + MERROR("Invalid db sync mode: " << db_sync_mode); + return 1; + } + + // If we wanted to use the memory pool, we would set up a fake_core. + + // Use Blockchain instead of lower-level BlockchainDB for two reasons: + // 1. Blockchain has the init() method for easy setup + // 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash() + // + // cannot match blockchain_storage setup above with just one line, + // e.g. + // Blockchain* core_storage = new Blockchain(NULL); + // because unlike blockchain_storage constructor, which takes a pointer to + // tx_memory_pool, Blockchain's constructor takes tx_memory_pool object. + MINFO("Initializing source blockchain (BlockchainDB)"); + std::array<std::unique_ptr<Blockchain>, 2> core_storage; + Blockchain *blockchain = NULL; + tx_memory_pool m_mempool(*blockchain); + boost::filesystem::path paths[2]; + bool already_pruned = false; + for (size_t n = 0; n < core_storage.size(); ++n) + { + core_storage[n].reset(new Blockchain(m_mempool)); + + BlockchainDB* db = new_db(db_type); + if (db == NULL) + { + MERROR("Attempted to use non-existent database type: " << db_type); + throw std::runtime_error("Attempting to use non-existent database type"); + } + MDEBUG("database: " << db_type); + + if (n == 1) + { + paths[1] = boost::filesystem::path(data_dir) / (db->get_db_name() + "-pruned"); + if (boost::filesystem::exists(paths[1])) + { + if (!boost::filesystem::is_directory(paths[1])) + { + MERROR("LMDB needs a directory path, but a file was passed: " << paths[1].string()); + return 1; + } + } + else + { + if (!boost::filesystem::create_directories(paths[1])) + { + MERROR("Failed to create directory: " << paths[1].string()); + return 1; + } + } + db_path = paths[1].string(); + } + else + { + paths[0] = boost::filesystem::path(data_dir) / db->get_db_name(); + } + + MINFO("Loading blockchain from folder " << paths[n] << " ..."); + + try + { + db->open(paths[n].string(), n == 0 ? DBF_RDONLY : 0); + } + catch (const std::exception& e) + { + MERROR("Error opening database: " << e.what()); + return 1; + } + r = core_storage[n]->init(db, net_type); + + std::string source_dest = n == 0 ? "source" : "pruned"; + CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize " << source_dest << " blockchain storage"); + MINFO(source_dest << " blockchain storage initialized OK"); + if (n == 0 && core_storage[0]->get_blockchain_pruning_seed()) + { + if (!opt_copy_pruned_database) + { + MERROR("Blockchain is already pruned, use --" << arg_copy_pruned_database.name << " to copy it anyway"); + return 1; + } + already_pruned = true; + } + } + core_storage[0]->deinit(); + core_storage[0].reset(NULL); + core_storage[1]->deinit(); + core_storage[1].reset(NULL); + + MINFO("Pruning..."); + MDB_env *env0 = NULL, *env1 = NULL; + open(env0, paths[0], db_flags, true); + open(env1, paths[1], db_flags, false); + copy_table(env0, env1, "blocks", MDB_INTEGERKEY, MDB_APPEND); + copy_table(env0, env1, "block_info", MDB_INTEGERKEY | MDB_DUPSORT| MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64); + copy_table(env0, env1, "block_heights", MDB_INTEGERKEY | MDB_DUPSORT| MDB_DUPFIXED, 0, BlockchainLMDB::compare_hash32); + //copy_table(env0, env1, "txs", MDB_INTEGERKEY); + copy_table(env0, env1, "txs_pruned", MDB_INTEGERKEY, MDB_APPEND); + copy_table(env0, env1, "txs_prunable_hash", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPEND); + // not copied: prunable, prunable_tip + copy_table(env0, env1, "tx_indices", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, 0, BlockchainLMDB::compare_hash32); + copy_table(env0, env1, "tx_outputs", MDB_INTEGERKEY, MDB_APPEND); + copy_table(env0, env1, "output_txs", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64); + copy_table(env0, env1, "output_amounts", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64); + copy_table(env0, env1, "spent_keys", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_NODUPDATA, BlockchainLMDB::compare_hash32); + copy_table(env0, env1, "txpool_meta", 0, MDB_NODUPDATA, BlockchainLMDB::compare_hash32); + copy_table(env0, env1, "txpool_blob", 0, MDB_NODUPDATA, BlockchainLMDB::compare_hash32); + copy_table(env0, env1, "hf_versions", MDB_INTEGERKEY, MDB_APPEND); + copy_table(env0, env1, "properties", 0, 0, BlockchainLMDB::compare_string); + if (already_pruned) + { + copy_table(env0, env1, "txs_prunable", MDB_INTEGERKEY, MDB_APPEND, BlockchainLMDB::compare_uint64); + copy_table(env0, env1, "txs_prunable_tip", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_NODUPDATA, BlockchainLMDB::compare_uint64); + } + else + { + prune(env0, env1); + } + close(env1); + close(env0); + + MINFO("Swapping databases, pre-pruning blockchain will be left in " << paths[0].string() + "-old and can be removed if desired"); + if (replace_file(paths[0].string(), paths[0].string() + "-old") || replace_file(paths[1].string(), paths[0].string())) + { + MERROR("Blockchain pruned OK, but renaming failed"); + return 1; + } + + MINFO("Blockchain pruned OK"); + return 0; + + CATCH_ENTRY("Pruning error", 1); +} diff --git a/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp b/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp new file mode 100644 index 000000000..f6136c1ba --- /dev/null +++ b/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp @@ -0,0 +1,305 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <boost/algorithm/string.hpp> +#include "common/command_line.h" +#include "serialization/crypto.h" +#include "cryptonote_core/tx_pool.h" +#include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_core/blockchain.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/db_types.h" +#include "version.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "bcutil" + +namespace po = boost::program_options; +using namespace epee; +using namespace cryptonote; + +static std::map<uint64_t, uint64_t> load_outputs(const std::string &filename) +{ + std::map<uint64_t, uint64_t> outputs; + uint64_t amount = std::numeric_limits<uint64_t>::max(); + FILE *f; + + f = fopen(filename.c_str(), "r"); + if (!f) + { + MERROR("Failed to load outputs from " << filename << ": " << strerror(errno)); + return {}; + } + while (1) + { + char s[256]; + if (!fgets(s, sizeof(s), f)) + break; + if (feof(f)) + break; + const size_t len = strlen(s); + if (len > 0 && s[len - 1] == '\n') + s[len - 1] = 0; + if (!s[0]) + continue; + std::pair<uint64_t, uint64_t> output; + uint64_t offset, num_offsets; + if (sscanf(s, "@%" PRIu64, &amount) == 1) + { + continue; + } + if (amount == std::numeric_limits<uint64_t>::max()) + { + MERROR("Bad format in " << filename); + continue; + } + if (sscanf(s, "%" PRIu64 "*%" PRIu64, &offset, &num_offsets) == 2 && num_offsets < std::numeric_limits<uint64_t>::max() - offset) + { + outputs[amount] += num_offsets; + } + else if (sscanf(s, "%" PRIu64, &offset) == 1) + { + outputs[amount] += 1; + } + else + { + MERROR("Bad format in " << filename); + continue; + } + } + fclose(f); + return outputs; +} + +int main(int argc, char* argv[]) +{ + TRY_ENTRY(); + + epee::string_tools::set_module_name_and_folder(argv[0]); + + std::string default_db_type = "lmdb"; + + std::string available_dbs = cryptonote::blockchain_db_types(", "); + available_dbs = "available: " + available_dbs; + + uint32_t log_level = 0; + + tools::on_startup(); + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""}; + const command_line::arg_descriptor<std::string> arg_database = { + "database", available_dbs.c_str(), default_db_type + }; + const command_line::arg_descriptor<bool> arg_verbose = {"verbose", "Verbose output", false}; + const command_line::arg_descriptor<bool> arg_dry_run = {"dry-run", "Do not actually prune", false}; + const command_line::arg_descriptor<std::string> arg_input = {"input", "Path to the known spent outputs file"}; + + command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_database); + command_line::add_arg(desc_cmd_sett, arg_verbose); + command_line::add_arg(desc_cmd_sett, arg_dry_run); + command_line::add_arg(desc_cmd_sett, arg_input); + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + auto parser = po::command_line_parser(argc, argv).options(desc_options); + po::store(parser.run(), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + mlog_configure(mlog_get_default_log_path("monero-blockchain-prune-known-spent-data.log"), true); + if (!command_line::is_arg_defaulted(vm, arg_log_level)) + mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); + else + mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str()); + + LOG_PRINT_L0("Starting..."); + + std::string opt_data_dir = command_line::get_arg(vm, cryptonote::arg_data_dir); + bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); + bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); + network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; + bool opt_verbose = command_line::get_arg(vm, arg_verbose); + bool opt_dry_run = command_line::get_arg(vm, arg_dry_run); + + std::string db_type = command_line::get_arg(vm, arg_database); + if (!cryptonote::blockchain_valid_db_type(db_type)) + { + std::cerr << "Invalid database type: " << db_type << std::endl; + return 1; + } + + const std::string input = command_line::get_arg(vm, arg_input); + + LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); + std::unique_ptr<Blockchain> core_storage; + tx_memory_pool m_mempool(*core_storage); + core_storage.reset(new Blockchain(m_mempool)); + BlockchainDB *db = new_db(db_type); + if (db == NULL) + { + LOG_ERROR("Attempted to use non-existent database type: " << db_type); + throw std::runtime_error("Attempting to use non-existent database type"); + } + LOG_PRINT_L0("database: " << db_type); + + const std::string filename = (boost::filesystem::path(opt_data_dir) / db->get_db_name()).string(); + LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); + + try + { + db->open(filename, 0); + } + catch (const std::exception& e) + { + LOG_PRINT_L0("Error opening database: " << e.what()); + return 1; + } + r = core_storage->init(db, net_type); + + CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage"); + LOG_PRINT_L0("Source blockchain storage initialized OK"); + + std::map<uint64_t, uint64_t> known_spent_outputs; + if (input.empty()) + { + std::map<uint64_t, std::pair<uint64_t, uint64_t>> outputs; + + LOG_PRINT_L0("Scanning for known spent data..."); + db->for_all_transactions([&](const crypto::hash &txid, const cryptonote::transaction &tx){ + const bool miner_tx = tx.vin.size() == 1 && tx.vin[0].type() == typeid(txin_gen); + for (const auto &in: tx.vin) + { + if (in.type() != typeid(txin_to_key)) + continue; + const auto &txin = boost::get<txin_to_key>(in); + if (txin.amount == 0) + continue; + + outputs[txin.amount].second++; + } + + for (const auto &out: tx.vout) + { + uint64_t amount = out.amount; + if (miner_tx && tx.version >= 2) + amount = 0; + if (amount == 0) + continue; + if (out.target.type() != typeid(txout_to_key)) + continue; + + outputs[amount].first++; + } + return true; + }, true); + + for (const auto &i: outputs) + { + known_spent_outputs[i.first] = i.second.second; + } + } + else + { + LOG_PRINT_L0("Loading known spent data..."); + known_spent_outputs = load_outputs(input); + } + + LOG_PRINT_L0("Pruning known spent data..."); + + bool stop_requested = false; + tools::signal_handler::install([&stop_requested](int type) { + stop_requested = true; + }); + + db->batch_start(); + + size_t num_total_outputs = 0, num_prunable_outputs = 0, num_known_spent_outputs = 0, num_eligible_outputs = 0, num_eligible_known_spent_outputs = 0; + for (auto i = known_spent_outputs.begin(); i != known_spent_outputs.end(); ++i) + { + uint64_t num_outputs = db->get_num_outputs(i->first); + num_total_outputs += num_outputs; + num_known_spent_outputs += i->second; + if (i->first == 0 || is_valid_decomposed_amount(i->first)) + { + if (opt_verbose) + MINFO("Ignoring output value " << i->first << ", with " << num_outputs << " outputs"); + continue; + } + num_eligible_outputs += num_outputs; + num_eligible_known_spent_outputs += i->second; + if (opt_verbose) + MINFO(i->first << ": " << i->second << "/" << num_outputs); + if (num_outputs > i->second) + continue; + if (num_outputs && num_outputs < i->second) + { + MERROR("More outputs are spent than known for amount " << i->first << ", not touching"); + continue; + } + if (opt_verbose) + MINFO("Pruning data for " << num_outputs << " outputs"); + if (!opt_dry_run) + db->prune_outputs(i->first); + num_prunable_outputs += i->second; + } + + db->batch_stop(); + + MINFO("Total outputs: " << num_total_outputs); + MINFO("Known spent outputs: " << num_known_spent_outputs); + MINFO("Eligible outputs: " << num_eligible_outputs); + MINFO("Eligible known spent outputs: " << num_eligible_known_spent_outputs); + MINFO("Prunable outputs: " << num_prunable_outputs); + + LOG_PRINT_L0("Blockchain known spent data pruned OK"); + core_storage->deinit(); + return 0; + + CATCH_ENTRY("Error", 1); +} diff --git a/src/blockchain_utilities/blockchain_stats.cpp b/src/blockchain_utilities/blockchain_stats.cpp new file mode 100644 index 000000000..aae8f333b --- /dev/null +++ b/src/blockchain_utilities/blockchain_stats.cpp @@ -0,0 +1,337 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <boost/algorithm/string.hpp> +#include "common/command_line.h" +#include "common/varint.h" +#include "cryptonote_basic/cryptonote_boost_serialization.h" +#include "cryptonote_core/tx_pool.h" +#include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_core/blockchain.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/db_types.h" +#include "version.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "bcutil" + +namespace po = boost::program_options; +using namespace epee; +using namespace cryptonote; + +static bool stop_requested = false; + +int main(int argc, char* argv[]) +{ + TRY_ENTRY(); + + epee::string_tools::set_module_name_and_folder(argv[0]); + + std::string default_db_type = "lmdb"; + + std::string available_dbs = cryptonote::blockchain_db_types(", "); + available_dbs = "available: " + available_dbs; + + uint32_t log_level = 0; + uint64_t block_start = 0; + uint64_t block_stop = 0; + + tools::on_startup(); + + boost::filesystem::path output_file_path; + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""}; + const command_line::arg_descriptor<std::string> arg_database = { + "database", available_dbs.c_str(), default_db_type + }; + const command_line::arg_descriptor<uint64_t> arg_block_start = {"block-start", "start at block number", block_start}; + const command_line::arg_descriptor<uint64_t> arg_block_stop = {"block-stop", "Stop at block number", block_stop}; + const command_line::arg_descriptor<bool> arg_inputs = {"with-inputs", "with input stats", false}; + const command_line::arg_descriptor<bool> arg_outputs = {"with-outputs", "with output stats", false}; + const command_line::arg_descriptor<bool> arg_ringsize = {"with-ringsize", "with ringsize stats", false}; + const command_line::arg_descriptor<bool> arg_hours = {"with-hours", "with txns per hour", false}; + + command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_database); + command_line::add_arg(desc_cmd_sett, arg_block_start); + command_line::add_arg(desc_cmd_sett, arg_block_stop); + command_line::add_arg(desc_cmd_sett, arg_inputs); + command_line::add_arg(desc_cmd_sett, arg_outputs); + command_line::add_arg(desc_cmd_sett, arg_ringsize); + command_line::add_arg(desc_cmd_sett, arg_hours); + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + auto parser = po::command_line_parser(argc, argv).options(desc_options); + po::store(parser.run(), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + mlog_configure(mlog_get_default_log_path("monero-blockchain-stats.log"), true); + if (!command_line::is_arg_defaulted(vm, arg_log_level)) + mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); + else + mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str()); + + LOG_PRINT_L0("Starting..."); + + std::string opt_data_dir = command_line::get_arg(vm, cryptonote::arg_data_dir); + bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); + bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); + network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; + block_start = command_line::get_arg(vm, arg_block_start); + block_stop = command_line::get_arg(vm, arg_block_stop); + bool do_inputs = command_line::get_arg(vm, arg_inputs); + bool do_outputs = command_line::get_arg(vm, arg_outputs); + bool do_ringsize = command_line::get_arg(vm, arg_ringsize); + bool do_hours = command_line::get_arg(vm, arg_hours); + + std::string db_type = command_line::get_arg(vm, arg_database); + if (!cryptonote::blockchain_valid_db_type(db_type)) + { + std::cerr << "Invalid database type: " << db_type << std::endl; + return 1; + } + + LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); + std::unique_ptr<Blockchain> core_storage; + tx_memory_pool m_mempool(*core_storage); + core_storage.reset(new Blockchain(m_mempool)); + BlockchainDB *db = new_db(db_type); + if (db == NULL) + { + LOG_ERROR("Attempted to use non-existent database type: " << db_type); + throw std::runtime_error("Attempting to use non-existent database type"); + } + LOG_PRINT_L0("database: " << db_type); + + const std::string filename = (boost::filesystem::path(opt_data_dir) / db->get_db_name()).string(); + LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); + + try + { + db->open(filename, DBF_RDONLY); + } + catch (const std::exception& e) + { + LOG_PRINT_L0("Error opening database: " << e.what()); + return 1; + } + r = core_storage->init(db, net_type); + + CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage"); + LOG_PRINT_L0("Source blockchain storage initialized OK"); + + tools::signal_handler::install([](int type) { + stop_requested = true; + }); + + const uint64_t db_height = db->height(); + if (!block_stop) + block_stop = db_height; + MINFO("Starting from height " << block_start << ", stopping at height " << block_stop); + +/* + * The default output can be plotted with GnuPlot using these commands: +set key autotitle columnhead +set title "Monero Blockchain Growth" +set timefmt "%Y-%m-%d" +set xdata time +set xrange ["2014-04-17":*] +set format x "%Y-%m-%d" +set yrange [0:*] +set y2range [0:*] +set ylabel "Txs/Day" +set y2label "Bytes" +set y2tics nomirror +plot 'stats.csv' index "DATA" using (timecolumn(1,"%Y-%m-%d")):4 with lines, '' using (timecolumn(1,"%Y-%m-%d")):7 axes x1y2 with lines + */ + + // spit out a comment that GnuPlot can use as an index + std::cout << ENDL << "# DATA" << ENDL; + std::cout << "Date\tBlocks/day\tBlocks\tTxs/Day\tTxs\tBytes/Day\tBytes"; + if (do_inputs) + std::cout << "\tInMin\tInMax\tInAvg"; + if (do_outputs) + std::cout << "\tOutMin\tOutMax\tOutAvg"; + if (do_ringsize) + std::cout << "\tRingMin\tRingMax\tRingAvg"; + if (do_hours) { + char buf[8]; + unsigned int i; + for (i=0; i<24; i++) { + sprintf(buf, "\t%02d:00", i); + std::cout << buf; + } + } + std::cout << ENDL; + + struct tm prevtm = {0}, currtm; + uint64_t prevsz = 0, currsz = 0; + uint64_t prevtxs = 0, currtxs = 0; + uint64_t currblks = 0; + uint64_t totins = 0, totouts = 0, totrings = 0; + uint32_t minins = 10, maxins = 0; + uint32_t minouts = 10, maxouts = 0; + uint32_t minrings = 50, maxrings = 0; + uint32_t io, tottxs = 0; + uint32_t txhr[24] = {0}; + unsigned int i; + + for (uint64_t h = block_start; h < block_stop; ++h) + { + cryptonote::blobdata bd = db->get_block_blob_from_height(h); + cryptonote::block blk; + if (!cryptonote::parse_and_validate_block_from_blob(bd, blk)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + time_t tt = blk.timestamp; + char timebuf[64]; + epee::misc_utils::get_gmt_time(tt, currtm); + if (!prevtm.tm_year) + prevtm = currtm; + // catch change of day + if (currtm.tm_mday > prevtm.tm_mday || (currtm.tm_mday == 1 && prevtm.tm_mday > 27)) + { + // check for timestamp fudging around month ends + if (prevtm.tm_mday == 1 && currtm.tm_mday > 27) + goto skip; + strftime(timebuf, sizeof(timebuf), "%Y-%m-%d", &prevtm); + prevtm = currtm; + std::cout << timebuf << "\t" << currblks << "\t" << h << "\t" << currtxs << "\t" << prevtxs + currtxs << "\t" << currsz << "\t" << prevsz + currsz; + prevsz += currsz; + currsz = 0; + currblks = 0; + prevtxs += currtxs; + currtxs = 0; + if (!tottxs) + tottxs = 1; + if (do_inputs) { + std::cout << "\t" << (maxins ? minins : 0) << "\t" << maxins << "\t" << totins / tottxs; + minins = 10; maxins = 0; totins = 0; + } + if (do_outputs) { + std::cout << "\t" << (maxouts ? minouts : 0) << "\t" << maxouts << "\t" << totouts / tottxs; + minouts = 10; maxouts = 0; totouts = 0; + } + if (do_ringsize) { + std::cout << "\t" << (maxrings ? minrings : 0) << "\t" << maxrings << "\t" << totrings / tottxs; + minrings = 50; maxrings = 0; totrings = 0; + } + tottxs = 0; + if (do_hours) { + for (i=0; i<24; i++) { + std::cout << "\t" << txhr[i]; + txhr[i] = 0; + } + } + std::cout << ENDL; + } +skip: + currsz += bd.size(); + for (const auto& tx_id : blk.tx_hashes) + { + if (tx_id == crypto::null_hash) + { + throw std::runtime_error("Aborting: tx == null_hash"); + } + if (!db->get_tx_blob(tx_id, bd)) + { + throw std::runtime_error("Aborting: tx not found"); + } + transaction tx; + if (!parse_and_validate_tx_from_blob(bd, tx)) + { + LOG_PRINT_L0("Bad txn from db"); + return 1; + } + currsz += bd.size(); + currtxs++; + if (do_hours) + txhr[currtm.tm_hour]++; + if (do_inputs) { + io = tx.vin.size(); + if (io < minins) + minins = io; + else if (io > maxins) + maxins = io; + totins += io; + } + if (do_ringsize) { + const cryptonote::txin_to_key& tx_in_to_key + = boost::get<cryptonote::txin_to_key>(tx.vin[0]); + io = tx_in_to_key.key_offsets.size(); + if (io < minrings) + minrings = io; + else if (io > maxrings) + maxrings = io; + totrings += io; + } + if (do_outputs) { + io = tx.vout.size(); + if (io < minouts) + minouts = io; + else if (io > maxouts) + maxouts = io; + totouts += io; + } + tottxs++; + } + currblks++; + + if (stop_requested) + break; + } + + core_storage->deinit(); + return 0; + + CATCH_ENTRY("Stats reporting error", 1); +} diff --git a/src/blockchain_utilities/bootstrap_file.cpp b/src/blockchain_utilities/bootstrap_file.cpp index beaad2abc..a8c46d661 100644 --- a/src/blockchain_utilities/bootstrap_file.cpp +++ b/src/blockchain_utilities/bootstrap_file.cpp @@ -304,7 +304,7 @@ bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem } if (m_cur_height % progress_interval == 0) { std::cout << refresh_string; - std::cout << "block " << m_cur_height << "/" << block_stop << std::flush; + std::cout << "block " << m_cur_height << "/" << block_stop << "\r" << std::flush; } } // NOTE: use of NUM_BLOCKS_PER_CHUNK is a placeholder in case multi-block chunks are later supported. @@ -479,7 +479,7 @@ uint64_t BootstrapFile::count_blocks(const std::string& import_file_path, std::s bytes_read += count_bytes(import_file, progress_interval, blocks, quit); h += blocks; std::cout << "\r" << "block height: " << h-1 << - " " << + " \r" << std::flush; // std::cout << refresh_string; diff --git a/src/blocks/CMakeLists.txt b/src/blocks/CMakeLists.txt index ebb5408cc..ff48af6dc 100644 --- a/src/blocks/CMakeLists.txt +++ b/src/blocks/CMakeLists.txt @@ -26,20 +26,23 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -if(APPLE) - add_library(blocks STATIC blockexports.c) - set_target_properties(blocks PROPERTIES LINKER_LANGUAGE C) -else() - if(LINUX_32) - add_custom_command(OUTPUT blocks.o MAIN_DEPENDENCY blocks.dat COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${CMAKE_LINKER} -m elf_i386 ${LD_RAW_FLAGS} -r -b binary -o ${CMAKE_CURRENT_BINARY_DIR}/blocks.o blocks.dat) - add_custom_command(OUTPUT testnet_blocks.o MAIN_DEPENDENCY testnet_blocks.dat COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${CMAKE_LINKER} -m elf_i386 ${LD_RAW_FLAGS} -r -b binary -o ${CMAKE_CURRENT_BINARY_DIR}/testnet_blocks.o testnet_blocks.dat) - add_custom_command(OUTPUT stagenet_blocks.o MAIN_DEPENDENCY stagenet_blocks.dat COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${CMAKE_LINKER} -m elf_i386 ${LD_RAW_FLAGS} -r -b binary -o ${CMAKE_CURRENT_BINARY_DIR}/stagenet_blocks.o stagenet_blocks.dat) - else() - add_custom_command(OUTPUT blocks.o MAIN_DEPENDENCY blocks.dat COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${CMAKE_LINKER} ${LD_RAW_FLAGS} -r -b binary -o ${CMAKE_CURRENT_BINARY_DIR}/blocks.o blocks.dat) - add_custom_command(OUTPUT testnet_blocks.o MAIN_DEPENDENCY testnet_blocks.dat COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${CMAKE_LINKER} ${LD_RAW_FLAGS} -r -b binary -o ${CMAKE_CURRENT_BINARY_DIR}/testnet_blocks.o testnet_blocks.dat) - add_custom_command(OUTPUT stagenet_blocks.o MAIN_DEPENDENCY stagenet_blocks.dat COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${CMAKE_LINKER} ${LD_RAW_FLAGS} -r -b binary -o ${CMAKE_CURRENT_BINARY_DIR}/stagenet_blocks.o stagenet_blocks.dat) - endif() - add_library(blocks STATIC blocks.o testnet_blocks.o stagenet_blocks.o blockexports.c) - set_target_properties(blocks PROPERTIES LINKER_LANGUAGE C) -endif() +set(GENERATED_SOURCES "") +foreach(BLOB_NAME checkpoints testnet_blocks stagenet_blocks) + set(OUTPUT_C_SOURCE "generated_${BLOB_NAME}.c") + list(APPEND GENERATED_SOURCES ${OUTPUT_C_SOURCE}) + set(INPUT_DAT_FILE "${BLOB_NAME}.dat") + add_custom_command( + OUTPUT ${OUTPUT_C_SOURCE} + MAIN_DEPENDENCY ${INPUT_DAT_FILE} + COMMAND + cd ${CMAKE_CURRENT_BINARY_DIR} && + echo "'#include\t<stddef.h>'" > ${OUTPUT_C_SOURCE} && + echo "'const\tunsigned\tchar\t${BLOB_NAME}[]={'" >> ${OUTPUT_C_SOURCE} && + od -v -An -tx1 ${CMAKE_CURRENT_SOURCE_DIR}/${INPUT_DAT_FILE} | sed -e "'s/[0-9a-fA-F]\\{1,\\}/0x&,/g'" -e "'$$s/.$$//'" >> ${OUTPUT_C_SOURCE} && + echo "'};'" >> ${OUTPUT_C_SOURCE} && + echo "'const\tsize_t\t${BLOB_NAME}_len\t=\tsizeof(${BLOB_NAME});'" >> ${OUTPUT_C_SOURCE} + ) +endforeach() + +add_library(blocks STATIC blocks.cpp ${GENERATED_SOURCES}) diff --git a/src/blocks/blockexports.c b/src/blocks/blockexports.c deleted file mode 100644 index 0154b0413..000000000 --- a/src/blocks/blockexports.c +++ /dev/null @@ -1,87 +0,0 @@ -#include <stddef.h> - -#if defined(__APPLE__) -#include <mach-o/getsect.h> -#ifdef BUILD_SHARED_LIBS -#if !defined(__LP64__) -const struct mach_header _mh_execute_header; -#else -const struct mach_header_64 _mh_execute_header; -#endif -#else -#if !defined(__LP64__) -extern const struct mach_header _mh_execute_header; -#else -extern const struct mach_header_64 _mh_execute_header; -#endif -#endif - -const unsigned char *get_blocks_dat_start(int testnet, int stagenet) -{ - size_t size; - if (testnet) - return getsectiondata(&_mh_execute_header, "__DATA", "__testnet_blocks_dat", &size); - else if (stagenet) - return getsectiondata(&_mh_execute_header, "__DATA", "__stagenet_blocks_dat", &size); - else - return getsectiondata(&_mh_execute_header, "__DATA", "__blocks_dat", &size); -} - -size_t get_blocks_dat_size(int testnet, int stagenet) -{ - size_t size; - if (testnet) - getsectiondata(&_mh_execute_header, "__DATA", "__testnet_blocks_dat", &size); - else if (stagenet) - getsectiondata(&_mh_execute_header, "__DATA", "__stagenet_blocks_dat", &size); - else - getsectiondata(&_mh_execute_header, "__DATA", "__blocks_dat", &size); - return size; -} - -#else - -#if defined(_WIN32) && !defined(_WIN64) -#define _binary_blocks_start binary_blocks_dat_start -#define _binary_blocks_end binary_blocks_dat_end -#define _binary_testnet_blocks_start binary_testnet_blocks_dat_start -#define _binary_testnet_blocks_end binary_testnet_blocks_dat_end -#define _binary_stagenet_blocks_start binary_stagenet_blocks_dat_start -#define _binary_stagenet_blocks_end binary_stagenet_blocks_dat_end -#else -#define _binary_blocks_start _binary_blocks_dat_start -#define _binary_blocks_end _binary_blocks_dat_end -#define _binary_testnet_blocks_start _binary_testnet_blocks_dat_start -#define _binary_testnet_blocks_end _binary_testnet_blocks_dat_end -#define _binary_stagenet_blocks_start _binary_stagenet_blocks_dat_start -#define _binary_stagenet_blocks_end _binary_stagenet_blocks_dat_end -#endif - -extern const unsigned char _binary_blocks_start[]; -extern const unsigned char _binary_blocks_end[]; -extern const unsigned char _binary_testnet_blocks_start[]; -extern const unsigned char _binary_testnet_blocks_end[]; -extern const unsigned char _binary_stagenet_blocks_start[]; -extern const unsigned char _binary_stagenet_blocks_end[]; - -const unsigned char *get_blocks_dat_start(int testnet, int stagenet) -{ - if (testnet) - return _binary_testnet_blocks_start; - else if (stagenet) - return _binary_stagenet_blocks_start; - else - return _binary_blocks_start; -} - -size_t get_blocks_dat_size(int testnet, int stagenet) -{ - if (testnet) - return (size_t) (_binary_testnet_blocks_end - _binary_testnet_blocks_start); - else if (stagenet) - return (size_t) (_binary_stagenet_blocks_end - _binary_stagenet_blocks_start); - else - return (size_t) (_binary_blocks_end - _binary_blocks_start); -} - -#endif diff --git a/src/blocks/blocks.cpp b/src/blocks/blocks.cpp new file mode 100644 index 000000000..0661f8448 --- /dev/null +++ b/src/blocks/blocks.cpp @@ -0,0 +1,31 @@ +#include "blocks.h" + +#include <unordered_map> + +extern const unsigned char checkpoints[]; +extern const size_t checkpoints_len; +extern const unsigned char stagenet_blocks[]; +extern const size_t stagenet_blocks_len; +extern const unsigned char testnet_blocks[]; +extern const size_t testnet_blocks_len; + +namespace blocks +{ + + const std::unordered_map<cryptonote::network_type, const epee::span<const unsigned char>, std::hash<size_t>> CheckpointsByNetwork = { + {cryptonote::network_type::MAINNET, {checkpoints, checkpoints_len}}, + {cryptonote::network_type::STAGENET, {stagenet_blocks, stagenet_blocks_len}}, + {cryptonote::network_type::TESTNET, {testnet_blocks, testnet_blocks_len}} + }; + + const epee::span<const unsigned char> GetCheckpointsData(cryptonote::network_type network) + { + const auto it = CheckpointsByNetwork.find(network); + if (it != CheckpointsByNetwork.end()) + { + return it->second; + } + return nullptr; + } + +} diff --git a/src/blocks/blocks.h b/src/blocks/blocks.h index ec683f47e..14e391319 100644 --- a/src/blocks/blocks.h +++ b/src/blocks/blocks.h @@ -1,16 +1,12 @@ #ifndef SRC_BLOCKS_BLOCKS_H_ #define SRC_BLOCKS_BLOCKS_H_ -#ifdef __cplusplus -extern "C" { -#endif +#include "cryptonote_config.h" +#include "span.h" -const unsigned char *get_blocks_dat_start(int testnet, int stagenet); -size_t get_blocks_dat_size(int testnet, int stagenet); - -#ifdef __cplusplus +namespace blocks +{ + const epee::span<const unsigned char> GetCheckpointsData(cryptonote::network_type network); } -#endif - #endif /* SRC_BLOCKS_BLOCKS_H_ */ diff --git a/src/checkpoints/checkpoints.cpp b/src/checkpoints/checkpoints.cpp index 6251fcc91..96f575b2d 100644 --- a/src/checkpoints/checkpoints.cpp +++ b/src/checkpoints/checkpoints.cpp @@ -28,17 +28,15 @@ // // Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers -#include "include_base_utils.h" - -using namespace epee; - #include "checkpoints.h" #include "common/dns_utils.h" -#include "include_base_utils.h" #include "string_tools.h" #include "storages/portable_storage_template_helper.h" // epee json include #include "serialization/keyvalue_serialization.h" +#include <vector> + +using namespace epee; #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "checkpoints" @@ -76,7 +74,7 @@ namespace cryptonote bool checkpoints::add_checkpoint(uint64_t height, const std::string& hash_str) { crypto::hash h = crypto::null_hash; - bool r = epee::string_tools::parse_tpod_from_hex_string(hash_str, h); + bool r = epee::string_tools::hex_to_pod(hash_str, h); CHECK_AND_ASSERT_MES(r, false, "Failed to parse checkpoint hash string into binary representation!"); // return false if adding at a height we already have AND the hash is different @@ -294,7 +292,7 @@ namespace cryptonote // parse the second part as crypto::hash, // if this fails move on to the next record std::string hashStr = record.substr(pos + 1); - if (!epee::string_tools::parse_tpod_from_hex_string(hashStr, hash)) + if (!epee::string_tools::hex_to_pod(hashStr, hash)) { continue; } diff --git a/src/checkpoints/checkpoints.h b/src/checkpoints/checkpoints.h index 61be2c27a..ad2b44d1a 100644 --- a/src/checkpoints/checkpoints.h +++ b/src/checkpoints/checkpoints.h @@ -30,7 +30,6 @@ #pragma once #include <map> -#include <vector> #include "misc_log_ex.h" #include "crypto/hash.h" #include "cryptonote_config.h" diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index aed9bfee7..212a1891e 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -40,15 +40,21 @@ set(common_sources notify.cpp password.cpp perf_timer.cpp + pruning.cpp spawn.cpp threadpool.cpp updates.cpp - aligned.c) + aligned.c + combinator.cpp) if (STACK_TRACE) list(APPEND common_sources stack_trace.cpp) endif() +if (BACKCOMPAT) + list(APPEND common_sources compat/glibc_compat.cpp) +endif() + set(common_headers) set(common_private_headers @@ -62,9 +68,9 @@ set(common_private_headers error.h expect.h http_connection.h - int-util.h notify.h pod-class.h + pruning.h rpc_client.h scoped_message_writer.h unordered_containers_boost_serialization.h @@ -77,7 +83,8 @@ set(common_private_headers stack_trace.h threadpool.h updates.h - aligned.h) + aligned.h + combinator.h) monero_private_headers(common ${common_private_headers}) diff --git a/src/common/base58.cpp b/src/common/base58.cpp index 75556cad9..3562af486 100644 --- a/src/common/base58.cpp +++ b/src/common/base58.cpp @@ -36,7 +36,6 @@ #include "crypto/hash.h" #include "int-util.h" -#include "util.h" #include "varint.h" namespace tools @@ -109,20 +108,8 @@ namespace tools assert(1 <= size && size <= sizeof(uint64_t)); uint64_t res = 0; - switch (9 - size) - { - case 1: res |= *data++; /* FALLTHRU */ - case 2: res <<= 8; res |= *data++; /* FALLTHRU */ - case 3: res <<= 8; res |= *data++; /* FALLTHRU */ - case 4: res <<= 8; res |= *data++; /* FALLTHRU */ - case 5: res <<= 8; res |= *data++; /* FALLTHRU */ - case 6: res <<= 8; res |= *data++; /* FALLTHRU */ - case 7: res <<= 8; res |= *data++; /* FALLTHRU */ - case 8: res <<= 8; res |= *data; break; - default: assert(false); - } - - return res; + memcpy(reinterpret_cast<uint8_t*>(&res) + sizeof(uint64_t) - size, data, size); + return SWAP64BE(res); } void uint_64_to_8be(uint64_t num, size_t size, uint8_t* data) @@ -247,7 +234,7 @@ namespace tools return encode(buf); } - bool decode_addr(std::string addr, uint64_t& tag, std::string& data) + bool decode_addr(const std::string &addr, uint64_t& tag, std::string& data) { std::string addr_data; bool r = decode(addr, addr_data); diff --git a/src/common/base58.h b/src/common/base58.h index 02ca96956..69611859d 100644 --- a/src/common/base58.h +++ b/src/common/base58.h @@ -41,6 +41,6 @@ namespace tools bool decode(const std::string& enc, std::string& data); std::string encode_addr(uint64_t tag, const std::string& data); - bool decode_addr(std::string addr, uint64_t& tag, std::string& data); + bool decode_addr(const std::string &addr, uint64_t& tag, std::string& data); } } diff --git a/src/common/combinator.cpp b/src/common/combinator.cpp new file mode 100644 index 000000000..cb4fbc908 --- /dev/null +++ b/src/common/combinator.cpp @@ -0,0 +1,50 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +#include "combinator.h" + +namespace tools { + +uint64_t combinations_count(uint32_t k, uint32_t n) +{ + if (k > n) { + throw std::runtime_error("k must not be greater than n"); + } + + uint64_t c = 1; + for (uint64_t i = 1; i <= k; ++i) { + c *= n--; + c /= i; + } + + return c; +} + +} diff --git a/src/common/combinator.h b/src/common/combinator.h new file mode 100644 index 000000000..72c6800d5 --- /dev/null +++ b/src/common/combinator.h @@ -0,0 +1,96 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +#pragma once + +#include <iostream> +#include <vector> + +namespace tools { + +uint64_t combinations_count(uint32_t k, uint32_t n); + +template<typename T> +class Combinator { +public: + Combinator(const std::vector<T>& v) : origin(v) { } + + std::vector<std::vector<T>> combine(size_t k); + +private: + void doCombine(size_t from, size_t k); + + std::vector<T> origin; + std::vector<std::vector<T>> combinations; + std::vector<size_t> current; +}; + +template<typename T> +std::vector<std::vector<T>> Combinator<T>::combine(size_t k) +{ + if (k > origin.size()) + { + throw std::runtime_error("k must be smaller than elements number"); + } + + if (k == 0) + { + throw std::runtime_error("k must be greater than zero"); + } + + combinations.clear(); + doCombine(0, k); + return combinations; +} + +template<typename T> +void Combinator<T>::doCombine(size_t from, size_t k) +{ + current.push_back(0); + + for (size_t i = from; i <= origin.size() - k; ++i) + { + current.back() = i; + + if (k > 1) { + doCombine(i + 1, k - 1); + } else { + std::vector<T> comb; + for (auto ind: current) { + comb.push_back(origin[ind]); + } + combinations.push_back(comb); + } + } + + current.pop_back(); +} + +} //namespace tools diff --git a/src/common/command_line.cpp b/src/common/command_line.cpp index 7980b381f..35135ea18 100644 --- a/src/common/command_line.cpp +++ b/src/common/command_line.cpp @@ -31,10 +31,7 @@ #include "command_line.h" #include <boost/algorithm/string/compare.hpp> #include <boost/algorithm/string/predicate.hpp> -#include <unordered_set> #include "common/i18n.h" -#include "cryptonote_config.h" -#include "string_tools.h" namespace command_line { diff --git a/src/common/compat/glibc_compat.cpp b/src/common/compat/glibc_compat.cpp new file mode 100644 index 000000000..bf567987d --- /dev/null +++ b/src/common/compat/glibc_compat.cpp @@ -0,0 +1,98 @@ +#include <cstddef> +#include <cstdint> +#include <strings.h> +#include <string.h> +#include <glob.h> +#include <unistd.h> +#include <fnmatch.h> + +#if defined(HAVE_SYS_SELECT_H) +#include <sys/select.h> +#endif + +// Prior to GLIBC_2.14, memcpy was aliased to memmove. +extern "C" void* memmove(void* a, const void* b, size_t c); +//extern "C" void* memset(void* a, int b, long unsigned int c); +extern "C" void* memcpy(void* a, const void* b, size_t c) +{ + return memmove(a, b, c); +} + +extern "C" void __chk_fail(void) __attribute__((__noreturn__)); + +#if defined(__i386__) || defined(__arm__) + +extern "C" int64_t __udivmoddi4(uint64_t u, uint64_t v, uint64_t* rp); + +extern "C" int64_t __wrap___divmoddi4(int64_t u, int64_t v, int64_t* rp) +{ + int32_t c1 = 0, c2 = 0; + int64_t uu = u, vv = v; + int64_t w; + int64_t r; + + if (uu < 0) { + c1 = ~c1, c2 = ~c2, uu = -uu; + } + if (vv < 0) { + c1 = ~c1, vv = -vv; + } + + w = __udivmoddi4(uu, vv, (uint64_t*)&r); + if (c1) + w = -w; + if (c2) + r = -r; + + *rp = r; + return w; +} +#endif + +/* glibc-internal users use __explicit_bzero_chk, and explicit_bzero + redirects to that. */ +#undef explicit_bzero +/* Set LEN bytes of S to 0. The compiler will not delete a call to + this function, even if S is dead after the call. */ +void +explicit_bzero (void *s, size_t len) +{ + memset (s, '\0', len); + /* Compiler barrier. */ + asm volatile ("" ::: "memory"); +} + +// Redefine explicit_bzero_chk +void +__explicit_bzero_chk (void *dst, size_t len, size_t dstlen) +{ + /* Inline __memset_chk to avoid a PLT reference to __memset_chk. */ + if (__glibc_unlikely (dstlen < len)) + __chk_fail (); + memset (dst, '\0', len); + /* Compiler barrier. */ + asm volatile ("" ::: "memory"); +} +/* libc-internal references use the hidden + __explicit_bzero_chk_internal symbol. This is necessary if + __explicit_bzero_chk is implemented as an IFUNC because some + targets do not support hidden references to IFUNC symbols. */ +#define strong_alias (__explicit_bzero_chk, __explicit_bzero_chk_internal) + +#undef glob +extern "C" int glob_old(const char * pattern, int flags, int (*errfunc) (const char *epath, int eerrno), glob_t *pglob); +#ifdef __i386__ +__asm__(".symver glob_old,glob@GLIBC_2.1"); +#elif defined(__amd64__) +__asm__(".symver glob_old,glob@GLIBC_2.2.5"); +#elif defined(__arm__) +__asm(".symver glob_old,glob@GLIBC_2.4"); +#elif defined(__aarch64__) +__asm__(".symver glob_old,glob@GLIBC_2.17"); +#endif + +extern "C" int __wrap_glob(const char * pattern, int flags, int (*errfunc) (const char *epath, int eerrno), glob_t *pglob) +{ + return glob_old(pattern, flags, errfunc, pglob); +} + diff --git a/src/common/dns_utils.cpp b/src/common/dns_utils.cpp index f2b270981..417b5b4ac 100644 --- a/src/common/dns_utils.cpp +++ b/src/common/dns_utils.cpp @@ -33,12 +33,11 @@ #include <stdlib.h> #include "include_base_utils.h" #include <random> -#include <boost/filesystem/fstream.hpp> #include <boost/thread/mutex.hpp> #include <boost/thread/thread.hpp> #include <boost/algorithm/string/join.hpp> +#include <boost/optional.hpp> using namespace epee; -namespace bf = boost::filesystem; #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net.dns" @@ -119,10 +118,25 @@ get_builtin_ds(void) namespace tools { +static const char *get_record_name(int record_type) +{ + switch (record_type) + { + case DNS_TYPE_A: return "A"; + case DNS_TYPE_TXT: return "TXT"; + case DNS_TYPE_AAAA: return "AAAA"; + default: return "unknown"; + } +} + // fuck it, I'm tired of dealing with getnameinfo()/inet_ntop/etc -std::string ipv4_to_string(const char* src, size_t len) +boost::optional<std::string> ipv4_to_string(const char* src, size_t len) { - assert(len >= 4); + if (len < 4) + { + MERROR("Invalid IPv4 address: " << std::string(src, len)); + return boost::none; + } std::stringstream ss; unsigned int bytes[4]; @@ -140,9 +154,13 @@ std::string ipv4_to_string(const char* src, size_t len) // this obviously will need to change, but is here to reflect the above // stop-gap measure and to make the tests pass at least... -std::string ipv6_to_string(const char* src, size_t len) +boost::optional<std::string> ipv6_to_string(const char* src, size_t len) { - assert(len >= 8); + if (len < 8) + { + MERROR("Invalid IPv4 address: " << std::string(src, len)); + return boost::none; + } std::stringstream ss; unsigned int bytes[8]; @@ -162,8 +180,10 @@ std::string ipv6_to_string(const char* src, size_t len) return ss.str(); } -std::string txt_to_string(const char* src, size_t len) +boost::optional<std::string> txt_to_string(const char* src, size_t len) { + if (len == 0) + return boost::none; return std::string(src+1, len-1); } @@ -266,7 +286,7 @@ DNSResolver::~DNSResolver() } } -std::vector<std::string> DNSResolver::get_record(const std::string& url, int record_type, std::string (*reader)(const char *,size_t), bool& dnssec_available, bool& dnssec_valid) +std::vector<std::string> DNSResolver::get_record(const std::string& url, int record_type, boost::optional<std::string> (*reader)(const char *,size_t), bool& dnssec_available, bool& dnssec_valid) { std::vector<std::string> addresses; dnssec_available = false; @@ -283,13 +303,18 @@ std::vector<std::string> DNSResolver::get_record(const std::string& url, int rec // call DNS resolver, blocking. if return value not zero, something went wrong if (!ub_resolve(m_data->m_ub_context, string_copy(url.c_str()), record_type, DNS_CLASS_IN, &result)) { - dnssec_available = (result->secure || (!result->secure && result->bogus)); + dnssec_available = (result->secure || result->bogus); dnssec_valid = result->secure && !result->bogus; if (result->havedata) { for (size_t i=0; result->data[i] != NULL; i++) { - addresses.push_back((*reader)(result->data[i], result->len[i])); + boost::optional<std::string> res = (*reader)(result->data[i], result->len[i]); + if (res) + { + MINFO("Found \"" << *res << "\" in " << get_record_name(record_type) << " record for " << url); + addresses.push_back(*res); + } } } } diff --git a/src/common/dns_utils.h b/src/common/dns_utils.h index f46bca3dd..3a6ef68a1 100644 --- a/src/common/dns_utils.h +++ b/src/common/dns_utils.h @@ -30,6 +30,7 @@ #include <vector> #include <string> #include <functional> +#include <boost/optional/optional_fwd.hpp> namespace tools { @@ -143,7 +144,7 @@ private: * @return A vector of strings containing the requested record; or an empty vector */ // TODO: modify this to accommodate DNSSEC - std::vector<std::string> get_record(const std::string& url, int record_type, std::string (*reader)(const char *,size_t), bool& dnssec_available, bool& dnssec_valid); + std::vector<std::string> get_record(const std::string& url, int record_type, boost::optional<std::string> (*reader)(const char *,size_t), bool& dnssec_available, bool& dnssec_valid); /** * @brief Checks a string to see if it looks like a URL diff --git a/src/common/download.cpp b/src/common/download.cpp index 6698a5abf..58ce0595f 100644 --- a/src/common/download.cpp +++ b/src/common/download.cpp @@ -29,10 +29,7 @@ #include <string> #include <atomic> #include <boost/filesystem.hpp> -#include <boost/asio.hpp> #include <boost/thread/thread.hpp> -#include "cryptonote_config.h" -#include "include_base_utils.h" #include "file_io_utils.h" #include "net/http_client.h" #include "download.h" diff --git a/src/common/http_connection.h b/src/common/http_connection.h index 9fc6be261..554dd832b 100644 --- a/src/common/http_connection.h +++ b/src/common/http_connection.h @@ -55,7 +55,8 @@ public: { if (m_ok) { - mp_http_client->disconnect(); + try { mp_http_client->disconnect(); } + catch (...) { /* do not propagate through dtor */ } } } diff --git a/src/common/i18n.cpp b/src/common/i18n.cpp index 4a89876fb..a32875945 100644 --- a/src/common/i18n.cpp +++ b/src/common/i18n.cpp @@ -31,15 +31,15 @@ #include <ctype.h> #include <string> #include <map> -#include "include_base_utils.h" #include "file_io_utils.h" -#include "common/util.h" #include "common/i18n.h" #include "translation_files.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "i18n" +#define MAX_LANGUAGE_SIZE 16 + static const unsigned char qm_magic[16] = {0x3c, 0xb8, 0x64, 0x18, 0xca, 0xef, 0x9c, 0x95, 0xcd, 0x21, 0x1c, 0xbf, 0x60, 0xa1, 0xbd, 0xdd}; static std::map<std::string,std::string> i18n_entries; @@ -64,7 +64,19 @@ std::string i18n_get_language() std::string language = e; language = language.substr(0, language.find(".")); + language = language.substr(0, language.find("@")); + + // check valid values + for (char c: language) + if (!strchr("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_-.@", c)) + return "en"; + std::transform(language.begin(), language.end(), language.begin(), tolower); + if (language.size() > MAX_LANGUAGE_SIZE) + { + i18n_log("Language from LANG/LC_ALL suspiciously long, defaulting to en"); + return "en"; + } return language; } diff --git a/src/common/int-util.h b/src/common/int-util.h deleted file mode 100644 index 3bcc085e2..000000000 --- a/src/common/int-util.h +++ /dev/null @@ -1,258 +0,0 @@ -// Copyright (c) 2014-2018, The Monero Project -// -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without modification, are -// permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this list of -// conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright notice, this list -// of conditions and the following disclaimer in the documentation and/or other -// materials provided with the distribution. -// -// 3. Neither the name of the copyright holder nor the names of its contributors may be -// used to endorse or promote products derived from this software without specific -// prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL -// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF -// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers - -#pragma once - -#include <assert.h> -#include <stdbool.h> -#include <stdint.h> -#include <string.h> - -#ifndef _MSC_VER -#include <sys/param.h> -#endif - -#if defined(__ANDROID__) -#include <byteswap.h> -#endif - -#if defined(__sun) && defined(__SVR4) -#include <endian.h> -#endif - -#if defined(_MSC_VER) -#include <stdlib.h> - -static inline uint32_t rol32(uint32_t x, int r) { - static_assert(sizeof(uint32_t) == sizeof(unsigned int), "this code assumes 32-bit integers"); - return _rotl(x, r); -} - -static inline uint64_t rol64(uint64_t x, int r) { - return _rotl64(x, r); -} - -#else - -static inline uint32_t rol32(uint32_t x, int r) { - return (x << (r & 31)) | (x >> (-r & 31)); -} - -static inline uint64_t rol64(uint64_t x, int r) { - return (x << (r & 63)) | (x >> (-r & 63)); -} - -#endif - -static inline uint64_t hi_dword(uint64_t val) { - return val >> 32; -} - -static inline uint64_t lo_dword(uint64_t val) { - return val & 0xFFFFFFFF; -} - -static inline uint64_t mul128(uint64_t multiplier, uint64_t multiplicand, uint64_t* product_hi) { - // multiplier = ab = a * 2^32 + b - // multiplicand = cd = c * 2^32 + d - // ab * cd = a * c * 2^64 + (a * d + b * c) * 2^32 + b * d - uint64_t a = hi_dword(multiplier); - uint64_t b = lo_dword(multiplier); - uint64_t c = hi_dword(multiplicand); - uint64_t d = lo_dword(multiplicand); - - uint64_t ac = a * c; - uint64_t ad = a * d; - uint64_t bc = b * c; - uint64_t bd = b * d; - - uint64_t adbc = ad + bc; - uint64_t adbc_carry = adbc < ad ? 1 : 0; - - // multiplier * multiplicand = product_hi * 2^64 + product_lo - uint64_t product_lo = bd + (adbc << 32); - uint64_t product_lo_carry = product_lo < bd ? 1 : 0; - *product_hi = ac + (adbc >> 32) + (adbc_carry << 32) + product_lo_carry; - assert(ac <= *product_hi); - - return product_lo; -} - -static inline uint64_t div_with_reminder(uint64_t dividend, uint32_t divisor, uint32_t* remainder) { - dividend |= ((uint64_t)*remainder) << 32; - *remainder = dividend % divisor; - return dividend / divisor; -} - -// Long division with 2^32 base -static inline uint32_t div128_32(uint64_t dividend_hi, uint64_t dividend_lo, uint32_t divisor, uint64_t* quotient_hi, uint64_t* quotient_lo) { - uint64_t dividend_dwords[4]; - uint32_t remainder = 0; - - dividend_dwords[3] = hi_dword(dividend_hi); - dividend_dwords[2] = lo_dword(dividend_hi); - dividend_dwords[1] = hi_dword(dividend_lo); - dividend_dwords[0] = lo_dword(dividend_lo); - - *quotient_hi = div_with_reminder(dividend_dwords[3], divisor, &remainder) << 32; - *quotient_hi |= div_with_reminder(dividend_dwords[2], divisor, &remainder); - *quotient_lo = div_with_reminder(dividend_dwords[1], divisor, &remainder) << 32; - *quotient_lo |= div_with_reminder(dividend_dwords[0], divisor, &remainder); - - return remainder; -} - -#define IDENT32(x) ((uint32_t) (x)) -#define IDENT64(x) ((uint64_t) (x)) - -#define SWAP32(x) ((((uint32_t) (x) & 0x000000ff) << 24) | \ - (((uint32_t) (x) & 0x0000ff00) << 8) | \ - (((uint32_t) (x) & 0x00ff0000) >> 8) | \ - (((uint32_t) (x) & 0xff000000) >> 24)) -#define SWAP64(x) ((((uint64_t) (x) & 0x00000000000000ff) << 56) | \ - (((uint64_t) (x) & 0x000000000000ff00) << 40) | \ - (((uint64_t) (x) & 0x0000000000ff0000) << 24) | \ - (((uint64_t) (x) & 0x00000000ff000000) << 8) | \ - (((uint64_t) (x) & 0x000000ff00000000) >> 8) | \ - (((uint64_t) (x) & 0x0000ff0000000000) >> 24) | \ - (((uint64_t) (x) & 0x00ff000000000000) >> 40) | \ - (((uint64_t) (x) & 0xff00000000000000) >> 56)) - -static inline uint32_t ident32(uint32_t x) { return x; } -static inline uint64_t ident64(uint64_t x) { return x; } - -#ifndef __OpenBSD__ -# if defined(__ANDROID__) && defined(__swap32) && !defined(swap32) -# define swap32 __swap32 -# elif !defined(swap32) -static inline uint32_t swap32(uint32_t x) { - x = ((x & 0x00ff00ff) << 8) | ((x & 0xff00ff00) >> 8); - return (x << 16) | (x >> 16); -} -# endif -# if defined(__ANDROID__) && defined(__swap64) && !defined(swap64) -# define swap64 __swap64 -# elif !defined(swap64) -static inline uint64_t swap64(uint64_t x) { - x = ((x & 0x00ff00ff00ff00ff) << 8) | ((x & 0xff00ff00ff00ff00) >> 8); - x = ((x & 0x0000ffff0000ffff) << 16) | ((x & 0xffff0000ffff0000) >> 16); - return (x << 32) | (x >> 32); -} -# endif -#endif /* __OpenBSD__ */ - -#if defined(__GNUC__) -#define UNUSED __attribute__((unused)) -#else -#define UNUSED -#endif -static inline void mem_inplace_ident(void *mem UNUSED, size_t n UNUSED) { } -#undef UNUSED - -static inline void mem_inplace_swap32(void *mem, size_t n) { - size_t i; - for (i = 0; i < n; i++) { - ((uint32_t *) mem)[i] = swap32(((const uint32_t *) mem)[i]); - } -} -static inline void mem_inplace_swap64(void *mem, size_t n) { - size_t i; - for (i = 0; i < n; i++) { - ((uint64_t *) mem)[i] = swap64(((const uint64_t *) mem)[i]); - } -} - -static inline void memcpy_ident32(void *dst, const void *src, size_t n) { - memcpy(dst, src, 4 * n); -} -static inline void memcpy_ident64(void *dst, const void *src, size_t n) { - memcpy(dst, src, 8 * n); -} - -static inline void memcpy_swap32(void *dst, const void *src, size_t n) { - size_t i; - for (i = 0; i < n; i++) { - ((uint32_t *) dst)[i] = swap32(((const uint32_t *) src)[i]); - } -} -static inline void memcpy_swap64(void *dst, const void *src, size_t n) { - size_t i; - for (i = 0; i < n; i++) { - ((uint64_t *) dst)[i] = swap64(((const uint64_t *) src)[i]); - } -} - -#ifdef _MSC_VER -# define LITTLE_ENDIAN 1234 -# define BIG_ENDIAN 4321 -# define BYTE_ORDER LITTLE_ENDIAN -#endif - -#if !defined(BYTE_ORDER) || !defined(LITTLE_ENDIAN) || !defined(BIG_ENDIAN) -static_assert(false, "BYTE_ORDER is undefined. Perhaps, GNU extensions are not enabled"); -#endif - -#if BYTE_ORDER == LITTLE_ENDIAN -#define SWAP32LE IDENT32 -#define SWAP32BE SWAP32 -#define swap32le ident32 -#define swap32be swap32 -#define mem_inplace_swap32le mem_inplace_ident -#define mem_inplace_swap32be mem_inplace_swap32 -#define memcpy_swap32le memcpy_ident32 -#define memcpy_swap32be memcpy_swap32 -#define SWAP64LE IDENT64 -#define SWAP64BE SWAP64 -#define swap64le ident64 -#define swap64be swap64 -#define mem_inplace_swap64le mem_inplace_ident -#define mem_inplace_swap64be mem_inplace_swap64 -#define memcpy_swap64le memcpy_ident64 -#define memcpy_swap64be memcpy_swap64 -#endif - -#if BYTE_ORDER == BIG_ENDIAN -#define SWAP32BE IDENT32 -#define SWAP32LE SWAP32 -#define swap32be ident32 -#define swap32le swap32 -#define mem_inplace_swap32be mem_inplace_ident -#define mem_inplace_swap32le mem_inplace_swap32 -#define memcpy_swap32be memcpy_ident32 -#define memcpy_swap32le memcpy_swap32 -#define SWAP64BE IDENT64 -#define SWAP64LE SWAP64 -#define swap64be ident64 -#define swap64le swap64 -#define mem_inplace_swap64be mem_inplace_ident -#define mem_inplace_swap64le mem_inplace_swap64 -#define memcpy_swap64be memcpy_ident64 -#define memcpy_swap64le memcpy_swap64 -#endif diff --git a/src/common/password.cpp b/src/common/password.cpp index b3c51128f..5f5cb800a 100644 --- a/src/common/password.cpp +++ b/src/common/password.cpp @@ -31,7 +31,6 @@ #include "password.h" #include <iostream> -#include <memory.h> #include <stdio.h> #if defined(_WIN32) @@ -42,8 +41,6 @@ #include <unistd.h> #endif -#include "memwipe.h" - #define EOT 0x4 namespace diff --git a/src/common/perf_timer.cpp b/src/common/perf_timer.cpp index 6910ebdd4..3e1357833 100644 --- a/src/common/perf_timer.cpp +++ b/src/common/perf_timer.cpp @@ -33,6 +33,13 @@ #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "perf" +#define PERF_LOG_ALWAYS(level, cat, x) \ + el::base::Writer(level, __FILE__, __LINE__, ELPP_FUNC, el::base::DispatchAction::FileOnlyLog).construct(cat) << x +#define PERF_LOG(level, cat, x) \ + do { \ + if (ELPP->vRegistry()->allowed(level, cat)) PERF_LOG_ALWAYS(level, cat, x); \ + } while(0) + namespace tools { uint64_t get_tick_count() @@ -104,20 +111,26 @@ PerformanceTimer::PerformanceTimer(bool paused): started(true), paused(paused) ticks = get_tick_count(); } -LoggingPerformanceTimer::LoggingPerformanceTimer(const std::string &s, uint64_t unit, el::Level l): PerformanceTimer(), name(s), unit(unit), level(l) +LoggingPerformanceTimer::LoggingPerformanceTimer(const std::string &s, const std::string &cat, uint64_t unit, el::Level l): PerformanceTimer(), name(s), cat(cat), unit(unit), level(l) { + const bool log = ELPP->vRegistry()->allowed(level, cat.c_str()); if (!performance_timers) { - MLOG(level, "PERF ----------"); + if (log) + PERF_LOG_ALWAYS(level, cat.c_str(), "PERF ----------"); performance_timers = new std::vector<LoggingPerformanceTimer*>(); + performance_timers->reserve(16); // how deep before realloc } else { LoggingPerformanceTimer *pt = performance_timers->back(); if (!pt->started && !pt->paused) { - size_t size = 0; for (const auto *tmp: *performance_timers) if (!tmp->paused) ++size; - MLOG(pt->level, "PERF " << std::string((size-1) * 2, ' ') << " " << pt->name); + if (log) + { + size_t size = 0; for (const auto *tmp: *performance_timers) if (!tmp->paused) ++size; + PERF_LOG_ALWAYS(pt->level, cat.c_str(), "PERF " << std::string((size-1) * 2, ' ') << " " << pt->name); + } pt->started = true; } } @@ -134,10 +147,14 @@ LoggingPerformanceTimer::~LoggingPerformanceTimer() { pause(); performance_timers->pop_back(); - char s[12]; - snprintf(s, sizeof(s), "%8llu ", (unsigned long long)(ticks_to_ns(ticks) / (1000000000 / unit))); - size_t size = 0; for (const auto *tmp: *performance_timers) if (!tmp->paused || tmp==this) ++size; - MLOG(level, "PERF " << s << std::string(size * 2, ' ') << " " << name); + const bool log = ELPP->vRegistry()->allowed(level, cat.c_str()); + if (log) + { + char s[12]; + snprintf(s, sizeof(s), "%8llu ", (unsigned long long)(ticks_to_ns(ticks) / (1000000000 / unit))); + size_t size = 0; for (const auto *tmp: *performance_timers) if (!tmp->paused || tmp==this) ++size; + PERF_LOG_ALWAYS(level, cat.c_str(), "PERF " << s << std::string(size * 2, ' ') << " " << name); + } if (performance_timers->empty()) { delete performance_timers; @@ -161,4 +178,20 @@ void PerformanceTimer::resume() paused = false; } +void PerformanceTimer::reset() +{ + if (paused) + ticks = 0; + else + ticks = get_tick_count(); +} + +uint64_t PerformanceTimer::value() const +{ + uint64_t v = ticks; + if (!paused) + v = get_tick_count() - v; + return ticks_to_ns(v); +} + } diff --git a/src/common/perf_timer.h b/src/common/perf_timer.h index 675d6234d..d859cf576 100644 --- a/src/common/perf_timer.h +++ b/src/common/perf_timer.h @@ -33,9 +33,6 @@ #include <memory> #include "misc_log_ex.h" -#undef MONERO_DEFAULT_LOG_CATEGORY -#define MONERO_DEFAULT_LOG_CATEGORY "perf" - namespace tools { @@ -54,9 +51,8 @@ public: ~PerformanceTimer(); void pause(); void resume(); - - uint64_t value() const { return ticks; } -void set(uint64_t v){ticks=v;} + void reset(); + uint64_t value() const; protected: uint64_t ticks; @@ -67,23 +63,24 @@ protected: class LoggingPerformanceTimer: public PerformanceTimer { public: - LoggingPerformanceTimer(const std::string &s, uint64_t unit, el::Level l = el::Level::Debug); + LoggingPerformanceTimer(const std::string &s, const std::string &cat, uint64_t unit, el::Level l = el::Level::Info); ~LoggingPerformanceTimer(); private: std::string name; + std::string cat; uint64_t unit; el::Level level; }; void set_performance_timer_log_level(el::Level level); -#define PERF_TIMER_UNIT(name, unit) tools::LoggingPerformanceTimer pt_##name(#name, unit, tools::performance_timer_log_level) -#define PERF_TIMER_UNIT_L(name, unit, l) tools::LoggingPerformanceTimer pt_##name(#name, unit, l) -#define PERF_TIMER(name) PERF_TIMER_UNIT(name, 1000) -#define PERF_TIMER_L(name, l) PERF_TIMER_UNIT_L(name, 1000, l) -#define PERF_TIMER_START_UNIT(name, unit) std::unique_ptr<tools::LoggingPerformanceTimer> pt_##name(new tools::LoggingPerformanceTimer(#name, unit, el::Level::Info)) -#define PERF_TIMER_START(name) PERF_TIMER_START_UNIT(name, 1000) +#define PERF_TIMER_UNIT(name, unit) tools::LoggingPerformanceTimer pt_##name(#name, "perf." MONERO_DEFAULT_LOG_CATEGORY, unit, tools::performance_timer_log_level) +#define PERF_TIMER_UNIT_L(name, unit, l) tools::LoggingPerformanceTimer pt_##name(#name, "perf." MONERO_DEFAULT_LOG_CATEGORY, unit, l) +#define PERF_TIMER(name) PERF_TIMER_UNIT(name, 1000000) +#define PERF_TIMER_L(name, l) PERF_TIMER_UNIT_L(name, 1000000, l) +#define PERF_TIMER_START_UNIT(name, unit) std::unique_ptr<tools::LoggingPerformanceTimer> pt_##name(new tools::LoggingPerformanceTimer(#name, "perf." MONERO_DEFAULT_LOG_CATEGORY, unit, el::Level::Info)) +#define PERF_TIMER_START(name) PERF_TIMER_START_UNIT(name, 1000000) #define PERF_TIMER_STOP(name) do { pt_##name.reset(NULL); } while(0) #define PERF_TIMER_PAUSE(name) pt_##name->pause() #define PERF_TIMER_RESUME(name) pt_##name->resume() diff --git a/src/common/pruning.cpp b/src/common/pruning.cpp new file mode 100644 index 000000000..442b24e4e --- /dev/null +++ b/src/common/pruning.cpp @@ -0,0 +1,116 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "cryptonote_config.h" +#include "misc_log_ex.h" +#include "crypto/crypto.h" +#include "pruning.h" + +namespace tools +{ + +uint32_t make_pruning_seed(uint32_t stripe, uint32_t log_stripes) +{ + CHECK_AND_ASSERT_THROW_MES(log_stripes <= PRUNING_SEED_LOG_STRIPES_MASK, "log_stripes out of range"); + CHECK_AND_ASSERT_THROW_MES(stripe > 0 && stripe <= (1ul << log_stripes), "stripe out of range"); + return (log_stripes << PRUNING_SEED_LOG_STRIPES_SHIFT) | ((stripe - 1) << PRUNING_SEED_STRIPE_SHIFT); +} + +bool has_unpruned_block(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed) +{ + const uint32_t stripe = get_pruning_stripe(pruning_seed); + if (stripe == 0) + return true; + const uint32_t log_stripes = get_pruning_log_stripes(pruning_seed); + uint32_t block_stripe = get_pruning_stripe(block_height, blockchain_height, log_stripes); + return block_stripe == 0 || block_stripe == stripe; +} + +uint32_t get_pruning_stripe(uint64_t block_height, uint64_t blockchain_height, uint32_t log_stripes) +{ + if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS >= blockchain_height) + return 0; + return ((block_height / CRYPTONOTE_PRUNING_STRIPE_SIZE) & (uint64_t)((1ul << log_stripes) - 1)) + 1; +} + +uint32_t get_pruning_seed(uint64_t block_height, uint64_t blockchain_height, uint32_t log_stripes) +{ + const uint32_t stripe = get_pruning_stripe(block_height, blockchain_height, log_stripes); + if (stripe == 0) + return 0; + return make_pruning_seed(stripe, log_stripes); +} + +uint64_t get_next_unpruned_block_height(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed) +{ + CHECK_AND_ASSERT_MES(block_height <= CRYPTONOTE_MAX_BLOCK_NUMBER+1, block_height, "block_height too large"); + CHECK_AND_ASSERT_MES(blockchain_height <= CRYPTONOTE_MAX_BLOCK_NUMBER+1, block_height, "blockchain_height too large"); + const uint32_t stripe = get_pruning_stripe(pruning_seed); + if (stripe == 0) + return block_height; + if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS >= blockchain_height) + return block_height; + const uint32_t seed_log_stripes = get_pruning_log_stripes(pruning_seed); + const uint64_t log_stripes = seed_log_stripes ? seed_log_stripes : CRYPTONOTE_PRUNING_LOG_STRIPES; + const uint64_t mask = (1ul << log_stripes) - 1; + const uint32_t block_pruning_stripe = ((block_height / CRYPTONOTE_PRUNING_STRIPE_SIZE) & mask) + 1; + if (block_pruning_stripe == stripe) + return block_height; + const uint64_t cycles = ((block_height / CRYPTONOTE_PRUNING_STRIPE_SIZE) >> log_stripes); + const uint64_t cycle_start = cycles + ((stripe > block_pruning_stripe) ? 0 : 1); + const uint64_t h = cycle_start * (CRYPTONOTE_PRUNING_STRIPE_SIZE << log_stripes) + (stripe - 1) * CRYPTONOTE_PRUNING_STRIPE_SIZE; + if (h + CRYPTONOTE_PRUNING_TIP_BLOCKS > blockchain_height) + return blockchain_height < CRYPTONOTE_PRUNING_TIP_BLOCKS ? 0 : blockchain_height - CRYPTONOTE_PRUNING_TIP_BLOCKS; + CHECK_AND_ASSERT_MES(h >= block_height, block_height, "h < block_height, unexpected"); + return h; +} + +uint64_t get_next_pruned_block_height(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed) +{ + const uint32_t stripe = get_pruning_stripe(pruning_seed); + if (stripe == 0) + return blockchain_height; + if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS >= blockchain_height) + return blockchain_height; + const uint32_t seed_log_stripes = get_pruning_log_stripes(pruning_seed); + const uint64_t log_stripes = seed_log_stripes ? seed_log_stripes : CRYPTONOTE_PRUNING_LOG_STRIPES; + const uint64_t mask = (1ul << log_stripes) - 1; + const uint32_t block_pruning_seed = ((block_height / CRYPTONOTE_PRUNING_STRIPE_SIZE) & mask) + 1; + if (block_pruning_seed != stripe) + return block_height; + const uint32_t next_stripe = 1 + (block_pruning_seed & mask); + return get_next_unpruned_block_height(block_height, blockchain_height, tools::make_pruning_seed(next_stripe, log_stripes)); +} + +uint32_t get_random_stripe() +{ + return 1 + crypto::rand<uint8_t>() % (1ul << CRYPTONOTE_PRUNING_LOG_STRIPES); +} + +} + diff --git a/src/common/pruning.h b/src/common/pruning.h new file mode 100644 index 000000000..3fac3c0fa --- /dev/null +++ b/src/common/pruning.h @@ -0,0 +1,52 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <stdint.h> + +namespace tools +{ + static constexpr uint32_t PRUNING_SEED_LOG_STRIPES_SHIFT = 7; + static constexpr uint32_t PRUNING_SEED_LOG_STRIPES_MASK = 0x7; + static constexpr uint32_t PRUNING_SEED_STRIPE_SHIFT = 0; + static constexpr uint32_t PRUNING_SEED_STRIPE_MASK = 0x7f; + + constexpr inline uint32_t get_pruning_log_stripes(uint32_t pruning_seed) { return (pruning_seed >> PRUNING_SEED_LOG_STRIPES_SHIFT) & PRUNING_SEED_LOG_STRIPES_MASK; } + inline uint32_t get_pruning_stripe(uint32_t pruning_seed) { if (pruning_seed == 0) return 0; return 1 + ((pruning_seed >> PRUNING_SEED_STRIPE_SHIFT) & PRUNING_SEED_STRIPE_MASK); } + + uint32_t make_pruning_seed(uint32_t stripe, uint32_t log_stripes); + + bool has_unpruned_block(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed); + uint32_t get_pruning_stripe(uint64_t block_height, uint64_t blockchain_height, uint32_t log_stripes); + uint32_t get_pruning_seed(uint64_t block_height, uint64_t blockchain_height, uint32_t log_stripes); + uint64_t get_next_unpruned_block_height(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed); + uint64_t get_next_pruned_block_height(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed); + uint32_t get_random_stripe(); +} + diff --git a/src/common/scoped_message_writer.h b/src/common/scoped_message_writer.h index d887a13c9..42f439ad8 100644 --- a/src/common/scoped_message_writer.h +++ b/src/common/scoped_message_writer.h @@ -101,13 +101,13 @@ public: MCLOG_FILE(m_log_level, "msgwriter", m_oss.str()); + PAUSE_READLINE(); if (epee::console_color_default == m_color) { std::cout << m_oss.str(); } else { - PAUSE_READLINE(); set_console_color(m_color, m_bright); std::cout << m_oss.str(); epee::reset_console_color(); diff --git a/src/common/spawn.cpp b/src/common/spawn.cpp index 0a2ce8387..b2d03f62f 100644 --- a/src/common/spawn.cpp +++ b/src/common/spawn.cpp @@ -35,6 +35,7 @@ #include <windows.h> #else #include <sys/wait.h> +#include <signal.h> #endif #include "misc_log_ex.h" @@ -114,7 +115,10 @@ int spawn(const char *filename, const std::vector<std::string>& args, bool wait) if (pid > 0) { if (!wait) + { + signal(SIGCHLD, SIG_IGN); return 0; + } while (1) { diff --git a/src/common/threadpool.cpp b/src/common/threadpool.cpp index 37825e31d..cbf7163c5 100644 --- a/src/common/threadpool.cpp +++ b/src/common/threadpool.cpp @@ -28,10 +28,6 @@ #include "misc_log_ex.h" #include "common/threadpool.h" -#include <cassert> -#include <limits> -#include <stdexcept> - #include "cryptonote_config.h" #include "common/util.h" diff --git a/src/common/util.cpp b/src/common/util.cpp index f91230528..28745eea4 100644 --- a/src/common/util.cpp +++ b/src/common/util.cpp @@ -58,6 +58,7 @@ #include "include_base_utils.h" #include "file_io_utils.h" #include "wipeable_string.h" +#include "misc_os_dependent.h" using namespace epee; #include "crypto/crypto.h" @@ -81,6 +82,32 @@ using namespace epee; #include <boost/asio.hpp> #include <openssl/sha.h> +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "util" + +namespace +{ + +#ifndef _WIN32 +static int flock_exnb(int fd) +{ + struct flock fl; + int ret; + + memset(&fl, 0, sizeof(fl)); + fl.l_type = F_WRLCK; + fl.l_whence = SEEK_SET; + fl.l_start = 0; + fl.l_len = 0; + ret = fcntl(fd, F_SETLK, &fl); + if (ret < 0) + MERROR("Error locking fd " << fd << ": " << errno << " (" << strerror(errno) << ")"); + return ret; +} +#endif + +} + namespace tools { std::function<void(int)> signal_handler::m_handler; @@ -181,7 +208,7 @@ namespace tools struct stat wstats = {}; if (fstat(fdw, std::addressof(wstats)) == 0 && rstats.st_dev == wstats.st_dev && rstats.st_ino == wstats.st_ino && - flock(fdw, (LOCK_EX | LOCK_NB)) == 0 && ftruncate(fdw, 0) == 0) + flock_exnb(fdw) == 0 && ftruncate(fdw, 0) == 0) { std::FILE* file = fdopen(fdw, "w"); if (file) return {file, std::move(name)}; @@ -234,10 +261,10 @@ namespace tools MERROR("Failed to open " << filename << ": " << std::error_code(GetLastError(), std::system_category())); } #else - m_fd = open(filename.c_str(), O_RDONLY | O_CREAT | O_CLOEXEC, 0666); + m_fd = open(filename.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0666); if (m_fd != -1) { - if (flock(m_fd, LOCK_EX | LOCK_NB) == -1) + if (flock_exnb(m_fd) == -1) { MERROR("Failed to lock " << filename << ": " << std::strerror(errno)); close(m_fd); @@ -308,10 +335,19 @@ namespace tools StringCchCopy(pszOS, BUFSIZE, TEXT("Microsoft ")); // Test for the specific product. + if ( osvi.dwMajorVersion == 10 ) + { + if ( osvi.dwMinorVersion == 0 ) + { + if( osvi.wProductType == VER_NT_WORKSTATION ) + StringCchCat(pszOS, BUFSIZE, TEXT("Windows 10 ")); + else StringCchCat(pszOS, BUFSIZE, TEXT("Windows Server 2016 " )); + } + } if ( osvi.dwMajorVersion == 6 ) { - if( osvi.dwMinorVersion == 0 ) + if ( osvi.dwMinorVersion == 0 ) { if( osvi.wProductType == VER_NT_WORKSTATION ) StringCchCat(pszOS, BUFSIZE, TEXT("Windows Vista ")); @@ -325,6 +361,20 @@ namespace tools else StringCchCat(pszOS, BUFSIZE, TEXT("Windows Server 2008 R2 " )); } + if ( osvi.dwMinorVersion == 2 ) + { + if( osvi.wProductType == VER_NT_WORKSTATION ) + StringCchCat(pszOS, BUFSIZE, TEXT("Windows 8 ")); + else StringCchCat(pszOS, BUFSIZE, TEXT("Windows Server 2012 " )); + } + + if ( osvi.dwMinorVersion == 3 ) + { + if( osvi.wProductType == VER_NT_WORKSTATION ) + StringCchCat(pszOS, BUFSIZE, TEXT("Windows 8.1 ")); + else StringCchCat(pszOS, BUFSIZE, TEXT("Windows Server 2012 R2 " )); + } + pGPI = (PGPI) GetProcAddress( GetModuleHandle(TEXT("kernel32.dll")), "GetProductInfo"); @@ -705,6 +755,21 @@ std::string get_nix_version_display_string() return true; } + ssize_t get_lockable_memory() + { +#ifdef __GLIBC__ + struct rlimit rlim; + if (getrlimit(RLIMIT_MEMLOCK, &rlim) < 0) + { + MERROR("Failed to determine the lockable memory limit"); + return -1; + } + return rlim.rlim_cur; +#else + return -1; +#endif + } + bool on_startup() { mlog_configure("", true); @@ -987,4 +1052,15 @@ std::string get_nix_version_display_string() #endif } + std::string get_human_readable_timestamp(uint64_t ts) + { + char buffer[64]; + if (ts < 1234567890) + return "<unknown>"; + time_t tt = ts; + struct tm tm; + misc_utils::get_gmt_time(tt, tm); + strftime(buffer, sizeof(buffer), "%Y-%m-%d %H:%M:%S", &tm); + return std::string(buffer); + } } diff --git a/src/common/util.h b/src/common/util.h index e793a42b5..d5aca15d1 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -221,6 +221,8 @@ namespace tools void set_strict_default_file_permissions(bool strict); + ssize_t get_lockable_memory(); + void set_max_concurrency(unsigned n); unsigned get_max_concurrency(); @@ -240,4 +242,6 @@ namespace tools #endif void closefrom(int fd); + + std::string get_human_readable_timestamp(uint64_t ts); } diff --git a/src/common/varint.h b/src/common/varint.h index 151d13dbf..904255afc 100644 --- a/src/common/varint.h +++ b/src/common/varint.h @@ -123,6 +123,6 @@ namespace tools { */ template<typename InputIt, typename T> int read_varint(InputIt &&first, InputIt &&last, T &i) { - return read_varint<std::numeric_limits<T>::digits, InputIt, T>(std::move(first), std::move(last), i); + return read_varint<std::numeric_limits<T>::digits>(std::forward<InputIt>(first), std::forward<InputIt>(last), i); } } diff --git a/src/crypto/aesb.c b/src/crypto/aesb.c index 5d57b8af4..efdeef8d1 100644 --- a/src/crypto/aesb.c +++ b/src/crypto/aesb.c @@ -19,6 +19,7 @@ Issue Date: 20/12/2007 */ #include <stdint.h> +#include "int-util.h" #if defined(__cplusplus) extern "C" @@ -50,7 +51,7 @@ extern "C" #define state_out(y,x) so(y,x,0); so(y,x,1); so(y,x,2); so(y,x,3) #define round(rm,y,x,k) rm(y,x,k,0); rm(y,x,k,1); rm(y,x,k,2); rm(y,x,k,3) #define to_byte(x) ((x) & 0xff) -#define bval(x,n) to_byte((x) >> (8 * (n))) +#define bval(x,n) to_byte(SWAP32LE(x) >> (8 * (n))) #define fwd_var(x,r,c)\ ( r == 0 ? ( c == 0 ? s(x,0) : c == 1 ? s(x,1) : c == 2 ? s(x,2) : s(x,3))\ @@ -58,7 +59,7 @@ extern "C" : r == 2 ? ( c == 0 ? s(x,2) : c == 1 ? s(x,3) : c == 2 ? s(x,0) : s(x,1))\ : ( c == 0 ? s(x,3) : c == 1 ? s(x,0) : c == 2 ? s(x,1) : s(x,2))) -#define fwd_rnd(y,x,k,c) (s(y,c) = (k)[c] ^ four_tables(x,t_use(f,n),fwd_var,rf1,c)) +#define fwd_rnd(y,x,k,c) (s(y,c) = (k)[c] ^ SWAP32LE(four_tables(x,t_use(f,n),fwd_var,rf1,c))) #define sb_data(w) {\ w(0x63), w(0x7c), w(0x77), w(0x7b), w(0xf2), w(0x6b), w(0x6f), w(0xc5),\ diff --git a/src/crypto/chacha.c b/src/crypto/chacha.c index 5d3edb98d..d734e8b1b 100644 --- a/src/crypto/chacha.c +++ b/src/crypto/chacha.c @@ -11,7 +11,7 @@ Public domain. #endif #include "chacha.h" -#include "common/int-util.h" +#include "int-util.h" #include "warnings.h" /* diff --git a/src/crypto/crypto.cpp b/src/crypto/crypto.cpp index ad7721cf0..ddf072f68 100644 --- a/src/crypto/crypto.cpp +++ b/src/crypto/crypto.cpp @@ -34,7 +34,6 @@ #include <cstdint> #include <cstdlib> #include <cstring> -#include <memory> #include <boost/thread/mutex.hpp> #include <boost/thread/lock_guard.hpp> #include <boost/shared_ptr.hpp> diff --git a/src/crypto/crypto.h b/src/crypto/crypto.h index 33cc0a25a..f22df1230 100644 --- a/src/crypto/crypto.h +++ b/src/crypto/crypto.h @@ -32,14 +32,11 @@ #include <cstddef> #include <iostream> -#include <boost/thread/mutex.hpp> -#include <boost/thread/lock_guard.hpp> #include <boost/optional.hpp> #include <type_traits> #include <vector> #include "common/pod-class.h" -#include "common/util.h" #include "memwipe.h" #include "mlocker.h" #include "generic-ops.h" diff --git a/src/crypto/groestl.c b/src/crypto/groestl.c index c8258add3..d5e2989a8 100644 --- a/src/crypto/groestl.c +++ b/src/crypto/groestl.c @@ -20,9 +20,15 @@ const uint8_t shift_Values[2][8] = {{0,1,2,3,4,5,6,7},{1,3,5,7,0,2,4,6}}; const uint8_t indices_cyclic[15] = {0,1,2,3,4,5,6,7,0,1,2,3,4,5,6}; +#if BYTE_ORDER == LITTLE_ENDIAN #define ROTATE_COLUMN_DOWN(v1, v2, amount_bytes, temp_var) {temp_var = (v1<<(8*amount_bytes))|(v2>>(8*(4-amount_bytes))); \ v2 = (v2<<(8*amount_bytes))|(v1>>(8*(4-amount_bytes))); \ v1 = temp_var;} +#else +#define ROTATE_COLUMN_DOWN(v1, v2, amount_bytes, temp_var) {temp_var = (v1>>(8*amount_bytes))|(v2<<(8*(4-amount_bytes))); \ + v2 = (v2>>(8*amount_bytes))|(v1<<(8*(4-amount_bytes))); \ + v1 = temp_var;} +#endif #define COLUMN(x,y,i,c0,c1,c2,c3,c4,c5,c6,c7,tv1,tv2,tu,tl,t) \ @@ -68,14 +74,14 @@ const uint8_t indices_cyclic[15] = {0,1,2,3,4,5,6,7,0,1,2,3,4,5,6}; static void RND512P(uint8_t *x, uint32_t *y, uint32_t r) { uint32_t temp_v1, temp_v2, temp_upper_value, temp_lower_value, temp; uint32_t* x32 = (uint32_t*)x; - x32[ 0] ^= 0x00000000^r; - x32[ 2] ^= 0x00000010^r; - x32[ 4] ^= 0x00000020^r; - x32[ 6] ^= 0x00000030^r; - x32[ 8] ^= 0x00000040^r; - x32[10] ^= 0x00000050^r; - x32[12] ^= 0x00000060^r; - x32[14] ^= 0x00000070^r; + x32[ 0] ^= SWAP32LE(0x00000000)^r; + x32[ 2] ^= SWAP32LE(0x00000010)^r; + x32[ 4] ^= SWAP32LE(0x00000020)^r; + x32[ 6] ^= SWAP32LE(0x00000030)^r; + x32[ 8] ^= SWAP32LE(0x00000040)^r; + x32[10] ^= SWAP32LE(0x00000050)^r; + x32[12] ^= SWAP32LE(0x00000060)^r; + x32[14] ^= SWAP32LE(0x00000070)^r; COLUMN(x,y, 0, 0, 2, 4, 6, 9, 11, 13, 15, temp_v1, temp_v2, temp_upper_value, temp_lower_value, temp); COLUMN(x,y, 2, 2, 4, 6, 8, 11, 13, 15, 1, temp_v1, temp_v2, temp_upper_value, temp_lower_value, temp); COLUMN(x,y, 4, 4, 6, 8, 10, 13, 15, 1, 3, temp_v1, temp_v2, temp_upper_value, temp_lower_value, temp); @@ -91,21 +97,22 @@ static void RND512Q(uint8_t *x, uint32_t *y, uint32_t r) { uint32_t temp_v1, temp_v2, temp_upper_value, temp_lower_value, temp; uint32_t* x32 = (uint32_t*)x; x32[ 0] = ~x32[ 0]; - x32[ 1] ^= 0xffffffff^r; + x32[ 1] ^= SWAP32LE(0xffffffff)^r; x32[ 2] = ~x32[ 2]; - x32[ 3] ^= 0xefffffff^r; + x32[ 3] ^= SWAP32LE(0xefffffff)^r; x32[ 4] = ~x32[ 4]; - x32[ 5] ^= 0xdfffffff^r; + x32[ 5] ^= SWAP32LE(0xdfffffff)^r; x32[ 6] = ~x32[ 6]; - x32[ 7] ^= 0xcfffffff^r; + x32[ 7] ^= SWAP32LE(0xcfffffff)^r; x32[ 8] = ~x32[ 8]; - x32[ 9] ^= 0xbfffffff^r; + x32[ 9] ^= SWAP32LE(0xbfffffff)^r; x32[10] = ~x32[10]; - x32[11] ^= 0xafffffff^r; + x32[11] ^= SWAP32LE(0xafffffff)^r; x32[12] = ~x32[12]; - x32[13] ^= 0x9fffffff^r; + x32[13] ^= SWAP32LE(0x9fffffff)^r; x32[14] = ~x32[14]; - x32[15] ^= 0x8fffffff^r; + x32[15] ^= SWAP32LE(0x8fffffff)^r; + COLUMN(x,y, 0, 2, 6, 10, 14, 1, 5, 9, 13, temp_v1, temp_v2, temp_upper_value, temp_lower_value, temp); COLUMN(x,y, 2, 4, 8, 12, 0, 3, 7, 11, 15, temp_v1, temp_v2, temp_upper_value, temp_lower_value, temp); COLUMN(x,y, 4, 6, 10, 14, 2, 5, 9, 13, 1, temp_v1, temp_v2, temp_upper_value, temp_lower_value, temp); @@ -130,28 +137,28 @@ static void F512(uint32_t *h, const uint32_t *m) { } /* compute Q(m) */ - RND512Q((uint8_t*)z, y, 0x00000000); - RND512Q((uint8_t*)y, z, 0x01000000); - RND512Q((uint8_t*)z, y, 0x02000000); - RND512Q((uint8_t*)y, z, 0x03000000); - RND512Q((uint8_t*)z, y, 0x04000000); - RND512Q((uint8_t*)y, z, 0x05000000); - RND512Q((uint8_t*)z, y, 0x06000000); - RND512Q((uint8_t*)y, z, 0x07000000); - RND512Q((uint8_t*)z, y, 0x08000000); - RND512Q((uint8_t*)y, Qtmp, 0x09000000); + RND512Q((uint8_t*)z, y, SWAP32LE(0x00000000)); + RND512Q((uint8_t*)y, z, SWAP32LE(0x01000000)); + RND512Q((uint8_t*)z, y, SWAP32LE(0x02000000)); + RND512Q((uint8_t*)y, z, SWAP32LE(0x03000000)); + RND512Q((uint8_t*)z, y, SWAP32LE(0x04000000)); + RND512Q((uint8_t*)y, z, SWAP32LE(0x05000000)); + RND512Q((uint8_t*)z, y, SWAP32LE(0x06000000)); + RND512Q((uint8_t*)y, z, SWAP32LE(0x07000000)); + RND512Q((uint8_t*)z, y, SWAP32LE(0x08000000)); + RND512Q((uint8_t*)y, Qtmp, SWAP32LE(0x09000000)); /* compute P(h+m) */ - RND512P((uint8_t*)Ptmp, y, 0x00000000); - RND512P((uint8_t*)y, z, 0x00000001); - RND512P((uint8_t*)z, y, 0x00000002); - RND512P((uint8_t*)y, z, 0x00000003); - RND512P((uint8_t*)z, y, 0x00000004); - RND512P((uint8_t*)y, z, 0x00000005); - RND512P((uint8_t*)z, y, 0x00000006); - RND512P((uint8_t*)y, z, 0x00000007); - RND512P((uint8_t*)z, y, 0x00000008); - RND512P((uint8_t*)y, Ptmp, 0x00000009); + RND512P((uint8_t*)Ptmp, y, SWAP32LE(0x00000000)); + RND512P((uint8_t*)y, z, SWAP32LE(0x00000001)); + RND512P((uint8_t*)z, y, SWAP32LE(0x00000002)); + RND512P((uint8_t*)y, z, SWAP32LE(0x00000003)); + RND512P((uint8_t*)z, y, SWAP32LE(0x00000004)); + RND512P((uint8_t*)y, z, SWAP32LE(0x00000005)); + RND512P((uint8_t*)z, y, SWAP32LE(0x00000006)); + RND512P((uint8_t*)y, z, SWAP32LE(0x00000007)); + RND512P((uint8_t*)z, y, SWAP32LE(0x00000008)); + RND512P((uint8_t*)y, Ptmp, SWAP32LE(0x00000009)); /* compute P(h+m) + Q(m) + h */ for (i = 0; i < 2*COLS512; i++) { @@ -188,16 +195,16 @@ static void OutputTransformation(hashState *ctx) { for (j = 0; j < 2*COLS512; j++) { temp[j] = ctx->chaining[j]; } - RND512P((uint8_t*)temp, y, 0x00000000); - RND512P((uint8_t*)y, z, 0x00000001); - RND512P((uint8_t*)z, y, 0x00000002); - RND512P((uint8_t*)y, z, 0x00000003); - RND512P((uint8_t*)z, y, 0x00000004); - RND512P((uint8_t*)y, z, 0x00000005); - RND512P((uint8_t*)z, y, 0x00000006); - RND512P((uint8_t*)y, z, 0x00000007); - RND512P((uint8_t*)z, y, 0x00000008); - RND512P((uint8_t*)y, temp, 0x00000009); + RND512P((uint8_t*)temp, y, SWAP32LE(0x00000000)); + RND512P((uint8_t*)y, z, SWAP32LE(0x00000001)); + RND512P((uint8_t*)z, y, SWAP32LE(0x00000002)); + RND512P((uint8_t*)y, z, SWAP32LE(0x00000003)); + RND512P((uint8_t*)z, y, SWAP32LE(0x00000004)); + RND512P((uint8_t*)y, z, SWAP32LE(0x00000005)); + RND512P((uint8_t*)z, y, SWAP32LE(0x00000006)); + RND512P((uint8_t*)y, z, SWAP32LE(0x00000007)); + RND512P((uint8_t*)z, y, SWAP32LE(0x00000008)); + RND512P((uint8_t*)y, temp, SWAP32LE(0x00000009)); for (j = 0; j < 2*COLS512; j++) { ctx->chaining[j] ^= temp[j]; } @@ -213,7 +220,7 @@ static void Init(hashState* ctx) { } /* set initial value */ - ctx->chaining[2*COLS512-1] = u32BIG((uint32_t)HASH_BIT_LEN); + ctx->chaining[2*COLS512-1] = SWAP32LE(u32BIG((uint32_t)HASH_BIT_LEN)); /* set other variables */ ctx->buf_ptr = 0; diff --git a/src/crypto/groestl_tables.h b/src/crypto/groestl_tables.h index c4b368584..12472dced 100644 --- a/src/crypto/groestl_tables.h +++ b/src/crypto/groestl_tables.h @@ -29,7 +29,10 @@ #ifndef __tables_h #define __tables_h +#include "int-util.h" + +#if BYTE_ORDER == LITTLE_ENDIAN const uint32_t T[512] = {0xa5f432c6, 0xc6a597f4, 0x84976ff8, 0xf884eb97, 0x99b05eee, 0xee99c7b0, 0x8d8c7af6, 0xf68df78c, 0xd17e8ff, 0xff0de517, 0xbddc0ad6, 0xd6bdb7dc, 0xb1c816de, 0xdeb1a7c8, 0x54fc6d91, 0x915439fc , 0x50f09060, 0x6050c0f0, 0x3050702, 0x2030405, 0xa9e02ece, 0xcea987e0, 0x7d87d156, 0x567dac87, 0x192bcce7, 0xe719d52b, 0x62a613b5, 0xb56271a6, 0xe6317c4d, 0x4de69a31, 0x9ab559ec, 0xec9ac3b5 , 0x45cf408f, 0x8f4505cf, 0x9dbca31f, 0x1f9d3ebc, 0x40c04989, 0x894009c0, 0x879268fa, 0xfa87ef92, 0x153fd0ef, 0xef15c53f, 0xeb2694b2, 0xb2eb7f26, 0xc940ce8e, 0x8ec90740, 0xb1de6fb, 0xfb0bed1d @@ -62,5 +65,39 @@ const uint32_t T[512] = {0xa5f432c6, 0xc6a597f4, 0x84976ff8, 0xf884eb97, 0x99b05 , 0xb6c1ec2d, 0x2db65ac1, 0x22665a3c, 0x3c227866, 0x92adb815, 0x15922aad, 0x2060a9c9, 0xc9208960, 0x49db5c87, 0x874915db, 0xff1ab0aa, 0xaaff4f1a, 0x7888d850, 0x5078a088, 0x7a8e2ba5, 0xa57a518e , 0x8f8a8903, 0x38f068a, 0xf8134a59, 0x59f8b213, 0x809b9209, 0x980129b, 0x1739231a, 0x1a173439, 0xda751065, 0x65daca75, 0x315384d7, 0xd731b553, 0xc651d584, 0x84c61351, 0xb8d303d0, 0xd0b8bbd3 , 0xc35edc82, 0x82c31f5e, 0xb0cbe229, 0x29b052cb, 0x7799c35a, 0x5a77b499, 0x11332d1e, 0x1e113c33, 0xcb463d7b, 0x7bcbf646, 0xfc1fb7a8, 0xa8fc4b1f, 0xd6610c6d, 0x6dd6da61, 0x3a4e622c, 0x2c3a584e}; +#else +const uint32_t T[512] = {0xc632f4a5, 0xf497a5c6, 0xf86f9784, 0x97eb84f8, 0xee5eb099, 0xb0c799ee, 0xf67a8c8d, 0x8cf78df6, 0xffe8170d, 0x17e50dff, 0xd60adcbd, 0xdcb7bdd6, 0xde16c8b1, 0xc8a7b1de, 0x916dfc54, 0xfc395491 +, 0x6090f050, 0xf0c05060, 0x02070503, 0x05040302, 0xce2ee0a9, 0xe087a9ce, 0x56d1877d, 0x87ac7d56, 0xe7cc2b19, 0x2bd519e7, 0xb513a662, 0xa67162b5, 0x4d7c31e6, 0x319ae64d, 0xec59b59a, 0xb5c39aec +, 0x8f40cf45, 0xcf05458f, 0x1fa3bc9d, 0xbc3e9d1f, 0x8949c040, 0xc0094089, 0xfa689287, 0x92ef87fa, 0xefd03f15, 0x3fc515ef, 0xb29426eb, 0x267febb2, 0x8ece40c9, 0x4007c98e, 0xfbe61d0b, 0x1ded0bfb +, 0x416e2fec, 0x2f82ec41, 0xb31aa967, 0xa97d67b3, 0x5f431cfd, 0x1cbefd5f, 0x456025ea, 0x258aea45, 0x23f9dabf, 0xda46bf23, 0x535102f7, 0x02a6f753, 0xe445a196, 0xa1d396e4, 0x9b76ed5b, 0xed2d5b9b +, 0x75285dc2, 0x5deac275, 0xe1c5241c, 0x24d91ce1, 0x3dd4e9ae, 0xe97aae3d, 0x4cf2be6a, 0xbe986a4c, 0x6c82ee5a, 0xeed85a6c, 0x7ebdc341, 0xc3fc417e, 0xf5f30602, 0x06f102f5, 0x8352d14f, 0xd11d4f83 +, 0x688ce45c, 0xe4d05c68, 0x515607f4, 0x07a2f451, 0xd18d5c34, 0x5cb934d1, 0xf9e11808, 0x18e908f9, 0xe24cae93, 0xaedf93e2, 0xab3e9573, 0x954d73ab, 0x6297f553, 0xf5c45362, 0x2a6b413f, 0x41543f2a +, 0x081c140c, 0x14100c08, 0x9563f652, 0xf6315295, 0x46e9af65, 0xaf8c6546, 0x9d7fe25e, 0xe2215e9d, 0x30487828, 0x78602830, 0x37cff8a1, 0xf86ea137, 0x0a1b110f, 0x11140f0a, 0x2febc4b5, 0xc45eb52f +, 0x0e151b09, 0x1b1c090e, 0x247e5a36, 0x5a483624, 0x1badb69b, 0xb6369b1b, 0xdf98473d, 0x47a53ddf, 0xcda76a26, 0x6a8126cd, 0x4ef5bb69, 0xbb9c694e, 0x7f334ccd, 0x4cfecd7f, 0xea50ba9f, 0xbacf9fea +, 0x123f2d1b, 0x2d241b12, 0x1da4b99e, 0xb93a9e1d, 0x58c49c74, 0x9cb07458, 0x3446722e, 0x72682e34, 0x3641772d, 0x776c2d36, 0xdc11cdb2, 0xcda3b2dc, 0xb49d29ee, 0x2973eeb4, 0x5b4d16fb, 0x16b6fb5b +, 0xa4a501f6, 0x0153f6a4, 0x76a1d74d, 0xd7ec4d76, 0xb714a361, 0xa37561b7, 0x7d3449ce, 0x49face7d, 0x52df8d7b, 0x8da47b52, 0xdd9f423e, 0x42a13edd, 0x5ecd9371, 0x93bc715e, 0x13b1a297, 0xa2269713 +, 0xa6a204f5, 0x0457f5a6, 0xb901b868, 0xb86968b9, 0x00000000, 0x00000000, 0xc1b5742c, 0x74992cc1, 0x40e0a060, 0xa0806040, 0xe3c2211f, 0x21dd1fe3, 0x793a43c8, 0x43f2c879, 0xb69a2ced, 0x2c77edb6 +, 0xd40dd9be, 0xd9b3bed4, 0x8d47ca46, 0xca01468d, 0x671770d9, 0x70ced967, 0x72afdd4b, 0xdde44b72, 0x94ed79de, 0x7933de94, 0x98ff67d4, 0x672bd498, 0xb09323e8, 0x237be8b0, 0x855bde4a, 0xde114a85 +, 0xbb06bd6b, 0xbd6d6bbb, 0xc5bb7e2a, 0x7e912ac5, 0x4f7b34e5, 0x349ee54f, 0xedd73a16, 0x3ac116ed, 0x86d254c5, 0x5417c586, 0x9af862d7, 0x622fd79a, 0x6699ff55, 0xffcc5566, 0x11b6a794, 0xa7229411 +, 0x8ac04acf, 0x4a0fcf8a, 0xe9d93010, 0x30c910e9, 0x040e0a06, 0x0a080604, 0xfe669881, 0x98e781fe, 0xa0ab0bf0, 0x0b5bf0a0, 0x78b4cc44, 0xccf04478, 0x25f0d5ba, 0xd54aba25, 0x4b753ee3, 0x3e96e34b +, 0xa2ac0ef3, 0x0e5ff3a2, 0x5d4419fe, 0x19bafe5d, 0x80db5bc0, 0x5b1bc080, 0x0580858a, 0x850a8a05, 0x3fd3ecad, 0xec7ead3f, 0x21fedfbc, 0xdf42bc21, 0x70a8d848, 0xd8e04870, 0xf1fd0c04, 0x0cf904f1 +, 0x63197adf, 0x7ac6df63, 0x772f58c1, 0x58eec177, 0xaf309f75, 0x9f4575af, 0x42e7a563, 0xa5846342, 0x20705030, 0x50403020, 0xe5cb2e1a, 0x2ed11ae5, 0xfdef120e, 0x12e10efd, 0xbf08b76d, 0xb7656dbf +, 0x8155d44c, 0xd4194c81, 0x18243c14, 0x3c301418, 0x26795f35, 0x5f4c3526, 0xc3b2712f, 0x719d2fc3, 0xbe8638e1, 0x3867e1be, 0x35c8fda2, 0xfd6aa235, 0x88c74fcc, 0x4f0bcc88, 0x2e654b39, 0x4b5c392e +, 0x936af957, 0xf93d5793, 0x55580df2, 0x0daaf255, 0xfc619d82, 0x9de382fc, 0x7ab3c947, 0xc9f4477a, 0xc827efac, 0xef8bacc8, 0xba8832e7, 0x326fe7ba, 0x324f7d2b, 0x7d642b32, 0xe642a495, 0xa4d795e6 +, 0xc03bfba0, 0xfb9ba0c0, 0x19aab398, 0xb3329819, 0x9ef668d1, 0x6827d19e, 0xa322817f, 0x815d7fa3, 0x44eeaa66, 0xaa886644, 0x54d6827e, 0x82a87e54, 0x3bdde6ab, 0xe676ab3b, 0x0b959e83, 0x9e16830b +, 0x8cc945ca, 0x4503ca8c, 0xc7bc7b29, 0x7b9529c7, 0x6b056ed3, 0x6ed6d36b, 0x286c443c, 0x44503c28, 0xa72c8b79, 0x8b5579a7, 0xbc813de2, 0x3d63e2bc, 0x1631271d, 0x272c1d16, 0xad379a76, 0x9a4176ad +, 0xdb964d3b, 0x4dad3bdb, 0x649efa56, 0xfac85664, 0x74a6d24e, 0xd2e84e74, 0x1436221e, 0x22281e14, 0x92e476db, 0x763fdb92, 0x0c121e0a, 0x1e180a0c, 0x48fcb46c, 0xb4906c48, 0xb88f37e4, 0x376be4b8 +, 0x9f78e75d, 0xe7255d9f, 0xbd0fb26e, 0xb2616ebd, 0x43692aef, 0x2a86ef43, 0xc435f1a6, 0xf193a6c4, 0x39dae3a8, 0xe372a839, 0x31c6f7a4, 0xf762a431, 0xd38a5937, 0x59bd37d3, 0xf274868b, 0x86ff8bf2 +, 0xd5835632, 0x56b132d5, 0x8b4ec543, 0xc50d438b, 0x6e85eb59, 0xebdc596e, 0xda18c2b7, 0xc2afb7da, 0x018e8f8c, 0x8f028c01, 0xb11dac64, 0xac7964b1, 0x9cf16dd2, 0x6d23d29c, 0x49723be0, 0x3b92e049 +, 0xd81fc7b4, 0xc7abb4d8, 0xacb915fa, 0x1543faac, 0xf3fa0907, 0x09fd07f3, 0xcfa06f25, 0x6f8525cf, 0xca20eaaf, 0xea8fafca, 0xf47d898e, 0x89f38ef4, 0x476720e9, 0x208ee947, 0x10382818, 0x28201810 +, 0x6f0b64d5, 0x64ded56f, 0xf0738388, 0x83fb88f0, 0x4afbb16f, 0xb1946f4a, 0x5cca9672, 0x96b8725c, 0x38546c24, 0x6c702438, 0x575f08f1, 0x08aef157, 0x732152c7, 0x52e6c773, 0x9764f351, 0xf3355197 +, 0xcbae6523, 0x658d23cb, 0xa125847c, 0x84597ca1, 0xe857bf9c, 0xbfcb9ce8, 0x3e5d6321, 0x637c213e, 0x96ea7cdd, 0x7c37dd96, 0x611e7fdc, 0x7fc2dc61, 0x0d9c9186, 0x911a860d, 0x0f9b9485, 0x941e850f +, 0xe04bab90, 0xabdb90e0, 0x7cbac642, 0xc6f8427c, 0x712657c4, 0x57e2c471, 0xcc29e5aa, 0xe583aacc, 0x90e373d8, 0x733bd890, 0x06090f05, 0x0f0c0506, 0xf7f40301, 0x03f501f7, 0x1c2a3612, 0x3638121c +, 0xc23cfea3, 0xfe9fa3c2, 0x6a8be15f, 0xe1d45f6a, 0xaebe10f9, 0x1047f9ae, 0x69026bd0, 0x6bd2d069, 0x17bfa891, 0xa82e9117, 0x9971e858, 0xe8295899, 0x3a536927, 0x6974273a, 0x27f7d0b9, 0xd04eb927 +, 0xd9914838, 0x48a938d9, 0xebde3513, 0x35cd13eb, 0x2be5ceb3, 0xce56b32b, 0x22775533, 0x55443322, 0xd204d6bb, 0xd6bfbbd2, 0xa9399070, 0x904970a9, 0x07878089, 0x800e8907, 0x33c1f2a7, 0xf266a733 +, 0x2decc1b6, 0xc15ab62d, 0x3c5a6622, 0x6678223c, 0x15b8ad92, 0xad2a9215, 0xc9a96020, 0x608920c9, 0x875cdb49, 0xdb154987, 0xaab01aff, 0x1a4fffaa, 0x50d88878, 0x88a07850, 0xa52b8e7a, 0x8e517aa5 +, 0x03898a8f, 0x8a068f03, 0x594a13f8, 0x13b2f859, 0x09929b80, 0x9b128009, 0x1a233917, 0x3934171a, 0x651075da, 0x75cada65, 0xd7845331, 0x53b531d7, 0x84d551c6, 0x5113c684, 0xd003d3b8, 0xd3bbb8d0 +, 0x82dc5ec3, 0x5e1fc382, 0x29e2cbb0, 0xcb52b029, 0x5ac39977, 0x99b4775a, 0x1e2d3311, 0x333c111e, 0x7b3d46cb, 0x46f6cb7b, 0xa8b71ffc, 0x1f4bfca8, 0x6d0c61d6, 0x61dad66d, 0x2c624e3a, 0x4e583a2c}; +#endif #endif /* __tables_h */ diff --git a/src/crypto/hash-ops.h b/src/crypto/hash-ops.h index d77d55cf3..77b52e2d4 100644 --- a/src/crypto/hash-ops.h +++ b/src/crypto/hash-ops.h @@ -37,7 +37,7 @@ #include <stddef.h> #include <stdint.h> -#include "common/int-util.h" +#include "int-util.h" #include "warnings.h" static inline void *padd(void *p, size_t i) { diff --git a/src/crypto/hash.c b/src/crypto/hash.c index 42f272e34..43ce32957 100644 --- a/src/crypto/hash.c +++ b/src/crypto/hash.c @@ -36,7 +36,14 @@ #include "keccak.h" void hash_permutation(union hash_state *state) { +#if BYTE_ORDER == LITTLE_ENDIAN keccakf((uint64_t*)state, 24); +#else + uint64_t le_state[25]; + memcpy_swap64le(le_state, state, 25); + keccakf(le_state, 24); + memcpy_swap64le(state, le_state, 25); +#endif } void hash_process(union hash_state *state, const uint8_t *buf, size_t count) { diff --git a/src/crypto/keccak.c b/src/crypto/keccak.c index b5946036e..170911262 100644 --- a/src/crypto/keccak.c +++ b/src/crypto/keccak.c @@ -5,7 +5,7 @@ #include <stdio.h> #include <stdlib.h> #include <unistd.h> -#include "common/int-util.h" +#include "int-util.h" #include "hash-ops.h" #include "keccak.h" @@ -145,7 +145,7 @@ void keccak1600(const uint8_t *in, size_t inlen, uint8_t *md) #define IS_ALIGNED_64(p) (0 == (7 & ((const char*)(p) - (const char*)0))) #define KECCAK_PROCESS_BLOCK(st, block) { \ for (int i_ = 0; i_ < KECCAK_WORDS; i_++){ \ - ((st))[i_] ^= ((block))[i_]; \ + ((st))[i_] ^= swap64le(((block))[i_]); \ }; \ keccakf(st, KECCAK_ROUNDS); } @@ -207,7 +207,8 @@ void keccak_finish(KECCAK_CTX * ctx, uint8_t *md){ } static_assert(KECCAK_BLOCKLEN > KECCAK_DIGESTSIZE, ""); + static_assert(KECCAK_DIGESTSIZE % sizeof(uint64_t) == 0, ""); if (md) { - memcpy(md, ctx->hash, KECCAK_DIGESTSIZE); + memcpy_swap64le(md, ctx->hash, KECCAK_DIGESTSIZE / sizeof(uint64_t)); } } diff --git a/src/crypto/skein_port.h b/src/crypto/skein_port.h index a50a28e6b..8a1640e57 100644 --- a/src/crypto/skein_port.h +++ b/src/crypto/skein_port.h @@ -114,7 +114,7 @@ typedef uint64_t u64b_t; /* 64-bit unsigned integer */ #ifndef SKEIN_NEED_SWAP /* compile-time "override" for endianness? */ -#include "common/int-util.h" +#include "int-util.h" #define IS_BIG_ENDIAN 4321 /* byte 0 is most significant (mc68k) */ #define IS_LITTLE_ENDIAN 1234 /* byte 0 is least significant (i386) */ diff --git a/src/crypto/slow-hash.c b/src/crypto/slow-hash.c index 40cfb0461..ae0bd4e98 100644 --- a/src/crypto/slow-hash.c +++ b/src/crypto/slow-hash.c @@ -35,7 +35,7 @@ #include <stdio.h> #include <unistd.h> -#include "common/int-util.h" +#include "int-util.h" #include "hash-ops.h" #include "oaes_lib.h" #include "variant2_int_sqrt.h" @@ -47,8 +47,8 @@ #define INIT_SIZE_BLK 8 #define INIT_SIZE_BYTE (INIT_SIZE_BLK * AES_BLOCK_SIZE) -extern int aesb_single_round(const uint8_t *in, uint8_t*out, const uint8_t *expandedKey); -extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *expandedKey); +extern void aesb_single_round(const uint8_t *in, uint8_t *out, const uint8_t *expandedKey); +extern void aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *expandedKey); #define VARIANT1_1(p) \ do if (variant == 1) \ @@ -109,8 +109,8 @@ extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *exp memcpy(b + AES_BLOCK_SIZE, state.hs.b + 64, AES_BLOCK_SIZE); \ xor64(b + AES_BLOCK_SIZE, state.hs.b + 80); \ xor64(b + AES_BLOCK_SIZE + 8, state.hs.b + 88); \ - division_result = state.hs.w[12]; \ - sqrt_result = state.hs.w[13]; \ + division_result = SWAP64LE(state.hs.w[12]); \ + sqrt_result = SWAP64LE(state.hs.w[13]); \ } while (0) #define VARIANT2_SHUFFLE_ADD_SSE2(base_ptr, offset) \ @@ -145,30 +145,31 @@ extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *exp const uint64_t chunk1_old[2] = { chunk1[0], chunk1[1] }; \ \ uint64_t b1[2]; \ - memcpy(b1, b + 16, 16); \ - chunk1[0] = chunk3[0] + b1[0]; \ - chunk1[1] = chunk3[1] + b1[1]; \ + memcpy_swap64le(b1, b + 16, 2); \ + chunk1[0] = SWAP64LE(SWAP64LE(chunk3[0]) + b1[0]); \ + chunk1[1] = SWAP64LE(SWAP64LE(chunk3[1]) + b1[1]); \ \ uint64_t a0[2]; \ - memcpy(a0, a, 16); \ - chunk3[0] = chunk2[0] + a0[0]; \ - chunk3[1] = chunk2[1] + a0[1]; \ + memcpy_swap64le(a0, a, 2); \ + chunk3[0] = SWAP64LE(SWAP64LE(chunk2[0]) + a0[0]); \ + chunk3[1] = SWAP64LE(SWAP64LE(chunk2[1]) + a0[1]); \ \ uint64_t b0[2]; \ - memcpy(b0, b, 16); \ - chunk2[0] = chunk1_old[0] + b0[0]; \ - chunk2[1] = chunk1_old[1] + b0[1]; \ + memcpy_swap64le(b0, b, 2); \ + chunk2[0] = SWAP64LE(SWAP64LE(chunk1_old[0]) + b0[0]); \ + chunk2[1] = SWAP64LE(SWAP64LE(chunk1_old[1]) + b0[1]); \ } while (0) #define VARIANT2_INTEGER_MATH_DIVISION_STEP(b, ptr) \ - ((uint64_t*)(b))[0] ^= division_result ^ (sqrt_result << 32); \ + uint64_t tmpx = division_result ^ (sqrt_result << 32); \ + ((uint64_t*)(b))[0] ^= SWAP64LE(tmpx); \ { \ - const uint64_t dividend = ((uint64_t*)(ptr))[1]; \ - const uint32_t divisor = (((uint64_t*)(ptr))[0] + (uint32_t)(sqrt_result << 1)) | 0x80000001UL; \ + const uint64_t dividend = SWAP64LE(((uint64_t*)(ptr))[1]); \ + const uint32_t divisor = (SWAP64LE(((uint64_t*)(ptr))[0]) + (uint32_t)(sqrt_result << 1)) | 0x80000001UL; \ division_result = ((uint32_t)(dividend / divisor)) + \ (((uint64_t)(dividend % divisor)) << 32); \ } \ - const uint64_t sqrt_input = ((uint64_t*)(ptr))[0] + division_result + const uint64_t sqrt_input = SWAP64LE(((uint64_t*)(ptr))[0]) + division_result #define VARIANT2_INTEGER_MATH_SSE2(b, ptr) \ do if (variant >= 2) \ @@ -207,10 +208,10 @@ extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *exp #define VARIANT2_2() \ do if (variant >= 2) \ { \ - *U64(hp_state + (j ^ 0x10)) ^= hi; \ - *(U64(hp_state + (j ^ 0x10)) + 1) ^= lo; \ - hi ^= *U64(hp_state + (j ^ 0x20)); \ - lo ^= *(U64(hp_state + (j ^ 0x20)) + 1); \ + *U64(hp_state + (j ^ 0x10)) ^= SWAP64LE(hi); \ + *(U64(hp_state + (j ^ 0x10)) + 1) ^= SWAP64LE(lo); \ + hi ^= SWAP64LE(*U64(hp_state + (j ^ 0x20))); \ + lo ^= SWAP64LE(*(U64(hp_state + (j ^ 0x20)) + 1)); \ } while (0) @@ -1408,10 +1409,7 @@ static void (*const extra_hashes[4])(const void *, size_t, char *) = { hash_extra_blake, hash_extra_groestl, hash_extra_jh, hash_extra_skein }; -extern int aesb_single_round(const uint8_t *in, uint8_t*out, const uint8_t *expandedKey); -extern int aesb_pseudo_round(const uint8_t *in, uint8_t *out, const uint8_t *expandedKey); - -static size_t e2i(const uint8_t* a, size_t count) { return (*((uint64_t*)a) / AES_BLOCK_SIZE) & (count - 1); } +static size_t e2i(const uint8_t* a, size_t count) { return (SWAP64LE(*((uint64_t*)a)) / AES_BLOCK_SIZE) & (count - 1); } static void mul(const uint8_t* a, const uint8_t* b, uint8_t* res) { uint64_t a0, b0; diff --git a/src/cryptonote_basic/account.cpp b/src/cryptonote_basic/account.cpp index e891a748d..edbc2c561 100644 --- a/src/cryptonote_basic/account.cpp +++ b/src/cryptonote_basic/account.cpp @@ -136,6 +136,16 @@ DISABLE_VS_WARNINGS(4244 4345) void account_base::set_null() { m_keys = account_keys(); + m_creation_timestamp = 0; + } + //----------------------------------------------------------------- + void account_base::deinit() + { + try{ + m_keys.get_device().disconnect(); + } catch (const std::exception &e){ + MERROR("Device disconnect exception: " << e.what()); + } } //----------------------------------------------------------------- void account_base::forget_spend_key() @@ -205,11 +215,16 @@ DISABLE_VS_WARNINGS(4244 4345) void account_base::create_from_device(hw::device &hwdev) { m_keys.set_device(hwdev); - MCDEBUG("ledger", "device type: "<<typeid(hwdev).name()); - hwdev.init(); - hwdev.connect(); - hwdev.get_public_address(m_keys.m_account_address); - hwdev.get_secret_keys(m_keys.m_view_secret_key, m_keys.m_spend_secret_key); + MCDEBUG("device", "device type: "<<typeid(hwdev).name()); + CHECK_AND_ASSERT_THROW_MES(hwdev.init(), "Device init failed"); + CHECK_AND_ASSERT_THROW_MES(hwdev.connect(), "Device connect failed"); + try { + CHECK_AND_ASSERT_THROW_MES(hwdev.get_public_address(m_keys.m_account_address), "Cannot get a device address"); + CHECK_AND_ASSERT_THROW_MES(hwdev.get_secret_keys(m_keys.m_view_secret_key, m_keys.m_spend_secret_key), "Cannot get device secret"); + } catch (const std::exception &e){ + hwdev.disconnect(); + throw; + } struct tm timestamp = {0}; timestamp.tm_year = 2014 - 1900; // year 2014 timestamp.tm_mon = 4 - 1; // month april diff --git a/src/cryptonote_basic/account.h b/src/cryptonote_basic/account.h index 98bba55b1..021f84029 100644 --- a/src/cryptonote_basic/account.h +++ b/src/cryptonote_basic/account.h @@ -89,6 +89,7 @@ namespace cryptonote hw::device& get_device() const {return m_keys.get_device();} void set_device( hw::device &hwdev) {m_keys.set_device(hwdev);} + void deinit(); uint64_t get_createtime() const { return m_creation_timestamp; } void set_createtime(uint64_t val) { m_creation_timestamp = val; } diff --git a/src/cryptonote_basic/blobdatatype.h b/src/cryptonote_basic/blobdatatype.h index 7d6ff0187..82484c0a8 100644 --- a/src/cryptonote_basic/blobdatatype.h +++ b/src/cryptonote_basic/blobdatatype.h @@ -30,7 +30,11 @@ #pragma once +#include <string> +#include "span.h" + namespace cryptonote { typedef std::string blobdata; + typedef epee::span<const char> blobdata_ref; } diff --git a/src/cryptonote_basic/connection_context.h b/src/cryptonote_basic/connection_context.h index eb73ab0ea..112c13049 100644 --- a/src/cryptonote_basic/connection_context.h +++ b/src/cryptonote_basic/connection_context.h @@ -40,7 +40,7 @@ namespace cryptonote struct cryptonote_connection_context: public epee::net_utils::connection_context_base { cryptonote_connection_context(): m_state(state_before_handshake), m_remote_blockchain_height(0), m_last_response_height(0), - m_last_request_time(boost::posix_time::microsec_clock::universal_time()), m_callback_request_count(0), m_last_known_hash(crypto::null_hash) {} + m_last_request_time(boost::date_time::not_a_date_time), m_callback_request_count(0), m_last_known_hash(crypto::null_hash), m_pruning_seed(0), m_anchor(false) {} enum state { @@ -59,6 +59,8 @@ namespace cryptonote boost::posix_time::ptime m_last_request_time; epee::copyable_atomic m_callback_request_count; //in debug purpose: problem with double callback rise crypto::hash m_last_known_hash; + uint32_t m_pruning_seed; + bool m_anchor; //size_t m_score; TODO: add score calculations }; @@ -81,4 +83,23 @@ namespace cryptonote } } + inline char get_protocol_state_char(cryptonote_connection_context::state s) + { + switch (s) + { + case cryptonote_connection_context::state_before_handshake: + return 'h'; + case cryptonote_connection_context::state_synchronizing: + return 's'; + case cryptonote_connection_context::state_standby: + return 'w'; + case cryptonote_connection_context::state_idle: + return 'i'; + case cryptonote_connection_context::state_normal: + return 'n'; + default: + return 'u'; + } + } + } diff --git a/src/cryptonote_basic/cryptonote_basic.h b/src/cryptonote_basic/cryptonote_basic.h index d4558ef7b..c9c783a56 100644 --- a/src/cryptonote_basic/cryptonote_basic.h +++ b/src/cryptonote_basic/cryptonote_basic.h @@ -47,7 +47,6 @@ #include "crypto/crypto.h" #include "crypto/hash.h" #include "misc_language.h" -#include "tx_extra.h" #include "ringct/rctTypes.h" #include "device/device.hpp" @@ -176,7 +175,15 @@ namespace cryptonote END_SERIALIZE() public: - transaction_prefix(){} + transaction_prefix(){ set_null(); } + void set_null() + { + version = 1; + unlock_time = 0; + vin.clear(); + vout.clear(); + extra.clear(); + } }; class transaction: public transaction_prefix @@ -194,9 +201,11 @@ namespace cryptonote mutable crypto::hash hash; mutable size_t blob_size; + bool pruned; + transaction(); - transaction(const transaction &t): transaction_prefix(t), hash_valid(false), blob_size_valid(false), signatures(t.signatures), rct_signatures(t.rct_signatures) { if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } } - transaction &operator=(const transaction &t) { transaction_prefix::operator=(t); set_hash_valid(false); set_blob_size_valid(false); signatures = t.signatures; rct_signatures = t.rct_signatures; if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } return *this; } + transaction(const transaction &t): transaction_prefix(t), hash_valid(false), blob_size_valid(false), signatures(t.signatures), rct_signatures(t.rct_signatures), pruned(t.pruned) { if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } } + transaction &operator=(const transaction &t) { transaction_prefix::operator=(t); set_hash_valid(false); set_blob_size_valid(false); signatures = t.signatures; rct_signatures = t.rct_signatures; if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } pruned = t.pruned; return *this; } virtual ~transaction(); void set_null(); void invalidate_hashes(); @@ -204,6 +213,8 @@ namespace cryptonote void set_hash_valid(bool v) const { hash_valid.store(v,std::memory_order_release); } bool is_blob_size_valid() const { return blob_size_valid.load(std::memory_order_acquire); } void set_blob_size_valid(bool v) const { blob_size_valid.store(v,std::memory_order_release); } + void set_hash(const crypto::hash &h) { hash = h; set_hash_valid(true); } + void set_blob_size(size_t sz) { blob_size = sz; set_blob_size_valid(true); } BEGIN_SERIALIZE_OBJECT() if (!typename Archive<W>::is_saving()) @@ -223,7 +234,7 @@ namespace cryptonote if (!signatures_not_expected && vin.size() != signatures.size()) return false; - for (size_t i = 0; i < vin.size(); ++i) + if (!pruned) for (size_t i = 0; i < vin.size(); ++i) { size_t signature_size = get_signature_size(vin[i]); if (signatures_not_expected) @@ -254,7 +265,7 @@ namespace cryptonote bool r = rct_signatures.serialize_rctsig_base(ar, vin.size(), vout.size()); if (!r || !ar.stream().good()) return false; ar.end_object(); - if (rct_signatures.type != rct::RCTTypeNull) + if (!pruned && rct_signatures.type != rct::RCTTypeNull) { ar.tag("rctsig_prunable"); ar.begin_object(); @@ -265,6 +276,8 @@ namespace cryptonote } } } + if (!typename Archive<W>::is_saving()) + pruned = false; END_SERIALIZE() template<bool W, template <bool> class Archive> @@ -286,6 +299,8 @@ namespace cryptonote ar.end_object(); } } + if (!typename Archive<W>::is_saving()) + pruned = true; return true; } @@ -303,21 +318,17 @@ namespace cryptonote inline transaction::~transaction() { - //set_null(); } inline void transaction::set_null() { - version = 1; - unlock_time = 0; - vin.clear(); - vout.clear(); - extra.clear(); + transaction_prefix::set_null(); signatures.clear(); rct_signatures.type = rct::RCTTypeNull; set_hash_valid(false); set_blob_size_valid(false); + pruned = false; } inline diff --git a/src/cryptonote_basic/cryptonote_basic_impl.cpp b/src/cryptonote_basic/cryptonote_basic_impl.cpp index b18ef1c5c..23a5bd5bd 100644 --- a/src/cryptonote_basic/cryptonote_basic_impl.cpp +++ b/src/cryptonote_basic/cryptonote_basic_impl.cpp @@ -40,7 +40,7 @@ using namespace epee; #include "misc_language.h" #include "common/base58.h" #include "crypto/hash.h" -#include "common/int-util.h" +#include "int-util.h" #include "common/dns_utils.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -322,13 +322,13 @@ namespace cryptonote { } //-------------------------------------------------------------------------------- -bool parse_hash256(const std::string str_hash, crypto::hash& hash) +bool parse_hash256(const std::string &str_hash, crypto::hash& hash) { std::string buf; bool res = epee::string_tools::parse_hexstr_to_binbuff(str_hash, buf); if (!res || buf.size() != sizeof(crypto::hash)) { - std::cout << "invalid hash format: <" << str_hash << '>' << std::endl; + MERROR("invalid hash format: " << str_hash); return false; } else diff --git a/src/cryptonote_basic/cryptonote_basic_impl.h b/src/cryptonote_basic/cryptonote_basic_impl.h index c804a88fa..0b8131a7a 100644 --- a/src/cryptonote_basic/cryptonote_basic_impl.h +++ b/src/cryptonote_basic/cryptonote_basic_impl.h @@ -124,5 +124,5 @@ namespace cryptonote { bool operator ==(const cryptonote::block& a, const cryptonote::block& b); } -bool parse_hash256(const std::string str_hash, crypto::hash& hash); +bool parse_hash256(const std::string &str_hash, crypto::hash& hash); diff --git a/src/cryptonote_basic/cryptonote_boost_serialization.h b/src/cryptonote_basic/cryptonote_boost_serialization.h index 0725a2bb8..a4228b849 100644 --- a/src/cryptonote_basic/cryptonote_boost_serialization.h +++ b/src/cryptonote_basic/cryptonote_boost_serialization.h @@ -45,6 +45,8 @@ #include "ringct/rctTypes.h" #include "ringct/rctOps.h" +BOOST_CLASS_VERSION(rct::ecdhTuple, 1) + //namespace cryptonote { namespace boost { @@ -247,9 +249,19 @@ namespace boost template <class Archive> inline void serialize(Archive &a, rct::ecdhTuple &x, const boost::serialization::version_type ver) { - a & x.mask; - a & x.amount; - // a & x.senderPk; // not serialized, as we do not use it in monero currently + if (ver < 1) + { + a & x.mask; + a & x.amount; + return; + } + crypto::hash8 &amount = (crypto::hash8&)x.amount; + if (!Archive::is_saving::value) + { + memset(&x.mask, 0, sizeof(x.mask)); + memset(&x.amount, 0, sizeof(x.amount)); + } + a & amount; } template <class Archive> @@ -295,7 +307,7 @@ namespace boost a & x.type; if (x.type == rct::RCTTypeNull) return; - if (x.type != rct::RCTTypeFull && x.type != rct::RCTTypeSimple && x.type != rct::RCTTypeBulletproof) + if (x.type != rct::RCTTypeFull && x.type != rct::RCTTypeSimple && x.type != rct::RCTTypeBulletproof && x.type != rct::RCTTypeBulletproof2) throw boost::archive::archive_exception(boost::archive::archive_exception::other_exception, "Unsupported rct type"); // a & x.message; message is not serialized, as it can be reconstructed from the tx data // a & x.mixRing; mixRing is not serialized, as it can be reconstructed from the offsets @@ -323,7 +335,7 @@ namespace boost a & x.type; if (x.type == rct::RCTTypeNull) return; - if (x.type != rct::RCTTypeFull && x.type != rct::RCTTypeSimple && x.type != rct::RCTTypeBulletproof) + if (x.type != rct::RCTTypeFull && x.type != rct::RCTTypeSimple && x.type != rct::RCTTypeBulletproof && x.type != rct::RCTTypeBulletproof2) throw boost::archive::archive_exception(boost::archive::archive_exception::other_exception, "Unsupported rct type"); // a & x.message; message is not serialized, as it can be reconstructed from the tx data // a & x.mixRing; mixRing is not serialized, as it can be reconstructed from the offsets @@ -337,7 +349,7 @@ namespace boost if (x.p.rangeSigs.empty()) a & x.p.bulletproofs; a & x.p.MGs; - if (x.type == rct::RCTTypeBulletproof) + if (x.type == rct::RCTTypeBulletproof || x.type == rct::RCTTypeBulletproof2) a & x.p.pseudoOuts; } } diff --git a/src/cryptonote_basic/cryptonote_format_utils.cpp b/src/cryptonote_basic/cryptonote_format_utils.cpp index 9e9c12605..f6daaab95 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.cpp +++ b/src/cryptonote_basic/cryptonote_format_utils.cpp @@ -28,9 +28,6 @@ // // Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers -#include "include_base_utils.h" -using namespace epee; - #include <atomic> #include <boost/algorithm/string.hpp> #include "wipeable_string.h" @@ -42,6 +39,8 @@ using namespace epee; #include "crypto/hash.h" #include "ringct/rctSigs.h" +using namespace epee; + #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "cn" @@ -185,6 +184,7 @@ namespace cryptonote CHECK_AND_ASSERT_MES(r, false, "Failed to parse transaction from blob"); CHECK_AND_ASSERT_MES(expand_transaction_1(tx, false), false, "Failed to expand transaction data"); tx.invalidate_hashes(); + tx.set_blob_size(tx_blob.size()); return true; } //--------------------------------------------------------------- @@ -196,6 +196,7 @@ namespace cryptonote bool r = tx.serialize_base(ba); CHECK_AND_ASSERT_MES(r, false, "Failed to parse transaction from blob"); CHECK_AND_ASSERT_MES(expand_transaction_1(tx, true), false, "Failed to expand transaction data"); + tx.invalidate_hashes(); return true; } //--------------------------------------------------------------- @@ -225,6 +226,22 @@ namespace cryptonote return true; } //--------------------------------------------------------------- + bool is_v1_tx(const blobdata_ref& tx_blob) + { + uint64_t version; + const char* begin = static_cast<const char*>(tx_blob.data()); + const char* end = begin + tx_blob.size(); + int read = tools::read_varint(begin, end, version); + if (read <= 0) + throw std::runtime_error("Internal error getting transaction version"); + return version <= 1; + } + //--------------------------------------------------------------- + bool is_v1_tx(const blobdata& tx_blob) + { + return is_v1_tx(blobdata_ref{tx_blob.data(), tx_blob.size()}); + } + //--------------------------------------------------------------- bool generate_key_image_helper(const account_keys& ack, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, const crypto::public_key& out_key, const crypto::public_key& tx_public_key, const std::vector<crypto::public_key>& additional_tx_public_keys, size_t real_output_index, keypair& in_ephemeral, crypto::key_image& ki, hw::device &hwdev) { crypto::key_derivation recv_derivation = AUTO_VAL_INIT(recv_derivation); @@ -380,11 +397,19 @@ namespace cryptonote //--------------------------------------------------------------- uint64_t get_transaction_weight(const transaction &tx) { - std::ostringstream s; - binary_archive<true> a(s); - ::serialization::serialize(a, const_cast<transaction&>(tx)); - const cryptonote::blobdata blob = s.str(); - return get_transaction_weight(tx, blob.size()); + size_t blob_size; + if (tx.is_blob_size_valid()) + { + blob_size = tx.blob_size; + } + else + { + std::ostringstream s; + binary_archive<true> a(s); + ::serialization::serialize(a, const_cast<transaction&>(tx)); + blob_size = s.str().size(); + } + return get_transaction_weight(tx, blob_size); } //--------------------------------------------------------------- bool get_tx_fee(const transaction& tx, uint64_t & fee) @@ -445,6 +470,91 @@ namespace cryptonote return true; } //--------------------------------------------------------------- + template<typename T> + static bool pick(binary_archive<true> &ar, std::vector<tx_extra_field> &fields, uint8_t tag) + { + std::vector<tx_extra_field>::iterator it; + while ((it = std::find_if(fields.begin(), fields.end(), [](const tx_extra_field &f) { return f.type() == typeid(T); })) != fields.end()) + { + bool r = ::do_serialize(ar, tag); + CHECK_AND_NO_ASSERT_MES_L1(r, false, "failed to serialize tx extra field"); + r = ::do_serialize(ar, boost::get<T>(*it)); + CHECK_AND_NO_ASSERT_MES_L1(r, false, "failed to serialize tx extra field"); + fields.erase(it); + } + return true; + } + //--------------------------------------------------------------- + bool sort_tx_extra(const std::vector<uint8_t>& tx_extra, std::vector<uint8_t> &sorted_tx_extra, bool allow_partial) + { + std::vector<tx_extra_field> tx_extra_fields; + + if(tx_extra.empty()) + { + sorted_tx_extra.clear(); + return true; + } + + std::string extra_str(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size()); + std::istringstream iss(extra_str); + binary_archive<false> ar(iss); + + bool eof = false; + size_t processed = 0; + while (!eof) + { + tx_extra_field field; + bool r = ::do_serialize(ar, field); + if (!r) + { + MWARNING("failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size()))); + if (!allow_partial) + return false; + break; + } + tx_extra_fields.push_back(field); + processed = iss.tellg(); + + std::ios_base::iostate state = iss.rdstate(); + eof = (EOF == iss.peek()); + iss.clear(state); + } + if (!::serialization::check_stream_state(ar)) + { + MWARNING("failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size()))); + if (!allow_partial) + return false; + } + MTRACE("Sorted " << processed << "/" << tx_extra.size()); + + std::ostringstream oss; + binary_archive<true> nar(oss); + + // sort by: + if (!pick<tx_extra_pub_key>(nar, tx_extra_fields, TX_EXTRA_TAG_PUBKEY)) return false; + if (!pick<tx_extra_additional_pub_keys>(nar, tx_extra_fields, TX_EXTRA_TAG_ADDITIONAL_PUBKEYS)) return false; + if (!pick<tx_extra_nonce>(nar, tx_extra_fields, TX_EXTRA_NONCE)) return false; + if (!pick<tx_extra_merge_mining_tag>(nar, tx_extra_fields, TX_EXTRA_MERGE_MINING_TAG)) return false; + if (!pick<tx_extra_mysterious_minergate>(nar, tx_extra_fields, TX_EXTRA_MYSTERIOUS_MINERGATE_TAG)) return false; + if (!pick<tx_extra_padding>(nar, tx_extra_fields, TX_EXTRA_TAG_PADDING)) return false; + + // if not empty, someone added a new type and did not add a case above + if (!tx_extra_fields.empty()) + { + MERROR("tx_extra_fields not empty after sorting, someone forgot to add a case above"); + return false; + } + + std::string oss_str = oss.str(); + if (allow_partial && processed < tx_extra.size()) + { + MDEBUG("Appending unparsed data"); + oss_str += std::string((const char*)tx_extra.data() + processed, tx_extra.size() - processed); + } + sorted_tx_extra = std::vector<uint8_t>(oss_str.begin(), oss_str.end()); + return true; + } + //--------------------------------------------------------------- crypto::public_key get_tx_pub_key_from_extra(const std::vector<uint8_t>& tx_extra, size_t pk_index) { std::vector<tx_extra_field> tx_extra_fields; @@ -798,7 +908,7 @@ namespace cryptonote { if (decimal_point == (unsigned int)-1) decimal_point = default_decimal_point; - switch (std::atomic_load(&default_decimal_point)) + switch (decimal_point) { case 12: return "monero"; @@ -811,7 +921,7 @@ namespace cryptonote case 0: return "piconero"; default: - ASSERT_MES_AND_THROW("Invalid decimal point specification: " << default_decimal_point); + ASSERT_MES_AND_THROW("Invalid decimal point specification: " << decimal_point); } } //--------------------------------------------------------------- diff --git a/src/cryptonote_basic/cryptonote_format_utils.h b/src/cryptonote_basic/cryptonote_format_utils.h index 725c75f4e..994978c10 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.h +++ b/src/cryptonote_basic/cryptonote_format_utils.h @@ -31,6 +31,7 @@ #pragma once #include "blobdatatype.h" #include "cryptonote_basic_impl.h" +#include "tx_extra.h" #include "account.h" #include "subaddress_index.h" #include "include_base_utils.h" @@ -52,6 +53,8 @@ namespace cryptonote bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx, crypto::hash& tx_hash, crypto::hash& tx_prefix_hash); bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx); bool parse_and_validate_tx_base_from_blob(const blobdata& tx_blob, transaction& tx); + bool is_v1_tx(const blobdata_ref& tx_blob); + bool is_v1_tx(const blobdata& tx_blob); template<typename T> bool find_tx_extra_field_by_type(const std::vector<tx_extra_field>& tx_extra_fields, T& field, size_t index = 0) @@ -65,6 +68,7 @@ namespace cryptonote } bool parse_tx_extra(const std::vector<uint8_t>& tx_extra, std::vector<tx_extra_field>& tx_extra_fields); + bool sort_tx_extra(const std::vector<uint8_t>& tx_extra, std::vector<uint8_t> &sorted_tx_extra, bool allow_partial = false); crypto::public_key get_tx_pub_key_from_extra(const std::vector<uint8_t>& tx_extra, size_t pk_index = 0); crypto::public_key get_tx_pub_key_from_extra(const transaction_prefix& tx, size_t pk_index = 0); crypto::public_key get_tx_pub_key_from_extra(const transaction& tx, size_t pk_index = 0); diff --git a/src/cryptonote_basic/difficulty.cpp b/src/cryptonote_basic/difficulty.cpp index cb2a00a12..55e3e93b3 100644 --- a/src/cryptonote_basic/difficulty.cpp +++ b/src/cryptonote_basic/difficulty.cpp @@ -34,7 +34,7 @@ #include <cstdint> #include <vector> -#include "common/int-util.h" +#include "int-util.h" #include "crypto/hash.h" #include "cryptonote_config.h" #include "difficulty.h" diff --git a/src/cryptonote_basic/hardfork.cpp b/src/cryptonote_basic/hardfork.cpp index f05b25901..447d79aee 100644 --- a/src/cryptonote_basic/hardfork.cpp +++ b/src/cryptonote_basic/hardfork.cpp @@ -56,12 +56,13 @@ static uint8_t get_block_version(const cryptonote::block &b) HardFork::HardFork(cryptonote::BlockchainDB &db, uint8_t original_version, uint64_t original_version_till_height, time_t forked_time, time_t update_time, uint64_t window_size, uint8_t default_threshold_percent): db(db), - original_version(original_version), - original_version_till_height(original_version_till_height), forked_time(forked_time), update_time(update_time), window_size(window_size), - default_threshold_percent(default_threshold_percent) + default_threshold_percent(default_threshold_percent), + original_version(original_version), + original_version_till_height(original_version_till_height), + current_fork_index(0) { if (window_size == 0) throw "window_size needs to be strictly positive"; @@ -221,7 +222,6 @@ bool HardFork::reorganize_from_block_height(uint64_t height) if (height >= db.height()) return false; - db.set_batch_transactions(true); bool stop_batch = db.batch_start(); versions.clear(); @@ -305,6 +305,29 @@ bool HardFork::rescan_from_chain_height(uint64_t height) return rescan_from_block_height(height - 1); } +void HardFork::on_block_popped(uint64_t nblocks) +{ + CHECK_AND_ASSERT_THROW_MES(nblocks > 0, "nblocks must be greater than 0"); + + CRITICAL_REGION_LOCAL(lock); + + const uint64_t new_chain_height = db.height(); + const uint64_t old_chain_height = new_chain_height + nblocks; + uint8_t version; + uint64_t height; + for (height = old_chain_height - 1; height >= new_chain_height; --height) + { + versions.pop_back(); + version = db.get_hard_fork_version(height); + versions.push_front(version); + } + + // does not take voting into account + for (current_fork_index = heights.size() - 1; current_fork_index > 0; --current_fork_index) + if (height >= heights[current_fork_index].height) + break; +} + int HardFork::get_voted_fork_index(uint64_t height) const { CRITICAL_REGION_LOCAL(lock); diff --git a/src/cryptonote_basic/hardfork.h b/src/cryptonote_basic/hardfork.h index a63a66976..a3fc25dfa 100644 --- a/src/cryptonote_basic/hardfork.h +++ b/src/cryptonote_basic/hardfork.h @@ -150,6 +150,16 @@ namespace cryptonote bool reorganize_from_chain_height(uint64_t height); /** + * @brief called when one or more blocks are popped from the blockchain + * + * The current fork will be updated by looking up the db, + * which is much cheaper than recomputing everything + * + * @param new_chain_height the height of the chain after popping + */ + void on_block_popped(uint64_t new_chain_height); + + /** * @brief returns current state at the given time * * Based on the approximate time of the last known hard fork, diff --git a/src/cryptonote_basic/miner.cpp b/src/cryptonote_basic/miner.cpp index d0b03593e..f4de2ed7e 100644 --- a/src/cryptonote_basic/miner.cpp +++ b/src/cryptonote_basic/miner.cpp @@ -33,8 +33,6 @@ #include <boost/utility/value_init.hpp> #include <boost/interprocess/detail/atomic.hpp> #include <boost/algorithm/string.hpp> -#include <boost/limits.hpp> -#include "include_base_utils.h" #include "misc_language.h" #include "syncobj.h" #include "cryptonote_basic_impl.h" @@ -54,19 +52,22 @@ #include <mach/mach_host.h> #include <AvailabilityMacros.h> #include <TargetConditionals.h> -#endif - -#ifdef __FreeBSD__ -#include <devstat.h> -#include <errno.h> -#include <fcntl.h> -#include <machine/apm_bios.h> -#include <stdio.h> -#include <sys/resource.h> -#include <sys/sysctl.h> -#include <sys/times.h> -#include <sys/types.h> -#include <unistd.h> +#elif defined(__linux__) + #include <unistd.h> + #include <sys/resource.h> + #include <sys/times.h> + #include <time.h> +#elif defined(__FreeBSD__) + #include <devstat.h> + #include <errno.h> + #include <fcntl.h> + #include <machine/apm_bios.h> + #include <stdio.h> + #include <sys/resource.h> + #include <sys/sysctl.h> + #include <sys/times.h> + #include <sys/types.h> + #include <unistd.h> #endif #undef MONERO_DEFAULT_LOG_CATEGORY @@ -146,7 +147,7 @@ namespace cryptonote //----------------------------------------------------------------------------------------------------- bool miner::request_block_template() { - block bl = AUTO_VAL_INIT(bl); + block bl; difficulty_type di = AUTO_VAL_INIT(di); uint64_t height = AUTO_VAL_INIT(height); uint64_t expected_reward; //only used for RPC calls - could possibly be useful here too? @@ -637,7 +638,7 @@ namespace cryptonote boost::tribool battery_powered(on_battery_power()); if(!indeterminate( battery_powered )) { - on_ac_power = !battery_powered; + on_ac_power = !(bool)battery_powered; } } diff --git a/src/cryptonote_basic/miner.h b/src/cryptonote_basic/miner.h index 2bff784c7..e16d9f3b8 100644 --- a/src/cryptonote_basic/miner.h +++ b/src/cryptonote_basic/miner.h @@ -38,11 +38,6 @@ #include "math_helper.h" #ifdef _WIN32 #include <windows.h> -#elif defined(__linux__) -#include <unistd.h> -#include <sys/resource.h> -#include <sys/times.h> -#include <time.h> #endif namespace cryptonote diff --git a/src/cryptonote_config.h b/src/cryptonote_config.h index a6858ce7c..4f652cd42 100644 --- a/src/cryptonote_config.h +++ b/src/cryptonote_config.h @@ -30,6 +30,7 @@ #pragma once +#include <stdexcept> #include <string> #include <boost/uuid/uuid.hpp> @@ -112,6 +113,8 @@ #define P2P_DEFAULT_HANDSHAKE_INVOKE_TIMEOUT 5000 //5 seconds #define P2P_DEFAULT_WHITELIST_CONNECTIONS_PERCENT 70 #define P2P_DEFAULT_ANCHOR_CONNECTIONS_COUNT 2 +#define P2P_DEFAULT_LIMIT_RATE_UP 2048 // kB/s +#define P2P_DEFAULT_LIMIT_RATE_DOWN 8192 // kB/s #define P2P_FAILED_ADDR_FORGET_SECONDS (60*60) //1 hour #define P2P_IP_BLOCKTIME (60*60*24) //24 hour @@ -138,6 +141,7 @@ #define HF_VERSION_MIN_MIXIN_10 8 #define HF_VERSION_ENFORCE_RCT 6 #define HF_VERSION_PER_BYTE_FEE 8 +#define HF_VERSION_SMALLER_BP 10 #define PER_KB_FEE_QUANTIZATION_DECIMALS 8 @@ -147,6 +151,11 @@ #define BULLETPROOF_MAX_OUTPUTS 16 +#define CRYPTONOTE_PRUNING_STRIPE_SIZE 4096 // the smaller, the smoother the increase +#define CRYPTONOTE_PRUNING_LOG_STRIPES 3 // the higher, the more space saved +#define CRYPTONOTE_PRUNING_TIP_BLOCKS 5500 // the smaller, the more space saved +//#define CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED + // New constants are intended to go here namespace config { diff --git a/src/cryptonote_core/CMakeLists.txt b/src/cryptonote_core/CMakeLists.txt index 72844db66..231489fdb 100644 --- a/src/cryptonote_core/CMakeLists.txt +++ b/src/cryptonote_core/CMakeLists.txt @@ -41,12 +41,6 @@ set(cryptonote_core_private_headers tx_pool.h cryptonote_tx_utils.h) -if(PER_BLOCK_CHECKPOINT) - set(Blocks "blocks") -else() - set(Blocks "") -endif() - monero_private_headers(cryptonote_core ${cryptonote_core_private_headers}) monero_add_library(cryptonote_core @@ -69,5 +63,4 @@ target_link_libraries(cryptonote_core ${Boost_SYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY} PRIVATE - ${Blocks} ${EXTRA_LIBRARIES}) diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 1ec2366e4..9f1e2a0c0 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -44,7 +44,7 @@ #include "misc_language.h" #include "profile_tools.h" #include "file_io_utils.h" -#include "common/int-util.h" +#include "int-util.h" #include "common/threadpool.h" #include "common/boost_serialization_helper.h" #include "warnings.h" @@ -53,9 +53,8 @@ #include "ringct/rctSigs.h" #include "common/perf_timer.h" #include "common/notify.h" -#if defined(PER_BLOCK_CHECKPOINT) -#include "blocks/blocks.h" -#endif +#include "common/varint.h" +#include "common/pruning.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "blockchain" @@ -174,6 +173,12 @@ Blockchain::Blockchain(tx_memory_pool& tx_pool) : LOG_PRINT_L3("Blockchain::" << __func__); } //------------------------------------------------------------------ +Blockchain::~Blockchain() +{ + try { deinit(); } + catch (const std::exception &e) { /* ignore */ } +} +//------------------------------------------------------------------ bool Blockchain::have_tx(const crypto::hash &id) const { LOG_PRINT_L3("Blockchain::" << __func__); @@ -232,7 +237,7 @@ bool Blockchain::scan_outputkeys_for_indexes(size_t tx_version, const txin_to_ke { try { - m_db->get_output_key(tx_in_to_key.amount, absolute_offsets, outputs, true); + m_db->get_output_key(epee::span<const uint64_t>(&tx_in_to_key.amount, 1), absolute_offsets, outputs, true); if (absolute_offsets.size() != outputs.size()) { MERROR_VER("Output does not exist! amount = " << tx_in_to_key.amount); @@ -258,7 +263,7 @@ bool Blockchain::scan_outputkeys_for_indexes(size_t tx_version, const txin_to_ke add_offsets.push_back(absolute_offsets[i]); try { - m_db->get_output_key(tx_in_to_key.amount, add_offsets, add_outputs, true); + m_db->get_output_key(epee::span<const uint64_t>(&tx_in_to_key.amount, 1), add_offsets, add_outputs, true); if (add_offsets.size() != add_outputs.size()) { MERROR_VER("Output does not exist! amount = " << tx_in_to_key.amount); @@ -341,7 +346,7 @@ uint64_t Blockchain::get_current_blockchain_height() const //------------------------------------------------------------------ //FIXME: possibly move this into the constructor, to avoid accidentally // dereferencing a null BlockchainDB pointer -bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline, const cryptonote::test_options *test_options, difficulty_type fixed_difficulty) +bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline, const cryptonote::test_options *test_options, difficulty_type fixed_difficulty, const GetCheckpointsCallback& get_checkpoints/* = nullptr*/) { LOG_PRINT_L3("Blockchain::" << __func__); @@ -407,7 +412,7 @@ bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline if(!m_db->height()) { MINFO("Blockchain not loaded, generating genesis block."); - block bl = boost::value_initialized<block>(); + block bl; block_verification_context bvc = boost::value_initialized<block_verification_context>(); generate_genesis_block(bl, get_config(m_nettype).GENESIS_TX, get_config(m_nettype).GENESIS_NONCE); add_new_block(bl, bvc); @@ -442,7 +447,7 @@ bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline #if defined(PER_BLOCK_CHECKPOINT) if (m_nettype != FAKECHAIN) - load_compiled_in_block_hashes(); + load_compiled_in_block_hashes(get_checkpoints); #endif MINFO("Blockchain initialized. last block: " << m_db->height() - 1 << ", " << epee::misc_utils::get_time_interval_string(timestamp_diff) << " time ago, current difficulty: " << get_difficulty_for_next_block()); @@ -553,15 +558,13 @@ bool Blockchain::deinit() // as this should be called if handling a SIGSEGV, need to check // if m_db is a NULL pointer (and thus may have caused the illegal // memory operation), otherwise we may cause a loop. - if (m_db == NULL) - { - throw DB_ERROR("The db pointer is null in Blockchain, the blockchain may be corrupt!"); - } - try { - m_db->close(); - MTRACE("Local blockchain read/write activity stopped successfully"); + if (m_db) + { + m_db->close(); + MTRACE("Local blockchain read/write activity stopped successfully"); + } } catch (const std::exception& e) { @@ -579,6 +582,38 @@ bool Blockchain::deinit() return true; } //------------------------------------------------------------------ +// This function removes blocks from the top of blockchain. +// It starts a batch and calls private method pop_block_from_blockchain(). +void Blockchain::pop_blocks(uint64_t nblocks) +{ + uint64_t i; + CRITICAL_REGION_LOCAL(m_tx_pool); + CRITICAL_REGION_LOCAL1(m_blockchain_lock); + + while (!m_db->batch_start()) + { + m_blockchain_lock.unlock(); + m_tx_pool.unlock(); + epee::misc_utils::sleep_no_w(1000); + m_tx_pool.lock(); + m_blockchain_lock.lock(); + } + + try + { + for (i=0; i < nblocks; ++i) + { + pop_block_from_blockchain(); + } + } + catch (const std::exception& e) + { + LOG_ERROR("Error when popping blocks, only " << i << " blocks are popped: " << e.what()); + } + + m_db->batch_stop(); +} +//------------------------------------------------------------------ // This function tells BlockchainDB to remove the top block from the // blockchain and then returns all transactions (except the miner tx, of course) // from it to the tx_pool @@ -609,9 +644,18 @@ block Blockchain::pop_block_from_blockchain() throw; } + // make sure the hard fork object updates its current version + m_hardfork->on_block_popped(1); + // return transactions from popped block to the tx_pool + size_t pruned = 0; for (transaction& tx : popped_txs) { + if (tx.pruned) + { + ++pruned; + continue; + } if (!is_coinbase(tx)) { cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc); @@ -619,12 +663,7 @@ block Blockchain::pop_block_from_blockchain() // FIXME: HardFork // Besides the below, popping a block should also remove the last entry // in hf_versions. - // - // FIXME: HardFork - // This is not quite correct, as we really want to add the txes - // to the pool based on the version determined after all blocks - // are popped. - uint8_t version = get_current_hard_fork_version(); + uint8_t version = get_ideal_hard_fork_version(m_db->height()); // We assume that if they were in a block, the transactions are already // known to the network as a whole. However, if we had mined that block, @@ -638,6 +677,8 @@ block Blockchain::pop_block_from_blockchain() } } } + if (pruned) + MWARNING(pruned << " pruned txes could not be added back to the txpool"); m_blocks_longhash_table.clear(); m_scan_table.clear(); @@ -834,6 +875,7 @@ difficulty_type Blockchain::get_difficulty_for_next_block() std::vector<uint64_t> timestamps; std::vector<difficulty_type> difficulties; auto height = m_db->height(); + top_hash = get_tail_id(); // get it again now that we have the lock // ND: Speedup // 1. Keep a list of the last 735 (or less) blocks that is used to compute difficulty, // then when the next block difficulty is queried, push the latest height data and @@ -856,7 +898,7 @@ difficulty_type Blockchain::get_difficulty_for_next_block() } else { - size_t offset = height - std::min < size_t > (height, static_cast<size_t>(DIFFICULTY_BLOCKS_COUNT)); + uint64_t offset = height - std::min <uint64_t> (height, static_cast<uint64_t>(DIFFICULTY_BLOCKS_COUNT)); if (offset == 0) ++offset; @@ -886,6 +928,17 @@ difficulty_type Blockchain::get_difficulty_for_next_block() return diff; } //------------------------------------------------------------------ +std::vector<time_t> Blockchain::get_last_block_timestamps(unsigned int blocks) const +{ + uint64_t height = m_db->height(); + if (blocks > height) + blocks = height; + std::vector<time_t> timestamps(blocks); + while (blocks--) + timestamps[blocks] = m_db->get_block_timestamp(height - blocks - 1); + return timestamps; +} +//------------------------------------------------------------------ // This function removes blocks from the blockchain until it gets to the // position where the blockchain switch started and then re-adds the blocks // that had been removed. @@ -922,7 +975,7 @@ bool Blockchain::rollback_blockchain_switching(std::list<block>& original_chain, m_hardfork->reorganize_from_chain_height(rollback_height); MINFO("Rollback to height " << rollback_height << " was successful."); - if (original_chain.size()) + if (!original_chain.empty()) { MINFO("Restoration to previous blockchain successful as well."); } @@ -1157,7 +1210,7 @@ bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_bl return false; } // From hard fork 2, we allow a miner to claim less block reward than is allowed, in case a miner wants less dust - if (m_hardfork->get_current_version() < 2) + if (version < 2) { if(base_reward + fee != money_in_use) { @@ -1230,7 +1283,9 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m uint64_t already_generated_coins; uint64_t pool_cookie; - CRITICAL_REGION_BEGIN(m_blockchain_lock); + m_tx_pool.lock(); + const auto unlock_guard = epee::misc_utils::create_scope_leave_handler([&]() { m_tx_pool.unlock(); }); + CRITICAL_REGION_LOCAL(m_blockchain_lock); height = m_db->height(); if (m_btc_valid) { // The pool cookie is atomic. The lack of locking is OK, as if it changes @@ -1266,8 +1321,6 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m median_weight = m_current_block_cumul_weight_limit / 2; already_generated_coins = m_db->get_block_already_generated_coins(height - 1); - CRITICAL_REGION_END(); - size_t txs_weight; uint64_t fee; if (!m_tx_pool.fill_block_template(b, median_weight, already_generated_coins, txs_weight, fee, expected_reward, m_hardfork->get_current_version())) @@ -1278,7 +1331,6 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) size_t real_txs_weight = 0; uint64_t real_fee = 0; - CRITICAL_REGION_BEGIN(m_tx_pool.m_transactions_lock); for(crypto::hash &cur_hash: b.tx_hashes) { auto cur_res = m_tx_pool.m_transactions.find(cur_hash); @@ -1322,7 +1374,6 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m { LOG_ERROR("Creating block template: error: wrongly calculated fee"); } - CRITICAL_REGION_END(); MDEBUG("Creating block template: height " << height << ", median weight " << median_weight << ", already generated coins " << already_generated_coins << @@ -1477,7 +1528,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id // if block to be added connects to known blocks that aren't part of the // main chain -- that is, if we're adding on to an alternate chain - if(alt_chain.size()) + if(!alt_chain.empty()) { // make sure alt chain doesn't somehow start past the end of the main chain CHECK_AND_ASSERT_MES(m_db->height() > alt_chain.front()->second.height, false, "main blockchain wrong height"); @@ -1760,16 +1811,34 @@ bool Blockchain::get_outs(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMA res.outs.clear(); res.outs.reserve(req.outputs.size()); + + std::vector<cryptonote::output_data_t> data; try { + std::vector<uint64_t> amounts, offsets; + amounts.reserve(req.outputs.size()); + offsets.reserve(req.outputs.size()); for (const auto &i: req.outputs) { - // get tx_hash, tx_out_index from DB - const output_data_t od = m_db->get_output_key(i.amount, i.index); - tx_out_index toi = m_db->get_output_tx_and_index(i.amount, i.index); - bool unlocked = is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first)); + amounts.push_back(i.amount); + offsets.push_back(i.index); + } + m_db->get_output_key(epee::span<const uint64_t>(amounts.data(), amounts.size()), offsets, data); + if (data.size() != req.outputs.size()) + { + MERROR("Unexpected output data size: expected " << req.outputs.size() << ", got " << data.size()); + return false; + } + for (const auto &t: data) + res.outs.push_back({t.pubkey, t.commitment, is_tx_spendtime_unlocked(t.unlock_time), t.height, crypto::null_hash}); - res.outs.push_back({od.pubkey, od.commitment, unlocked, od.height, toi.first}); + if (req.get_txid) + { + for (size_t i = 0; i < req.outputs.size(); ++i) + { + tx_out_index toi = m_db->get_output_tx_and_index(req.outputs[i].amount, req.outputs[i].index); + res.outs[i].txid = toi.first; + } } } catch (const std::exception &e) @@ -1798,6 +1867,7 @@ bool Blockchain::get_output_distribution(uint64_t amount, uint64_t from_height, case STAGENET: start_height = stagenet_hard_forks[3].height; break; case TESTNET: start_height = testnet_hard_forks[3].height; break; case MAINNET: start_height = mainnet_hard_forks[3].height; break; + case FAKECHAIN: start_height = 0; break; default: return false; } } @@ -1816,18 +1886,21 @@ bool Blockchain::get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t db_height = m_db->height(); if (db_height == 0) return false; - if (to_height == 0) - to_height = db_height - 1; if (start_height >= db_height || to_height >= db_height) return false; if (amount == 0) { std::vector<uint64_t> heights; heights.reserve(to_height + 1 - start_height); - for (uint64_t h = start_height; h <= to_height; ++h) + uint64_t real_start_height = start_height > 0 ? start_height-1 : start_height; + for (uint64_t h = real_start_height; h <= to_height; ++h) heights.push_back(h); distribution = m_db->get_block_cumulative_rct_outputs(heights); - base = 0; + if (start_height > 0) + { + base = distribution[0]; + distribution.erase(distribution.begin()); + } return true; } else @@ -1846,7 +1919,7 @@ bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qbloc // make sure the request includes at least the genesis block, otherwise // how can we expect to sync from the client that the block list came from? - if(!qblock_ids.size()) + if(qblock_ids.empty()) { MCERROR("net.p2p", "Client sent wrong NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << qblock_ids.size() << ", dropping connection"); return false; @@ -1981,6 +2054,51 @@ bool Blockchain::get_transactions_blobs(const t_ids_container& txs_ids, t_tx_con return true; } //------------------------------------------------------------------ +size_t get_transaction_version(const cryptonote::blobdata &bd) +{ + size_t version; + const char* begin = static_cast<const char*>(bd.data()); + const char* end = begin + bd.size(); + int read = tools::read_varint(begin, end, version); + if (read <= 0) + throw std::runtime_error("Internal error getting transaction version"); + return version; +} +//------------------------------------------------------------------ +template<class t_ids_container, class t_tx_container, class t_missed_container> +bool Blockchain::get_split_transactions_blobs(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + reserve_container(txs, txs_ids.size()); + for (const auto& tx_hash : txs_ids) + { + try + { + cryptonote::blobdata tx; + if (m_db->get_pruned_tx_blob(tx_hash, tx)) + { + txs.push_back(std::make_tuple(tx_hash, std::move(tx), crypto::null_hash, cryptonote::blobdata())); + if (!is_v1_tx(std::get<1>(txs.back())) && !m_db->get_prunable_tx_hash(tx_hash, std::get<2>(txs.back()))) + { + MERROR("Prunable data hash not found for " << tx_hash); + return false; + } + if (!m_db->get_prunable_tx_blob(tx_hash, std::get<3>(txs.back()))) + std::get<3>(txs.back()).clear(); + } + else + missed_txs.push_back(tx_hash); + } + catch (const std::exception& e) + { + return false; + } + } + return true; +} +//------------------------------------------------------------------ template<class t_ids_container, class t_tx_container, class t_missed_container> bool Blockchain::get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const { @@ -2029,9 +2147,12 @@ bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qbloc m_db->block_txn_start(true); current_height = get_current_blockchain_height(); + const uint32_t pruning_seed = get_blockchain_pruning_seed(); + start_height = tools::get_next_unpruned_block_height(start_height, current_height, pruning_seed); + uint64_t stop_height = tools::get_next_pruned_block_height(start_height, current_height, pruning_seed); size_t count = 0; - hashes.reserve(std::max((size_t)(current_height - start_height), (size_t)BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT)); - for(size_t i = start_height; i < current_height && count < BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT; i++, count++) + hashes.reserve(std::min((size_t)(stop_height - start_height), (size_t)BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT)); + for(size_t i = start_height; i < stop_height && count < BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT; i++, count++) { hashes.push_back(m_db->get_block_hash_from_height(i)); } @@ -2236,7 +2357,7 @@ bool Blockchain::check_for_double_spend(const transaction& tx, key_images_contai return true; } //------------------------------------------------------------------ -bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<uint64_t>& indexs) const +bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, size_t n_txes, std::vector<std::vector<uint64_t>>& indexs) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -2246,16 +2367,25 @@ bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<u MERROR_VER("get_tx_outputs_gindexs failed to find transaction with id = " << tx_id); return false; } + indexs = m_db->get_tx_amount_output_indices(tx_index, n_txes); + CHECK_AND_ASSERT_MES(n_txes == indexs.size(), false, "Wrong indexs size"); - // get amount output indexes, currently referred to in parts as "output global indices", but they are actually specific to amounts - indexs = m_db->get_tx_amount_output_indices(tx_index); - if (indexs.empty()) + return true; +} +//------------------------------------------------------------------ +bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<uint64_t>& indexs) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + uint64_t tx_index; + if (!m_db->tx_exists(tx_id, tx_index)) { - // empty indexs is only valid if the vout is empty, which is legal but rare - cryptonote::transaction tx = m_db->get_tx(tx_id); - CHECK_AND_ASSERT_MES(tx.vout.empty(), false, "internal error: global indexes for transaction " << tx_id << " is empty, and tx vout is not"); + MERROR_VER("get_tx_outputs_gindexs failed to find transaction with id = " << tx_id); + return false; } - + std::vector<std::vector<uint64_t>> indices = m_db->get_tx_amount_output_indices(tx_index, 1); + CHECK_AND_ASSERT_MES(indices.size() == 1, false, "Wrong indices size"); + indexs = indices.front(); return true; } //------------------------------------------------------------------ @@ -2387,6 +2517,18 @@ bool Blockchain::check_tx_outputs(const transaction& tx, tx_verification_context } } + // from v10, allow bulletproofs v2 + if (hf_version < HF_VERSION_SMALLER_BP) { + if (tx.version >= 2) { + if (tx.rct_signatures.type == rct::RCTTypeBulletproof2) + { + MERROR_VER("Bulletproofs v2 are not allowed before v" << HF_VERSION_SMALLER_BP); + tvc.m_invalid_output = true; + return false; + } + } + } + return true; } //------------------------------------------------------------------ @@ -2427,7 +2569,7 @@ bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_pr } } } - else if (rv.type == rct::RCTTypeSimple || rv.type == rct::RCTTypeBulletproof) + else if (rv.type == rct::RCTTypeSimple || rv.type == rct::RCTTypeBulletproof || rv.type == rct::RCTTypeBulletproof2) { CHECK_AND_ASSERT_MES(!pubkeys.empty() && !pubkeys[0].empty(), false, "empty pubkeys"); rv.mixRing.resize(pubkeys.size()); @@ -2453,7 +2595,7 @@ bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_pr for (size_t n = 0; n < tx.vin.size(); ++n) rv.p.MGs[0].II[n] = rct::ki2rct(boost::get<txin_to_key>(tx.vin[n]).k_image); } - else if (rv.type == rct::RCTTypeSimple || rv.type == rct::RCTTypeBulletproof) + else if (rv.type == rct::RCTTypeSimple || rv.type == rct::RCTTypeBulletproof || rct::RCTTypeBulletproof2) { CHECK_AND_ASSERT_MES(rv.p.MGs.size() == tx.vin.size(), false, "Bad MGs size"); for (size_t n = 0; n < tx.vin.size(); ++n) @@ -2524,7 +2666,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, } } - if (hf_version >= HF_VERSION_MIN_MIXIN_10 && mixin != 10) + if (((hf_version == HF_VERSION_MIN_MIXIN_10 || hf_version == HF_VERSION_MIN_MIXIN_10+1) && mixin != 10) || (hf_version >= HF_VERSION_MIN_MIXIN_10+2 && mixin > 10)) { MERROR_VER("Tx " << get_transaction_hash(tx) << " has invalid ring size (" << (mixin + 1) << "), it should be 11"); tvc.m_low_mixin = true; @@ -2727,6 +2869,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, } case rct::RCTTypeSimple: case rct::RCTTypeBulletproof: + case rct::RCTTypeBulletproof2: { // check all this, either reconstructed (so should really pass), or not { @@ -3297,7 +3440,7 @@ leave: { if (memcmp(&hash, &expected_hash, sizeof(hash)) != 0) { - MERROR_VER("Block with id is INVALID: " << id); + MERROR_VER("Block with id is INVALID: " << id << ", expected " << expected_hash); bvc.m_verifivation_failed = true; goto leave; } @@ -3563,6 +3706,35 @@ leave: return true; } //------------------------------------------------------------------ +bool Blockchain::prune_blockchain(uint32_t pruning_seed) +{ + uint8_t hf_version = m_hardfork->get_current_version(); + if (hf_version < 10) + { + MERROR("Most of the network will only be ready for pruned blockchains from v10, not pruning"); + return false; + } + return m_db->prune_blockchain(pruning_seed); +} +//------------------------------------------------------------------ +bool Blockchain::update_blockchain_pruning() +{ + m_tx_pool.lock(); + epee::misc_utils::auto_scope_leave_caller unlocker = epee::misc_utils::create_scope_leave_handler([&](){m_tx_pool.unlock();}); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + return m_db->update_pruning(); +} +//------------------------------------------------------------------ +bool Blockchain::check_blockchain_pruning() +{ + m_tx_pool.lock(); + epee::misc_utils::auto_scope_leave_caller unlocker = epee::misc_utils::create_scope_leave_handler([&](){m_tx_pool.unlock();}); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + return m_db->check_pruning(); +} +//------------------------------------------------------------------ bool Blockchain::update_next_cumulative_weight_limit() { uint64_t full_reward_zone = get_min_block_weight(get_current_hard_fork_version()); @@ -3694,7 +3866,7 @@ void Blockchain::set_enforce_dns_checkpoints(bool enforce_checkpoints) } //------------------------------------------------------------------ -void Blockchain::block_longhash_worker(uint64_t height, const std::vector<block> &blocks, std::unordered_map<crypto::hash, crypto::hash> &map) const +void Blockchain::block_longhash_worker(uint64_t height, const epee::span<const block> &blocks, std::unordered_map<crypto::hash, crypto::hash> &map) const { TIME_MEASURE_START(t); slow_hash_allocate_state(); @@ -3776,16 +3948,17 @@ bool Blockchain::cleanup_handle_incoming_blocks(bool force_sync) CRITICAL_REGION_END(); m_tx_pool.unlock(); + update_blockchain_pruning(); + return success; } //------------------------------------------------------------------ -//FIXME: unused parameter txs -void Blockchain::output_scan_worker(const uint64_t amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs, std::unordered_map<crypto::hash, cryptonote::transaction> &txs) const +void Blockchain::output_scan_worker(const uint64_t amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs) const { try { - m_db->get_output_key(amount, offsets, outputs, true); + m_db->get_output_key(epee::span<const uint64_t>(&amount, 1), offsets, outputs, true); } catch (const std::exception& e) { @@ -3946,42 +4119,40 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete m_blockchain_lock.lock(); } - if ((m_db->height() + blocks_entry.size()) < m_blocks_hash_check.size()) + const uint64_t height = m_db->height(); + if ((height + blocks_entry.size()) < m_blocks_hash_check.size()) return true; bool blocks_exist = false; tools::threadpool& tpool = tools::threadpool::getInstance(); - uint64_t threads = tpool.get_max_concurrency(); + unsigned threads = tpool.get_max_concurrency(); + std::vector<block> blocks; + blocks.resize(blocks_entry.size()); - if (blocks_entry.size() > 1 && threads > 1 && m_max_prepare_blocks_threads > 1) + if (1) { // limit threads, default limit = 4 if(threads > m_max_prepare_blocks_threads) threads = m_max_prepare_blocks_threads; - uint64_t height = m_db->height(); - int batches = blocks_entry.size() / threads; - int extra = blocks_entry.size() % threads; + unsigned int batches = blocks_entry.size() / threads; + unsigned int extra = blocks_entry.size() % threads; MDEBUG("block_batches: " << batches); std::vector<std::unordered_map<crypto::hash, crypto::hash>> maps(threads); - std::vector < std::vector < block >> blocks(threads); auto it = blocks_entry.begin(); + unsigned blockidx = 0; - for (uint64_t i = 0; i < threads; i++) + for (unsigned i = 0; i < threads; i++) { - blocks[i].reserve(batches + 1); - for (int j = 0; j < batches; j++) + for (unsigned int j = 0; j < batches; j++, ++blockidx) { - block block; + block &block = blocks[blockidx]; if (!parse_and_validate_block_from_blob(it->block, block)) - { - std::advance(it, 1); - continue; - } + return false; // check first block and skip all blocks if its not chained properly - if (i == 0 && j == 0) + if (blockidx == 0) { crypto::hash tophash = m_db->top_block_hash(); if (block.prev_id != tophash) @@ -3996,20 +4167,16 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete break; } - blocks[i].push_back(std::move(block)); std::advance(it, 1); } } - for (int i = 0; i < extra && !blocks_exist; i++) + for (unsigned i = 0; i < extra && !blocks_exist; i++, blockidx++) { - block block; + block &block = blocks[blockidx]; if (!parse_and_validate_block_from_blob(it->block, block)) - { - std::advance(it, 1); - continue; - } + return false; if (have_block(get_block_hash(block))) { @@ -4017,7 +4184,6 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete break; } - blocks[i].push_back(std::move(block)); std::advance(it, 1); } @@ -4026,10 +4192,13 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete m_blocks_longhash_table.clear(); uint64_t thread_height = height; tools::threadpool::waiter waiter; - for (uint64_t i = 0; i < threads; i++) + for (unsigned int i = 0; i < threads; i++) { - tpool.submit(&waiter, boost::bind(&Blockchain::block_longhash_worker, this, thread_height, std::cref(blocks[i]), std::ref(maps[i])), true); - thread_height += blocks[i].size(); + unsigned nblocks = batches; + if (i < extra) + ++nblocks; + tpool.submit(&waiter, boost::bind(&Blockchain::block_longhash_worker, this, thread_height, epee::span<const block>(&blocks[i], nblocks), std::ref(maps[i])), true); + thread_height += nblocks; } waiter.wait(&tpool); @@ -4083,7 +4252,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete } while(0); \ // generate sorted tables for all amounts and absolute offsets - size_t tx_index = 0; + size_t tx_index = 0, block_index = 0; for (const auto &entry : blocks_entry) { if (m_cancel) @@ -4148,6 +4317,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete } } + ++block_index; } // sort and remove duplicate absolute_offsets in offset_map @@ -4158,21 +4328,19 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete offsets.second.erase(last, offsets.second.end()); } - // [output] stores all transactions for each tx_out_index::hash found - std::vector<std::unordered_map<crypto::hash, cryptonote::transaction>> transactions(amounts.size()); - + // gather all the output keys threads = tpool.get_max_concurrency(); if (!m_db->can_thread_bulk_indices()) threads = 1; - if (threads > 1) + if (threads > 1 && amounts.size() > 1) { tools::threadpool::waiter waiter; for (size_t i = 0; i < amounts.size(); i++) { uint64_t amount = amounts[i]; - tpool.submit(&waiter, boost::bind(&Blockchain::output_scan_worker, this, amount, std::cref(offset_map[amount]), std::ref(tx_map[amount]), std::ref(transactions[i])), true); + tpool.submit(&waiter, boost::bind(&Blockchain::output_scan_worker, this, amount, std::cref(offset_map[amount]), std::ref(tx_map[amount])), true); } waiter.wait(&tpool); } @@ -4181,7 +4349,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete for (size_t i = 0; i < amounts.size(); i++) { uint64_t amount = amounts[i]; - output_scan_worker(amount, offset_map[amount], tx_map[amount], transactions[i]); + output_scan_worker(amount, offset_map[amount], tx_map[amount]); } } @@ -4248,9 +4416,9 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete return true; } -void Blockchain::add_txpool_tx(transaction &tx, const txpool_tx_meta_t &meta) +void Blockchain::add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const txpool_tx_meta_t &meta) { - m_db->add_txpool_tx(tx, meta); + m_db->add_txpool_tx(txid, blob, meta); } void Blockchain::update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t &meta) @@ -4407,19 +4575,21 @@ void Blockchain::cancel() #if defined(PER_BLOCK_CHECKPOINT) static const char expected_block_hashes_hash[] = "954cb2bbfa2fe6f74b2cdd22a1a4c767aea249ad47ad4f7c9445f0f03260f511"; -void Blockchain::load_compiled_in_block_hashes() +void Blockchain::load_compiled_in_block_hashes(const GetCheckpointsCallback& get_checkpoints) { - const bool testnet = m_nettype == TESTNET; - const bool stagenet = m_nettype == STAGENET; - if (m_fast_sync && get_blocks_dat_start(testnet, stagenet) != nullptr && get_blocks_dat_size(testnet, stagenet) > 0) + if (get_checkpoints == nullptr || !m_fast_sync) { - MINFO("Loading precomputed blocks (" << get_blocks_dat_size(testnet, stagenet) << " bytes)"); - + return; + } + const epee::span<const unsigned char> &checkpoints = get_checkpoints(m_nettype); + if (!checkpoints.empty()) + { + MINFO("Loading precomputed blocks (" << checkpoints.size() << " bytes)"); if (m_nettype == MAINNET) { // first check hash crypto::hash hash; - if (!tools::sha256sum(get_blocks_dat_start(testnet, stagenet), get_blocks_dat_size(testnet, stagenet), hash)) + if (!tools::sha256sum(checkpoints.data(), checkpoints.size(), hash)) { MERROR("Failed to hash precomputed blocks data"); return; @@ -4439,9 +4609,9 @@ void Blockchain::load_compiled_in_block_hashes() } } - if (get_blocks_dat_size(testnet, stagenet) > 4) + if (checkpoints.size() > 4) { - const unsigned char *p = get_blocks_dat_start(testnet, stagenet); + const unsigned char *p = checkpoints.data(); const uint32_t nblocks = *p | ((*(p+1))<<8) | ((*(p+2))<<16) | ((*(p+3))<<24); if (nblocks > (std::numeric_limits<uint32_t>::max() - 4) / sizeof(hash)) { @@ -4449,7 +4619,7 @@ void Blockchain::load_compiled_in_block_hashes() return; } const size_t size_needed = 4 + nblocks * sizeof(crypto::hash); - if(nblocks > 0 && nblocks > (m_db->height() + HASH_OF_HASHES_STEP - 1) / HASH_OF_HASHES_STEP && get_blocks_dat_size(testnet, stagenet) >= size_needed) + if(nblocks > 0 && nblocks > (m_db->height() + HASH_OF_HASHES_STEP - 1) / HASH_OF_HASHES_STEP && checkpoints.size() >= size_needed) { p += sizeof(uint32_t); m_blocks_hash_of_hashes.reserve(nblocks); @@ -4553,4 +4723,5 @@ void Blockchain::cache_block_template(const block &b, const cryptonote::account_ namespace cryptonote { template bool Blockchain::get_transactions(const std::vector<crypto::hash>&, std::vector<transaction>&, std::vector<crypto::hash>&) const; template bool Blockchain::get_transactions_blobs(const std::vector<crypto::hash>&, std::vector<cryptonote::blobdata>&, std::vector<crypto::hash>&, bool) const; +template bool Blockchain::get_split_transactions_blobs(const std::vector<crypto::hash>&, std::vector<std::tuple<crypto::hash, cryptonote::blobdata, crypto::hash, cryptonote::blobdata>>&, std::vector<crypto::hash>&) const; } diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index ab66fac8b..4952116ac 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -38,9 +38,11 @@ #include <boost/multi_index/hashed_index.hpp> #include <boost/multi_index/member.hpp> #include <atomic> +#include <functional> #include <unordered_map> #include <unordered_set> +#include "span.h" #include "syncobj.h" #include "string_tools.h" #include "cryptonote_basic/cryptonote_basic.h" @@ -73,6 +75,15 @@ namespace cryptonote db_nosync //!< Leave syncing up to the backing db (safest, but slowest because of disk I/O) }; + /** + * @brief Callback routine that returns checkpoints data for specific network type + * + * @param network network type + * + * @return checkpoints data, empty span if there ain't any checkpoints for specific network type + */ + typedef std::function<const epee::span<const unsigned char>(cryptonote::network_type network)> GetCheckpointsCallback; + /************************************************************************/ /* */ /************************************************************************/ @@ -110,6 +121,11 @@ namespace cryptonote Blockchain(tx_memory_pool& tx_pool); /** + * @brief Blockchain destructor + */ + ~Blockchain(); + + /** * @brief Initialize the Blockchain state * * @param db a pointer to the backing store to use for the blockchain @@ -117,10 +133,11 @@ namespace cryptonote * @param offline true if running offline, else false * @param test_options test parameters * @param fixed_difficulty fixed difficulty for testing purposes; 0 means disabled + * @param get_checkpoints if set, will be called to get checkpoints data * * @return true on success, false if any initialization steps fail */ - bool init(BlockchainDB* db, const network_type nettype = MAINNET, bool offline = false, const cryptonote::test_options *test_options = NULL, difficulty_type fixed_difficulty = 0); + bool init(BlockchainDB* db, const network_type nettype = MAINNET, bool offline = false, const cryptonote::test_options *test_options = NULL, difficulty_type fixed_difficulty = 0, const GetCheckpointsCallback& get_checkpoints = nullptr); /** * @brief Initialize the Blockchain state @@ -504,10 +521,12 @@ namespace cryptonote * * @param tx_id the hash of the transaction to fetch indices for * @param indexs return-by-reference the global indices for the transaction's outputs + * @param n_txes how many txes in a row to get results for * * @return false if the transaction does not exist, or if no indices are found, otherwise true */ bool get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<uint64_t>& indexs) const; + bool get_tx_outputs_gindexs(const crypto::hash& tx_id, size_t n_txes, std::vector<std::vector<uint64_t>>& indexs) const; /** * @brief stores the blockchain @@ -658,6 +677,8 @@ namespace cryptonote template<class t_ids_container, class t_tx_container, class t_missed_container> bool get_transactions_blobs(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs, bool pruned = false) const; template<class t_ids_container, class t_tx_container, class t_missed_container> + bool get_split_transactions_blobs(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const; + template<class t_ids_container, class t_tx_container, class t_missed_container> bool get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const; //debug functions @@ -904,11 +925,9 @@ namespace cryptonote * @param amount the amount * @param offsets the indices (indexed to the amount) of the outputs * @param outputs return-by-reference the outputs collected - * @param txs unused, candidate for removal */ void output_scan_worker(const uint64_t amount,const std::vector<uint64_t> &offsets, - std::vector<output_data_t> &outputs, std::unordered_map<crypto::hash, - cryptonote::transaction> &txs) const; + std::vector<output_data_t> &outputs) const; /** * @brief computes the "short" and "long" hashes for a set of blocks @@ -917,7 +936,7 @@ namespace cryptonote * @param blocks the blocks to be hashed * @param map return-by-reference the hashes for each block */ - void block_longhash_worker(uint64_t height, const std::vector<block> &blocks, + void block_longhash_worker(uint64_t height, const epee::span<const block> &blocks, std::unordered_map<crypto::hash, crypto::hash> &map) const; /** @@ -927,7 +946,7 @@ namespace cryptonote */ std::list<std::pair<block_extended_info,std::vector<crypto::hash>>> get_alternative_chains() const; - void add_txpool_tx(transaction &tx, const txpool_tx_meta_t &meta); + void add_txpool_tx(const crypto::hash &txid, const cryptonote::blobdata &blob, const txpool_tx_meta_t &meta); void update_txpool_tx(const crypto::hash &txid, const txpool_tx_meta_t &meta); void remove_txpool_tx(const crypto::hash &txid); uint64_t get_txpool_tx_count(bool include_unrelayed_txes = true) const; @@ -939,6 +958,10 @@ namespace cryptonote bool is_within_compiled_block_hash_area(uint64_t height) const; bool is_within_compiled_block_hash_area() const { return is_within_compiled_block_hash_area(m_db->height()); } uint64_t prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes); + uint32_t get_blockchain_pruning_seed() const { return m_db->get_blockchain_pruning_seed(); } + bool prune_blockchain(uint32_t pruning_seed = 0); + bool update_blockchain_pruning(); + bool check_blockchain_pruning(); void lock(); void unlock(); @@ -952,6 +975,18 @@ namespace cryptonote */ void on_new_tx_from_block(const cryptonote::transaction &tx); + /** + * @brief returns the timestamps of the last N blocks + */ + std::vector<time_t> get_last_block_timestamps(unsigned int blocks) const; + + /** + * @brief removes blocks from the top of the blockchain + * + * @param nblocks number of blocks to be removed + */ + void pop_blocks(uint64_t nblocks); + private: // TODO: evaluate whether or not each of these typedefs are left over from blockchain_storage @@ -1369,8 +1404,10 @@ namespace cryptonote * A (possibly empty) set of block hashes can be compiled into the * monero daemon binary. This function loads those hashes into * a useful state. + * + * @param get_checkpoints if set, will be called to get checkpoints data */ - void load_compiled_in_block_hashes(); + void load_compiled_in_block_hashes(const GetCheckpointsCallback& get_checkpoints); /** * @brief expands v2 transaction data from blockchain diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 2fec6b613..48c607ff1 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -30,13 +30,11 @@ #include <boost/algorithm/string.hpp> -#include "include_base_utils.h" #include "string_tools.h" using namespace epee; #include <unordered_set> #include "cryptonote_core.h" -#include "common/command_line.h" #include "common/util.h" #include "common/updates.h" #include "common/download.h" @@ -45,7 +43,6 @@ using namespace epee; #include "warnings.h" #include "crypto/crypto.h" #include "cryptonote_config.h" -#include "cryptonote_tx_utils.h" #include "misc_language.h" #include "file_io_utils.h" #include <csignal> @@ -108,6 +105,11 @@ namespace cryptonote "disable-dns-checkpoints" , "Do not retrieve checkpoints from DNS" }; + const command_line::arg_descriptor<size_t> arg_block_download_max_size = { + "block-download-max-size" + , "Set maximum size of block download queue in bytes (0 for default)" + , 0 + }; static const command_line::arg_descriptor<bool> arg_test_drop_download = { "test-drop-download" @@ -163,6 +165,11 @@ namespace cryptonote , "Relay blocks as normal blocks" , false }; + static const command_line::arg_descriptor<bool> arg_pad_transactions = { + "pad-transactions" + , "Pad relayed transactions to help defend against traffic volume analysis" + , false + }; static const command_line::arg_descriptor<size_t> arg_max_txpool_weight = { "max-txpool-weight" , "Set maximum txpool weight in bytes." @@ -173,6 +180,11 @@ namespace cryptonote , "Run a program for each new block, '%s' will be replaced by the block hash" , "" }; + static const command_line::arg_descriptor<bool> arg_prune_blockchain = { + "prune-blockchain" + , "Prune blockchain" + , false + }; //----------------------------------------------------------------------------------------------- core::core(i_cryptonote_protocol* pprotocol): @@ -188,7 +200,8 @@ namespace cryptonote m_disable_dns_checkpoints(false), m_update_download(0), m_nettype(UNDEFINED), - m_update_available(false) + m_update_available(false), + m_pad_transactions(false) { m_checkpoints_updating.clear(); set_cryptonote_protocol(pprotocol); @@ -247,6 +260,7 @@ namespace cryptonote //----------------------------------------------------------------------------------- void core::stop() { + m_miner.stop(); m_blockchain_storage.cancel(); tools::download_async_handle handle; @@ -281,8 +295,11 @@ namespace cryptonote command_line::add_arg(desc, arg_test_dbg_lock_sleep); command_line::add_arg(desc, arg_offline); command_line::add_arg(desc, arg_disable_dns_checkpoints); + command_line::add_arg(desc, arg_block_download_max_size); command_line::add_arg(desc, arg_max_txpool_weight); + command_line::add_arg(desc, arg_pad_transactions); command_line::add_arg(desc, arg_block_notify); + command_line::add_arg(desc, arg_prune_blockchain); miner::init_options(desc); BlockchainDB::init_options(desc); @@ -320,6 +337,7 @@ namespace cryptonote set_enforce_dns_checkpoints(command_line::get_arg(vm, arg_dns_checkpoints)); test_drop_download_height(command_line::get_arg(vm, arg_test_drop_download_height)); m_fluffy_blocks_enabled = !get_arg(vm, arg_no_fluffy_blocks); + m_pad_transactions = get_arg(vm, arg_pad_transactions); m_offline = get_arg(vm, arg_offline); m_disable_dns_checkpoints = get_arg(vm, arg_disable_dns_checkpoints); if (!command_line::is_arg_defaulted(vm, arg_fluffy_blocks)) @@ -368,6 +386,11 @@ namespace cryptonote return m_blockchain_storage.get_transactions_blobs(txs_ids, txs, missed_txs); } //----------------------------------------------------------------------------------------------- + bool core::get_split_transactions_blobs(const std::vector<crypto::hash>& txs_ids, std::vector<std::tuple<crypto::hash, cryptonote::blobdata, crypto::hash, cryptonote::blobdata>>& txs, std::vector<crypto::hash>& missed_txs) const + { + return m_blockchain_storage.get_split_transactions_blobs(txs_ids, txs, missed_txs); + } + //----------------------------------------------------------------------------------------------- bool core::get_txpool_backlog(std::vector<tx_backlog_entry>& backlog) const { m_mempool.get_transaction_backlog(backlog); @@ -389,7 +412,7 @@ namespace cryptonote return m_blockchain_storage.get_alternative_blocks_count(); } //----------------------------------------------------------------------------------------------- - bool core::init(const boost::program_options::variables_map& vm, const char *config_subdir, const cryptonote::test_options *test_options) + bool core::init(const boost::program_options::variables_map& vm, const cryptonote::test_options *test_options, const GetCheckpointsCallback& get_checkpoints/* = nullptr */) { start_time = std::time(nullptr); @@ -399,10 +422,6 @@ namespace cryptonote m_nettype = FAKECHAIN; } bool r = handle_command_line(vm); - std::string m_config_folder_mempool = m_config_folder; - - if (config_subdir) - m_config_folder_mempool = m_config_folder_mempool + "/" + config_subdir; std::string db_type = command_line::get_arg(vm, cryptonote::arg_db_type); std::string db_sync_mode = command_line::get_arg(vm, cryptonote::arg_db_sync_mode); @@ -411,6 +430,7 @@ namespace cryptonote uint64_t blocks_threads = command_line::get_arg(vm, arg_prep_blocks_threads); std::string check_updates_string = command_line::get_arg(vm, arg_check_updates); size_t max_txpool_weight = command_line::get_arg(vm, arg_max_txpool_weight); + bool prune_blockchain = command_line::get_arg(vm, arg_prune_blockchain); boost::filesystem::path folder(m_config_folder); if (m_nettype == FAKECHAIN) @@ -567,7 +587,7 @@ namespace cryptonote regtest_hard_forks }; const difficulty_type fixed_difficulty = command_line::get_arg(vm, arg_fixed_difficulty); - r = m_blockchain_storage.init(db.release(), m_nettype, m_offline, regtest ? ®test_test_options : test_options, fixed_difficulty); + r = m_blockchain_storage.init(db.release(), m_nettype, m_offline, regtest ? ®test_test_options : test_options, fixed_difficulty, get_checkpoints); r = m_mempool.init(max_txpool_weight); CHECK_AND_ASSERT_MES(r, false, "Failed to initialize memory pool"); @@ -605,6 +625,14 @@ namespace cryptonote r = m_miner.init(vm, m_nettype); CHECK_AND_ASSERT_MES(r, false, "Failed to initialize miner instance"); + if (prune_blockchain) + { + // display a message if the blockchain is not pruned yet + if (m_blockchain_storage.get_current_blockchain_height() > 1 && !m_blockchain_storage.get_blockchain_pruning_seed()) + MGINFO("Pruning blockchain..."); + CHECK_AND_ASSERT_MES(m_blockchain_storage.prune_blockchain(), false, "Failed to prune blockchain"); + } + return load_state_data(); } //----------------------------------------------------------------------------------------------- @@ -788,6 +816,7 @@ namespace cryptonote } break; case rct::RCTTypeBulletproof: + case rct::RCTTypeBulletproof2: if (!is_canonical_bulletproof_layout(rv.p.bulletproofs)) { MERROR_VER("Bulletproof does not have canonical form"); @@ -815,7 +844,7 @@ namespace cryptonote { if (!tx_info[n].result) continue; - if (tx_info[n].tx->rct_signatures.type != rct::RCTTypeBulletproof) + if (tx_info[n].tx->rct_signatures.type != rct::RCTTypeBulletproof && tx_info[n].tx->rct_signatures.type != rct::RCTTypeBulletproof2) continue; if (assumed_bad || !rct::verRctSemanticsSimple(tx_info[n].tx->rct_signatures)) { @@ -834,7 +863,7 @@ namespace cryptonote TRY_ENTRY(); CRITICAL_REGION_LOCAL(m_incoming_tx_lock); - struct result { bool res; cryptonote::transaction tx; crypto::hash hash; crypto::hash prefix_hash; bool in_txpool; bool in_blockchain; }; + struct result { bool res; cryptonote::transaction tx; crypto::hash hash; crypto::hash prefix_hash; }; std::vector<result> results(tx_blobs.size()); tvc.resize(tx_blobs.size()); @@ -900,16 +929,18 @@ namespace cryptonote bool ok = true; it = tx_blobs.begin(); for (size_t i = 0; i < tx_blobs.size(); i++, ++it) { - if (already_have[i]) - continue; if (!results[i].res) { ok = false; continue; } + if (keeped_by_block) + get_blockchain_storage().on_new_tx_from_block(results[i].tx); + if (already_have[i]) + continue; const size_t weight = get_transaction_weight(results[i].tx, it->size()); - ok &= add_new_tx(results[i].tx, results[i].hash, results[i].prefix_hash, weight, tvc[i], keeped_by_block, relayed, do_not_relay); + ok &= add_new_tx(results[i].tx, results[i].hash, tx_blobs[i], results[i].prefix_hash, weight, tvc[i], keeped_by_block, relayed, do_not_relay); if(tvc[i].m_verifivation_failed) {MERROR_VER("Transaction verification failed: " << results[i].hash);} else if(tvc[i].m_verifivation_impossible) @@ -1127,7 +1158,7 @@ namespace cryptonote blobdata bl; t_serializable_object_to_blob(tx, bl); size_t tx_weight = get_transaction_weight(tx, bl.size()); - return add_new_tx(tx, tx_hash, tx_prefix_hash, tx_weight, tvc, keeped_by_block, relayed, do_not_relay); + return add_new_tx(tx, tx_hash, bl, tx_prefix_hash, tx_weight, tvc, keeped_by_block, relayed, do_not_relay); } //----------------------------------------------------------------------------------------------- size_t core::get_blockchain_total_transactions() const @@ -1135,11 +1166,8 @@ namespace cryptonote return m_blockchain_storage.get_total_transactions(); } //----------------------------------------------------------------------------------------------- - bool core::add_new_tx(transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t tx_weight, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) + bool core::add_new_tx(transaction& tx, const crypto::hash& tx_hash, const cryptonote::blobdata &blob, const crypto::hash& tx_prefix_hash, size_t tx_weight, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { - if (keeped_by_block) - get_blockchain_storage().on_new_tx_from_block(tx); - if(m_mempool.have_tx(tx_hash)) { LOG_PRINT_L2("tx " << tx_hash << "already have transaction in tx_pool"); @@ -1153,7 +1181,7 @@ namespace cryptonote } uint8_t version = m_blockchain_storage.get_current_hard_fork_version(); - return m_mempool.add_tx(tx, tx_hash, tx_weight, tvc, keeped_by_block, relayed, do_not_relay, version); + return m_mempool.add_tx(tx, tx_hash, blob, tx_weight, tvc, keeped_by_block, relayed, do_not_relay, version); } //----------------------------------------------------------------------------------------------- bool core::relay_txpool_transactions() @@ -1219,6 +1247,11 @@ namespace cryptonote return m_blockchain_storage.get_tx_outputs_gindexs(tx_id, indexs); } //----------------------------------------------------------------------------------------------- + bool core::get_tx_outputs_gindexs(const crypto::hash& tx_id, size_t n_txes, std::vector<std::vector<uint64_t>>& indexs) const + { + return m_blockchain_storage.get_tx_outputs_gindexs(tx_id, n_txes, indexs); + } + //----------------------------------------------------------------------------------------------- void core::pause_mine() { m_miner.pause(); @@ -1494,6 +1527,8 @@ namespace cryptonote m_txpool_auto_relayer.do_call(boost::bind(&core::relay_txpool_transactions, this)); m_check_updates_interval.do_call(boost::bind(&core::check_updates, this)); m_check_disk_space_interval.do_call(boost::bind(&core::check_disk_space, this)); + m_block_rate_interval.do_call(boost::bind(&core::check_block_rate, this)); + m_blockchain_pruning_interval.do_call(boost::bind(&core::update_blockchain_pruning, this)); m_miner.on_idle(); m_mempool.on_idle(); return true; @@ -1682,6 +1717,63 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- + double factorial(unsigned int n) + { + if (n <= 1) + return 1.0; + double f = n; + while (n-- > 1) + f *= n; + return f; + } + //----------------------------------------------------------------------------------------------- + static double probability(unsigned int blocks, unsigned int expected) + { + // https://www.umass.edu/wsp/resources/poisson/#computing + return pow(expected, blocks) / (factorial(blocks) * exp(expected)); + } + //----------------------------------------------------------------------------------------------- + bool core::check_block_rate() + { + if (m_offline || m_target_blockchain_height > get_current_blockchain_height()) + { + MDEBUG("Not checking block rate, offline or syncing"); + return true; + } + + static constexpr double threshold = 1. / (864000 / DIFFICULTY_TARGET_V2); // one false positive every 10 days + + const time_t now = time(NULL); + const std::vector<time_t> timestamps = m_blockchain_storage.get_last_block_timestamps(60); + + static const unsigned int seconds[] = { 5400, 1800, 600 }; + for (size_t n = 0; n < sizeof(seconds)/sizeof(seconds[0]); ++n) + { + unsigned int b = 0; + const time_t time_boundary = now - static_cast<time_t>(seconds[n]); + for (time_t ts: timestamps) b += ts >= time_boundary; + const double p = probability(b, seconds[n] / DIFFICULTY_TARGET_V2); + MDEBUG("blocks in the last " << seconds[n] / 60 << " minutes: " << b << " (probability " << p << ")"); + if (p < threshold) + { + MWARNING("There were " << b << " blocks in the last " << seconds[n] / 60 << " minutes, there might be large hash rate changes, or we might be partitioned, cut off from the Monero network or under attack. Or it could be just sheer bad luck."); + break; // no need to look further + } + } + + return true; + } + //----------------------------------------------------------------------------------------------- + bool core::update_blockchain_pruning() + { + return m_blockchain_storage.update_blockchain_pruning(); + } + //----------------------------------------------------------------------------------------------- + bool core::check_blockchain_pruning() + { + return m_blockchain_storage.check_blockchain_pruning(); + } + //----------------------------------------------------------------------------------------------- void core::set_target_blockchain_height(uint64_t target_blockchain_height) { m_target_blockchain_height = target_blockchain_height; @@ -1704,6 +1796,16 @@ namespace cryptonote return si.available; } //----------------------------------------------------------------------------------------------- + uint32_t core::get_blockchain_pruning_seed() const + { + return get_blockchain_storage().get_blockchain_pruning_seed(); + } + //----------------------------------------------------------------------------------------------- + bool core::prune_blockchain(uint32_t pruning_seed) + { + return get_blockchain_storage().prune_blockchain(pruning_seed); + } + //----------------------------------------------------------------------------------------------- std::time_t core::get_start_time() const { return start_time; diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 58fe5b7b5..4810fc891 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -34,7 +34,6 @@ #include <boost/program_options/options_description.hpp> #include <boost/program_options/variables_map.hpp> -#include <boost/interprocess/sync/file_lock.hpp> #include "cryptonote_protocol/cryptonote_protocol_handler_common.h" #include "storages/portable_storage_template_helper.h" @@ -63,6 +62,7 @@ namespace cryptonote extern const command_line::arg_descriptor<bool, false> arg_regtest_on; extern const command_line::arg_descriptor<difficulty_type> arg_fixed_difficulty; extern const command_line::arg_descriptor<bool> arg_offline; + extern const command_line::arg_descriptor<size_t> arg_block_download_max_size; /************************************************************************/ /* */ @@ -117,7 +117,7 @@ namespace cryptonote * @param relayed whether or not the transaction was relayed to us * @param do_not_relay whether to prevent the transaction from being relayed * - * @return true if the transaction made it to the transaction pool, otherwise false + * @return true if the transaction was accepted, false otherwise */ bool handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); @@ -133,7 +133,7 @@ namespace cryptonote * @param relayed whether or not the transactions were relayed to us * @param do_not_relay whether to prevent the transactions from being relayed * - * @return true if the transactions made it to the transaction pool, otherwise false + * @return true if the transactions were accepted, false otherwise */ bool handle_incoming_txs(const std::vector<blobdata>& tx_blobs, std::vector<tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); @@ -242,12 +242,12 @@ namespace cryptonote * a miner instance with parameters given on the command line (or defaults) * * @param vm command line parameters - * @param config_subdir subdirectory for config storage * @param test_options configuration options for testing + * @param get_checkpoints if set, will be called to get checkpoints data, must return checkpoints data pointer and size or nullptr if there ain't any checkpoints for specific network type * * @return false if one of the init steps fails, otherwise true */ - bool init(const boost::program_options::variables_map& vm, const char *config_subdir = NULL, const test_options *test_options = NULL); + bool init(const boost::program_options::variables_map& vm, const test_options *test_options = NULL, const GetCheckpointsCallback& get_checkpoints = nullptr); /** * @copydoc Blockchain::reset_and_set_genesis_block @@ -360,6 +360,13 @@ namespace cryptonote * * @note see Blockchain::get_transactions */ + bool get_split_transactions_blobs(const std::vector<crypto::hash>& txs_ids, std::vector<std::tuple<crypto::hash, cryptonote::blobdata, crypto::hash, cryptonote::blobdata>>& txs, std::vector<crypto::hash>& missed_txs) const; + + /** + * @copydoc Blockchain::get_transactions + * + * @note see Blockchain::get_transactions + */ bool get_transactions(const std::vector<crypto::hash>& txs_ids, std::vector<transaction>& txs, std::vector<crypto::hash>& missed_txs) const; /** @@ -535,6 +542,7 @@ namespace cryptonote * @note see Blockchain::get_tx_outputs_gindexs */ bool get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<uint64_t>& indexs) const; + bool get_tx_outputs_gindexs(const crypto::hash& tx_id, size_t n_txes, std::vector<std::vector<uint64_t>>& indexs) const; /** * @copydoc Blockchain::get_tail_id @@ -756,6 +764,13 @@ namespace cryptonote bool fluffy_blocks_enabled() const { return m_fluffy_blocks_enabled; } /** + * @brief get whether transaction relay should be padded + * + * @return whether transaction relay should be padded + */ + bool pad_transactions() const { return m_pad_transactions; } + + /** * @brief check a set of hashes against the precompiled hash set * * @return number of usable blocks @@ -776,19 +791,50 @@ namespace cryptonote */ bool offline() const { return m_offline; } + /** + * @brief get the blockchain pruning seed + * + * @return the blockchain pruning seed + */ + uint32_t get_blockchain_pruning_seed() const; + + /** + * @brief prune the blockchain + * + * @param pruning_seed the seed to use to prune the chain (0 for default, highly recommended) + * + * @return true iff success + */ + bool prune_blockchain(uint32_t pruning_seed = 0); + + /** + * @brief incrementally prunes blockchain + * + * @return true on success, false otherwise + */ + bool update_blockchain_pruning(); + + /** + * @brief checks the blockchain pruning if enabled + * + * @return true on success, false otherwise + */ + bool check_blockchain_pruning(); + private: /** * @copydoc add_new_tx(transaction&, tx_verification_context&, bool) * * @param tx_hash the transaction's hash + * @param blob the transaction as a blob * @param tx_prefix_hash the transaction prefix' hash * @param tx_weight the weight of the transaction * @param relayed whether or not the transaction was relayed to us * @param do_not_relay whether to prevent the transaction from being relayed * */ - bool add_new_tx(transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prefix_hash, size_t tx_weight, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); + bool add_new_tx(transaction& tx, const crypto::hash& tx_hash, const cryptonote::blobdata &blob, const crypto::hash& tx_prefix_hash, size_t tx_weight, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); /** * @brief add a new transaction to the transaction pool @@ -945,6 +991,13 @@ namespace cryptonote */ bool check_disk_space(); + /** + * @brief checks block rate, and warns if it's too slow + * + * @return true on success, false otherwise + */ + bool check_block_rate(); + bool m_test_drop_download = true; //!< whether or not to drop incoming blocks (for testing) uint64_t m_test_drop_download_height = 0; //!< height under which to drop incoming blocks, if doing so @@ -969,6 +1022,8 @@ namespace cryptonote epee::math_helper::once_a_time_seconds<60*2, false> m_txpool_auto_relayer; //!< interval for checking re-relaying txpool transactions epee::math_helper::once_a_time_seconds<60*60*12, true> m_check_updates_interval; //!< interval for checking for new versions epee::math_helper::once_a_time_seconds<60*10, true> m_check_disk_space_interval; //!< interval for checking for disk space + epee::math_helper::once_a_time_seconds<90, false> m_block_rate_interval; //!< interval for checking block rate + epee::math_helper::once_a_time_seconds<60*60*5, true> m_blockchain_pruning_interval; //!< interval for incremental blockchain pruning std::atomic<bool> m_starter_message_showed; //!< has the "daemon will sync now" message been shown? @@ -1005,6 +1060,7 @@ namespace cryptonote bool m_fluffy_blocks_enabled; bool m_offline; + bool m_pad_transactions; }; } diff --git a/src/cryptonote_core/cryptonote_tx_utils.cpp b/src/cryptonote_core/cryptonote_tx_utils.cpp index 4bc33b56b..0a04e0d38 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.cpp +++ b/src/cryptonote_core/cryptonote_tx_utils.cpp @@ -38,6 +38,7 @@ using namespace epee; #include "cryptonote_tx_utils.h" #include "cryptonote_config.h" #include "cryptonote_basic/miner.h" +#include "cryptonote_basic/tx_extra.h" #include "crypto/crypto.h" #include "crypto/hash.h" #include "ringct/rctSigs.h" @@ -84,6 +85,8 @@ namespace cryptonote if(!extra_nonce.empty()) if(!add_extra_nonce_to_tx_extra(tx.extra, extra_nonce)) return false; + if (!sort_tx_extra(tx.extra, tx.extra)) + return false; txin_gen in; in.height = height; @@ -195,7 +198,7 @@ namespace cryptonote return addr.m_view_public_key; } //--------------------------------------------------------------- - bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct, rct::RangeProofType range_proof_type, rct::multisig_out *msout, bool shuffle_outs) + bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct, const rct::RCTConfig &rct_config, rct::multisig_out *msout, bool shuffle_outs) { hw::device &hwdev = sender_account_keys.get_device(); @@ -223,13 +226,15 @@ namespace cryptonote std::vector<tx_extra_field> tx_extra_fields; if (parse_tx_extra(tx.extra, tx_extra_fields)) { + bool add_dummy_payment_id = true; tx_extra_nonce extra_nonce; if (find_tx_extra_field_by_type(tx_extra_fields, extra_nonce)) { - crypto::hash8 payment_id = null_hash8; - if (get_encrypted_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id)) + crypto::hash payment_id = null_hash; + crypto::hash8 payment_id8 = null_hash8; + if (get_encrypted_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id8)) { - LOG_PRINT_L2("Encrypting payment id " << payment_id); + LOG_PRINT_L2("Encrypting payment id " << payment_id8); crypto::public_key view_key_pub = get_destination_view_key_pub(destinations, change_addr); if (view_key_pub == null_pkey) { @@ -237,28 +242,60 @@ namespace cryptonote return false; } - if (!hwdev.encrypt_payment_id(payment_id, view_key_pub, tx_key)) + if (!hwdev.encrypt_payment_id(payment_id8, view_key_pub, tx_key)) { LOG_ERROR("Failed to encrypt payment id"); return false; } std::string extra_nonce; - set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, payment_id); + set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, payment_id8); remove_field_from_tx_extra(tx.extra, typeid(tx_extra_nonce)); if (!add_extra_nonce_to_tx_extra(tx.extra, extra_nonce)) { LOG_ERROR("Failed to add encrypted payment id to tx extra"); return false; } - LOG_PRINT_L1("Encrypted payment ID: " << payment_id); + LOG_PRINT_L1("Encrypted payment ID: " << payment_id8); + add_dummy_payment_id = false; + } + else if (get_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id)) + { + add_dummy_payment_id = false; + } + } + + // we don't add one if we've got more than the usual 1 destination plus change + if (destinations.size() > 2) + add_dummy_payment_id = false; + + if (add_dummy_payment_id) + { + // if we have neither long nor short payment id, add a dummy short one, + // this should end up being the vast majority of txes as time goes on + std::string extra_nonce; + crypto::hash8 payment_id8 = null_hash8; + crypto::public_key view_key_pub = get_destination_view_key_pub(destinations, change_addr); + if (view_key_pub == null_pkey) + { + LOG_ERROR("Failed to get key to encrypt dummy payment id with"); + } + else + { + hwdev.encrypt_payment_id(payment_id8, view_key_pub, tx_key); + set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, payment_id8); + if (!add_extra_nonce_to_tx_extra(tx.extra, extra_nonce)) + { + LOG_ERROR("Failed to add dummy encrypted payment id to tx extra"); + // continue anyway + } } } } else { - LOG_ERROR("Failed to parse tx extra"); - return false; + MWARNING("Failed to parse tx extra"); + tx_extra_fields.clear(); } struct input_generation_context_data @@ -434,6 +471,9 @@ namespace cryptonote add_additional_tx_pub_keys_to_extra(tx.extra, additional_tx_public_keys); } + if (!sort_tx_extra(tx.extra, tx.extra)) + return false; + //check money if(summary_outs_money > summary_inputs_money ) { @@ -491,7 +531,7 @@ namespace cryptonote // the non-simple version is slightly smaller, but assumes all real inputs // are on the same index, so can only be used if there just one ring. - bool use_simple_rct = sources.size() > 1 || range_proof_type != rct::RangeProofBorromean; + bool use_simple_rct = sources.size() > 1 || rct_config.range_proof_type != rct::RangeProofBorromean; if (!use_simple_rct) { @@ -589,9 +629,9 @@ namespace cryptonote get_transaction_prefix_hash(tx, tx_prefix_hash); rct::ctkeyV outSk; if (use_simple_rct) - tx.rct_signatures = rct::genRctSimple(rct::hash2rct(tx_prefix_hash), inSk, destinations, inamounts, outamounts, amount_in - amount_out, mixRing, amount_keys, msout ? &kLRki : NULL, msout, index, outSk, range_proof_type, hwdev); + tx.rct_signatures = rct::genRctSimple(rct::hash2rct(tx_prefix_hash), inSk, destinations, inamounts, outamounts, amount_in - amount_out, mixRing, amount_keys, msout ? &kLRki : NULL, msout, index, outSk, rct_config, hwdev); else - tx.rct_signatures = rct::genRct(rct::hash2rct(tx_prefix_hash), inSk, destinations, outamounts, mixRing, amount_keys, msout ? &kLRki[0] : NULL, msout, sources[0].real_output, outSk, hwdev); // same index assumption + tx.rct_signatures = rct::genRct(rct::hash2rct(tx_prefix_hash), inSk, destinations, outamounts, mixRing, amount_keys, msout ? &kLRki[0] : NULL, msout, sources[0].real_output, outSk, rct_config, hwdev); // same index assumption memwipe(inSk.data(), inSk.size() * sizeof(rct::ctkey)); CHECK_AND_ASSERT_MES(tx.vout.size() == outSk.size(), false, "outSk size does not match vout"); @@ -604,7 +644,7 @@ namespace cryptonote return true; } //--------------------------------------------------------------- - bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys, bool rct, rct::RangeProofType range_proof_type, rct::multisig_out *msout) + bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys, bool rct, const rct::RCTConfig &rct_config, rct::multisig_out *msout) { hw::device &hwdev = sender_account_keys.get_device(); hwdev.open_tx(tx_key); @@ -622,19 +662,19 @@ namespace cryptonote additional_tx_keys.push_back(keypair::generate(sender_account_keys.get_device()).sec); } - bool r = construct_tx_with_tx_key(sender_account_keys, subaddresses, sources, destinations, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, rct, range_proof_type, msout); + bool r = construct_tx_with_tx_key(sender_account_keys, subaddresses, sources, destinations, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, rct, rct_config, msout); hwdev.close_tx(); return r; } //--------------------------------------------------------------- - bool construct_tx(const account_keys& sender_account_keys, std::vector<tx_source_entry>& sources, const std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time) + bool construct_tx(const account_keys& sender_account_keys, std::vector<tx_source_entry>& sources, const std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time) { std::unordered_map<crypto::public_key, cryptonote::subaddress_index> subaddresses; subaddresses[sender_account_keys.m_account_address.m_spend_public_key] = {0,0}; crypto::secret_key tx_key; std::vector<crypto::secret_key> additional_tx_keys; std::vector<tx_destination_entry> destinations_copy = destinations; - return construct_tx_and_get_tx_key(sender_account_keys, subaddresses, sources, destinations_copy, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, rct::RangeProofBorromean, NULL); + return construct_tx_and_get_tx_key(sender_account_keys, subaddresses, sources, destinations_copy, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, { rct::RangeProofBorromean, 0}, NULL); } //--------------------------------------------------------------- bool generate_genesis_block( diff --git a/src/cryptonote_core/cryptonote_tx_utils.h b/src/cryptonote_core/cryptonote_tx_utils.h index f2cf7b6ca..85061668b 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.h +++ b/src/cryptonote_core/cryptonote_tx_utils.h @@ -73,25 +73,30 @@ namespace cryptonote struct tx_destination_entry { + std::string original; uint64_t amount; //money account_public_address addr; //destination address bool is_subaddress; + bool is_integrated; - tx_destination_entry() : amount(0), addr(AUTO_VAL_INIT(addr)), is_subaddress(false) { } - tx_destination_entry(uint64_t a, const account_public_address &ad, bool is_subaddress) : amount(a), addr(ad), is_subaddress(is_subaddress) { } + tx_destination_entry() : amount(0), addr(AUTO_VAL_INIT(addr)), is_subaddress(false), is_integrated(false) { } + tx_destination_entry(uint64_t a, const account_public_address &ad, bool is_subaddress) : amount(a), addr(ad), is_subaddress(is_subaddress), is_integrated(false) { } + tx_destination_entry(const std::string &o, uint64_t a, const account_public_address &ad, bool is_subaddress) : original(o), amount(a), addr(ad), is_subaddress(is_subaddress), is_integrated(false) { } BEGIN_SERIALIZE_OBJECT() + FIELD(original) VARINT_FIELD(amount) FIELD(addr) FIELD(is_subaddress) + FIELD(is_integrated) END_SERIALIZE() }; //--------------------------------------------------------------- crypto::public_key get_destination_view_key_pub(const std::vector<tx_destination_entry> &destinations, const boost::optional<cryptonote::account_public_address>& change_addr); - bool construct_tx(const account_keys& sender_account_keys, std::vector<tx_source_entry> &sources, const std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time); - bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, rct::RangeProofType range_proof_type = rct::RangeProofBorromean, rct::multisig_out *msout = NULL, bool shuffle_outs = true); - bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, std::vector<uint8_t> extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, rct::RangeProofType range_proof_type = rct::RangeProofBorromean, rct::multisig_out *msout = NULL); + bool construct_tx(const account_keys& sender_account_keys, std::vector<tx_source_entry> &sources, const std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time); + bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, const rct::RCTConfig &rct_config = { rct::RangeProofBorromean, 0 }, rct::multisig_out *msout = NULL, bool shuffle_outs = true); + bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, const rct::RCTConfig &rct_config = { rct::RangeProofBorromean, 0 }, rct::multisig_out *msout = NULL); bool generate_genesis_block( block& bl @@ -102,7 +107,7 @@ namespace cryptonote } BOOST_CLASS_VERSION(cryptonote::tx_source_entry, 1) -BOOST_CLASS_VERSION(cryptonote::tx_destination_entry, 1) +BOOST_CLASS_VERSION(cryptonote::tx_destination_entry, 2) namespace boost { @@ -132,6 +137,13 @@ namespace boost if (ver < 1) return; a & x.is_subaddress; + if (ver < 2) + { + x.is_integrated = false; + return; + } + a & x.original; + a & x.is_integrated; } } } diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index 553a22298..1c8f1c62c 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -40,7 +40,7 @@ #include "blockchain.h" #include "blockchain_db/blockchain_db.h" #include "common/boost_serialization_helper.h" -#include "common/int-util.h" +#include "int-util.h" #include "misc_language.h" #include "warnings.h" #include "common/perf_timer.h" @@ -111,7 +111,7 @@ namespace cryptonote } //--------------------------------------------------------------------------------- - bool tx_memory_pool::add_tx(transaction &tx, /*const crypto::hash& tx_prefix_hash,*/ const crypto::hash &id, size_t tx_weight, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version) + bool tx_memory_pool::add_tx(transaction &tx, /*const crypto::hash& tx_prefix_hash,*/ const crypto::hash &id, const cryptonote::blobdata &blob, size_t tx_weight, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version) { // this should already be called with that lock, but let's make it explicit for clarity CRITICAL_REGION_LOCAL(m_transactions_lock); @@ -247,9 +247,11 @@ namespace cryptonote memset(meta.padding, 0, sizeof(meta.padding)); try { + if (kept_by_block) + m_parsed_tx_cache.insert(std::make_pair(id, tx)); CRITICAL_REGION_LOCAL1(m_blockchain); LockedTXN lock(m_blockchain); - m_blockchain.add_txpool_tx(tx, meta); + m_blockchain.add_txpool_tx(id, blob, meta); if (!insert_key_images(tx, id, kept_by_block)) return false; m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)tx_weight, receive_time), id); @@ -288,12 +290,13 @@ namespace cryptonote try { + if (kept_by_block) + m_parsed_tx_cache.insert(std::make_pair(id, tx)); CRITICAL_REGION_LOCAL1(m_blockchain); LockedTXN lock(m_blockchain); - const crypto::hash txid = get_transaction_hash(tx); - m_blockchain.remove_txpool_tx(txid); - m_blockchain.add_txpool_tx(tx, meta); - if (!insert_key_images(tx, txid, kept_by_block)) + m_blockchain.remove_txpool_tx(id); + m_blockchain.add_txpool_tx(id, blob, meta); + if (!insert_key_images(tx, id, kept_by_block)) return false; m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)tx_weight, receive_time), id); } @@ -324,9 +327,11 @@ namespace cryptonote { crypto::hash h = null_hash; size_t blob_size = 0; - if (!get_transaction_hash(tx, h, blob_size) || blob_size == 0) + cryptonote::blobdata bl; + t_serializable_object_to_blob(tx, bl); + if (bl.size() == 0 || !get_transaction_hash(tx, h)) return false; - return add_tx(tx, h, get_transaction_weight(tx, blob_size), tvc, keeped_by_block, relayed, do_not_relay, version); + return add_tx(tx, h, bl, get_transaction_weight(tx, bl.size()), tvc, keeped_by_block, relayed, do_not_relay, version); } //--------------------------------------------------------------------------------- size_t tx_memory_pool::get_txpool_weight() const @@ -379,11 +384,11 @@ namespace cryptonote return; } // remove first, in case this throws, so key images aren't removed - MINFO("Pruning tx " << txid << " from txpool: weight: " << it->first.second << ", fee/byte: " << it->first.first); + MINFO("Pruning tx " << txid << " from txpool: weight: " << meta.weight << ", fee/byte: " << it->first.first); m_blockchain.remove_txpool_tx(txid); - m_txpool_weight -= it->first.second; + m_txpool_weight -= meta.weight; remove_transaction_keyimages(tx, txid); - MINFO("Pruned tx " << txid << " from txpool: weight: " << it->first.second << ", fee/byte: " << it->first.first); + MINFO("Pruned tx " << txid << " from txpool: weight: " << meta.weight << ", fee/byte: " << it->first.first); m_txs_by_fee_and_receive_time.erase(it--); changed = true; } @@ -454,8 +459,6 @@ namespace cryptonote CRITICAL_REGION_LOCAL1(m_blockchain); auto sorted_it = find_tx_in_sorted_container(id); - if (sorted_it == m_txs_by_fee_and_receive_time.end()) - return false; try { @@ -467,11 +470,20 @@ namespace cryptonote return false; } cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(id); - if (!parse_and_validate_tx_from_blob(txblob, tx)) + auto ci = m_parsed_tx_cache.find(id); + if (ci != m_parsed_tx_cache.end()) + { + tx = ci->second; + } + else if (!parse_and_validate_tx_from_blob(txblob, tx)) { MERROR("Failed to parse tx from txpool"); return false; } + else + { + tx.set_hash(id); + } tx_weight = meta.weight; fee = meta.fee; relayed = meta.relayed; @@ -489,7 +501,8 @@ namespace cryptonote return false; } - m_txs_by_fee_and_receive_time.erase(sorted_it); + if (sorted_it != m_txs_by_fee_and_receive_time.end()) + m_txs_by_fee_and_receive_time.erase(sorted_it); ++m_cookie; return true; } @@ -650,7 +663,8 @@ namespace cryptonote // continue return true; } - txs.push_back(tx); + tx.set_hash(txid); + txs.push_back(std::move(tx)); return true; }, true, include_unrelayed_txes); } @@ -773,6 +787,7 @@ namespace cryptonote // continue return true; } + tx.set_hash(txid); txi.tx_json = obj_to_json_str(tx); txi.blob_size = bd->size(); txi.weight = meta.weight; @@ -789,7 +804,7 @@ namespace cryptonote txi.last_relayed_time = include_sensitive_data ? meta.last_relayed_time : 0; txi.do_not_relay = meta.do_not_relay; txi.double_spend_seen = meta.double_spend_seen; - tx_infos.push_back(txi); + tx_infos.push_back(std::move(txi)); return true; }, true, include_sensitive_data); @@ -838,14 +853,13 @@ namespace cryptonote m_blockchain.for_all_txpool_txes([&tx_infos, key_image_infos](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ cryptonote::rpc::tx_in_pool txi; txi.tx_hash = txid; - transaction tx; - if (!parse_and_validate_tx_from_blob(*bd, tx)) + if (!parse_and_validate_tx_from_blob(*bd, txi.tx)) { MERROR("Failed to parse tx from txpool"); // continue return true; } - txi.tx = tx; + txi.tx.set_hash(txid); txi.blob_size = bd->size(); txi.weight = meta.weight; txi.fee = meta.fee; @@ -872,7 +886,7 @@ namespace cryptonote } const crypto::key_image& k_image = kee.first; - key_image_infos[k_image] = tx_hashes; + key_image_infos[k_image] = std::move(tx_hashes); } return true; } @@ -910,6 +924,7 @@ namespace cryptonote { CRITICAL_REGION_LOCAL(m_transactions_lock); m_input_cache.clear(); + m_parsed_tx_cache.clear(); return true; } //--------------------------------------------------------------------------------- @@ -917,6 +932,7 @@ namespace cryptonote { CRITICAL_REGION_LOCAL(m_transactions_lock); m_input_cache.clear(); + m_parsed_tx_cache.clear(); return true; } //--------------------------------------------------------------------------------- @@ -979,21 +995,23 @@ namespace cryptonote { struct transction_parser { - transction_parser(const cryptonote::blobdata &txblob, transaction &tx): txblob(txblob), tx(tx), parsed(false) {} + transction_parser(const cryptonote::blobdata &txblob, const crypto::hash &txid, transaction &tx): txblob(txblob), txid(txid), tx(tx), parsed(false) {} cryptonote::transaction &operator()() { if (!parsed) { if (!parse_and_validate_tx_from_blob(txblob, tx)) throw std::runtime_error("failed to parse transaction blob"); + tx.set_hash(txid); parsed = true; } return tx; } const cryptonote::blobdata &txblob; + const crypto::hash &txid; transaction &tx; bool parsed; - } lazy_tx(txblob, tx); + } lazy_tx(txblob, txid, tx); //not the best implementation at this time, sorry :( //check is ring_signature already checked ? diff --git a/src/cryptonote_core/tx_pool.h b/src/cryptonote_core/tx_pool.h index 7a0cc23bf..670d70d77 100644 --- a/src/cryptonote_core/tx_pool.h +++ b/src/cryptonote_core/tx_pool.h @@ -107,7 +107,7 @@ namespace cryptonote * @param id the transaction's hash * @param tx_weight the transaction's weight */ - bool add_tx(transaction &tx, const crypto::hash &id, size_t tx_weight, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version); + bool add_tx(transaction &tx, const crypto::hash &id, const cryptonote::blobdata &blob, size_t tx_weight, tx_verification_context& tvc, bool kept_by_block, bool relayed, bool do_not_relay, uint8_t version); /** * @brief add a transaction to the transaction pool @@ -510,7 +510,7 @@ namespace cryptonote * @param txd the transaction to check (and info about it) * @param txid the txid of the transaction to check * @param txblob the transaction blob to check - * @param tx the parsed transaction prefix, if successful + * @param tx the parsed transaction, if successful * * @return true if the transaction is good to go, otherwise false */ @@ -584,6 +584,8 @@ private: size_t m_txpool_weight; mutable std::unordered_map<crypto::hash, std::tuple<bool, tx_verification_context, uint64_t, crypto::hash>> m_input_cache; + + std::unordered_map<crypto::hash, transaction> m_parsed_tx_cache; }; } diff --git a/src/cryptonote_protocol/block_queue.cpp b/src/cryptonote_protocol/block_queue.cpp index 05f4189fb..672696944 100644 --- a/src/cryptonote_protocol/block_queue.cpp +++ b/src/cryptonote_protocol/block_queue.cpp @@ -31,8 +31,10 @@ #include <vector> #include <unordered_map> #include <boost/uuid/nil_generator.hpp> +#include <boost/uuid/uuid_io.hpp> #include "string_tools.h" #include "cryptonote_protocol_defs.h" +#include "common/pruning.h" #include "block_queue.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -59,7 +61,10 @@ void block_queue::add_blocks(uint64_t height, std::vector<cryptonote::block_comp if (has_hashes) { for (const crypto::hash &h: hashes) + { requested_hashes.insert(h); + have_blocks.insert(h); + } set_span_hashes(height, connection_id, hashes); } } @@ -89,7 +94,10 @@ void block_queue::erase_block(block_map::iterator j) { CHECK_AND_ASSERT_THROW_MES(j != blocks.end(), "Invalid iterator"); for (const crypto::hash &h: j->hashes) + { requested_hashes.erase(h); + have_blocks.erase(h); + } blocks.erase(j); } @@ -97,12 +105,10 @@ void block_queue::flush_stale_spans(const std::set<boost::uuids::uuid> &live_con { boost::unique_lock<boost::recursive_mutex> lock(mutex); block_map::iterator i = blocks.begin(); - if (i != blocks.end() && is_blockchain_placeholder(*i)) - ++i; while (i != blocks.end()) { block_map::iterator j = i++; - if (live_connections.find(j->connection_id) == live_connections.end() && j->blocks.size() == 0) + if (j->blocks.empty() && live_connections.find(j->connection_id) == live_connections.end()) { erase_block(j); } @@ -151,23 +157,56 @@ uint64_t block_queue::get_max_block_height() const return height; } +uint64_t block_queue::get_next_needed_height(uint64_t blockchain_height) const +{ + boost::unique_lock<boost::recursive_mutex> lock(mutex); + if (blocks.empty()) + return blockchain_height; + uint64_t last_needed_height = blockchain_height; + bool first = true; + for (const auto &span: blocks) + { + if (span.start_block_height + span.nblocks - 1 < blockchain_height) + continue; + if (span.start_block_height != last_needed_height || (first && span.blocks.empty())) + return last_needed_height; + last_needed_height = span.start_block_height + span.nblocks; + first = false; + } + return last_needed_height; +} + void block_queue::print() const { boost::unique_lock<boost::recursive_mutex> lock(mutex); MDEBUG("Block queue has " << blocks.size() << " spans"); for (const auto &span: blocks) - MDEBUG(" " << span.start_block_height << " - " << (span.start_block_height+span.nblocks-1) << " (" << span.nblocks << ") - " << (is_blockchain_placeholder(span) ? "blockchain" : span.blocks.empty() ? "scheduled" : "filled ") << " " << span.connection_id << " (" << ((unsigned)(span.rate*10/1024.f))/10.f << " kB/s)"); + MDEBUG(" " << span.start_block_height << " - " << (span.start_block_height+span.nblocks-1) << " (" << span.nblocks << ") - " << (span.blocks.empty() ? "scheduled" : "filled ") << " " << span.connection_id << " (" << ((unsigned)(span.rate*10/1024.f))/10.f << " kB/s)"); } -std::string block_queue::get_overview() const +std::string block_queue::get_overview(uint64_t blockchain_height) const { boost::unique_lock<boost::recursive_mutex> lock(mutex); if (blocks.empty()) return "[]"; block_map::const_iterator i = blocks.begin(); - std::string s = std::string("[") + std::to_string(i->start_block_height + i->nblocks - 1) + ":"; - while (++i != blocks.end()) - s += i->blocks.empty() ? "." : "o"; + std::string s = std::string("["); + uint64_t expected = blockchain_height; + while (i != blocks.end()) + { + if (expected > i->start_block_height) + { + s += "<"; + } + else + { + if (expected < i->start_block_height) + s += std::string(std::max((uint64_t)1, (i->start_block_height - expected) / (i->nblocks ? i->nblocks : 1)), '_'); + s += i->blocks.empty() ? "." : i->start_block_height == blockchain_height ? "m" : "o"; + expected = i->start_block_height + i->nblocks; + } + ++i; + } s += "]"; return s; } @@ -183,16 +222,31 @@ bool block_queue::requested(const crypto::hash &hash) const return requested_internal(hash); } -std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, const std::vector<crypto::hash> &block_hashes, boost::posix_time::ptime time) +bool block_queue::have(const crypto::hash &hash) const +{ + boost::unique_lock<boost::recursive_mutex> lock(mutex); + return have_blocks.find(hash) != have_blocks.end(); +} + +std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, uint32_t pruning_seed, uint64_t blockchain_height, const std::vector<crypto::hash> &block_hashes, boost::posix_time::ptime time) { boost::unique_lock<boost::recursive_mutex> lock(mutex); + MDEBUG("reserve_span: first_block_height " << first_block_height << ", last_block_height " << last_block_height + << ", max " << max_blocks << ", seed " << epee::string_tools::to_string_hex(pruning_seed) << ", blockchain_height " << + blockchain_height << ", block hashes size " << block_hashes.size()); if (last_block_height < first_block_height || max_blocks == 0) { MDEBUG("reserve_span: early out: first_block_height " << first_block_height << ", last_block_height " << last_block_height << ", max_blocks " << max_blocks); return std::make_pair(0, 0); } + if (block_hashes.size() >= last_block_height) + { + MDEBUG("reserve_span: more block hashes than fit within last_block_height: " << block_hashes.size() << " and " << last_block_height); + return std::make_pair(0, 0); + } + // skip everything we've already requested uint64_t span_start_height = last_block_height - block_hashes.size() + 1; std::vector<crypto::hash>::const_iterator i = block_hashes.begin(); while (i != block_hashes.end() && requested_internal(*i)) @@ -200,55 +254,57 @@ std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_hei ++i; ++span_start_height; } + + // if the peer's pruned for the starting block and its unpruned stripe comes next, start downloading from there + const uint32_t next_unpruned_height = tools::get_next_unpruned_block_height(span_start_height, blockchain_height, pruning_seed); + MDEBUG("reserve_span: next_unpruned_height " << next_unpruned_height << " from " << span_start_height << " and seed " + << epee::string_tools::to_string_hex(pruning_seed) << ", limit " << span_start_height + CRYPTONOTE_PRUNING_STRIPE_SIZE); + if (next_unpruned_height > span_start_height && next_unpruned_height < span_start_height + CRYPTONOTE_PRUNING_STRIPE_SIZE) + { + MDEBUG("We can download from next span: ideal height " << span_start_height << ", next unpruned height " << next_unpruned_height << + "(+" << next_unpruned_height - span_start_height << "), current seed " << pruning_seed); + span_start_height = next_unpruned_height; + } + MDEBUG("span_start_height: " <<span_start_height); + const uint64_t block_hashes_start_height = last_block_height - block_hashes.size() + 1; + if (span_start_height >= block_hashes.size() + block_hashes_start_height) + { + MDEBUG("Out of hashes, cannot reserve"); + return std::make_pair(0, 0); + } + + i = block_hashes.begin() + span_start_height - block_hashes_start_height; + while (i != block_hashes.end() && requested_internal(*i)) + { + ++i; + ++span_start_height; + } + uint64_t span_length = 0; std::vector<crypto::hash> hashes; - while (i != block_hashes.end() && span_length < max_blocks) + while (i != block_hashes.end() && span_length < max_blocks && tools::has_unpruned_block(span_start_height + span_length, blockchain_height, pruning_seed)) { hashes.push_back(*i); ++i; ++span_length; } if (span_length == 0) + { + MDEBUG("span_length 0, cannot reserve"); return std::make_pair(0, 0); + } MDEBUG("Reserving span " << span_start_height << " - " << (span_start_height + span_length - 1) << " for " << connection_id); add_blocks(span_start_height, span_length, connection_id, time); set_span_hashes(span_start_height, connection_id, hashes); return std::make_pair(span_start_height, span_length); } -bool block_queue::is_blockchain_placeholder(const span &span) const -{ - return span.connection_id == boost::uuids::nil_uuid(); -} - -std::pair<uint64_t, uint64_t> block_queue::get_start_gap_span() const -{ - boost::unique_lock<boost::recursive_mutex> lock(mutex); - if (blocks.empty()) - return std::make_pair(0, 0); - block_map::const_iterator i = blocks.begin(); - if (!is_blockchain_placeholder(*i)) - return std::make_pair(0, 0); - uint64_t current_height = i->start_block_height + i->nblocks - 1; - ++i; - if (i == blocks.end()) - return std::make_pair(0, 0); - uint64_t first_span_height = i->start_block_height; - if (first_span_height <= current_height + 1) - return std::make_pair(0, 0); - MDEBUG("Found gap at start of spans: last blockchain block height " << current_height << ", first span's block height " << first_span_height); - print(); - return std::make_pair(current_height + 1, first_span_height - current_height - 1); -} - std::pair<uint64_t, uint64_t> block_queue::get_next_span_if_scheduled(std::vector<crypto::hash> &hashes, boost::uuids::uuid &connection_id, boost::posix_time::ptime &time) const { boost::unique_lock<boost::recursive_mutex> lock(mutex); if (blocks.empty()) return std::make_pair(0, 0); block_map::const_iterator i = blocks.begin(); - if (is_blockchain_placeholder(*i)) - ++i; if (i == blocks.end()) return std::make_pair(0, 0); if (!i->blocks.empty()) @@ -259,6 +315,16 @@ std::pair<uint64_t, uint64_t> block_queue::get_next_span_if_scheduled(std::vecto return std::make_pair(i->start_block_height, i->nblocks); } +void block_queue::reset_next_span_time(boost::posix_time::ptime t) +{ + boost::unique_lock<boost::recursive_mutex> lock(mutex); + CHECK_AND_ASSERT_THROW_MES(!blocks.empty(), "No next span to reset time"); + block_map::iterator i = blocks.begin(); + CHECK_AND_ASSERT_THROW_MES(i != blocks.end(), "No next span to reset time"); + CHECK_AND_ASSERT_THROW_MES(i->blocks.empty(), "Next span is not empty"); + (boost::posix_time::ptime&)i->time = t; // sod off, time doesn't influence sorting +} + void block_queue::set_span_hashes(uint64_t start_height, const boost::uuids::uuid &connection_id, std::vector<crypto::hash> hashes) { boost::unique_lock<boost::recursive_mutex> lock(mutex); @@ -283,8 +349,6 @@ bool block_queue::get_next_span(uint64_t &height, std::vector<cryptonote::block_ if (blocks.empty()) return false; block_map::const_iterator i = blocks.begin(); - if (is_blockchain_placeholder(*i)) - ++i; for (; i != blocks.end(); ++i) { if (!filled || !i->blocks.empty()) @@ -298,19 +362,34 @@ bool block_queue::get_next_span(uint64_t &height, std::vector<cryptonote::block_ return false; } -bool block_queue::has_next_span(const boost::uuids::uuid &connection_id, bool &filled) const +bool block_queue::has_next_span(const boost::uuids::uuid &connection_id, bool &filled, boost::posix_time::ptime &time) const { boost::unique_lock<boost::recursive_mutex> lock(mutex); if (blocks.empty()) return false; block_map::const_iterator i = blocks.begin(); - if (is_blockchain_placeholder(*i)) - ++i; if (i == blocks.end()) return false; if (i->connection_id != connection_id) return false; filled = !i->blocks.empty(); + time = i->time; + return true; +} + +bool block_queue::has_next_span(uint64_t height, bool &filled, boost::posix_time::ptime &time, boost::uuids::uuid &connection_id) const +{ + boost::unique_lock<boost::recursive_mutex> lock(mutex); + if (blocks.empty()) + return false; + block_map::const_iterator i = blocks.begin(); + if (i == blocks.end()) + return false; + if (i->start_block_height > height) + return false; + filled = !i->blocks.empty(); + time = i->time; + connection_id = i->connection_id; return true; } @@ -330,8 +409,6 @@ size_t block_queue::get_num_filled_spans_prefix() const if (blocks.empty()) return 0; block_map::const_iterator i = blocks.begin(); - if (is_blockchain_placeholder(*i)) - ++i; size_t size = 0; while (i != blocks.end() && !i->blocks.empty()) { @@ -416,12 +493,35 @@ float block_queue::get_speed(const boost::uuids::uuid &connection_id) const return speed; } -bool block_queue::foreach(std::function<bool(const span&)> f, bool include_blockchain_placeholder) const +float block_queue::get_download_rate(const boost::uuids::uuid &connection_id) const +{ + boost::unique_lock<boost::recursive_mutex> lock(mutex); + float conn_rate = -1.f; + for (const auto &span: blocks) + { + if (span.blocks.empty()) + continue; + if (span.connection_id != connection_id) + continue; + // note that the average below does not average over the whole set, but over the + // previous pseudo average and the latest rate: this gives much more importance + // to the latest measurements, which is fine here + if (conn_rate < 0.f) + conn_rate = span.rate; + else + conn_rate = (conn_rate + span.rate) / 2; + } + + if (conn_rate < 0) + conn_rate = 0.0f; + MTRACE("Download rate for " << connection_id << ": " << conn_rate << " b/s"); + return conn_rate; +} + +bool block_queue::foreach(std::function<bool(const span&)> f) const { boost::unique_lock<boost::recursive_mutex> lock(mutex); block_map::const_iterator i = blocks.begin(); - if (!include_blockchain_placeholder && i != blocks.end() && is_blockchain_placeholder(*i)) - ++i; while (i != blocks.end()) if (!f(*i++)) return false; diff --git a/src/cryptonote_protocol/block_queue.h b/src/cryptonote_protocol/block_queue.h index 9cce95075..c7d3af7ac 100644 --- a/src/cryptonote_protocol/block_queue.h +++ b/src/cryptonote_protocol/block_queue.h @@ -76,22 +76,26 @@ namespace cryptonote void remove_spans(const boost::uuids::uuid &connection_id, uint64_t start_block_height); uint64_t get_max_block_height() const; void print() const; - std::string get_overview() const; - std::pair<uint64_t, uint64_t> reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, const std::vector<crypto::hash> &block_hashes, boost::posix_time::ptime time = boost::posix_time::microsec_clock::universal_time()); - bool is_blockchain_placeholder(const span &span) const; - std::pair<uint64_t, uint64_t> get_start_gap_span() const; + std::string get_overview(uint64_t blockchain_height) const; + bool has_unpruned_height(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed) const; + std::pair<uint64_t, uint64_t> reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, uint32_t pruning_seed, uint64_t blockchain_height, const std::vector<crypto::hash> &block_hashes, boost::posix_time::ptime time = boost::posix_time::microsec_clock::universal_time()); + uint64_t get_next_needed_height(uint64_t blockchain_height) const; std::pair<uint64_t, uint64_t> get_next_span_if_scheduled(std::vector<crypto::hash> &hashes, boost::uuids::uuid &connection_id, boost::posix_time::ptime &time) const; + void reset_next_span_time(boost::posix_time::ptime t = boost::posix_time::microsec_clock::universal_time()); void set_span_hashes(uint64_t start_height, const boost::uuids::uuid &connection_id, std::vector<crypto::hash> hashes); bool get_next_span(uint64_t &height, std::vector<cryptonote::block_complete_entry> &bcel, boost::uuids::uuid &connection_id, bool filled = true) const; - bool has_next_span(const boost::uuids::uuid &connection_id, bool &filled) const; + bool has_next_span(const boost::uuids::uuid &connection_id, bool &filled, boost::posix_time::ptime &time) const; + bool has_next_span(uint64_t height, bool &filled, boost::posix_time::ptime &time, boost::uuids::uuid &connection_id) const; size_t get_data_size() const; size_t get_num_filled_spans_prefix() const; size_t get_num_filled_spans() const; crypto::hash get_last_known_hash(const boost::uuids::uuid &connection_id) const; bool has_spans(const boost::uuids::uuid &connection_id) const; float get_speed(const boost::uuids::uuid &connection_id) const; - bool foreach(std::function<bool(const span&)> f, bool include_blockchain_placeholder = false) const; + float get_download_rate(const boost::uuids::uuid &connection_id) const; + bool foreach(std::function<bool(const span&)> f) const; bool requested(const crypto::hash &hash) const; + bool have(const crypto::hash &hash) const; private: void erase_block(block_map::iterator j); @@ -101,5 +105,6 @@ namespace cryptonote block_map blocks; mutable boost::recursive_mutex mutex; std::unordered_set<crypto::hash> requested_hashes; + std::unordered_set<crypto::hash> have_blocks; }; } diff --git a/src/cryptonote_protocol/cryptonote_protocol_defs.h b/src/cryptonote_protocol/cryptonote_protocol_defs.h index db159f0f4..c49371d48 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_defs.h +++ b/src/cryptonote_protocol/cryptonote_protocol_defs.h @@ -78,6 +78,8 @@ namespace cryptonote uint64_t height; + uint32_t pruning_seed; + BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(incoming) KV_SERIALIZE(localhost) @@ -100,6 +102,7 @@ namespace cryptonote KV_SERIALIZE(support_flags) KV_SERIALIZE(connection_id) KV_SERIALIZE(height) + KV_SERIALIZE(pruning_seed) END_KV_SERIALIZE_MAP() }; @@ -146,9 +149,11 @@ namespace cryptonote struct request { std::vector<blobdata> txs; + std::string _; // padding BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(txs) + KV_SERIALIZE(_) END_KV_SERIALIZE_MAP() }; }; @@ -198,12 +203,14 @@ namespace cryptonote uint64_t cumulative_difficulty; crypto::hash top_id; uint8_t top_version; + uint32_t pruning_seed; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(current_height) KV_SERIALIZE(cumulative_difficulty) KV_SERIALIZE_VAL_POD_AS_BLOB(top_id) KV_SERIALIZE_OPT(top_version, (uint8_t)0) + KV_SERIALIZE_OPT(pruning_seed, (uint32_t)0) END_KV_SERIALIZE_MAP() }; diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp b/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp index c9fd40d88..6d9ad9028 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp +++ b/src/cryptonote_protocol/cryptonote_protocol_handler-base.cpp @@ -30,20 +30,8 @@ // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#include <boost/asio.hpp> #include <string> #include <vector> -#include <boost/noncopyable.hpp> -#include <boost/shared_ptr.hpp> -#include <atomic> - -#include <boost/asio.hpp> -#include <boost/array.hpp> -#include <boost/noncopyable.hpp> -#include <boost/shared_ptr.hpp> -#include <boost/enable_shared_from_this.hpp> -#include <boost/interprocess/detail/atomic.hpp> -#include <boost/thread/thread.hpp> #include <memory> @@ -51,24 +39,14 @@ #include "net/net_utils_base.h" #include "misc_log_ex.h" -#include <boost/lambda/bind.hpp> -#include <boost/lambda/lambda.hpp> -#include <boost/uuid/random_generator.hpp> #include <boost/chrono.hpp> -#include <boost/utility/value_init.hpp> -#include <boost/asio/deadline_timer.hpp> #include <boost/date_time/posix_time/posix_time.hpp> #include <boost/thread/thread.hpp> #include "misc_language.h" #include "pragma_comp_defs.h" -#include <sstream> -#include <iomanip> #include <algorithm> -#include <boost/asio/basic_socket.hpp> -#include <boost/asio/ip/unicast.hpp> - #include "cryptonote_protocol_handler.h" #include "net/network_throttle.hpp" diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.h b/src/cryptonote_protocol/cryptonote_protocol_handler.h index d7b74c06d..a1bd9171c 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.h @@ -43,6 +43,7 @@ #include "cryptonote_protocol_defs.h" #include "cryptonote_protocol_handler_common.h" #include "block_queue.h" +#include "common/perf_timer.h" #include "cryptonote_basic/connection_context.h" #include "cryptonote_basic/cryptonote_stat_info.h" #include <boost/circular_buffer.hpp> @@ -109,6 +110,10 @@ namespace cryptonote const block_queue &get_block_queue() const { return m_block_queue; } void stop(); void on_connection_close(cryptonote_connection_context &context); + void set_max_out_peers(unsigned int max) { m_max_out_peers = max; } + std::string get_peers_overview() const; + std::pair<uint32_t, uint32_t> get_next_needed_pruning_stripe() const; + bool needs_new_sync_connections() const; private: //----------------- commands handlers ---------------------------------------------- int handle_notify_new_block(int command, NOTIFY_NEW_BLOCK::request& arg, cryptonote_connection_context& context); @@ -125,13 +130,17 @@ namespace cryptonote virtual bool relay_transactions(NOTIFY_NEW_TRANSACTIONS::request& arg, cryptonote_connection_context& exclude_context); //---------------------------------------------------------------------------------- //bool get_payload_sync_data(HANDSHAKE_DATA::request& hshd, cryptonote_connection_context& context); + bool should_drop_connection(cryptonote_connection_context& context, uint32_t next_stripe); bool request_missing_objects(cryptonote_connection_context& context, bool check_having_blocks, bool force_next_span = false); size_t get_synchronizing_connections_count(); bool on_connection_synchronized(); - bool should_download_next_span(cryptonote_connection_context& context) const; + bool should_download_next_span(cryptonote_connection_context& context, bool standby); void drop_connection(cryptonote_connection_context &context, bool add_fail, bool flush_all_spans); bool kick_idle_peers(); + bool check_standby_peers(); int try_add_next_blocks(cryptonote_connection_context &context); + void notify_new_stripe(cryptonote_connection_context &context, uint32_t stripe); + void skip_unneeded_hashes(cryptonote_connection_context& context, bool check_block_queue) const; t_core& m_core; @@ -143,6 +152,13 @@ namespace cryptonote boost::mutex m_sync_lock; block_queue m_block_queue; epee::math_helper::once_a_time_seconds<30> m_idle_peer_kicker; + epee::math_helper::once_a_time_milliseconds<100> m_standby_checker; + std::atomic<unsigned int> m_max_out_peers; + tools::PerformanceTimer m_sync_timer, m_add_timer; + uint64_t m_last_add_end_time; + uint64_t m_sync_spans_downloaded, m_sync_old_spans_downloaded, m_sync_bad_spans_downloaded; + uint64_t m_sync_download_chain_size, m_sync_download_objects_size; + size_t m_block_download_max_size; boost::mutex m_buffer_mutex; double get_avg_block_size(); @@ -155,7 +171,7 @@ namespace cryptonote std::string blob; epee::serialization::store_t_to_binary(arg, blob); //handler_response_blocks_now(blob.size()); // XXX - return m_p2p->invoke_notify_to_peer(t_parameter::ID, blob, context); + return m_p2p->invoke_notify_to_peer(t_parameter::ID, epee::strspan<uint8_t>(blob), context); } template<class t_parameter> @@ -164,7 +180,7 @@ namespace cryptonote LOG_PRINT_L2("[" << epee::net_utils::print_connection_context_short(exclude_context) << "] post relay " << typeid(t_parameter).name() << " -->"); std::string arg_buff; epee::serialization::store_t_to_binary(arg, arg_buff); - return m_p2p->relay_notify_to_all(t_parameter::ID, arg_buff, exclude_context); + return m_p2p->relay_notify_to_all(t_parameter::ID, epee::strspan<uint8_t>(arg_buff), exclude_context); } }; diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index c2c660e8c..61a211094 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -42,17 +42,24 @@ #include "cryptonote_basic/cryptonote_format_utils.h" #include "profile_tools.h" #include "net/network_throttle-detail.hpp" +#include "common/pruning.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net.cn" #define MLOG_P2P_MESSAGE(x) MCINFO("net.p2p.msg", context << x) +#define MLOG_PEER_STATE(x) \ + MCINFO(MONERO_DEFAULT_LOG_CATEGORY, context << "[" << epee::string_tools::to_string_hex(context.m_pruning_seed) << "] state: " << x << " in state " << cryptonote::get_protocol_state_string(context.m_state)) -#define BLOCK_QUEUE_NBLOCKS_THRESHOLD 10 // chunks of N blocks +#define BLOCK_QUEUE_NSPANS_THRESHOLD 10 // chunks of N blocks #define BLOCK_QUEUE_SIZE_THRESHOLD (100*1024*1024) // MB -#define REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD (5 * 1000000) // microseconds +#define BLOCK_QUEUE_FORCE_DOWNLOAD_NEAR_BLOCKS 1000 +#define REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD_STANDBY (5 * 1000000) // microseconds +#define REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD (30 * 1000000) // microseconds #define IDLE_PEER_KICK_TIME (600 * 1000000) // microseconds #define PASSIVE_PEER_KICK_TIME (60 * 1000000) // microseconds +#define DROP_ON_SYNC_WEDGE_THRESHOLD (30 * 1000000000ull) // nanoseconds +#define LAST_ACTIVITY_STALL_THRESHOLD (2.0f) // seconds namespace cryptonote { @@ -75,6 +82,19 @@ namespace cryptonote template<class t_core> bool t_cryptonote_protocol_handler<t_core>::init(const boost::program_options::variables_map& vm) { + m_sync_timer.pause(); + m_sync_timer.reset(); + m_add_timer.pause(); + m_add_timer.reset(); + m_last_add_end_time = 0; + m_sync_spans_downloaded = 0; + m_sync_old_spans_downloaded = 0; + m_sync_bad_spans_downloaded = 0; + m_sync_download_chain_size = 0; + m_sync_download_objects_size = 0; + + m_block_download_max_size = command_line::get_arg(vm, cryptonote::arg_block_download_max_size); + return true; } //------------------------------------------------------------------------------------------------------------------------ @@ -103,9 +123,12 @@ namespace cryptonote if(context.m_state == cryptonote_connection_context::state_synchronizing) { NOTIFY_REQUEST_CHAIN::request r = boost::value_initialized<NOTIFY_REQUEST_CHAIN::request>(); + context.m_needed_objects.clear(); m_core.get_short_chain_history(r.block_ids); - LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); + handler_request_blocks_history( r.block_ids ); // change the limit(?), sleep(?) + MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); post_notify<NOTIFY_REQUEST_CHAIN>(r, context); + MLOG_PEER_STATE("requesting chain"); } else if(context.m_state == cryptonote_connection_context::state_standby) { @@ -247,6 +270,7 @@ namespace cryptonote cnx.connection_id = epee::string_tools::pod_to_hex(cntxt.m_connection_id); cnx.height = cntxt.m_remote_blockchain_height; + cnx.pruning_seed = cntxt.m_pruning_seed; connections.push_back(cnx); @@ -279,7 +303,23 @@ namespace cryptonote } } + // reject weird pruning schemes + if (hshd.pruning_seed) + { + const uint32_t log_stripes = tools::get_pruning_log_stripes(hshd.pruning_seed); + if (log_stripes != CRYPTONOTE_PRUNING_LOG_STRIPES || tools::get_pruning_stripe(hshd.pruning_seed) > (1u << log_stripes)) + { + MWARNING(context << " peer claim unexpected pruning seed " << epee::string_tools::to_string_hex(hshd.pruning_seed) << ", disconnecting"); + return false; + } + } + context.m_remote_blockchain_height = hshd.current_height; + context.m_pruning_seed = hshd.pruning_seed; +#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED + context.m_pruning_seed = tools::make_pruning_seed(1 + (context.m_remote_address.as<epee::net_utils::ipv4_network_address>().ip()) % (1 << CRYPTONOTE_PRUNING_LOG_STRIPES), CRYPTONOTE_PRUNING_LOG_STRIPES); + LOG_INFO_CC(context, "New connection posing as pruning seed " << epee::string_tools::to_string_hex(context.m_pruning_seed) << ", seed address " << &context.m_pruning_seed); +#endif uint64_t target = m_core.get_target_blockchain_height(); if (target == 0) @@ -298,7 +338,6 @@ namespace cryptonote /* As I don't know if accessing hshd from core could be a good practice, I prefer pushing target height to the core at the same time it is pushed to the user. Nz. */ - m_core.set_target_blockchain_height((hshd.current_height)); int64_t diff = static_cast<int64_t>(hshd.current_height) - static_cast<int64_t>(m_core.get_current_blockchain_height()); uint64_t abs_diff = std::abs(diff); uint64_t max_block_height = std::max(hshd.current_height,m_core.get_current_blockchain_height()); @@ -309,14 +348,31 @@ namespace cryptonote << (0 <= diff ? std::string("behind") : std::string("ahead")) << "] " << ENDL << "SYNCHRONIZATION started"); if (hshd.current_height >= m_core.get_current_blockchain_height() + 5) // don't switch to unsafe mode just for a few blocks + { m_core.safesyncmode(false); + } + if (m_core.get_target_blockchain_height() == 0) // only when sync starts + { + m_sync_timer.resume(); + m_sync_timer.reset(); + m_add_timer.pause(); + m_add_timer.reset(); + m_last_add_end_time = 0; + m_sync_spans_downloaded = 0; + m_sync_old_spans_downloaded = 0; + m_sync_bad_spans_downloaded = 0; + m_sync_download_chain_size = 0; + m_sync_download_objects_size = 0; + } + m_core.set_target_blockchain_height((hshd.current_height)); } - LOG_PRINT_L1("Remote blockchain height: " << hshd.current_height << ", id: " << hshd.top_id); + MINFO(context << "Remote blockchain height: " << hshd.current_height << ", id: " << hshd.top_id); context.m_state = cryptonote_connection_context::state_synchronizing; //let the socket to send response to handshake, but request callback, to let send request data after response LOG_PRINT_CCONTEXT_L2("requesting callback"); ++context.m_callback_request_count; m_p2p->request_callback(context); + MLOG_PEER_STATE("requesting callback"); return true; } //------------------------------------------------------------------------------------------------------------------------ @@ -327,6 +383,7 @@ namespace cryptonote hshd.top_version = m_core.get_ideal_hard_fork_version(hshd.current_height); hshd.cumulative_difficulty = m_core.get_block_cumulative_difficulty(hshd.current_height); hshd.current_height +=1; + hshd.pruning_seed = m_core.get_blockchain_pruning_seed(); return true; } //------------------------------------------------------------------------------------------------------------------------ @@ -389,11 +446,14 @@ namespace cryptonote relay_block(arg, context); }else if(bvc.m_marked_as_orphaned) { + context.m_needed_objects.clear(); context.m_state = cryptonote_connection_context::state_synchronizing; NOTIFY_REQUEST_CHAIN::request r = boost::value_initialized<NOTIFY_REQUEST_CHAIN::request>(); m_core.get_short_chain_history(r.block_ids); - LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); + handler_request_blocks_history( r.block_ids ); // change the limit(?), sleep(?) + MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); post_notify<NOTIFY_REQUEST_CHAIN>(r, context); + MLOG_PEER_STATE("requesting chain"); } return 1; @@ -616,6 +676,7 @@ namespace cryptonote missing_tx_req.missing_tx_indices = std::move(need_tx_indices); m_core.resume_mine(); + MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_FLUFFY_MISSING_TX: missing_tx_indices.size()=" << missing_tx_req.missing_tx_indices.size() ); post_notify<NOTIFY_REQUEST_FLUFFY_MISSING_TX>(missing_tx_req, context); } else // whoo-hoo we've got em all .. @@ -656,11 +717,14 @@ namespace cryptonote } else if( bvc.m_marked_as_orphaned ) { + context.m_needed_objects.clear(); context.m_state = cryptonote_connection_context::state_synchronizing; NOTIFY_REQUEST_CHAIN::request r = boost::value_initialized<NOTIFY_REQUEST_CHAIN::request>(); m_core.get_short_chain_history(r.block_ids); - LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); + handler_request_blocks_history( r.block_ids ); // change the limit(?), sleep(?) + MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); post_notify<NOTIFY_REQUEST_CHAIN>(r, context); + MLOG_PEER_STATE("requesting chain"); } } } @@ -747,7 +811,7 @@ namespace cryptonote fluffy_response.b.txs.push_back(t_serializable_object_to_blob(tx)); } - LOG_PRINT_CCONTEXT_L2 + MLOG_P2P_MESSAGE ( "-->>NOTIFY_RESPONSE_FLUFFY_MISSING_TX: " << ", txs.size()=" << fluffy_response.b.txs.size() @@ -811,7 +875,7 @@ namespace cryptonote drop_connection(context, false, false); return 1; } - LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_RESPONSE_GET_OBJECTS: blocks.size()=" << rsp.blocks.size() << ", txs.size()=" << rsp.txs.size() + MLOG_P2P_MESSAGE("-->>NOTIFY_RESPONSE_GET_OBJECTS: blocks.size()=" << rsp.blocks.size() << ", txs.size()=" << rsp.txs.size() << ", rsp.m_current_blockchain_height=" << rsp.current_blockchain_height << ", missed_ids.size()=" << rsp.missed_ids.size()); post_notify<NOTIFY_RESPONSE_GET_OBJECTS>(rsp, context); //handler_response_blocks_now(sizeof(rsp)); // XXX @@ -834,11 +898,14 @@ namespace cryptonote return avg / m_avg_buffer.size(); } - template<class t_core> int t_cryptonote_protocol_handler<t_core>::handle_response_get_objects(int command, NOTIFY_RESPONSE_GET_OBJECTS::request& arg, cryptonote_connection_context& context) { MLOG_P2P_MESSAGE("Received NOTIFY_RESPONSE_GET_OBJECTS (" << arg.blocks.size() << " blocks, " << arg.txs.size() << " txes)"); + MLOG_PEER_STATE("received objects"); + + boost::posix_time::ptime request_time = context.m_last_request_time; + context.m_last_request_time = boost::date_time::not_a_date_time; // calculate size of request size_t size = 0; @@ -860,6 +927,8 @@ namespace cryptonote CRITICAL_REGION_LOCAL(m_buffer_mutex); m_avg_buffer.push_back(size); } + ++m_sync_spans_downloaded; + m_sync_download_objects_size += size; MDEBUG(context << " downloaded " << size << " bytes worth of blocks"); /*using namespace boost::chrono; @@ -874,6 +943,7 @@ namespace cryptonote LOG_ERROR_CCONTEXT("sent wrong NOTIFY_HAVE_OBJECTS: arg.m_current_blockchain_height=" << arg.current_blockchain_height << " < m_last_response_height=" << context.m_last_response_height << ", dropping connection"); drop_connection(context, false, false); + ++m_sync_bad_spans_downloaded; return 1; } @@ -898,6 +968,7 @@ namespace cryptonote LOG_ERROR_CCONTEXT("sent wrong block: failed to parse and validate block: " << epee::string_tools::buff_to_hex_nodelimer(block_entry.block) << ", dropping connection"); drop_connection(context, false, false); + ++m_sync_bad_spans_downloaded; return 1; } if (b.miner_tx.vin.size() != 1 || b.miner_tx.vin.front().type() != typeid(txin_gen)) @@ -905,6 +976,7 @@ namespace cryptonote LOG_ERROR_CCONTEXT("sent wrong block: block: miner tx does not have exactly one txin_gen input" << epee::string_tools::buff_to_hex_nodelimer(block_entry.block) << ", dropping connection"); drop_connection(context, false, false); + ++m_sync_bad_spans_downloaded; return 1; } if (start_height == std::numeric_limits<uint64_t>::max()) @@ -917,6 +989,7 @@ namespace cryptonote LOG_ERROR_CCONTEXT("sent wrong NOTIFY_RESPONSE_GET_OBJECTS: block with id=" << epee::string_tools::pod_to_hex(get_blob_hash(block_entry.block)) << " wasn't requested, dropping connection"); drop_connection(context, false, false); + ++m_sync_bad_spans_downloaded; return 1; } if(b.tx_hashes.size() != block_entry.txs.size()) @@ -924,6 +997,7 @@ namespace cryptonote LOG_ERROR_CCONTEXT("sent wrong NOTIFY_RESPONSE_GET_OBJECTS: block with id=" << epee::string_tools::pod_to_hex(get_blob_hash(block_entry.block)) << ", tx_hashes.size()=" << b.tx_hashes.size() << " mismatch with block_complete_entry.m_txs.size()=" << block_entry.txs.size() << ", dropping connection"); drop_connection(context, false, false); + ++m_sync_bad_spans_downloaded; return 1; } @@ -931,34 +1005,27 @@ namespace cryptonote block_hashes.push_back(block_hash); } - if(context.m_requested_objects.size()) + if(!context.m_requested_objects.empty()) { - MERROR("returned not all requested objects (context.m_requested_objects.size()=" + MERROR(context << "returned not all requested objects (context.m_requested_objects.size()=" << context.m_requested_objects.size() << "), dropping connection"); drop_connection(context, false, false); + ++m_sync_bad_spans_downloaded; return 1; } - // get the last parsed block, which should be the highest one - const crypto::hash last_block_hash = cryptonote::get_block_hash(b); - if(m_core.have_block(last_block_hash)) - { - const uint64_t subchain_height = start_height + arg.blocks.size(); - LOG_DEBUG_CC(context, "These are old blocks, ignoring: blocks " << start_height << " - " << (subchain_height-1) << ", blockchain height " << m_core.get_current_blockchain_height()); - m_block_queue.remove_spans(context.m_connection_id, start_height); - goto skip; - } - { MLOG_YELLOW(el::Level::Debug, context << " Got NEW BLOCKS inside of " << __FUNCTION__ << ": size: " << arg.blocks.size() - << ", blocks: " << start_height << " - " << (start_height + arg.blocks.size() - 1)); + << ", blocks: " << start_height << " - " << (start_height + arg.blocks.size() - 1) << + " (pruning seed " << epee::string_tools::to_string_hex(context.m_pruning_seed) << ")"); // add that new span to the block queue - const boost::posix_time::time_duration dt = now - context.m_last_request_time; + const boost::posix_time::time_duration dt = now - request_time; const float rate = size * 1e6 / (dt.total_microseconds() + 1); - MDEBUG(context << " adding span: " << arg.blocks.size() << " at height " << start_height << ", " << dt.total_microseconds()/1e6 << " seconds, " << (rate/1e3) << " kB/s, size now " << (m_block_queue.get_data_size() + blocks_size) / 1048576.f << " MB"); + MDEBUG(context << " adding span: " << arg.blocks.size() << " at height " << start_height << ", " << dt.total_microseconds()/1e6 << " seconds, " << (rate/1024) << " kB/s, size now " << (m_block_queue.get_data_size() + blocks_size) / 1048576.f << " MB"); m_block_queue.add_blocks(start_height, arg.blocks, context.m_connection_id, rate, blocks_size); + const crypto::hash last_block_hash = cryptonote::get_block_hash(b); context.m_last_known_hash = last_block_hash; if (!m_core.get_test_drop_download() || !m_core.get_test_drop_download_height()) { // DISCARD BLOCKS for testing @@ -966,7 +1033,6 @@ namespace cryptonote } } -skip: try_add_next_blocks(context); return 1; } @@ -983,15 +1049,22 @@ skip: const boost::unique_lock<boost::mutex> sync{m_sync_lock, boost::try_to_lock}; if (!sync.owns_lock()) { - MINFO("Failed to lock m_sync_lock, going back to download"); + MINFO(context << "Failed to lock m_sync_lock, going back to download"); goto skip; } MDEBUG(context << " lock m_sync_lock, adding blocks to chain..."); + MLOG_PEER_STATE("adding blocks"); { m_core.pause_mine(); - epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler( - boost::bind(&t_core::resume_mine, &m_core)); + m_add_timer.resume(); + bool starting = true; + epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([this, &starting]() { + m_add_timer.pause(); + m_core.resume_mine(); + if (!starting) + m_last_add_end_time = tools::get_tick_count(); + }); while (1) { @@ -1004,22 +1077,38 @@ skip: MDEBUG(context << " no next span found, going back to download"); break; } - MDEBUG(context << " next span in the queue has blocks " << start_height << "-" << (start_height + blocks.size() - 1) - << ", we need " << previous_height); if (blocks.empty()) { - MERROR("Next span has no blocks"); + MERROR(context << "Next span has no blocks"); m_block_queue.remove_spans(span_connection_id, start_height); - break; + continue; } + MDEBUG(context << " next span in the queue has blocks " << start_height << "-" << (start_height + blocks.size() - 1) + << ", we need " << previous_height); + block new_block; + if (!parse_and_validate_block_from_blob(blocks.back().block, new_block)) + { + MERROR(context << "Failed to parse block, but it should already have been parsed"); + m_block_queue.remove_spans(span_connection_id, start_height); + continue; + } + const crypto::hash last_block_hash = cryptonote::get_block_hash(new_block); + if (m_core.have_block(last_block_hash)) + { + const uint64_t subchain_height = start_height + blocks.size(); + LOG_DEBUG_CC(context, "These are old blocks, ignoring: blocks " << start_height << " - " << (subchain_height-1) << ", blockchain height " << m_core.get_current_blockchain_height()); + m_block_queue.remove_spans(span_connection_id, start_height); + ++m_sync_old_spans_downloaded; + continue; + } if (!parse_and_validate_block_from_blob(blocks.front().block, new_block)) { - MERROR("Failed to parse block, but it should already have been parsed"); + MERROR(context << "Failed to parse block, but it should already have been parsed"); m_block_queue.remove_spans(span_connection_id, start_height); - break; + continue; } bool parent_known = m_core.have_block(new_block.prev_id); if (!parent_known) @@ -1032,6 +1121,24 @@ skip: bool parent_requested = m_block_queue.requested(new_block.prev_id); if (!parent_requested) { + // we might be able to ask for that block directly, as we now can request out of order, + // otherwise we continue out of order, unless this block is the one we need, in which + // case we request block hashes, though it might be safer to disconnect ? + if (start_height > previous_height) + { + if (should_drop_connection(context, get_next_needed_pruning_stripe().first)) + { + MDEBUG(context << "Got block with unknown parent which was not requested, but peer does not have that block - dropping connection"); + if (!context.m_is_income) + m_p2p->add_used_stripe_peer(context); + drop_connection(context, false, true); + return 1; + } + MDEBUG(context << "Got block with unknown parent which was not requested, but peer does not have that block - back to download"); + + goto skip; + } + // this can happen if a connection was sicced onto a late span, if it did not have those blocks, // since we don't know that at the sic time LOG_ERROR_CCONTEXT("Got block with unknown parent which was not requested - querying block hashes"); @@ -1047,7 +1154,17 @@ skip: } const boost::posix_time::ptime start = boost::posix_time::microsec_clock::universal_time(); - context.m_last_request_time = start; + + if (starting) + { + starting = false; + if (m_last_add_end_time) + { + const uint64_t tnow = tools::get_tick_count(); + const uint64_t ns = tools::ticks_to_ns(tnow - m_last_add_end_time); + MINFO("Restarting adding block after idle for " << ns/1e9 << " seconds"); + } + } m_core.prepare_handle_incoming_blocks(blocks); @@ -1077,8 +1194,10 @@ skip: if(tvc[i].m_verifivation_failed) { if (!m_p2p->for_connection(span_connection_id, [&](cryptonote_connection_context& context, nodetool::peerid_type peer_id, uint32_t f)->bool{ + cryptonote::transaction tx; + parse_and_validate_tx_from_blob(*it, tx); // must succeed if we got here LOG_ERROR_CCONTEXT("transaction verification failed on NOTIFY_RESPONSE_GET_OBJECTS, tx_id = " - << epee::string_tools::pod_to_hex(get_blob_hash(*it)) << ", dropping connection"); + << epee::string_tools::pod_to_hex(cryptonote::get_transaction_hash(tx)) << ", dropping connection"); drop_connection(context, false, true); return 1; })) @@ -1148,7 +1267,7 @@ skip: } // each download block - MCINFO("sync-info", "Block process time (" << blocks.size() << " blocks, " << num_txs << " txs): " << block_process_time_full + transactions_process_time_full << " (" << transactions_process_time_full << "/" << block_process_time_full << ") ms"); + MDEBUG(context << "Block process time (" << blocks.size() << " blocks, " << num_txs << " txs): " << block_process_time_full + transactions_process_time_full << " (" << transactions_process_time_full << "/" << block_process_time_full << ") ms"); if (!m_core.cleanup_handle_incoming_blocks()) { @@ -1158,40 +1277,54 @@ skip: m_block_queue.remove_spans(span_connection_id, start_height); - if (m_core.get_current_blockchain_height() > previous_height) + const uint64_t current_blockchain_height = m_core.get_current_blockchain_height(); + if (current_blockchain_height > previous_height) { + const uint64_t target_blockchain_height = m_core.get_target_blockchain_height(); const boost::posix_time::time_duration dt = boost::posix_time::microsec_clock::universal_time() - start; + std::string progress_message = ""; + if (current_blockchain_height < target_blockchain_height) + { + uint64_t completion_percent = (current_blockchain_height * 100 / target_blockchain_height); + if (completion_percent == 100) // never show 100% if not actually up to date + completion_percent = 99; + progress_message = " (" + std::to_string(completion_percent) + "%, " + + std::to_string(target_blockchain_height - current_blockchain_height) + " left)"; + } + const uint32_t previous_stripe = tools::get_pruning_stripe(previous_height, target_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES); + const uint32_t current_stripe = tools::get_pruning_stripe(current_blockchain_height, target_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES); std::string timing_message = ""; if (ELPP->vRegistry()->allowed(el::Level::Info, "sync-info")) timing_message = std::string(" (") + std::to_string(dt.total_microseconds()/1e6) + " sec, " - + std::to_string((m_core.get_current_blockchain_height() - previous_height) * 1e6 / dt.total_microseconds()) - + " blocks/sec), " + std::to_string(m_block_queue.get_data_size() / 1048576.f) + " MB queued"; + + std::to_string((current_blockchain_height - previous_height) * 1e6 / dt.total_microseconds()) + + " blocks/sec), " + std::to_string(m_block_queue.get_data_size() / 1048576.f) + " MB queued in " + + std::to_string(m_block_queue.get_num_filled_spans()) + " spans, stripe " + + std::to_string(previous_stripe) + " -> " + std::to_string(current_stripe); if (ELPP->vRegistry()->allowed(el::Level::Debug, "sync-info")) - timing_message += std::string(": ") + m_block_queue.get_overview(); - if(m_core.get_target_blockchain_height() == 0){ - MGINFO_YELLOW(context << " Synced " << m_core.get_current_blockchain_height() << "/" << m_core.get_target_blockchain_height() - << timing_message); - } else { - const int completition_percent = (m_core.get_current_blockchain_height() * 100 / m_core.get_target_blockchain_height()); - if(completition_percent < 99) {//printing completion percent only if % is < of 99 cause for 99 >= this is useless - MGINFO_YELLOW(context << " Synced " << m_core.get_current_blockchain_height() << "/" << m_core.get_target_blockchain_height() - << " (" << completition_percent << "% " << (m_core.get_target_blockchain_height() - m_core.get_current_blockchain_height()) - << " blocks remaining)" << timing_message); - } else { - MGINFO_YELLOW(context << " Synced " << m_core.get_current_blockchain_height() << "/" << m_core.get_target_blockchain_height() - << timing_message); - } - } + timing_message += std::string(": ") + m_block_queue.get_overview(current_blockchain_height); + MGINFO_YELLOW("Synced " << current_blockchain_height << "/" << target_blockchain_height + << progress_message << timing_message); + if (previous_stripe != current_stripe) + notify_new_stripe(context, current_stripe); } } } - if (should_download_next_span(context)) + MLOG_PEER_STATE("stopping adding blocks"); + + if (should_download_next_span(context, false)) { - context.m_needed_objects.clear(); - context.m_last_response_height = 0; force_next_span = true; } + else if (should_drop_connection(context, get_next_needed_pruning_stripe().first)) + { + if (!context.m_is_income) + { + m_p2p->add_used_stripe_peer(context); + drop_connection(context, false, false); + } + return 1; + } } skip: @@ -1205,9 +1338,32 @@ skip: } //------------------------------------------------------------------------------------------------------------------------ template<class t_core> + void t_cryptonote_protocol_handler<t_core>::notify_new_stripe(cryptonote_connection_context& cntxt, uint32_t stripe) + { + m_p2p->for_each_connection([&](cryptonote_connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags)->bool + { + if (cntxt.m_connection_id == context.m_connection_id) + return true; + if (context.m_state == cryptonote_connection_context::state_normal) + { + const uint32_t peer_stripe = tools::get_pruning_stripe(context.m_pruning_seed); + if (stripe && peer_stripe && peer_stripe != stripe) + return true; + context.m_state = cryptonote_connection_context::state_synchronizing; + LOG_PRINT_CCONTEXT_L2("requesting callback"); + ++context.m_callback_request_count; + m_p2p->request_callback(context); + MLOG_PEER_STATE("requesting callback"); + } + return true; + }); + } + //------------------------------------------------------------------------------------------------------------------------ + template<class t_core> bool t_cryptonote_protocol_handler<t_core>::on_idle() { m_idle_peer_kicker.do_call(boost::bind(&t_cryptonote_protocol_handler<t_core>::kick_idle_peers, this)); + m_standby_checker.do_call(boost::bind(&t_cryptonote_protocol_handler<t_core>::check_standby_peers, this)); return m_core.on_idle(); } //------------------------------------------------------------------------------------------------------------------------ @@ -1215,30 +1371,40 @@ skip: bool t_cryptonote_protocol_handler<t_core>::kick_idle_peers() { MTRACE("Checking for idle peers..."); - std::vector<boost::uuids::uuid> kick_connections; m_p2p->for_each_connection([&](cryptonote_connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags)->bool { - if (context.m_state == cryptonote_connection_context::state_synchronizing || context.m_state == cryptonote_connection_context::state_before_handshake) + if (context.m_state == cryptonote_connection_context::state_synchronizing && context.m_last_request_time != boost::date_time::not_a_date_time) { - const bool passive = context.m_state == cryptonote_connection_context::state_before_handshake; const boost::posix_time::ptime now = boost::posix_time::microsec_clock::universal_time(); const boost::posix_time::time_duration dt = now - context.m_last_request_time; - const int64_t threshold = passive ? PASSIVE_PEER_KICK_TIME : IDLE_PEER_KICK_TIME; - if (dt.total_microseconds() > threshold) + if (dt.total_microseconds() > IDLE_PEER_KICK_TIME) { - MINFO(context << " kicking " << (passive ? "passive" : "idle") << " peer"); - kick_connections.push_back(context.m_connection_id); + MINFO(context << " kicking idle peer, last update " << (dt.total_microseconds() / 1.e6) << " seconds ago"); + LOG_PRINT_CCONTEXT_L2("requesting callback"); + context.m_last_request_time = boost::date_time::not_a_date_time; + context.m_state = cryptonote_connection_context::state_standby; // we'll go back to adding, then (if we can't), download + ++context.m_callback_request_count; + m_p2p->request_callback(context); } } return true; }); - for (const boost::uuids::uuid &conn_id: kick_connections) + return true; + } + //------------------------------------------------------------------------------------------------------------------------ + template<class t_core> + bool t_cryptonote_protocol_handler<t_core>::check_standby_peers() + { + m_p2p->for_each_connection([&](cryptonote_connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags)->bool { - m_p2p->for_connection(conn_id, [this](cryptonote_connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags) { - drop_connection(context, false, false); - return true; - }); - } + if (context.m_state == cryptonote_connection_context::state_standby) + { + LOG_PRINT_CCONTEXT_L2("requesting callback"); + ++context.m_callback_request_count; + m_p2p->request_callback(context); + } + return true; + }); return true; } //------------------------------------------------------------------------------------------------------------------------ @@ -1253,75 +1419,174 @@ skip: drop_connection(context, false, false); return 1; } - LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_RESPONSE_CHAIN_ENTRY: m_start_height=" << r.start_height << ", m_total_height=" << r.total_height << ", m_block_ids.size()=" << r.m_block_ids.size()); + MLOG_P2P_MESSAGE("-->>NOTIFY_RESPONSE_CHAIN_ENTRY: m_start_height=" << r.start_height << ", m_total_height=" << r.total_height << ", m_block_ids.size()=" << r.m_block_ids.size()); post_notify<NOTIFY_RESPONSE_CHAIN_ENTRY>(r, context); return 1; } //------------------------------------------------------------------------------------------------------------------------ template<class t_core> - bool t_cryptonote_protocol_handler<t_core>::should_download_next_span(cryptonote_connection_context& context) const + bool t_cryptonote_protocol_handler<t_core>::should_download_next_span(cryptonote_connection_context& context, bool standby) { std::vector<crypto::hash> hashes; boost::uuids::uuid span_connection_id; boost::posix_time::ptime request_time; + boost::uuids::uuid connection_id; std::pair<uint64_t, uint64_t> span; + bool filled; - span = m_block_queue.get_start_gap_span(); - if (span.second > 0) - { - MDEBUG(context << " we should download it as there is a gap"); - return true; - } - - // if the next span is not scheduled (or there is none) - span = m_block_queue.get_next_span_if_scheduled(hashes, span_connection_id, request_time); - if (span.second == 0) + const uint64_t blockchain_height = m_core.get_current_blockchain_height(); + if (context.m_remote_blockchain_height <= blockchain_height) + return false; + const boost::posix_time::ptime now = boost::posix_time::microsec_clock::universal_time(); + const bool has_next_block = tools::has_unpruned_block(blockchain_height, context.m_remote_blockchain_height, context.m_pruning_seed); + if (has_next_block) { - // we might be in a weird case where there is a filled next span, - // but it starts higher than the current height - uint64_t height; - std::vector<cryptonote::block_complete_entry> bcel; - if (!m_block_queue.get_next_span(height, bcel, span_connection_id, true)) - return false; - if (height > m_core.get_current_blockchain_height()) + if (!m_block_queue.has_next_span(blockchain_height, filled, request_time, connection_id)) { - MDEBUG(context << " we should download it as the next block isn't scheduled"); + MDEBUG(context << " we should download it as no peer reserved it"); return true; } + if (!filled) + { + const long dt = (now - request_time).total_microseconds(); + if (dt >= REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD) + { + MDEBUG(context << " we should download it as it's not been received yet after " << dt/1e6); + return true; + } + + // in standby, be ready to double download early since we're idling anyway + // let the fastest peer trigger first + long threshold; + const double dl_speed = context.m_max_speed_down; + if (standby && dt >= REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD_STANDBY && dl_speed > 0) + { + bool download = false; + if (m_p2p->for_connection(connection_id, [&](cryptonote_connection_context& ctx, nodetool::peerid_type peer_id, uint32_t f)->bool{ + const time_t nowt = time(NULL); + const time_t time_since_last_recv = nowt - ctx.m_last_recv; + const float last_activity = std::min((float)time_since_last_recv, dt/1e6f); + const bool stalled = last_activity > LAST_ACTIVITY_STALL_THRESHOLD; + if (stalled) + { + MDEBUG(context << " we should download it as the downloading peer is stalling for " << nowt - ctx.m_last_recv << " seconds"); + download = true; + return true; + } + + // estimate the standby peer can give us 80% of its max speed + // and let it download if that speed is > N times as fast as the current one + // N starts at 10 after REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD_STANDBY, + // decreases to 1.25 at REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD, + // so that at times goes without the download being done, a retry becomes easier + const float max_multiplier = 10.f; + const float min_multiplier = 1.25f; + float multiplier = max_multiplier; + if (dt/1e6 >= REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD_STANDBY) + { + multiplier = max_multiplier - (dt/1e6-REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD_STANDBY) * (max_multiplier - min_multiplier) / (REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD - REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD_STANDBY); + multiplier = std::min(max_multiplier, std::max(min_multiplier, multiplier)); + } + if (dl_speed * .8f > ctx.m_current_speed_down * multiplier) + { + MDEBUG(context << " we should download it as we are substantially faster (" << dl_speed << " vs " + << ctx.m_current_speed_down << ", multiplier " << multiplier << " after " << dt/1e6 << " seconds)"); + download = true; + return true; + } + return true; + })) + { + if (download) + return true; + } + else + { + MWARNING(context << " we should download it as the downloading peer is unexpectedly not known to us"); + return true; + } + } + } + } + + return false; + } + //------------------------------------------------------------------------------------------------------------------------ + template<class t_core> + bool t_cryptonote_protocol_handler<t_core>::should_drop_connection(cryptonote_connection_context& context, uint32_t next_stripe) + { + if (context.m_anchor) + { + MDEBUG(context << "This is an anchor peer, not dropping"); return false; } - // if it was scheduled by this particular peer - if (span_connection_id == context.m_connection_id) + if (context.m_pruning_seed == 0) + { + MDEBUG(context << "This peer is not striped, not dropping"); return false; + } - float span_speed = m_block_queue.get_speed(span_connection_id); - float speed = m_block_queue.get_speed(context.m_connection_id); - MDEBUG(context << " next span is scheduled for " << span_connection_id << ", speed " << span_speed << ", ours " << speed); - - // we try for that span too if: - // - we're substantially faster, or: - // - we're the fastest and the other one isn't (avoids a peer being waaaay slow but yet unmeasured) - // - the other one asked at least 5 seconds ago - if (span_speed < .25 && speed > .75f) + const uint32_t peer_stripe = tools::get_pruning_stripe(context.m_pruning_seed); + if (next_stripe == peer_stripe) { - MDEBUG(context << " we should download it as we're substantially faster"); - return true; + MDEBUG(context << "This peer has needed stripe " << peer_stripe << ", not dropping"); + return false; } - if (speed == 1.0f && span_speed != 1.0f) + + if (!context.m_needed_objects.empty()) { - MDEBUG(context << " we should download it as we're the fastest peer"); - return true; + const uint64_t next_available_block_height = context.m_last_response_height - context.m_needed_objects.size() + 1; + if (tools::has_unpruned_block(next_available_block_height, context.m_remote_blockchain_height, context.m_pruning_seed)) + { + MDEBUG(context << "This peer has unpruned next block at height " << next_available_block_height << ", not dropping"); + return false; + } } - const boost::posix_time::ptime now = boost::posix_time::microsec_clock::universal_time(); - if ((now - request_time).total_microseconds() > REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD) + + if (next_stripe > 0) { - MDEBUG(context << " we should download it as this span was requested long ago"); - return true; + unsigned int n_out_peers = 0, n_peers_on_next_stripe = 0; + m_p2p->for_each_connection([&](cryptonote_connection_context& ctx, nodetool::peerid_type peer_id, uint32_t support_flags)->bool{ + if (!ctx.m_is_income) + ++n_out_peers; + if (ctx.m_state >= cryptonote_connection_context::state_synchronizing && tools::get_pruning_stripe(ctx.m_pruning_seed) == next_stripe) + ++n_peers_on_next_stripe; + return true; + }); + const uint32_t distance = (peer_stripe + (1<<CRYPTONOTE_PRUNING_LOG_STRIPES) - next_stripe) % (1<<CRYPTONOTE_PRUNING_LOG_STRIPES); + if ((n_out_peers >= m_max_out_peers && n_peers_on_next_stripe == 0) || (distance > 1 && n_peers_on_next_stripe <= 2) || distance > 2) + { + MDEBUG(context << "we want seed " << next_stripe << ", and either " << n_out_peers << " is at max out peers (" + << m_max_out_peers << ") or distance " << distance << " from " << next_stripe << " to " << peer_stripe << + " is too large and we have only " << n_peers_on_next_stripe << " peers on next seed, dropping connection to make space"); + return true; + } } + MDEBUG(context << "End of checks, not dropping"); return false; } //------------------------------------------------------------------------------------------------------------------------ template<class t_core> + void t_cryptonote_protocol_handler<t_core>::skip_unneeded_hashes(cryptonote_connection_context& context, bool check_block_queue) const + { + // take out blocks we already have + size_t skip = 0; + while (skip < context.m_needed_objects.size() && (m_core.have_block(context.m_needed_objects[skip]) || (check_block_queue && m_block_queue.have(context.m_needed_objects[skip])))) + { + // if we're popping the last hash, record it so we can ask again from that hash, + // this prevents never being able to progress on peers we get old hash lists from + if (skip + 1 == context.m_needed_objects.size()) + context.m_last_known_hash = context.m_needed_objects[skip]; + ++skip; + } + if (skip > 0) + { + MDEBUG(context << "skipping " << skip << "/" << context.m_needed_objects.size() << " blocks"); + context.m_needed_objects = std::vector<crypto::hash>(context.m_needed_objects.begin() + skip, context.m_needed_objects.end()); + } + } + //------------------------------------------------------------------------------------------------------------------------ + template<class t_core> bool t_cryptonote_protocol_handler<t_core>::request_missing_objects(cryptonote_connection_context& context, bool check_having_blocks, bool force_next_span) { // flush stale spans @@ -1336,64 +1601,114 @@ skip: bool start_from_current_chain = false; if (!force_next_span) { - bool first = true; - while (1) + do { - size_t nblocks = m_block_queue.get_num_filled_spans(); + size_t nspans = m_block_queue.get_num_filled_spans(); size_t size = m_block_queue.get_data_size(); - if (nblocks < BLOCK_QUEUE_NBLOCKS_THRESHOLD || size < BLOCK_QUEUE_SIZE_THRESHOLD) + const uint64_t bc_height = m_core.get_current_blockchain_height(); + const auto next_needed_pruning_stripe = get_next_needed_pruning_stripe(); + const uint32_t add_stripe = tools::get_pruning_stripe(bc_height, context.m_remote_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES); + const uint32_t peer_stripe = tools::get_pruning_stripe(context.m_pruning_seed); + const size_t block_queue_size_threshold = m_block_download_max_size ? m_block_download_max_size : BLOCK_QUEUE_SIZE_THRESHOLD; + bool queue_proceed = nspans < BLOCK_QUEUE_NSPANS_THRESHOLD || size < block_queue_size_threshold; + // get rid of blocks we already requested, or already have + skip_unneeded_hashes(context, true); + uint64_t next_needed_height = m_block_queue.get_next_needed_height(bc_height); + uint64_t next_block_height; + if (context.m_needed_objects.empty()) + next_block_height = next_needed_height; + else + next_block_height = context.m_last_response_height - context.m_needed_objects.size() + 1; + bool stripe_proceed_main = (add_stripe == 0 || peer_stripe == 0 || add_stripe == peer_stripe) && (next_block_height < bc_height + BLOCK_QUEUE_FORCE_DOWNLOAD_NEAR_BLOCKS || next_needed_height < bc_height + BLOCK_QUEUE_FORCE_DOWNLOAD_NEAR_BLOCKS); + bool stripe_proceed_secondary = tools::has_unpruned_block(next_block_height, context.m_remote_blockchain_height, context.m_pruning_seed); + bool proceed = stripe_proceed_main || (queue_proceed && stripe_proceed_secondary); + if (!stripe_proceed_main && !stripe_proceed_secondary && should_drop_connection(context, tools::get_pruning_stripe(next_block_height, context.m_remote_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES))) { - if (!first) - { - LOG_DEBUG_CC(context, "Block queue is " << nblocks << " and " << size << ", resuming"); - } - break; + if (!context.m_is_income) + m_p2p->add_used_stripe_peer(context); + return false; // drop outgoing connections } - // this one triggers if all threads are in standby, which should not happen, - // but happened at least once, so we unblock at least one thread if so - const boost::unique_lock<boost::mutex> sync{m_sync_lock, boost::try_to_lock}; - if (sync.owns_lock()) - { - LOG_DEBUG_CC(context, "No other thread is adding blocks, resuming"); - break; - } + MDEBUG(context << "proceed " << proceed << " (queue " << queue_proceed << ", stripe " << stripe_proceed_main << "/" << + stripe_proceed_secondary << "), " << next_needed_pruning_stripe.first << "-" << next_needed_pruning_stripe.second << + " needed, bc add stripe " << add_stripe << ", we have " << peer_stripe << "), bc_height " << bc_height); + MDEBUG(context << " - next_block_height " << next_block_height << ", seed " << epee::string_tools::to_string_hex(context.m_pruning_seed) << + ", next_needed_height "<< next_needed_height); + MDEBUG(context << " - last_response_height " << context.m_last_response_height << ", m_needed_objects size " << context.m_needed_objects.size()); - if (should_download_next_span(context)) + // if we're waiting for next span, try to get it before unblocking threads below, + // or a runaway downloading of future spans might happen + if (stripe_proceed_main && should_download_next_span(context, true)) { MDEBUG(context << " we should try for that next span too, we think we could get it faster, resuming"); force_next_span = true; + MLOG_PEER_STATE("resuming"); break; } - if (first) + if (proceed) { - LOG_DEBUG_CC(context, "Block queue is " << nblocks << " and " << size << ", pausing"); - first = false; - context.m_state = cryptonote_connection_context::state_standby; + if (context.m_state != cryptonote_connection_context::state_standby) + { + LOG_DEBUG_CC(context, "Block queue is " << nspans << " and " << size << ", resuming"); + MLOG_PEER_STATE("resuming"); + } + break; } - // this needs doing after we went to standby, so the callback knows what to do - bool filled; - if (m_block_queue.has_next_span(context.m_connection_id, filled) && !filled) + // this one triggers if all threads are in standby, which should not happen, + // but happened at least once, so we unblock at least one thread if so + boost::unique_lock<boost::mutex> sync{m_sync_lock, boost::try_to_lock}; + if (sync.owns_lock()) { - MDEBUG(context << " we have the next span, and it is scheduled, resuming"); - ++context.m_callback_request_count; - m_p2p->request_callback(context); - return true; + bool filled = false; + boost::posix_time::ptime time; + boost::uuids::uuid connection_id; + if (m_block_queue.has_next_span(m_core.get_current_blockchain_height(), filled, time, connection_id) && filled) + { + LOG_DEBUG_CC(context, "No other thread is adding blocks, and next span needed is ready, resuming"); + MLOG_PEER_STATE("resuming"); + context.m_state = cryptonote_connection_context::state_standby; + ++context.m_callback_request_count; + m_p2p->request_callback(context); + return true; + } + else + { + sync.unlock(); + + // if this has gone on for too long, drop incoming connection to guard against some wedge state + if (!context.m_is_income) + { + const uint64_t now = tools::get_tick_count(); + const uint64_t dt = now - m_last_add_end_time; + if (tools::ticks_to_ns(dt) >= DROP_ON_SYNC_WEDGE_THRESHOLD) + { + MDEBUG(context << "Block addition seems to have wedged, dropping connection"); + return false; + } + } + } } - for (size_t n = 0; n < 50; ++n) + if (context.m_state != cryptonote_connection_context::state_standby) { - if (m_stopping) - return true; - boost::this_thread::sleep_for(boost::chrono::milliseconds(100)); + if (!queue_proceed) + LOG_DEBUG_CC(context, "Block queue is " << nspans << " and " << size << ", pausing"); + else if (!stripe_proceed_main && !stripe_proceed_secondary) + LOG_DEBUG_CC(context, "We do not have the stripe required to download another block, pausing"); + context.m_state = cryptonote_connection_context::state_standby; + MLOG_PEER_STATE("pausing"); } - } + + return true; + } while(0); context.m_state = cryptonote_connection_context::state_synchronizing; } - MDEBUG(context << " request_missing_objects: check " << check_having_blocks << ", force_next_span " << force_next_span << ", m_needed_objects " << context.m_needed_objects.size() << " lrh " << context.m_last_response_height << ", chain " << m_core.get_current_blockchain_height()); + MDEBUG(context << " request_missing_objects: check " << check_having_blocks << ", force_next_span " << force_next_span + << ", m_needed_objects " << context.m_needed_objects.size() << " lrh " << context.m_last_response_height << ", chain " + << m_core.get_current_blockchain_height() << ", pruning seed " << epee::string_tools::to_string_hex(context.m_pruning_seed)); if(context.m_needed_objects.size() || force_next_span) { //we know objects that we need, request this objects @@ -1402,28 +1717,6 @@ skip: size_t count = 0; const size_t count_limit = m_core.get_block_sync_size(m_core.get_current_blockchain_height()); std::pair<uint64_t, uint64_t> span = std::make_pair(0, 0); - { - MDEBUG(context << " checking for gap"); - span = m_block_queue.get_start_gap_span(); - if (span.second > 0) - { - const uint64_t first_block_height_known = context.m_last_response_height - context.m_needed_objects.size() + 1; - const uint64_t last_block_height_known = context.m_last_response_height; - const uint64_t first_block_height_needed = span.first; - const uint64_t last_block_height_needed = span.first + std::min(span.second, (uint64_t)count_limit) - 1; - MDEBUG(context << " gap found, span: " << span.first << " - " << span.first + span.second - 1 << " (" << last_block_height_needed << ")"); - MDEBUG(context << " current known hashes from " << first_block_height_known << " to " << last_block_height_known); - if (first_block_height_needed < first_block_height_known || last_block_height_needed > last_block_height_known) - { - MDEBUG(context << " we are missing some of the necessary hashes for this gap, requesting chain again"); - context.m_needed_objects.clear(); - context.m_last_response_height = 0; - start_from_current_chain = true; - goto skip; - } - MDEBUG(context << " we have the hashes for this gap"); - } - } if (force_next_span) { if (span.second == 0) @@ -1440,6 +1733,7 @@ skip: req.blocks.push_back(hash); context.m_requested_objects.insert(hash); } + m_block_queue.reset_next_span_time(); } } } @@ -1453,22 +1747,20 @@ skip: context.m_last_response_height = 0; goto skip; } - // take out blocks we already have - size_t skip = 0; - while (skip < context.m_needed_objects.size() && m_core.have_block(context.m_needed_objects[skip])) - { - // if we're popping the last hash, record it so we can ask again from that hash, - // this prevents never being able to progress on peers we get old hash lists from - if (skip + 1 == context.m_needed_objects.size()) - context.m_last_known_hash = context.m_needed_objects[skip]; - ++skip; - } - if (skip > 0) - context.m_needed_objects = std::vector<crypto::hash>(context.m_needed_objects.begin() + skip, context.m_needed_objects.end()); + skip_unneeded_hashes(context, false); const uint64_t first_block_height = context.m_last_response_height - context.m_needed_objects.size() + 1; - span = m_block_queue.reserve_span(first_block_height, context.m_last_response_height, count_limit, context.m_connection_id, context.m_needed_objects); + span = m_block_queue.reserve_span(first_block_height, context.m_last_response_height, count_limit, context.m_connection_id, context.m_pruning_seed, context.m_remote_blockchain_height, context.m_needed_objects); MDEBUG(context << " span from " << first_block_height << ": " << span.first << "/" << span.second); + if (span.second > 0) + { + const uint32_t stripe = tools::get_pruning_stripe(span.first, context.m_remote_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES); + if (context.m_pruning_seed && stripe != tools::get_pruning_stripe(context.m_pruning_seed)) + { + MDEBUG(context << " starting early on next seed (" << span.first << " with stripe " << stripe << + ", context seed " << epee::string_tools::to_string_hex(context.m_pruning_seed) << ")"); + } + } } if (span.second == 0 && !force_next_span) { @@ -1477,6 +1769,8 @@ skip: boost::uuids::uuid span_connection_id; boost::posix_time::ptime time; span = m_block_queue.get_next_span_if_scheduled(hashes, span_connection_id, time); + if (span.second > 0 && !tools::has_unpruned_block(span.first, context.m_remote_blockchain_height, context.m_pruning_seed)) + span = std::make_pair(0, 0); if (span.second > 0) { is_next = true; @@ -1522,17 +1816,67 @@ skip: } context.m_last_request_time = boost::posix_time::microsec_clock::universal_time(); - LOG_PRINT_CCONTEXT_L1("-->>NOTIFY_REQUEST_GET_OBJECTS: blocks.size()=" << req.blocks.size() << ", txs.size()=" << req.txs.size() + MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_GET_OBJECTS: blocks.size()=" << req.blocks.size() << ", txs.size()=" << req.txs.size() << "requested blocks count=" << count << " / " << count_limit << " from " << span.first << ", first hash " << req.blocks.front()); //epee::net_utils::network_throttle_manager::get_global_throttle_inreq().logger_handle_net("log/dr-monero/net/req-all.data", sec, get_avg_block_size()); post_notify<NOTIFY_REQUEST_GET_OBJECTS>(req, context); + MLOG_PEER_STATE("requesting objects"); + return true; + } + + // if we're still around, we might be at a point where the peer is pruned, so we could either + // drop it to make space for other peers, or ask for a span further down the line + const uint32_t next_stripe = get_next_needed_pruning_stripe().first; + const uint32_t peer_stripe = tools::get_pruning_stripe(context.m_pruning_seed); + if (next_stripe && peer_stripe && next_stripe != peer_stripe) + { + // at this point, we have to either close the connection, or start getting blocks past the + // current point, or become dormant + MDEBUG(context << "this peer is pruned at seed " << epee::string_tools::to_string_hex(context.m_pruning_seed) << + ", next stripe needed is " << next_stripe); + if (!context.m_is_income) + { + if (should_drop_connection(context, next_stripe)) + { + m_p2p->add_used_stripe_peer(context); + return false; // drop outgoing connections + } + } + // we'll get back stuck waiting for the go ahead + context.m_state = cryptonote_connection_context::state_normal; + MLOG_PEER_STATE("Nothing to do for now, switching to normal state"); return true; } } skip: context.m_needed_objects.clear(); + + // we might have been called from the "received chain entry" handler, and end up + // here because we can't use any of those blocks (maybe because all of them are + // actually already requested). In this case, if we can add blocks instead, do so + if (m_core.get_current_blockchain_height() < m_core.get_target_blockchain_height()) + { + const boost::unique_lock<boost::mutex> sync{m_sync_lock, boost::try_to_lock}; + if (sync.owns_lock()) + { + uint64_t start_height; + std::vector<cryptonote::block_complete_entry> blocks; + boost::uuids::uuid span_connection_id; + bool filled = false; + if (m_block_queue.get_next_span(start_height, blocks, span_connection_id, filled) && filled) + { + LOG_DEBUG_CC(context, "No other thread is adding blocks, resuming"); + MLOG_PEER_STATE("will try to add blocks next"); + context.m_state = cryptonote_connection_context::state_standby; + ++context.m_callback_request_count; + m_p2p->request_callback(context); + return true; + } + } + } + if(context.m_last_response_height < context.m_remote_blockchain_height-1) {//we have to fetch more objects ids, request blockchain entry @@ -1555,8 +1899,9 @@ skip: //LOG_PRINT_CCONTEXT_L1("r = " << 200); context.m_last_request_time = boost::posix_time::microsec_clock::universal_time(); - LOG_PRINT_CCONTEXT_L1("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() << ", start_from_current_chain " << start_from_current_chain); + MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() << ", start_from_current_chain " << start_from_current_chain); post_notify<NOTIFY_REQUEST_CHAIN>(r, context); + MLOG_PEER_STATE("requesting chain"); }else { CHECK_AND_ASSERT_MES(context.m_last_response_height == context.m_remote_blockchain_height-1 @@ -1596,9 +1941,25 @@ skip: << ENDL << "Use the \"help\" command to see the list of available commands." << ENDL << "**********************************************************************"); + m_sync_timer.pause(); + if (ELPP->vRegistry()->allowed(el::Level::Info, "sync-info")) + { + const uint64_t sync_time = m_sync_timer.value(); + const uint64_t add_time = m_add_timer.value(); + if (sync_time && add_time) + { + MCLOG_YELLOW(el::Level::Info, "sync-info", "Sync time: " << sync_time/1e9/60 << " min, idle time " << + (100.f * (1.0f - add_time / (float)sync_time)) << "%" << ", " << + (10 * m_sync_download_objects_size / 1024 / 1024) / 10.f << " + " << + (10 * m_sync_download_chain_size / 1024 / 1024) / 10.f << " MB downloaded, " << + 100.0f * m_sync_old_spans_downloaded / m_sync_spans_downloaded << "% old spans, " << + 100.0f * m_sync_bad_spans_downloaded / m_sync_spans_downloaded << "% bad spans"); + } + } m_core.on_synchronized(); } m_core.safesyncmode(true); + m_p2p->clear_used_stripe_peers(); return true; } //------------------------------------------------------------------------------------------------------------------------ @@ -1619,6 +1980,11 @@ skip: { MLOG_P2P_MESSAGE("Received NOTIFY_RESPONSE_CHAIN_ENTRY: m_block_ids.size()=" << arg.m_block_ids.size() << ", m_start_height=" << arg.start_height << ", m_total_height=" << arg.total_height); + MLOG_PEER_STATE("received chain"); + + context.m_last_request_time = boost::date_time::not_a_date_time; + + m_sync_download_chain_size += arg.m_block_ids.size() * sizeof(crypto::hash); if(!arg.m_block_ids.size()) { @@ -1632,7 +1998,14 @@ skip: drop_connection(context, true, false); return 1; } + MDEBUG(context << "first block hash " << arg.m_block_ids.front() << ", last " << arg.m_block_ids.back()); + if (arg.total_height >= CRYPTONOTE_MAX_BLOCK_NUMBER || arg.m_block_ids.size() >= CRYPTONOTE_MAX_BLOCK_NUMBER) + { + LOG_ERROR_CCONTEXT("sent wrong NOTIFY_RESPONSE_CHAIN_ENTRY, with total_height=" << arg.total_height << " and block_ids=" << arg.m_block_ids.size()); + drop_connection(context, false, false); + return 1; + } context.m_remote_blockchain_height = arg.total_height; context.m_last_response_height = arg.start_height + arg.m_block_ids.size()-1; if(context.m_last_response_height > context.m_remote_blockchain_height) @@ -1652,6 +2025,7 @@ skip: return 1; } + context.m_needed_objects.clear(); uint64_t added = 0; for(auto& bl_id: arg.m_block_ids) { @@ -1708,13 +2082,13 @@ skip: { std::string fluffyBlob; epee::serialization::store_t_to_binary(fluffy_arg, fluffyBlob); - m_p2p->relay_notify_to_list(NOTIFY_NEW_FLUFFY_BLOCK::ID, fluffyBlob, fluffyConnections); + m_p2p->relay_notify_to_list(NOTIFY_NEW_FLUFFY_BLOCK::ID, epee::strspan<uint8_t>(fluffyBlob), fluffyConnections); } if (!fullConnections.empty()) { std::string fullBlob; epee::serialization::store_t_to_binary(arg, fullBlob); - m_p2p->relay_notify_to_list(NOTIFY_NEW_BLOCK::ID, fullBlob, fullConnections); + m_p2p->relay_notify_to_list(NOTIFY_NEW_BLOCK::ID, epee::strspan<uint8_t>(fullBlob), fullConnections); } return true; @@ -1724,23 +2098,129 @@ skip: bool t_cryptonote_protocol_handler<t_core>::relay_transactions(NOTIFY_NEW_TRANSACTIONS::request& arg, cryptonote_connection_context& exclude_context) { // no check for success, so tell core they're relayed unconditionally + const bool pad_transactions = m_core.pad_transactions(); + size_t bytes = pad_transactions ? 9 /* header */ + 4 /* 1 + 'txs' */ + tools::get_varint_data(arg.txs.size()).size() : 0; for(auto tx_blob_it = arg.txs.begin(); tx_blob_it!=arg.txs.end(); ++tx_blob_it) + { m_core.on_transaction_relayed(*tx_blob_it); + if (pad_transactions) + bytes += tools::get_varint_data(tx_blob_it->size()).size() + tx_blob_it->size(); + } + + if (pad_transactions) + { + // stuff some dummy bytes in to stay safe from traffic volume analysis + static constexpr size_t granularity = 1024; + size_t padding = granularity - bytes % granularity; + const size_t overhead = 2 /* 1 + '_' */ + tools::get_varint_data(padding).size(); + if (overhead > padding) + padding = 0; + else + padding -= overhead; + arg._ = std::string(padding, ' '); + + std::string arg_buff; + epee::serialization::store_t_to_binary(arg, arg_buff); + + // we probably lowballed the payload size a bit, so added a but too much. Fix this now. + size_t remove = arg_buff.size() % granularity; + if (remove > arg._.size()) + arg._.clear(); + else + arg._.resize(arg._.size() - remove); + // if the size of _ moved enough, we might lose byte in size encoding, we don't care + } + return relay_post_notify<NOTIFY_NEW_TRANSACTIONS>(arg, exclude_context); } //------------------------------------------------------------------------------------------------------------------------ template<class t_core> + std::string t_cryptonote_protocol_handler<t_core>::get_peers_overview() const + { + std::stringstream ss; + const boost::posix_time::ptime now = boost::posix_time::microsec_clock::universal_time(); + m_p2p->for_each_connection([&](const connection_context &ctx, nodetool::peerid_type peer_id, uint32_t support_flags) { + const uint32_t stripe = tools::get_pruning_stripe(ctx.m_pruning_seed); + char state_char = cryptonote::get_protocol_state_char(ctx.m_state); + ss << stripe + state_char; + if (ctx.m_last_request_time != boost::date_time::not_a_date_time) + ss << (((now - ctx.m_last_request_time).total_microseconds() > IDLE_PEER_KICK_TIME) ? "!" : "?"); + ss << + " "; + return true; + }); + return ss.str(); + } + //------------------------------------------------------------------------------------------------------------------------ + template<class t_core> + std::pair<uint32_t, uint32_t> t_cryptonote_protocol_handler<t_core>::get_next_needed_pruning_stripe() const + { + const uint64_t want_height_from_blockchain = m_core.get_current_blockchain_height(); + const uint64_t want_height_from_block_queue = m_block_queue.get_next_needed_height(want_height_from_blockchain); + const uint64_t want_height = std::max(want_height_from_blockchain, want_height_from_block_queue); + uint64_t blockchain_height = m_core.get_target_blockchain_height(); + // if we don't know the remote chain size yet, assume infinitely large so we get the right stripe if we're not near the tip + if (blockchain_height == 0) + blockchain_height = CRYPTONOTE_MAX_BLOCK_NUMBER; + const uint32_t next_pruning_stripe = tools::get_pruning_stripe(want_height, blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES); + if (next_pruning_stripe == 0) + return std::make_pair(0, 0); + // if we already have a few peers on this stripe, but none on next one, try next one + unsigned int n_next = 0, n_subsequent = 0, n_others = 0; + const uint32_t subsequent_pruning_stripe = 1 + next_pruning_stripe % (1<<CRYPTONOTE_PRUNING_LOG_STRIPES); + m_p2p->for_each_connection([&](const connection_context &context, nodetool::peerid_type peer_id, uint32_t support_flags) { + if (context.m_state >= cryptonote_connection_context::state_synchronizing) + { + if (context.m_pruning_seed == 0 || tools::get_pruning_stripe(context.m_pruning_seed) == next_pruning_stripe) + ++n_next; + else if (tools::get_pruning_stripe(context.m_pruning_seed) == subsequent_pruning_stripe) + ++n_subsequent; + else + ++n_others; + } + return true; + }); + const bool use_next = (n_next > m_max_out_peers / 2 && n_subsequent <= 1) || (n_next > 2 && n_subsequent == 0); + const uint32_t ret_stripe = use_next ? subsequent_pruning_stripe: next_pruning_stripe; + MIDEBUG(const std::string po = get_peers_overview(), "get_next_needed_pruning_stripe: want height " << want_height << " (" << + want_height_from_blockchain << " from blockchain, " << want_height_from_block_queue << " from block queue), stripe " << + next_pruning_stripe << " (" << n_next << "/" << m_max_out_peers << " on it and " << n_subsequent << " on " << + subsequent_pruning_stripe << ", " << n_others << " others) -> " << ret_stripe << " (+" << + (ret_stripe - next_pruning_stripe + (1 << CRYPTONOTE_PRUNING_LOG_STRIPES)) % (1 << CRYPTONOTE_PRUNING_LOG_STRIPES) << + "), current peers " << po); + return std::make_pair(next_pruning_stripe, ret_stripe); + } + //------------------------------------------------------------------------------------------------------------------------ + template<class t_core> + bool t_cryptonote_protocol_handler<t_core>::needs_new_sync_connections() const + { + const uint64_t target = m_core.get_target_blockchain_height(); + const uint64_t height = m_core.get_current_blockchain_height(); + if (target && target <= height) + return false; + size_t n_out_peers = 0; + m_p2p->for_each_connection([&](cryptonote_connection_context& ctx, nodetool::peerid_type peer_id, uint32_t support_flags)->bool{ + if (!ctx.m_is_income) + ++n_out_peers; + return true; + }); + if (n_out_peers >= m_max_out_peers) + return false; + return true; + } + //------------------------------------------------------------------------------------------------------------------------ + template<class t_core> void t_cryptonote_protocol_handler<t_core>::drop_connection(cryptonote_connection_context &context, bool add_fail, bool flush_all_spans) { - LOG_DEBUG_CC(context, "dropping connection id " << context.m_connection_id << - ", add_fail " << add_fail << ", flush_all_spans " << flush_all_spans); + LOG_DEBUG_CC(context, "dropping connection id " << context.m_connection_id << " (pruning seed " << + epee::string_tools::to_string_hex(context.m_pruning_seed) << + "), add_fail " << add_fail << ", flush_all_spans " << flush_all_spans); if (add_fail) m_p2p->add_host_fail(context.m_remote_address); - m_p2p->drop_connection(context); - m_block_queue.flush_spans(context.m_connection_id, flush_all_spans); + + m_p2p->drop_connection(context); } //------------------------------------------------------------------------------------------------------------------------ template<class t_core> @@ -1760,6 +2240,7 @@ skip: } m_block_queue.flush_spans(context.m_connection_id, false); + MLOG_PEER_STATE("closed"); } //------------------------------------------------------------------------------------------------------------------------ diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt index f645836a4..117790455 100644 --- a/src/daemon/CMakeLists.txt +++ b/src/daemon/CMakeLists.txt @@ -26,20 +26,6 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -set(blocksdat "") -if(PER_BLOCK_CHECKPOINT) - if(APPLE AND DEPENDS) - add_custom_command(OUTPUT blocksdat.o MAIN_DEPENDENCY ../blocks/checkpoints.dat COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && touch stub.c && ${CMAKE_C_COMPILER} --target=x86_64-apple-darwin11 -o stub.o -c stub.c COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${CMAKE_LINKER} -r -sectcreate __DATA __blocks_dat ../blocks/checkpoints.dat -o ${CMAKE_CURRENT_BINARY_DIR}/blocksdat.o stub.o && rm -f stub.*) - elseif(APPLE AND NOT DEPENDS) - add_custom_command(OUTPUT blocksdat.o MAIN_DEPENDENCY ../blocks/checkpoints.dat COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && touch stub.c && ${CMAKE_C_COMPILER} -o stub.o -c stub.c COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && ${CMAKE_LINKER} ${LD_RAW_FLAGS} -r -sectcreate __DATA __blocks_dat ../blocks/checkpoints.dat -o ${CMAKE_CURRENT_BINARY_DIR}/blocksdat.o stub.o && rm -f stub.*) - elseif(LINUX_32) - add_custom_command(OUTPUT blocksdat.o MAIN_DEPENDENCY ../blocks/checkpoints.dat COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && cp ../blocks/checkpoints.dat blocks.dat && ${CMAKE_LINKER} -m elf_i386 ${LD_RAW_FLAGS} -r -b binary -o ${CMAKE_CURRENT_BINARY_DIR}/blocksdat.o blocks.dat && rm -f blocks.dat) - else() - add_custom_command(OUTPUT blocksdat.o MAIN_DEPENDENCY ../blocks/checkpoints.dat COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && cp ../blocks/checkpoints.dat blocks.dat && ${CMAKE_LINKER} ${LD_RAW_FLAGS} -r -b binary -o ${CMAKE_CURRENT_BINARY_DIR}/blocksdat.o blocks.dat && rm -f blocks.dat) - endif() - set(blocksdat "blocksdat.o") -endif() - set(daemon_sources command_parser_executor.cpp command_server.cpp @@ -81,9 +67,7 @@ monero_private_headers(daemon monero_add_executable(daemon ${daemon_sources} ${daemon_headers} - ${daemon_private_headers} - ${blocksdat} -) + ${daemon_private_headers}) target_link_libraries(daemon PRIVATE rpc @@ -106,7 +90,8 @@ target_link_libraries(daemon ${CMAKE_THREAD_LIBS_INIT} ${ZMQ_LIB} ${GNU_READLINE_LIBRARY} - ${EXTRA_LIBRARIES}) + ${EXTRA_LIBRARIES} + ${Blocks}) set_property(TARGET daemon PROPERTY OUTPUT_NAME "monerod") diff --git a/src/daemon/command_parser_executor.cpp b/src/daemon/command_parser_executor.cpp index 1638cf505..37cb55ec8 100644 --- a/src/daemon/command_parser_executor.cpp +++ b/src/daemon/command_parser_executor.cpp @@ -163,9 +163,21 @@ bool t_command_parser_executor::print_height(const std::vector<std::string>& arg bool t_command_parser_executor::print_block(const std::vector<std::string>& args) { + bool include_hex = false; + + // Assumes that optional flags come after mandatory argument <transaction_hash> + for (unsigned int i = 1; i < args.size(); ++i) { + if (args[i] == "+hex") + include_hex = true; + else + { + std::cout << "unexpected argument: " << args[i] << std::endl; + return true; + } + } if (args.empty()) { - std::cout << "expected: print_block (<block_hash> | <block_height>)" << std::endl; + std::cout << "expected: print_block (<block_hash> | <block_height>) [+hex]" << std::endl; return false; } @@ -173,14 +185,14 @@ bool t_command_parser_executor::print_block(const std::vector<std::string>& args try { uint64_t height = boost::lexical_cast<uint64_t>(arg); - return m_executor.print_block_by_height(height); + return m_executor.print_block_by_height(height, include_hex); } catch (const boost::bad_lexical_cast&) { crypto::hash block_hash; if (parse_hash256(arg, block_hash)) { - return m_executor.print_block_by_hash(block_hash); + return m_executor.print_block_by_hash(block_hash, include_hex); } } @@ -674,10 +686,58 @@ bool t_command_parser_executor::sync_info(const std::vector<std::string>& args) return m_executor.sync_info(); } +bool t_command_parser_executor::pop_blocks(const std::vector<std::string>& args) +{ + if (args.size() != 1) + { + std::cout << "Exactly one parameter is needed" << std::endl; + return false; + } + + try + { + uint64_t nblocks = boost::lexical_cast<uint64_t>(args[0]); + if (nblocks < 1) + { + std::cout << "number of blocks must be greater than 0" << std::endl; + return false; + } + return m_executor.pop_blocks(nblocks); + } + catch (const boost::bad_lexical_cast&) + { + std::cout << "number of blocks must be a number greater than 0" << std::endl; + } + return false; +} + bool t_command_parser_executor::version(const std::vector<std::string>& args) { std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << std::endl; return true; } +bool t_command_parser_executor::prune_blockchain(const std::vector<std::string>& args) +{ + if (args.size() > 1) return false; + + if (args.empty() || args[0] != "confirm") + { + std::cout << "Warning: pruning from within monerod will not shrink the database file size." << std::endl; + std::cout << "Instead, parts of the file will be marked as free, so the file will not grow" << std::endl; + std::cout << "until that newly free space is used up. If you want a smaller file size now," << std::endl; + std::cout << "exit monerod and run monero-blockchain-prune (you will temporarily need more" << std::endl; + std::cout << "disk space for the database conversion though). If you are OK with the database" << std::endl; + std::cout << "file keeping the same size, re-run this command with the \"confirm\" parameter." << std::endl; + return true; + } + + return m_executor.prune_blockchain(); +} + +bool t_command_parser_executor::check_blockchain_pruning(const std::vector<std::string>& args) +{ + return m_executor.check_blockchain_pruning(); +} + } // namespace daemonize diff --git a/src/daemon/command_parser_executor.h b/src/daemon/command_parser_executor.h index a70070171..5b8927908 100644 --- a/src/daemon/command_parser_executor.h +++ b/src/daemon/command_parser_executor.h @@ -139,7 +139,13 @@ public: bool sync_info(const std::vector<std::string>& args); + bool pop_blocks(const std::vector<std::string>& args); + bool version(const std::vector<std::string>& args); + + bool prune_blockchain(const std::vector<std::string>& args); + + bool check_blockchain_pruning(const std::vector<std::string>& args); }; } // namespace daemonize diff --git a/src/daemon/command_server.cpp b/src/daemon/command_server.cpp index 35504f2c9..786d1a601 100644 --- a/src/daemon/command_server.cpp +++ b/src/daemon/command_server.cpp @@ -282,10 +282,26 @@ t_command_server::t_command_server( , "Print information about the blockchain sync state." ); m_command_lookup.set_handler( + "pop_blocks" + , std::bind(&t_command_parser_executor::pop_blocks, &m_parser, p::_1) + , "pop_blocks <nblocks>" + , "Remove blocks from end of blockchain" + ); + m_command_lookup.set_handler( "version" , std::bind(&t_command_parser_executor::version, &m_parser, p::_1) , "Print version information." ); + m_command_lookup.set_handler( + "prune_blockchain" + , std::bind(&t_command_parser_executor::prune_blockchain, &m_parser, p::_1) + , "Prune the blockchain." + ); + m_command_lookup.set_handler( + "check_blockchain_pruning" + , std::bind(&t_command_parser_executor::check_blockchain_pruning, &m_parser, p::_1) + , "Check the blockchain pruning." + ); } bool t_command_server::process_command_str(const std::string& cmd) diff --git a/src/daemon/core.h b/src/daemon/core.h index 475f418d6..c15d8d236 100644 --- a/src/daemon/core.h +++ b/src/daemon/core.h @@ -28,6 +28,7 @@ #pragma once +#include "blocks/blocks.h" #include "cryptonote_core/cryptonote_core.h" #include "cryptonote_protocol/cryptonote_protocol_handler.h" #include "misc_log_ex.h" @@ -66,26 +67,16 @@ public: m_core.set_cryptonote_protocol(&protocol); } - std::string get_config_subdir() const - { - bool testnet = command_line::get_arg(m_vm_HACK, cryptonote::arg_testnet_on); - bool stagenet = command_line::get_arg(m_vm_HACK, cryptonote::arg_stagenet_on); - bool mainnet = !testnet && !stagenet; - std::string port = command_line::get_arg(m_vm_HACK, nodetool::arg_p2p_bind_port); - if ((mainnet && port != std::to_string(::config::P2P_DEFAULT_PORT)) - || (testnet && port != std::to_string(::config::testnet::P2P_DEFAULT_PORT)) - || (stagenet && port != std::to_string(::config::stagenet::P2P_DEFAULT_PORT))) { - return port; - } - return std::string(); - } - bool run() { //initialize core here MGINFO("Initializing core..."); - std::string config_subdir = get_config_subdir(); - if (!m_core.init(m_vm_HACK, config_subdir.empty() ? NULL : config_subdir.c_str())) +#if defined(PER_BLOCK_CHECKPOINT) + const cryptonote::GetCheckpointsCallback& get_checkpoints = blocks::GetCheckpointsData; +#else + const cryptonote::GetCheckpointsCallback& get_checkpoints = nullptr; +#endif + if (!m_core.init(m_vm_HACK, nullptr, get_checkpoints)) { return false; } diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 49d6d49cf..88fba8372 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -75,18 +75,15 @@ public: protocol.set_p2p_endpoint(p2p.get()); core.set_protocol(protocol.get()); - const auto testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); - const auto stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); - const auto regtest = command_line::get_arg(vm, cryptonote::arg_regtest_on); const auto restricted = command_line::get_arg(vm, cryptonote::core_rpc_server::arg_restricted_rpc); const auto main_rpc_port = command_line::get_arg(vm, cryptonote::core_rpc_server::arg_rpc_bind_port); - rpcs.emplace_back(new t_rpc{vm, core, p2p, restricted, testnet ? cryptonote::TESTNET : stagenet ? cryptonote::STAGENET : regtest ? cryptonote::FAKECHAIN : cryptonote::MAINNET, main_rpc_port, "core"}); + rpcs.emplace_back(new t_rpc{vm, core, p2p, restricted, main_rpc_port, "core"}); auto restricted_rpc_port_arg = cryptonote::core_rpc_server::arg_rpc_restricted_bind_port; if(!command_line::is_arg_defaulted(vm, restricted_rpc_port_arg)) { auto restricted_rpc_port = command_line::get_arg(vm, restricted_rpc_port_arg); - rpcs.emplace_back(new t_rpc{vm, core, p2p, true, testnet ? cryptonote::TESTNET : stagenet ? cryptonote::STAGENET : cryptonote::MAINNET, restricted_rpc_port, "restricted"}); + rpcs.emplace_back(new t_rpc{vm, core, p2p, true, restricted_rpc_port, "restricted"}); } } }; @@ -198,7 +195,6 @@ bool t_daemon::run(bool interactive) for(auto& rpc : mp_internals->rpcs) rpc->stop(); - mp_internals->core.get().get_miner().stop(); MGINFO("Node stopped."); return true; } @@ -220,7 +216,6 @@ void t_daemon::stop() { throw std::runtime_error{"Can't stop stopped daemon"}; } - mp_internals->core.get().get_miner().stop(); mp_internals->p2p.stop(); for(auto& rpc : mp_internals->rpcs) rpc->stop(); diff --git a/src/daemon/main.cpp b/src/daemon/main.cpp index f483ba6c9..35017d9ef 100644 --- a/src/daemon/main.cpp +++ b/src/daemon/main.cpp @@ -216,6 +216,16 @@ int main(int argc, char const * argv[]) // after logs initialized tools::create_directories_if_necessary(data_dir.string()); +#ifdef STACK_TRACE + tools::set_stack_trace_log(log_file_path.filename().string()); +#endif // STACK_TRACE + + if (!command_line::is_arg_defaulted(vm, daemon_args::arg_max_concurrency)) + tools::set_max_concurrency(command_line::get_arg(vm, daemon_args::arg_max_concurrency)); + + // logging is now set up + MGINFO("Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")"); + // If there are positional options, we're running a daemon command { auto command = command_line::get_arg(vm, daemon_args::arg_command); @@ -276,16 +286,6 @@ int main(int argc, char const * argv[]) } } -#ifdef STACK_TRACE - tools::set_stack_trace_log(log_file_path.filename().string()); -#endif // STACK_TRACE - - if (!command_line::is_arg_defaulted(vm, daemon_args::arg_max_concurrency)) - tools::set_max_concurrency(command_line::get_arg(vm, daemon_args::arg_max_concurrency)); - - // logging is now set up - MGINFO("Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")"); - MINFO("Moving from main() into the daemonize now."); return daemonizer::daemonize(argc, argv, daemonize::t_executor{}, vm) ? 0 : 1; diff --git a/src/daemon/rpc.h b/src/daemon/rpc.h index 9621b0d01..37dffc097 100644 --- a/src/daemon/rpc.h +++ b/src/daemon/rpc.h @@ -54,7 +54,6 @@ public: , t_core & core , t_p2p & p2p , const bool restricted - , const cryptonote::network_type nettype , const std::string & port , const std::string & description ) @@ -62,7 +61,7 @@ public: { MGINFO("Initializing " << m_description << " RPC server..."); - if (!m_server.init(vm, restricted, nettype, port)) + if (!m_server.init(vm, restricted, port)) { throw std::runtime_error("Failed to initialize " + m_description + " RPC server."); } diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp index 6464d372f..7cd61934f 100644 --- a/src/daemon/rpc_command_executor.cpp +++ b/src/daemon/rpc_command_executor.cpp @@ -31,6 +31,7 @@ #include "string_tools.h" #include "common/password.h" #include "common/scoped_message_writer.h" +#include "common/pruning.h" #include "daemon/rpc_command_executor.h" #include "rpc/core_rpc_server_commands_defs.h" #include "cryptonote_core/cryptonote_core.h" @@ -60,13 +61,14 @@ namespace { peer_id_str >> id_str; epee::string_tools::xtype_to_string(peer.port, port_str); std::string addr_str = ip_str + ":" + port_str; - tools::msg_writer() << boost::format("%-10s %-25s %-25s %s") % prefix % id_str % addr_str % elapsed; + std::string pruning_seed = epee::string_tools::to_string_hex(peer.pruning_seed); + tools::msg_writer() << boost::format("%-10s %-25s %-25s %-4s %s") % prefix % id_str % addr_str % pruning_seed % elapsed; } void print_block_header(cryptonote::block_header_response const & header) { tools::success_msg_writer() - << "timestamp: " << boost::lexical_cast<std::string>(header.timestamp) << std::endl + << "timestamp: " << boost::lexical_cast<std::string>(header.timestamp) << " (" << tools::get_human_readable_timestamp(header.timestamp) << ")" << std::endl << "previous hash: " << header.prev_hash << std::endl << "nonce: " << boost::lexical_cast<std::string>(header.nonce) << std::endl << "is orphan: " << header.orphan_status << std::endl @@ -449,7 +451,7 @@ bool t_rpc_command_executor::show_status() { % get_sync_percentage(ires) % (ires.testnet ? "testnet" : ires.stagenet ? "stagenet" : "mainnet") % bootstrap_msg - % (!has_mining_info ? "mining info unavailable" : mining_busy ? "syncing" : mres.active ? ( ( mres.is_background_mining_enabled ? "smart " : "" ) + std::string("mining at ") + get_mining_speed(mres.speed) ) : "not mining") + % (!has_mining_info ? "mining info unavailable" : mining_busy ? "syncing" : mres.active ? ( ( mres.is_background_mining_enabled ? "smart " : "" ) + std::string("mining at ") + get_mining_speed(mres.speed) + std::string(" to ") + mres.address ) : "not mining") % get_mining_speed(ires.difficulty / ires.target) % (unsigned)hfres.version % get_fork_extra_info(hfres.earliest_height, net_height, ires.target) @@ -567,9 +569,9 @@ bool t_rpc_command_executor::print_blockchain_info(uint64_t start_block_index, u for (auto & header : res.headers) { if (!first) - std::cout << std::endl; - std::cout - << "height: " << header.height << ", timestamp: " << header.timestamp + tools::msg_writer() << "" << std::endl; + tools::msg_writer() + << "height: " << header.height << ", timestamp: " << header.timestamp << " (" << tools::get_human_readable_timestamp(header.timestamp) << ")" << ", size: " << header.block_size << ", weight: " << header.block_weight << ", transactions: " << header.num_txes << std::endl << "major version: " << (unsigned)header.major_version << ", minor version: " << (unsigned)header.minor_version << std::endl << "block id: " << header.hash << ", previous block id: " << header.prev_hash << std::endl @@ -663,7 +665,7 @@ bool t_rpc_command_executor::print_height() { return true; } -bool t_rpc_command_executor::print_block_by_hash(crypto::hash block_hash) { +bool t_rpc_command_executor::print_block_by_hash(crypto::hash block_hash, bool include_hex) { cryptonote::COMMAND_RPC_GET_BLOCK::request req; cryptonote::COMMAND_RPC_GET_BLOCK::response res; epee::json_rpc::error error_resp; @@ -689,13 +691,15 @@ bool t_rpc_command_executor::print_block_by_hash(crypto::hash block_hash) { } } + if (include_hex) + tools::success_msg_writer() << res.blob << std::endl; print_block_header(res.block_header); tools::success_msg_writer() << res.json << ENDL; return true; } -bool t_rpc_command_executor::print_block_by_height(uint64_t height) { +bool t_rpc_command_executor::print_block_by_height(uint64_t height, bool include_hex) { cryptonote::COMMAND_RPC_GET_BLOCK::request req; cryptonote::COMMAND_RPC_GET_BLOCK::response res; epee::json_rpc::error error_resp; @@ -721,6 +725,8 @@ bool t_rpc_command_executor::print_block_by_height(uint64_t height) { } } + if (include_hex) + tools::success_msg_writer() << res.blob << std::endl; print_block_header(res.block_header); tools::success_msg_writer() << res.json << ENDL; @@ -737,6 +743,7 @@ bool t_rpc_command_executor::print_transaction(crypto::hash transaction_hash, req.txs_hashes.push_back(epee::string_tools::pod_to_hex(transaction_hash)); req.decode_as_json = false; + req.split = true; req.prune = false; if (m_is_rpc) { @@ -762,13 +769,25 @@ bool t_rpc_command_executor::print_transaction(crypto::hash transaction_hash, if (res.txs.front().in_pool) tools::success_msg_writer() << "Found in pool"; else - tools::success_msg_writer() << "Found in blockchain at height " << res.txs.front().block_height; + tools::success_msg_writer() << "Found in blockchain at height " << res.txs.front().block_height << (res.txs.front().prunable_as_hex.empty() ? " (pruned)" : ""); } const std::string &as_hex = (1 == res.txs.size()) ? res.txs.front().as_hex : res.txs_as_hex.front(); + const std::string &pruned_as_hex = (1 == res.txs.size()) ? res.txs.front().pruned_as_hex : ""; + const std::string &prunable_as_hex = (1 == res.txs.size()) ? res.txs.front().prunable_as_hex : ""; // Print raw hex if requested if (include_hex) - tools::success_msg_writer() << as_hex << std::endl; + { + if (!as_hex.empty()) + { + tools::success_msg_writer() << as_hex << std::endl; + } + else + { + std::string output = pruned_as_hex + prunable_as_hex; + tools::success_msg_writer() << output << std::endl; + } + } // Print json if requested if (include_json) @@ -776,17 +795,27 @@ bool t_rpc_command_executor::print_transaction(crypto::hash transaction_hash, crypto::hash tx_hash, tx_prefix_hash; cryptonote::transaction tx; cryptonote::blobdata blob; - if (!string_tools::parse_hexstr_to_binbuff(as_hex, blob)) + std::string source = as_hex.empty() ? pruned_as_hex + prunable_as_hex : as_hex; + bool pruned = !pruned_as_hex.empty() && prunable_as_hex.empty(); + if (!string_tools::parse_hexstr_to_binbuff(source, blob)) { tools::fail_msg_writer() << "Failed to parse tx to get json format"; } - else if (!cryptonote::parse_and_validate_tx_from_blob(blob, tx, tx_hash, tx_prefix_hash)) - { - tools::fail_msg_writer() << "Failed to parse tx blob to get json format"; - } else { - tools::success_msg_writer() << cryptonote::obj_to_json_str(tx) << std::endl; + bool ret; + if (pruned) + ret = cryptonote::parse_and_validate_tx_base_from_blob(blob, tx); + else + ret = cryptonote::parse_and_validate_tx_from_blob(blob, tx); + if (!ret) + { + tools::fail_msg_writer() << "Failed to parse tx blob to get json format"; + } + else + { + tools::success_msg_writer() << cryptonote::obj_to_json_str(tx) << std::endl; + } } } } @@ -1313,7 +1342,7 @@ bool t_rpc_command_executor::out_peers(uint64_t limit) } } - std::cout << "Max number of out peers set to " << limit << std::endl; + tools::msg_writer() << "Max number of out peers set to " << limit << std::endl; return true; } @@ -1345,7 +1374,7 @@ bool t_rpc_command_executor::in_peers(uint64_t limit) } } - std::cout << "Max number of in peers set to " << limit << std::endl; + tools::msg_writer() << "Max number of in peers set to " << limit << std::endl; return true; } @@ -1717,11 +1746,14 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks) cryptonote::COMMAND_RPC_GET_BLOCK_HEADERS_RANGE::response bhres; cryptonote::COMMAND_RPC_GET_BASE_FEE_ESTIMATE::request fereq; cryptonote::COMMAND_RPC_GET_BASE_FEE_ESTIMATE::response feres; + cryptonote::COMMAND_RPC_HARD_FORK_INFO::request hfreq; + cryptonote::COMMAND_RPC_HARD_FORK_INFO::response hfres; epee::json_rpc::error error_resp; std::string fail_message = "Problem fetching info"; fereq.grace_blocks = 0; + hfreq.version = HF_VERSION_PER_BYTE_FEE; if (m_is_rpc) { if (!m_rpc_client->rpc_request(ireq, ires, "/getinfo", fail_message.c_str())) @@ -1732,6 +1764,10 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks) { return true; } + if (!m_rpc_client->json_rpc_request(hfreq, hfres, "hard_fork_info", fail_message.c_str())) + { + return true; + } } else { @@ -1745,10 +1781,15 @@ bool t_rpc_command_executor::print_blockchain_dynamic_stats(uint64_t nblocks) tools::fail_msg_writer() << make_error(fail_message, feres.status); return true; } + if (!m_rpc_server->on_hard_fork_info(hfreq, hfres, error_resp) || hfres.status != CORE_RPC_STATUS_OK) + { + tools::fail_msg_writer() << make_error(fail_message, hfres.status); + return true; + } } tools::msg_writer() << "Height: " << ires.height << ", diff " << ires.difficulty << ", cum. diff " << ires.cumulative_difficulty - << ", target " << ires.target << " sec" << ", dyn fee " << cryptonote::print_money(feres.fee) << "/kB"; + << ", target " << ires.target << " sec" << ", dyn fee " << cryptonote::print_money(feres.fee) << "/" << (hfres.enabled ? "byte" : "kB"); if (nblocks > 0) { @@ -1923,6 +1964,8 @@ bool t_rpc_command_executor::sync_info() for (const auto &p: res.peers) current_download += p.info.current_download; tools::success_msg_writer() << "Downloading at " << current_download << " kB/s"; + if (res.next_needed_pruning_seed) + tools::success_msg_writer() << "Next needed pruning seed: " << res.next_needed_pruning_seed; tools::success_msg_writer() << std::to_string(res.peers.size()) << " peers"; for (const auto &p: res.peers) @@ -1930,29 +1973,126 @@ bool t_rpc_command_executor::sync_info() std::string address = epee::string_tools::pad_string(p.info.address, 24); uint64_t nblocks = 0, size = 0; for (const auto &s: res.spans) - if (s.rate > 0.0f && s.connection_id == p.info.connection_id) + if (s.connection_id == p.info.connection_id) nblocks += s.nblocks, size += s.size; - tools::success_msg_writer() << address << " " << epee::string_tools::pad_string(p.info.peer_id, 16, '0', true) << " " << epee::string_tools::pad_string(p.info.state, 16) << " " << p.info.height << " " << p.info.current_download << " kB/s, " << nblocks << " blocks / " << size/1e6 << " MB queued"; + tools::success_msg_writer() << address << " " << epee::string_tools::pad_string(p.info.peer_id, 16, '0', true) << " " << + epee::string_tools::pad_string(p.info.state, 16) << " " << + epee::string_tools::pad_string(epee::string_tools::to_string_hex(p.info.pruning_seed), 8) << " " << p.info.height << " " << + p.info.current_download << " kB/s, " << nblocks << " blocks / " << size/1e6 << " MB queued"; } uint64_t total_size = 0; for (const auto &s: res.spans) total_size += s.size; tools::success_msg_writer() << std::to_string(res.spans.size()) << " spans, " << total_size/1e6 << " MB"; + tools::success_msg_writer() << res.overview; for (const auto &s: res.spans) { std::string address = epee::string_tools::pad_string(s.remote_address, 24); + std::string pruning_seed = epee::string_tools::to_string_hex(tools::get_pruning_seed(s.start_block_height, std::numeric_limits<uint64_t>::max(), CRYPTONOTE_PRUNING_LOG_STRIPES)); if (s.size == 0) { - tools::success_msg_writer() << address << " " << s.nblocks << " (" << s.start_block_height << " - " << (s.start_block_height + s.nblocks - 1) << ") -"; + tools::success_msg_writer() << address << " " << s.nblocks << "/" << pruning_seed << " (" << s.start_block_height << " - " << (s.start_block_height + s.nblocks - 1) << ") -"; } else { - tools::success_msg_writer() << address << " " << s.nblocks << " (" << s.start_block_height << " - " << (s.start_block_height + s.nblocks - 1) << ", " << (uint64_t)(s.size/1e3) << " kB) " << (unsigned)(s.rate/1e3) << " kB/s (" << s.speed/100.0f << ")"; + tools::success_msg_writer() << address << " " << s.nblocks << "/" << pruning_seed << " (" << s.start_block_height << " - " << (s.start_block_height + s.nblocks - 1) << ", " << (uint64_t)(s.size/1e3) << " kB) " << (unsigned)(s.rate/1e3) << " kB/s (" << s.speed/100.0f << ")"; } } return true; } +bool t_rpc_command_executor::pop_blocks(uint64_t num_blocks) +{ + cryptonote::COMMAND_RPC_POP_BLOCKS::request req; + cryptonote::COMMAND_RPC_POP_BLOCKS::response res; + std::string fail_message = "pop_blocks failed"; + + req.nblocks = num_blocks; + if (m_is_rpc) + { + if (!m_rpc_client->rpc_request(req, res, "/pop_blocks", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_pop_blocks(req, res) || res.status != CORE_RPC_STATUS_OK) + { + tools::fail_msg_writer() << make_error(fail_message, res.status); + return true; + } + } + tools::success_msg_writer() << "new height: " << res.height; + + return true; +} + +bool t_rpc_command_executor::prune_blockchain() +{ + cryptonote::COMMAND_RPC_PRUNE_BLOCKCHAIN::request req; + cryptonote::COMMAND_RPC_PRUNE_BLOCKCHAIN::response res; + std::string fail_message = "Unsuccessful"; + epee::json_rpc::error error_resp; + + req.check = false; + + if (m_is_rpc) + { + if (!m_rpc_client->json_rpc_request(req, res, "prune_blockchain", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_prune_blockchain(req, res, error_resp) || res.status != CORE_RPC_STATUS_OK) + { + tools::fail_msg_writer() << make_error(fail_message, res.status); + return true; + } + } + + tools::success_msg_writer() << "Blockchain pruned: seed " << epee::string_tools::to_string_hex(res.pruning_seed); + return true; +} + +bool t_rpc_command_executor::check_blockchain_pruning() +{ + cryptonote::COMMAND_RPC_PRUNE_BLOCKCHAIN::request req; + cryptonote::COMMAND_RPC_PRUNE_BLOCKCHAIN::response res; + std::string fail_message = "Unsuccessful"; + epee::json_rpc::error error_resp; + + req.check = true; + + if (m_is_rpc) + { + if (!m_rpc_client->json_rpc_request(req, res, "prune_blockchain", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_prune_blockchain(req, res, error_resp) || res.status != CORE_RPC_STATUS_OK) + { + tools::fail_msg_writer() << make_error(fail_message, res.status); + return true; + } + } + + if (res.pruning_seed) + { + tools::success_msg_writer() << "Blockchain pruning checked"; + } + else + { + tools::success_msg_writer() << "Blockchain is not pruned"; + } + return true; +} + }// namespace daemonize diff --git a/src/daemon/rpc_command_executor.h b/src/daemon/rpc_command_executor.h index 9e6010c5b..de743065b 100644 --- a/src/daemon/rpc_command_executor.h +++ b/src/daemon/rpc_command_executor.h @@ -91,9 +91,9 @@ public: bool print_height(); - bool print_block_by_hash(crypto::hash block_hash); + bool print_block_by_hash(crypto::hash block_hash, bool include_hex); - bool print_block_by_height(uint64_t height); + bool print_block_by_height(uint64_t height, bool include_hex); bool print_transaction(crypto::hash transaction_hash, bool include_hex, bool include_json); @@ -152,6 +152,12 @@ public: bool relay_tx(const std::string &txid); bool sync_info(); + + bool pop_blocks(uint64_t num_blocks); + + bool prune_blockchain(); + + bool check_blockchain_pruning(); }; } // namespace daemonize diff --git a/src/daemonizer/posix_fork.cpp b/src/daemonizer/posix_fork.cpp index 3cbee9c51..5af4e1a4a 100644 --- a/src/daemonizer/posix_fork.cpp +++ b/src/daemonizer/posix_fork.cpp @@ -12,7 +12,6 @@ #include <unistd.h> #include <stdexcept> #include <string> -#include <sys/stat.h> #ifndef TMPDIR #define TMPDIR "/tmp" diff --git a/src/daemonizer/windows_service.cpp b/src/daemonizer/windows_service.cpp index b344e1a4b..1302fa578 100644 --- a/src/daemonizer/windows_service.cpp +++ b/src/daemonizer/windows_service.cpp @@ -70,8 +70,9 @@ namespace { } else { - return std::string{p_error_text}; + std::string ret{p_error_text}; LocalFree(p_error_text); + return ret; } } diff --git a/src/debug_utilities/cn_deserialize.cpp b/src/debug_utilities/cn_deserialize.cpp index 3e2552230..f08f2b928 100644 --- a/src/debug_utilities/cn_deserialize.cpp +++ b/src/debug_utilities/cn_deserialize.cpp @@ -169,6 +169,7 @@ int main(int argc, char* argv[]) return 1; } + bool full; cryptonote::block block; cryptonote::transaction tx; std::vector<cryptonote::tx_extra_field> fields; @@ -179,11 +180,9 @@ int main(int argc, char* argv[]) } else if (cryptonote::parse_and_validate_tx_from_blob(blob, tx) || cryptonote::parse_and_validate_tx_base_from_blob(blob, tx)) { -/* if (tx.pruned) std::cout << "Parsed pruned transaction:" << std::endl; else -*/ std::cout << "Parsed transaction:" << std::endl; std::cout << cryptonote::obj_to_json_str(tx) << std::endl; @@ -200,9 +199,9 @@ int main(int argc, char* argv[]) std::cout << "No fields were found in tx_extra" << std::endl; } } - else if (cryptonote::parse_tx_extra(std::vector<uint8_t>(blob.begin(), blob.end()), fields) && !fields.empty()) + else if (((full = cryptonote::parse_tx_extra(std::vector<uint8_t>(blob.begin(), blob.end()), fields)) || true) && !fields.empty()) { - std::cout << "Parsed tx_extra:" << std::endl; + std::cout << "Parsed" << (full ? "" : " partial") << " tx_extra:" << std::endl; print_extra_fields(fields); } else diff --git a/src/device/CMakeLists.txt b/src/device/CMakeLists.txt index 8f446f42a..91d670b73 100644 --- a/src/device/CMakeLists.txt +++ b/src/device/CMakeLists.txt @@ -44,6 +44,7 @@ set(device_headers device.hpp device_io.hpp device_default.hpp + device_cold.hpp log.hpp ) @@ -58,12 +59,6 @@ endif() set(device_private_headers) -if(PER_BLOCK_CHECKPOINT) - set(Blocks "blocks") -else() - set(Blocks "") -endif() - monero_private_headers(device ${device_private_headers}) @@ -78,6 +73,6 @@ target_link_libraries(device cncrypto ringct_basic ${OPENSSL_CRYPTO_LIBRARIES} + ${Boost_SERIALIZATION_LIBRARY} PRIVATE - ${Blocks} ${EXTRA_LIBRARIES}) diff --git a/src/device/device.hpp b/src/device/device.hpp index cb9117650..bdb608907 100644 --- a/src/device/device.hpp +++ b/src/device/device.hpp @@ -47,6 +47,7 @@ #include "crypto/crypto.h" #include "crypto/chacha.h" #include "ringct/rctTypes.h" +#include "cryptonote_config.h" #ifndef USE_DEVICE_LEDGER @@ -79,13 +80,21 @@ namespace hw { return false; } + class i_device_callback { + public: + virtual void on_button_request() {} + virtual void on_pin_request(epee::wipeable_string & pin) {} + virtual void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) {} + virtual ~i_device_callback() = default; + }; + class device { protected: std::string name; public: - device() {} + device(): mode(NONE) {} device(const device &hwdev) {} virtual ~device() {} @@ -99,10 +108,17 @@ namespace hw { enum device_type { SOFTWARE = 0, - LEDGER = 1 + LEDGER = 1, + TREZOR = 2 }; + enum device_protocol_t { + PROTOCOL_DEFAULT, + PROTOCOL_PROXY, // Originally defined by Ledger + PROTOCOL_COLD, // Originally defined by Trezor + }; + /* ======================================================================= */ /* SETUP/TEARDOWN */ /* ======================================================================= */ @@ -115,10 +131,14 @@ namespace hw { virtual bool connect(void) = 0; virtual bool disconnect(void) = 0; - virtual bool set_mode(device_mode mode) = 0; + virtual bool set_mode(device_mode mode) { this->mode = mode; return true; } + virtual device_mode get_mode() const { return mode; } virtual device_type get_type() const = 0; + virtual device_protocol_t device_protocol() const { return PROTOCOL_DEFAULT; }; + virtual void set_callback(i_device_callback * callback) {}; + virtual void set_derivation_path(const std::string &derivation_path) {}; /* ======================================================================= */ /* LOCKER */ @@ -188,8 +208,8 @@ namespace hw { return encrypt_payment_id(payment_id, public_key, secret_key); } - virtual bool ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & sharedSec) = 0; - virtual bool ecdhDecode(rct::ecdhTuple & masked, const rct::key & sharedSec) = 0; + virtual bool ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & sharedSec, bool short_amount) = 0; + virtual bool ecdhDecode(rct::ecdhTuple & masked, const rct::key & sharedSec, bool short_amount) = 0; virtual bool add_output_key_mapping(const crypto::public_key &Aout, const crypto::public_key &Bout, const bool is_subaddress, const size_t real_output_index, const rct::key &amount_key, const crypto::public_key &out_eph_public_key) = 0; @@ -202,6 +222,14 @@ namespace hw { virtual bool mlsag_sign(const rct::key &c, const rct::keyV &xx, const rct::keyV &alpha, const size_t rows, const size_t dsRows, rct::keyV &ss) = 0; virtual bool close_tx(void) = 0; + + virtual bool has_ki_cold_sync(void) const { return false; } + virtual bool has_tx_cold_sign(void) const { return false; } + + virtual void set_network_type(cryptonote::network_type network_type) { } + + protected: + device_mode mode; } ; struct reset_mode { diff --git a/src/device/device_cold.hpp b/src/device/device_cold.hpp new file mode 100644 index 000000000..22128cec1 --- /dev/null +++ b/src/device/device_cold.hpp @@ -0,0 +1,71 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef MONERO_DEVICE_COLD_H +#define MONERO_DEVICE_COLD_H + +#include "wallet/wallet2.h" +#include <boost/function.hpp> + + +namespace hw { + + typedef struct wallet_shim { + boost::function<crypto::public_key (const tools::wallet2::transfer_details &td)> get_tx_pub_key_from_received_outs; + } wallet_shim; + + class tx_aux_data { + public: + std::vector<std::string> tx_device_aux; // device generated aux data + std::vector<cryptonote::address_parse_info> tx_recipients; // as entered by user + }; + + class device_cold { + public: + + using exported_key_image = std::vector<std::pair<crypto::key_image, crypto::signature>>; + + /** + * Key image sync with the cold protocol. + */ + virtual void ki_sync(wallet_shim * wallet, + const std::vector<::tools::wallet2::transfer_details> & transfers, + exported_key_image & ski) =0; + + /** + * Signs unsigned transaction with the cold protocol. + */ + virtual void tx_sign(wallet_shim * wallet, + const ::tools::wallet2::unsigned_tx_set & unsigned_tx, + ::tools::wallet2::signed_tx_set & signed_tx, + tx_aux_data & aux_data) =0; + }; +} + +#endif //MONERO_DEVICE_COLD_H diff --git a/src/device/device_default.cpp b/src/device/device_default.cpp index a4f40e041..cb2f4e266 100644 --- a/src/device/device_default.cpp +++ b/src/device/device_default.cpp @@ -31,7 +31,7 @@ #include "device_default.hpp" -#include "common/int-util.h" +#include "int-util.h" #include "cryptonote_basic/account.h" #include "cryptonote_basic/subaddress_index.h" #include "ringct/rctOps.h" @@ -69,21 +69,21 @@ namespace hw { } bool device_default::init(void) { - dfns(); + return true; } bool device_default::release() { - dfns(); + return true; } bool device_default::connect(void) { - dfns(); + return true; } bool device_default::disconnect() { - dfns(); + return true; } bool device_default::set_mode(device_mode mode) { - return true; + return device::set_mode(mode); } /* ======================================================================= */ @@ -302,13 +302,13 @@ namespace hw { return true; } - bool device_default::ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & sharedSec) { - rct::ecdhEncode(unmasked, sharedSec); + bool device_default::ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & sharedSec, bool short_amount) { + rct::ecdhEncode(unmasked, sharedSec, short_amount); return true; } - bool device_default::ecdhDecode(rct::ecdhTuple & masked, const rct::key & sharedSec) { - rct::ecdhDecode(masked, sharedSec); + bool device_default::ecdhDecode(rct::ecdhTuple & masked, const rct::key & sharedSec, bool short_amount) { + rct::ecdhDecode(masked, sharedSec, short_amount); return true; } diff --git a/src/device/device_default.hpp b/src/device/device_default.hpp index 5c59a9066..54d159b11 100644 --- a/src/device/device_default.hpp +++ b/src/device/device_default.hpp @@ -111,8 +111,8 @@ namespace hw { bool encrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key) override; - bool ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & sharedSec) override; - bool ecdhDecode(rct::ecdhTuple & masked, const rct::key & sharedSec) override; + bool ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & sharedSec, bool short_amount) override; + bool ecdhDecode(rct::ecdhTuple & masked, const rct::key & sharedSec, bool short_amount) override; bool add_output_key_mapping(const crypto::public_key &Aout, const crypto::public_key &Bout, const bool is_subaddress, const size_t real_output_index, const rct::key &amount_key, const crypto::public_key &out_eph_public_key) override; diff --git a/src/device/device_io.hpp b/src/device/device_io.hpp index 96163a211..1d5e3564c 100644 --- a/src/device/device_io.hpp +++ b/src/device/device_io.hpp @@ -50,7 +50,7 @@ namespace hw { virtual void disconnect() = 0; virtual bool connected() const = 0; - virtual int exchange(unsigned char *command, unsigned int cmd_len, unsigned char *response, unsigned int max_resp_len) = 0; + virtual int exchange(unsigned char *command, unsigned int cmd_len, unsigned char *response, unsigned int max_resp_len, bool user_input) = 0; }; }; }; diff --git a/src/device/device_io_hid.cpp b/src/device/device_io_hid.cpp index 666255cb3..36c7a241b 100644 --- a/src/device/device_io_hid.cpp +++ b/src/device/device_io_hid.cpp @@ -1,3 +1,18 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be // used to endorse or promote products derived from this software without specific // prior written permission. // @@ -133,7 +148,7 @@ namespace hw { return this->usb_device != NULL; } - int device_io_hid::exchange(unsigned char *command, unsigned int cmd_len, unsigned char *response, unsigned int max_resp_len) { + int device_io_hid::exchange(unsigned char *command, unsigned int cmd_len, unsigned char *response, unsigned int max_resp_len, bool user_input) { unsigned char buffer[400]; unsigned char padding_buffer[MAX_BLOCK+1]; unsigned int result; @@ -162,7 +177,11 @@ namespace hw { //get first response memset(buffer, 0, sizeof(buffer)); - hid_ret = hid_read_timeout(this->usb_device, buffer, MAX_BLOCK, this->timeout); + if (!user_input) { + hid_ret = hid_read_timeout(this->usb_device, buffer, MAX_BLOCK, this->timeout); + } else { + hid_ret = hid_read(this->usb_device, buffer, MAX_BLOCK); + } ASSERT_X(hid_ret>=0, "Unable to read hidapi response. Error "+std::to_string(result)+": "+ safe_hid_error(this->usb_device)); result = (unsigned int)hid_ret; io_hid_log(1, buffer, result); diff --git a/src/device/device_io_hid.hpp b/src/device/device_io_hid.hpp index bb0f0a814..c47eefad2 100644 --- a/src/device/device_io_hid.hpp +++ b/src/device/device_io_hid.hpp @@ -100,7 +100,7 @@ namespace hw { void connect(void *params); void connect(unsigned int vid, unsigned int pid, boost::optional<int> interface_number, boost::optional<unsigned short> usage_page); bool connected() const; - int exchange(unsigned char *command, unsigned int cmd_len, unsigned char *response, unsigned int max_resp_len); + int exchange(unsigned char *command, unsigned int cmd_len, unsigned char *response, unsigned int max_resp_len, bool user_input); void disconnect(); void release(); }; diff --git a/src/device/device_ledger.cpp b/src/device/device_ledger.cpp index d879ee95a..9daf62d37 100644 --- a/src/device/device_ledger.cpp +++ b/src/device/device_ledger.cpp @@ -176,7 +176,7 @@ namespace hw { #define INS_GET_RESPONSE 0xc0 - device_ledger::device_ledger(): hw_device(0x0101, 0x05, 64, 120000) { + device_ledger::device_ledger(): hw_device(0x0101, 0x05, 64, 2000) { this->id = device_id++; this->reset_buffer(); this->mode = NONE; @@ -235,6 +235,9 @@ namespace hw { /* IO */ /* ======================================================================= */ + #define IO_SW_DENY 0x6982 + #define IO_SECRET_KEY 0x02 + void device_ledger::logCMD() { if (apdu_verbose) { char strbuffer[1024]; @@ -283,7 +286,12 @@ namespace hw { void device_ledger::send_simple(unsigned char ins, unsigned char p1) { this->length_send = set_command_header_noopt(ins, p1); - this->exchange(); + if (ins == INS_GET_KEY && p1 == IO_SECRET_KEY) { + // export view key user input + this->exchange_wait_on_input(); + } else { + this->exchange(); + } } bool device_ledger::reset() { @@ -294,7 +302,7 @@ namespace hw { unsigned int device_ledger::exchange(unsigned int ok, unsigned int mask) { logCMD(); - this->length_recv = hw_device.exchange(this->buffer_send, this->length_send, this->buffer_recv, BUFFER_SEND_SIZE); + this->length_recv = hw_device.exchange(this->buffer_send, this->length_send, this->buffer_recv, BUFFER_SEND_SIZE, false); ASSERT_X(this->length_recv>=2, "Communication error, less than tow bytes received"); this->length_recv -= 2; @@ -305,6 +313,25 @@ namespace hw { return this->sw; } + unsigned int device_ledger::exchange_wait_on_input(unsigned int ok, unsigned int mask) { + logCMD(); + unsigned int deny = 0; + this->length_recv = hw_device.exchange(this->buffer_send, this->length_send, this->buffer_recv, BUFFER_SEND_SIZE, true); + ASSERT_X(this->length_recv>=2, "Communication error, less than two bytes received"); + + this->length_recv -= 2; + this->sw = (this->buffer_recv[length_recv]<<8) | this->buffer_recv[length_recv+1]; + if (this->sw == IO_SW_DENY) { + // cancel on device + deny = 1; + } else { + ASSERT_SW(this->sw,ok,msk); + } + + logRESP(); + return deny; + } + void device_ledger::reset_buffer() { this->length_send = 0; memset(this->buffer_send, 0, BUFFER_SEND_SIZE); @@ -322,7 +349,7 @@ namespace hw { } const std::string device_ledger::get_name() const { - if (this->full_name.empty() || !this->connected()) { + if (!this->connected()) { return std::string("<disconnected:").append(this->name).append(">"); } return this->name; @@ -396,7 +423,7 @@ namespace hw { CHECK_AND_ASSERT_THROW_MES(false, " device_ledger::set_mode(unsigned int mode): invalid mode: "<<mode); } MDEBUG("Switch to mode: " <<mode); - return true; + return device::set_mode(mode); } @@ -789,8 +816,6 @@ namespace hw { } #ifdef DEBUG_HWDEVICE - bool recover_x = recover; - const crypto::secret_key recovery_key_x = recovery_key; crypto::public_key pub_x; crypto::secret_key sec_x; #endif @@ -1142,13 +1167,13 @@ namespace hw { return true; } - bool device_ledger::ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & AKout) { + bool device_ledger::ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & AKout, bool short_amount) { AUTO_LOCK_CMD(); #ifdef DEBUG_HWDEVICE const rct::key AKout_x = hw::ledger::decrypt(AKout); rct::ecdhTuple unmasked_x = unmasked; - this->controle_device->ecdhEncode(unmasked_x, AKout_x); + this->controle_device->ecdhEncode(unmasked_x, AKout_x, short_amount); #endif int offset = set_command_header_noopt(INS_BLIND); @@ -1179,13 +1204,13 @@ namespace hw { return true; } - bool device_ledger::ecdhDecode(rct::ecdhTuple & masked, const rct::key & AKout) { + bool device_ledger::ecdhDecode(rct::ecdhTuple & masked, const rct::key & AKout, bool short_amount) { AUTO_LOCK_CMD(); #ifdef DEBUG_HWDEVICE const rct::key AKout_x = hw::ledger::decrypt(AKout); rct::ecdhTuple masked_x = masked; - this->controle_device->ecdhDecode(masked_x, AKout_x); + this->controle_device->ecdhDecode(masked_x, AKout_x, short_amount); #endif int offset = set_command_header_noopt(INS_UNBLIND); @@ -1262,7 +1287,8 @@ namespace hw { this->buffer_send[4] = offset-5; this->length_send = offset; - this->exchange(); + // check fee user input + CHECK_AND_ASSERT_THROW_MES(this->exchange_wait_on_input() == 0, "Fee denied on device."); //pseudoOuts if (type == rct::RCTTypeSimple) { @@ -1330,7 +1356,8 @@ namespace hw { this->buffer_send[4] = offset-5; this->length_send = offset; - this->exchange(); + // check transaction user input + CHECK_AND_ASSERT_THROW_MES(this->exchange_wait_on_input() == 0, "Transaction denied on device."); #ifdef DEBUG_HWDEVICE hw::ledger::log_hexbuffer("Prehash AKV input", (char*)&this->buffer_recv[64], 3*32); #endif diff --git a/src/device/device_ledger.hpp b/src/device/device_ledger.hpp index dde69fbfd..584f1e096 100644 --- a/src/device/device_ledger.hpp +++ b/src/device/device_ledger.hpp @@ -85,7 +85,6 @@ namespace hw { //IO hw::io::device_io_hid hw_device; - std::string full_name; unsigned int length_send; unsigned char buffer_send[BUFFER_SEND_SIZE]; unsigned int length_recv; @@ -95,6 +94,7 @@ namespace hw { void logCMD(void); void logRESP(void); unsigned int exchange(unsigned int ok=0x9000, unsigned int mask=0xFFFF); + unsigned int exchange_wait_on_input(unsigned int ok=0x9000, unsigned int mask=0xFFFF); void reset_buffer(void); int set_command_header(unsigned char ins, unsigned char p1 = 0x00, unsigned char p2 = 0x00); int set_command_header_noopt(unsigned char ins, unsigned char p1 = 0x00, unsigned char p2 = 0x00); @@ -141,6 +141,7 @@ namespace hw { bool set_mode(device_mode mode) override; device_type get_type() const override {return device_type::LEDGER;}; + device_protocol_t device_protocol() const override { return PROTOCOL_PROXY; }; /* ======================================================================= */ /* LOCKER */ @@ -190,8 +191,8 @@ namespace hw { bool encrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key) override; - bool ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & sharedSec) override; - bool ecdhDecode(rct::ecdhTuple & masked, const rct::key & sharedSec) override; + bool ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & sharedSec, bool short_format) override; + bool ecdhDecode(rct::ecdhTuple & masked, const rct::key & sharedSec, bool short_format) override; bool add_output_key_mapping(const crypto::public_key &Aout, const crypto::public_key &Bout, const bool is_subaddress, const size_t real_output_index, const rct::key &amount_key, const crypto::public_key &out_eph_public_key) override; diff --git a/src/device_trezor/CMakeLists.txt b/src/device_trezor/CMakeLists.txt new file mode 100644 index 000000000..7f979389a --- /dev/null +++ b/src/device_trezor/CMakeLists.txt @@ -0,0 +1,97 @@ +# Copyright (c) 2014-2017, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set(TREZOR_PROTOB_H + trezor/messages/messages.pb.h + trezor/messages/messages-common.pb.h + trezor/messages/messages-management.pb.h + trezor/messages/messages-monero.pb.h +) + +set(TREZOR_PROTOB_CPP + trezor/messages/messages.pb.cc + trezor/messages/messages-common.pb.cc + trezor/messages/messages-management.pb.cc + trezor/messages/messages-monero.pb.cc +) + +set(trezor_headers + trezor/exceptions.hpp + trezor/messages_map.hpp + trezor/protocol.hpp + trezor/transport.hpp + device_trezor_base.hpp + device_trezor.hpp + trezor.hpp + ${TREZOR_PROTOB_H} +) + +set(trezor_sources + trezor/messages_map.cpp + trezor/protocol.cpp + trezor/transport.cpp + device_trezor_base.cpp + device_trezor.cpp + ${TREZOR_PROTOB_CPP} +) + +set(trezor_private_headers) + + +# Protobuf and LibUSB processed by CheckTrezor +if(DEVICE_TREZOR_READY) + message(STATUS "Trezor support enabled") + + monero_private_headers(device_trezor + ${device_private_headers}) + + monero_add_library(device_trezor + ${trezor_sources} + ${trezor_headers} + ${trezor_private_headers}) + + target_link_libraries(device_trezor + PUBLIC + device + cncrypto + ringct_basic + cryptonote_core + common + ${SODIUM_LIBRARY} + ${Boost_CHRONO_LIBRARY} + ${Protobuf_LIBRARY} + ${TREZOR_LIBUSB_LIBRARIES} + PRIVATE + ${EXTRA_LIBRARIES}) + +else() + message(STATUS "Trezor support disabled") + monero_private_headers(device_trezor) + monero_add_library(device_trezor device_trezor.cpp) + target_link_libraries(device_trezor PUBLIC cncrypto) +endif() diff --git a/src/device_trezor/device_trezor.cpp b/src/device_trezor/device_trezor.cpp new file mode 100644 index 000000000..8868fb995 --- /dev/null +++ b/src/device_trezor/device_trezor.cpp @@ -0,0 +1,420 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "device_trezor.hpp" + +namespace hw { +namespace trezor { + +#ifdef WITH_DEVICE_TREZOR + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "device.trezor" + +#define HW_TREZOR_NAME "Trezor" + + static device_trezor *trezor_device = nullptr; + static device_trezor *ensure_trezor_device(){ + if (!trezor_device) { + trezor_device = new device_trezor(); + trezor_device->set_name(HW_TREZOR_NAME); + } + return trezor_device; + } + + void register_all(std::map<std::string, std::unique_ptr<device>> ®istry) { + registry.insert(std::make_pair(HW_TREZOR_NAME, std::unique_ptr<device>(ensure_trezor_device()))); + } + + void register_all() { + hw::register_device(HW_TREZOR_NAME, ensure_trezor_device()); + } + + device_trezor::device_trezor() { + + } + + device_trezor::~device_trezor() { + try { + disconnect(); + release(); + } catch(std::exception const& e){ + MWARNING("Could not disconnect and release: " << e.what()); + } + } + + /* ======================================================================= */ + /* WALLET & ADDRESS */ + /* ======================================================================= */ + + bool device_trezor::get_public_address(cryptonote::account_public_address &pubkey) { + try { + auto res = get_address(); + + cryptonote::address_parse_info info{}; + bool r = cryptonote::get_account_address_from_str(info, this->network_type, res->address()); + CHECK_AND_ASSERT_MES(r, false, "Could not parse returned address. Address parse failed: " + res->address()); + CHECK_AND_ASSERT_MES(!info.is_subaddress, false, "Trezor returned a sub address"); + + pubkey = info.address; + return true; + + } catch(std::exception const& e){ + MERROR("Get public address exception: " << e.what()); + return false; + } + } + + bool device_trezor::get_secret_keys(crypto::secret_key &viewkey , crypto::secret_key &spendkey) { + try { + MDEBUG("Loading view-only key from the Trezor. Please check the Trezor for a confirmation."); + auto res = get_view_key(); + CHECK_AND_ASSERT_MES(res->watch_key().size() == 32, false, "Trezor returned invalid view key"); + + spendkey = crypto::null_skey; // not given + memcpy(viewkey.data, res->watch_key().data(), 32); + + return true; + + } catch(std::exception const& e){ + MERROR("Get secret keys exception: " << e.what()); + return false; + } + } + + /* ======================================================================= */ + /* Helpers */ + /* ======================================================================= */ + + /* ======================================================================= */ + /* TREZOR PROTOCOL */ + /* ======================================================================= */ + + std::shared_ptr<messages::monero::MoneroAddress> device_trezor::get_address( + const boost::optional<std::vector<uint32_t>> & path, + const boost::optional<cryptonote::network_type> & network_type){ + AUTO_LOCK_CMD(); + require_connected(); + device_state_reset_unsafe(); + require_initialized(); + + auto req = std::make_shared<messages::monero::MoneroGetAddress>(); + this->set_msg_addr<messages::monero::MoneroGetAddress>(req.get(), path, network_type); + + auto response = this->client_exchange<messages::monero::MoneroAddress>(req); + MTRACE("Get address response received"); + return response; + } + + std::shared_ptr<messages::monero::MoneroWatchKey> device_trezor::get_view_key( + const boost::optional<std::vector<uint32_t>> & path, + const boost::optional<cryptonote::network_type> & network_type){ + AUTO_LOCK_CMD(); + require_connected(); + device_state_reset_unsafe(); + require_initialized(); + + auto req = std::make_shared<messages::monero::MoneroGetWatchKey>(); + this->set_msg_addr<messages::monero::MoneroGetWatchKey>(req.get(), path, network_type); + + auto response = this->client_exchange<messages::monero::MoneroWatchKey>(req); + MTRACE("Get watch key response received"); + return response; + } + + void device_trezor::ki_sync(wallet_shim * wallet, + const std::vector<tools::wallet2::transfer_details> & transfers, + hw::device_cold::exported_key_image & ski) + { + AUTO_LOCK_CMD(); + require_connected(); + device_state_reset_unsafe(); + require_initialized(); + + std::shared_ptr<messages::monero::MoneroKeyImageExportInitRequest> req; + + std::vector<protocol::ki::MoneroTransferDetails> mtds; + std::vector<protocol::ki::MoneroExportedKeyImage> kis; + protocol::ki::key_image_data(wallet, transfers, mtds); + protocol::ki::generate_commitment(mtds, transfers, req); + + this->set_msg_addr<messages::monero::MoneroKeyImageExportInitRequest>(req.get()); + auto ack1 = this->client_exchange<messages::monero::MoneroKeyImageExportInitAck>(req); + + const auto batch_size = 10; + const auto num_batches = (mtds.size() + batch_size - 1) / batch_size; + for(uint64_t cur = 0; cur < num_batches; ++cur){ + auto step_req = std::make_shared<messages::monero::MoneroKeyImageSyncStepRequest>(); + auto idx_finish = std::min(static_cast<uint64_t>((cur + 1) * batch_size), static_cast<uint64_t>(mtds.size())); + for(uint64_t idx = cur * batch_size; idx < idx_finish; ++idx){ + auto added_tdis = step_req->add_tdis(); + CHECK_AND_ASSERT_THROW_MES(idx < mtds.size(), "Invalid transfer detail index"); + *added_tdis = mtds[idx]; + } + + auto step_ack = this->client_exchange<messages::monero::MoneroKeyImageSyncStepAck>(step_req); + auto kis_size = step_ack->kis_size(); + kis.reserve(static_cast<size_t>(kis_size)); + for(int i = 0; i < kis_size; ++i){ + auto ckis = step_ack->kis(i); + kis.push_back(ckis); + } + + MTRACE("Batch " << cur << " / " << num_batches << " batches processed"); + } + + auto final_req = std::make_shared<messages::monero::MoneroKeyImageSyncFinalRequest>(); + auto final_ack = this->client_exchange<messages::monero::MoneroKeyImageSyncFinalAck>(final_req); + ski.reserve(kis.size()); + + for(auto & sub : kis){ + char buff[32*3]; + protocol::crypto::chacha::decrypt(sub.blob().data(), sub.blob().size(), + reinterpret_cast<const uint8_t *>(final_ack->enc_key().data()), + reinterpret_cast<const uint8_t *>(sub.iv().data()), buff); + + ::crypto::signature sig{}; + ::crypto::key_image ki; + memcpy(ki.data, buff, 32); + memcpy(sig.c.data, buff + 32, 32); + memcpy(sig.r.data, buff + 64, 32); + ski.push_back(std::make_pair(ki, sig)); + } + } + + + void device_trezor::tx_sign(wallet_shim * wallet, + const tools::wallet2::unsigned_tx_set & unsigned_tx, + tools::wallet2::signed_tx_set & signed_tx, + hw::tx_aux_data & aux_data) + { + CHECK_AND_ASSERT_THROW_MES(unsigned_tx.transfers.first == 0, "Unsuported non zero offset"); + size_t num_tx = unsigned_tx.txes.size(); + signed_tx.key_images.clear(); + signed_tx.key_images.resize(unsigned_tx.transfers.second.size()); + + for(size_t tx_idx = 0; tx_idx < num_tx; ++tx_idx) { + std::shared_ptr<protocol::tx::Signer> signer; + tx_sign(wallet, unsigned_tx, tx_idx, aux_data, signer); + + auto & cdata = signer->tdata(); + auto aux_info_cur = signer->store_tx_aux_info(); + aux_data.tx_device_aux.emplace_back(aux_info_cur); + + // Pending tx reconstruction + signed_tx.ptx.emplace_back(); + auto & cpend = signed_tx.ptx.back(); + cpend.tx = cdata.tx; + cpend.dust = 0; + cpend.fee = 0; + cpend.dust_added_to_fee = false; + cpend.change_dts = cdata.tx_data.change_dts; + cpend.selected_transfers = cdata.tx_data.selected_transfers; + cpend.key_images = ""; + cpend.dests = cdata.tx_data.dests; + cpend.construction_data = cdata.tx_data; + + // Transaction check + try { + transaction_check(cdata, aux_data); + } catch(const std::exception &e){ + throw exc::ProtocolException(std::string("Transaction verification failed: ") + e.what()); + } + + std::string key_images; + bool all_are_txin_to_key = std::all_of(cdata.tx.vin.begin(), cdata.tx.vin.end(), [&](const cryptonote::txin_v& s_e) -> bool + { + CHECKED_GET_SPECIFIC_VARIANT(s_e, const cryptonote::txin_to_key, in, false); + key_images += boost::to_string(in.k_image) + " "; + return true; + }); + if(!all_are_txin_to_key) { + throw std::invalid_argument("Not all are txin_to_key"); + } + cpend.key_images = key_images; + + // KI sync + size_t num_sources = cdata.tx_data.sources.size(); + CHECK_AND_ASSERT_THROW_MES(num_sources == cdata.source_permutation.size(), "Invalid permutation size"); + CHECK_AND_ASSERT_THROW_MES(num_sources == cdata.tx.vin.size(), "Invalid tx.vin size"); + for(size_t src_idx = 0; src_idx < num_sources; ++src_idx){ + size_t idx_mapped = cdata.source_permutation[src_idx]; + CHECK_AND_ASSERT_THROW_MES(idx_mapped < cdata.tx_data.selected_transfers.size(), "Invalid idx_mapped"); + CHECK_AND_ASSERT_THROW_MES(src_idx < cdata.tx.vin.size(), "Invalid idx_mapped"); + + size_t idx_map_src = cdata.tx_data.selected_transfers[idx_mapped]; + auto vini = boost::get<cryptonote::txin_to_key>(cdata.tx.vin[src_idx]); + + CHECK_AND_ASSERT_THROW_MES(idx_map_src < signed_tx.key_images.size(), "Invalid key image index"); + signed_tx.key_images[idx_map_src] = vini.k_image; + } + } + } + + void device_trezor::tx_sign(wallet_shim * wallet, + const tools::wallet2::unsigned_tx_set & unsigned_tx, + size_t idx, + hw::tx_aux_data & aux_data, + std::shared_ptr<protocol::tx::Signer> & signer) + { + AUTO_LOCK_CMD(); + require_connected(); + device_state_reset_unsafe(); + require_initialized(); + + CHECK_AND_ASSERT_THROW_MES(idx < unsigned_tx.txes.size(), "Invalid transaction index"); + signer = std::make_shared<protocol::tx::Signer>(wallet, &unsigned_tx, idx, &aux_data); + const tools::wallet2::tx_construction_data & cur_tx = unsigned_tx.txes[idx]; + unsigned long num_sources = cur_tx.sources.size(); + unsigned long num_outputs = cur_tx.splitted_dsts.size(); + + // Step: Init + auto init_msg = signer->step_init(); + this->set_msg_addr(init_msg.get()); + transaction_pre_check(init_msg); + + auto response = this->client_exchange<messages::monero::MoneroTransactionInitAck>(init_msg); + signer->step_init_ack(response); + + // Step: Set transaction inputs + for(size_t cur_src = 0; cur_src < num_sources; ++cur_src){ + auto src = signer->step_set_input(cur_src); + auto ack = this->client_exchange<messages::monero::MoneroTransactionSetInputAck>(src); + signer->step_set_input_ack(ack); + } + + // Step: sort + auto perm_req = signer->step_permutation(); + if (perm_req){ + auto perm_ack = this->client_exchange<messages::monero::MoneroTransactionInputsPermutationAck>(perm_req); + signer->step_permutation_ack(perm_ack); + } + + // Step: input_vini + if (!signer->in_memory()){ + for(size_t cur_src = 0; cur_src < num_sources; ++cur_src){ + auto src = signer->step_set_vini_input(cur_src); + auto ack = this->client_exchange<messages::monero::MoneroTransactionInputViniAck>(src); + signer->step_set_vini_input_ack(ack); + } + } + + // Step: all inputs set + auto all_inputs_set = signer->step_all_inputs_set(); + auto ack_all_inputs = this->client_exchange<messages::monero::MoneroTransactionAllInputsSetAck>(all_inputs_set); + signer->step_all_inputs_set_ack(ack_all_inputs); + + // Step: outputs + for(size_t cur_dst = 0; cur_dst < num_outputs; ++cur_dst){ + auto src = signer->step_set_output(cur_dst); + auto ack = this->client_exchange<messages::monero::MoneroTransactionSetOutputAck>(src); + signer->step_set_output_ack(ack); + } + + // Step: all outs set + auto all_out_set = signer->step_all_outs_set(); + auto ack_all_out_set = this->client_exchange<messages::monero::MoneroTransactionAllOutSetAck>(all_out_set); + signer->step_all_outs_set_ack(ack_all_out_set, *this); + + // Step: sign each input + for(size_t cur_src = 0; cur_src < num_sources; ++cur_src){ + auto src = signer->step_sign_input(cur_src); + auto ack_sign = this->client_exchange<messages::monero::MoneroTransactionSignInputAck>(src); + signer->step_sign_input_ack(ack_sign); + } + + // Step: final + auto final_msg = signer->step_final(); + auto ack_final = this->client_exchange<messages::monero::MoneroTransactionFinalAck>(final_msg); + signer->step_final_ack(ack_final); + } + + void device_trezor::transaction_pre_check(std::shared_ptr<messages::monero::MoneroTransactionInitRequest> init_msg) + { + CHECK_AND_ASSERT_THROW_MES(init_msg, "TransactionInitRequest is empty"); + CHECK_AND_ASSERT_THROW_MES(init_msg->has_tsx_data(), "TransactionInitRequest has no transaction data"); + CHECK_AND_ASSERT_THROW_MES(m_features, "Device state not initialized"); // make sure the caller did not reset features + const bool nonce_required = init_msg->tsx_data().has_payment_id() && init_msg->tsx_data().payment_id().size() > 0; + + if (nonce_required){ + // Versions 2.0.9 and lower do not support payment ID + CHECK_AND_ASSERT_THROW_MES(m_features->has_major_version() && m_features->has_minor_version() && m_features->has_patch_version(), "Invalid Trezor firmware version information"); + const uint32_t vma = m_features->major_version(); + const uint32_t vmi = m_features->minor_version(); + const uint32_t vpa = m_features->patch_version(); + if (vma < 2 || (vma == 2 && vmi == 0 && vpa <= 9)) { + throw exc::TrezorException("Trezor firmware 2.0.9 and lower does not support transactions with short payment IDs or integrated addresses. Please update."); + } + } + } + + void device_trezor::transaction_check(const protocol::tx::TData & tdata, const hw::tx_aux_data & aux_data) + { + // Simple serialization check + cryptonote::blobdata tx_blob; + cryptonote::transaction tx_deserialized; + bool r = cryptonote::t_serializable_object_to_blob(tdata.tx, tx_blob); + CHECK_AND_ASSERT_THROW_MES(r, "Transaction serialization failed"); + r = cryptonote::parse_and_validate_tx_from_blob(tx_blob, tx_deserialized); + CHECK_AND_ASSERT_THROW_MES(r, "Transaction deserialization failed"); + + // Extras check + std::vector<cryptonote::tx_extra_field> tx_extra_fields; + cryptonote::tx_extra_nonce nonce; + + r = cryptonote::parse_tx_extra(tdata.tx.extra, tx_extra_fields); + CHECK_AND_ASSERT_THROW_MES(r, "tx.extra parsing failed"); + + const bool nonce_required = tdata.tsx_data.has_payment_id() && tdata.tsx_data.payment_id().size() > 0; + const bool has_nonce = cryptonote::find_tx_extra_field_by_type(tx_extra_fields, nonce); + CHECK_AND_ASSERT_THROW_MES(has_nonce == nonce_required, "Transaction nonce presence inconsistent"); + + if (nonce_required){ + const std::string & payment_id = tdata.tsx_data.payment_id(); + if (payment_id.size() == 32){ + crypto::hash payment_id_long{}; + CHECK_AND_ASSERT_THROW_MES(cryptonote::get_payment_id_from_tx_extra_nonce(nonce.nonce, payment_id_long), "Long payment ID not present"); + + } else if (payment_id.size() == 8){ + crypto::hash8 payment_id_short{}; + CHECK_AND_ASSERT_THROW_MES(cryptonote::get_encrypted_payment_id_from_tx_extra_nonce(nonce.nonce, payment_id_short), "Short payment ID not present"); + } + } + } + +#else //WITH_DEVICE_TREZOR + + void register_all(std::map<std::string, std::unique_ptr<device>> ®istry) { + } + + void register_all() { + } + +#endif //WITH_DEVICE_TREZOR +}} diff --git a/src/device_trezor/device_trezor.hpp b/src/device_trezor/device_trezor.hpp new file mode 100644 index 000000000..1f08be887 --- /dev/null +++ b/src/device_trezor/device_trezor.hpp @@ -0,0 +1,131 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef MONERO_DEVICE_TREZOR_H +#define MONERO_DEVICE_TREZOR_H + + +#include <cstddef> +#include <string> +#include "device/device.hpp" +#include "device/device_default.hpp" +#include "device/device_cold.hpp" +#include <boost/scope_exit.hpp> +#include <boost/thread/mutex.hpp> +#include <boost/thread/recursive_mutex.hpp> +#include "cryptonote_config.h" +#include "trezor.hpp" +#include "device_trezor_base.hpp" + +namespace hw { +namespace trezor { + + void register_all(); + void register_all(std::map<std::string, std::unique_ptr<device>> ®istry); + +#ifdef WITH_DEVICE_TREZOR + class device_trezor; + + /** + * Main device + */ + class device_trezor : public hw::trezor::device_trezor_base, public hw::device_cold { + protected: + void transaction_pre_check(std::shared_ptr<messages::monero::MoneroTransactionInitRequest> init_msg); + void transaction_check(const protocol::tx::TData & tdata, const hw::tx_aux_data & aux_data); + + public: + device_trezor(); + virtual ~device_trezor() override; + + device_trezor(const device_trezor &device) = delete ; + device_trezor& operator=(const device_trezor &device) = delete; + + explicit operator bool() const override {return true;} + + device_protocol_t device_protocol() const override { return PROTOCOL_COLD; }; + + bool has_ki_cold_sync() const override { return true; } + bool has_tx_cold_sign() const override { return true; } + void set_network_type(cryptonote::network_type network_type) override { this->network_type = network_type; } + + /* ======================================================================= */ + /* WALLET & ADDRESS */ + /* ======================================================================= */ + bool get_public_address(cryptonote::account_public_address &pubkey) override; + bool get_secret_keys(crypto::secret_key &viewkey , crypto::secret_key &spendkey) override; + + /* ======================================================================= */ + /* TREZOR PROTOCOL */ + /* ======================================================================= */ + + /** + * Get address. Throws. + */ + std::shared_ptr<messages::monero::MoneroAddress> get_address( + const boost::optional<std::vector<uint32_t>> & path = boost::none, + const boost::optional<cryptonote::network_type> & network_type = boost::none); + + /** + * Get watch key from device. Throws. + */ + std::shared_ptr<messages::monero::MoneroWatchKey> get_view_key( + const boost::optional<std::vector<uint32_t>> & path = boost::none, + const boost::optional<cryptonote::network_type> & network_type = boost::none); + + /** + * Key image sync with the Trezor. + */ + void ki_sync(wallet_shim * wallet, + const std::vector<::tools::wallet2::transfer_details> & transfers, + hw::device_cold::exported_key_image & ski) override; + + /** + * Signs particular transaction idx in the unsigned set, keeps state in the signer + */ + void tx_sign(wallet_shim * wallet, + const ::tools::wallet2::unsigned_tx_set & unsigned_tx, + size_t idx, + hw::tx_aux_data & aux_data, + std::shared_ptr<protocol::tx::Signer> & signer); + + /** + * Signs unsigned transaction with the Trezor. + */ + void tx_sign(wallet_shim * wallet, + const ::tools::wallet2::unsigned_tx_set & unsigned_tx, + ::tools::wallet2::signed_tx_set & signed_tx, + hw::tx_aux_data & aux_data) override; + }; + +#endif + +} +} +#endif //MONERO_DEVICE_TREZOR_H diff --git a/src/device_trezor/device_trezor_base.cpp b/src/device_trezor/device_trezor_base.cpp new file mode 100644 index 000000000..5071932ee --- /dev/null +++ b/src/device_trezor/device_trezor_base.cpp @@ -0,0 +1,425 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "device_trezor_base.hpp" +#include <boost/algorithm/string/classification.hpp> +#include <boost/algorithm/string/split.hpp> +#include <boost/regex.hpp> + +namespace hw { +namespace trezor { + +#ifdef WITH_DEVICE_TREZOR + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "device.trezor" +#define TREZOR_BIP44_HARDENED_ZERO 0x80000000 + + const uint32_t device_trezor_base::DEFAULT_BIP44_PATH[] = {0x8000002c, 0x80000080}; + + device_trezor_base::device_trezor_base(): m_callback(nullptr) { + + } + + device_trezor_base::~device_trezor_base() { + try { + disconnect(); + release(); + } catch(std::exception const& e){ + MERROR("Could not disconnect and release: " << e.what()); + } + } + + /* ======================================================================= */ + /* SETUP/TEARDOWN */ + /* ======================================================================= */ + + bool device_trezor_base::reset() { + return false; + } + + bool device_trezor_base::set_name(const std::string & name) { + this->m_full_name = name; + this->name = ""; + + auto delim = name.find(':'); + if (delim != std::string::npos && delim + 1 < name.length()) { + this->name = name.substr(delim + 1); + } + + return true; + } + + const std::string device_trezor_base::get_name() const { + if (this->m_full_name.empty()) { + return std::string("<disconnected:").append(this->name).append(">"); + } + return this->m_full_name; + } + + bool device_trezor_base::init() { + if (!release()){ + MERROR("Release failed"); + return false; + } + + return true; + } + + bool device_trezor_base::release() { + try { + disconnect(); + return true; + + } catch(std::exception const& e){ + MERROR("Release exception: " << e.what()); + return false; + } + } + + bool device_trezor_base::connect() { + disconnect(); + + // Enumerate all available devices + try { + hw::trezor::t_transport_vect trans; + + MDEBUG("Enumerating Trezor devices..."); + enumerate(trans); + + MDEBUG("Enumeration yielded " << trans.size() << " devices"); + for (auto &cur : trans) { + MDEBUG(" device: " << *(cur.get())); + std::string cur_path = cur->get_path(); + if (boost::starts_with(cur_path, this->name)) { + MDEBUG("Device Match: " << cur_path); + m_transport = cur; + break; + } + } + + if (!m_transport) { + MERROR("No matching Trezor device found. Device specifier: \"" + this->name + "\""); + return false; + } + + m_transport->open(); + return true; + + } catch(std::exception const& e){ + MERROR("Open exception: " << e.what()); + return false; + } + } + + bool device_trezor_base::disconnect() { + m_device_state.clear(); + m_features.reset(); + + if (m_transport){ + try { + m_transport->close(); + m_transport = nullptr; + + } catch(std::exception const& e){ + MERROR("Disconnect exception: " << e.what()); + m_transport = nullptr; + return false; + } + } + return true; + } + + /* ======================================================================= */ + /* LOCKER */ + /* ======================================================================= */ + + //lock the device for a long sequence + void device_trezor_base::lock() { + MTRACE("Ask for LOCKING for device " << this->name << " in thread "); + device_locker.lock(); + MTRACE("Device " << this->name << " LOCKed"); + } + + //lock the device for a long sequence + bool device_trezor_base::try_lock() { + MTRACE("Ask for LOCKING(try) for device " << this->name << " in thread "); + bool r = device_locker.try_lock(); + if (r) { + MTRACE("Device " << this->name << " LOCKed(try)"); + } else { + MDEBUG("Device " << this->name << " not LOCKed(try)"); + } + return r; + } + + //unlock the device + void device_trezor_base::unlock() { + MTRACE("Ask for UNLOCKING for device " << this->name << " in thread "); + device_locker.unlock(); + MTRACE("Device " << this->name << " UNLOCKed"); + } + + /* ======================================================================= */ + /* Helpers */ + /* ======================================================================= */ + + void device_trezor_base::require_connected(){ + if (!m_transport){ + throw exc::NotConnectedException(); + } + } + + void device_trezor_base::require_initialized(){ + if (!m_features){ + throw exc::TrezorException("Device state not initialized"); + } + + if (m_features->has_bootloader_mode() && m_features->bootloader_mode()){ + throw exc::TrezorException("Device is in the bootloader mode"); + } + + if (m_features->has_firmware_present() && !m_features->firmware_present()){ + throw exc::TrezorException("Device has no firmware loaded"); + } + + // Hard requirement on initialized field, has to be there. + if (!m_features->has_initialized() || !m_features->initialized()){ + throw exc::TrezorException("Device is not initialized"); + } + } + + void device_trezor_base::call_ping_unsafe(){ + auto pingMsg = std::make_shared<messages::management::Ping>(); + pingMsg->set_message("PING"); + + auto success = this->client_exchange<messages::common::Success>(pingMsg); // messages::MessageType_Success + MDEBUG("Ping response " << success->message()); + (void)success; + } + + void device_trezor_base::test_ping(){ + require_connected(); + + try { + this->call_ping_unsafe(); + + } catch(exc::TrezorException const& e){ + MINFO("Trezor does not respond: " << e.what()); + throw exc::DeviceNotResponsiveException(std::string("Trezor not responding: ") + e.what()); + } + } + + void device_trezor_base::write_raw(const google::protobuf::Message * msg){ + require_connected(); + CHECK_AND_ASSERT_THROW_MES(msg, "Empty message"); + this->get_transport()->write(*msg); + } + + GenericMessage device_trezor_base::read_raw(){ + require_connected(); + std::shared_ptr<google::protobuf::Message> msg_resp; + hw::trezor::messages::MessageType msg_resp_type; + + this->get_transport()->read(msg_resp, &msg_resp_type); + return GenericMessage(msg_resp_type, msg_resp); + } + + GenericMessage device_trezor_base::call_raw(const google::protobuf::Message * msg) { + write_raw(msg); + return read_raw(); + } + + bool device_trezor_base::message_handler(GenericMessage & input){ + // Later if needed this generic message handler can be replaced by a pointer to + // a protocol message handler which by default points to the device class which implements + // the default handler. + switch(input.m_type){ + case messages::MessageType_ButtonRequest: + on_button_request(input, dynamic_cast<const messages::common::ButtonRequest*>(input.m_msg.get())); + return true; + case messages::MessageType_PassphraseRequest: + on_passphrase_request(input, dynamic_cast<const messages::common::PassphraseRequest*>(input.m_msg.get())); + return true; + case messages::MessageType_PassphraseStateRequest: + on_passphrase_state_request(input, dynamic_cast<const messages::common::PassphraseStateRequest*>(input.m_msg.get())); + return true; + case messages::MessageType_PinMatrixRequest: + on_pin_request(input, dynamic_cast<const messages::common::PinMatrixRequest*>(input.m_msg.get())); + return true; + default: + return false; + } + } + + void device_trezor_base::ensure_derivation_path() noexcept { + if (m_wallet_deriv_path.empty()){ + m_wallet_deriv_path.push_back(TREZOR_BIP44_HARDENED_ZERO); // default 0' + } + } + + void device_trezor_base::set_derivation_path(const std::string &deriv_path){ + this->m_wallet_deriv_path.clear(); + + if (deriv_path.empty() || deriv_path == "-"){ + ensure_derivation_path(); + return; + } + + CHECK_AND_ASSERT_THROW_MES(deriv_path.size() <= 255, "Derivation path is too long"); + + std::vector<std::string> fields; + boost::split(fields, deriv_path, boost::is_any_of("/")); + CHECK_AND_ASSERT_THROW_MES(fields.size() <= 10, "Derivation path is too long"); + + boost::regex rgx("^([0-9]+)'?$"); + boost::cmatch match; + + this->m_wallet_deriv_path.reserve(fields.size()); + for(const std::string & cur : fields){ + const bool ok = boost::regex_match(cur.c_str(), match, rgx); + CHECK_AND_ASSERT_THROW_MES(ok, "Invalid wallet code: " << deriv_path << ". Invalid path element: " << cur); + CHECK_AND_ASSERT_THROW_MES(match[0].length() > 0, "Invalid wallet code: " << deriv_path << ". Invalid path element: " << cur); + + const unsigned long cidx = std::stoul(match[0].str()) | TREZOR_BIP44_HARDENED_ZERO; + this->m_wallet_deriv_path.push_back((unsigned int)cidx); + } + } + + /* ======================================================================= */ + /* TREZOR PROTOCOL */ + /* ======================================================================= */ + + bool device_trezor_base::ping() { + AUTO_LOCK_CMD(); + if (!m_transport){ + MINFO("Ping failed, device not connected"); + return false; + } + + try { + this->call_ping_unsafe(); + return true; + + } catch(std::exception const& e) { + MERROR("Ping failed, exception thrown " << e.what()); + } catch(...){ + MERROR("Ping failed, general exception thrown" << boost::current_exception_diagnostic_information()); + } + + return false; + } + + void device_trezor_base::device_state_reset_unsafe() + { + require_connected(); + auto initMsg = std::make_shared<messages::management::Initialize>(); + + if(!m_device_state.empty()) { + initMsg->set_allocated_state(&m_device_state); + } + + m_features = this->client_exchange<messages::management::Features>(initMsg); + initMsg->release_state(); + } + + void device_trezor_base::device_state_reset() + { + AUTO_LOCK_CMD(); + device_state_reset_unsafe(); + } + + void device_trezor_base::on_button_request(GenericMessage & resp, const messages::common::ButtonRequest * msg) + { + CHECK_AND_ASSERT_THROW_MES(msg, "Empty message"); + MDEBUG("on_button_request, code: " << msg->code()); + + messages::common::ButtonAck ack; + write_raw(&ack); + + if (m_callback){ + m_callback->on_button_request(); + } + + resp = read_raw(); + } + + void device_trezor_base::on_pin_request(GenericMessage & resp, const messages::common::PinMatrixRequest * msg) + { + MDEBUG("on_pin_request"); + CHECK_AND_ASSERT_THROW_MES(msg, "Empty message"); + + epee::wipeable_string pin; + + if (m_callback){ + m_callback->on_pin_request(pin); + } + + // TODO: remove PIN from memory + messages::common::PinMatrixAck m; + m.set_pin(pin.data(), pin.size()); + resp = call_raw(&m); + } + + void device_trezor_base::on_passphrase_request(GenericMessage & resp, const messages::common::PassphraseRequest * msg) + { + CHECK_AND_ASSERT_THROW_MES(msg, "Empty message"); + MDEBUG("on_passhprase_request, on device: " << msg->on_device()); + epee::wipeable_string passphrase; + + if (m_callback){ + m_callback->on_passphrase_request(msg->on_device(), passphrase); + } + + messages::common::PassphraseAck m; + if (!msg->on_device()){ + // TODO: remove passphrase from memory + m.set_passphrase(passphrase.data(), passphrase.size()); + } + + if (!m_device_state.empty()){ + m.set_allocated_state(&m_device_state); + } + + resp = call_raw(&m); + m.release_state(); + } + + void device_trezor_base::on_passphrase_state_request(GenericMessage & resp, const messages::common::PassphraseStateRequest * msg) + { + MDEBUG("on_passhprase_state_request"); + CHECK_AND_ASSERT_THROW_MES(msg, "Empty message"); + + m_device_state = msg->state(); + messages::common::PassphraseStateAck m; + resp = call_raw(&m); + } + +#endif //WITH_DEVICE_TREZOR +}} diff --git a/src/device_trezor/device_trezor_base.hpp b/src/device_trezor/device_trezor_base.hpp new file mode 100644 index 000000000..3c35e8aca --- /dev/null +++ b/src/device_trezor/device_trezor_base.hpp @@ -0,0 +1,277 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef MONERO_DEVICE_TREZOR_BASE_H +#define MONERO_DEVICE_TREZOR_BASE_H + + +#include <cstddef> +#include <string> +#include "device/device.hpp" +#include "device/device_default.hpp" +#include "device/device_cold.hpp" +#include <boost/scope_exit.hpp> +#include <boost/thread/mutex.hpp> +#include <boost/thread/recursive_mutex.hpp> +#include "cryptonote_config.h" +#include "trezor.hpp" + +//automatic lock one more level on device ensuring the current thread is allowed to use it +#define AUTO_LOCK_CMD() \ + /* lock both mutexes without deadlock*/ \ + boost::lock(device_locker, command_locker); \ + /* make sure both already-locked mutexes are unlocked at the end of scope */ \ + boost::lock_guard<boost::recursive_mutex> lock1(device_locker, boost::adopt_lock); \ + boost::lock_guard<boost::mutex> lock2(command_locker, boost::adopt_lock) + + +namespace hw { +namespace trezor { + +#ifdef WITH_DEVICE_TREZOR + class device_trezor_base; + + /** + * TREZOR device template with basic functions + */ + class device_trezor_base : public hw::core::device_default { + protected: + + // Locker for concurrent access + mutable boost::recursive_mutex device_locker; + mutable boost::mutex command_locker; + + std::shared_ptr<Transport> m_transport; + i_device_callback * m_callback; + + std::string m_full_name; + std::vector<unsigned int> m_wallet_deriv_path; + std::string m_device_state; // returned after passphrase entry, session + std::shared_ptr<messages::management::Features> m_features; // features from the last device reset + + cryptonote::network_type network_type; + + // + // Internal methods + // + + void require_connected(); + void require_initialized(); + void call_ping_unsafe(); + void test_ping(); + void device_state_reset_unsafe(); + void ensure_derivation_path() noexcept; + + // Communication methods + + void write_raw(const google::protobuf::Message * msg); + GenericMessage read_raw(); + GenericMessage call_raw(const google::protobuf::Message * msg); + + // Trezor message protocol handler. Handles specific signalling messages. + bool message_handler(GenericMessage & input); + + /** + * Client communication wrapper, handles specific Trezor protocol. + * + * @throws UnexpectedMessageException if the response message type is different than expected. + * Exception contains message type and the message itself. + */ + template<class t_message> + std::shared_ptr<t_message> + client_exchange(const std::shared_ptr<const google::protobuf::Message> &req, + const boost::optional<messages::MessageType> & resp_type = boost::none, + const boost::optional<std::vector<messages::MessageType>> & resp_types = boost::none, + const boost::optional<messages::MessageType*> & resp_type_ptr = boost::none, + bool open_session = false) + { + // Require strictly protocol buffers response in the template. + BOOST_STATIC_ASSERT(boost::is_base_of<google::protobuf::Message, t_message>::value); + const bool accepting_base = boost::is_same<google::protobuf::Message, t_message>::value; + if (resp_types && !accepting_base){ + throw std::invalid_argument("Cannot specify list of accepted types and not using generic response"); + } + + // Determine type of expected message response + const messages::MessageType required_type = accepting_base ? messages::MessageType_Success : + (resp_type ? resp_type.get() : MessageMapper::get_message_wire_number<t_message>()); + + // Open session if required + if (open_session){ + try { + m_transport->open(); + } catch (const std::exception& e) { + std::throw_with_nested(exc::SessionException("Could not open session")); + } + } + + // Scoped session closer + BOOST_SCOPE_EXIT_ALL(&, this) { + if (open_session){ + this->get_transport()->close(); + } + }; + + // Write/read the request + CHECK_AND_ASSERT_THROW_MES(req, "Request is null"); + auto msg_resp = call_raw(req.get()); + + bool processed = false; + do { + processed = message_handler(msg_resp); + } while(processed); + + // Response section + if (resp_type_ptr){ + *(resp_type_ptr.get()) = msg_resp.m_type; + } + + if (msg_resp.m_type == messages::MessageType_Failure) { + throw_failure_exception(dynamic_cast<messages::common::Failure *>(msg_resp.m_msg.get())); + + } else if (!accepting_base && msg_resp.m_type == required_type) { + return message_ptr_retype<t_message>(msg_resp.m_msg); + + } else if (accepting_base && (!resp_types || + std::find(resp_types.get().begin(), resp_types.get().end(), msg_resp.m_type) != resp_types.get().end())) { + return message_ptr_retype<t_message>(msg_resp.m_msg); + + } else { + throw exc::UnexpectedMessageException(msg_resp.m_type, msg_resp.m_msg); + } + } + + /** + * Utility method to set address_n and network type to the message requets. + */ + template<class t_message> + void set_msg_addr(t_message * msg, + const boost::optional<std::vector<uint32_t>> & path = boost::none, + const boost::optional<cryptonote::network_type> & network_type = boost::none) + { + CHECK_AND_ASSERT_THROW_MES(msg, "Message is null"); + msg->clear_address_n(); + if (path){ + for(auto x : path.get()){ + msg->add_address_n(x); + } + } else { + ensure_derivation_path(); + for (unsigned int i : DEFAULT_BIP44_PATH) { + msg->add_address_n(i); + } + for (unsigned int i : m_wallet_deriv_path) { + msg->add_address_n(i); + } + } + + if (network_type){ + msg->set_network_type(static_cast<uint32_t>(network_type.get())); + } else { + msg->set_network_type(static_cast<uint32_t>(this->network_type)); + } + } + + public: + device_trezor_base(); + ~device_trezor_base() override; + + device_trezor_base(const device_trezor_base &device) = delete ; + device_trezor_base& operator=(const device_trezor_base &device) = delete; + + explicit operator bool() const override {return true;} + device_type get_type() const override {return device_type::TREZOR;}; + + bool reset(); + + // Default derivation path for Monero + static const uint32_t DEFAULT_BIP44_PATH[2]; + + std::shared_ptr<Transport> get_transport(){ + return m_transport; + } + + void set_callback(i_device_callback * callback) override { + m_callback = callback; + } + + i_device_callback * get_callback(){ + return m_callback; + } + + std::shared_ptr<messages::management::Features> & get_features() { + return m_features; + } + + void set_derivation_path(const std::string &deriv_path) override; + + /* ======================================================================= */ + /* SETUP/TEARDOWN */ + /* ======================================================================= */ + bool set_name(const std::string &name) override; + + const std::string get_name() const override; + bool init() override; + bool release() override; + bool connect() override; + bool disconnect() override; + + /* ======================================================================= */ + /* LOCKER */ + /* ======================================================================= */ + void lock() override; + void unlock() override; + bool try_lock() override; + + /* ======================================================================= */ + /* TREZOR PROTOCOL */ + /* ======================================================================= */ + + /** + * Device ping, no-throw + */ + bool ping(); + + /** + * Performs Initialize call to the Trezor, resets to known state. + */ + void device_state_reset(); + + // Protocol callbacks + void on_button_request(GenericMessage & resp, const messages::common::ButtonRequest * msg); + void on_pin_request(GenericMessage & resp, const messages::common::PinMatrixRequest * msg); + void on_passphrase_request(GenericMessage & resp, const messages::common::PassphraseRequest * msg); + void on_passphrase_state_request(GenericMessage & resp, const messages::common::PassphraseStateRequest * msg); + }; + +#endif + +} +} +#endif //MONERO_DEVICE_TREZOR_BASE_H diff --git a/src/device_trezor/trezor.hpp b/src/device_trezor/trezor.hpp new file mode 100644 index 000000000..97dc0a957 --- /dev/null +++ b/src/device_trezor/trezor.hpp @@ -0,0 +1,44 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef MONERO_TREZOR_HPP +#define MONERO_TREZOR_HPP + +#include "trezor/trezor_defs.hpp" + +#ifdef WITH_DEVICE_TREZOR +#include "trezor/transport.hpp" +#include "trezor/messages/messages.pb.h" +#include "trezor/messages/messages-common.pb.h" +#include "trezor/messages/messages-management.pb.h" +#include "trezor/messages/messages-monero.pb.h" +#include "trezor/protocol.hpp" +#endif + +#endif //MONERO_TREZOR_HPP diff --git a/src/device_trezor/trezor/exceptions.hpp b/src/device_trezor/trezor/exceptions.hpp new file mode 100644 index 000000000..197dc43a4 --- /dev/null +++ b/src/device_trezor/trezor/exceptions.hpp @@ -0,0 +1,193 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef MONERO_EXCEPTIONS_H +#define MONERO_EXCEPTIONS_H + +#include <exception> +#include <string> +#include <boost/optional.hpp> + +namespace hw { +namespace trezor { +namespace exc { + + class SecurityException : public std::exception { + protected: + boost::optional<std::string> reason; + + public: + SecurityException(): reason("General Security exception"){} + explicit SecurityException(std::string what): reason(what){} + + virtual const char* what() const throw() { + return reason.get().c_str(); + } + }; + + class Poly1305TagInvalid: public SecurityException { + public: + using SecurityException::SecurityException; + Poly1305TagInvalid(): SecurityException("Poly1305 authentication tag invalid"){} + }; + + class TrezorException : public std::exception { + protected: + boost::optional<std::string> reason; + + public: + TrezorException(): reason("General Trezor exception"){} + explicit TrezorException(std::string what): reason(what){} + + virtual const char* what() const throw() { + return reason.get().c_str(); + } + }; + + class CommunicationException: public TrezorException { + public: + using TrezorException::TrezorException; + CommunicationException(): TrezorException("Trezor communication error"){} + }; + + class EncodingException: public CommunicationException { + public: + using CommunicationException::CommunicationException; + EncodingException(): CommunicationException("Trezor message encoding error"){} + }; + + class NotConnectedException : public CommunicationException { + public: + using CommunicationException::CommunicationException; + NotConnectedException(): CommunicationException("Trezor not connected"){} + }; + + class DeviceNotResponsiveException : public CommunicationException { + public: + using CommunicationException::CommunicationException; + DeviceNotResponsiveException(): CommunicationException("Trezor does not respond to ping"){} + }; + + class DeviceAcquireException : public CommunicationException { + public: + using CommunicationException::CommunicationException; + DeviceAcquireException(): CommunicationException("Trezor could not be acquired"){} + }; + + class SessionException: public CommunicationException { + public: + using CommunicationException::CommunicationException; + SessionException(): CommunicationException("Trezor session expired"){} + }; + + class TimeoutException: public CommunicationException { + public: + using CommunicationException::CommunicationException; + TimeoutException(): CommunicationException("Trezor communication timeout"){} + }; + + class ProtocolException: public CommunicationException { + public: + using CommunicationException::CommunicationException; + ProtocolException(): CommunicationException("Trezor protocol error"){} + }; + + // Communication protocol namespace + // Separated to distinguish between client and Trezor side exceptions. +namespace proto { + + class SecurityException : public ProtocolException { + public: + using ProtocolException::ProtocolException; + SecurityException(): ProtocolException("Security assertion violated in the protocol"){} + }; + + class FailureException : public ProtocolException { + private: + boost::optional<uint32_t> code; + boost::optional<std::string> message; + public: + using ProtocolException::ProtocolException; + FailureException(): ProtocolException("Trezor returned failure"){} + FailureException(boost::optional<uint32_t> code, + boost::optional<std::string> message) + : code(code), message(message) { + reason = "Trezor returned failure: code=" + + (code ? std::to_string(code.get()) : "") + + ", message=" + (message ? message.get() : ""); + }; + }; + + class UnexpectedMessageException : public FailureException { + public: + using FailureException::FailureException; + UnexpectedMessageException(): FailureException("Trezor claims unexpected message received"){} + }; + + class CancelledException : public FailureException { + public: + using FailureException::FailureException; + CancelledException(): FailureException("Trezor returned: cancelled operation"){} + }; + + class PinExpectedException : public FailureException { + public: + using FailureException::FailureException; + PinExpectedException(): FailureException("Trezor claims PIN is expected"){} + }; + + class InvalidPinException : public FailureException { + public: + using FailureException::FailureException; + InvalidPinException(): FailureException("Trezor claims PIN is invalid"){} + }; + + class NotEnoughFundsException : public FailureException { + public: + using FailureException::FailureException; + NotEnoughFundsException(): FailureException("Trezor claims not enough funds"){} + }; + + class NotInitializedException : public FailureException { + public: + using FailureException::FailureException; + NotInitializedException(): FailureException("Trezor claims not initialized"){} + }; + + class FirmwareErrorException : public FailureException { + public: + using FailureException::FailureException; + FirmwareErrorException(): FailureException("Trezor returned firmware error"){} + }; + +} +} +} +} +#endif //MONERO_EXCEPTIONS_H diff --git a/src/device_trezor/trezor/messages/.gitignore b/src/device_trezor/trezor/messages/.gitignore new file mode 100644 index 000000000..32f7a77e5 --- /dev/null +++ b/src/device_trezor/trezor/messages/.gitignore @@ -0,0 +1,2 @@ +# protobuf generated code +* diff --git a/src/device_trezor/trezor/messages_map.cpp b/src/device_trezor/trezor/messages_map.cpp new file mode 100644 index 000000000..b0d1aa254 --- /dev/null +++ b/src/device_trezor/trezor/messages_map.cpp @@ -0,0 +1,125 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "messages_map.hpp" +#include "messages/messages.pb.h" +#include "messages/messages-common.pb.h" +#include "messages/messages-management.pb.h" +#include "messages/messages-monero.pb.h" + +using namespace std; +using namespace hw::trezor; + +namespace hw{ +namespace trezor +{ + + const char * TYPE_PREFIX = "MessageType_"; + const char * PACKAGES[] = { + "hw.trezor.messages.", + "hw.trezor.messages.common.", + "hw.trezor.messages.management.", + "hw.trezor.messages.monero." + }; + + google::protobuf::Message * MessageMapper::get_message(int wire_number) { + return MessageMapper::get_message(static_cast<messages::MessageType>(wire_number)); + } + + google::protobuf::Message * MessageMapper::get_message(messages::MessageType wire_number) { + const string &messageTypeName = hw::trezor::messages::MessageType_Name(wire_number); + if (messageTypeName.empty()) { + throw exc::EncodingException(std::string("Message descriptor not found: ") + std::to_string(wire_number)); + } + + string messageName = messageTypeName.substr(strlen(TYPE_PREFIX)); + return MessageMapper::get_message(messageName); + } + + google::protobuf::Message * MessageMapper::get_message(const std::string & msg_name) { + // Each package instantiation so lookup works + hw::trezor::messages::common::Success::default_instance(); + hw::trezor::messages::management::Cancel::default_instance(); + hw::trezor::messages::monero::MoneroGetAddress::default_instance(); + + google::protobuf::Descriptor const * desc = nullptr; + for(const string &text : PACKAGES){ + desc = google::protobuf::DescriptorPool::generated_pool() + ->FindMessageTypeByName(text + msg_name); + if (desc != nullptr){ + break; + } + } + + if (desc == nullptr){ + throw exc::EncodingException(std::string("Message not found: ") + msg_name); + } + + google::protobuf::Message* message = + google::protobuf::MessageFactory::generated_factory() + ->GetPrototype(desc)->New(); + + return message; + +// // CODEGEN way, fast +// switch(wire_number){ +// case 501: +// return new messages::monero::MoneroTransactionSignRequest(); +// default: +// throw std::runtime_error("not implemented"); +// } +// +// // CODEGEN message -> number: specification +// // messages::MessageType get_message_wire_number(const messages::monero::MoneroTransactionSignRequest * msg) { return 501; } +// // messages::MessageType get_message_wire_number(const messages::management::ping * msg) +// + } + + messages::MessageType MessageMapper::get_message_wire_number(const google::protobuf::Message * msg){ + return MessageMapper::get_message_wire_number(msg->GetDescriptor()->name()); + } + + messages::MessageType MessageMapper::get_message_wire_number(const google::protobuf::Message & msg){ + return MessageMapper::get_message_wire_number(msg.GetDescriptor()->name()); + } + + messages::MessageType MessageMapper::get_message_wire_number(const std::string & msg_name){ + string enumMessageName = std::string(TYPE_PREFIX) + msg_name; + + messages::MessageType res; + bool r = hw::trezor::messages::MessageType_Parse(enumMessageName, &res); + if (!r){ + throw exc::EncodingException(std::string("Message ") + msg_name + " not found"); + } + + return res; + } + +} +} diff --git a/src/device_trezor/trezor/messages_map.hpp b/src/device_trezor/trezor/messages_map.hpp new file mode 100644 index 000000000..f61338f09 --- /dev/null +++ b/src/device_trezor/trezor/messages_map.hpp @@ -0,0 +1,94 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef MONERO_MESSAGES_MAP_H +#define MONERO_MESSAGES_MAP_H + +#include <string> +#include <type_traits> +#include <memory> +#include "exceptions.hpp" + +#include "trezor_defs.hpp" + +#include <google/protobuf/stubs/common.h> +#include <google/protobuf/generated_message_util.h> +#include <google/protobuf/repeated_field.h> +#include <google/protobuf/extension_set.h> +#include <google/protobuf/generated_enum_reflection.h> +#include "google/protobuf/descriptor.pb.h" + +#include "messages/messages.pb.h" + +namespace hw { +namespace trezor { + + class MessageMapper{ + public: + MessageMapper() { + + } + + static ::google::protobuf::Message * get_message(int wire_number); + static ::google::protobuf::Message * get_message(messages::MessageType); + static ::google::protobuf::Message * get_message(const std::string & msg_name); + static messages::MessageType get_message_wire_number(const google::protobuf::Message * msg); + static messages::MessageType get_message_wire_number(const google::protobuf::Message & msg); + static messages::MessageType get_message_wire_number(const std::string & msg_name); + + template<class t_message> + static messages::MessageType get_message_wire_number() { + BOOST_STATIC_ASSERT(boost::is_base_of<google::protobuf::Message, t_message>::value); + return get_message_wire_number(t_message::default_instance().GetDescriptor()->name()); + } + }; + + template<class t_message> + std::shared_ptr<t_message> message_ptr_retype(std::shared_ptr<google::protobuf::Message> & in){ + BOOST_STATIC_ASSERT(boost::is_base_of<google::protobuf::Message, t_message>::value); + if (!in){ + return nullptr; + } + + return std::dynamic_pointer_cast<t_message>(in); + } + + template<class t_message> + std::shared_ptr<t_message> message_ptr_retype_static(std::shared_ptr<google::protobuf::Message> & in){ + BOOST_STATIC_ASSERT(boost::is_base_of<google::protobuf::Message, t_message>::value); + if (!in){ + return nullptr; + } + + return std::static_pointer_cast<t_message>(in); + } + +}} + +#endif //MONERO_MESSAGES_MAP_H diff --git a/src/device_trezor/trezor/protocol.cpp b/src/device_trezor/trezor/protocol.cpp new file mode 100644 index 000000000..13506a67f --- /dev/null +++ b/src/device_trezor/trezor/protocol.cpp @@ -0,0 +1,894 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "protocol.hpp" +#include <unordered_map> +#include <set> +#include <utility> +#include <boost/endian/conversion.hpp> +#include <common/apply_permutation.h> +#include <ringct/rctSigs.h> +#include <ringct/bulletproofs.h> +#include "cryptonote_config.h" +#include <sodium.h> +#include <sodium/crypto_verify_32.h> +#include <sodium/crypto_aead_chacha20poly1305.h> + +namespace hw{ +namespace trezor{ +namespace protocol{ + + std::string key_to_string(const ::crypto::ec_point & key){ + return std::string(key.data, sizeof(key.data)); + } + + std::string key_to_string(const ::crypto::ec_scalar & key){ + return std::string(key.data, sizeof(key.data)); + } + + std::string key_to_string(const ::crypto::hash & key){ + return std::string(key.data, sizeof(key.data)); + } + + std::string key_to_string(const ::rct::key & key){ + return std::string(reinterpret_cast<const char*>(key.bytes), sizeof(key.bytes)); + } + + void string_to_key(::crypto::ec_scalar & key, const std::string & str){ + if (str.size() != sizeof(key.data)){ + throw std::invalid_argument(std::string("Key has to have ") + std::to_string(sizeof(key.data)) + " B"); + } + memcpy(key.data, str.data(), sizeof(key.data)); + } + + void string_to_key(::crypto::ec_point & key, const std::string & str){ + if (str.size() != sizeof(key.data)){ + throw std::invalid_argument(std::string("Key has to have ") + std::to_string(sizeof(key.data)) + " B"); + } + memcpy(key.data, str.data(), sizeof(key.data)); + } + + void string_to_key(::rct::key & key, const std::string & str){ + if (str.size() != sizeof(key.bytes)){ + throw std::invalid_argument(std::string("Key has to have ") + std::to_string(sizeof(key.bytes)) + " B"); + } + memcpy(key.bytes, str.data(), sizeof(key.bytes)); + } + +namespace crypto { +namespace chacha { + + void decrypt(const void* ciphertext, size_t length, const uint8_t* key, const uint8_t* iv, char* plaintext){ + if (length < 16){ + throw std::invalid_argument("Ciphertext length too small"); + } + + unsigned long long int cip_len = length; + auto r = crypto_aead_chacha20poly1305_ietf_decrypt( + reinterpret_cast<unsigned char *>(plaintext), &cip_len, nullptr, + static_cast<const unsigned char *>(ciphertext), length, nullptr, 0, iv, key); + + if (r != 0){ + throw exc::Poly1305TagInvalid(); + } + } + +} +} + + +// Cold Key image sync +namespace ki { + + bool key_image_data(wallet_shim * wallet, + const std::vector<tools::wallet2::transfer_details> & transfers, + std::vector<MoneroTransferDetails> & res) + { + for(auto & td : transfers){ + ::crypto::public_key tx_pub_key = wallet->get_tx_pub_key_from_received_outs(td); + const std::vector<::crypto::public_key> additional_tx_pub_keys = cryptonote::get_additional_tx_pub_keys_from_extra(td.m_tx); + + res.emplace_back(); + auto & cres = res.back(); + + cres.set_out_key(key_to_string(boost::get<cryptonote::txout_to_key>(td.m_tx.vout[td.m_internal_output_index].target).key)); + cres.set_tx_pub_key(key_to_string(tx_pub_key)); + cres.set_internal_output_index(td.m_internal_output_index); + for(auto & aux : additional_tx_pub_keys){ + cres.add_additional_tx_pub_keys(key_to_string(aux)); + } + } + + return true; + } + + std::string compute_hash(const MoneroTransferDetails & rr){ + KECCAK_CTX kck; + uint8_t md[32]; + + CHECK_AND_ASSERT_THROW_MES(rr.out_key().size() == 32, "Invalid out_key size"); + CHECK_AND_ASSERT_THROW_MES(rr.tx_pub_key().size() == 32, "Invalid tx_pub_key size"); + + keccak_init(&kck); + keccak_update(&kck, reinterpret_cast<const uint8_t *>(rr.out_key().data()), 32); + keccak_update(&kck, reinterpret_cast<const uint8_t *>(rr.tx_pub_key().data()), 32); + for (const auto &aux : rr.additional_tx_pub_keys()){ + CHECK_AND_ASSERT_THROW_MES(aux.size() == 32, "Invalid aux size"); + keccak_update(&kck, reinterpret_cast<const uint8_t *>(aux.data()), 32); + } + + auto index_serialized = tools::get_varint_data(rr.internal_output_index()); + keccak_update(&kck, reinterpret_cast<const uint8_t *>(index_serialized.data()), index_serialized.size()); + keccak_finish(&kck, md); + return std::string(reinterpret_cast<const char*>(md), sizeof(md)); + } + + void generate_commitment(std::vector<MoneroTransferDetails> & mtds, + const std::vector<tools::wallet2::transfer_details> & transfers, + std::shared_ptr<messages::monero::MoneroKeyImageExportInitRequest> & req) + { + req = std::make_shared<messages::monero::MoneroKeyImageExportInitRequest>(); + + KECCAK_CTX kck; + uint8_t final_hash[32]; + keccak_init(&kck); + + for(auto &cur : mtds){ + auto hash = compute_hash(cur); + keccak_update(&kck, reinterpret_cast<const uint8_t *>(hash.data()), hash.size()); + } + keccak_finish(&kck, final_hash); + + req = std::make_shared<messages::monero::MoneroKeyImageExportInitRequest>(); + req->set_hash(std::string(reinterpret_cast<const char*>(final_hash), 32)); + req->set_num(transfers.size()); + + std::unordered_map<uint32_t, std::set<uint32_t>> sub_indices; + for (auto &cur : transfers){ + auto search = sub_indices.emplace(cur.m_subaddr_index.major, std::set<uint32_t>()); + auto & st = search.first->second; + st.insert(cur.m_subaddr_index.minor); + } + + for (auto& x: sub_indices){ + auto subs = req->add_subs(); + subs->set_account(x.first); + for(auto minor : x.second){ + subs->add_minor_indices(minor); + } + } + } + +} + +// Cold transaction signing +namespace tx { + + void translate_address(MoneroAccountPublicAddress * dst, const cryptonote::account_public_address * src){ + dst->set_view_public_key(key_to_string(src->m_view_public_key)); + dst->set_spend_public_key(key_to_string(src->m_spend_public_key)); + } + + void translate_dst_entry(MoneroTransactionDestinationEntry * dst, const cryptonote::tx_destination_entry * src){ + dst->set_amount(src->amount); + dst->set_is_subaddress(src->is_subaddress); + translate_address(dst->mutable_addr(), &(src->addr)); + } + + void translate_src_entry(MoneroTransactionSourceEntry * dst, const cryptonote::tx_source_entry * src){ + for(auto & cur : src->outputs){ + auto out = dst->add_outputs(); + out->set_idx(cur.first); + translate_rct_key(out->mutable_key(), &(cur.second)); + } + + dst->set_real_output(src->real_output); + dst->set_real_out_tx_key(key_to_string(src->real_out_tx_key)); + for(auto & cur : src->real_out_additional_tx_keys){ + dst->add_real_out_additional_tx_keys(key_to_string(cur)); + } + + dst->set_real_output_in_tx_index(src->real_output_in_tx_index); + dst->set_amount(src->amount); + dst->set_rct(src->rct); + dst->set_mask(key_to_string(src->mask)); + translate_klrki(dst->mutable_multisig_klrki(), &(src->multisig_kLRki)); + } + + void translate_klrki(MoneroMultisigKLRki * dst, const rct::multisig_kLRki * src){ + dst->set_k(key_to_string(src->k)); + dst->set_l(key_to_string(src->L)); + dst->set_r(key_to_string(src->R)); + dst->set_ki(key_to_string(src->ki)); + } + + void translate_rct_key(MoneroRctKey * dst, const rct::ctkey * src){ + dst->set_dest(key_to_string(src->dest)); + dst->set_commitment(key_to_string(src->mask)); + } + + std::string hash_addr(const MoneroAccountPublicAddress * addr, boost::optional<uint64_t> amount, boost::optional<bool> is_subaddr){ + return hash_addr(addr->spend_public_key(), addr->view_public_key(), amount, is_subaddr); + } + + std::string hash_addr(const std::string & spend_key, const std::string & view_key, boost::optional<uint64_t> amount, boost::optional<bool> is_subaddr){ + ::crypto::public_key spend{}, view{}; + if (spend_key.size() != 32 || view_key.size() != 32){ + throw std::invalid_argument("Public keys have invalid sizes"); + } + + memcpy(spend.data, spend_key.data(), 32); + memcpy(view.data, view_key.data(), 32); + return hash_addr(&spend, &view, amount, is_subaddr); + } + + std::string hash_addr(const ::crypto::public_key * spend_key, const ::crypto::public_key * view_key, boost::optional<uint64_t> amount, boost::optional<bool> is_subaddr){ + char buff[64+8+1]; + size_t offset = 0; + + memcpy(buff + offset, spend_key->data, 32); offset += 32; + memcpy(buff + offset, view_key->data, 32); offset += 32; + + if (amount){ + memcpy(buff + offset, (uint8_t*) &(amount.get()), sizeof(amount.get())); offset += sizeof(amount.get()); + } + + if (is_subaddr){ + buff[offset] = is_subaddr.get(); + offset += 1; + } + + return std::string(buff, offset); + } + + TData::TData() { + in_memory = false; + rsig_type = 0; + cur_input_idx = 0; + cur_output_idx = 0; + cur_batch_idx = 0; + cur_output_in_batch_idx = 0; + } + + Signer::Signer(wallet_shim *wallet2, const unsigned_tx_set * unsigned_tx, size_t tx_idx, hw::tx_aux_data * aux_data) { + m_wallet2 = wallet2; + m_unsigned_tx = unsigned_tx; + m_aux_data = aux_data; + m_tx_idx = tx_idx; + m_ct.tx_data = cur_tx(); + m_multisig = false; + } + + void Signer::extract_payment_id(){ + const std::vector<uint8_t>& tx_extra = cur_tx().extra; + m_ct.tsx_data.set_payment_id(""); + + std::vector<cryptonote::tx_extra_field> tx_extra_fields; + cryptonote::parse_tx_extra(tx_extra, tx_extra_fields); // ok if partially parsed + cryptonote::tx_extra_nonce extra_nonce; + + ::crypto::hash payment_id{}; + if (find_tx_extra_field_by_type(tx_extra_fields, extra_nonce)) + { + ::crypto::hash8 payment_id8{}; + if(cryptonote::get_encrypted_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id8)) + { + m_ct.tsx_data.set_payment_id(std::string(payment_id8.data, 8)); + } + else if (cryptonote::get_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id)) + { + m_ct.tsx_data.set_payment_id(std::string(payment_id.data, 32)); + } + } + } + + static unsigned get_rsig_type(bool use_bulletproof, size_t num_outputs){ + if (!use_bulletproof){ + return rct::RangeProofBorromean; + } else if (num_outputs > BULLETPROOF_MAX_OUTPUTS){ + return rct::RangeProofMultiOutputBulletproof; + } else { + return rct::RangeProofPaddedBulletproof; + } + } + + static void generate_rsig_batch_sizes(std::vector<uint64_t> &batches, unsigned rsig_type, size_t num_outputs){ + size_t amount_batched = 0; + + while(amount_batched < num_outputs){ + if (rsig_type == rct::RangeProofBorromean || rsig_type == rct::RangeProofBulletproof) { + batches.push_back(1); + amount_batched += 1; + + } else if (rsig_type == rct::RangeProofPaddedBulletproof){ + if (num_outputs > BULLETPROOF_MAX_OUTPUTS){ + throw std::invalid_argument("BP padded can support only BULLETPROOF_MAX_OUTPUTS statements"); + } + batches.push_back(num_outputs); + amount_batched += num_outputs; + + } else if (rsig_type == rct::RangeProofMultiOutputBulletproof){ + size_t batch_size = 1; + while (batch_size * 2 + amount_batched <= num_outputs && batch_size * 2 <= BULLETPROOF_MAX_OUTPUTS){ + batch_size *= 2; + } + batch_size = std::min(batch_size, num_outputs - amount_batched); + batches.push_back(batch_size); + amount_batched += batch_size; + + } else { + throw std::invalid_argument("Unknown rsig type"); + } + } + } + + void Signer::compute_integrated_indices(TsxData * tsx_data){ + if (m_aux_data == nullptr || m_aux_data->tx_recipients.empty()){ + return; + } + + auto & chg = tsx_data->change_dts(); + std::string change_hash = hash_addr(&chg.addr(), chg.amount(), chg.is_subaddress()); + + std::vector<uint32_t> integrated_indices; + std::set<std::string> integrated_hashes; + for (auto & cur : m_aux_data->tx_recipients){ + if (!cur.has_payment_id){ + continue; + } + integrated_hashes.emplace(hash_addr(&cur.address.m_spend_public_key, &cur.address.m_view_public_key)); + } + + ssize_t idx = -1; + for (auto & cur : tsx_data->outputs()){ + idx += 1; + + std::string c_hash = hash_addr(&cur.addr(), cur.amount(), cur.is_subaddress()); + if (c_hash == change_hash || cur.is_subaddress()){ + continue; + } + + c_hash = hash_addr(&cur.addr()); + if (integrated_hashes.find(c_hash) != integrated_hashes.end()){ + integrated_indices.push_back((uint32_t)idx); + } + } + + if (!integrated_indices.empty()){ + assign_to_repeatable(tsx_data->mutable_integrated_indices(), integrated_indices.begin(), integrated_indices.end()); + } + } + + std::shared_ptr<messages::monero::MoneroTransactionInitRequest> Signer::step_init(){ + // extract payment ID from construction data + auto & tsx_data = m_ct.tsx_data; + auto & tx = cur_tx(); + + m_ct.tx.version = 2; + m_ct.tx.unlock_time = tx.unlock_time; + + tsx_data.set_version(1); + tsx_data.set_unlock_time(tx.unlock_time); + tsx_data.set_num_inputs(static_cast<google::protobuf::uint32>(tx.sources.size())); + tsx_data.set_mixin(static_cast<google::protobuf::uint32>(tx.sources[0].outputs.size() - 1)); + tsx_data.set_account(tx.subaddr_account); + assign_to_repeatable(tsx_data.mutable_minor_indices(), tx.subaddr_indices.begin(), tx.subaddr_indices.end()); + + // Rsig decision + auto rsig_data = tsx_data.mutable_rsig_data(); + m_ct.rsig_type = get_rsig_type(tx.use_bulletproofs, tx.splitted_dsts.size()); + rsig_data->set_rsig_type(m_ct.rsig_type); + + generate_rsig_batch_sizes(m_ct.grouping_vct, m_ct.rsig_type, tx.splitted_dsts.size()); + assign_to_repeatable(rsig_data->mutable_grouping(), m_ct.grouping_vct.begin(), m_ct.grouping_vct.end()); + + translate_dst_entry(tsx_data.mutable_change_dts(), &(tx.change_dts)); + for(auto & cur : tx.splitted_dsts){ + auto dst = tsx_data.mutable_outputs()->Add(); + translate_dst_entry(dst, &cur); + } + + compute_integrated_indices(&tsx_data); + + int64_t fee = 0; + for(auto & cur_in : tx.sources){ + fee += cur_in.amount; + } + for(auto & cur_out : tx.splitted_dsts){ + fee -= cur_out.amount; + } + if (fee < 0){ + throw std::invalid_argument("Fee cannot be negative"); + } + + tsx_data.set_fee(static_cast<google::protobuf::uint64>(fee)); + this->extract_payment_id(); + + auto init_req = std::make_shared<messages::monero::MoneroTransactionInitRequest>(); + init_req->set_version(0); + init_req->mutable_tsx_data()->CopyFrom(tsx_data); + return init_req; + } + + void Signer::step_init_ack(std::shared_ptr<const messages::monero::MoneroTransactionInitAck> ack){ + m_ct.in_memory = false; + if (ack->has_rsig_data()){ + m_ct.rsig_param = std::make_shared<MoneroRsigData>(ack->rsig_data()); + } + + assign_from_repeatable(&(m_ct.tx_out_entr_hmacs), ack->hmacs().begin(), ack->hmacs().end()); + } + + std::shared_ptr<messages::monero::MoneroTransactionSetInputRequest> Signer::step_set_input(size_t idx){ + CHECK_AND_ASSERT_THROW_MES(idx < cur_tx().sources.size(), "Invalid source index"); + m_ct.cur_input_idx = idx; + auto res = std::make_shared<messages::monero::MoneroTransactionSetInputRequest>(); + translate_src_entry(res->mutable_src_entr(), &(cur_tx().sources[idx])); + return res; + } + + void Signer::step_set_input_ack(std::shared_ptr<const messages::monero::MoneroTransactionSetInputAck> ack){ + auto & vini_str = ack->vini(); + + cryptonote::txin_v vini; + if (!cn_deserialize(vini_str.data(), vini_str.size(), vini)){ + throw exc::ProtocolException("Cannot deserialize vin[i]"); + } + + m_ct.tx.vin.emplace_back(vini); + m_ct.tx_in_hmacs.push_back(ack->vini_hmac()); + m_ct.pseudo_outs.push_back(ack->pseudo_out()); + m_ct.pseudo_outs_hmac.push_back(ack->pseudo_out_hmac()); + m_ct.alphas.push_back(ack->pseudo_out_alpha()); + m_ct.spend_encs.push_back(ack->spend_key()); + } + + void Signer::sort_ki(){ + const size_t input_size = cur_tx().sources.size(); + + m_ct.source_permutation.clear(); + for (size_t n = 0; n < input_size; ++n){ + m_ct.source_permutation.push_back(n); + } + + CHECK_AND_ASSERT_THROW_MES(m_ct.tx.vin.size() == input_size, "Invalid vector size"); + std::sort(m_ct.source_permutation.begin(), m_ct.source_permutation.end(), [&](const size_t i0, const size_t i1) { + const cryptonote::txin_to_key &tk0 = boost::get<cryptonote::txin_to_key>(m_ct.tx.vin[i0]); + const cryptonote::txin_to_key &tk1 = boost::get<cryptonote::txin_to_key>(m_ct.tx.vin[i1]); + return memcmp(&tk0.k_image, &tk1.k_image, sizeof(tk0.k_image)) > 0; + }); + + CHECK_AND_ASSERT_THROW_MES(m_ct.tx_in_hmacs.size() == input_size, "Invalid vector size"); + CHECK_AND_ASSERT_THROW_MES(m_ct.pseudo_outs.size() == input_size, "Invalid vector size"); + CHECK_AND_ASSERT_THROW_MES(m_ct.pseudo_outs_hmac.size() == input_size, "Invalid vector size"); + CHECK_AND_ASSERT_THROW_MES(m_ct.alphas.size() == input_size, "Invalid vector size"); + CHECK_AND_ASSERT_THROW_MES(m_ct.spend_encs.size() == input_size, "Invalid vector size"); + CHECK_AND_ASSERT_THROW_MES(m_ct.tx_data.sources.size() == input_size, "Invalid vector size"); + + tools::apply_permutation(m_ct.source_permutation, [&](size_t i0, size_t i1){ + std::swap(m_ct.tx.vin[i0], m_ct.tx.vin[i1]); + std::swap(m_ct.tx_in_hmacs[i0], m_ct.tx_in_hmacs[i1]); + std::swap(m_ct.pseudo_outs[i0], m_ct.pseudo_outs[i1]); + std::swap(m_ct.pseudo_outs_hmac[i0], m_ct.pseudo_outs_hmac[i1]); + std::swap(m_ct.alphas[i0], m_ct.alphas[i1]); + std::swap(m_ct.spend_encs[i0], m_ct.spend_encs[i1]); + std::swap(m_ct.tx_data.sources[i0], m_ct.tx_data.sources[i1]); + }); + } + + std::shared_ptr<messages::monero::MoneroTransactionInputsPermutationRequest> Signer::step_permutation(){ + sort_ki(); + + if (in_memory()){ + return nullptr; + } + + auto res = std::make_shared<messages::monero::MoneroTransactionInputsPermutationRequest>(); + assign_to_repeatable(res->mutable_perm(), m_ct.source_permutation.begin(), m_ct.source_permutation.end()); + + return res; + } + + void Signer::step_permutation_ack(std::shared_ptr<const messages::monero::MoneroTransactionInputsPermutationAck> ack){ + if (in_memory()){ + return; + } + } + + std::shared_ptr<messages::monero::MoneroTransactionInputViniRequest> Signer::step_set_vini_input(size_t idx){ + if (in_memory()){ + return nullptr; + } + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_data.sources.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx.vin.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_in_hmacs.size(), "Invalid transaction index"); + + m_ct.cur_input_idx = idx; + auto tx = m_ct.tx_data; + auto res = std::make_shared<messages::monero::MoneroTransactionInputViniRequest>(); + auto & vini = m_ct.tx.vin[idx]; + translate_src_entry(res->mutable_src_entr(), &(tx.sources[idx])); + res->set_vini(cryptonote::t_serializable_object_to_blob(vini)); + res->set_vini_hmac(m_ct.tx_in_hmacs[idx]); + if (!in_memory()) { + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs_hmac.size(), "Invalid transaction index"); + res->set_pseudo_out(m_ct.pseudo_outs[idx]); + res->set_pseudo_out_hmac(m_ct.pseudo_outs_hmac[idx]); + } + + return res; + } + + void Signer::step_set_vini_input_ack(std::shared_ptr<const messages::monero::MoneroTransactionInputViniAck> ack){ + if (in_memory()){ + return; + } + } + + std::shared_ptr<messages::monero::MoneroTransactionAllInputsSetRequest> Signer::step_all_inputs_set(){ + return std::make_shared<messages::monero::MoneroTransactionAllInputsSetRequest>(); + } + + void Signer::step_all_inputs_set_ack(std::shared_ptr<const messages::monero::MoneroTransactionAllInputsSetAck> ack){ + if (is_offloading()){ + // If offloading, expect rsig configuration. + if (!ack->has_rsig_data()){ + throw exc::ProtocolException("Rsig offloading requires rsig param"); + } + + auto & rsig_data = ack->rsig_data(); + if (!rsig_data.has_mask()){ + throw exc::ProtocolException("Gamma masks not present in offloaded version"); + } + + auto & mask = rsig_data.mask(); + if (mask.size() != 32 * num_outputs()){ + throw exc::ProtocolException("Invalid number of gamma masks"); + } + + m_ct.rsig_gamma.reserve(num_outputs()); + for(size_t c=0; c < num_outputs(); ++c){ + rct::key cmask{}; + memcpy(cmask.bytes, mask.data() + c * 32, 32); + m_ct.rsig_gamma.emplace_back(cmask); + } + } + } + + std::shared_ptr<messages::monero::MoneroTransactionSetOutputRequest> Signer::step_set_output(size_t idx){ + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_data.splitted_dsts.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_out_entr_hmacs.size(), "Invalid transaction index"); + + m_ct.cur_output_idx = idx; + m_ct.cur_output_in_batch_idx += 1; // assumes sequential call to step_set_output() + + auto res = std::make_shared<messages::monero::MoneroTransactionSetOutputRequest>(); + auto & cur_dst = m_ct.tx_data.splitted_dsts[idx]; + translate_dst_entry(res->mutable_dst_entr(), &cur_dst); + res->set_dst_entr_hmac(m_ct.tx_out_entr_hmacs[idx]); + + // Range sig offloading to the host + if (!is_offloading()) { + return res; + } + + CHECK_AND_ASSERT_THROW_MES(m_ct.cur_batch_idx < m_ct.grouping_vct.size(), "Invalid batch index"); + if (m_ct.grouping_vct[m_ct.cur_batch_idx] > m_ct.cur_output_in_batch_idx) { + return res; + } + + auto rsig_data = res->mutable_rsig_data(); + auto batch_size = m_ct.grouping_vct[m_ct.cur_batch_idx]; + + if (!is_req_bulletproof()){ + if (batch_size > 1){ + throw std::invalid_argument("Borromean cannot batch outputs"); + } + + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.rsig_gamma.size(), "Invalid gamma index"); + rct::key C{}, mask = m_ct.rsig_gamma[idx]; + auto genRsig = rct::proveRange(C, mask, cur_dst.amount); // TODO: rsig with given mask + auto serRsig = cn_serialize(genRsig); + m_ct.tx_out_rsigs.emplace_back(genRsig); + rsig_data->set_rsig(serRsig); + + } else { + std::vector<uint64_t> amounts; + rct::keyV masks; + CHECK_AND_ASSERT_THROW_MES(idx + 1 >= batch_size, "Invalid index for batching"); + + for(size_t i = 0; i < batch_size; ++i){ + const size_t bidx = 1 + idx - batch_size + i; + CHECK_AND_ASSERT_THROW_MES(bidx < m_ct.tx_data.splitted_dsts.size(), "Invalid gamma index"); + CHECK_AND_ASSERT_THROW_MES(bidx < m_ct.rsig_gamma.size(), "Invalid gamma index"); + + amounts.push_back(m_ct.tx_data.splitted_dsts[bidx].amount); + masks.push_back(m_ct.rsig_gamma[bidx]); + } + + auto bp = bulletproof_PROVE(amounts, masks); + auto serRsig = cn_serialize(bp); + m_ct.tx_out_rsigs.emplace_back(bp); + rsig_data->set_rsig(serRsig); + } + + return res; + } + + void Signer::step_set_output_ack(std::shared_ptr<const messages::monero::MoneroTransactionSetOutputAck> ack){ + cryptonote::tx_out tx_out; + rct::rangeSig range_sig{}; + rct::Bulletproof bproof{}; + rct::ctkey out_pk{}; + rct::ecdhTuple ecdh{}; + + bool has_rsig = false; + std::string rsig_buff; + + if (ack->has_rsig_data()){ + auto & rsig_data = ack->rsig_data(); + + if (rsig_data.has_rsig() && !rsig_data.rsig().empty()){ + has_rsig = true; + rsig_buff = rsig_data.rsig(); + + } else if (rsig_data.rsig_parts_size() > 0){ + has_rsig = true; + for (const auto &it : rsig_data.rsig_parts()) { + rsig_buff += it; + } + } + } + + if (!cn_deserialize(ack->tx_out(), tx_out)){ + throw exc::ProtocolException("Cannot deserialize vout[i]"); + } + + if (!cn_deserialize(ack->out_pk(), out_pk)){ + throw exc::ProtocolException("Cannot deserialize out_pk"); + } + + if (!cn_deserialize(ack->ecdh_info(), ecdh)){ + throw exc::ProtocolException("Cannot deserialize ecdhtuple"); + } + + if (has_rsig && !is_req_bulletproof() && !cn_deserialize(rsig_buff, range_sig)){ + throw exc::ProtocolException("Cannot deserialize rangesig"); + } + + if (has_rsig && is_req_bulletproof() && !cn_deserialize(rsig_buff, bproof)){ + throw exc::ProtocolException("Cannot deserialize bulletproof rangesig"); + } + + m_ct.tx.vout.emplace_back(tx_out); + m_ct.tx_out_hmacs.push_back(ack->vouti_hmac()); + m_ct.tx_out_pk.emplace_back(out_pk); + m_ct.tx_out_ecdh.emplace_back(ecdh); + + if (!has_rsig){ + return; + } + + if (is_req_bulletproof()){ + CHECK_AND_ASSERT_THROW_MES(m_ct.cur_batch_idx < m_ct.grouping_vct.size(), "Invalid batch index"); + auto batch_size = m_ct.grouping_vct[m_ct.cur_batch_idx]; + for (size_t i = 0; i < batch_size; ++i){ + const size_t bidx = 1 + m_ct.cur_output_idx - batch_size + i; + CHECK_AND_ASSERT_THROW_MES(bidx < m_ct.tx_out_pk.size(), "Invalid out index"); + + rct::key commitment = m_ct.tx_out_pk[bidx].mask; + commitment = rct::scalarmultKey(commitment, rct::INV_EIGHT); + bproof.V.push_back(commitment); + } + + m_ct.tx_out_rsigs.emplace_back(bproof); + if (!rct::bulletproof_VERIFY(boost::get<rct::Bulletproof>(m_ct.tx_out_rsigs.back()))) { + throw exc::ProtocolException("Returned range signature is invalid"); + } + + } else { + m_ct.tx_out_rsigs.emplace_back(range_sig); + + if (!rct::verRange(out_pk.mask, boost::get<rct::rangeSig>(m_ct.tx_out_rsigs.back()))) { + throw exc::ProtocolException("Returned range signature is invalid"); + } + } + + m_ct.cur_batch_idx += 1; + m_ct.cur_output_in_batch_idx = 0; + } + + std::shared_ptr<messages::monero::MoneroTransactionAllOutSetRequest> Signer::step_all_outs_set(){ + return std::make_shared<messages::monero::MoneroTransactionAllOutSetRequest>(); + } + + void Signer::step_all_outs_set_ack(std::shared_ptr<const messages::monero::MoneroTransactionAllOutSetAck> ack, hw::device &hwdev){ + m_ct.rv = std::make_shared<rct::rctSig>(); + m_ct.rv->txnFee = ack->rv().txn_fee(); + m_ct.rv->type = static_cast<uint8_t>(ack->rv().rv_type()); + string_to_key(m_ct.rv->message, ack->rv().message()); + + // Extra copy + m_ct.tx.extra.clear(); + auto extra = ack->extra(); + auto extra_data = extra.data(); + m_ct.tx.extra.reserve(extra.size()); + for(size_t i = 0; i < extra.size(); ++i){ + m_ct.tx.extra.push_back(static_cast<uint8_t>(extra_data[i])); + } + + ::crypto::hash tx_prefix_hash{}; + cryptonote::get_transaction_prefix_hash(m_ct.tx, tx_prefix_hash); + m_ct.tx_prefix_hash = key_to_string(tx_prefix_hash); + if (crypto_verify_32(reinterpret_cast<const unsigned char *>(tx_prefix_hash.data), + reinterpret_cast<const unsigned char *>(ack->tx_prefix_hash().data()))){ + throw exc::proto::SecurityException("Transaction prefix has does not match to the computed value"); + } + + // RctSig + auto num_sources = m_ct.tx_data.sources.size(); + if (is_simple() || is_req_bulletproof()){ + auto dst = &m_ct.rv->pseudoOuts; + if (is_bulletproof()){ + dst = &m_ct.rv->p.pseudoOuts; + } + + dst->clear(); + for (const auto &pseudo_out : m_ct.pseudo_outs) { + dst->emplace_back(); + string_to_key(dst->back(), pseudo_out); + } + + m_ct.rv->mixRing.resize(num_sources); + } else { + m_ct.rv->mixRing.resize(m_ct.tsx_data.mixin()); + m_ct.rv->mixRing[0].resize(num_sources); + } + + CHECK_AND_ASSERT_THROW_MES(m_ct.tx_out_pk.size() == m_ct.tx_out_ecdh.size(), "Invalid vector sizes"); + for(size_t i = 0; i < m_ct.tx_out_ecdh.size(); ++i){ + m_ct.rv->outPk.push_back(m_ct.tx_out_pk[i]); + m_ct.rv->ecdhInfo.push_back(m_ct.tx_out_ecdh[i]); + } + + for(size_t i = 0; i < m_ct.tx_out_rsigs.size(); ++i){ + if (is_bulletproof()){ + m_ct.rv->p.bulletproofs.push_back(boost::get<rct::Bulletproof>(m_ct.tx_out_rsigs[i])); + } else { + m_ct.rv->p.rangeSigs.push_back(boost::get<rct::rangeSig>(m_ct.tx_out_rsigs[i])); + } + } + + rct::key hash_computed = rct::get_pre_mlsag_hash(*(m_ct.rv), hwdev); + auto & hash = ack->full_message_hash(); + + if (hash.size() != 32){ + throw exc::ProtocolException("Returned mlsag hash has invalid size"); + } + + if (crypto_verify_32(reinterpret_cast<const unsigned char *>(hash_computed.bytes), + reinterpret_cast<const unsigned char *>(hash.data()))){ + throw exc::proto::SecurityException("Computed MLSAG does not match"); + } + } + + std::shared_ptr<messages::monero::MoneroTransactionSignInputRequest> Signer::step_sign_input(size_t idx){ + m_ct.cur_input_idx = idx; + + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_data.sources.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx.vin.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.tx_in_hmacs.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.alphas.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.spend_encs.size(), "Invalid transaction index"); + + auto res = std::make_shared<messages::monero::MoneroTransactionSignInputRequest>(); + translate_src_entry(res->mutable_src_entr(), &(m_ct.tx_data.sources[idx])); + res->set_vini(cryptonote::t_serializable_object_to_blob(m_ct.tx.vin[idx])); + res->set_vini_hmac(m_ct.tx_in_hmacs[idx]); + res->set_pseudo_out_alpha(m_ct.alphas[idx]); + res->set_spend_key(m_ct.spend_encs[idx]); + if (!in_memory()){ + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs.size(), "Invalid transaction index"); + CHECK_AND_ASSERT_THROW_MES(idx < m_ct.pseudo_outs_hmac.size(), "Invalid transaction index"); + res->set_pseudo_out(m_ct.pseudo_outs[idx]); + res->set_pseudo_out_hmac(m_ct.pseudo_outs_hmac[idx]); + } + return res; + } + + void Signer::step_sign_input_ack(std::shared_ptr<const messages::monero::MoneroTransactionSignInputAck> ack){ + rct::mgSig mg; + if (!cn_deserialize(ack->signature(), mg)){ + throw exc::ProtocolException("Cannot deserialize mg[i]"); + } + + m_ct.rv->p.MGs.push_back(mg); + } + + std::shared_ptr<messages::monero::MoneroTransactionFinalRequest> Signer::step_final(){ + m_ct.tx.rct_signatures = *(m_ct.rv); + return std::make_shared<messages::monero::MoneroTransactionFinalRequest>(); + } + + void Signer::step_final_ack(std::shared_ptr<const messages::monero::MoneroTransactionFinalAck> ack){ + if (m_multisig){ + auto & cout_key = ack->cout_key(); + for(auto & cur : m_ct.couts){ + if (cur.size() != 12 + 32){ + throw std::invalid_argument("Encrypted cout has invalid length"); + } + + char buff[32]; + auto data = cur.data(); + + crypto::chacha::decrypt(data + 12, 32, reinterpret_cast<const uint8_t *>(cout_key.data()), reinterpret_cast<const uint8_t *>(data), buff); + m_ct.couts_dec.emplace_back(buff, 32); + } + } + + m_ct.enc_salt1 = ack->salt(); + m_ct.enc_salt2 = ack->rand_mult(); + m_ct.enc_keys = ack->tx_enc_keys(); + } + + std::string Signer::store_tx_aux_info(){ + rapidjson::StringBuffer sb; + rapidjson::Writer<rapidjson::StringBuffer> writer(sb); + + rapidjson::Document json; + json.SetObject(); + + rapidjson::Value valueS(rapidjson::kStringType); + rapidjson::Value valueI(rapidjson::kNumberType); + + valueI.SetInt(1); + json.AddMember("version", valueI, json.GetAllocator()); + + valueS.SetString(m_ct.enc_salt1.c_str(), m_ct.enc_salt1.size()); + json.AddMember("salt1", valueS, json.GetAllocator()); + + valueS.SetString(m_ct.enc_salt2.c_str(), m_ct.enc_salt2.size()); + json.AddMember("salt2", valueS, json.GetAllocator()); + + valueS.SetString(m_ct.tx_prefix_hash.c_str(), m_ct.tx_prefix_hash.size()); + json.AddMember("tx_prefix_hash", valueS, json.GetAllocator()); + + valueS.SetString(m_ct.enc_keys.c_str(), m_ct.enc_keys.size()); + json.AddMember("enc_keys", valueS, json.GetAllocator()); + + json.Accept(writer); + return sb.GetString(); + } + + +} +} +} +} diff --git a/src/device_trezor/trezor/protocol.hpp b/src/device_trezor/trezor/protocol.hpp new file mode 100644 index 000000000..ce0361640 --- /dev/null +++ b/src/device_trezor/trezor/protocol.hpp @@ -0,0 +1,300 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef MONERO_PROTOCOL_H +#define MONERO_PROTOCOL_H + +#include "trezor_defs.hpp" +#include "device/device_cold.hpp" +#include "messages_map.hpp" +#include "transport.hpp" +#include "wallet/wallet2.h" + +namespace hw{ +namespace trezor{ +namespace protocol{ + + std::string key_to_string(const ::crypto::ec_point & key); + std::string key_to_string(const ::crypto::ec_scalar & key); + std::string key_to_string(const ::crypto::hash & key); + std::string key_to_string(const ::rct::key & key); + + void string_to_key(::crypto::ec_scalar & key, const std::string & str); + void string_to_key(::crypto::ec_point & key, const std::string & str); + void string_to_key(::rct::key & key, const std::string & str); + + template<class sub_t, class InputIterator> + void assign_to_repeatable(::google::protobuf::RepeatedField<sub_t> * dst, const InputIterator begin, const InputIterator end){ + for (InputIterator it = begin; it != end; it++) { + auto s = dst->Add(); + *s = *it; + } + } + + template<class sub_t, class InputIterator> + void assign_from_repeatable(std::vector<sub_t> * dst, const InputIterator begin, const InputIterator end){ + for (InputIterator it = begin; it != end; it++) { + dst->push_back(*it); + } + }; + + template<typename T> + bool cn_deserialize(const void * buff, size_t len, T & dst){ + std::stringstream ss; + ss.write(static_cast<const char *>(buff), len); //ss << tx_blob; + binary_archive<false> ba(ss); + bool r = ::serialization::serialize(ba, dst); + return r; + } + + template<typename T> + bool cn_deserialize(const std::string & str, T & dst){ + return cn_deserialize(str.data(), str.size(), dst); + } + + template<typename T> + std::string cn_serialize(T & obj){ + std::ostringstream oss; + binary_archive<true> oar(oss); + bool success = ::serialization::serialize(oar, obj); + if (!success){ + throw exc::EncodingException("Could not CN serialize given object"); + } + return oss.str(); + } + +// Crypto / encryption +namespace crypto { +namespace chacha { + + /** + * Chacha20Poly1305 decryption with tag verification. RFC 7539. + */ + void decrypt(const void* ciphertext, size_t length, const uint8_t* key, const uint8_t* iv, char* plaintext); + +} +} + + +// Cold Key image sync +namespace ki { + + using MoneroTransferDetails = messages::monero::MoneroKeyImageSyncStepRequest_MoneroTransferDetails; + using MoneroSubAddressIndicesList = messages::monero::MoneroKeyImageExportInitRequest_MoneroSubAddressIndicesList; + using MoneroExportedKeyImage = messages::monero::MoneroKeyImageSyncStepAck_MoneroExportedKeyImage; + using exported_key_image = hw::device_cold::exported_key_image; + + /** + * Converts transfer details to the MoneroTransferDetails required for KI sync + */ + bool key_image_data(wallet_shim * wallet, + const std::vector<tools::wallet2::transfer_details> & transfers, + std::vector<MoneroTransferDetails> & res); + + /** + * Computes a hash over MoneroTransferDetails. Commitment used in the KI sync. + */ + std::string compute_hash(const MoneroTransferDetails & rr); + + /** + * Generates KI sync request with commitments computed. + */ + void generate_commitment(std::vector<MoneroTransferDetails> & mtds, + const std::vector<tools::wallet2::transfer_details> & transfers, + std::shared_ptr<messages::monero::MoneroKeyImageExportInitRequest> & req); + +} + +// Cold transaction signing +namespace tx { + using TsxData = messages::monero::MoneroTransactionInitRequest_MoneroTransactionData; + using MoneroTransactionDestinationEntry = messages::monero::MoneroTransactionDestinationEntry; + using MoneroAccountPublicAddress = messages::monero::MoneroTransactionDestinationEntry_MoneroAccountPublicAddress; + using MoneroTransactionSourceEntry = messages::monero::MoneroTransactionSourceEntry; + using MoneroMultisigKLRki = messages::monero::MoneroTransactionSourceEntry_MoneroMultisigKLRki; + using MoneroOutputEntry = messages::monero::MoneroTransactionSourceEntry_MoneroOutputEntry; + using MoneroRctKey = messages::monero::MoneroTransactionSourceEntry_MoneroOutputEntry_MoneroRctKeyPublic; + using MoneroRsigData = messages::monero::MoneroTransactionRsigData; + + using tx_construction_data = tools::wallet2::tx_construction_data; + using unsigned_tx_set = tools::wallet2::unsigned_tx_set; + + void translate_address(MoneroAccountPublicAddress * dst, const cryptonote::account_public_address * src); + void translate_dst_entry(MoneroTransactionDestinationEntry * dst, const cryptonote::tx_destination_entry * src); + void translate_src_entry(MoneroTransactionSourceEntry * dst, const cryptonote::tx_source_entry * src); + void translate_klrki(MoneroMultisigKLRki * dst, const rct::multisig_kLRki * src); + void translate_rct_key(MoneroRctKey * dst, const rct::ctkey * src); + std::string hash_addr(const MoneroAccountPublicAddress * addr, boost::optional<uint64_t> amount = boost::none, boost::optional<bool> is_subaddr = boost::none); + std::string hash_addr(const std::string & spend_key, const std::string & view_key, boost::optional<uint64_t> amount = boost::none, boost::optional<bool> is_subaddr = boost::none); + std::string hash_addr(const ::crypto::public_key * spend_key, const ::crypto::public_key * view_key, boost::optional<uint64_t> amount = boost::none, boost::optional<bool> is_subaddr = boost::none); + + typedef boost::variant<rct::rangeSig, rct::Bulletproof> rsig_v; + + /** + * Transaction signer state holder. + */ + class TData { + public: + TsxData tsx_data; + tx_construction_data tx_data; + cryptonote::transaction tx; + bool in_memory; + unsigned rsig_type; + std::vector<uint64_t> grouping_vct; + std::shared_ptr<MoneroRsigData> rsig_param; + size_t cur_input_idx; + size_t cur_output_idx; + size_t cur_batch_idx; + size_t cur_output_in_batch_idx; + + std::vector<std::string> tx_in_hmacs; + std::vector<std::string> tx_out_entr_hmacs; + std::vector<std::string> tx_out_hmacs; + std::vector<rsig_v> tx_out_rsigs; + std::vector<rct::ctkey> tx_out_pk; + std::vector<rct::ecdhTuple> tx_out_ecdh; + std::vector<size_t> source_permutation; + std::vector<std::string> alphas; + std::vector<std::string> spend_encs; + std::vector<std::string> pseudo_outs; + std::vector<std::string> pseudo_outs_hmac; + std::vector<std::string> couts; + std::vector<std::string> couts_dec; + std::vector<rct::key> rsig_gamma; + std::string tx_prefix_hash; + std::string enc_salt1; + std::string enc_salt2; + std::string enc_keys; + + std::shared_ptr<rct::rctSig> rv; + + TData(); + }; + + class Signer { + private: + TData m_ct; + wallet_shim * m_wallet2; + + size_t m_tx_idx; + const unsigned_tx_set * m_unsigned_tx; + hw::tx_aux_data * m_aux_data; + + bool m_multisig; + + const tx_construction_data & cur_tx(){ + CHECK_AND_ASSERT_THROW_MES(m_tx_idx < m_unsigned_tx->txes.size(), "Invalid transaction index"); + return m_unsigned_tx->txes[m_tx_idx]; + } + + void extract_payment_id(); + void compute_integrated_indices(TsxData * tsx_data); + + public: + Signer(wallet_shim * wallet2, const unsigned_tx_set * unsigned_tx, size_t tx_idx = 0, hw::tx_aux_data * aux_data = nullptr); + + std::shared_ptr<messages::monero::MoneroTransactionInitRequest> step_init(); + void step_init_ack(std::shared_ptr<const messages::monero::MoneroTransactionInitAck> ack); + + std::shared_ptr<messages::monero::MoneroTransactionSetInputRequest> step_set_input(size_t idx); + void step_set_input_ack(std::shared_ptr<const messages::monero::MoneroTransactionSetInputAck> ack); + + void sort_ki(); + std::shared_ptr<messages::monero::MoneroTransactionInputsPermutationRequest> step_permutation(); + void step_permutation_ack(std::shared_ptr<const messages::monero::MoneroTransactionInputsPermutationAck> ack); + + std::shared_ptr<messages::monero::MoneroTransactionInputViniRequest> step_set_vini_input(size_t idx); + void step_set_vini_input_ack(std::shared_ptr<const messages::monero::MoneroTransactionInputViniAck> ack); + + std::shared_ptr<messages::monero::MoneroTransactionAllInputsSetRequest> step_all_inputs_set(); + void step_all_inputs_set_ack(std::shared_ptr<const messages::monero::MoneroTransactionAllInputsSetAck> ack); + + std::shared_ptr<messages::monero::MoneroTransactionSetOutputRequest> step_set_output(size_t idx); + void step_set_output_ack(std::shared_ptr<const messages::monero::MoneroTransactionSetOutputAck> ack); + + std::shared_ptr<messages::monero::MoneroTransactionAllOutSetRequest> step_all_outs_set(); + void step_all_outs_set_ack(std::shared_ptr<const messages::monero::MoneroTransactionAllOutSetAck> ack, hw::device &hwdev); + + std::shared_ptr<messages::monero::MoneroTransactionSignInputRequest> step_sign_input(size_t idx); + void step_sign_input_ack(std::shared_ptr<const messages::monero::MoneroTransactionSignInputAck> ack); + + std::shared_ptr<messages::monero::MoneroTransactionFinalRequest> step_final(); + void step_final_ack(std::shared_ptr<const messages::monero::MoneroTransactionFinalAck> ack); + + std::string store_tx_aux_info(); + + bool in_memory() const { + return m_ct.in_memory; + } + + bool is_simple() const { + if (!m_ct.rv){ + throw std::invalid_argument("RV not initialized"); + } + auto tp = m_ct.rv->type; + return tp == rct::RCTTypeSimple; + } + + bool is_req_bulletproof() const { + return m_ct.tx_data.use_bulletproofs; + } + + bool is_bulletproof() const { + if (!m_ct.rv){ + throw std::invalid_argument("RV not initialized"); + } + auto tp = m_ct.rv->type; + return tp == rct::RCTTypeBulletproof || tp == rct::RCTTypeBulletproof2; + } + + bool is_offloading() const { + return m_ct.rsig_param && m_ct.rsig_param->offload_type() != 0; + } + + size_t num_outputs() const { + return m_ct.tx_data.splitted_dsts.size(); + } + + size_t num_inputs() const { + return m_ct.tx_data.sources.size(); + } + + const TData & tdata() const { + return m_ct; + } + }; + +} + +} +} +} + + +#endif //MONERO_PROTOCOL_H diff --git a/src/device_trezor/trezor/tools/README.md b/src/device_trezor/trezor/tools/README.md new file mode 100644 index 000000000..0802e734a --- /dev/null +++ b/src/device_trezor/trezor/tools/README.md @@ -0,0 +1,45 @@ +# Trezor + +## Messages rebuild + +Install `protoc` for your distribution. Requirements: + +- `protobuf-compiler` +- `libprotobuf-dev` +- `python` + + +Soft requirement: Python 3, can be easily installed with [pyenv]. +If Python 3 is used there are no additional python dependencies. + +Since Cmake 3.12 the `FindPython` module is used to locate the Python +interpreter in your system. It preferably searches for Python 3, if none +is found, it searches for Python 2. + +Lower version of the cmake uses another module which does not guarantee +ordering. If you want to override the selected python you can do it in +the following way: + +```bash +export TREZOR_PYTHON=`which python3` +``` + + +### Python 2.7+ + +Python 3 has `tempfile.TemporaryDirectory` available but Python 2 lacks +this class so the message generation code uses `backports.tempfile` package +bundled in the repository. + +The minimal Python versions are 2.7 and 3.4 + +### Regenerate messages + +```bash +cd src/device_trezor/trezor +python tools/build_protob.py +``` + +The messages regeneration is done also automatically via cmake. + +[pyenv]: https://github.com/pyenv/pyenv diff --git a/src/device_trezor/trezor/tools/build_protob.py b/src/device_trezor/trezor/tools/build_protob.py new file mode 100644 index 000000000..2611f3296 --- /dev/null +++ b/src/device_trezor/trezor/tools/build_protob.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python +import os +import subprocess +import sys + +CWD = os.path.dirname(os.path.realpath(__file__)) +ROOT_DIR = os.path.abspath(os.path.join(CWD, "..", "..", "..", "..")) +TREZOR_COMMON = os.path.join(ROOT_DIR, "external", "trezor-common") +TREZOR_MESSAGES = os.path.join(CWD, "..", "messages") + +# check for existence of the submodule directory +common_defs = os.path.join(TREZOR_COMMON, "defs") +if not os.path.exists(common_defs): + raise ValueError( + "trezor-common submodule seems to be missing.\n" + + 'Use "git submodule update --init --recursive" to retrieve it.' + ) + +# regenerate messages +try: + selected = [ + "messages.proto", + "messages-common.proto", + "messages-management.proto", + "messages-monero.proto", + ] + proto_srcs = [os.path.join(TREZOR_COMMON, "protob", x) for x in selected] + exec_args = [ + sys.executable, + os.path.join(CWD, "pb2cpp.py"), + "-o", + TREZOR_MESSAGES, + ] + proto_srcs + + subprocess.check_call(exec_args) + +except Exception as e: + raise diff --git a/src/device_trezor/trezor/tools/pb2cpp.py b/src/device_trezor/trezor/tools/pb2cpp.py new file mode 100644 index 000000000..3e0318ea5 --- /dev/null +++ b/src/device_trezor/trezor/tools/pb2cpp.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python +# Converts Google's protobuf python definitions of TREZOR wire messages +# to plain-python objects as used in TREZOR Core and python-trezor + +import argparse +import logging +import os +import re +import shutil +import subprocess +import glob +import hashlib + +try: + from tempfile import TemporaryDirectory +except: + # Py2 backward compatibility, using bundled sources. + # Original source: pip install backports.tempfile + try: + # Try bundled python version + import sys + sys.path.append(os.path.dirname(__file__)) + from py2backports.tempfile import TemporaryDirectory + + except: + raise EnvironmentError('Python 2.7+ or 3.4+ is required. ' + 'TemporaryDirectory is not available in Python 2.' + 'Try to specify python to use, e.g.: "export TREZOR_PYTHON=`which python3`"') + + +AUTO_HEADER = "# Automatically generated by pb2cpp\n" + +# Fixing GCC7 compilation error +UNDEF_STATEMENT = """ +#ifdef minor +#undef minor +#endif +""" + +PROTOC = None +PROTOC_INCLUDE = None + + +def which(pgm): + path = os.getenv('PATH') + for p in path.split(os.path.pathsep): + p = os.path.join(p, pgm) + if os.path.exists(p) and os.access(p, os.X_OK): + return p + + +def namespace_file(fpath, package): + """Adds / replaces package name. Simple regex parsing, may use https://github.com/ph4r05/plyprotobuf later""" + with open(fpath) as fh: + fdata = fh.read() + + re_syntax = re.compile(r"^syntax\s*=") + re_package = re.compile(r"^package\s+([^;]+?)\s*;\s*$") + lines = fdata.split("\n") + + line_syntax = None + line_package = None + for idx, line in enumerate(lines): + if line_syntax is None and re_syntax.match(line): + line_syntax = idx + if line_package is None and re_package.match(line): + line_package = idx + + if package is None: + if line_package is None: + return + else: + lines.pop(line_package) + + else: + new_package = "package %s;" % package + if line_package is None: + lines.insert(line_syntax + 1 if line_syntax is not None else 0, new_package) + else: + lines[line_package] = new_package + + new_fdat = "\n".join(lines) + with open(fpath, "w+") as fh: + fh.write(new_fdat) + return new_fdat + + +def protoc(files, out_dir, additional_includes=(), package=None, force=False): + """Compile code with protoc and return the data.""" + + include_dirs = set() + include_dirs.add(PROTOC_INCLUDE) + if additional_includes: + include_dirs.update(additional_includes) + + with TemporaryDirectory() as tmpdir_protob, TemporaryDirectory() as tmpdir_out: + include_dirs.add(tmpdir_protob) + + new_files = [] + for file in files: + bname = os.path.basename(file) + tmp_file = os.path.join(tmpdir_protob, bname) + + shutil.copy(file, tmp_file) + if package is not None: + namespace_file(tmp_file, package) + new_files.append(tmp_file) + + protoc_includes = ["-I" + dir for dir in include_dirs if dir] + + exec_args = ( + [ + PROTOC, + "--cpp_out", + tmpdir_out, + ] + + protoc_includes + + new_files + ) + + subprocess.check_call(exec_args) + + # Fixing gcc compilation and clashes with "minor" field name + add_undef(tmpdir_out) + + # Scan output dir, check file differences + update_message_files(tmpdir_out, out_dir, force) + + +def update_message_files(tmpdir_out, out_dir, force=False): + files = glob.glob(os.path.join(tmpdir_out, '*.pb.*')) + for fname in files: + bname = os.path.basename(fname) + dest_file = os.path.join(out_dir, bname) + if not force and os.path.exists(dest_file): + data = open(fname, 'rb').read() + data_hash = hashlib.sha256(data).digest() + data_dest = open(dest_file, 'rb').read() + data_dest_hash = hashlib.sha256(data_dest).digest() + if data_hash == data_dest_hash: + continue + + shutil.copy(fname, dest_file) + + +def add_undef(out_dir): + files = glob.glob(os.path.join(out_dir, '*.pb.*')) + for fname in files: + with open(fname) as fh: + lines = fh.readlines() + + idx_insertion = None + for idx in range(len(lines)): + if '@@protoc_insertion_point(includes)' in lines[idx]: + idx_insertion = idx + break + + if idx_insertion is None: + pass + + lines.insert(idx_insertion + 1, UNDEF_STATEMENT) + with open(fname, 'w') as fh: + fh.write("".join(lines)) + + +def strip_leader(s, prefix): + """Remove given prefix from underscored name.""" + leader = prefix + "_" + if s.startswith(leader): + return s[len(leader) :] + else: + return s + + +def main(): + global PROTOC, PROTOC_INCLUDE + logging.basicConfig(level=logging.DEBUG) + + parser = argparse.ArgumentParser() + # fmt: off + parser.add_argument("proto", nargs="+", help="Protobuf definition files") + parser.add_argument("-o", "--out-dir", help="Directory for generated source code") + parser.add_argument("-n", "--namespace", default=None, help="Message namespace") + parser.add_argument("-I", "--protoc-include", action="append", help="protoc include path") + parser.add_argument("-P", "--protobuf-module", default="protobuf", help="Name of protobuf module") + parser.add_argument("-f", "--force", default=False, help="Overwrite existing files") + # fmt: on + args = parser.parse_args() + + protoc_includes = args.protoc_include or (os.environ.get("PROTOC_INCLUDE"),) + + PROTOBUF_INCLUDE_DIRS = os.getenv("PROTOBUF_INCLUDE_DIRS", None) + PROTOBUF_PROTOC_EXECUTABLE = os.getenv("PROTOBUF_PROTOC_EXECUTABLE", None) + + if PROTOBUF_PROTOC_EXECUTABLE and not os.path.exists(PROTOBUF_PROTOC_EXECUTABLE): + raise ValueError("PROTOBUF_PROTOC_EXECUTABLE set but not found: %s" % PROTOBUF_PROTOC_EXECUTABLE) + + elif PROTOBUF_PROTOC_EXECUTABLE: + PROTOC = PROTOBUF_PROTOC_EXECUTABLE + + else: + if os.name == "nt": + PROTOC = which("protoc.exe") + else: + PROTOC = which("protoc") + + if not PROTOC: + raise ValueError("protoc command not found. Set PROTOBUF_PROTOC_EXECUTABLE env var to the protoc binary and optionally PROTOBUF_INCLUDE_DIRS") + + PROTOC_PREFIX = os.path.dirname(os.path.dirname(PROTOC)) + PROTOC_INCLUDE = PROTOBUF_INCLUDE_DIRS if PROTOBUF_INCLUDE_DIRS else os.path.join(PROTOC_PREFIX, "include") + + protoc( + args.proto, args.out_dir, protoc_includes, package=args.namespace, force=args.force + ) + + +if __name__ == "__main__": + main() diff --git a/src/blocks/blocks.dat b/src/device_trezor/trezor/tools/py2backports/__init__.py index e69de29bb..e69de29bb 100644 --- a/src/blocks/blocks.dat +++ b/src/device_trezor/trezor/tools/py2backports/__init__.py diff --git a/src/device_trezor/trezor/tools/py2backports/tempfile.py b/src/device_trezor/trezor/tools/py2backports/tempfile.py new file mode 100644 index 000000000..e259dea3f --- /dev/null +++ b/src/device_trezor/trezor/tools/py2backports/tempfile.py @@ -0,0 +1,72 @@ +""" +https://github.com/pjdelport/backports.tempfile/blob/master/src/backports/tempfile.py +Partial backport of Python 3.5's tempfile module: + TemporaryDirectory +Backport modifications are marked with marked with "XXX backport". +""" +from __future__ import absolute_import + +import sys +import warnings as _warnings +from shutil import rmtree as _rmtree + +from py2backports.weakref import finalize + + +# XXX backport: Rather than backporting all of mkdtemp(), we just create a +# thin wrapper implementing its Python 3.5 signature. +if sys.version_info < (3, 5): + from tempfile import mkdtemp as old_mkdtemp + + def mkdtemp(suffix=None, prefix=None, dir=None): + """ + Wrap `tempfile.mkdtemp()` to make the suffix and prefix optional (like Python 3.5). + """ + kwargs = {k: v for (k, v) in + dict(suffix=suffix, prefix=prefix, dir=dir).items() + if v is not None} + return old_mkdtemp(**kwargs) + +else: + from tempfile import mkdtemp + + +# XXX backport: ResourceWarning was added in Python 3.2. +# For earlier versions, fall back to RuntimeWarning instead. +_ResourceWarning = RuntimeWarning if sys.version_info < (3, 2) else ResourceWarning + + +class TemporaryDirectory(object): + """Create and return a temporary directory. This has the same + behavior as mkdtemp but can be used as a context manager. For + example: + with TemporaryDirectory() as tmpdir: + ... + Upon exiting the context, the directory and everything contained + in it are removed. + """ + + def __init__(self, suffix=None, prefix=None, dir=None): + self.name = mkdtemp(suffix, prefix, dir) + self._finalizer = finalize( + self, self._cleanup, self.name, + warn_message="Implicitly cleaning up {!r}".format(self)) + + @classmethod + def _cleanup(cls, name, warn_message): + _rmtree(name) + _warnings.warn(warn_message, _ResourceWarning) + + + def __repr__(self): + return "<{} {!r}>".format(self.__class__.__name__, self.name) + + def __enter__(self): + return self.name + + def __exit__(self, exc, value, tb): + self.cleanup() + + def cleanup(self): + if self._finalizer.detach(): + _rmtree(self.name) diff --git a/src/device_trezor/trezor/tools/py2backports/weakref.py b/src/device_trezor/trezor/tools/py2backports/weakref.py new file mode 100644 index 000000000..eb646812b --- /dev/null +++ b/src/device_trezor/trezor/tools/py2backports/weakref.py @@ -0,0 +1,148 @@ +""" +https://github.com/pjdelport/backports.weakref/blob/master/src/backports/weakref.py +Partial backport of Python 3.6's weakref module: + finalize (new in Python 3.4) +Backport modifications are marked with "XXX backport". +""" +from __future__ import absolute_import + +import itertools +import sys +from weakref import ref + +__all__ = ['finalize'] + + +class finalize(object): + """Class for finalization of weakrefable objects + finalize(obj, func, *args, **kwargs) returns a callable finalizer + object which will be called when obj is garbage collected. The + first time the finalizer is called it evaluates func(*arg, **kwargs) + and returns the result. After this the finalizer is dead, and + calling it just returns None. + When the program exits any remaining finalizers for which the + atexit attribute is true will be run in reverse order of creation. + By default atexit is true. + """ + + # Finalizer objects don't have any state of their own. They are + # just used as keys to lookup _Info objects in the registry. This + # ensures that they cannot be part of a ref-cycle. + + __slots__ = () + _registry = {} + _shutdown = False + _index_iter = itertools.count() + _dirty = False + _registered_with_atexit = False + + class _Info(object): + __slots__ = ("weakref", "func", "args", "kwargs", "atexit", "index") + + def __init__(self, obj, func, *args, **kwargs): + if not self._registered_with_atexit: + # We may register the exit function more than once because + # of a thread race, but that is harmless + import atexit + atexit.register(self._exitfunc) + finalize._registered_with_atexit = True + info = self._Info() + info.weakref = ref(obj, self) + info.func = func + info.args = args + info.kwargs = kwargs or None + info.atexit = True + info.index = next(self._index_iter) + self._registry[self] = info + finalize._dirty = True + + def __call__(self, _=None): + """If alive then mark as dead and return func(*args, **kwargs); + otherwise return None""" + info = self._registry.pop(self, None) + if info and not self._shutdown: + return info.func(*info.args, **(info.kwargs or {})) + + def detach(self): + """If alive then mark as dead and return (obj, func, args, kwargs); + otherwise return None""" + info = self._registry.get(self) + obj = info and info.weakref() + if obj is not None and self._registry.pop(self, None): + return (obj, info.func, info.args, info.kwargs or {}) + + def peek(self): + """If alive then return (obj, func, args, kwargs); + otherwise return None""" + info = self._registry.get(self) + obj = info and info.weakref() + if obj is not None: + return (obj, info.func, info.args, info.kwargs or {}) + + @property + def alive(self): + """Whether finalizer is alive""" + return self in self._registry + + @property + def atexit(self): + """Whether finalizer should be called at exit""" + info = self._registry.get(self) + return bool(info) and info.atexit + + @atexit.setter + def atexit(self, value): + info = self._registry.get(self) + if info: + info.atexit = bool(value) + + def __repr__(self): + info = self._registry.get(self) + obj = info and info.weakref() + if obj is None: + return '<%s object at %#x; dead>' % (type(self).__name__, id(self)) + else: + return '<%s object at %#x; for %r at %#x>' % \ + (type(self).__name__, id(self), type(obj).__name__, id(obj)) + + @classmethod + def _select_for_exit(cls): + # Return live finalizers marked for exit, oldest first + L = [(f,i) for (f,i) in cls._registry.items() if i.atexit] + L.sort(key=lambda item:item[1].index) + return [f for (f,i) in L] + + @classmethod + def _exitfunc(cls): + # At shutdown invoke finalizers for which atexit is true. + # This is called once all other non-daemonic threads have been + # joined. + reenable_gc = False + try: + if cls._registry: + import gc + if gc.isenabled(): + reenable_gc = True + gc.disable() + pending = None + while True: + if pending is None or finalize._dirty: + pending = cls._select_for_exit() + finalize._dirty = False + if not pending: + break + f = pending.pop() + try: + # gc is disabled, so (assuming no daemonic + # threads) the following is the only line in + # this function which might trigger creation + # of a new finalizer + f() + except Exception: + sys.excepthook(*sys.exc_info()) + assert f not in cls._registry + finally: + # prevent any more finalizers from executing during shutdown + finalize._shutdown = True + if reenable_gc: + gc.enable() diff --git a/src/device_trezor/trezor/transport.cpp b/src/device_trezor/trezor/transport.cpp new file mode 100644 index 000000000..cd66e59e8 --- /dev/null +++ b/src/device_trezor/trezor/transport.cpp @@ -0,0 +1,1050 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifdef WITH_DEVICE_TREZOR_WEBUSB +#include <libusb.h> +#endif + +#include <boost/endian/conversion.hpp> +#include <boost/asio/io_service.hpp> +#include <boost/asio/ip/udp.hpp> +#include <boost/date_time/posix_time/posix_time_types.hpp> +#include <boost/format.hpp> +#include "transport.hpp" +#include "messages/messages-common.pb.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "device.trezor.transport" + +using namespace std; +using json = rapidjson::Document; + + +namespace hw{ +namespace trezor{ + + bool t_serialize(const std::string & in, std::string & out){ + out = in; + return true; + } + + bool t_serialize(const json_val & in, std::string & out){ + rapidjson::StringBuffer sb; + rapidjson::Writer<rapidjson::StringBuffer> writer(sb); + in.Accept(writer); + out = sb.GetString(); + return true; + } + + std::string t_serialize(const json_val & in){ + std::string ret; + t_serialize(in, ret); + return ret; + } + + bool t_deserialize(const std::string & in, std::string & out){ + out = in; + return true; + } + + bool t_deserialize(const std::string & in, json & out){ + if (out.Parse(in.c_str()).HasParseError()) { + throw exc::CommunicationException("JSON parse error"); + } + return true; + } + + static std::string json_get_string(const rapidjson::Value & in){ + return std::string(in.GetString()); + } + + // + // Helpers + // + +#define PROTO_HEADER_SIZE 6 + + static size_t message_size(const google::protobuf::Message &req){ + return static_cast<size_t>(req.ByteSize()); + } + + static size_t serialize_message_buffer_size(size_t msg_size) { + return PROTO_HEADER_SIZE + msg_size; // tag 2B + len 4B + } + + static void serialize_message_header(void * buff, uint16_t tag, uint32_t len){ + uint16_t wire_tag = boost::endian::native_to_big(static_cast<uint16_t>(tag)); + uint32_t wire_len = boost::endian::native_to_big(static_cast<uint32_t>(len)); + memcpy(buff, (void *) &wire_tag, 2); + memcpy((uint8_t*)buff + 2, (void *) &wire_len, 4); + } + + static void deserialize_message_header(const void * buff, uint16_t & tag, uint32_t & len){ + uint16_t wire_tag; + uint32_t wire_len; + memcpy(&wire_tag, buff, 2); + memcpy(&wire_len, (uint8_t*)buff + 2, 4); + + tag = boost::endian::big_to_native(wire_tag); + len = boost::endian::big_to_native(wire_len); + } + + static void serialize_message(const google::protobuf::Message &req, size_t msg_size, uint8_t * buff, size_t buff_size) { + auto msg_wire_num = MessageMapper::get_message_wire_number(req); + const auto req_buffer_size = serialize_message_buffer_size(msg_size); + if (req_buffer_size > buff_size){ + throw std::invalid_argument("Buffer too small"); + } + + serialize_message_header(buff, msg_wire_num, msg_size); + if (!req.SerializeToArray(buff + 6, msg_size)){ + throw exc::EncodingException("Message serialization error"); + } + } + + // + // Communication protocol + // + +#define REPLEN 64 + + void ProtocolV1::write(Transport & transport, const google::protobuf::Message & req){ + const auto msg_size = message_size(req); + const auto buff_size = serialize_message_buffer_size(msg_size) + 2; + + std::unique_ptr<uint8_t[]> req_buff(new uint8_t[buff_size]); + uint8_t * req_buff_raw = req_buff.get(); + req_buff_raw[0] = '#'; + req_buff_raw[1] = '#'; + + serialize_message(req, msg_size, req_buff_raw + 2, buff_size - 2); + + size_t offset = 0; + uint8_t chunk_buff[REPLEN]; + + // Chunk by chunk upload + while(offset < buff_size){ + auto to_copy = std::min((size_t)(buff_size - offset), (size_t)(REPLEN - 1)); + + chunk_buff[0] = '?'; + memcpy(chunk_buff + 1, req_buff_raw + offset, to_copy); + + // Pad with zeros + if (to_copy < REPLEN - 1){ + memset(chunk_buff + 1 + to_copy, 0, REPLEN - 1 - to_copy); + } + + transport.write_chunk(chunk_buff, REPLEN); + offset += REPLEN - 1; + } + } + + void ProtocolV1::read(Transport & transport, std::shared_ptr<google::protobuf::Message> & msg, messages::MessageType * msg_type){ + char chunk[REPLEN]; + + // Initial chunk read + size_t nread = transport.read_chunk(chunk, REPLEN); + if (nread != REPLEN){ + throw exc::CommunicationException("Read chunk has invalid size"); + } + + if (strncmp(chunk, "?##", 3) != 0){ + throw exc::CommunicationException("Malformed chunk"); + } + + uint16_t tag; + uint32_t len; + nread -= 3 + 6; + deserialize_message_header(chunk + 3, tag, len); + + std::string data_acc(chunk + 3 + 6, nread); + data_acc.reserve(len); + + while(nread < len){ + const size_t cur = transport.read_chunk(chunk, REPLEN); + if (chunk[0] != '?'){ + throw exc::CommunicationException("Chunk malformed"); + } + + data_acc.append(chunk + 1, cur - 1); + nread += cur - 1; + } + + if (msg_type){ + *msg_type = static_cast<messages::MessageType>(tag); + } + + if (nread < len){ + throw exc::CommunicationException("Response incomplete"); + } + + std::shared_ptr<google::protobuf::Message> msg_wrap(MessageMapper::get_message(tag)); + if (!msg_wrap->ParseFromArray(data_acc.c_str(), len)){ + throw exc::CommunicationException("Message could not be parsed"); + } + + msg = msg_wrap; + } + + // + // Bridge transport + // + + const char * BridgeTransport::PATH_PREFIX = "bridge:"; + + std::string BridgeTransport::get_path() const { + if (!m_device_path){ + return ""; + } + + std::string path(PATH_PREFIX); + return path + m_device_path.get(); + } + + void BridgeTransport::enumerate(t_transport_vect & res) { + json bridge_res; + std::string req; + + bool req_status = invoke_bridge_http("/enumerate", req, bridge_res, m_http_client); + if (!req_status){ + throw exc::CommunicationException("Bridge enumeration failed"); + } + + for(rapidjson::Value::ConstValueIterator itr = bridge_res.Begin(); itr != bridge_res.End(); ++itr){ + auto element = itr->GetObject(); + auto t = std::make_shared<BridgeTransport>(boost::make_optional(json_get_string(element["path"]))); + t->m_device_info.emplace(); + t->m_device_info->CopyFrom(*itr, t->m_device_info->GetAllocator()); + res.push_back(t); + } + } + + void BridgeTransport::open() { + if (!m_device_path){ + throw exc::CommunicationException("Coud not open, empty device path"); + } + + std::string uri = "/acquire/" + m_device_path.get() + "/null"; + std::string req; + json bridge_res; + bool req_status = invoke_bridge_http(uri, req, bridge_res, m_http_client); + if (!req_status){ + throw exc::CommunicationException("Failed to acquire device"); + } + + m_session = boost::make_optional(json_get_string(bridge_res["session"])); + } + + void BridgeTransport::close() { + if (!m_device_path || !m_session){ + throw exc::CommunicationException("Device not open"); + } + + std::string uri = "/release/" + m_session.get(); + std::string req; + json bridge_res; + bool req_status = invoke_bridge_http(uri, req, bridge_res, m_http_client); + if (!req_status){ + throw exc::CommunicationException("Failed to release device"); + } + + m_session = boost::none; + } + + void BridgeTransport::write(const google::protobuf::Message &req) { + m_response = boost::none; + + const auto msg_size = message_size(req); + const auto buff_size = serialize_message_buffer_size(msg_size); + + std::unique_ptr<uint8_t[]> req_buff(new uint8_t[buff_size]); + uint8_t * req_buff_raw = req_buff.get(); + + serialize_message(req, msg_size, req_buff_raw, buff_size); + + std::string uri = "/call/" + m_session.get(); + std::string req_hex = epee::to_hex::string(epee::span<const std::uint8_t>(req_buff_raw, buff_size)); + std::string res_hex; + + bool req_status = invoke_bridge_http(uri, req_hex, res_hex, m_http_client); + if (!req_status){ + throw exc::CommunicationException("Call method failed"); + } + + m_response = res_hex; + } + + void BridgeTransport::read(std::shared_ptr<google::protobuf::Message> & msg, messages::MessageType * msg_type) { + if (!m_response){ + throw exc::CommunicationException("Could not read, no response stored"); + } + + std::string bin_data; + if (!epee::string_tools::parse_hexstr_to_binbuff(m_response.get(), bin_data)){ + throw exc::CommunicationException("Response is not well hexcoded"); + } + + uint16_t msg_tag; + uint32_t msg_len; + deserialize_message_header(bin_data.c_str(), msg_tag, msg_len); + if (bin_data.size() != msg_len + 6){ + throw exc::CommunicationException("Response is not well hexcoded"); + } + + if (msg_type){ + *msg_type = static_cast<messages::MessageType>(msg_tag); + } + + std::shared_ptr<google::protobuf::Message> msg_wrap(MessageMapper::get_message(msg_tag)); + if (!msg_wrap->ParseFromArray(bin_data.c_str() + 6, msg_len)){ + throw exc::EncodingException("Response is not well hexcoded"); + } + msg = msg_wrap; + } + + const boost::optional<json> & BridgeTransport::device_info() const { + return m_device_info; + } + + std::ostream& BridgeTransport::dump(std::ostream& o) const { + return o << "BridgeTransport<path=" << (m_device_path ? get_path() : "None") + << ", info=" << (m_device_info ? t_serialize(m_device_info.get()) : "None") + << ", session=" << (m_session ? m_session.get() : "None") + << ">"; + } + + // + // UdpTransport + // + const char * UdpTransport::PATH_PREFIX = "udp:"; + const char * UdpTransport::DEFAULT_HOST = "127.0.0.1"; + const int UdpTransport::DEFAULT_PORT = 21324; + + UdpTransport::UdpTransport(boost::optional<std::string> device_path, + boost::optional<std::shared_ptr<Protocol>> proto) : + m_io_service(), m_deadline(m_io_service) + { + m_device_port = DEFAULT_PORT; + if (device_path) { + const std::string device_str = device_path.get(); + auto delim = device_str.find(':'); + if (delim == std::string::npos) { + m_device_host = device_str; + } else { + m_device_host = device_str.substr(0, delim); + m_device_port = std::stoi(device_str.substr(delim + 1)); + } + } else { + m_device_host = DEFAULT_HOST; + } + + if (m_device_port <= 1024 || m_device_port > 65535){ + throw std::invalid_argument("Port number invalid"); + } + + if (m_device_host != "localhost" && m_device_host != DEFAULT_HOST){ + throw std::invalid_argument("Local endpoint allowed only"); + } + + m_proto = proto ? proto.get() : std::make_shared<ProtocolV1>(); + } + + std::string UdpTransport::get_path() const { + std::string path(PATH_PREFIX); + return path + m_device_host + ":" + std::to_string(m_device_port); + } + + void UdpTransport::require_socket(){ + if (!m_socket){ + throw exc::NotConnectedException("Socket not connected"); + } + } + + bool UdpTransport::ping(){ + return ping_int(); + } + + bool UdpTransport::ping_int(boost::posix_time::time_duration timeout){ + require_socket(); + try { + std::string req = "PINGPING"; + char res[8]; + + m_socket->send_to(boost::asio::buffer(req.c_str(), req.size()), m_endpoint); + receive(res, 8, nullptr, false, timeout); + + return memcmp(res, "PONGPONG", 8) == 0; + + } catch(...){ + return false; + } + } + + void UdpTransport::enumerate(t_transport_vect & res) { + std::shared_ptr<UdpTransport> t = std::make_shared<UdpTransport>(); + bool t_works = false; + + try{ + t->open(); + t_works = t->ping(); + } catch(...) { + + } + t->close(); + if (t_works){ + res.push_back(t); + } + } + + void UdpTransport::open() { + udp::resolver resolver(m_io_service); + udp::resolver::query query(udp::v4(), m_device_host, std::to_string(m_device_port)); + m_endpoint = *resolver.resolve(query); + + m_socket.reset(new udp::socket(m_io_service)); + m_socket->open(udp::v4()); + + m_deadline.expires_at(boost::posix_time::pos_infin); + check_deadline(); + + m_proto->session_begin(*this); + } + + void UdpTransport::close() { + if (!m_socket){ + throw exc::CommunicationException("Socket is already closed"); + } + + m_proto->session_end(*this); + m_socket->close(); + m_socket = nullptr; + } + + void UdpTransport::write_chunk(const void * buff, size_t size){ + require_socket(); + + if (size != 64){ + throw exc::CommunicationException("Invalid chunk size"); + } + + auto written = m_socket->send_to(boost::asio::buffer(buff, size), m_endpoint); + if (size != written){ + throw exc::CommunicationException("Could not send the whole chunk"); + } + } + + size_t UdpTransport::read_chunk(void * buff, size_t size){ + require_socket(); + if (size < 64){ + throw std::invalid_argument("Buffer too small"); + } + + ssize_t len; + while(true) { + try { + boost::system::error_code ec; + len = receive(buff, size, &ec, true); + if (ec == boost::asio::error::operation_aborted) { + continue; + } else if (ec) { + throw exc::CommunicationException(std::string("Comm error: ") + ec.message()); + } + + if (len != 64) { + throw exc::CommunicationException("Invalid chunk size"); + } + + break; + + } catch(exc::CommunicationException const& e){ + throw; + } catch(std::exception const& e){ + MWARNING("Error reading chunk, reason: " << e.what()); + throw exc::CommunicationException(std::string("Chunk read error: ") + std::string(e.what())); + } + } + + return static_cast<size_t>(len); + } + + ssize_t UdpTransport::receive(void * buff, size_t size, boost::system::error_code * error_code, bool no_throw, boost::posix_time::time_duration timeout){ + boost::system::error_code ec; + boost::asio::mutable_buffer buffer = boost::asio::buffer(buff, size); + + require_socket(); + + // Set a deadline for the asynchronous operation. + m_deadline.expires_from_now(timeout); + + // Set up the variables that receive the result of the asynchronous + // operation. The error code is set to would_block to signal that the + // operation is incomplete. Asio guarantees that its asynchronous + // operations will never fail with would_block, so any other value in + // ec indicates completion. + ec = boost::asio::error::would_block; + std::size_t length = 0; + + // Start the asynchronous operation itself. The handle_receive function + // used as a callback will update the ec and length variables. + m_socket->async_receive_from(boost::asio::buffer(buffer), m_endpoint, + boost::bind(&UdpTransport::handle_receive, _1, _2, &ec, &length)); + + // Block until the asynchronous operation has completed. + do { + m_io_service.run_one(); + } + while (ec == boost::asio::error::would_block); + + if (error_code){ + *error_code = ec; + } + + if (no_throw){ + return length; + } + + // Operation result + if (ec == boost::asio::error::operation_aborted){ + throw exc::TimeoutException(); + + } else if (ec) { + MWARNING("Reading from UDP socket failed: " << ec.message()); + throw exc::CommunicationException(); + + } + + return length; + } + + void UdpTransport::write(const google::protobuf::Message &req) { + m_proto->write(*this, req); + } + + void UdpTransport::read(std::shared_ptr<google::protobuf::Message> & msg, messages::MessageType * msg_type) { + m_proto->read(*this, msg, msg_type); + } + + void UdpTransport::check_deadline(){ + if (!m_socket){ + return; // no active socket. + } + + // Check whether the deadline has passed. We compare the deadline against + // the current time since a new asynchronous operation may have moved the + // deadline before this actor had a chance to run. + if (m_deadline.expires_at() <= boost::asio::deadline_timer::traits_type::now()) + { + // The deadline has passed. The outstanding asynchronous operation needs + // to be cancelled so that the blocked receive() function will return. + // + // Please note that cancel() has portability issues on some versions of + // Microsoft Windows, and it may be necessary to use close() instead. + // Consult the documentation for cancel() for further information. + m_socket->cancel(); + + // There is no longer an active deadline. The expiry is set to positive + // infinity so that the actor takes no action until a new deadline is set. + m_deadline.expires_at(boost::posix_time::pos_infin); + } + + // Put the actor back to sleep. + m_deadline.async_wait(boost::bind(&UdpTransport::check_deadline, this)); + } + + void UdpTransport::handle_receive(const boost::system::error_code &ec, std::size_t length, + boost::system::error_code *out_ec, std::size_t *out_length) { + *out_ec = ec; + *out_length = length; + } + + std::ostream& UdpTransport::dump(std::ostream& o) const { + return o << "UdpTransport<path=" << get_path() + << ", socket_alive=" << (m_socket ? "true" : "false") + << ">"; + } + +#ifdef WITH_DEVICE_TREZOR_WEBUSB + + static bool is_trezor1(libusb_device_descriptor * info){ + return info->idVendor == 0x534C && info->idProduct == 0x0001; + } + + static bool is_trezor2(libusb_device_descriptor * info){ + return info->idVendor == 0x1209 && info->idProduct == 0x53C1; + } + + static bool is_trezor2_bl(libusb_device_descriptor * info){ + return info->idVendor == 0x1209 && info->idProduct == 0x53C0; + } + + static uint8_t get_trezor_dev_mask(libusb_device_descriptor * info){ + uint8_t mask = 0; + CHECK_AND_ASSERT_THROW_MES(info, "Empty device descriptor"); + mask |= is_trezor1(info) ? 1 : 0; + mask |= is_trezor2(info) ? 2 : 0; + mask |= is_trezor2_bl(info) ? 4 : 0; + return mask; + } + + static void set_libusb_log(libusb_context *ctx){ + CHECK_AND_ASSERT_THROW_MES(ctx, "Null libusb context"); + + // http://libusb.sourceforge.net/api-1.0/group__libusb__lib.html +#if defined(LIBUSB_API_VERSION) && (LIBUSB_API_VERSION >= 0x01000106) +# define TREZOR_LIBUSB_SET_DEBUG(ctx, level) libusb_set_option(ctx, LIBUSB_OPTION_LOG_LEVEL, level) +#else +# define TREZOR_LIBUSB_SET_DEBUG(ctx, level) libusb_set_debug(ctx, level) +#endif + + if (ELPP->vRegistry()->allowed(el::Level::Debug, MONERO_DEFAULT_LOG_CATEGORY)) + TREZOR_LIBUSB_SET_DEBUG(ctx, 3); + else if (ELPP->vRegistry()->allowed(el::Level::Warning, MONERO_DEFAULT_LOG_CATEGORY)) + TREZOR_LIBUSB_SET_DEBUG(ctx, 2); + else if (ELPP->vRegistry()->allowed(el::Level::Error, MONERO_DEFAULT_LOG_CATEGORY)) + TREZOR_LIBUSB_SET_DEBUG(ctx, 1); + +#undef TREZOR_LIBUSB_SET_DEBUG + } + + static int get_libusb_ports(libusb_device *dev, std::vector<uint8_t> &path){ + uint8_t tmp_path[16]; + int r = libusb_get_port_numbers(dev, tmp_path, sizeof(tmp_path)); + CHECK_AND_ASSERT_MES(r != LIBUSB_ERROR_OVERFLOW, -1, "Libusb path array too small"); + CHECK_AND_ASSERT_MES(r >= 0, -1, "Libusb path array error"); + + path.resize(r); + for (int i = 0; i < r; i++){ + path[i] = tmp_path[i]; + } + + return 0; + } + + static std::string get_usb_path(uint8_t bus_id, const std::vector<uint8_t> &path){ + std::stringstream ss; + ss << WebUsbTransport::PATH_PREFIX << (boost::format("%03d") % ((int)bus_id)); + for(uint8_t port : path){ + ss << ":" << ((int) port); + } + return ss.str(); + } + + const char * WebUsbTransport::PATH_PREFIX = "webusb:"; + + WebUsbTransport::WebUsbTransport( + boost::optional<libusb_device_descriptor*> descriptor, + boost::optional<std::shared_ptr<Protocol>> proto + ): m_conn_count(0), + m_usb_session(nullptr), m_usb_device(nullptr), m_usb_device_handle(nullptr), + m_bus_id(-1), m_device_addr(-1) + { + if (descriptor){ + libusb_device_descriptor * desc = new libusb_device_descriptor; + memcpy(desc, descriptor.get(), sizeof(libusb_device_descriptor)); + this->m_usb_device_desc.reset(desc); + } + + m_proto = proto ? proto.get() : std::make_shared<ProtocolV1>(); + +#ifdef WITH_TREZOR_DEBUG + m_debug_mode = false; +#endif + } + + WebUsbTransport::~WebUsbTransport(){ + if (m_usb_device){ + close(); + } + + if (m_usb_session) { + libusb_exit(m_usb_session); + m_usb_session = nullptr; + } + } + + void WebUsbTransport::require_device() const{ + if (!m_usb_device_desc){ + throw std::runtime_error("No USB device specified"); + } + } + + void WebUsbTransport::require_connected() const{ + require_device(); + if (!m_usb_device_handle){ + throw std::runtime_error("USB Device not opened"); + } + } + + void WebUsbTransport::enumerate(t_transport_vect & res) { + int r; + libusb_device **devs; + libusb_context *ctx = nullptr; + + r = libusb_init(&ctx); + CHECK_AND_ASSERT_THROW_MES(r >= 0, "Unable to init libusb"); + + set_libusb_log(ctx); + + ssize_t cnt = libusb_get_device_list(ctx, &devs); + if (cnt < 0){ + libusb_exit(ctx); + throw std::runtime_error("Unable to enumerate libusb devices"); + } + + MTRACE("Libusb devices: " << cnt); + + for(ssize_t i = 0; i < cnt; i++) { + libusb_device_descriptor desc{}; + r = libusb_get_device_descriptor(devs[i], &desc); + if (r < 0){ + MERROR("Unable to get libusb device descriptor " << i); + continue; + } + + const auto trezor_mask = get_trezor_dev_mask(&desc); + if (!trezor_mask){ + continue; + } + + MTRACE("Found Trezor device: " << desc.idVendor << ":" << desc.idProduct << " mask " << (int)trezor_mask); + + auto t = std::make_shared<WebUsbTransport>(boost::make_optional(&desc)); + t->m_bus_id = libusb_get_bus_number(devs[i]); + t->m_device_addr = libusb_get_device_address(devs[i]); + + // Port resolution may fail. Non-critical error, just addressing precision is decreased. + get_libusb_ports(devs[i], t->m_port_numbers); + + res.push_back(t); + } + + libusb_free_device_list(devs, 1); + libusb_exit(ctx); + } + + std::string WebUsbTransport::get_path() const { + if (!m_usb_device_desc){ + return ""; + } + + return get_usb_path(static_cast<uint8_t>(m_bus_id), m_port_numbers); + }; + + void WebUsbTransport::open() { + const int interface = get_interface(); + if (m_conn_count > 0){ + MTRACE("Already opened, count: " << m_conn_count); + m_conn_count += 1; + return; + } + +#define TREZOR_DESTROY_SESSION() do { libusb_exit(m_usb_session); m_usb_session = nullptr; } while(0) + + int r; + libusb_device **devs = nullptr; + + if (m_usb_session) { + TREZOR_DESTROY_SESSION(); + } + + r = libusb_init(&m_usb_session); + CHECK_AND_ASSERT_THROW_MES(r >= 0, "Unable to init libusb"); + set_libusb_log(m_usb_session); + + bool found = false; + int open_res = 0; + + ssize_t cnt = libusb_get_device_list(m_usb_session, &devs); + if (cnt < 0){ + TREZOR_DESTROY_SESSION(); + throw std::runtime_error("Unable to enumerate libusb devices"); + } + + for (ssize_t i = 0; i < cnt; i++) { + libusb_device_descriptor desc{}; + r = libusb_get_device_descriptor(devs[i], &desc); + if (r < 0){ + MERROR("Unable to get libusb device descriptor " << i); + continue; + } + + const auto trezor_mask = get_trezor_dev_mask(&desc); + if (!trezor_mask) { + continue; + } + + auto bus_id = libusb_get_bus_number(devs[i]); + std::vector<uint8_t> path; + + // Port resolution may fail. Non-critical error, just addressing precision is decreased. + get_libusb_ports(devs[i], path); + + MTRACE("Found Trezor device: " << desc.idVendor << ":" << desc.idProduct + << ", mask: " << (int)trezor_mask + << ". path: " << get_usb_path(bus_id, path)); + + if (bus_id == m_bus_id && path == m_port_numbers) { + found = true; + m_usb_device = devs[i]; + open_res = libusb_open(m_usb_device, &m_usb_device_handle); + break; + } + } + + libusb_free_device_list(devs, 1); + + if (!found){ + TREZOR_DESTROY_SESSION(); + throw exc::DeviceAcquireException("Device not found"); + + } else if (found && open_res != 0) { + m_usb_device_handle = nullptr; + m_usb_device = nullptr; + TREZOR_DESTROY_SESSION(); + throw exc::DeviceAcquireException("Unable to open libusb device"); + } + + r = libusb_claim_interface(m_usb_device_handle, interface); + + if (r != 0){ + libusb_close(m_usb_device_handle); + m_usb_device_handle = nullptr; + m_usb_device = nullptr; + TREZOR_DESTROY_SESSION(); + throw exc::DeviceAcquireException("Unable to claim libusb device"); + } + + m_conn_count = 1; + m_proto->session_begin(*this); + +#undef TREZOR_DESTROY_SESSION + }; + + void WebUsbTransport::close() { + m_conn_count -= 1; + + if (m_conn_count < 0){ + MERROR("Close counter is negative: " << m_conn_count); + + } else if (m_conn_count == 0){ + MTRACE("Closing webusb device"); + + m_proto->session_end(*this); + + int r = libusb_release_interface(m_usb_device_handle, get_interface()); + if (r != 0){ + MERROR("Could not release libusb interface: " << r); + } + + m_usb_device = nullptr; + if (m_usb_device_handle) { + libusb_close(m_usb_device_handle); + m_usb_device_handle = nullptr; + } + + if (m_usb_session) { + libusb_exit(m_usb_session); + m_usb_session = nullptr; + } + } + }; + + + int WebUsbTransport::get_interface() const{ + const int INTERFACE_NORMAL = 0; +#ifdef WITH_TREZOR_DEBUG + const int INTERFACE_DEBUG = 1; + return m_debug_mode ? INTERFACE_DEBUG : INTERFACE_NORMAL; +#else + return INTERFACE_NORMAL; +#endif + } + + unsigned char WebUsbTransport::get_endpoint() const{ + const unsigned char ENDPOINT_NORMAL = 1; +#ifdef WITH_TREZOR_DEBUG + const unsigned char ENDPOINT_DEBUG = 2; + return m_debug_mode ? ENDPOINT_DEBUG : ENDPOINT_NORMAL; +#else + return ENDPOINT_NORMAL; +#endif + } + + void WebUsbTransport::write(const google::protobuf::Message &req) { + m_proto->write(*this, req); + }; + + void WebUsbTransport::read(std::shared_ptr<google::protobuf::Message> & msg, messages::MessageType * msg_type) { + m_proto->read(*this, msg, msg_type); + }; + + void WebUsbTransport::write_chunk(const void * buff, size_t size) { + require_connected(); + if (size != REPLEN){ + throw exc::CommunicationException("Invalid chunk size: "); + } + + unsigned char endpoint = get_endpoint(); + endpoint = (endpoint & ~LIBUSB_ENDPOINT_DIR_MASK) | LIBUSB_ENDPOINT_OUT; + + int transferred = 0; + int r = libusb_interrupt_transfer(m_usb_device_handle, endpoint, (unsigned char*)buff, (int)size, &transferred, 0); + CHECK_AND_ASSERT_THROW_MES(r == 0, "Unable to transfer, r: " << r); + if (transferred != (int)size){ + throw exc::CommunicationException("Could not transfer chunk"); + } + }; + + size_t WebUsbTransport::read_chunk(void * buff, size_t size) { + require_connected(); + unsigned char endpoint = get_endpoint(); + endpoint = (endpoint & ~LIBUSB_ENDPOINT_DIR_MASK) | LIBUSB_ENDPOINT_IN; + + int transferred = 0; + int r = libusb_interrupt_transfer(m_usb_device_handle, endpoint, (unsigned char*)buff, (int)size, &transferred, 0); + CHECK_AND_ASSERT_THROW_MES(r == 0, "Unable to transfer, r: " << r); + if (transferred != (int)size){ + throw exc::CommunicationException("Could not read the chunk"); + } + + return transferred; + }; + + std::ostream& WebUsbTransport::dump(std::ostream& o) const { + o << "WebUsbTransport<path=" << get_path() + << ", vendorId=" << (m_usb_device_desc ? std::to_string(m_usb_device_desc->idVendor) : "?") + << ", productId=" << (m_usb_device_desc ? std::to_string(m_usb_device_desc->idProduct) : "?") + << ", deviceType="; + + if (m_usb_device_desc){ + if (is_trezor1(m_usb_device_desc.get())) + o << "TrezorOne"; + else if (is_trezor2(m_usb_device_desc.get())) + o << "TrezorT"; + else if (is_trezor2_bl(m_usb_device_desc.get())) + o << "TrezorT BL"; + } else { + o << "?"; + } + + return o << ">"; + }; + +#endif // WITH_DEVICE_TREZOR_WEBUSB + + void enumerate(t_transport_vect & res){ + BridgeTransport bt; + try{ + bt.enumerate(res); + } catch (const std::exception & e){ + MERROR("BridgeTransport enumeration failed:" << e.what()); + } + +#ifdef WITH_DEVICE_TREZOR_WEBUSB + hw::trezor::WebUsbTransport btw; + try{ + btw.enumerate(res); + } catch (const std::exception & e){ + MERROR("WebUsbTransport enumeration failed:" << e.what()); + } +#endif + +#ifdef WITH_DEVICE_TREZOR_UDP + hw::trezor::UdpTransport btu; + try{ + btu.enumerate(res); + } catch (const std::exception & e){ + MERROR("UdpTransport enumeration failed:" << e.what()); + } +#endif + } + + std::shared_ptr<Transport> transport(const std::string & path){ + if (boost::starts_with(path, BridgeTransport::PATH_PREFIX)){ + return std::make_shared<BridgeTransport>(path.substr(strlen(BridgeTransport::PATH_PREFIX))); + + } else if (boost::starts_with(path, UdpTransport::PATH_PREFIX)){ + return std::make_shared<UdpTransport>(path.substr(strlen(UdpTransport::PATH_PREFIX))); + + } else { + throw std::invalid_argument("Unknown Trezor device path: " + path); + + } + } + + void throw_failure_exception(const messages::common::Failure * failure) { + if (failure == nullptr){ + throw std::invalid_argument("Failure message cannot be null"); + } + + boost::optional<std::string> message = failure->has_message() ? boost::make_optional(failure->message()) : boost::none; + boost::optional<uint32_t> code = failure->has_code() ? boost::make_optional(static_cast<uint32_t>(failure->code())) : boost::none; + if (!code){ + throw exc::proto::FailureException(code, message); + } + + auto ecode = failure->code(); + if (ecode == messages::common::Failure_FailureType_Failure_UnexpectedMessage){ + throw exc::proto::UnexpectedMessageException(code, message); + } else if (ecode == messages::common::Failure_FailureType_Failure_ActionCancelled){ + throw exc::proto::CancelledException(code, message); + } else if (ecode == messages::common::Failure_FailureType_Failure_PinExpected){ + throw exc::proto::PinExpectedException(code, message); + } else if (ecode == messages::common::Failure_FailureType_Failure_PinInvalid){ + throw exc::proto::InvalidPinException(code, message); + } else if (ecode == messages::common::Failure_FailureType_Failure_NotEnoughFunds){ + throw exc::proto::NotEnoughFundsException(code, message); + } else if (ecode == messages::common::Failure_FailureType_Failure_NotInitialized){ + throw exc::proto::NotInitializedException(code, message); + } else if (ecode == messages::common::Failure_FailureType_Failure_FirmwareError){ + throw exc::proto::FirmwareErrorException(code, message); + } else { + throw exc::proto::FailureException(code, message); + } + } + + GenericMessage::GenericMessage(messages::MessageType m_type, const shared_ptr<google::protobuf::Message> &m_msg) + : m_type(m_type), m_msg(m_msg), m_empty(false) {} + + std::ostream& operator<<(std::ostream& o, hw::trezor::Transport const& t){ + return t.dump(o); + } + + std::ostream& operator<<(std::ostream& o, std::shared_ptr<hw::trezor::Transport> const& t){ + if (!t){ + return o << "None"; + } + + return t->dump(o); + } + +} +} + + diff --git a/src/device_trezor/trezor/transport.hpp b/src/device_trezor/trezor/transport.hpp new file mode 100644 index 000000000..50c31cf73 --- /dev/null +++ b/src/device_trezor/trezor/transport.hpp @@ -0,0 +1,398 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef MONERO_TRANSPORT_H +#define MONERO_TRANSPORT_H + + +#include <boost/asio.hpp> +#include <boost/asio/deadline_timer.hpp> +#include <boost/array.hpp> +#include <boost/utility/string_ref.hpp> + +#include <typeinfo> +#include <type_traits> +#include "net/http_client.h" + +#include "rapidjson/document.h" +#include "rapidjson/writer.h" +#include "rapidjson/stringbuffer.h" + +#include "exceptions.hpp" +#include "trezor_defs.hpp" +#include "messages_map.hpp" + +#include "messages/messages.pb.h" +#include "messages/messages-common.pb.h" +#include "messages/messages-management.pb.h" +#include "messages/messages-monero.pb.h" + +namespace hw { +namespace trezor { + + using json = rapidjson::Document; + using json_val = rapidjson::Value; + namespace http = epee::net_utils::http; + + const std::string DEFAULT_BRIDGE = "127.0.0.1:21325"; + + // Base HTTP comm serialization. + bool t_serialize(const std::string & in, std::string & out); + bool t_serialize(const json_val & in, std::string & out); + std::string t_serialize(const json_val & in); + + bool t_deserialize(const std::string & in, std::string & out); + bool t_deserialize(const std::string & in, json & out); + + // Flexible json serialization. HTTP client tailored for bridge API + template<class t_req, class t_res, class t_transport> + bool invoke_bridge_http(const boost::string_ref uri, const t_req & out_struct, t_res & result_struct, t_transport& transport, const boost::string_ref method = "POST", std::chrono::milliseconds timeout = std::chrono::seconds(180)) + { + std::string req_param; + t_serialize(out_struct, req_param); + + http::fields_list additional_params; + additional_params.push_back(std::make_pair("Origin","https://monero.trezor.io")); + additional_params.push_back(std::make_pair("Content-Type","application/json; charset=utf-8")); + + const http::http_response_info* pri = nullptr; + if(!transport.invoke(uri, method, req_param, timeout, &pri, std::move(additional_params))) + { + MERROR("Failed to invoke http request to " << uri); + return false; + } + + if(!pri) + { + MERROR("Failed to invoke http request to " << uri << ", internal error (null response ptr)"); + return false; + } + + if(pri->m_response_code != 200) + { + MERROR("Failed to invoke http request to " << uri << ", wrong response code: " << pri->m_response_code + << " Response Body: " << pri->m_body); + return false; + } + + return t_deserialize(pri->m_body, result_struct); + } + + // Forward decl + class Transport; + class Protocol; + + // Communication protocol + class Protocol { + public: + Protocol() = default; + virtual ~Protocol() = default; + virtual void session_begin(Transport & transport){ }; + virtual void session_end(Transport & transport){ }; + virtual void write(Transport & transport, const google::protobuf::Message & req)= 0; + virtual void read(Transport & transport, std::shared_ptr<google::protobuf::Message> & msg, messages::MessageType * msg_type=nullptr)= 0; + }; + + class ProtocolV1 : public Protocol { + public: + ProtocolV1() = default; + virtual ~ProtocolV1() = default; + + void write(Transport & transport, const google::protobuf::Message & req) override; + void read(Transport & transport, std::shared_ptr<google::protobuf::Message> & msg, messages::MessageType * msg_type=nullptr) override; + }; + + + // Base transport + typedef std::vector<std::shared_ptr<Transport>> t_transport_vect; + + class Transport { + public: + Transport() = default; + virtual ~Transport() = default; + + virtual bool ping() { return false; }; + virtual std::string get_path() const { return ""; }; + virtual void enumerate(t_transport_vect & res){}; + virtual void open(){}; + virtual void close(){}; + virtual void write(const google::protobuf::Message & req) =0; + virtual void read(std::shared_ptr<google::protobuf::Message> & msg, messages::MessageType * msg_type=nullptr) =0; + + virtual void write_chunk(const void * buff, size_t size) { }; + virtual size_t read_chunk(void * buff, size_t size) { return 0; }; + virtual std::ostream& dump(std::ostream& o) const { return o << "Transport<>"; } + }; + + // Bridge transport + class BridgeTransport : public Transport { + public: + BridgeTransport( + boost::optional<std::string> device_path = boost::none, + boost::optional<std::string> bridge_host = boost::none): + m_device_path(device_path), + m_bridge_host(bridge_host ? bridge_host.get() : DEFAULT_BRIDGE), + m_response(boost::none), + m_session(boost::none), + m_device_info(boost::none) + { + m_http_client.set_server(m_bridge_host, boost::none, false); + } + + virtual ~BridgeTransport() = default; + + static const char * PATH_PREFIX; + + std::string get_path() const override; + void enumerate(t_transport_vect & res) override; + + void open() override; + void close() override; + + void write(const google::protobuf::Message &req) override; + void read(std::shared_ptr<google::protobuf::Message> & msg, messages::MessageType * msg_type=nullptr) override; + + const boost::optional<json> & device_info() const; + std::ostream& dump(std::ostream& o) const override; + + private: + epee::net_utils::http::http_simple_client m_http_client; + std::string m_bridge_host; + boost::optional<std::string> m_device_path; + boost::optional<std::string> m_session; + boost::optional<std::string> m_response; + boost::optional<json> m_device_info; + }; + + // UdpTransport transport + using boost::asio::ip::udp; + + class UdpTransport : public Transport { + public: + + explicit UdpTransport( + boost::optional<std::string> device_path=boost::none, + boost::optional<std::shared_ptr<Protocol>> proto=boost::none); + + virtual ~UdpTransport() = default; + + static const char * PATH_PREFIX; + static const char * DEFAULT_HOST; + static const int DEFAULT_PORT; + + bool ping() override; + std::string get_path() const override; + void enumerate(t_transport_vect & res) override; + + void open() override; + void close() override; + + void write(const google::protobuf::Message &req) override; + void read(std::shared_ptr<google::protobuf::Message> & msg, messages::MessageType * msg_type=nullptr) override; + + void write_chunk(const void * buff, size_t size) override; + size_t read_chunk(void * buff, size_t size) override; + + std::ostream& dump(std::ostream& o) const override; + + private: + void require_socket(); + ssize_t receive(void * buff, size_t size, boost::system::error_code * error_code=nullptr, bool no_throw=false, boost::posix_time::time_duration timeout=boost::posix_time::seconds(10)); + void check_deadline(); + static void handle_receive(const boost::system::error_code& ec, std::size_t length, + boost::system::error_code* out_ec, std::size_t* out_length); + bool ping_int(boost::posix_time::time_duration timeout=boost::posix_time::milliseconds(1500)); + + std::shared_ptr<Protocol> m_proto; + std::string m_device_host; + int m_device_port; + + std::unique_ptr<udp::socket> m_socket; + boost::asio::io_service m_io_service; + boost::asio::deadline_timer m_deadline; + udp::endpoint m_endpoint; + }; + +#ifdef WITH_DEVICE_TREZOR_WEBUSB +#include <libusb.h> + + class WebUsbTransport : public Transport { + public: + + explicit WebUsbTransport( + boost::optional<libusb_device_descriptor*> descriptor = boost::none, + boost::optional<std::shared_ptr<Protocol>> proto = boost::none + ); + + virtual ~WebUsbTransport(); + + static const char * PATH_PREFIX; + + std::string get_path() const override; + void enumerate(t_transport_vect & res) override; + + void open() override; + void close() override; + + void write(const google::protobuf::Message &req) override; + void read(std::shared_ptr<google::protobuf::Message> & msg, messages::MessageType * msg_type=nullptr) override; + + void write_chunk(const void * buff, size_t size) override; + size_t read_chunk(void * buff, size_t size) override; + + std::ostream& dump(std::ostream& o) const override; + + private: + void require_device() const; + void require_connected() const; + int get_interface() const; + unsigned char get_endpoint() const; + + int m_conn_count; + std::shared_ptr<Protocol> m_proto; + + libusb_context *m_usb_session; + libusb_device *m_usb_device; + libusb_device_handle *m_usb_device_handle; + std::unique_ptr<libusb_device_descriptor> m_usb_device_desc; + std::vector<uint8_t> m_port_numbers; + int m_bus_id; + int m_device_addr; + +#ifdef WITH_TREZOR_DEBUG + bool m_debug_mode; +#endif + }; + +#endif + + // + // General helpers + // + + /** + * Enumerates all transports + */ + void enumerate(t_transport_vect & res); + + /** + * Transforms path to the transport + */ + std::shared_ptr<Transport> transport(const std::string & path); + + /** + * Transforms path to the particular transport + */ + template<class t_transport> + std::shared_ptr<t_transport> transport_typed(const std::string & path){ + auto t = transport(path); + if (!t){ + return nullptr; + } + + return std::dynamic_pointer_cast<t_transport>(t); + } + + // Exception carries unexpected message being received + namespace exc { + class UnexpectedMessageException: public ProtocolException { + protected: + hw::trezor::messages::MessageType recvType; + std::shared_ptr<google::protobuf::Message> recvMsg; + + public: + using ProtocolException::ProtocolException; + UnexpectedMessageException(): ProtocolException("Trezor returned unexpected message") {}; + UnexpectedMessageException(hw::trezor::messages::MessageType recvType, + const std::shared_ptr<google::protobuf::Message> & recvMsg) + : recvType(recvType), recvMsg(recvMsg) { + reason = std::string("Trezor returned unexpected message: ") + std::to_string(recvType); + } + }; + } + + /** + * Throws corresponding failure exception. + */ + [[ noreturn ]] void throw_failure_exception(const messages::common::Failure * failure); + + /** + * Generic message holder, type + obj + */ + class GenericMessage { + public: + GenericMessage(): m_empty(true) {} + GenericMessage(messages::MessageType m_type, const std::shared_ptr<google::protobuf::Message> &m_msg); + bool empty() const { return m_empty; } + + hw::trezor::messages::MessageType m_type; + std::shared_ptr<google::protobuf::Message> m_msg; + bool m_empty; + }; + + /** + * Simple wrapper for write-read message exchange with expected message response type. + * + * @throws UnexpectedMessageException if the response message type is different than expected. + * Exception contains message type and the message itself. + */ + template<class t_message> + std::shared_ptr<t_message> + exchange_message(Transport & transport, const google::protobuf::Message & req, + boost::optional<messages::MessageType> resp_type = boost::none) + { + // Require strictly protocol buffers response in the template. + BOOST_STATIC_ASSERT(boost::is_base_of<google::protobuf::Message, t_message>::value); + + // Write the request + transport.write(req); + + // Read the response + std::shared_ptr<google::protobuf::Message> msg_resp; + hw::trezor::messages::MessageType msg_resp_type; + transport.read(msg_resp, &msg_resp_type); + + // Determine type of expected message response + messages::MessageType required_type = resp_type ? resp_type.get() : MessageMapper::get_message_wire_number<t_message>(); + + if (msg_resp_type == required_type) { + return message_ptr_retype<t_message>(msg_resp); + } else if (msg_resp_type == messages::MessageType_Failure){ + throw_failure_exception(dynamic_cast<messages::common::Failure*>(msg_resp.get())); + } else { + throw exc::UnexpectedMessageException(msg_resp_type, msg_resp); + } + } + + std::ostream& operator<<(std::ostream& o, hw::trezor::Transport const& t); + std::ostream& operator<<(std::ostream& o, std::shared_ptr<hw::trezor::Transport> const& t); +}} + + +#endif //MONERO_TRANSPORT_H diff --git a/src/device_trezor/trezor/trezor_defs.hpp b/src/device_trezor/trezor/trezor_defs.hpp new file mode 100644 index 000000000..30e76eadc --- /dev/null +++ b/src/device_trezor/trezor/trezor_defs.hpp @@ -0,0 +1,75 @@ +// Copyright (c) 2017-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef USE_DEVICE_TREZOR +#define USE_DEVICE_TREZOR 1 +#endif + +// HAVE_TREZOR_READY set by cmake after protobuf messages +// were generated successfully and all minimal dependencies are met. +#ifndef DEVICE_TREZOR_READY +#undef USE_DEVICE_TREZOR +#define USE_DEVICE_TREZOR 0 +#endif + +#if USE_DEVICE_TREZOR +#define WITH_DEVICE_TREZOR 1 +#endif + +#ifndef WITH_DEVICE_TREZOR +#undef WITH_DEVICE_TREZOR_LITE +#endif + +#if defined(HAVE_TREZOR_LIBUSB) && USE_DEVICE_TREZOR +#define WITH_DEVICE_TREZOR_WEBUSB 1 +#endif + +// Enable / disable UDP in the enumeration +#ifndef USE_DEVICE_TREZOR_UDP +#define USE_DEVICE_TREZOR_UDP 1 +#endif + +// Enable / disable UDP in the enumeration for release +#ifndef USE_DEVICE_TREZOR_UDP_RELEASE +#define USE_DEVICE_TREZOR_UDP_RELEASE 0 +#endif + +#if USE_DEVICE_TREZOR_UDP && (USE_DEVICE_TREZOR_UDP_RELEASE || defined(TREZOR_DEBUG)) +#define WITH_DEVICE_TREZOR_UDP 1 +#endif + +// Avoids protobuf undefined macro warning +#ifndef PROTOBUF_INLINE_NOT_IN_HEADERS +#define PROTOBUF_INLINE_NOT_IN_HEADERS 0 +#endif + +// Fixes gcc7 problem with minor macro defined clashing with minor() field. +#ifdef minor +#undef minor +#endif diff --git a/src/mnemonics/electrum-words.cpp b/src/mnemonics/electrum-words.cpp index 3d6338856..68d5b84d3 100644 --- a/src/mnemonics/electrum-words.cpp +++ b/src/mnemonics/electrum-words.cpp @@ -37,22 +37,14 @@ */ #include <string> -#include <cassert> -#include <map> #include <cstdint> #include <vector> #include <unordered_map> -#include <boost/algorithm/string.hpp> #include "wipeable_string.h" #include "misc_language.h" -#include "crypto/crypto.h" // for declaration of crypto::secret_key -#include <fstream> -#include "common/int-util.h" +#include "int-util.h" #include "mnemonics/electrum-words.h" -#include <stdexcept> -#include <boost/filesystem.hpp> #include <boost/crc.hpp> -#include <boost/algorithm/string/join.hpp> #include "chinese_simplified.h" #include "english.h" @@ -84,8 +76,8 @@ namespace crypto namespace { uint32_t create_checksum_index(const std::vector<epee::wipeable_string> &word_list, - uint32_t unique_prefix_length); - bool checksum_test(std::vector<epee::wipeable_string> seed, uint32_t unique_prefix_length); + const Language::Base *language); + bool checksum_test(std::vector<epee::wipeable_string> seed, const Language::Base *language); /*! * \brief Finds the word list that contains the seed words and puts the indices @@ -124,8 +116,8 @@ namespace for (std::vector<Language::Base*>::iterator it1 = language_instances.begin(); it1 != language_instances.end(); it1++) { - const std::unordered_map<epee::wipeable_string, uint32_t> &word_map = (*it1)->get_word_map(); - const std::unordered_map<epee::wipeable_string, uint32_t> &trimmed_word_map = (*it1)->get_trimmed_word_map(); + const std::unordered_map<epee::wipeable_string, uint32_t, Language::WordHash, Language::WordEqual> &word_map = (*it1)->get_word_map(); + const std::unordered_map<epee::wipeable_string, uint32_t, Language::WordHash, Language::WordEqual> &trimmed_word_map = (*it1)->get_trimmed_word_map(); // To iterate through seed words bool full_match = true; @@ -159,7 +151,7 @@ namespace // if we were using prefix only, and we have a checksum, check it now // to avoid false positives due to prefix set being too common if (has_checksum) - if (!checksum_test(seed, (*it1)->get_unique_prefix_length())) + if (!checksum_test(seed, *it1)) { fallback = *it1; full_match = false; @@ -198,24 +190,24 @@ namespace * \return Checksum index */ uint32_t create_checksum_index(const std::vector<epee::wipeable_string> &word_list, - uint32_t unique_prefix_length) + const Language::Base *language) { - epee::wipeable_string trimmed_words = ""; + epee::wipeable_string trimmed_words = "", word; + const auto &word_map = language->get_word_map(); + const auto &trimmed_word_map = language->get_trimmed_word_map(); + const uint32_t unique_prefix_length = language->get_unique_prefix_length(); for (std::vector<epee::wipeable_string>::const_iterator it = word_list.begin(); it != word_list.end(); it++) { - if (it->length() > unique_prefix_length) - { - trimmed_words += Language::utf8prefix(*it, unique_prefix_length); - } - else - { - trimmed_words += *it; - } + word = Language::utf8prefix(*it, unique_prefix_length); + auto it2 = trimmed_word_map.find(word); + if (it2 == trimmed_word_map.end()) + throw std::runtime_error("Word \"" + std::string(word.data(), word.size()) + "\" not found in trimmed word map in " + language->get_english_language_name()); + trimmed_words += it2->first; } boost::crc_32_type result; result.process_bytes(trimmed_words.data(), trimmed_words.length()); - return result.checksum() % crypto::ElectrumWords::seed_length; + return result.checksum() % word_list.size(); } /*! @@ -224,7 +216,7 @@ namespace * \param unique_prefix_length the prefix length of each word to use for checksum * \return True if the test passed false if not. */ - bool checksum_test(std::vector<epee::wipeable_string> seed, uint32_t unique_prefix_length) + bool checksum_test(std::vector<epee::wipeable_string> seed, const Language::Base *language) { if (seed.empty()) return false; @@ -232,13 +224,16 @@ namespace epee::wipeable_string last_word = seed.back(); seed.pop_back(); - epee::wipeable_string checksum = seed[create_checksum_index(seed, unique_prefix_length)]; + const uint32_t unique_prefix_length = language->get_unique_prefix_length(); + + auto idx = create_checksum_index(seed, language); + epee::wipeable_string checksum = seed[idx]; epee::wipeable_string trimmed_checksum = checksum.length() > unique_prefix_length ? Language::utf8prefix(checksum, unique_prefix_length) : checksum; epee::wipeable_string trimmed_last_word = last_word.length() > unique_prefix_length ? Language::utf8prefix(last_word, unique_prefix_length) : last_word; - bool ret = trimmed_checksum == trimmed_last_word; + bool ret = Language::WordEqual()(trimmed_checksum, trimmed_last_word); MINFO("Checksum is " << (ret ? "valid" : "invalid")); return ret; } @@ -309,7 +304,7 @@ namespace crypto if (has_checksum) { - if (!checksum_test(seed, language->get_unique_prefix_length())) + if (!checksum_test(seed, language)) { // Checksum fail MERROR("Invalid seed: invalid checksum"); @@ -335,6 +330,7 @@ namespace crypto return false; } + w[0] = SWAP32LE(w[0]); dst.append((const char*)&w[0], 4); // copy 4 bytes to position memwipe(w, sizeof(w)); } @@ -431,7 +427,7 @@ namespace crypto memwipe(w, sizeof(w)); } - words += words_store[create_checksum_index(words_store, language->get_unique_prefix_length())]; + words += words_store[create_checksum_index(words_store, language)]; return true; } diff --git a/src/mnemonics/electrum-words.h b/src/mnemonics/electrum-words.h index 5401b9779..60d2c5f15 100644 --- a/src/mnemonics/electrum-words.h +++ b/src/mnemonics/electrum-words.h @@ -41,7 +41,6 @@ #include <string> #include <cstdint> -#include <map> #include "crypto/crypto.h" // for declaration of crypto::secret_key namespace epee { class wipeable_string; } diff --git a/src/mnemonics/language_base.h b/src/mnemonics/language_base.h index 52e784cef..a6f969604 100644 --- a/src/mnemonics/language_base.h +++ b/src/mnemonics/language_base.h @@ -38,7 +38,9 @@ #include <vector>
#include <unordered_map>
#include <string>
+#include <boost/algorithm/string.hpp>
#include "misc_log_ex.h"
+#include "fnv1.h"
/*!
* \namespace Language
@@ -71,6 +73,92 @@ namespace Language return prefix;
}
+ template<typename T>
+ inline T utf8canonical(const T &s)
+ {
+ T sc = "";
+ size_t avail = s.size();
+ const char *ptr = s.data();
+ wint_t cp = 0;
+ int bytes = 1;
+ char wbuf[8], *wptr;
+ while (avail--)
+ {
+ if ((*ptr & 0x80) == 0)
+ {
+ cp = *ptr++;
+ bytes = 1;
+ }
+ else if ((*ptr & 0xe0) == 0xc0)
+ {
+ if (avail < 1)
+ throw std::runtime_error("Invalid UTF-8");
+ cp = (*ptr++ & 0x1f) << 6;
+ cp |= *ptr++ & 0x3f;
+ --avail;
+ bytes = 2;
+ }
+ else if ((*ptr & 0xf0) == 0xe0)
+ {
+ if (avail < 2)
+ throw std::runtime_error("Invalid UTF-8");
+ cp = (*ptr++ & 0xf) << 12;
+ cp |= (*ptr++ & 0x3f) << 6;
+ cp |= *ptr++ & 0x3f;
+ avail -= 2;
+ bytes = 3;
+ }
+ else if ((*ptr & 0xf8) == 0xf0)
+ {
+ if (avail < 3)
+ throw std::runtime_error("Invalid UTF-8");
+ cp = (*ptr++ & 0x7) << 18;
+ cp |= (*ptr++ & 0x3f) << 12;
+ cp |= (*ptr++ & 0x3f) << 6;
+ cp |= *ptr++ & 0x3f;
+ avail -= 3;
+ bytes = 4;
+ }
+ else
+ throw std::runtime_error("Invalid UTF-8");
+
+ cp = std::towlower(cp);
+ wptr = wbuf;
+ switch (bytes)
+ {
+ case 1: *wptr++ = cp; break;
+ case 2: *wptr++ = 0xc0 | (cp >> 6); *wptr++ = 0x80 | (cp & 0x3f); break;
+ case 3: *wptr++ = 0xe0 | (cp >> 12); *wptr++ = 0x80 | ((cp >> 6) & 0x3f); *wptr++ = 0x80 | (cp & 0x3f); break;
+ case 4: *wptr++ = 0xf0 | (cp >> 18); *wptr += 0x80 | ((cp >> 12) & 0x3f); *wptr++ = 0x80 | ((cp >> 6) & 0x3f); *wptr++ = 0x80 | (cp & 0x3f); break;
+ default: throw std::runtime_error("Invalid UTF-8");
+ }
+ *wptr = 0;
+ sc += T(wbuf, bytes);
+ cp = 0;
+ bytes = 1;
+ }
+ return sc;
+ }
+
+ struct WordHash
+ {
+ std::size_t operator()(const epee::wipeable_string &s) const
+ {
+ const epee::wipeable_string sc = utf8canonical(s);
+ return epee::fnv::FNV1a(sc.data(), sc.size());
+ }
+ };
+
+ struct WordEqual
+ {
+ bool operator()(const epee::wipeable_string &s0, const epee::wipeable_string &s1) const
+ {
+ const epee::wipeable_string s0c = utf8canonical(s0);
+ const epee::wipeable_string s1c = utf8canonical(s1);
+ return s0c == s1c;
+ }
+ };
+
/*!
* \class Base
* \brief A base language class which all languages have to inherit from for
@@ -87,8 +175,8 @@ namespace Language NWORDS = 1626
};
std::vector<std::string> word_list; /*!< A pointer to the array of words */
- std::unordered_map<epee::wipeable_string, uint32_t> word_map; /*!< hash table to find word's index */
- std::unordered_map<epee::wipeable_string, uint32_t> trimmed_word_map; /*!< hash table to find word's trimmed index */
+ std::unordered_map<epee::wipeable_string, uint32_t, WordHash, WordEqual> word_map; /*!< hash table to find word's index */
+ std::unordered_map<epee::wipeable_string, uint32_t, WordHash, WordEqual> trimmed_word_map; /*!< hash table to find word's trimmed index */
std::string language_name; /*!< Name of language */
std::string english_language_name; /*!< Name of language */
uint32_t unique_prefix_length; /*!< Number of unique starting characters to trim the wordlist to when matching */
@@ -159,7 +247,7 @@ namespace Language * \brief Returns a pointer to the word map.
* \return A pointer to the word map.
*/
- const std::unordered_map<epee::wipeable_string, uint32_t>& get_word_map() const
+ const std::unordered_map<epee::wipeable_string, uint32_t, WordHash, WordEqual>& get_word_map() const
{
return word_map;
}
@@ -167,7 +255,7 @@ namespace Language * \brief Returns a pointer to the trimmed word map.
* \return A pointer to the trimmed word map.
*/
- const std::unordered_map<epee::wipeable_string, uint32_t>& get_trimmed_word_map() const
+ const std::unordered_map<epee::wipeable_string, uint32_t, WordHash, WordEqual>& get_trimmed_word_map() const
{
return trimmed_word_map;
}
diff --git a/src/mnemonics/singleton.h b/src/mnemonics/singleton.h index a15c2b9ae..d317a2d8d 100644 --- a/src/mnemonics/singleton.h +++ b/src/mnemonics/singleton.h @@ -50,8 +50,8 @@ namespace Language class Singleton
{
Singleton() {}
- Singleton(Singleton &s) {}
- Singleton& operator=(const Singleton&) {}
+ Singleton(Singleton &s) = delete;
+ Singleton& operator=(const Singleton&) = delete;
public:
static T* instance()
{
diff --git a/src/p2p/net_node.cpp b/src/p2p/net_node.cpp index e9d2061e8..7cad6e077 100644 --- a/src/p2p/net_node.cpp +++ b/src/p2p/net_node.cpp @@ -62,8 +62,8 @@ namespace nodetool const command_line::arg_descriptor<int64_t> arg_in_peers = {"in-peers", "set max number of in peers", -1}; const command_line::arg_descriptor<int> arg_tos_flag = {"tos-flag", "set TOS flag", -1}; - const command_line::arg_descriptor<int64_t> arg_limit_rate_up = {"limit-rate-up", "set limit-rate-up [kB/s]", -1}; - const command_line::arg_descriptor<int64_t> arg_limit_rate_down = {"limit-rate-down", "set limit-rate-down [kB/s]", -1}; + const command_line::arg_descriptor<int64_t> arg_limit_rate_up = {"limit-rate-up", "set limit-rate-up [kB/s]", P2P_DEFAULT_LIMIT_RATE_UP}; + const command_line::arg_descriptor<int64_t> arg_limit_rate_down = {"limit-rate-down", "set limit-rate-down [kB/s]", P2P_DEFAULT_LIMIT_RATE_DOWN}; const command_line::arg_descriptor<int64_t> arg_limit_rate = {"limit-rate", "set limit-rate [kB/s]", -1}; const command_line::arg_descriptor<bool> arg_save_graph = {"save-graph", "Save data for dr monero", false}; diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index 90e2f78b1..09c219a44 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -29,6 +29,7 @@ // Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers #pragma once +#include <array> #include <boost/thread.hpp> #include <boost/program_options/options_description.hpp> #include <boost/program_options/variables_map.hpp> @@ -38,7 +39,9 @@ #include "cryptonote_config.h" #include "warnings.h" -#include "net/levin_server_cp2.h" +#include "net/abstract_tcp_server2.h" +#include "net/levin_protocol_handler.h" +#include "net/levin_protocol_handler_async.h" #include "p2p_protocol_defs.h" #include "storages/levin_abstract_invoke2.h" #include "net_peerlist.h" @@ -125,6 +128,11 @@ namespace nodetool virtual bool block_host(const epee::net_utils::network_address &adress, time_t seconds = P2P_IP_BLOCKTIME); virtual bool unblock_host(const epee::net_utils::network_address &address); virtual std::map<std::string, time_t> get_blocked_hosts() { CRITICAL_REGION_LOCAL(m_blocked_hosts_lock); return m_blocked_hosts; } + + virtual void add_used_stripe_peer(const typename t_payload_net_handler::connection_context &context); + virtual void remove_used_stripe_peer(const typename t_payload_net_handler::connection_context &context); + virtual void clear_used_stripe_peers(); + private: const std::vector<std::string> m_seed_nodes_list = { "seeds.moneroseeds.se" @@ -178,10 +186,10 @@ namespace nodetool virtual void on_connection_close(p2p_connection_context& context); virtual void callback(p2p_connection_context& context); //----------------- i_p2p_endpoint ------------------------------------------------------------- - virtual bool relay_notify_to_list(int command, const std::string& data_buff, const std::list<boost::uuids::uuid> &connections); - virtual bool relay_notify_to_all(int command, const std::string& data_buff, const epee::net_utils::connection_context_base& context); - virtual bool invoke_command_to_peer(int command, const std::string& req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context); - virtual bool invoke_notify_to_peer(int command, const std::string& req_buff, const epee::net_utils::connection_context_base& context); + virtual bool relay_notify_to_list(int command, const epee::span<const uint8_t> data_buff, const std::list<boost::uuids::uuid> &connections); + virtual bool relay_notify_to_all(int command, const epee::span<const uint8_t> data_buff, const epee::net_utils::connection_context_base& context); + virtual bool invoke_command_to_peer(int command, const epee::span<const uint8_t> req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context); + virtual bool invoke_notify_to_peer(int command, const epee::span<const uint8_t> req_buff, const epee::net_utils::connection_context_base& context); virtual bool drop_connection(const epee::net_utils::connection_context_base& context); virtual void request_callback(const epee::net_utils::connection_context_base& context); virtual void for_each_connection(std::function<bool(typename t_payload_net_handler::connection_context&, peerid_type, uint32_t)> f); @@ -195,12 +203,12 @@ namespace nodetool const boost::program_options::variables_map& vm ); bool idle_worker(); - bool handle_remote_peerlist(const std::list<peerlist_entry>& peerlist, time_t local_time, const epee::net_utils::connection_context_base& context); + bool handle_remote_peerlist(const std::vector<peerlist_entry>& peerlist, time_t local_time, const epee::net_utils::connection_context_base& context); bool get_local_node_data(basic_node_data& node_data); //bool get_local_handshake_data(handshake_data& hshd); - bool merge_peerlist_with_local(const std::list<peerlist_entry>& bs); - bool fix_time_delta(std::list<peerlist_entry>& local_peerlist, time_t local_time, int64_t& delta); + bool merge_peerlist_with_local(const std::vector<peerlist_entry>& bs); + bool fix_time_delta(std::vector<peerlist_entry>& local_peerlist, time_t local_time, int64_t& delta); bool connections_maker(); bool peer_sync_idle_maker(); @@ -233,7 +241,13 @@ namespace nodetool bool parse_peers_and_add_to_container(const boost::program_options::variables_map& vm, const command_line::arg_descriptor<std::vector<std::string> > & arg, Container& container); bool set_max_out_peers(const boost::program_options::variables_map& vm, int64_t max); + bool get_max_out_peers() const { return m_config.m_net_config.max_out_connection_count; } + bool get_current_out_peers() const { return m_current_number_of_out_peers; } + bool set_max_in_peers(const boost::program_options::variables_map& vm, int64_t max); + bool get_max_in_peers() const { return m_config.m_net_config.max_in_connection_count; } + bool get_current_in_peers() const { return m_current_number_of_in_peers; } + bool set_tos_flag(const boost::program_options::variables_map& vm, int limit); bool set_rate_up_limit(const boost::program_options::variables_map& vm, int64_t limit); @@ -308,7 +322,7 @@ namespace nodetool epee::math_helper::once_a_time_seconds<1> m_connections_maker_interval; epee::math_helper::once_a_time_seconds<60*30, false> m_peerlist_store_interval; epee::math_helper::once_a_time_seconds<60> m_gray_peerlist_housekeeping_interval; - epee::math_helper::once_a_time_seconds<900, false> m_incoming_connections_interval; + epee::math_helper::once_a_time_seconds<3600, false> m_incoming_connections_interval; std::string m_bind_ip; std::string m_port; @@ -334,11 +348,14 @@ namespace nodetool epee::critical_section m_host_fails_score_lock; std::map<std::string, uint64_t> m_host_fails_score; + boost::mutex m_used_stripe_peers_mutex; + std::array<std::list<epee::net_utils::network_address>, 1 << CRYPTONOTE_PRUNING_LOG_STRIPES> m_used_stripe_peers; + cryptonote::network_type m_nettype; }; - const int64_t default_limit_up = 2048; // kB/s - const int64_t default_limit_down = 8192; // kB/s + const int64_t default_limit_up = P2P_DEFAULT_LIMIT_RATE_UP; // kB/s + const int64_t default_limit_down = P2P_DEFAULT_LIMIT_RATE_DOWN; // kB/s extern const command_line::arg_descriptor<std::string> arg_p2p_bind_ip; extern const command_line::arg_descriptor<std::string, false, true, 2> arg_p2p_bind_port; extern const command_line::arg_descriptor<uint32_t> arg_p2p_external_port; diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index 9390626a8..c594984d4 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -33,6 +33,7 @@ #include <algorithm> #include <boost/date_time/posix_time/posix_time.hpp> #include <boost/thread/thread.hpp> +#include <boost/uuid/uuid_io.hpp> #include <boost/bind.hpp> #include <atomic> @@ -40,6 +41,7 @@ #include "string_tools.h" #include "common/util.h" #include "common/dns_utils.h" +#include "common/pruning.h" #include "net/net_helper.h" #include "math_helper.h" #include "p2p_protocol_defs.h" @@ -132,6 +134,28 @@ namespace nodetool make_default_config(); } +#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED + std::list<peerlist_entry> plw; + while (m_peerlist.get_white_peers_count()) + { + plw.push_back(peerlist_entry()); + m_peerlist.get_white_peer_by_index(plw.back(), 0); + m_peerlist.remove_from_peer_white(plw.back()); + } + for (auto &e:plw) + m_peerlist.append_with_peer_white(e); + + std::list<peerlist_entry> plg; + while (m_peerlist.get_gray_peers_count()) + { + plg.push_back(peerlist_entry()); + m_peerlist.get_gray_peer_by_index(plg.back(), 0); + m_peerlist.remove_from_peer_gray(plg.back()); + } + for (auto &e:plg) + m_peerlist.append_with_peer_gray(e); +#endif + // always recreate a new peer id make_default_peer_id(); @@ -655,10 +679,14 @@ namespace nodetool { kill(); m_peerlist.deinit(); - m_net_server.deinit_server(); - // remove UPnP port mapping - if(!m_no_igd) - delete_upnp_port_mapping(m_listening_port); + + if (!m_offline) + { + m_net_server.deinit_server(); + // remove UPnP port mapping + if(!m_no_igd) + delete_upnp_port_mapping(m_listening_port); + } return store_config(); } //----------------------------------------------------------------------------------- @@ -724,7 +752,7 @@ namespace nodetool std::atomic<bool> hsh_result(false); bool r = epee::net_utils::async_invoke_remote_command2<typename COMMAND_HANDSHAKE::response>(context_.m_connection_id, COMMAND_HANDSHAKE::ID, arg, m_net_server.get_config_object(), - [this, &pi, &ev, &hsh_result, &just_take_peerlist](int code, const typename COMMAND_HANDSHAKE::response& rsp, p2p_connection_context& context) + [this, &pi, &ev, &hsh_result, &just_take_peerlist, &context_](int code, const typename COMMAND_HANDSHAKE::response& rsp, p2p_connection_context& context) { epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){ev.raise();}); @@ -736,7 +764,7 @@ namespace nodetool if(rsp.node_data.network_id != m_network_id) { - LOG_WARNING_CC(context, "COMMAND_HANDSHAKE Failed, wrong network! (" << epee::string_tools::get_str_from_guid_a(rsp.node_data.network_id) << "), closing connection."); + LOG_WARNING_CC(context, "COMMAND_HANDSHAKE Failed, wrong network! (" << rsp.node_data.network_id << "), closing connection."); return; } @@ -757,7 +785,7 @@ namespace nodetool } pi = context.peer_id = rsp.node_data.peer_id; - m_peerlist.set_peer_just_seen(rsp.node_data.peer_id, context.m_remote_address); + m_peerlist.set_peer_just_seen(rsp.node_data.peer_id, context.m_remote_address, context.m_pruning_seed); if(rsp.node_data.peer_id == m_config.m_peer_id) { @@ -765,11 +793,13 @@ namespace nodetool hsh_result = false; return; } + LOG_INFO_CC(context, "New connection handshaked, pruning seed " << epee::string_tools::to_string_hex(context.m_pruning_seed)); LOG_DEBUG_CC(context, " COMMAND_HANDSHAKE INVOKED OK"); }else { LOG_DEBUG_CC(context, " COMMAND_HANDSHAKE(AND CLOSE) INVOKED OK"); } + context_ = context; }, P2P_DEFAULT_HANDSHAKE_INVOKE_TIMEOUT); if(r) @@ -816,7 +846,7 @@ namespace nodetool add_host_fail(context.m_remote_address); } if(!context.m_is_income) - m_peerlist.set_peer_just_seen(context.peer_id, context.m_remote_address); + m_peerlist.set_peer_just_seen(context.peer_id, context.m_remote_address, context.m_pruning_seed); m_payload_handler.process_payload_sync_data(rsp.payload_data, context, false); }); @@ -934,6 +964,7 @@ namespace nodetool const epee::net_utils::ipv4_network_address &ipv4 = na.as<const epee::net_utils::ipv4_network_address>(); typename net_server::t_connection_context con = AUTO_VAL_INIT(con); + con.m_anchor = peer_type == anchor; bool res = m_net_server.connect(epee::string_tools::get_ip_string_from_int32(ipv4.ip()), epee::string_tools::num_to_string_fast(ipv4.port()), m_config.m_net_config.connection_timeout, @@ -973,6 +1004,7 @@ namespace nodetool time_t last_seen; time(&last_seen); pe_local.last_seen = static_cast<int64_t>(last_seen); + pe_local.pruning_seed = con.m_pruning_seed; m_peerlist.append_with_peer_white(pe_local); //update last seen and push it to peerlist manager @@ -999,6 +1031,7 @@ namespace nodetool const epee::net_utils::ipv4_network_address &ipv4 = na.as<epee::net_utils::ipv4_network_address>(); typename net_server::t_connection_context con = AUTO_VAL_INIT(con); + con.m_anchor = false; bool res = m_net_server.connect(epee::string_tools::get_ip_string_from_int32(ipv4.ip()), epee::string_tools::num_to_string_fast(ipv4.port()), m_config.m_net_config.connection_timeout, @@ -1051,7 +1084,7 @@ namespace nodetool bool node_server<t_payload_net_handler>::make_new_connection_from_anchor_peerlist(const std::vector<anchor_peerlist_entry>& anchor_peerlist) { for (const auto& pe: anchor_peerlist) { - _note("Considering connecting (out) to peer: " << peerid_type(pe.id) << " " << pe.adr.str()); + _note("Considering connecting (out) to anchor peer: " << peerid_type(pe.id) << " " << pe.adr.str()); if(is_peer_used(pe)) { _note("Peer is used"); @@ -1084,11 +1117,7 @@ namespace nodetool template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::make_new_connection_from_peerlist(bool use_white_list) { - size_t local_peers_count = use_white_list ? m_peerlist.get_white_peers_count():m_peerlist.get_gray_peers_count(); - if(!local_peers_count) - return false;//no peers - - size_t max_random_index = std::min<uint64_t>(local_peers_count -1, 20); + size_t max_random_index = 0; std::set<size_t> tried_peers; @@ -1098,21 +1127,54 @@ namespace nodetool { ++rand_count; size_t random_index; + const uint32_t next_needed_pruning_stripe = m_payload_handler.get_next_needed_pruning_stripe().second; - if (use_white_list) { - local_peers_count = m_peerlist.get_white_peers_count(); - if (!local_peers_count) - return false; - max_random_index = std::min<uint64_t>(local_peers_count -1, 20); - random_index = get_random_index_with_fixed_probability(max_random_index); - } else { - local_peers_count = m_peerlist.get_gray_peers_count(); - if (!local_peers_count) + std::deque<size_t> filtered; + const size_t limit = use_white_list ? 20 : std::numeric_limits<size_t>::max(); + size_t idx = 0; + m_peerlist.foreach (use_white_list, [&filtered, &idx, limit, next_needed_pruning_stripe](const peerlist_entry &pe){ + if (filtered.size() >= limit) return false; - random_index = crypto::rand<size_t>() % local_peers_count; + if (next_needed_pruning_stripe == 0 || pe.pruning_seed == 0) + filtered.push_back(idx); + else if (next_needed_pruning_stripe == tools::get_pruning_stripe(pe.pruning_seed)) + filtered.push_front(idx); + ++idx; + return true; + }); + if (filtered.empty()) + { + MDEBUG("No available peer in " << (use_white_list ? "white" : "gray") << " list filtered by " << next_needed_pruning_stripe); + return false; } + if (use_white_list) + { + // if using the white list, we first pick in the set of peers we've already been using earlier + random_index = get_random_index_with_fixed_probability(std::min<uint64_t>(filtered.size() - 1, 20)); + CRITICAL_REGION_LOCAL(m_used_stripe_peers_mutex); + if (next_needed_pruning_stripe > 0 && next_needed_pruning_stripe <= (1ul << CRYPTONOTE_PRUNING_LOG_STRIPES) && !m_used_stripe_peers[next_needed_pruning_stripe-1].empty()) + { + const epee::net_utils::network_address na = m_used_stripe_peers[next_needed_pruning_stripe-1].front(); + m_used_stripe_peers[next_needed_pruning_stripe-1].pop_front(); + for (size_t i = 0; i < filtered.size(); ++i) + { + peerlist_entry pe; + if (m_peerlist.get_white_peer_by_index(pe, filtered[i]) && pe.adr == na) + { + MDEBUG("Reusing stripe " << next_needed_pruning_stripe << " peer " << pe.adr.str()); + random_index = i; + break; + } + } + } + } + else + random_index = crypto::rand<size_t>() % filtered.size(); - CHECK_AND_ASSERT_MES(random_index < local_peers_count, false, "random_starter_index < peers_local.size() failed!!"); + CHECK_AND_ASSERT_MES(random_index < filtered.size(), false, "random_index < filtered.size() failed!!"); + random_index = filtered[random_index]; + CHECK_AND_ASSERT_MES(random_index < (use_white_list ? m_peerlist.get_white_peers_count() : m_peerlist.get_gray_peers_count()), + false, "random_index < peers size failed!!"); if(tried_peers.count(random_index)) continue; @@ -1124,7 +1186,9 @@ namespace nodetool ++try_count; - _note("Considering connecting (out) to peer: " << peerid_to_string(pe.id) << " " << pe.adr.str()); + _note("Considering connecting (out) to " << (use_white_list ? "white" : "gray") << " list peer: " << + peerid_to_string(pe.id) << " " << pe.adr.str() << ", pruning seed " << epee::string_tools::to_string_hex(pe.pruning_seed) << + " (stripe " << next_needed_pruning_stripe << " needed)"); if(is_peer_used(pe)) { _note("Peer is used"); @@ -1138,6 +1202,7 @@ namespace nodetool continue; MDEBUG("Selected peer: " << peerid_to_string(pe.id) << " " << pe.adr.str() + << ", pruning seed " << epee::string_tools::to_string_hex(pe.pruning_seed) << " " << "[peer_list=" << (use_white_list ? white : gray) << "] last_seen: " << (pe.last_seen ? epee::misc_utils::get_time_interval_string(time(NULL) - pe.last_seen) : "never")); @@ -1214,31 +1279,35 @@ namespace nodetool if (!connect_to_peerlist(m_priority_peers)) return false; - size_t expected_white_connections = (m_config.m_net_config.max_out_connection_count*P2P_DEFAULT_WHITELIST_CONNECTIONS_PERCENT)/100; + size_t base_expected_white_connections = (m_config.m_net_config.max_out_connection_count*P2P_DEFAULT_WHITELIST_CONNECTIONS_PERCENT)/100; size_t conn_count = get_outgoing_connections_count(); - if(conn_count < m_config.m_net_config.max_out_connection_count) + while(conn_count < m_config.m_net_config.max_out_connection_count) { + const size_t expected_white_connections = m_payload_handler.get_next_needed_pruning_stripe().second ? m_config.m_net_config.max_out_connection_count : base_expected_white_connections; if(conn_count < expected_white_connections) { //start from anchor list - if(!make_expected_connections_count(anchor, P2P_DEFAULT_ANCHOR_CONNECTIONS_COUNT)) - return false; + while (get_outgoing_connections_count() < P2P_DEFAULT_ANCHOR_CONNECTIONS_COUNT + && make_expected_connections_count(anchor, P2P_DEFAULT_ANCHOR_CONNECTIONS_COUNT)); //then do white list - if(!make_expected_connections_count(white, expected_white_connections)) - return false; + while (get_outgoing_connections_count() < expected_white_connections + && make_expected_connections_count(white, expected_white_connections)); //then do grey list - if(!make_expected_connections_count(gray, m_config.m_net_config.max_out_connection_count)) - return false; + while (get_outgoing_connections_count() < m_config.m_net_config.max_out_connection_count + && make_expected_connections_count(gray, m_config.m_net_config.max_out_connection_count)); }else { //start from grey list - if(!make_expected_connections_count(gray, m_config.m_net_config.max_out_connection_count)) - return false; + while (get_outgoing_connections_count() < m_config.m_net_config.max_out_connection_count + && make_expected_connections_count(gray, m_config.m_net_config.max_out_connection_count)); //and then do white list - if(!make_expected_connections_count(white, m_config.m_net_config.max_out_connection_count)) - return false; + while (get_outgoing_connections_count() < m_config.m_net_config.max_out_connection_count + && make_expected_connections_count(white, m_config.m_net_config.max_out_connection_count)); } + if(m_net_server.is_stop_signal_sent()) + return false; + conn_count = get_outgoing_connections_count(); } if (start_conn_count == get_outgoing_connections_count() && start_conn_count < m_config.m_net_config.max_out_connection_count) @@ -1255,7 +1324,7 @@ namespace nodetool bool node_server<t_payload_net_handler>::make_expected_connections_count(PeerType peer_type, size_t expected_connections) { if (m_offline) - return true; + return false; std::vector<anchor_peerlist_entry> apl; @@ -1265,24 +1334,24 @@ namespace nodetool size_t conn_count = get_outgoing_connections_count(); //add new connections from white peers - while(conn_count < expected_connections) + if(conn_count < expected_connections) { if(m_net_server.is_stop_signal_sent()) return false; + MDEBUG("Making expected connection, type " << peer_type << ", " << conn_count << "/" << expected_connections << " connections"); + if (peer_type == anchor && !make_new_connection_from_anchor_peerlist(apl)) { - break; + return false; } if (peer_type == white && !make_new_connection_from_peerlist(true)) { - break; + return false; } if (peer_type == gray && !make_new_connection_from_peerlist(false)) { - break; + return false; } - - conn_count = get_outgoing_connections_count(); } return true; } @@ -1330,12 +1399,19 @@ namespace nodetool template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::check_incoming_connections() { - if (m_offline || m_hide_my_port) + if (m_offline) return true; if (get_incoming_connections_count() == 0) { - const el::Level level = el::Level::Warning; - MCLOG_RED(level, "global", "No incoming connections - check firewalls/routers allow port " << get_this_peer_port()); + if (m_hide_my_port || m_config.m_net_config.max_in_connection_count == 0) + { + MGINFO("Incoming connections disabled, enable them for full connectivity"); + } + else + { + const el::Level level = el::Level::Warning; + MCLOG_RED(level, "global", "No incoming connections - check firewalls/routers allow port " << get_this_peer_port()); + } } return true; } @@ -1363,7 +1439,7 @@ namespace nodetool } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::fix_time_delta(std::list<peerlist_entry>& local_peerlist, time_t local_time, int64_t& delta) + bool node_server<t_payload_net_handler>::fix_time_delta(std::vector<peerlist_entry>& local_peerlist, time_t local_time, int64_t& delta) { //fix time delta time_t now = 0; @@ -1378,15 +1454,18 @@ namespace nodetool return false; } be.last_seen += delta; +#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED + be.pruning_seed = tools::make_pruning_seed(1 + (be.adr.as<epee::net_utils::ipv4_network_address>().ip()) % (1ul << CRYPTONOTE_PRUNING_LOG_STRIPES), CRYPTONOTE_PRUNING_LOG_STRIPES); +#endif } return true; } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::handle_remote_peerlist(const std::list<peerlist_entry>& peerlist, time_t local_time, const epee::net_utils::connection_context_base& context) + bool node_server<t_payload_net_handler>::handle_remote_peerlist(const std::vector<peerlist_entry>& peerlist, time_t local_time, const epee::net_utils::connection_context_base& context) { int64_t delta = 0; - std::list<peerlist_entry> peerlist_ = peerlist; + std::vector<peerlist_entry> peerlist_ = peerlist; if(!fix_time_delta(peerlist_, local_time, delta)) return false; LOG_DEBUG_CC(context, "REMOTE PEERLIST: TIME_DELTA: " << delta << ", remote peerlist size=" << peerlist_.size()); @@ -1505,7 +1584,7 @@ namespace nodetool } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::relay_notify_to_list(int command, const std::string& data_buff, const std::list<boost::uuids::uuid> &connections) + bool node_server<t_payload_net_handler>::relay_notify_to_list(int command, const epee::span<const uint8_t> data_buff, const std::list<boost::uuids::uuid> &connections) { for(const auto& c_id: connections) { @@ -1515,7 +1594,7 @@ namespace nodetool } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::relay_notify_to_all(int command, const std::string& data_buff, const epee::net_utils::connection_context_base& context) + bool node_server<t_payload_net_handler>::relay_notify_to_all(int command, const epee::span<const uint8_t> data_buff, const epee::net_utils::connection_context_base& context) { std::list<boost::uuids::uuid> connections; m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) @@ -1534,14 +1613,14 @@ namespace nodetool } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::invoke_notify_to_peer(int command, const std::string& req_buff, const epee::net_utils::connection_context_base& context) + bool node_server<t_payload_net_handler>::invoke_notify_to_peer(int command, const epee::span<const uint8_t> req_buff, const epee::net_utils::connection_context_base& context) { int res = m_net_server.get_config_object().notify(command, req_buff, context.m_connection_id); return res > 0; } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::invoke_command_to_peer(int command, const std::string& req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context) + bool node_server<t_payload_net_handler>::invoke_command_to_peer(int command, const epee::span<const uint8_t> req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context) { int res = m_net_server.get_config_object().invoke(command, req_buff, resp_buff, context.m_connection_id); return res > 0; @@ -1675,7 +1754,7 @@ namespace nodetool if(arg.node_data.network_id != m_network_id) { - LOG_INFO_CC(context, "WRONG NETWORK AGENT CONNECTED! id=" << epee::string_tools::get_str_from_guid_a(arg.node_data.network_id)); + LOG_INFO_CC(context, "WRONG NETWORK AGENT CONNECTED! id=" << arg.node_data.network_id); drop_connection(context); add_host_fail(context.m_remote_address); return 1; @@ -1738,6 +1817,7 @@ namespace nodetool time(&last_seen); pe.last_seen = static_cast<int64_t>(last_seen); pe.id = peer_id_l; + pe.pruning_seed = context.m_pruning_seed; this->m_peerlist.append_with_peer_white(pe); LOG_DEBUG_CC(context, "PING SUCCESS " << context.m_remote_address.host_str() << ":" << port_l); }); @@ -1768,8 +1848,8 @@ namespace nodetool template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::log_peerlist() { - std::list<peerlist_entry> pl_white; - std::list<peerlist_entry> pl_gray; + std::vector<peerlist_entry> pl_white; + std::vector<peerlist_entry> pl_gray; m_peerlist.get_peerlist_full(pl_gray, pl_white); MINFO(ENDL << "Peerlist white:" << ENDL << print_peerlist_to_string(pl_white) << ENDL << "Peerlist gray:" << ENDL << print_peerlist_to_string(pl_gray) ); return true; @@ -1791,7 +1871,7 @@ namespace nodetool { ss << cntxt.m_remote_address.str() << " \t\tpeer_id " << cntxt.peer_id - << " \t\tconn_id " << epee::string_tools::get_str_from_guid_a(cntxt.m_connection_id) << (cntxt.m_is_income ? " INC":" OUT") + << " \t\tconn_id " << cntxt.m_connection_id << (cntxt.m_is_income ? " INC":" OUT") << std::endl; return true; }); @@ -1873,21 +1953,16 @@ namespace nodetool template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::set_max_out_peers(const boost::program_options::variables_map& vm, int64_t max) { - if(max == -1) { - m_config.m_net_config.max_out_connection_count = P2P_DEFAULT_CONNECTIONS_COUNT; - return true; - } + if(max == -1) + max = P2P_DEFAULT_CONNECTIONS_COUNT; m_config.m_net_config.max_out_connection_count = max; + m_payload_handler.set_max_out_peers(max); return true; } template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::set_max_in_peers(const boost::program_options::variables_map& vm, int64_t max) { - if(max == -1) { - m_config.m_net_config.max_in_connection_count = -1; - return true; - } m_config.m_net_config.max_in_connection_count = max; return true; } @@ -1998,6 +2073,7 @@ namespace nodetool { if (m_offline) return true; if (!m_exclusive_peers.empty()) return true; + if (m_payload_handler.needs_new_sync_connections()) return true; peerlist_entry pe = AUTO_VAL_INIT(pe); @@ -2005,7 +2081,7 @@ namespace nodetool return false; if (!m_peerlist.get_random_gray_peer(pe)) { - return false; + return true; } bool success = check_connection_and_handshake_with_peer(pe.adr, pe.last_seen); @@ -2018,7 +2094,7 @@ namespace nodetool return true; } - m_peerlist.set_peer_just_seen(pe.id, pe.adr); + m_peerlist.set_peer_just_seen(pe.id, pe.adr, pe.pruning_seed); LOG_PRINT_L2("PEER PROMOTED TO WHITE PEER LIST IP address: " << pe.adr.host_str() << " Peer ID: " << peerid_type(pe.id)); @@ -2026,6 +2102,42 @@ namespace nodetool } template<class t_payload_net_handler> + void node_server<t_payload_net_handler>::add_used_stripe_peer(const typename t_payload_net_handler::connection_context &context) + { + const uint32_t stripe = tools::get_pruning_stripe(context.m_pruning_seed); + if (stripe == 0 || stripe > (1ul << CRYPTONOTE_PRUNING_LOG_STRIPES)) + return; + const uint32_t index = stripe - 1; + CRITICAL_REGION_LOCAL(m_used_stripe_peers_mutex); + MINFO("adding stripe " << stripe << " peer: " << context.m_remote_address.str()); + std::remove_if(m_used_stripe_peers[index].begin(), m_used_stripe_peers[index].end(), + [&context](const epee::net_utils::network_address &na){ return context.m_remote_address == na; }); + m_used_stripe_peers[index].push_back(context.m_remote_address); + } + + template<class t_payload_net_handler> + void node_server<t_payload_net_handler>::remove_used_stripe_peer(const typename t_payload_net_handler::connection_context &context) + { + const uint32_t stripe = tools::get_pruning_stripe(context.m_pruning_seed); + if (stripe == 0 || stripe > (1ul << CRYPTONOTE_PRUNING_LOG_STRIPES)) + return; + const uint32_t index = stripe - 1; + CRITICAL_REGION_LOCAL(m_used_stripe_peers_mutex); + MINFO("removing stripe " << stripe << " peer: " << context.m_remote_address.str()); + std::remove_if(m_used_stripe_peers[index].begin(), m_used_stripe_peers[index].end(), + [&context](const epee::net_utils::network_address &na){ return context.m_remote_address == na; }); + } + + template<class t_payload_net_handler> + void node_server<t_payload_net_handler>::clear_used_stripe_peers() + { + CRITICAL_REGION_LOCAL(m_used_stripe_peers_mutex); + MINFO("clearing used stripe peers"); + for (auto &e: m_used_stripe_peers) + e.clear(); + } + + template<class t_payload_net_handler> void node_server<t_payload_net_handler>::add_upnp_port_mapping(uint32_t port) { MDEBUG("Attempting to add IGD port mapping."); @@ -2042,7 +2154,7 @@ namespace nodetool char lanAddress[64]; result = UPNP_GetValidIGD(deviceList, &urls, &igdData, lanAddress, sizeof lanAddress); freeUPNPDevlist(deviceList); - if (result != 0) { + if (result > 0) { if (result == 1) { std::ostringstream portString; portString << port; @@ -2088,7 +2200,7 @@ namespace nodetool char lanAddress[64]; result = UPNP_GetValidIGD(deviceList, &urls, &igdData, lanAddress, sizeof lanAddress); freeUPNPDevlist(deviceList); - if (result != 0) { + if (result > 0) { if (result == 1) { std::ostringstream portString; portString << port; diff --git a/src/p2p/net_node_common.h b/src/p2p/net_node_common.h index 218250efa..075c18ef9 100644 --- a/src/p2p/net_node_common.h +++ b/src/p2p/net_node_common.h @@ -43,10 +43,10 @@ namespace nodetool template<class t_connection_context> struct i_p2p_endpoint { - virtual bool relay_notify_to_list(int command, const std::string& data_buff, const std::list<boost::uuids::uuid>& connections)=0; - virtual bool relay_notify_to_all(int command, const std::string& data_buff, const epee::net_utils::connection_context_base& context)=0; - virtual bool invoke_command_to_peer(int command, const std::string& req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context)=0; - virtual bool invoke_notify_to_peer(int command, const std::string& req_buff, const epee::net_utils::connection_context_base& context)=0; + virtual bool relay_notify_to_list(int command, const epee::span<const uint8_t> data_buff, const std::list<boost::uuids::uuid>& connections)=0; + virtual bool relay_notify_to_all(int command, const epee::span<const uint8_t> data_buff, const epee::net_utils::connection_context_base& context)=0; + virtual bool invoke_command_to_peer(int command, const epee::span<const uint8_t> req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context)=0; + virtual bool invoke_notify_to_peer(int command, const epee::span<const uint8_t> req_buff, const epee::net_utils::connection_context_base& context)=0; virtual bool drop_connection(const epee::net_utils::connection_context_base& context)=0; virtual void request_callback(const epee::net_utils::connection_context_base& context)=0; virtual uint64_t get_connections_count()=0; @@ -56,24 +56,27 @@ namespace nodetool virtual bool unblock_host(const epee::net_utils::network_address &address)=0; virtual std::map<std::string, time_t> get_blocked_hosts()=0; virtual bool add_host_fail(const epee::net_utils::network_address &address)=0; + virtual void add_used_stripe_peer(const t_connection_context &context)=0; + virtual void remove_used_stripe_peer(const t_connection_context &context)=0; + virtual void clear_used_stripe_peers()=0; }; template<class t_connection_context> struct p2p_endpoint_stub: public i_p2p_endpoint<t_connection_context> { - virtual bool relay_notify_to_list(int command, const std::string& data_buff, const std::list<boost::uuids::uuid>& connections) + virtual bool relay_notify_to_list(int command, const epee::span<const uint8_t> data_buff, const std::list<boost::uuids::uuid>& connections) { return false; } - virtual bool relay_notify_to_all(int command, const std::string& data_buff, const epee::net_utils::connection_context_base& context) + virtual bool relay_notify_to_all(int command, const epee::span<const uint8_t> data_buff, const epee::net_utils::connection_context_base& context) { return false; } - virtual bool invoke_command_to_peer(int command, const std::string& req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context) + virtual bool invoke_command_to_peer(int command, const epee::span<const uint8_t> req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context) { return false; } - virtual bool invoke_notify_to_peer(int command, const std::string& req_buff, const epee::net_utils::connection_context_base& context) + virtual bool invoke_notify_to_peer(int command, const epee::span<const uint8_t> req_buff, const epee::net_utils::connection_context_base& context) { return true; } @@ -114,5 +117,14 @@ namespace nodetool { return true; } + virtual void add_used_stripe_peer(const t_connection_context &context) + { + } + virtual void remove_used_stripe_peer(const t_connection_context &context) + { + } + virtual void clear_used_stripe_peers() + { + } }; } diff --git a/src/p2p/net_peerlist.h b/src/p2p/net_peerlist.h index 1d609a37e..685fdc193 100644 --- a/src/p2p/net_peerlist.h +++ b/src/p2p/net_peerlist.h @@ -68,21 +68,23 @@ namespace nodetool bool deinit(); size_t get_white_peers_count(){CRITICAL_REGION_LOCAL(m_peerlist_lock); return m_peers_white.size();} size_t get_gray_peers_count(){CRITICAL_REGION_LOCAL(m_peerlist_lock); return m_peers_gray.size();} - bool merge_peerlist(const std::list<peerlist_entry>& outer_bs); - bool get_peerlist_head(std::list<peerlist_entry>& bs_head, uint32_t depth = P2P_DEFAULT_PEERS_IN_HANDSHAKE); - bool get_peerlist_full(std::list<peerlist_entry>& pl_gray, std::list<peerlist_entry>& pl_white); + bool merge_peerlist(const std::vector<peerlist_entry>& outer_bs); + bool get_peerlist_head(std::vector<peerlist_entry>& bs_head, uint32_t depth = P2P_DEFAULT_PEERS_IN_HANDSHAKE); + bool get_peerlist_full(std::vector<peerlist_entry>& pl_gray, std::vector<peerlist_entry>& pl_white); bool get_white_peer_by_index(peerlist_entry& p, size_t i); bool get_gray_peer_by_index(peerlist_entry& p, size_t i); + template<typename F> bool foreach(bool white, const F &f); bool append_with_peer_white(const peerlist_entry& pr); bool append_with_peer_gray(const peerlist_entry& pr); bool append_with_peer_anchor(const anchor_peerlist_entry& ple); - bool set_peer_just_seen(peerid_type peer, const epee::net_utils::network_address& addr); + bool set_peer_just_seen(peerid_type peer, const epee::net_utils::network_address& addr, uint32_t pruning_seed); bool set_peer_unreachable(const peerlist_entry& pr); bool is_host_allowed(const epee::net_utils::network_address &address); bool get_random_gray_peer(peerlist_entry& pe); bool remove_from_peer_gray(const peerlist_entry& pe); bool get_and_empty_anchor_peerlist(std::vector<anchor_peerlist_entry>& apl); bool remove_from_peer_anchor(const epee::net_utils::network_address& addr); + bool remove_from_peer_white(const peerlist_entry& pe); private: struct by_time{}; @@ -265,7 +267,7 @@ namespace nodetool } //-------------------------------------------------------------------------------------------------- inline - bool peerlist_manager::merge_peerlist(const std::list<peerlist_entry>& outer_bs) + bool peerlist_manager::merge_peerlist(const std::vector<peerlist_entry>& outer_bs) { CRITICAL_REGION_LOCAL(m_peerlist_lock); for(const peerlist_entry& be: outer_bs) @@ -315,12 +317,13 @@ namespace nodetool } //-------------------------------------------------------------------------------------------------- inline - bool peerlist_manager::get_peerlist_head(std::list<peerlist_entry>& bs_head, uint32_t depth) + bool peerlist_manager::get_peerlist_head(std::vector<peerlist_entry>& bs_head, uint32_t depth) { CRITICAL_REGION_LOCAL(m_peerlist_lock); peers_indexed::index<by_time>::type& by_time_index=m_peers_white.get<by_time>(); uint32_t cnt = 0; + bs_head.reserve(depth); for(const peers_indexed::value_type& vl: boost::adaptors::reverse(by_time_index)) { if(!vl.last_seen) @@ -335,16 +338,18 @@ namespace nodetool } //-------------------------------------------------------------------------------------------------- inline - bool peerlist_manager::get_peerlist_full(std::list<peerlist_entry>& pl_gray, std::list<peerlist_entry>& pl_white) + bool peerlist_manager::get_peerlist_full(std::vector<peerlist_entry>& pl_gray, std::vector<peerlist_entry>& pl_white) { CRITICAL_REGION_LOCAL(m_peerlist_lock); peers_indexed::index<by_time>::type& by_time_index_gr=m_peers_gray.get<by_time>(); + pl_gray.resize(pl_gray.size() + by_time_index_gr.size()); for(const peers_indexed::value_type& vl: boost::adaptors::reverse(by_time_index_gr)) { pl_gray.push_back(vl); } peers_indexed::index<by_time>::type& by_time_index_wt=m_peers_white.get<by_time>(); + pl_white.resize(pl_white.size() + by_time_index_wt.size()); for(const peers_indexed::value_type& vl: boost::adaptors::reverse(by_time_index_wt)) { pl_white.push_back(vl); @@ -353,8 +358,19 @@ namespace nodetool return true; } //-------------------------------------------------------------------------------------------------- + template<typename F> inline + bool peerlist_manager::foreach(bool white, const F &f) + { + CRITICAL_REGION_LOCAL(m_peerlist_lock); + peers_indexed::index<by_time>::type& by_time_index = white ? m_peers_white.get<by_time>() : m_peers_gray.get<by_time>(); + for(const peers_indexed::value_type& vl: boost::adaptors::reverse(by_time_index)) + if (!f(vl)) + return false; + return true; + } + //-------------------------------------------------------------------------------------------------- inline - bool peerlist_manager::set_peer_just_seen(peerid_type peer, const epee::net_utils::network_address& addr) + bool peerlist_manager::set_peer_just_seen(peerid_type peer, const epee::net_utils::network_address& addr, uint32_t pruning_seed) { TRY_ENTRY(); CRITICAL_REGION_LOCAL(m_peerlist_lock); @@ -363,6 +379,7 @@ namespace nodetool ple.adr = addr; ple.id = peer; ple.last_seen = time(NULL); + ple.pruning_seed = pruning_seed; return append_with_peer_white(ple); CATCH_ENTRY_L0("peerlist_manager::set_peer_just_seen()", false); } @@ -466,6 +483,24 @@ namespace nodetool } //-------------------------------------------------------------------------------------------------- inline + bool peerlist_manager::remove_from_peer_white(const peerlist_entry& pe) + { + TRY_ENTRY(); + + CRITICAL_REGION_LOCAL(m_peerlist_lock); + + peers_indexed::index_iterator<by_addr>::type iterator = m_peers_white.get<by_addr>().find(pe.adr); + + if (iterator != m_peers_white.get<by_addr>().end()) { + m_peers_white.erase(iterator); + } + + return true; + + CATCH_ENTRY_L0("peerlist_manager::remove_from_peer_white()", false); + } + //-------------------------------------------------------------------------------------------------- + inline bool peerlist_manager::remove_from_peer_gray(const peerlist_entry& pe) { TRY_ENTRY(); diff --git a/src/p2p/net_peerlist_boost_serialization.h b/src/p2p/net_peerlist_boost_serialization.h index e79207888..2f4a7e661 100644 --- a/src/p2p/net_peerlist_boost_serialization.h +++ b/src/p2p/net_peerlist_boost_serialization.h @@ -33,6 +33,10 @@ #include "net/net_utils_base.h" #include "p2p/p2p_protocol_defs.h" +#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED +#include "common/pruning.h" +#endif + namespace boost { namespace serialization @@ -77,6 +81,19 @@ namespace boost a & pl.adr; a & pl.id; a & pl.last_seen; + if (ver < 1) + { + if (!typename Archive::is_saving()) + pl.pruning_seed = 0; + return; + } + a & pl.pruning_seed; +#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED + if (!typename Archive::is_saving()) + { + pl.pruning_seed = tools::make_pruning_seed(1+pl.adr.as<epee::net_utils::ipv4_network_address>().ip() % (1<<CRYPTONOTE_PRUNING_LOG_STRIPES), CRYPTONOTE_PRUNING_LOG_STRIPES); + } +#endif } template <class Archive, class ver_type> diff --git a/src/p2p/p2p_protocol_defs.h b/src/p2p/p2p_protocol_defs.h index e793e19b6..939cedf42 100644 --- a/src/p2p/p2p_protocol_defs.h +++ b/src/p2p/p2p_protocol_defs.h @@ -31,6 +31,7 @@ #pragma once #include <boost/uuid/uuid.hpp> +#include <boost/serialization/version.hpp> #include "serialization/keyvalue_serialization.h" #include "net/net_utils_base.h" #include "misc_language.h" @@ -72,11 +73,13 @@ namespace nodetool AddressType adr; peerid_type id; int64_t last_seen; + uint32_t pruning_seed; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(adr) KV_SERIALIZE(id) KV_SERIALIZE(last_seen) + KV_SERIALIZE_OPT(pruning_seed, (uint32_t)0) END_KV_SERIALIZE_MAP() }; typedef peerlist_entry_base<epee::net_utils::network_address> peerlist_entry; @@ -114,7 +117,7 @@ namespace nodetool #pragma pack(pop) inline - std::string print_peerlist_to_string(const std::list<peerlist_entry>& pl) + std::string print_peerlist_to_string(const std::vector<peerlist_entry>& pl) { time_t now_time = 0; time(&now_time); @@ -122,7 +125,7 @@ namespace nodetool ss << std::setfill ('0') << std::setw (8) << std::hex << std::noshowbase; for(const peerlist_entry& pe: pl) { - ss << pe.id << "\t" << pe.adr.str() << " \tlast_seen: " << epee::misc_utils::get_time_interval_string(now_time - pe.last_seen) << std::endl; + ss << pe.id << "\t" << pe.adr.str() << " \tpruning seed " << pe.pruning_seed << " \tlast_seen: " << epee::misc_utils::get_time_interval_string(now_time - pe.last_seen) << std::endl; } return ss.str(); } @@ -189,7 +192,7 @@ namespace nodetool { basic_node_data node_data; t_playload_type payload_data; - std::list<peerlist_entry> local_peerlist_new; + std::vector<peerlist_entry> local_peerlist_new; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(node_data) @@ -198,14 +201,14 @@ namespace nodetool { // saving: save both, so old and new peers can understand it KV_SERIALIZE(local_peerlist_new) - std::list<peerlist_entry_base<network_address_old>> local_peerlist; + std::vector<peerlist_entry_base<network_address_old>> local_peerlist; for (const auto &p: this_ref.local_peerlist_new) { if (p.adr.get_type_id() == epee::net_utils::ipv4_network_address::ID) { const epee::net_utils::network_address &na = p.adr; const epee::net_utils::ipv4_network_address &ipv4 = na.as<const epee::net_utils::ipv4_network_address>(); - local_peerlist.push_back(peerlist_entry_base<network_address_old>({{ipv4.ip(), ipv4.port()}, p.id, p.last_seen})); + local_peerlist.push_back(peerlist_entry_base<network_address_old>({{ipv4.ip(), ipv4.port()}, p.id, p.last_seen, p.pruning_seed})); } else MDEBUG("Not including in legacy peer list: " << p.adr.str()); @@ -217,10 +220,10 @@ namespace nodetool // loading: load old list only if there is no new one if (!epee::serialization::selector<is_store>::serialize(this_ref.local_peerlist_new, stg, hparent_section, "local_peerlist_new")) { - std::list<peerlist_entry_base<network_address_old>> local_peerlist; + std::vector<peerlist_entry_base<network_address_old>> local_peerlist; epee::serialization::selector<is_store>::serialize_stl_container_pod_val_as_blob(local_peerlist, stg, hparent_section, "local_peerlist"); for (const auto &p: local_peerlist) - ((response&)this_ref).local_peerlist_new.push_back(peerlist_entry({epee::net_utils::ipv4_network_address(p.adr.ip, p.adr.port), p.id, p.last_seen})); + ((response&)this_ref).local_peerlist_new.push_back(peerlist_entry({epee::net_utils::ipv4_network_address(p.adr.ip, p.adr.port), p.id, p.last_seen, p.pruning_seed})); } } END_KV_SERIALIZE_MAP() @@ -248,7 +251,7 @@ namespace nodetool { uint64_t local_time; t_playload_type payload_data; - std::list<peerlist_entry> local_peerlist_new; + std::vector<peerlist_entry> local_peerlist_new; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(local_time) @@ -257,7 +260,7 @@ namespace nodetool { // saving: save both, so old and new peers can understand it KV_SERIALIZE(local_peerlist_new) - std::list<peerlist_entry_base<network_address_old>> local_peerlist; + std::vector<peerlist_entry_base<network_address_old>> local_peerlist; for (const auto &p: this_ref.local_peerlist_new) { if (p.adr.get_type_id() == epee::net_utils::ipv4_network_address::ID) @@ -276,7 +279,7 @@ namespace nodetool // loading: load old list only if there is no new one if (!epee::serialization::selector<is_store>::serialize(this_ref.local_peerlist_new, stg, hparent_section, "local_peerlist_new")) { - std::list<peerlist_entry_base<network_address_old>> local_peerlist; + std::vector<peerlist_entry_base<network_address_old>> local_peerlist; epee::serialization::selector<is_store>::serialize_stl_container_pod_val_as_blob(local_peerlist, stg, hparent_section, "local_peerlist"); for (const auto &p: local_peerlist) ((response&)this_ref).local_peerlist_new.push_back(peerlist_entry({epee::net_utils::ipv4_network_address(p.adr.ip, p.adr.port), p.id, p.last_seen})); @@ -389,9 +392,9 @@ namespace nodetool struct response { - std::list<peerlist_entry> local_peerlist_white; - std::list<peerlist_entry> local_peerlist_gray; - std::list<connection_entry> connections_list; + std::vector<peerlist_entry> local_peerlist_white; + std::vector<peerlist_entry> local_peerlist_gray; + std::vector<connection_entry> connections_list; peerid_type my_id; uint64_t local_time; BEGIN_KV_SERIALIZE_MAP() @@ -463,5 +466,6 @@ namespace nodetool } +BOOST_CLASS_VERSION(nodetool::peerlist_entry, 1) diff --git a/src/ringct/bulletproofs.cc b/src/ringct/bulletproofs.cc index 381f50872..d485fb748 100644 --- a/src/ringct/bulletproofs.cc +++ b/src/ringct/bulletproofs.cc @@ -29,10 +29,10 @@ // Adapted from Java code by Sarang Noether #include <stdlib.h> -#include <openssl/ssl.h> -#include <openssl/bn.h> #include <boost/thread/mutex.hpp> +#include <boost/thread/lock_guard.hpp> #include "misc_log_ex.h" +#include "span.h" #include "common/perf_timer.h" #include "cryptonote_config.h" extern "C" @@ -48,9 +48,15 @@ extern "C" //#define DEBUG_BP +#if 1 #define PERF_TIMER_START_BP(x) PERF_TIMER_START_UNIT(x, 1000000) +#define PERF_TIMER_STOP_BP(x) PERF_TIMER_STOP(x) +#else +#define PERF_TIMER_START_BP(x) ((void*)0) +#define PERF_TIMER_STOP_BP(x) ((void*)0) +#endif -#define STRAUS_SIZE_LIMIT 128 +#define STRAUS_SIZE_LIMIT 232 #define PIPPENGER_SIZE_LIMIT 0 namespace rct @@ -75,65 +81,20 @@ static const rct::keyV twoN = vector_powers(TWO, maxN); static const rct::key ip12 = inner_product(oneN, twoN); static boost::mutex init_mutex; -static inline rct::key multiexp(const std::vector<MultiexpData> &data, bool HiGi) +static inline rct::key multiexp(const std::vector<MultiexpData> &data, size_t HiGi_size) { - if (HiGi) + if (HiGi_size > 0) { - static_assert(128 <= STRAUS_SIZE_LIMIT, "Straus in precalc mode can only be calculated till STRAUS_SIZE_LIMIT"); - return data.size() <= 128 ? straus(data, straus_HiGi_cache, 0) : pippenger(data, pippenger_HiGi_cache, get_pippenger_c(data.size())); + static_assert(232 <= STRAUS_SIZE_LIMIT, "Straus in precalc mode can only be calculated till STRAUS_SIZE_LIMIT"); + return HiGi_size <= 232 && data.size() == HiGi_size ? straus(data, straus_HiGi_cache, 0) : pippenger(data, pippenger_HiGi_cache, HiGi_size, get_pippenger_c(data.size())); } else - return data.size() <= 64 ? straus(data, NULL, 0) : pippenger(data, NULL, get_pippenger_c(data.size())); -} - -static bool is_reduced(const rct::key &scalar) -{ - rct::key reduced = scalar; - sc_reduce32(reduced.bytes); - return scalar == reduced; -} - -static void addKeys_acc_p3(ge_p3 *acc_p3, const rct::key &a, const rct::key &point) -{ - ge_p3 p3; - CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&p3, point.bytes) == 0, "ge_frombytes_vartime failed"); - ge_scalarmult_p3(&p3, a.bytes, &p3); - ge_cached cached; - ge_p3_to_cached(&cached, acc_p3); - ge_p1p1 p1; - ge_add(&p1, &p3, &cached); - ge_p1p1_to_p3(acc_p3, &p1); -} - -static void add_acc_p3(ge_p3 *acc_p3, const rct::key &point) -{ - ge_p3 p3; - CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&p3, point.bytes) == 0, "ge_frombytes_vartime failed"); - ge_cached cached; - ge_p3_to_cached(&cached, &p3); - ge_p1p1 p1; - ge_add(&p1, acc_p3, &cached); - ge_p1p1_to_p3(acc_p3, &p1); + return data.size() <= 95 ? straus(data, NULL, 0) : pippenger(data, NULL, 0, get_pippenger_c(data.size())); } -static void sub_acc_p3(ge_p3 *acc_p3, const rct::key &point) +static inline bool is_reduced(const rct::key &scalar) { - ge_p3 p3; - CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&p3, point.bytes) == 0, "ge_frombytes_vartime failed"); - ge_cached cached; - ge_p3_to_cached(&cached, &p3); - ge_p1p1 p1; - ge_sub(&p1, acc_p3, &cached); - ge_p1p1_to_p3(acc_p3, &p1); -} - -static rct::key scalarmultKey(const ge_p3 &P, const rct::key &a) -{ - ge_p2 R; - ge_scalarmult(&R, a.bytes, &P); - rct::key aP; - ge_tobytes(aP.bytes, &R); - return aP; + return sc_check(scalar.bytes) == 0; } static rct::key get_exponent(const rct::key &base, size_t idx) @@ -153,6 +114,7 @@ static void init_exponents() if (init_done) return; std::vector<MultiexpData> data; + data.reserve(maxN*maxM*2); for (size_t i = 0; i < maxN*maxM; ++i) { Hi[i] = get_exponent(rct::H, i * 2); @@ -160,12 +122,12 @@ static void init_exponents() Gi[i] = get_exponent(rct::H, i * 2 + 1); CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&Gi_p3[i], Gi[i].bytes) == 0, "ge_frombytes_vartime failed"); - data.push_back({rct::zero(), Gi[i]}); - data.push_back({rct::zero(), Hi[i]}); + data.push_back({rct::zero(), Gi_p3[i]}); + data.push_back({rct::zero(), Hi_p3[i]}); } straus_HiGi_cache = straus_init_cache(data, STRAUS_SIZE_LIMIT); - pippenger_HiGi_cache = pippenger_init_cache(data, PIPPENGER_SIZE_LIMIT); + pippenger_HiGi_cache = pippenger_init_cache(data, 0, PIPPENGER_SIZE_LIMIT); MINFO("Hi/Gi cache size: " << (sizeof(Hi)+sizeof(Gi))/1024 << " kB"); MINFO("Hi_p3/Gi_p3 cache size: " << (sizeof(Hi_p3)+sizeof(Gi_p3))/1024 << " kB"); @@ -189,29 +151,37 @@ static rct::key vector_exponent(const rct::keyV &a, const rct::keyV &b) multiexp_data.emplace_back(a[i], Gi_p3[i]); multiexp_data.emplace_back(b[i], Hi_p3[i]); } - return multiexp(multiexp_data, true); + return multiexp(multiexp_data, 2 * a.size()); } /* Compute a custom vector-scalar commitment */ -static rct::key vector_exponent_custom(const rct::keyV &A, const rct::keyV &B, const rct::keyV &a, const rct::keyV &b) +static rct::key cross_vector_exponent8(size_t size, const std::vector<ge_p3> &A, size_t Ao, const std::vector<ge_p3> &B, size_t Bo, const rct::keyV &a, size_t ao, const rct::keyV &b, size_t bo, const rct::keyV *scale, const ge_p3 *extra_point, const rct::key *extra_scalar) { - CHECK_AND_ASSERT_THROW_MES(A.size() == B.size(), "Incompatible sizes of A and B"); - CHECK_AND_ASSERT_THROW_MES(a.size() == b.size(), "Incompatible sizes of a and b"); - CHECK_AND_ASSERT_THROW_MES(a.size() == A.size(), "Incompatible sizes of a and A"); - CHECK_AND_ASSERT_THROW_MES(a.size() <= maxN*maxM, "Incompatible sizes of a and maxN"); + CHECK_AND_ASSERT_THROW_MES(size + Ao <= A.size(), "Incompatible size for A"); + CHECK_AND_ASSERT_THROW_MES(size + Bo <= B.size(), "Incompatible size for B"); + CHECK_AND_ASSERT_THROW_MES(size + ao <= a.size(), "Incompatible size for a"); + CHECK_AND_ASSERT_THROW_MES(size + bo <= b.size(), "Incompatible size for b"); + CHECK_AND_ASSERT_THROW_MES(size <= maxN*maxM, "size is too large"); + CHECK_AND_ASSERT_THROW_MES(!scale || size == scale->size() / 2, "Incompatible size for scale"); + CHECK_AND_ASSERT_THROW_MES(!!extra_point == !!extra_scalar, "only one of extra point/scalar present"); std::vector<MultiexpData> multiexp_data; - multiexp_data.reserve(a.size()*2); - for (size_t i = 0; i < a.size(); ++i) + multiexp_data.resize(size*2 + (!!extra_point)); + for (size_t i = 0; i < size; ++i) + { + sc_mul(multiexp_data[i*2].scalar.bytes, a[ao+i].bytes, INV_EIGHT.bytes);; + multiexp_data[i*2].point = A[Ao+i]; + sc_mul(multiexp_data[i*2+1].scalar.bytes, b[bo+i].bytes, INV_EIGHT.bytes); + if (scale) + sc_mul(multiexp_data[i*2+1].scalar.bytes, multiexp_data[i*2+1].scalar.bytes, (*scale)[Bo+i].bytes); + multiexp_data[i*2+1].point = B[Bo+i]; + } + if (extra_point) { - multiexp_data.resize(multiexp_data.size() + 1); - multiexp_data.back().scalar = a[i]; - CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&multiexp_data.back().point, A[i].bytes) == 0, "ge_frombytes_vartime failed"); - multiexp_data.resize(multiexp_data.size() + 1); - multiexp_data.back().scalar = b[i]; - CHECK_AND_ASSERT_THROW_MES(ge_frombytes_vartime(&multiexp_data.back().point, B[i].bytes) == 0, "ge_frombytes_vartime failed"); + sc_mul(multiexp_data.back().scalar.bytes, extra_scalar->bytes, INV_EIGHT.bytes); + multiexp_data.back().point = *extra_point; } - return multiexp(multiexp_data, false); + return multiexp(multiexp_data, 0); } /* Given a scalar, construct a vector of powers */ @@ -250,7 +220,7 @@ static rct::key vector_power_sum(const rct::key &x, size_t n) } /* Given two scalar arrays, construct the inner product */ -static rct::key inner_product(const rct::keyV &a, const rct::keyV &b) +static rct::key inner_product(const epee::span<const rct::key> &a, const epee::span<const rct::key> &b) { CHECK_AND_ASSERT_THROW_MES(a.size() == b.size(), "Incompatible sizes of a and b"); rct::key res = rct::zero(); @@ -261,6 +231,11 @@ static rct::key inner_product(const rct::keyV &a, const rct::keyV &b) return res; } +static rct::key inner_product(const rct::keyV &a, const rct::keyV &b) +{ + return inner_product(epee::span<const rct::key>(a.data(), a.size()), epee::span<const rct::key>(b.data(), b.size())); +} + /* Given two scalar arrays, construct the Hadamard product */ static rct::keyV hadamard(const rct::keyV &a, const rct::keyV &b) { @@ -273,16 +248,22 @@ static rct::keyV hadamard(const rct::keyV &a, const rct::keyV &b) return res; } -/* Given two curvepoint arrays, construct the Hadamard product */ -static rct::keyV hadamard2(const rct::keyV &a, const rct::keyV &b) +/* folds a curvepoint array using a two way scaled Hadamard product */ +static void hadamard_fold(std::vector<ge_p3> &v, const rct::keyV *scale, const rct::key &a, const rct::key &b) { - CHECK_AND_ASSERT_THROW_MES(a.size() == b.size(), "Incompatible sizes of a and b"); - rct::keyV res(a.size()); - for (size_t i = 0; i < a.size(); ++i) + CHECK_AND_ASSERT_THROW_MES((v.size() & 1) == 0, "Vector size should be even"); + const size_t sz = v.size() / 2; + for (size_t n = 0; n < sz; ++n) { - rct::addKeys(res[i], a[i], b[i]); + ge_dsmp c[2]; + ge_dsm_precomp(c[0], &v[n]); + ge_dsm_precomp(c[1], &v[sz + n]); + rct::key sa, sb; + if (scale) sc_mul(sa.bytes, a.bytes, (*scale)[n].bytes); else sa = a; + if (scale) sc_mul(sb.bytes, b.bytes, (*scale)[sz + n].bytes); else sb = b; + ge_double_scalarmult_precomp_vartime2_p3(&v[n], sa.bytes, c[0], sb.bytes, c[1]); } - return res; + v.resize(sz); } /* Add two vectors */ @@ -297,88 +278,103 @@ static rct::keyV vector_add(const rct::keyV &a, const rct::keyV &b) return res; } -/* Subtract two vectors */ -static rct::keyV vector_subtract(const rct::keyV &a, const rct::keyV &b) +/* Add a scalar to all elements of a vector */ +static rct::keyV vector_add(const rct::keyV &a, const rct::key &b) { - CHECK_AND_ASSERT_THROW_MES(a.size() == b.size(), "Incompatible sizes of a and b"); rct::keyV res(a.size()); for (size_t i = 0; i < a.size(); ++i) { - sc_sub(res[i].bytes, a[i].bytes, b[i].bytes); + sc_add(res[i].bytes, a[i].bytes, b.bytes); } return res; } -/* Multiply a scalar and a vector */ -static rct::keyV vector_scalar(const rct::keyV &a, const rct::key &x) +/* Subtract a scalar from all elements of a vector */ +static rct::keyV vector_subtract(const rct::keyV &a, const rct::key &b) { rct::keyV res(a.size()); for (size_t i = 0; i < a.size(); ++i) { - sc_mul(res[i].bytes, a[i].bytes, x.bytes); + sc_sub(res[i].bytes, a[i].bytes, b.bytes); } return res; } -/* Create a vector from copies of a single value */ -static rct::keyV vector_dup(const rct::key &x, size_t N) -{ - return rct::keyV(N, x); -} - -/* Exponentiate a curve vector by a scalar */ -static rct::keyV vector_scalar2(const rct::keyV &a, const rct::key &x) +/* Multiply a scalar and a vector */ +static rct::keyV vector_scalar(const epee::span<const rct::key> &a, const rct::key &x) { rct::keyV res(a.size()); for (size_t i = 0; i < a.size(); ++i) { - rct::scalarmultKey(res[i], a[i], x); + sc_mul(res[i].bytes, a[i].bytes, x.bytes); } return res; } -/* Get the sum of a vector's elements */ -static rct::key vector_sum(const rct::keyV &a) +static rct::keyV vector_scalar(const rct::keyV &a, const rct::key &x) { - rct::key res = rct::zero(); - for (size_t i = 0; i < a.size(); ++i) - { - sc_add(res.bytes, res.bytes, a[i].bytes); - } - return res; + return vector_scalar(epee::span<const rct::key>(a.data(), a.size()), x); } -static rct::key switch_endianness(rct::key k) +/* Create a vector from copies of a single value */ +static rct::keyV vector_dup(const rct::key &x, size_t N) { - std::reverse(k.bytes, k.bytes + sizeof(k)); - return k; + return rct::keyV(N, x); } -/* Compute the inverse of a scalar, the stupid way */ -static rct::key invert(const rct::key &x) +static rct::key sm(rct::key y, int n, const rct::key &x) { - rct::key inv; - - BN_CTX *ctx = BN_CTX_new(); - BIGNUM *X = BN_new(); - BIGNUM *L = BN_new(); - BIGNUM *I = BN_new(); - - BN_bin2bn(switch_endianness(x).bytes, sizeof(rct::key), X); - BN_bin2bn(switch_endianness(rct::curveOrder()).bytes, sizeof(rct::key), L); - - CHECK_AND_ASSERT_THROW_MES(BN_mod_inverse(I, X, L, ctx), "Failed to invert"); + while (n--) + sc_mul(y.bytes, y.bytes, y.bytes); + sc_mul(y.bytes, y.bytes, x.bytes); + return y; +} - const int len = BN_num_bytes(I); - CHECK_AND_ASSERT_THROW_MES((size_t)len <= sizeof(rct::key), "Invalid number length"); - inv = rct::zero(); - BN_bn2bin(I, inv.bytes); - std::reverse(inv.bytes, inv.bytes + len); +/* Compute the inverse of a scalar, the clever way */ +static rct::key invert(const rct::key &x) +{ + rct::key _1, _10, _100, _11, _101, _111, _1001, _1011, _1111; + + _1 = x; + sc_mul(_10.bytes, _1.bytes, _1.bytes); + sc_mul(_100.bytes, _10.bytes, _10.bytes); + sc_mul(_11.bytes, _10.bytes, _1.bytes); + sc_mul(_101.bytes, _10.bytes, _11.bytes); + sc_mul(_111.bytes, _10.bytes, _101.bytes); + sc_mul(_1001.bytes, _10.bytes, _111.bytes); + sc_mul(_1011.bytes, _10.bytes, _1001.bytes); + sc_mul(_1111.bytes, _100.bytes, _1011.bytes); - BN_free(I); - BN_free(L); - BN_free(X); - BN_CTX_free(ctx); + rct::key inv; + sc_mul(inv.bytes, _1111.bytes, _1.bytes); + + inv = sm(inv, 123 + 3, _101); + inv = sm(inv, 2 + 2, _11); + inv = sm(inv, 1 + 4, _1111); + inv = sm(inv, 1 + 4, _1111); + inv = sm(inv, 4, _1001); + inv = sm(inv, 2, _11); + inv = sm(inv, 1 + 4, _1111); + inv = sm(inv, 1 + 3, _101); + inv = sm(inv, 3 + 3, _101); + inv = sm(inv, 3, _111); + inv = sm(inv, 1 + 4, _1111); + inv = sm(inv, 2 + 3, _111); + inv = sm(inv, 2 + 2, _11); + inv = sm(inv, 1 + 4, _1011); + inv = sm(inv, 2 + 4, _1011); + inv = sm(inv, 6 + 4, _1001); + inv = sm(inv, 2 + 2, _11); + inv = sm(inv, 3 + 2, _11); + inv = sm(inv, 3 + 2, _11); + inv = sm(inv, 1 + 4, _1001); + inv = sm(inv, 1 + 3, _111); + inv = sm(inv, 2 + 4, _1111); + inv = sm(inv, 1 + 4, _1011); + inv = sm(inv, 3, _101); + inv = sm(inv, 2 + 4, _1111); + inv = sm(inv, 3, _101); + inv = sm(inv, 1 + 2, _11); #ifdef DEBUG_BP rct::key tmp; @@ -388,18 +384,41 @@ static rct::key invert(const rct::key &x) return inv; } +static rct::keyV invert(rct::keyV x) +{ + rct::keyV scratch; + scratch.reserve(x.size()); + + rct::key acc = rct::identity(); + for (size_t n = 0; n < x.size(); ++n) + { + scratch.push_back(acc); + if (n == 0) + acc = x[0]; + else + sc_mul(acc.bytes, acc.bytes, x[n].bytes); + } + + acc = invert(acc); + + rct::key tmp; + for (int i = x.size(); i-- > 0; ) + { + sc_mul(tmp.bytes, acc.bytes, x[i].bytes); + sc_mul(x[i].bytes, acc.bytes, scratch[i].bytes); + acc = tmp; + } + + return x; +} + /* Compute the slice of a vector */ -static rct::keyV slice(const rct::keyV &a, size_t start, size_t stop) +static epee::span<const rct::key> slice(const rct::keyV &a, size_t start, size_t stop) { CHECK_AND_ASSERT_THROW_MES(start < a.size(), "Invalid start index"); CHECK_AND_ASSERT_THROW_MES(stop <= a.size(), "Invalid stop index"); CHECK_AND_ASSERT_THROW_MES(start < stop, "Invalid start/stop indices"); - rct::keyV res(stop - start); - for (size_t i = start; i < stop; ++i) - { - res[i - start] = a[i]; - } - return res; + return epee::span<const rct::key>(&a[start], stop - start); } static rct::key hash_cache_mash(rct::key &hash_cache, const rct::key &mash0, const rct::key &mash1) @@ -438,270 +457,12 @@ static rct::key hash_cache_mash(rct::key &hash_cache, const rct::key &mash0, con /* Given a value v (0..2^N-1) and a mask gamma, construct a range proof */ Bulletproof bulletproof_PROVE(const rct::key &sv, const rct::key &gamma) { - init_exponents(); - - PERF_TIMER_UNIT(PROVE, 1000000); - - constexpr size_t logN = 6; // log2(64) - constexpr size_t N = 1<<logN; - - rct::key V; - rct::keyV aL(N), aR(N); - - PERF_TIMER_START_BP(PROVE_v); - rct::addKeys2(V, gamma, sv, rct::H); - V = rct::scalarmultKey(V, INV_EIGHT); - PERF_TIMER_STOP(PROVE_v); - - PERF_TIMER_START_BP(PROVE_aLaR); - for (size_t i = N; i-- > 0; ) - { - if (sv[i/8] & (((uint64_t)1)<<(i%8))) - { - aL[i] = rct::identity(); - } - else - { - aL[i] = rct::zero(); - } - sc_sub(aR[i].bytes, aL[i].bytes, rct::identity().bytes); - } - PERF_TIMER_STOP(PROVE_aLaR); - - rct::key hash_cache = rct::hash_to_scalar(V); - - // DEBUG: Test to ensure this recovers the value -#ifdef DEBUG_BP - uint64_t test_aL = 0, test_aR = 0; - for (size_t i = 0; i < N; ++i) - { - if (aL[i] == rct::identity()) - test_aL += ((uint64_t)1)<<i; - if (aR[i] == rct::zero()) - test_aR += ((uint64_t)1)<<i; - } - uint64_t v_test = 0; - for (int n = 0; n < 8; ++n) v_test |= (((uint64_t)sv[n]) << (8*n)); - CHECK_AND_ASSERT_THROW_MES(test_aL == v_test, "test_aL failed"); - CHECK_AND_ASSERT_THROW_MES(test_aR == v_test, "test_aR failed"); -#endif - -try_again: - PERF_TIMER_START_BP(PROVE_step1); - // PAPER LINES 38-39 - rct::key alpha = rct::skGen(); - rct::key ve = vector_exponent(aL, aR); - rct::key A; - rct::addKeys(A, ve, rct::scalarmultBase(alpha)); - A = rct::scalarmultKey(A, INV_EIGHT); - - // PAPER LINES 40-42 - rct::keyV sL = rct::skvGen(N), sR = rct::skvGen(N); - rct::key rho = rct::skGen(); - ve = vector_exponent(sL, sR); - rct::key S; - rct::addKeys(S, ve, rct::scalarmultBase(rho)); - S = rct::scalarmultKey(S, INV_EIGHT); - - // PAPER LINES 43-45 - rct::key y = hash_cache_mash(hash_cache, A, S); - if (y == rct::zero()) - { - PERF_TIMER_STOP(PROVE_step1); - MINFO("y is 0, trying again"); - goto try_again; - } - rct::key z = hash_cache = rct::hash_to_scalar(y); - if (z == rct::zero()) - { - PERF_TIMER_STOP(PROVE_step1); - MINFO("z is 0, trying again"); - goto try_again; - } - - // Polynomial construction before PAPER LINE 46 - rct::key t0 = rct::zero(); - rct::key t1 = rct::zero(); - rct::key t2 = rct::zero(); - - const auto yN = vector_powers(y, N); - - rct::key ip1y = vector_sum(yN); - rct::key tmp; - sc_muladd(t0.bytes, z.bytes, ip1y.bytes, t0.bytes); - - rct::key zsq; - sc_mul(zsq.bytes, z.bytes, z.bytes); - sc_muladd(t0.bytes, zsq.bytes, sv.bytes, t0.bytes); - - rct::key k = rct::zero(); - sc_mulsub(k.bytes, zsq.bytes, ip1y.bytes, k.bytes); - - rct::key zcu; - sc_mul(zcu.bytes, zsq.bytes, z.bytes); - sc_mulsub(k.bytes, zcu.bytes, ip12.bytes, k.bytes); - sc_add(t0.bytes, t0.bytes, k.bytes); - - // DEBUG: Test the value of t0 has the correct form -#ifdef DEBUG_BP - rct::key test_t0 = rct::zero(); - rct::key iph = inner_product(aL, hadamard(aR, yN)); - sc_add(test_t0.bytes, test_t0.bytes, iph.bytes); - rct::key ips = inner_product(vector_subtract(aL, aR), yN); - sc_muladd(test_t0.bytes, z.bytes, ips.bytes, test_t0.bytes); - rct::key ipt = inner_product(twoN, aL); - sc_muladd(test_t0.bytes, zsq.bytes, ipt.bytes, test_t0.bytes); - sc_add(test_t0.bytes, test_t0.bytes, k.bytes); - CHECK_AND_ASSERT_THROW_MES(t0 == test_t0, "t0 check failed"); -#endif - PERF_TIMER_STOP(PROVE_step1); - - PERF_TIMER_START_BP(PROVE_step2); - const auto HyNsR = hadamard(yN, sR); - const auto vpIz = vector_dup(z, N); - const auto vp2zsq = vector_scalar(twoN, zsq); - const auto aL_vpIz = vector_subtract(aL, vpIz); - const auto aR_vpIz = vector_add(aR, vpIz); - - rct::key ip1 = inner_product(aL_vpIz, HyNsR); - sc_add(t1.bytes, t1.bytes, ip1.bytes); - - rct::key ip2 = inner_product(sL, vector_add(hadamard(yN, aR_vpIz), vp2zsq)); - sc_add(t1.bytes, t1.bytes, ip2.bytes); - - rct::key ip3 = inner_product(sL, HyNsR); - sc_add(t2.bytes, t2.bytes, ip3.bytes); - - // PAPER LINES 47-48 - rct::key tau1 = rct::skGen(), tau2 = rct::skGen(); - - rct::key T1 = rct::addKeys(rct::scalarmultH(t1), rct::scalarmultBase(tau1)); - T1 = rct::scalarmultKey(T1, INV_EIGHT); - rct::key T2 = rct::addKeys(rct::scalarmultH(t2), rct::scalarmultBase(tau2)); - T2 = rct::scalarmultKey(T2, INV_EIGHT); - - // PAPER LINES 49-51 - rct::key x = hash_cache_mash(hash_cache, z, T1, T2); - if (x == rct::zero()) - { - PERF_TIMER_STOP(PROVE_step2); - MINFO("x is 0, trying again"); - goto try_again; - } - - // PAPER LINES 52-53 - rct::key taux = rct::zero(); - sc_mul(taux.bytes, tau1.bytes, x.bytes); - rct::key xsq; - sc_mul(xsq.bytes, x.bytes, x.bytes); - sc_muladd(taux.bytes, tau2.bytes, xsq.bytes, taux.bytes); - sc_muladd(taux.bytes, gamma.bytes, zsq.bytes, taux.bytes); - rct::key mu; - sc_muladd(mu.bytes, x.bytes, rho.bytes, alpha.bytes); - - // PAPER LINES 54-57 - rct::keyV l = vector_add(aL_vpIz, vector_scalar(sL, x)); - rct::keyV r = vector_add(hadamard(yN, vector_add(aR_vpIz, vector_scalar(sR, x))), vp2zsq); - PERF_TIMER_STOP(PROVE_step2); - - PERF_TIMER_START_BP(PROVE_step3); - rct::key t = inner_product(l, r); - - // DEBUG: Test if the l and r vectors match the polynomial forms -#ifdef DEBUG_BP - rct::key test_t; - sc_muladd(test_t.bytes, t1.bytes, x.bytes, t0.bytes); - sc_muladd(test_t.bytes, t2.bytes, xsq.bytes, test_t.bytes); - CHECK_AND_ASSERT_THROW_MES(test_t == t, "test_t check failed"); -#endif - - // PAPER LINES 32-33 - rct::key x_ip = hash_cache_mash(hash_cache, x, taux, mu, t); - - // These are used in the inner product rounds - size_t nprime = N; - rct::keyV Gprime(N); - rct::keyV Hprime(N); - rct::keyV aprime(N); - rct::keyV bprime(N); - const rct::key yinv = invert(y); - rct::key yinvpow = rct::identity(); - for (size_t i = 0; i < N; ++i) - { - Gprime[i] = Gi[i]; - Hprime[i] = scalarmultKey(Hi_p3[i], yinvpow); - sc_mul(yinvpow.bytes, yinvpow.bytes, yinv.bytes); - aprime[i] = l[i]; - bprime[i] = r[i]; - } - rct::keyV L(logN); - rct::keyV R(logN); - int round = 0; - rct::keyV w(logN); // this is the challenge x in the inner product protocol - PERF_TIMER_STOP(PROVE_step3); - - PERF_TIMER_START_BP(PROVE_step4); - // PAPER LINE 13 - while (nprime > 1) - { - // PAPER LINE 15 - nprime /= 2; - - // PAPER LINES 16-17 - rct::key cL = inner_product(slice(aprime, 0, nprime), slice(bprime, nprime, bprime.size())); - rct::key cR = inner_product(slice(aprime, nprime, aprime.size()), slice(bprime, 0, nprime)); - - // PAPER LINES 18-19 - L[round] = vector_exponent_custom(slice(Gprime, nprime, Gprime.size()), slice(Hprime, 0, nprime), slice(aprime, 0, nprime), slice(bprime, nprime, bprime.size())); - sc_mul(tmp.bytes, cL.bytes, x_ip.bytes); - rct::addKeys(L[round], L[round], rct::scalarmultH(tmp)); - L[round] = rct::scalarmultKey(L[round], INV_EIGHT); - R[round] = vector_exponent_custom(slice(Gprime, 0, nprime), slice(Hprime, nprime, Hprime.size()), slice(aprime, nprime, aprime.size()), slice(bprime, 0, nprime)); - sc_mul(tmp.bytes, cR.bytes, x_ip.bytes); - rct::addKeys(R[round], R[round], rct::scalarmultH(tmp)); - R[round] = rct::scalarmultKey(R[round], INV_EIGHT); - - // PAPER LINES 21-22 - w[round] = hash_cache_mash(hash_cache, L[round], R[round]); - if (w[round] == rct::zero()) - { - PERF_TIMER_STOP(PROVE_step4); - MINFO("w[round] is 0, trying again"); - goto try_again; - } - - // PAPER LINES 24-25 - const rct::key winv = invert(w[round]); - Gprime = hadamard2(vector_scalar2(slice(Gprime, 0, nprime), winv), vector_scalar2(slice(Gprime, nprime, Gprime.size()), w[round])); - Hprime = hadamard2(vector_scalar2(slice(Hprime, 0, nprime), w[round]), vector_scalar2(slice(Hprime, nprime, Hprime.size()), winv)); - - // PAPER LINES 28-29 - aprime = vector_add(vector_scalar(slice(aprime, 0, nprime), w[round]), vector_scalar(slice(aprime, nprime, aprime.size()), winv)); - bprime = vector_add(vector_scalar(slice(bprime, 0, nprime), winv), vector_scalar(slice(bprime, nprime, bprime.size()), w[round])); - - ++round; - } - PERF_TIMER_STOP(PROVE_step4); - - // PAPER LINE 58 (with inclusions from PAPER LINE 8 and PAPER LINE 20) - return Bulletproof(V, A, S, T1, T2, taux, mu, L, R, aprime[0], bprime[0], t); + return bulletproof_PROVE(rct::keyV(1, sv), rct::keyV(1, gamma)); } Bulletproof bulletproof_PROVE(uint64_t v, const rct::key &gamma) { - // vG + gammaH - PERF_TIMER_START_BP(PROVE_v); - rct::key sv = rct::zero(); - sv.bytes[0] = v & 255; - sv.bytes[1] = (v >> 8) & 255; - sv.bytes[2] = (v >> 16) & 255; - sv.bytes[3] = (v >> 24) & 255; - sv.bytes[4] = (v >> 32) & 255; - sv.bytes[5] = (v >> 40) & 255; - sv.bytes[6] = (v >> 48) & 255; - sv.bytes[7] = (v >> 56) & 255; - PERF_TIMER_STOP(PROVE_v); - return bulletproof_PROVE(sv, gamma); + return bulletproof_PROVE(std::vector<uint64_t>(1, v), rct::keyV(1, gamma)); } /* Given a set of values v (0..2^N-1) and masks gamma, construct a range proof */ @@ -728,37 +489,39 @@ Bulletproof bulletproof_PROVE(const rct::keyV &sv, const rct::keyV &gamma) rct::keyV V(sv.size()); rct::keyV aL(MN), aR(MN); - rct::key tmp; + rct::keyV aL8(MN), aR8(MN); + rct::key tmp, tmp2; PERF_TIMER_START_BP(PROVE_v); for (size_t i = 0; i < sv.size(); ++i) { - rct::addKeys2(V[i], gamma[i], sv[i], rct::H); - V[i] = rct::scalarmultKey(V[i], INV_EIGHT); + rct::key gamma8, sv8; + sc_mul(gamma8.bytes, gamma[i].bytes, INV_EIGHT.bytes); + sc_mul(sv8.bytes, sv[i].bytes, INV_EIGHT.bytes); + rct::addKeys2(V[i], gamma8, sv8, rct::H); } - PERF_TIMER_STOP(PROVE_v); + PERF_TIMER_STOP_BP(PROVE_v); PERF_TIMER_START_BP(PROVE_aLaR); for (size_t j = 0; j < M; ++j) { for (size_t i = N; i-- > 0; ) { - if (j >= sv.size()) - { - aL[j*N+i] = rct::zero(); - } - else if (sv[j][i/8] & (((uint64_t)1)<<(i%8))) + if (j < sv.size() && (sv[j][i/8] & (((uint64_t)1)<<(i%8)))) { aL[j*N+i] = rct::identity(); + aL8[j*N+i] = INV_EIGHT; + aR[j*N+i] = aR8[j*N+i] = rct::zero(); } else { - aL[j*N+i] = rct::zero(); + aL[j*N+i] = aL8[j*N+i] = rct::zero(); + aR[j*N+i] = MINUS_ONE; + aR8[j*N+i] = MINUS_INV_EIGHT; } - sc_sub(aR[j*N+i].bytes, aL[j*N+i].bytes, rct::identity().bytes); } } - PERF_TIMER_STOP(PROVE_aLaR); + PERF_TIMER_STOP_BP(PROVE_aLaR); // DEBUG: Test to ensure this recovers the value #ifdef DEBUG_BP @@ -786,10 +549,10 @@ try_again: PERF_TIMER_START_BP(PROVE_step1); // PAPER LINES 38-39 rct::key alpha = rct::skGen(); - rct::key ve = vector_exponent(aL, aR); + rct::key ve = vector_exponent(aL8, aR8); rct::key A; - rct::addKeys(A, ve, rct::scalarmultBase(alpha)); - A = rct::scalarmultKey(A, INV_EIGHT); + sc_mul(tmp.bytes, alpha.bytes, INV_EIGHT.bytes); + rct::addKeys(A, ve, rct::scalarmultBase(tmp)); // PAPER LINES 40-42 rct::keyV sL = rct::skvGen(MN), sR = rct::skvGen(MN); @@ -803,21 +566,20 @@ try_again: rct::key y = hash_cache_mash(hash_cache, A, S); if (y == rct::zero()) { - PERF_TIMER_STOP(PROVE_step1); + PERF_TIMER_STOP_BP(PROVE_step1); MINFO("y is 0, trying again"); goto try_again; } rct::key z = hash_cache = rct::hash_to_scalar(y); if (z == rct::zero()) { - PERF_TIMER_STOP(PROVE_step1); + PERF_TIMER_STOP_BP(PROVE_step1); MINFO("z is 0, trying again"); goto try_again; } // Polynomial construction by coefficients - const auto zMN = vector_dup(z, MN); - rct::keyV l0 = vector_subtract(aL, zMN); + rct::keyV l0 = vector_subtract(aL, z); const rct::keyV &l1 = sL; // This computes the ugly sum/concatenation from PAPER LINE 65 @@ -837,7 +599,7 @@ try_again: } } - rct::keyV r0 = vector_add(aR, zMN); + rct::keyV r0 = vector_add(aR, z); const auto yMN = vector_powers(y, MN); r0 = hadamard(r0, yMN); r0 = vector_add(r0, zero_twos); @@ -850,22 +612,28 @@ try_again: sc_add(t1.bytes, t1_1.bytes, t1_2.bytes); rct::key t2 = inner_product(l1, r1); - PERF_TIMER_STOP(PROVE_step1); + PERF_TIMER_STOP_BP(PROVE_step1); PERF_TIMER_START_BP(PROVE_step2); // PAPER LINES 47-48 rct::key tau1 = rct::skGen(), tau2 = rct::skGen(); - rct::key T1 = rct::addKeys(rct::scalarmultH(t1), rct::scalarmultBase(tau1)); - T1 = rct::scalarmultKey(T1, INV_EIGHT); - rct::key T2 = rct::addKeys(rct::scalarmultH(t2), rct::scalarmultBase(tau2)); - T2 = rct::scalarmultKey(T2, INV_EIGHT); + rct::key T1, T2; + ge_p3 p3; + sc_mul(tmp.bytes, t1.bytes, INV_EIGHT.bytes); + sc_mul(tmp2.bytes, tau1.bytes, INV_EIGHT.bytes); + ge_double_scalarmult_base_vartime_p3(&p3, tmp.bytes, &ge_p3_H, tmp2.bytes); + ge_p3_tobytes(T1.bytes, &p3); + sc_mul(tmp.bytes, t2.bytes, INV_EIGHT.bytes); + sc_mul(tmp2.bytes, tau2.bytes, INV_EIGHT.bytes); + ge_double_scalarmult_base_vartime_p3(&p3, tmp.bytes, &ge_p3_H, tmp2.bytes); + ge_p3_tobytes(T2.bytes, &p3); // PAPER LINES 49-51 rct::key x = hash_cache_mash(hash_cache, z, T1, T2); if (x == rct::zero()) { - PERF_TIMER_STOP(PROVE_step2); + PERF_TIMER_STOP_BP(PROVE_step2); MINFO("x is 0, trying again"); goto try_again; } @@ -889,7 +657,7 @@ try_again: l = vector_add(l, vector_scalar(l1, x)); rct::keyV r = r0; r = vector_add(r, vector_scalar(r1, x)); - PERF_TIMER_STOP(PROVE_step2); + PERF_TIMER_STOP_BP(PROVE_step2); PERF_TIMER_START_BP(PROVE_step3); rct::key t = inner_product(l, r); @@ -907,24 +675,27 @@ try_again: rct::key x_ip = hash_cache_mash(hash_cache, x, taux, mu, t); if (x_ip == rct::zero()) { - PERF_TIMER_STOP(PROVE_step3); + PERF_TIMER_STOP_BP(PROVE_step3); MINFO("x_ip is 0, trying again"); goto try_again; } // These are used in the inner product rounds size_t nprime = MN; - rct::keyV Gprime(MN); - rct::keyV Hprime(MN); + std::vector<ge_p3> Gprime(MN); + std::vector<ge_p3> Hprime(MN); rct::keyV aprime(MN); rct::keyV bprime(MN); const rct::key yinv = invert(y); - rct::key yinvpow = rct::identity(); + rct::keyV yinvpow(MN); + yinvpow[0] = rct::identity(); + yinvpow[1] = yinv; for (size_t i = 0; i < MN; ++i) { - Gprime[i] = Gi[i]; - Hprime[i] = scalarmultKey(Hi_p3[i], yinvpow); - sc_mul(yinvpow.bytes, yinvpow.bytes, yinv.bytes); + Gprime[i] = Gi_p3[i]; + Hprime[i] = Hi_p3[i]; + if (i > 1) + sc_mul(yinvpow[i].bytes, yinvpow[i-1].bytes, yinv.bytes); aprime[i] = l[i]; bprime[i] = r[i]; } @@ -932,53 +703,62 @@ try_again: rct::keyV R(logMN); int round = 0; rct::keyV w(logMN); // this is the challenge x in the inner product protocol - PERF_TIMER_STOP(PROVE_step3); + PERF_TIMER_STOP_BP(PROVE_step3); PERF_TIMER_START_BP(PROVE_step4); // PAPER LINE 13 + const rct::keyV *scale = &yinvpow; while (nprime > 1) { // PAPER LINE 15 nprime /= 2; // PAPER LINES 16-17 + PERF_TIMER_START_BP(PROVE_inner_product); rct::key cL = inner_product(slice(aprime, 0, nprime), slice(bprime, nprime, bprime.size())); rct::key cR = inner_product(slice(aprime, nprime, aprime.size()), slice(bprime, 0, nprime)); + PERF_TIMER_STOP_BP(PROVE_inner_product); // PAPER LINES 18-19 - L[round] = vector_exponent_custom(slice(Gprime, nprime, Gprime.size()), slice(Hprime, 0, nprime), slice(aprime, 0, nprime), slice(bprime, nprime, bprime.size())); + PERF_TIMER_START_BP(PROVE_LR); sc_mul(tmp.bytes, cL.bytes, x_ip.bytes); - rct::addKeys(L[round], L[round], rct::scalarmultH(tmp)); - L[round] = rct::scalarmultKey(L[round], INV_EIGHT); - R[round] = vector_exponent_custom(slice(Gprime, 0, nprime), slice(Hprime, nprime, Hprime.size()), slice(aprime, nprime, aprime.size()), slice(bprime, 0, nprime)); + L[round] = cross_vector_exponent8(nprime, Gprime, nprime, Hprime, 0, aprime, 0, bprime, nprime, scale, &ge_p3_H, &tmp); sc_mul(tmp.bytes, cR.bytes, x_ip.bytes); - rct::addKeys(R[round], R[round], rct::scalarmultH(tmp)); - R[round] = rct::scalarmultKey(R[round], INV_EIGHT); + R[round] = cross_vector_exponent8(nprime, Gprime, 0, Hprime, nprime, aprime, nprime, bprime, 0, scale, &ge_p3_H, &tmp); + PERF_TIMER_STOP_BP(PROVE_LR); // PAPER LINES 21-22 w[round] = hash_cache_mash(hash_cache, L[round], R[round]); if (w[round] == rct::zero()) { - PERF_TIMER_STOP(PROVE_step4); + PERF_TIMER_STOP_BP(PROVE_step4); MINFO("w[round] is 0, trying again"); goto try_again; } // PAPER LINES 24-25 const rct::key winv = invert(w[round]); - Gprime = hadamard2(vector_scalar2(slice(Gprime, 0, nprime), winv), vector_scalar2(slice(Gprime, nprime, Gprime.size()), w[round])); - Hprime = hadamard2(vector_scalar2(slice(Hprime, 0, nprime), w[round]), vector_scalar2(slice(Hprime, nprime, Hprime.size()), winv)); + if (nprime > 1) + { + PERF_TIMER_START_BP(PROVE_hadamard2); + hadamard_fold(Gprime, NULL, winv, w[round]); + hadamard_fold(Hprime, scale, w[round], winv); + PERF_TIMER_STOP_BP(PROVE_hadamard2); + } // PAPER LINES 28-29 + PERF_TIMER_START_BP(PROVE_prime); aprime = vector_add(vector_scalar(slice(aprime, 0, nprime), w[round]), vector_scalar(slice(aprime, nprime, aprime.size()), winv)); bprime = vector_add(vector_scalar(slice(bprime, 0, nprime), winv), vector_scalar(slice(bprime, nprime, bprime.size()), w[round])); + PERF_TIMER_STOP_BP(PROVE_prime); + scale = NULL; ++round; } - PERF_TIMER_STOP(PROVE_step4); + PERF_TIMER_STOP_BP(PROVE_step4); // PAPER LINE 58 (with inclusions from PAPER LINE 8 and PAPER LINE 20) - return Bulletproof(V, A, S, T1, T2, taux, mu, L, R, aprime[0], bprime[0], t); + return Bulletproof(std::move(V), A, S, T1, T2, taux, mu, std::move(L), std::move(R), aprime[0], bprime[0], t); } Bulletproof bulletproof_PROVE(const std::vector<uint64_t> &v, const rct::keyV &gamma) @@ -1000,10 +780,17 @@ Bulletproof bulletproof_PROVE(const std::vector<uint64_t> &v, const rct::keyV &g sv[i].bytes[6] = (v[i] >> 48) & 255; sv[i].bytes[7] = (v[i] >> 56) & 255; } - PERF_TIMER_STOP(PROVE_v); + PERF_TIMER_STOP_BP(PROVE_v); return bulletproof_PROVE(sv, gamma); } +struct proof_data_t +{ + rct::key x, y, z, x_ip; + std::vector<rct::key> w; + size_t logM, inv_offset; +}; + /* Given a range proof, determine if it is valid */ bool bulletproof_VERIFY(const std::vector<const Bulletproof*> &proofs) { @@ -1011,8 +798,17 @@ bool bulletproof_VERIFY(const std::vector<const Bulletproof*> &proofs) PERF_TIMER_START_BP(VERIFY); + const size_t logN = 6; + const size_t N = 1 << logN; + // sanity and figure out which proof is longest size_t max_length = 0; + size_t nV = 0; + std::vector<proof_data_t> proof_data; + proof_data.reserve(proofs.size()); + size_t inv_offset = 0; + std::vector<rct::key> to_invert; + to_invert.reserve(11 * sizeof(proofs)); for (const Bulletproof *p: proofs) { const Bulletproof &proof = *p; @@ -1029,44 +825,76 @@ bool bulletproof_VERIFY(const std::vector<const Bulletproof*> &proofs) CHECK_AND_ASSERT_MES(proof.L.size() > 0, false, "Empty proof"); max_length = std::max(max_length, proof.L.size()); + nV += proof.V.size(); + + // Reconstruct the challenges + PERF_TIMER_START_BP(VERIFY_start); + proof_data.resize(proof_data.size() + 1); + proof_data_t &pd = proof_data.back(); + rct::key hash_cache = rct::hash_to_scalar(proof.V); + pd.y = hash_cache_mash(hash_cache, proof.A, proof.S); + CHECK_AND_ASSERT_MES(!(pd.y == rct::zero()), false, "y == 0"); + pd.z = hash_cache = rct::hash_to_scalar(pd.y); + CHECK_AND_ASSERT_MES(!(pd.z == rct::zero()), false, "z == 0"); + pd.x = hash_cache_mash(hash_cache, pd.z, proof.T1, proof.T2); + CHECK_AND_ASSERT_MES(!(pd.x == rct::zero()), false, "x == 0"); + pd.x_ip = hash_cache_mash(hash_cache, pd.x, proof.taux, proof.mu, proof.t); + CHECK_AND_ASSERT_MES(!(pd.x_ip == rct::zero()), false, "x_ip == 0"); + PERF_TIMER_STOP_BP(VERIFY_start); + + size_t M; + for (pd.logM = 0; (M = 1<<pd.logM) <= maxM && M < proof.V.size(); ++pd.logM); + CHECK_AND_ASSERT_MES(proof.L.size() == 6+pd.logM, false, "Proof is not the expected size"); + + const size_t rounds = pd.logM+logN; + CHECK_AND_ASSERT_MES(rounds > 0, false, "Zero rounds"); + + PERF_TIMER_START_BP(VERIFY_line_21_22); + // PAPER LINES 21-22 + // The inner product challenges are computed per round + pd.w.resize(rounds); + for (size_t i = 0; i < rounds; ++i) + { + pd.w[i] = hash_cache_mash(hash_cache, proof.L[i], proof.R[i]); + CHECK_AND_ASSERT_MES(!(pd.w[i] == rct::zero()), false, "w[i] == 0"); + } + PERF_TIMER_STOP_BP(VERIFY_line_21_22); + + pd.inv_offset = inv_offset; + for (size_t i = 0; i < rounds; ++i) + to_invert.push_back(pd.w[i]); + to_invert.push_back(pd.y); + inv_offset += rounds + 1; } CHECK_AND_ASSERT_MES(max_length < 32, false, "At least one proof is too large"); size_t maxMN = 1u << max_length; - const size_t logN = 6; - const size_t N = 1 << logN; rct::key tmp; + std::vector<MultiexpData> multiexp_data; + multiexp_data.reserve(nV + (2 * (10/*logM*/ + logN) + 4) * proofs.size() + 2 * maxMN); + multiexp_data.resize(2 * maxMN); + + PERF_TIMER_START_BP(VERIFY_line_24_25_invert); + const std::vector<rct::key> inverses = invert(to_invert); + PERF_TIMER_STOP_BP(VERIFY_line_24_25_invert); + // setup weighted aggregates - rct::key Z0 = rct::identity(); rct::key z1 = rct::zero(); - rct::key Z2 = rct::identity(); rct::key z3 = rct::zero(); - rct::keyV z4(maxMN, rct::zero()), z5(maxMN, rct::zero()); - rct::key Y2 = rct::identity(), Y3 = rct::identity(), Y4 = rct::identity(); - rct::key y0 = rct::zero(), y1 = rct::zero(); + rct::keyV m_z4(maxMN, rct::zero()), m_z5(maxMN, rct::zero()); + rct::key m_y0 = rct::zero(), y1 = rct::zero(); + int proof_data_index = 0; for (const Bulletproof *p: proofs) { const Bulletproof &proof = *p; + const proof_data_t &pd = proof_data[proof_data_index++]; - size_t M, logM; - for (logM = 0; (M = 1<<logM) <= maxM && M < proof.V.size(); ++logM); - CHECK_AND_ASSERT_MES(proof.L.size() == 6+logM, false, "Proof is not the expected size"); + CHECK_AND_ASSERT_MES(proof.L.size() == 6+pd.logM, false, "Proof is not the expected size"); + const size_t M = 1 << pd.logM; const size_t MN = M*N; - rct::key weight = rct::skGen(); - - // Reconstruct the challenges - PERF_TIMER_START_BP(VERIFY_start); - rct::key hash_cache = rct::hash_to_scalar(proof.V); - rct::key y = hash_cache_mash(hash_cache, proof.A, proof.S); - CHECK_AND_ASSERT_MES(!(y == rct::zero()), false, "y == 0"); - rct::key z = hash_cache = rct::hash_to_scalar(y); - CHECK_AND_ASSERT_MES(!(z == rct::zero()), false, "z == 0"); - rct::key x = hash_cache_mash(hash_cache, z, proof.T1, proof.T2); - CHECK_AND_ASSERT_MES(!(x == rct::zero()), false, "x == 0"); - rct::key x_ip = hash_cache_mash(hash_cache, x, proof.taux, proof.mu, proof.t); - CHECK_AND_ASSERT_MES(!(x_ip == rct::zero()), false, "x_ip == 0"); - PERF_TIMER_STOP(VERIFY_start); + const rct::key weight_y = rct::skGen(); + const rct::key weight_z = rct::skGen(); // pre-multiply some points by 8 rct::keyV proof8_V = proof.V; for (rct::key &k: proof8_V) k = rct::scalarmult8(k); @@ -1075,183 +903,168 @@ bool bulletproof_VERIFY(const std::vector<const Bulletproof*> &proofs) rct::key proof8_T1 = rct::scalarmult8(proof.T1); rct::key proof8_T2 = rct::scalarmult8(proof.T2); rct::key proof8_S = rct::scalarmult8(proof.S); + rct::key proof8_A = rct::scalarmult8(proof.A); PERF_TIMER_START_BP(VERIFY_line_61); // PAPER LINE 61 - sc_muladd(y0.bytes, proof.taux.bytes, weight.bytes, y0.bytes); + sc_mulsub(m_y0.bytes, proof.taux.bytes, weight_y.bytes, m_y0.bytes); - const rct::keyV zpow = vector_powers(z, M+3); + const rct::keyV zpow = vector_powers(pd.z, M+3); rct::key k; - const rct::key ip1y = vector_power_sum(y, MN); + const rct::key ip1y = vector_power_sum(pd.y, MN); sc_mulsub(k.bytes, zpow[2].bytes, ip1y.bytes, rct::zero().bytes); for (size_t j = 1; j <= M; ++j) { CHECK_AND_ASSERT_MES(j+2 < zpow.size(), false, "invalid zpow index"); sc_mulsub(k.bytes, zpow[j+2].bytes, ip12.bytes, k.bytes); } - PERF_TIMER_STOP(VERIFY_line_61); + PERF_TIMER_STOP_BP(VERIFY_line_61); PERF_TIMER_START_BP(VERIFY_line_61rl_new); - sc_muladd(tmp.bytes, z.bytes, ip1y.bytes, k.bytes); - std::vector<MultiexpData> multiexp_data; - multiexp_data.reserve(proof.V.size()); + sc_muladd(tmp.bytes, pd.z.bytes, ip1y.bytes, k.bytes); sc_sub(tmp.bytes, proof.t.bytes, tmp.bytes); - sc_muladd(y1.bytes, tmp.bytes, weight.bytes, y1.bytes); + sc_muladd(y1.bytes, tmp.bytes, weight_y.bytes, y1.bytes); for (size_t j = 0; j < proof8_V.size(); j++) { - multiexp_data.emplace_back(zpow[j+2], proof8_V[j]); + sc_mul(tmp.bytes, zpow[j+2].bytes, weight_y.bytes); + multiexp_data.emplace_back(tmp, proof8_V[j]); } - rct::addKeys(Y2, Y2, rct::scalarmultKey(multiexp(multiexp_data, false), weight)); - sc_mul(tmp.bytes, x.bytes, weight.bytes); - rct::addKeys(Y3, Y3, rct::scalarmultKey(proof8_T1, tmp)); + sc_mul(tmp.bytes, pd.x.bytes, weight_y.bytes); + multiexp_data.emplace_back(tmp, proof8_T1); rct::key xsq; - sc_mul(xsq.bytes, x.bytes, x.bytes); - sc_mul(tmp.bytes, xsq.bytes, weight.bytes); - rct::addKeys(Y4, Y4, rct::scalarmultKey(proof8_T2, tmp)); - PERF_TIMER_STOP(VERIFY_line_61rl_new); + sc_mul(xsq.bytes, pd.x.bytes, pd.x.bytes); + sc_mul(tmp.bytes, xsq.bytes, weight_y.bytes); + multiexp_data.emplace_back(tmp, proof8_T2); + PERF_TIMER_STOP_BP(VERIFY_line_61rl_new); PERF_TIMER_START_BP(VERIFY_line_62); // PAPER LINE 62 - rct::addKeys(Z0, Z0, rct::scalarmultKey(rct::addKeys(rct::scalarmult8(proof.A), rct::scalarmultKey(proof8_S, x)), weight)); - PERF_TIMER_STOP(VERIFY_line_62); + multiexp_data.emplace_back(weight_z, proof8_A); + sc_mul(tmp.bytes, pd.x.bytes, weight_z.bytes); + multiexp_data.emplace_back(tmp, proof8_S); + PERF_TIMER_STOP_BP(VERIFY_line_62); // Compute the number of rounds for the inner product - const size_t rounds = logM+logN; + const size_t rounds = pd.logM+logN; CHECK_AND_ASSERT_MES(rounds > 0, false, "Zero rounds"); - PERF_TIMER_START_BP(VERIFY_line_21_22); - // PAPER LINES 21-22 - // The inner product challenges are computed per round - rct::keyV w(rounds); - for (size_t i = 0; i < rounds; ++i) - { - w[i] = hash_cache_mash(hash_cache, proof.L[i], proof.R[i]); - CHECK_AND_ASSERT_MES(!(w[i] == rct::zero()), false, "w[i] == 0"); - } - PERF_TIMER_STOP(VERIFY_line_21_22); - PERF_TIMER_START_BP(VERIFY_line_24_25); // Basically PAPER LINES 24-25 // Compute the curvepoints from G[i] and H[i] rct::key yinvpow = rct::identity(); rct::key ypow = rct::identity(); - PERF_TIMER_START_BP(VERIFY_line_24_25_invert); - const rct::key yinv = invert(y); - rct::keyV winv(rounds); - for (size_t i = 0; i < rounds; ++i) - winv[i] = invert(w[i]); - PERF_TIMER_STOP(VERIFY_line_24_25_invert); + const rct::key *winv = &inverses[pd.inv_offset]; + const rct::key yinv = inverses[pd.inv_offset + rounds]; + + // precalc + PERF_TIMER_START_BP(VERIFY_line_24_25_precalc); + rct::keyV w_cache(1<<rounds); + w_cache[0] = winv[0]; + w_cache[1] = pd.w[0]; + for (size_t j = 1; j < rounds; ++j) + { + const size_t slots = 1<<(j+1); + for (size_t s = slots; s-- > 0; --s) + { + sc_mul(w_cache[s].bytes, w_cache[s/2].bytes, pd.w[j].bytes); + sc_mul(w_cache[s-1].bytes, w_cache[s/2].bytes, winv[j].bytes); + } + } + PERF_TIMER_STOP_BP(VERIFY_line_24_25_precalc); for (size_t i = 0; i < MN; ++i) { - // Convert the index to binary IN REVERSE and construct the scalar exponent rct::key g_scalar = proof.a; rct::key h_scalar; - sc_mul(h_scalar.bytes, proof.b.bytes, yinvpow.bytes); + if (i == 0) + h_scalar = proof.b; + else + sc_mul(h_scalar.bytes, proof.b.bytes, yinvpow.bytes); - for (size_t j = rounds; j-- > 0; ) - { - size_t J = w.size() - j - 1; - - if ((i & (((size_t)1)<<j)) == 0) - { - sc_mul(g_scalar.bytes, g_scalar.bytes, winv[J].bytes); - sc_mul(h_scalar.bytes, h_scalar.bytes, w[J].bytes); - } - else - { - sc_mul(g_scalar.bytes, g_scalar.bytes, w[J].bytes); - sc_mul(h_scalar.bytes, h_scalar.bytes, winv[J].bytes); - } - } + // Convert the index to binary IN REVERSE and construct the scalar exponent + sc_mul(g_scalar.bytes, g_scalar.bytes, w_cache[i].bytes); + sc_mul(h_scalar.bytes, h_scalar.bytes, w_cache[(~i) & (MN-1)].bytes); // Adjust the scalars using the exponents from PAPER LINE 62 - sc_add(g_scalar.bytes, g_scalar.bytes, z.bytes); + sc_add(g_scalar.bytes, g_scalar.bytes, pd.z.bytes); CHECK_AND_ASSERT_MES(2+i/N < zpow.size(), false, "invalid zpow index"); CHECK_AND_ASSERT_MES(i%N < twoN.size(), false, "invalid twoN index"); sc_mul(tmp.bytes, zpow[2+i/N].bytes, twoN[i%N].bytes); - sc_muladd(tmp.bytes, z.bytes, ypow.bytes, tmp.bytes); - sc_mulsub(h_scalar.bytes, tmp.bytes, yinvpow.bytes, h_scalar.bytes); + if (i == 0) + { + sc_add(tmp.bytes, tmp.bytes, pd.z.bytes); + sc_sub(h_scalar.bytes, h_scalar.bytes, tmp.bytes); + } + else + { + sc_muladd(tmp.bytes, pd.z.bytes, ypow.bytes, tmp.bytes); + sc_mulsub(h_scalar.bytes, tmp.bytes, yinvpow.bytes, h_scalar.bytes); + } - sc_muladd(z4[i].bytes, g_scalar.bytes, weight.bytes, z4[i].bytes); - sc_muladd(z5[i].bytes, h_scalar.bytes, weight.bytes, z5[i].bytes); + sc_mulsub(m_z4[i].bytes, g_scalar.bytes, weight_z.bytes, m_z4[i].bytes); + sc_mulsub(m_z5[i].bytes, h_scalar.bytes, weight_z.bytes, m_z5[i].bytes); - if (i != MN-1) + if (i == 0) + { + yinvpow = yinv; + ypow = pd.y; + } + else if (i != MN-1) { sc_mul(yinvpow.bytes, yinvpow.bytes, yinv.bytes); - sc_mul(ypow.bytes, ypow.bytes, y.bytes); + sc_mul(ypow.bytes, ypow.bytes, pd.y.bytes); } } - PERF_TIMER_STOP(VERIFY_line_24_25); + PERF_TIMER_STOP_BP(VERIFY_line_24_25); // PAPER LINE 26 PERF_TIMER_START_BP(VERIFY_line_26_new); - multiexp_data.clear(); - multiexp_data.reserve(2*rounds); - - sc_muladd(z1.bytes, proof.mu.bytes, weight.bytes, z1.bytes); + sc_muladd(z1.bytes, proof.mu.bytes, weight_z.bytes, z1.bytes); for (size_t i = 0; i < rounds; ++i) { - sc_mul(tmp.bytes, w[i].bytes, w[i].bytes); + sc_mul(tmp.bytes, pd.w[i].bytes, pd.w[i].bytes); + sc_mul(tmp.bytes, tmp.bytes, weight_z.bytes); multiexp_data.emplace_back(tmp, proof8_L[i]); sc_mul(tmp.bytes, winv[i].bytes, winv[i].bytes); + sc_mul(tmp.bytes, tmp.bytes, weight_z.bytes); multiexp_data.emplace_back(tmp, proof8_R[i]); } - rct::key acc = multiexp(multiexp_data, false); - rct::addKeys(Z2, Z2, rct::scalarmultKey(acc, weight)); sc_mulsub(tmp.bytes, proof.a.bytes, proof.b.bytes, proof.t.bytes); - sc_mul(tmp.bytes, tmp.bytes, x_ip.bytes); - sc_muladd(z3.bytes, tmp.bytes, weight.bytes, z3.bytes); - PERF_TIMER_STOP(VERIFY_line_26_new); + sc_mul(tmp.bytes, tmp.bytes, pd.x_ip.bytes); + sc_muladd(z3.bytes, tmp.bytes, weight_z.bytes, z3.bytes); + PERF_TIMER_STOP_BP(VERIFY_line_26_new); } // now check all proofs at once PERF_TIMER_START_BP(VERIFY_step2_check); - ge_p3 check1; - ge_scalarmult_base(&check1, y0.bytes); - addKeys_acc_p3(&check1, y1, rct::H); - sub_acc_p3(&check1, Y2); - sub_acc_p3(&check1, Y3); - sub_acc_p3(&check1, Y4); - if (!ge_p3_is_point_at_infinity(&check1)) - { - MERROR("Verification failure at step 1"); - return false; - } - ge_p3 check2; - sc_sub(tmp.bytes, rct::zero().bytes, z1.bytes); - ge_double_scalarmult_base_vartime_p3(&check2, z3.bytes, &ge_p3_H, tmp.bytes); - add_acc_p3(&check2, Z0); - add_acc_p3(&check2, Z2); - - std::vector<MultiexpData> multiexp_data; - multiexp_data.reserve(2 * maxMN); + sc_sub(tmp.bytes, m_y0.bytes, z1.bytes); + multiexp_data.emplace_back(tmp, rct::G); + sc_sub(tmp.bytes, z3.bytes, y1.bytes); + multiexp_data.emplace_back(tmp, rct::H); for (size_t i = 0; i < maxMN; ++i) { - sc_sub(tmp.bytes, rct::zero().bytes, z4[i].bytes); - multiexp_data.emplace_back(tmp, Gi_p3[i]); - sc_sub(tmp.bytes, rct::zero().bytes, z5[i].bytes); - multiexp_data.emplace_back(tmp, Hi_p3[i]); + multiexp_data[i * 2] = {m_z4[i], Gi_p3[i]}; + multiexp_data[i * 2 + 1] = {m_z5[i], Hi_p3[i]}; } - add_acc_p3(&check2, multiexp(multiexp_data, true)); - PERF_TIMER_STOP(VERIFY_step2_check); - - if (!ge_p3_is_point_at_infinity(&check2)) + if (!(multiexp(multiexp_data, 2 * maxMN) == rct::identity())) { - MERROR("Verification failure at step 2"); + PERF_TIMER_STOP_BP(VERIFY_step2_check); + MERROR("Verification failure"); return false; } + PERF_TIMER_STOP_BP(VERIFY_step2_check); - PERF_TIMER_STOP(VERIFY); + PERF_TIMER_STOP_BP(VERIFY); return true; } bool bulletproof_VERIFY(const std::vector<Bulletproof> &proofs) { std::vector<const Bulletproof*> proof_pointers; + proof_pointers.reserve(proofs.size()); for (const Bulletproof &proof: proofs) proof_pointers.push_back(&proof); return bulletproof_VERIFY(proof_pointers); diff --git a/src/ringct/multiexp.cc b/src/ringct/multiexp.cc index 21957b94c..6f77fed34 100644 --- a/src/ringct/multiexp.cc +++ b/src/ringct/multiexp.cc @@ -79,6 +79,25 @@ extern "C" // Best/cached Straus Straus Straus Straus Straus Straus Straus Straus Pip Pip Pip Pip // Best/uncached Straus Straus Straus Straus Straus Straus Pip Pip Pip Pip Pip Pip +// New timings: +// Pippenger: +// 2/1 always +// 3/2 at ~13 +// 4/3 at ~29 +// 5/4 at ~83 +// 6/5 < 200 +// 7/6 at ~470 +// 8/7 at ~1180 +// 9/8 at ~2290 +// Cached Pippenger: +// 6/5 < 200 +// 7/6 at 460 +// 8/7 at 1180 +// 9/8 at 2300 +// +// Cached Straus/Pippenger cross at 232 +// + namespace rct { @@ -320,7 +339,7 @@ rct::key bos_coster_heap_conv_robust(std::vector<MultiexpData> data) return res; } -static constexpr unsigned int STRAUS_C = 4; +#define STRAUS_C 4 struct straus_cached_data { @@ -447,28 +466,26 @@ rct::key straus(const std::vector<MultiexpData> &data, const std::shared_ptr<str #endif MULTIEXP_PERF(PERF_TIMER_START_UNIT(digits, 1000000)); +#if STRAUS_C==4 + std::unique_ptr<uint8_t[]> digits{new uint8_t[64 * data.size()]}; +#else std::unique_ptr<uint8_t[]> digits{new uint8_t[256 * data.size()]}; +#endif for (size_t j = 0; j < data.size(); ++j) { - unsigned char bytes33[33]; - memcpy(bytes33, data[j].scalar.bytes, 32); - bytes33[32] = 0; - const unsigned char *bytes = bytes33; -#if 1 - static_assert(STRAUS_C == 4, "optimized version needs STRAUS_C == 4"); + const unsigned char *bytes = data[j].scalar.bytes; +#if STRAUS_C==4 unsigned int i; - for (i = 0; i < 256; i += 8, bytes++) + for (i = 0; i < 64; i += 2, bytes++) { - digits[j*256+i] = bytes[0] & 0xf; - digits[j*256+i+1] = (bytes[0] >> 1) & 0xf; - digits[j*256+i+2] = (bytes[0] >> 2) & 0xf; - digits[j*256+i+3] = (bytes[0] >> 3) & 0xf; - digits[j*256+i+4] = ((bytes[0] >> 4) | (bytes[1]<<4)) & 0xf; - digits[j*256+i+5] = ((bytes[0] >> 5) | (bytes[1]<<3)) & 0xf; - digits[j*256+i+6] = ((bytes[0] >> 6) | (bytes[1]<<2)) & 0xf; - digits[j*256+i+7] = ((bytes[0] >> 7) | (bytes[1]<<1)) & 0xf; + digits[j*64+i] = bytes[0] & 0xf; + digits[j*64+i+1] = bytes[0] >> 4; } #elif 1 + unsigned char bytes33[33]; + memcpy(bytes33, data[j].scalar.bytes, 32); + bytes33[32] = 0; + bytes = bytes33; for (size_t i = 0; i < 256; ++i) digits[j*256+i] = ((bytes[i>>3] | (bytes[(i>>3)+1]<<8)) >> (i&7)) & mask; #else @@ -521,7 +538,11 @@ skipfirst: if (skip[j]) continue; #endif +#if STRAUS_C==4 + const uint8_t digit = digits[j*64+i/4]; +#else const uint8_t digit = digits[j*256+i]; +#endif if (digit) { ge_add(&p1, &band_p3, &CACHE_OFFSET(local_cache, j, digit)); @@ -542,16 +563,13 @@ skipfirst: size_t get_pippenger_c(size_t N) { -// uncached: 2:1, 4:2, 8:2, 16:3, 32:4, 64:4, 128:5, 256:6, 512:7, 1024:7, 2048:8, 4096:9 -// cached: 2:1, 4:2, 8:2, 16:3, 32:4, 64:4, 128:5, 256:6, 512:7, 1024:7, 2048:8, 4096:9 - if (N <= 2) return 1; - if (N <= 8) return 2; - if (N <= 16) return 3; - if (N <= 64) return 4; - if (N <= 128) return 5; - if (N <= 256) return 6; - if (N <= 1024) return 7; - if (N <= 2048) return 8; + if (N <= 13) return 2; + if (N <= 29) return 3; + if (N <= 83) return 4; + if (N <= 185) return 5; + if (N <= 465) return 6; + if (N <= 1180) return 7; + if (N <= 2295) return 8; return 9; } @@ -563,12 +581,13 @@ struct pippenger_cached_data ~pippenger_cached_data() { aligned_free(cached); } }; -std::shared_ptr<pippenger_cached_data> pippenger_init_cache(const std::vector<MultiexpData> &data, size_t N) +std::shared_ptr<pippenger_cached_data> pippenger_init_cache(const std::vector<MultiexpData> &data, size_t start_offset, size_t N) { MULTIEXP_PERF(PERF_TIMER_START_UNIT(pippenger_init_cache, 1000000)); + CHECK_AND_ASSERT_THROW_MES(start_offset <= data.size(), "Bad cache base data"); if (N == 0) - N = data.size(); - CHECK_AND_ASSERT_THROW_MES(N <= data.size(), "Bad cache base data"); + N = data.size() - start_offset; + CHECK_AND_ASSERT_THROW_MES(N <= data.size() - start_offset, "Bad cache base data"); ge_cached cached; std::shared_ptr<pippenger_cached_data> cache(new pippenger_cached_data()); @@ -576,7 +595,7 @@ std::shared_ptr<pippenger_cached_data> pippenger_init_cache(const std::vector<Mu cache->cached = (ge_cached*)aligned_realloc(cache->cached, N * sizeof(ge_cached), 4096); CHECK_AND_ASSERT_THROW_MES(cache->cached, "Out of memory"); for (size_t i = 0; i < N; ++i) - ge_p3_to_cached(&cache->cached[i], &data[i].point); + ge_p3_to_cached(&cache->cached[i], &data[i+start_offset].point); MULTIEXP_PERF(PERF_TIMER_STOP(pippenger_init_cache)); return cache; @@ -587,16 +606,21 @@ size_t pippenger_get_cache_size(const std::shared_ptr<pippenger_cached_data> &ca return cache->size * sizeof(*cache->cached); } -rct::key pippenger(const std::vector<MultiexpData> &data, const std::shared_ptr<pippenger_cached_data> &cache, size_t c) +rct::key pippenger(const std::vector<MultiexpData> &data, const std::shared_ptr<pippenger_cached_data> &cache, size_t cache_size, size_t c) { - CHECK_AND_ASSERT_THROW_MES(cache == NULL || cache->size >= data.size(), "Cache is too small"); + if (cache != NULL && cache_size == 0) + cache_size = cache->size; + CHECK_AND_ASSERT_THROW_MES(cache == NULL || cache_size <= cache->size, "Cache is too small"); if (c == 0) c = get_pippenger_c(data.size()); CHECK_AND_ASSERT_THROW_MES(c <= 9, "c is too large"); ge_p3 result = ge_p3_identity; + bool result_init = false; std::unique_ptr<ge_p3[]> buckets{new ge_p3[1<<c]}; + bool buckets_init[1<<9]; std::shared_ptr<pippenger_cached_data> local_cache = cache == NULL ? pippenger_init_cache(data) : cache; + std::shared_ptr<pippenger_cached_data> local_cache_2 = data.size() > cache_size ? pippenger_init_cache(data, cache_size) : NULL; rct::key maxscalar = rct::zero(); for (size_t i = 0; i < data.size(); ++i) @@ -611,7 +635,7 @@ rct::key pippenger(const std::vector<MultiexpData> &data, const std::shared_ptr< for (size_t k = groups; k-- > 0; ) { - if (!ge_p3_is_point_at_infinity(&result)) + if (result_init) { ge_p2 p2; ge_p3_to_p2(&p2, &result); @@ -625,8 +649,7 @@ rct::key pippenger(const std::vector<MultiexpData> &data, const std::shared_ptr< ge_p1p1_to_p2(&p2, &p1); } } - for (size_t i = 0; i < (1u<<c); ++i) - buckets[i] = ge_p3_identity; + memset(buckets_init, 0, 1u<<c); // partition scalars into buckets for (size_t i = 0; i < data.size(); ++i) @@ -638,22 +661,45 @@ rct::key pippenger(const std::vector<MultiexpData> &data, const std::shared_ptr< if (bucket == 0) continue; CHECK_AND_ASSERT_THROW_MES(bucket < (1u<<c), "bucket overflow"); - if (!ge_p3_is_point_at_infinity(&buckets[bucket])) + if (buckets_init[bucket]) { - add(buckets[bucket], local_cache->cached[i]); + if (i < cache_size) + add(buckets[bucket], local_cache->cached[i]); + else + add(buckets[bucket], local_cache_2->cached[i - cache_size]); } else + { buckets[bucket] = data[i].point; + buckets_init[bucket] = true; + } } // sum the buckets - ge_p3 pail = ge_p3_identity; + ge_p3 pail; + bool pail_init = false; for (size_t i = (1<<c)-1; i > 0; --i) { - if (!ge_p3_is_point_at_infinity(&buckets[i])) - add(pail, buckets[i]); - if (!ge_p3_is_point_at_infinity(&pail)) - add(result, pail); + if (buckets_init[i]) + { + if (pail_init) + add(pail, buckets[i]); + else + { + pail = buckets[i]; + pail_init = true; + } + } + if (pail_init) + { + if (result_init) + add(result, pail); + else + { + result = pail; + result_init = true; + } + } } } diff --git a/src/ringct/multiexp.h b/src/ringct/multiexp.h index 559ab664a..b52707933 100644 --- a/src/ringct/multiexp.h +++ b/src/ringct/multiexp.h @@ -61,10 +61,10 @@ rct::key bos_coster_heap_conv_robust(std::vector<MultiexpData> data); std::shared_ptr<straus_cached_data> straus_init_cache(const std::vector<MultiexpData> &data, size_t N =0); size_t straus_get_cache_size(const std::shared_ptr<straus_cached_data> &cache); rct::key straus(const std::vector<MultiexpData> &data, const std::shared_ptr<straus_cached_data> &cache = NULL, size_t STEP = 0); -std::shared_ptr<pippenger_cached_data> pippenger_init_cache(const std::vector<MultiexpData> &data, size_t N =0); +std::shared_ptr<pippenger_cached_data> pippenger_init_cache(const std::vector<MultiexpData> &data, size_t start_offset = 0, size_t N =0); size_t pippenger_get_cache_size(const std::shared_ptr<pippenger_cached_data> &cache); size_t get_pippenger_c(size_t N); -rct::key pippenger(const std::vector<MultiexpData> &data, const std::shared_ptr<pippenger_cached_data> &cache = NULL, size_t c = 0); +rct::key pippenger(const std::vector<MultiexpData> &data, const std::shared_ptr<pippenger_cached_data> &cache = NULL, size_t cache_size = 0, size_t c = 0); } diff --git a/src/ringct/rctOps.cpp b/src/ringct/rctOps.cpp index 41bbf6ca3..e39ba16fd 100644 --- a/src/ringct/rctOps.cpp +++ b/src/ringct/rctOps.cpp @@ -30,6 +30,7 @@ #include <boost/lexical_cast.hpp> #include "misc_log_ex.h" +#include "cryptonote_basic/cryptonote_format_utils.h" #include "rctOps.h" using namespace crypto; using namespace std; @@ -39,6 +40,183 @@ using namespace std; #define CHECK_AND_ASSERT_THROW_MES_L1(expr, message) {if(!(expr)) {MWARNING(message); throw std::runtime_error(message);}} +struct zero_commitment { uint64_t amount; rct::key commitment; }; +static const zero_commitment zero_commitments[] = { + { (uint64_t)0ull, {{0x58, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66}} }, + { (uint64_t)1ull, {{0x17, 0x38, 0xeb, 0x7a, 0x67, 0x7c, 0x61, 0x49, 0x22, 0x8a, 0x2b, 0xea, 0xa2, 0x1b, 0xea, 0x9e, 0x33, 0x70, 0x80, 0x2d, 0x72, 0xa3, 0xee, 0xc7, 0x90, 0x11, 0x95, 0x80, 0xe0, 0x2b, 0xd5, 0x22}} }, + { (uint64_t)2ull, {{0x76, 0x24, 0x84, 0x63, 0xa, 0x6, 0x17, 0x17, 0x8d, 0xe, 0x33, 0xf3, 0x2e, 0xe, 0x11, 0x3e, 0xa8, 0x46, 0x86, 0x9d, 0x46, 0x4b, 0xb, 0x6f, 0xf1, 0x3b, 0x29, 0x97, 0x4, 0x9c, 0xda, 0x7d}} }, + { (uint64_t)3ull, {{0xcf, 0xf7, 0x7b, 0x56, 0x62, 0x1c, 0x4f, 0xef, 0x74, 0xcf, 0x37, 0xc1, 0x78, 0xd4, 0xb5, 0x8a, 0xf4, 0xad, 0x8c, 0xd4, 0x35, 0xfc, 0xb9, 0x62, 0x76, 0xbc, 0x15, 0x9c, 0x7c, 0x6a, 0x28, 0x8c}} }, + { (uint64_t)4ull, {{0x9a, 0xb8, 0x6c, 0x31, 0xf4, 0x22, 0xd8, 0x21, 0xb5, 0x22, 0x57, 0x30, 0xd1, 0xbf, 0x73, 0xa, 0x9b, 0x91, 0xd2, 0xee, 0xe3, 0x14, 0xb8, 0x4e, 0xbd, 0x4b, 0x93, 0xa6, 0x81, 0x61, 0x82, 0x66}} }, + { (uint64_t)5ull, {{0x32, 0xee, 0x2f, 0x65, 0x9a, 0xf6, 0x38, 0x58, 0xc2, 0xf7, 0xdc, 0x11, 0x1b, 0x3b, 0xb8, 0xfe, 0xc0, 0x2c, 0xac, 0x42, 0x38, 0x3b, 0xb7, 0x36, 0xde, 0x1, 0x8, 0x6f, 0x38, 0xf0, 0x12, 0x3c}} }, + { (uint64_t)6ull, {{0x47, 0x26, 0x2b, 0x1e, 0xa6, 0x43, 0x1, 0x6e, 0x38, 0x24, 0x17, 0x53, 0xa4, 0xfb, 0x39, 0x92, 0x9e, 0x31, 0xea, 0x9b, 0xd3, 0x41, 0x1a, 0xb1, 0x7f, 0x16, 0x6e, 0x61, 0xf6, 0xc, 0xe5, 0xa7}} }, + { (uint64_t)7ull, {{0xc6, 0x32, 0x93, 0x68, 0x79, 0x9a, 0xd, 0xed, 0x4c, 0x20, 0x25, 0x6b, 0xff, 0xe6, 0x45, 0x47, 0xf1, 0x7b, 0xc4, 0x23, 0x95, 0x4, 0xbe, 0x82, 0x4d, 0xff, 0x8a, 0x2b, 0xe1, 0xaf, 0xe3, 0xcd}} }, + { (uint64_t)8ull, {{0xd5, 0xf1, 0x50, 0x74, 0x33, 0x46, 0x19, 0xf, 0x84, 0x2b, 0x6, 0xb8, 0xfa, 0xe1, 0x20, 0xeb, 0x85, 0x24, 0x7e, 0x9f, 0x6d, 0xec, 0x88, 0xff, 0xa2, 0x23, 0xbf, 0x69, 0x94, 0xe9, 0xc8, 0xc2}} }, + { (uint64_t)9ull, {{0x56, 00, 0x23, 0x32, 0x9e, 0xc0, 0xfa, 0xf3, 0x3b, 0x5e, 0x3a, 0x5c, 0xb4, 0xea, 0xef, 0xee, 0x38, 0xf8, 0x96, 0x1c, 0x88, 0xb6, 0x6a, 0x2f, 0x19, 0xd4, 0x59, 0x51, 0x96, 0x9c, 0x6d, 0x1f}} }, + { (uint64_t)10ull, {{0x3, 0x80, 0xdc, 0x24, 0xcc, 0x97, 0xcc, 0xe6, 0x58, 0xc3, 0xa9, 0x47, 0xc5, 0x10, 0x25, 0xde, 0x1a, 0x69, 0x80, 0x3b, 0xdb, 0x50, 0x5, 0xe3, 0xb7, 0xdd, 0xa9, 0xd, 0x68, 0x59, 0xb0, 0x1c}} }, + { (uint64_t)20ull, {{0x9, 0x3, 0xf6, 0x2e, 0x97, 0x76, 0x47, 0x58, 0xfe, 0xf8, 0x9e, 0x5b, 0xec, 0x29, 0xef, 0x4f, 0xc5, 0xe6, 0x45, 0x4b, 0x2d, 0x47, 0x44, 0x47, 0x36, 0x4, 0x4c, 0x25, 0x2e, 0xe2, 0x8e, 0xba}} }, + { (uint64_t)30ull, {{0xa2, 0x8b, 0x89, 0xe0, 0xb, 0xed, 0x62, 0x31, 0x68, 0x5b, 0xf9, 0x74, 0x36, 0xf2, 0xba, 0x51, 0xa2, 0x51, 0x55, 0x7f, 0x8d, 0x17, 0xa, 0x78, 0xe3, 0x12, 0xd6, 0x24, 0xbf, 0x60, 0xff, 0xfe}} }, + { (uint64_t)40ull, {{0xb5, 0xc6, 0x95, 0x55, 0x6a, 0x28, 0x47, 0xb2, 0xe, 0x1c, 0xbb, 0x26, 0xe6, 0xa9, 0xc6, 0x8a, 0x61, 0xc5, 0x50, 0xce, 0xb7, 0xc3, 0x4, 0xfe, 0x92, 0x28, 0x3d, 0x29, 0xa9, 0xb2, 0x43, 0xcb}} }, + { (uint64_t)50ull, {{0x12, 0x8e, 0xc6, 0xcd, 0xc0, 0x6b, 0x43, 0xc5, 0xd0, 0x9c, 0x3f, 0x65, 0x2a, 0xe3, 0x44, 0x7f, 0x9b, 0x3f, 0x2c, 0x30, 0x91, 0x2d, 0xf0, 0x80, 0x37, 00, 0x85, 0xbc, 0xc, 0x9, 0xef, 0x78}} }, + { (uint64_t)60ull, {{0x1f, 0x9f, 0x40, 0x3a, 0xae, 0xa7, 0x16, 0xfb, 0xe2, 0x98, 0xa8, 0x14, 0xf1, 0xee, 0xbc, 0x1b, 0x73, 0x16, 0x8c, 0x37, 0xfa, 0xe3, 0x16, 0xeb, 0x65, 0x5, 0x81, 0x6f, 0xc2, 0x20, 0xeb, 0xfb}} }, + { (uint64_t)70ull, {{0x10, 0xa2, 0x38, 0xc5, 0xe4, 0x8e, 0x4b, 0x93, 0x99, 0xdb, 0xa6, 0xcb, 0xd9, 0x8e, 0x63, 0x54, 0x41, 0x59, 0xe9, 0x8c, 0x93, 0x5a, 0xc0, 0x60, 0x3d, 0x72, 0xde, 0xf, 0xff, 0x31, 0x53, 0xbb}} }, + { (uint64_t)80ull, {{0x75, 0xab, 0x78, 0xc7, 0x28, 0x1f, 0x69, 0x28, 0xf0, 0x94, 0x86, 0x5, 0x7a, 0x63, 0x64, 0x18, 0x27, 0xc5, 0x74, 0x84, 0xe3, 0xe9, 0x9a, 0x39, 0xf3, 0x12, 0xa4, 0x3a, 0x51, 0x9b, 0xda, 0x8}} }, + { (uint64_t)90ull, {{0xe9, 0x56, 0x7b, 0xa7, 0x88, 0xb8, 0x5b, 0x82, 0xc8, 0x65, 0x7a, 0x15, 0xa5, 0x48, 0x99, 0x5c, 0xf6, 0xb0, 0xbd, 0xd1, 0xc6, 0x2a, 0xda, 0x77, 0x55, 0xf2, 0x32, 0x3a, 0xd8, 0xa4, 0x8, 0x51}} }, + { (uint64_t)100ull, {{0xb6, 0x17, 0x36, 0xd5, 0xf2, 0x8d, 0xef, 0x28, 0x61, 0x6a, 0xfc, 0x47, 0x93, 0xe9, 0x9b, 0x27, 0xcd, 0x3e, 0x89, 0xfb, 0x91, 0xc1, 0x13, 0xd4, 0x30, 0x73, 0x65, 0xfb, 0x75, 0xde, 0xdf, 0x88}} }, + { (uint64_t)200ull, {{0x71, 0x3, 0xeb, 0x72, 0x19, 0x28, 0xd7, 0x91, 0x99, 0x87, 0xf3, 0x50, 0xca, 0xa5, 0x7a, 0xe7, 0xb0, 0x81, 0x57, 0x15, 0x3b, 0x4c, 0x43, 0xd, 0x3e, 0xde, 0xc0, 0xc2, 0x3, 0x7, 0x97, 0x44}} }, + { (uint64_t)300ull, {{0x24, 0x40, 0x9e, 0x92, 0x2e, 0xce, 0xd1, 0xa0, 0x5e, 0x4e, 0xac, 0xa3, 0xdf, 0x91, 0x19, 0xc3, 0x8a, 0x92, 0x2e, 0xb, 0x66, 0xd0, 0x2d, 0x9d, 0xd2, 0xfb, 0x1d, 0xcc, 0x20, 0xb9, 0xaf, 0xc7}} }, + { (uint64_t)400ull, {{0xa7, 0x72, 0x9f, 0xa9, 0x32, 0x81, 0x82, 0x99, 0x34, 0x11, 0x5d, 0x47, 0x5a, 0x67, 0x86, 0xa, 0x14, 0x12, 0xc5, 0xe5, 0x95, 0x12, 0x20, 0xd9, 0x60, 0xc2, 0x41, 0xa0, 0x19, 0x1a, 0x9e, 0x65}} }, + { (uint64_t)500ull, {{0x2e, 0x53, 0xc, 0x6, 0x1c, 0x6d, 0x9e, 0x97, 0xab, 0xaf, 0x46, 0x8c, 0x32, 0xb0, 0xad, 0xa7, 0x49, 0x22, 0x57, 0x72, 0xfc, 0xd1, 0x17, 0x41, 0xcb, 0x5c, 0x3, 0x5c, 0xdd, 0x26, 0x14, 0xe}} }, + { (uint64_t)600ull, {{0xa5, 0xb, 0x91, 0x9, 0x9d, 0xf1, 0xb1, 0x69, 0x4f, 0x30, 0xb5, 0x8f, 0xe6, 0x77, 0x68, 0x50, 0xdb, 0xdb, 0xf4, 0x6c, 0xed, 0x99, 0x7f, 0x52, 0x62, 0xa8, 0x51, 0x59, 0x40, 0x74, 0xa5, 0x9d}} }, + { (uint64_t)700ull, {{0x51, 0x2c, 0xf, 0xae, 0xcc, 0xbe, 0xf2, 0xfe, 0xe5, 0x75, 0x4c, 0x6a, 0x45, 0xfd, 0xc0, 0x75, 0x2d, 0x4f, 0x15, 0x22, 0xe7, 0x7f, 0xf0, 0xc4, 0x8d, 0xcb, 0x19, 0x91, 0x8a, 0x68, 0x84, 0xe0}} }, + { (uint64_t)800ull, {{0xda, 0xa9, 0xf9, 0xa5, 0xb9, 0x71, 0x33, 0x33, 0xe9, 0x8c, 0x5, 0xac, 0xe7, 0x27, 0xcc, 0xe, 0x7d, 0xc3, 0xf1, 0x59, 0x49, 0xe1, 0xef, 0x4d, 0x94, 0xfa, 0x47, 0xd6, 0x8a, 0x34, 0xc6, 0x75}} }, + { (uint64_t)900ull, {{0xc7, 0x2b, 0x18, 0xc9, 0x17, 0xcd, 0x43, 0xee, 0x78, 0x40, 0x5e, 0x39, 0x83, 0x98, 0xb8, 0x3a, 0xc0, 0x97, 0x7b, 0x25, 0x19, 0x90, 0xd8, 0x13, 0xc, 0x38, 0xba, 0x53, 0xb6, 0x3d, 0xb4, 0xf7}} }, + { (uint64_t)1000ull, {{0x1, 0xdf, 0x60, 0x91, 0xeb, 0x6a, 0x48, 0xe9, 0xe4, 0x22, 0x25, 0xb, 0xe3, 0x83, 0x88, 0xc8, 0x61, 0xb6, 0x55, 0x55, 0xa7, 0x20, 0xad, 0x15, 0x35, 0x86, 0xfe, 0x2b, 0xd2, 0x2f, 0xa2, 0x3d}} }, + { (uint64_t)2000ull, {{0x24, 0xf5, 0xb1, 0x34, 0x78, 0x46, 0xaf, 0x22, 0xb5, 0x6f, 0x41, 0x25, 0xb3, 0xe7, 0x67, 0x8c, 0xf8, 0x4b, 0x4f, 0xd2, 0xf9, 0x2e, 0x1c, 0x40, 0xaa, 0x3a, 0x1b, 0xe0, 0xc7, 0x4d, 0x95, 0xe6}} }, + { (uint64_t)3000ull, {{0xa7, 0x1c, 0x9a, 0x8f, 0x40, 0xc1, 0x25, 0x9c, 0x36, 0x26, 0x27, 0x73, 0xe0, 0x8, 0x20, 0x18, 0x3e, 0x6b, 0x59, 0xe0, 0x71, 0xc9, 0x9b, 0x34, 0x9b, 0xef, 0x8f, 0x7e, 0xd2, 0xc6, 0xad, 0xb9}} }, + { (uint64_t)4000ull, {{0x98, 0xdc, 0x74, 0xaf, 0x19, 0x89, 0xd3, 0x4b, 0x64, 0x2e, 0xb3, 0x6, 0x2d, 0xbc, 0x9d, 0xca, 0xd8, 0x1, 0xc5, 0x65, 0x27, 0x6, 0x93, 0x99, 0xe7, 0xc4, 0x11, 0xad, 0x14, 0x28, 0x82, 0xf6}} }, + { (uint64_t)5000ull, {{0x61, 0x76, 0xac, 0x4a, 0xc0, 0x6, 0x5e, 0x49, 0xd6, 0xc4, 0x41, 0xcf, 0x40, 0x4f, 0xad, 0xda, 0xad, 0x44, 0x93, 0xe, 0xf0, 0x3c, 0x68, 0x9, 0xad, 0xd7, 0x77, 0xe4, 0x2f, 0xee, 0x7f, 0x10}} }, + { (uint64_t)6000ull, {{0x78, 0x79, 0x4, 0x65, 0xf6, 0x60, 0x5b, 0x5a, 0x84, 0x77, 0x36, 0x5a, 0xa6, 0xc2, 0xa4, 0xa5, 0x84, 0x91, 0xc, 0x23, 0x95, 0x2, 0x92, 0x97, 0x52, 0x49, 0xa1, 0xad, 0x7d, 0xf0, 0xf7, 0xe8}} }, + { (uint64_t)7000ull, {{0x20, 0xa5, 0x60, 0x6b, 0x60, 0x23, 0x95, 0xd6, 0x8e, 0x2f, 0xad, 0x8e, 0xc6, 0x7f, 0x92, 0xde, 0x89, 0xc6, 0x3e, 0x1e, 0x7f, 0xc1, 0xdd, 0x7f, 0x92, 0xff, 0xed, 0xb8, 0xf6, 0x55, 0xfb, 0xd}} }, + { (uint64_t)8000ull, {{0x9a, 0x78, 0x97, 0x43, 0x98, 0x65, 0x17, 0xd9, 0x5f, 0x4e, 0x80, 0x8b, 0xeb, 0xe6, 0x52, 0xd, 0xe6, 0xcf, 0x8c, 0x51, 0x35, 0xab, 0x36, 0x8, 0x7e, 0x87, 0xe2, 0x76, 0xac, 0x6a, 0x34, 0x1}} }, + { (uint64_t)9000ull, {{0x5f, 0xc7, 0xaa, 0x48, 0xbb, 0x19, 0x13, 0x58, 0xc7, 0xe3, 0x4d, 0x24, 0xcf, 0x9c, 0x31, 0x16, 0x74, 0x12, 0x7a, 0xb2, 0x45, 0xd0, 0x8f, 0x4e, 0x2c, 0xfd, 0xbf, 0x8f, 0x5, 0xc9, 0x5b, 0xf5}} }, + { (uint64_t)10000ull, {{0x61, 0x20, 0xe7, 0x76, 0xe9, 0x12, 0xab, 0x10, 0x5a, 0x49, 0xf9, 0xda, 0x2, 0xa6, 0x75, 0x17, 0xc0, 0xa9, 0xb, 0x2b, 0x3e, 0x2d, 0xa3, 0xd, 0xff, 0x34, 0x39, 0x93, 0xdb, 0xec, 0x95, 0x97}} }, + { (uint64_t)20000ull, {{0x77, 0xbf, 0xb5, 0x37, 0xac, 0xa, 0xbc, 0x41, 0xaa, 0x21, 0xd0, 0xec, 0xd9, 0x18, 0x13, 0x34, 0xd8, 0x6b, 0xa7, 0x86, 0x5a, 0x94, 0x47, 0xf5, 0xc1, 0x58, 0x9a, 0x81, 0xd7, 0xef, 0xb3, 0xbb}} }, + { (uint64_t)30000ull, {{0x35, 0xf4, 0x5, 0xa9, 0x5f, 0x75, 0x19, 0x2a, 0xe9, 0xc0, 0xd4, 0xf5, 0x88, 0x84, 0x47, 0x14, 0xf6, 0x85, 0x1b, 0x97, 0xce, 0xbd, 0x9f, 0x7c, 0x2, 0xc5, 0xdd, 0xd7, 0xbf, 0x58, 0xff, 0x31}} }, + { (uint64_t)40000ull, {{0x77, 0x55, 0xbb, 0x3f, 0x38, 0x7c, 0x21, 0xb8, 0xa0, 0xf4, 0x48, 0x1f, 0xbf, 0xa8, 0x8a, 0xbe, 0xee, 0xce, 0xc7, 0x56, 0x53, 0xfc, 0xa1, 0x89, 0x58, 0x39, 0xc1, 0xba, 0x6, 0x47, 0x9f, 0x96}} }, + { (uint64_t)50000ull, {{0x8b, 0x7e, 0x84, 0xa3, 0x37, 0xb7, 0xb9, 0xcd, 0x5d, 0xb3, 0x63, 0x33, 0x8, 0xad, 0x51, 0x86, 0xa3, 0x59, 0xd, 0xff, 0xb8, 0x23, 0x1e, 0x2f, 0x31, 0xfd, 0x20, 0x42, 0x54, 0x9f, 0xfb, 0xe2}} }, + { (uint64_t)60000ull, {{0xef, 0xfd, 0xa6, 0x25, 0x15, 0xea, 0xb1, 0xbc, 0x1e, 0xbd, 0x74, 0x92, 0x94, 0x9b, 0x1, 0x22, 0xc3, 0x9f, 0x71, 0xa, 0x65, 0x16, 0xec, 0x66, 0x8c, 0x37, 0x61, 0xe6, 0xcc, 0x36, 0x1f, 0x25}} }, + { (uint64_t)70000ull, {{0x16, 0xba, 0x89, 00, 0xf3, 0x6f, 0xf, 0x6c, 0x46, 0x1c, 0xb, 0xe7, 0x64, 0xae, 0xee, 0x48, 0x86, 0x6, 0xb0, 0x53, 0xed, 0xdc, 0x10, 0xb5, 0x9a, 0x3e, 0xde, 0xcd, 0x23, 0xd4, 0x4f, 0xc0}} }, + { (uint64_t)80000ull, {{0x4d, 0xd4, 0x70, 0x3b, 0x7b, 0x7f, 0xcf, 0xe7, 0x2a, 0x2e, 0x4f, 0x31, 0xa4, 0x34, 0x17, 0xf9, 0xc0, 0xda, 0x64, 0x2f, 0xd0, 0xa9, 0x29, 0xb8, 0xf5, 0xed, 0xd8, 0x3, 0x7f, 0x93, 0xc5, 0xb3}} }, + { (uint64_t)90000ull, {{0x8e, 0xfc, 0x3, 0x20, 0x40, 0xbd, 0x90, 0x41, 0xda, 0x3d, 0xb0, 0x9b, 0xa1, 0x3d, 0xa2, 0xa5, 0xd1, 0xb8, 0x12, 0x3, 0xa, 0x5a, 0x36, 0x7c, 0x58, 0x94, 0xbd, 0x54, 0x11, 0x9, 0xe7, 0x30}} }, + { (uint64_t)100000ull, {{0xbd, 0x2e, 0xb1, 0x97, 0x83, 0x57, 0x1c, 0xf2, 0x22, 0x2c, 0x81, 0xb, 0x69, 0xf, 0xc7, 0x66, 0x64, 0x57, 0xae, 0x20, 0x92, 0x5b, 0x90, 0x5, 0xce, 0xe6, 0x1d, 0xf2, 0x66, 0x6f, 0xdc, 0xb7}} }, + { (uint64_t)200000ull, {{0x83, 0xd4, 0xcd, 0xdd, 0xc1, 0x44, 0x87, 0x32, 0xf2, 0x97, 0x7c, 0x41, 0xaa, 0xa7, 0x1f, 0xe6, 0xde, 0x9c, 0x17, 0x6d, 0xa8, 0x99, 0xee, 0xbf, 0xfc, 0x1b, 0xb, 0xa9, 0xea, 0x92, 0x97, 0x90}} }, + { (uint64_t)300000ull, {{0xcc, 0xc0, 0x6b, 0x44, 0xc3, 0x1, 0x38, 0x6, 0x30, 0x45, 0xed, 0x1, 0xd2, 0x45, 0xd8, 0x14, 0x3, 0xb6, 0x36, 0x52, 0xeb, 0xc4, 0xf9, 0x96, 0x7f, 0xd, 0x7f, 0x38, 0x69, 0x7f, 0x46, 0x16}} }, + { (uint64_t)400000ull, {{0x1b, 0xbf, 0xe7, 0xe, 0xca, 0xf1, 0xdd, 0xd7, 0xf1, 0x2, 0x36, 0xf6, 0x8a, 0x41, 00, 0xb, 0x5d, 0xab, 0x2d, 0x47, 0x5c, 0xb9, 0x2f, 0x62, 0xc2, 0xd6, 0x84, 0xcf, 0x57, 0x69, 0xfb, 0x84}} }, + { (uint64_t)500000ull, {{0xf1, 0xb1, 0xcd, 0xaa, 0x78, 0x14, 0x95, 0x36, 0xf, 0x53, 0x31, 0x81, 0xaa, 0x58, 0xc8, 0xbd, 0xae, 0x6a, 0x77, 0x98, 0xd0, 0x2d, 0xab, 0x6d, 0x56, 0x26, 0x81, 0x27, 0x67, 0x9, 0xe7, 0x1}} }, + { (uint64_t)600000ull, {{0xd5, 0x26, 0x7d, 0x60, 0xd4, 0xfe, 0x9b, 0xc5, 0xfe, 0xfa, 0x7d, 0x3f, 0xe0, 0x7c, 0xd1, 0xfa, 0xd4, 0x55, 0x73, 0xd5, 0xae, 0x19, 0x10, 0xda, 0x7, 0x3e, 0x6d, 0x2d, 0xf9, 0xe2, 0x4, 0x39}} }, + { (uint64_t)700000ull, {{0xb, 0x58, 0x11, 0x25, 0xc2, 0xc4, 0x83, 0xc9, 0xa3, 0xd8, 0xbc, 0x8, 0x32, 0x2f, 0x26, 0xaa, 0x1f, 0xc5, 0xe, 0x41, 0x53, 0x2c, 0x1b, 0x9d, 0xf6, 0x26, 0xb0, 0x9, 0xd7, 0x88, 0x67, 0xcf}} }, + { (uint64_t)800000ull, {{0xf5, 0xb3, 0xd1, 0x8f, 0x66, 0xd0, 0xf9, 0x17, 0x5c, 0x30, 0x83, 0xb5, 0xf8, 0x7, 0x8e, 0xaf, 0xa8, 0x9e, 0xf8, 0x1d, 0xe7, 0x15, 0x8, 0xbc, 0x25, 0x1f, 0x5c, 0x5f, 0xe7, 0x25, 0x2e, 0x6}} }, + { (uint64_t)900000ull, {{0x1, 0xde, 0x40, 0x2c, 0x4b, 00, 0x43, 0x4, 0x2e, 0xae, 0x9e, 0xde, 0xa1, 0x49, 0x2b, 0x9d, 0x82, 0xb7, 0xbc, 0x36, 0x68, 0xe9, 0xb5, 0x84, 0xb0, 0x31, 0x3d, 0x44, 0x50, 0x53, 0x40, 0x74}} }, + { (uint64_t)1000000ull, {{0x53, 0x4b, 0x85, 0xc7, 0x89, 0x3f, 0x66, 0xf0, 0x26, 0xb6, 0x5e, 0xd7, 0xe7, 0xa4, 0xb8, 0xc9, 0xf4, 0xb, 0xe3, 0x1b, 0xcd, 0xa, 0x3d, 0xcd, 0x27, 0xc4, 0x71, 0x2, 0x56, 0x51, 0x65, 0x3}} }, + { (uint64_t)2000000ull, {{0xcd, 0xb5, 0xda, 0xfa, 0x53, 0x10, 0xf5, 0x26, 0x2f, 0xfc, 0x9, 0x26, 0xd0, 0xdf, 0x6e, 0xeb, 0xee, 0x2d, 0x52, 0xa9, 0x8d, 0xc6, 0x9f, 0xd, 0xc5, 0xe4, 0xeb, 0xf0, 0xc1, 0xa8, 0x77, 0x2e}} }, + { (uint64_t)3000000ull, {{0x9e, 0x75, 0x63, 0xf0, 0x33, 0x59, 0xea, 0x31, 0xe2, 0x91, 0xe7, 0xf0, 0xb8, 0x74, 0x17, 0xbc, 0xf5, 0xb2, 0x34, 0xee, 0x8b, 0x7e, 0x5b, 0x4, 0x41, 0x73, 0xbf, 00, 0x46, 0x86, 0x7c, 0x57}} }, + { (uint64_t)4000000ull, {{0xfd, 0x2a, 0xeb, 0xd, 0x5e, 0xe5, 0x3b, 0x77, 0xf2, 0xb1, 0xe3, 0xac, 0x75, 0x2d, 0x19, 0x38, 0x9f, 0xc5, 0xba, 0xa0, 0xf8, 0xd7, 0x64, 0x48, 0xa5, 0x9f, 0x99, 0x85, 0xa4, 0x8d, 0xa, 0x25}} }, + { (uint64_t)5000000ull, {{0xc0, 0xbe, 0x4f, 0xb8, 0x77, 0xb9, 0xce, 0x50, 0x87, 0x71, 0x32, 0x3b, 0xcf, 0x1f, 0xb9, 0x48, 0x47, 0x10, 0xee, 0x23, 0x2, 00, 0x6, 0xc3, 0xe8, 0xca, 0xac, 0x6e, 0x4f, 0x2, 0xfa, 0xbf}} }, + { (uint64_t)6000000ull, {{0xfc, 0x44, 0x5c, 0xa3, 0x84, 0xf3, 0x3e, 0x55, 0x8d, 0xc1, 0x56, 0x44, 0x9d, 0x3f, 0xba, 0x6a, 0xfd, 0x54, 0xc3, 0x42, 0xe6, 0x35, 0x11, 0xf, 0xe7, 0x9c, 0x16, 0xc7, 0x17, 0xf7, 0xd4, 0xf7}} }, + { (uint64_t)7000000ull, {{0xd8, 0x9, 0x2b, 0x8d, 0x45, 0xdb, 0x54, 0xa5, 0x6d, 0x64, 0xe8, 0x9, 0x4a, 0x6, 0x22, 0xe2, 0x6e, 0x8a, 0x2e, 0xec, 0xb9, 0x3, 0xb2, 0xe1, 0xf7, 0x5a, 0x83, 0x7b, 0x3a, 0xd8, 0x55, 0x4a}} }, + { (uint64_t)8000000ull, {{0x10, 0x4, 0x5c, 0x91, 0xdb, 0xad, 0x8a, 0x6a, 0x81, 0x62, 0x4a, 0xe0, 0xcf, 0x20, 0x5d, 0xb9, 0x97, 0x3e, 0xe8, 0x42, 0x3e, 0x97, 0xaf, 0x58, 0xa6, 0x1c, 0xfa, 0x7a, 0x78, 0x66, 0xf4, 0x1}} }, + { (uint64_t)9000000ull, {{0x11, 0x5c, 0x20, 0x9e, 0xe1, 0xde, 0xf3, 0x10, 0xce, 0xc9, 0xa6, 0xd1, 0x6c, 0xe6, 0x27, 0xec, 0xbd, 0xb9, 0xff, 0x2c, 0x23, 0x9, 0x3c, 0x24, 0xc8, 0x6c, 0x1b, 0xf2, 0x50, 0xd4, 0xb5, 0x85}} }, + { (uint64_t)10000000ull, {{0x2f, 0x99, 0xd9, 0x74, 0x44, 0x18, 0x66, 0x9, 0x49, 0xba, 0x43, 0x35, 0x61, 0xc6, 0x5, 0xb6, 0xf7, 0xbe, 0x8f, 0x82, 0xa, 0x93, 0xcb, 0x2a, 0xed, 0xa9, 0x7c, 0x87, 0x32, 0x92, 0x56, 0x49}} }, + { (uint64_t)20000000ull, {{0xc6, 0x77, 0x1f, 0xab, 0x14, 0xb, 0x75, 0xf4, 0xef, 0xd0, 0x97, 0xfc, 0xe1, 0x82, 0x6b, 0x80, 0xba, 0xe3, 0x16, 0xbc, 0xec, 0x28, 0x86, 0x9b, 0x3a, 0x1b, 0xf1, 0xbc, 0x6e, 0x4d, 0x20, 0x43}} }, + { (uint64_t)30000000ull, {{0xa1, 0xe9, 0xed, 0x3e, 0xf6, 0x5a, 0x9d, 0x52, 0x6c, 0xc2, 0x62, 0x5, 0x88, 0x12, 0x1, 0xd8, 0xa8, 0xf2, 0xc4, 0x40, 0x9f, 0xa3, 0x64, 0x10, 0x72, 0x96, 0xb9, 0xf9, 0x6a, 0x61, 0xb3, 0x58}} }, + { (uint64_t)40000000ull, {{0xb5, 0x31, 0x2d, 0xc7, 0x72, 0x94, 0xab, 0x9b, 0xc8, 0xbf, 0xd1, 0x39, 0x1e, 0x9a, 0xca, 0x92, 0x45, 0xe2, 0x28, 0xf7, 0x4b, 0x49, 0x74, 0xfc, 0x29, 0xad, 0x1c, 0x31, 0xcb, 0xe3, 0xe6, 0xa3}} }, + { (uint64_t)50000000ull, {{0xb8, 0xab, 0xc9, 0xff, 0xf6, 0x84, 0x1d, 0x2e, 0xa0, 0x13, 0x5a, 0x21, 0x72, 0xd3, 0xa7, 0xb, 0xfc, 0x2b, 0x70, 0x22, 0x8, 0xcd, 0x4a, 0x43, 0xc6, 0x30, 0xbe, 0xb1, 0xb8, 0xa0, 0x32, 0x8b}} }, + { (uint64_t)60000000ull, {{0x63, 0x90, 0xe1, 0xdb, 0x81, 0xb0, 0xea, 0x5c, 0xe2, 0x73, 0x94, 0x14, 0xe5, 0x2b, 0x7, 0x98, 0xd8, 0x2e, 0xb8, 0xe9, 0xae, 0xc5, 0x6d, 0xfe, 0x7e, 0x2c, 0x64, 0x11, 0xab, 0x79, 0x41, 0x87}} }, + { (uint64_t)70000000ull, {{0x7e, 0x51, 0xaf, 0xee, 0x5b, 0xc9, 0x71, 0x52, 0x9d, 0x64, 0x4d, 0xcd, 0x7f, 0x2a, 0x2a, 0xb0, 0x26, 0x69, 0xce, 0x2c, 0xb5, 0x7, 0xa6, 0x2d, 0xfc, 0x93, 0x17, 0x6c, 0xb6, 0xdf, 0x41, 0x38}} }, + { (uint64_t)80000000ull, {{0xf3, 0x7b, 0x94, 0x6b, 0x8b, 0x24, 0x88, 0xeb, 0xee, 0x1c, 0x6, 0xc1, 0x27, 0xfb, 0xe5, 0xfa, 0x5e, 0xfd, 0x62, 0x36, 0x9d, 0xd5, 0xaa, 0xda, 0xed, 0xd8, 0x88, 0x50, 0x1d, 0x3b, 0x7e, 0x3b}} }, + { (uint64_t)90000000ull, {{0x46, 0xcb, 0x76, 0x57, 0xf6, 0x1c, 0x83, 0x7c, 0xec, 0x80, 0x74, 0xbb, 0xb0, 0xf5, 0x2e, 0x7f, 0xc5, 0x9a, 0xd, 0x94, 0xe0, 0x17, 00, 0x9a, 0xbe, 0x25, 0x65, 0x2e, 0x4a, 0xd2, 0xe5, 0x3d}} }, + { (uint64_t)100000000ull, {{0x66, 0x7b, 0x8e, 0x6f, 0x6a, 0x4b, 0x91, 0x89, 0x76, 0xd9, 0x73, 0x5a, 0x43, 0x36, 0x7d, 0xc7, 0x59, 0x2c, 0x87, 0xd0, 0xa1, 0xf8, 0x15, 0xc6, 0xe8, 0x7d, 0xf1, 0x1a, 0x13, 0x50, 0x9f, 0xb2}} }, + { (uint64_t)200000000ull, {{0x3b, 0xcb, 0x51, 0x48, 0x1, 0x64, 0x1b, 0x62, 0x55, 0x93, 0x8c, 0xc5, 0x3, 0x76, 0x2d, 0x35, 0xce, 0x6, 0xd7, 0x5f, 0xe9, 0x50, 0x95, 0x9a, 0x1a, 0xab, 0x21, 0x4b, 0x50, 0x9b, 0x10, 0xb}} }, + { (uint64_t)300000000ull, {{0xa5, 0x92, 0x6f, 0x3, 0x1e, 0x6b, 0x15, 0xeb, 0x86, 0x23, 0x51, 0x8, 0xab, 0xb1, 0xaf, 0x90, 0xc5, 0xb1, 0x62, 0xc3, 0x99, 0x8c, 0x8b, 0xbb, 0x3f, 0xfb, 0xb0, 0x72, 0x9d, 0xa9, 0x45, 0x7b}} }, + { (uint64_t)400000000ull, {{0xfe, 0x35, 0xb6, 0x99, 0x44, 0x41, 0xe, 0xaf, 0x81, 0x5b, 0xdc, 0xd0, 0xa4, 0xd7, 0x1e, 0xf9, 0xfc, 0x66, 0x86, 0x48, 0xad, 0x43, 0x74, 0x3b, 0x3, 0x5a, 0xed, 0x2c, 0x17, 0xc1, 0x38, 0x7a}} }, + { (uint64_t)500000000ull, {{0x22, 0x22, 0xd6, 0x70, 0xb8, 0x7d, 0x9b, 0x47, 0xb8, 0xb9, 0x5c, 0x8c, 0x39, 0x7b, 0xc5, 0x2e, 0x2b, 0x46, 0xa6, 0x48, 0xb0, 0x2, 0xa0, 0x48, 0x5a, 0x37, 0x5c, 0xd8, 0x1f, 0x4a, 0x54, 0x5f}} }, + { (uint64_t)600000000ull, {{0xd3, 0x23, 0x8a, 0x4a, 0x8b, 0x71, 0xab, 0x46, 0xd1, 0x53, 0x4, 0xac, 0xfa, 0x2f, 0x40, 0xbf, 0x5e, 0xa6, 0x3b, 0x3d, 0x86, 0x4a, 0x79, 0xfa, 0x84, 0x25, 0xd2, 0x65, 0x5a, 0xe7, 0x7, 0x6f}} }, + { (uint64_t)700000000ull, {{0xa8, 0xff, 0x28, 0x3f, 0xcf, 0xf0, 0x53, 0xd3, 0x44, 0xc8, 0xf7, 0x56, 0x4f, 0x40, 0x24, 0xb6, 0x6b, 0xfa, 0x45, 0x9f, 0x47, 0x6f, 0xd, 0x73, 0xc, 0x91, 0x39, 0x90, 0x8b, 0x2d, 0x64, 0x7e}} }, + { (uint64_t)800000000ull, {{0xf2, 0xda, 0xf8, 0x88, 0xc4, 0x46, 0x57, 0x1, 0xc0, 0xe6, 0x1e, 0x12, 0xc3, 0xfb, 0xd4, 0xea, 0x79, 0xc7, 0xec, 0xb4, 0xf0, 0xc4, 0xb1, 0x54, 0xc5, 0x1a, 0x24, 0xd1, 0xe9, 0x21, 0x28, 0xba}} }, + { (uint64_t)900000000ull, {{0x11, 0x6a, 0xe5, 0xd2, 0x9c, 0xec, 0x72, 0xaa, 0xc5, 0x57, 0xcb, 0x14, 0xe2, 0xcd, 0xd5, 0x53, 0xe5, 0x88, 0xff, 0x8b, 0x81, 0x78, 0x26, 0x1, 0x99, 0xc4, 0xc, 0xae, 0xa2, 0x12, 0xcb, 0x63}} }, + { (uint64_t)1000000000ull, {{0x8c, 0xe6, 0x48, 0x33, 0xce, 0xc9, 00, 0xcb, 0x6d, 0x5a, 0xc4, 0x6f, 0xc0, 0x23, 0x7d, 0x8f, 0x24, 0x39, 0xc3, 0xdf, 0xa2, 0x38, 0xba, 0xf9, 0xcc, 0x94, 0x16, 0x6a, 0xd2, 0xe8, 0x98, 0x87}} }, + { (uint64_t)2000000000ull, {{0x37, 0x8d, 0x3c, 0x5d, 0xbb, 0xa4, 0x82, 0x3d, 0x33, 0x12, 0xbb, 0x61, 0xfc, 0x6, 0x75, 0xa1, 0xbb, 0x39, 0x89, 0xf3, 0x97, 0x1, 0xeb, 0xd, 0x5c, 0xe4, 0xde, 0x5b, 0xd, 0x90, 0x74, 0x72}} }, + { (uint64_t)3000000000ull, {{0x7f, 0xa2, 0xd0, 0xa5, 0x99, 0xe7, 0x97, 0x2e, 0x74, 0xcb, 0x75, 0xf9, 0x8a, 0xf4, 0x84, 0xfc, 0x85, 0x19, 0xcb, 0x7e, 0x25, 0xb9, 0x84, 0xa7, 0x6d, 0x8b, 0xc2, 0xba, 0x8d, 0xaf, 0xde, 0xd8}} }, + { (uint64_t)4000000000ull, {{0xda, 0x3a, 0xcb, 00, 0xab, 0x2d, 0x8d, 0xcc, 0xac, 0xec, 0x8f, 0x77, 0x59, 0x21, 0xc4, 0xe, 0x26, 0xb1, 0xff, 0xbe, 0xca, 0x9e, 0xb7, 0xe6, 0x57, 0x25, 0x6f, 0x59, 0x68, 0xf2, 0x34, 0x1c}} }, + { (uint64_t)5000000000ull, {{0x34, 0x6, 0xd7, 0x9a, 0x50, 0xd8, 0x14, 0xa9, 0xcc, 0xed, 0x3b, 0x24, 0x4, 0xed, 0x3e, 0x1b, 0x8d, 0xa6, 0x21, 0x98, 0x8c, 0x43, 0xb1, 0x93, 0x69, 0x42, 0xf4, 0x94, 0xa, 0xc5, 0xbf, 0x6a}} }, + { (uint64_t)6000000000ull, {{0xf8, 0x3e, 0xe8, 0xc1, 0x62, 0xfc, 0x52, 0xa0, 0x8, 0x9f, 0x46, 0xe8, 0x29, 0xc2, 0xea, 0xf6, 0xa1, 0x9f, 0xd5, 0x96, 0xcd, 0x12, 0xb3, 0xe8, 0x19, 0xd5, 0x67, 0x69, 0x44, 0xf, 0x7b, 0x4e}} }, + { (uint64_t)7000000000ull, {{0x8c, 0x72, 0x7d, 0x24, 0x57, 0xf3, 0x4b, 0x2f, 0xdb, 0x6a, 0xdf, 0x69, 0x1a, 0xb3, 0x5f, 0xaa, 0xe4, 0xff, 0x23, 0x4c, 0x28, 0xb4, 0x4e, 0x9f, 0xd3, 0x71, 0x8e, 0xef, 0xec, 0x41, 0x75, 0x80}} }, + { (uint64_t)8000000000ull, {{0x4a, 0x2e, 0x2f, 0x76, 0xe3, 0x5d, 0xcb, 0xa8, 0x97, 0xa3, 0xae, 0x72, 0xc4, 0x27, 0xd, 0x9c, 0x13, 0x17, 0x14, 0xed, 0x19, 0x1b, 0x55, 0x5c, 0x5e, 0x1, 0xe4, 0x75, 0x7c, 0xba, 0xe7, 0x2c}} }, + { (uint64_t)9000000000ull, {{0x3f, 0x9f, 0xc, 0x4, 0xc0, 0xb9, 0xec, 0x9b, 0x4d, 0x11, 0x7c, 0x5f, 0xc9, 0xf1, 0x8a, 0x20, 0xf2, 0xb3, 0xfa, 0xcc, 0xa4, 0xc8, 0xae, 0x41, 0xaf, 0x7c, 0x8, 0xe9, 0xe0, 0xef, 0xb9, 0x81}} }, + { (uint64_t)10000000000ull, {{0x97, 0xc9, 0x2a, 0x29, 0x1, 0x5e, 0xcb, 0x49, 0xf8, 0x9, 0x5, 0x45, 0xe0, 0x1f, 0xf9, 0x78, 0x6c, 0xae, 0x40, 0x57, 0x73, 0x47, 0x61, 0x18, 0x24, 0xf4, 0xb6, 0x59, 0x9f, 0xf5, 0xd3, 0x64}} }, + { (uint64_t)20000000000ull, {{0x9, 0xba, 0xed, 0x9a, 0x3c, 0x44, 0xb2, 0x22, 0x85, 0xa0, 0xae, 0xa4, 0x14, 0x8c, 0xa7, 0xde, 0x9b, 0xea, 0x96, 0x3c, 0xf6, 0x96, 0x23, 0xb6, 0x83, 0x44, 0x5c, 0xa, 0x10, 0xa5, 0x86, 0x77}} }, + { (uint64_t)30000000000ull, {{0x45, 0xac, 0xaf, 0x1d, 0xe2, 0x89, 0x6d, 0xe8, 0x72, 0x84, 0xff, 0xed, 0x57, 0x8b, 0x77, 0x14, 0xf5, 0x18, 0xa6, 0x18, 0xe2, 0xae, 0x6f, 0x90, 0xae, 0x4f, 0x70, 0x13, 0xa2, 0x8e, 0x99, 0xe0}} }, + { (uint64_t)40000000000ull, {{0x8, 0xb8, 0x47, 0x36, 0x42, 0x24, 0xe2, 0x9c, 0xe3, 0x36, 0x63, 0x93, 0xc2, 0xe1, 0x1e, 0xfc, 0x75, 0x55, 0xde, 0xe1, 0xa0, 0x5f, 0x91, 0xa7, 0x2e, 0x61, 0x11, 0x76, 0x84, 0xdd, 0xbe, 0x29}} }, + { (uint64_t)50000000000ull, {{0x6c, 0x8e, 0xe, 0x4a, 0x63, 0x4f, 0x85, 0x9a, 0x31, 0xab, 0x2f, 0x7a, 0x78, 0xc0, 0xc4, 0xa5, 0x93, 0x8c, 0xb7, 0x7f, 0x3, 0x35, 0x50, 0xa4, 0x7d, 0x7e, 0x31, 0x81, 0xb6, 0xb2, 0x6e, 0xc0}} }, + { (uint64_t)60000000000ull, {{0x66, 0xc2, 0xa0, 0x9, 0x65, 0xf9, 0xbf, 0xcb, 0xb1, 0x1e, 0xa0, 0x3c, 0xf1, 0xd6, 0x31, 0xb0, 0xe, 0x8a, 0x1e, 0xf7, 0xa6, 0xb, 0x1b, 0xe4, 0xa5, 0xac, 0x9, 0x23, 0xb, 0xf8, 0x17, 0x3f}} }, + { (uint64_t)70000000000ull, {{0x63, 0x51, 0xd7, 0x74, 0xc0, 0x2c, 0x5a, 0x9d, 0xee, 0xcf, 0xdb, 0xab, 0x70, 0x96, 0x68, 0x59, 0x8c, 0x47, 0xe4, 0xb1, 0x78, 0x2c, 0xe5, 0xae, 0x31, 0x6a, 0xf7, 0x40, 0xa6, 0x6f, 0x7e, 0x30}} }, + { (uint64_t)80000000000ull, {{0x5a, 0xcc, 0xfd, 0x16, 0x22, 0x79, 0xa5, 0x1c, 0x8b, 0x3b, 0xd5, 0xd3, 0x67, 0x9e, 0x91, 0x89, 0x67, 0xa2, 0x64, 0xea, 0x6, 0x3d, 0x37, 0xdf, 0xf5, 0xe3, 0x45, 0x7e, 0xc3, 0x7, 0xd4, 0x57}} }, + { (uint64_t)90000000000ull, {{0xb7, 0x47, 0xfc, 0x1, 0xc6, 0xf0, 0xc7, 0x49, 0x67, 0x3a, 0x29, 0x10, 0x25, 0xc, 0x2e, 0x23, 0xcb, 0x38, 0x27, 0x4d, 0x63, 0xb4, 0x2f, 0x52, 0x1b, 0x84, 0x63, 0x56, 0xe4, 0x13, 0x61, 0x8f}} }, + { (uint64_t)100000000000ull, {{0x9, 0x42, 0x84, 0x3d, 0x6f, 0x69, 0xe1, 0xcf, 0x3d, 0x99, 0xc9, 0x9f, 0xc, 0x97, 0xc0, 0xe6, 0xe5, 0x78, 0x93, 0x5a, 0xf6, 0xa8, 0xbd, 0xb8, 0xf8, 0x1d, 0x5b, 0x90, 0xbd, 0xe7, 0xcc, 0x10}} }, + { (uint64_t)200000000000ull, {{0x56, 0x4c, 0x64, 0xea, 0x50, 0xe4, 0xbd, 0x20, 0xdb, 0x58, 0x5d, 0xb5, 0x87, 0xb1, 0xf7, 0x64, 0xa2, 0x62, 0xd8, 0x46, 0xa6, 0xb0, 0xa2, 0x4b, 0x43, 0x27, 0x60, 0xd2, 0xf9, 0xde, 0x66, 0x5b}} }, + { (uint64_t)300000000000ull, {{0xac, 0x65, 0x83, 0x41, 0x5b, 0xd6, 0x4c, 0x3, 0x35, 0x97, 0xf9, 0x28, 0xa4, 0xb5, 0xd4, 0xf4, 0x78, 0x9e, 0xa8, 0xb2, 0x87, 0x82, 0x73, 0x89, 0xa8, 0x1e, 0xb6, 0x62, 0x9e, 0xc5, 0xb8, 0x50}} }, + { (uint64_t)400000000000ull, {{0x52, 0xf4, 0x9d, 0x89, 0xcf, 0x74, 0x13, 0x2f, 0xc7, 0x43, 0x2e, 0x6a, 0x6b, 0xef, 0xcf, 0xf3, 0xfd, 0x13, 0xd6, 0x3b, 0x51, 0x60, 0xab, 0x1c, 0xe6, 0x4a, 0xb0, 0xd1, 0x21, 0xcd, 0xa9, 0x9a}} }, + { (uint64_t)500000000000ull, {{0xe9, 0xaa, 0x7c, 0x81, 0xcd, 0xb5, 0xb3, 0x14, 0x8f, 0xb7, 0x62, 0x80, 0x63, 0xcd, 0x7a, 0x7, 0xd1, 0xad, 0xd1, 0x64, 0x3c, 0xed, 0xd3, 0xfa, 0x34, 0x47, 0x9d, 0x85, 0x9c, 0xc5, 0x62, 0x65}} }, + { (uint64_t)600000000000ull, {{0x98, 0x27, 0xae, 0x31, 0xe5, 0xc2, 0xa7, 0x78, 0x39, 0xf6, 0xb, 0x83, 0xab, 0x45, 0x78, 0xe2, 0xa0, 0x1e, 0xfa, 0x4b, 0x3b, 0x14, 0xcc, 0x72, 0x73, 0x14, 0xff, 0xd7, 0x15, 0x53, 0x63, 0xbf}} }, + { (uint64_t)700000000000ull, {{0x72, 0x91, 0x6a, 0x79, 0x27, 0xff, 0x13, 0x24, 0xd4, 0x98, 0x40, 0xec, 0xc0, 0x98, 0x68, 0xb8, 0xf3, 0x15, 0xe4, 0xf1, 0xf6, 0xd4, 0x45, 0x8d, 0x37, 0x5e, 0xc7, 0x45, 0xfc, 0x2e, 0x63, 0x53}} }, + { (uint64_t)800000000000ull, {{0x66, 0x76, 0xe0, 0x4, 0xf, 0xa4, 0xb8, 0x22, 0x9c, 0x61, 0x69, 0xc, 0x71, 0x32, 0x22, 0xcf, 0x3d, 0x37, 0xb9, 0x49, 0x3b, 0x49, 0x6, 0x80, 0xbb, 0x48, 0xd8, 0xd5, 0x1a, 0xde, 0x95, 0xf2}} }, + { (uint64_t)900000000000ull, {{0x41, 0x54, 0xb3, 0x46, 0x5a, 0x43, 0x72, 0x67, 0x1e, 0xa9, 0xe0, 0x64, 0xa7, 0xca, 0xa6, 0x6e, 0x14, 0xb4, 0x98, 0x6a, 0x46, 0x68, 0x91, 0x8a, 0xfa, 0x57, 0x9b, 0xf1, 0xed, 0x25, 0x6, 0xdd}} }, + { (uint64_t)1000000000000ull, {{0xbb, 0x6f, 0x70, 0x62, 0xca, 0x30, 0x6d, 0x67, 0x2a, 0x73, 0xe, 0x2a, 0x2f, 0x21, 0x9b, 0xdb, 0xe4, 0xc, 0x9f, 0xb3, 0xfe, 0x4d, 0x60, 0x13, 0x69, 0x2a, 0xf9, 0x3c, 0xdb, 0x2e, 0xc, 0xd1}} }, + { (uint64_t)2000000000000ull, {{0xbc, 0xe, 0xae, 0x5b, 0x9c, 0x6a, 0xd6, 0x38, 0x7a, 0x41, 0x19, 0x3c, 0x46, 0xf3, 0xc1, 0xd0, 0x71, 0x6d, 0x77, 0xd6, 0x4e, 0x22, 0xb2, 0xe0, 0x7b, 0x4b, 0xce, 0x75, 0x67, 0x65, 0xa2, 0xb}} }, + { (uint64_t)3000000000000ull, {{0x10, 0xc, 0x6f, 0x13, 0x42, 0xb7, 0x1b, 0x73, 0xed, 0xdd, 0xc5, 0x49, 0x2b, 0xe9, 0x23, 0x18, 0x2f, 00, 0xa6, 0x83, 0x48, 0x8e, 0xc3, 0xa2, 0xa1, 0xc7, 0xa9, 0x49, 0xcb, 0xe5, 0x77, 0x68}} }, + { (uint64_t)4000000000000ull, {{0x41, 0x9f, 0x7c, 0x94, 0x91, 0x4, 0x34, 0xf, 0xd3, 0xce, 0x85, 0x94, 0x8d, 0x2e, 0xf9, 0xf0, 0xdd, 0x4b, 0xb3, 0xd9, 0x2f, 0x5a, 0x78, 0x2c, 0x5f, 0x78, 0x4, 0xb7, 0x52, 0x9a, 0x13, 0xc6}} }, + { (uint64_t)5000000000000ull, {{0x40, 0x65, 0x34, 0x98, 0xbe, 0xa0, 0x22, 0xe3, 0x36, 0x5a, 0x3, 0xe5, 0x75, 0x25, 0xba, 0x65, 0x96, 0x53, 0x76, 0x24, 0x4f, 0xff, 0x10, 0x73, 0xe, 0xd9, 0x7a, 0x73, 0xb7, 0x53, 0x1, 0x91}} }, + { (uint64_t)6000000000000ull, {{0xdb, 0x1c, 0x7c, 0xf6, 0x8, 0x91, 0xf9, 0x65, 0xeb, 0xa9, 0xc6, 0x2, 0x24, 00, 0x63, 0xe, 00, 0x47, 0x95, 0x34, 0xe6, 0xf5, 0xb5, 0x33, 0xdc, 0xfc, 0x83, 0x19, 0x38, 0x52, 0x2c, 0x78}} }, + { (uint64_t)7000000000000ull, {{0x59, 0xa0, 0x3a, 0x31, 0x53, 0xa9, 0x94, 0xd7, 0x23, 0x27, 0xe4, 0xd9, 0x24, 0x21, 0xd3, 0xe3, 0x29, 0x1b, 0x1f, 0xa1, 0xb2, 0x40, 0xde, 0x44, 0xb9, 0x2d, 0x7f, 0x62, 0xec, 0x1, 0x28, 0xf1}} }, + { (uint64_t)8000000000000ull, {{0xb2, 0x80, 0xb9, 0x3b, 0x1e, 0x43, 0x88, 00, 0x73, 0xea, 0x4a, 0xa0, 0xef, 0x11, 0x4, 0xf8, 0x24, 0xbd, 0x12, 0x7a, 0x4a, 0x3d, 0xa2, 0x13, 0x92, 0x65, 0xf, 0xe8, 0xc6, 0x55, 0xb6, 0xc5}} }, + { (uint64_t)9000000000000ull, {{0xda, 0xf0, 0xd3, 0xe9, 0x32, 0x17, 0xd8, 0xe9, 0x5a, 0xbf, 0xdd, 0xf1, 0x3b, 0x7f, 0xd4, 0x8e, 0x34, 0x47, 0xad, 0x9, 0x23, 0x26, 0xb8, 0x99, 0xed, 0x58, 0x1f, 0xd5, 0xf8, 0x6, 0xc5, 0x6}} }, + { (uint64_t)10000000000000ull, {{0x16, 0x3d, 0xd6, 0x82, 0xec, 0x97, 0x7c, 0xdd, 0xa5, 0x95, 0x31, 0xda, 0x3f, 0xfa, 0x72, 0x99, 0x8a, 0x6f, 0x88, 0x37, 0xab, 0xad, 0xc6, 0x36, 0xaa, 0xed, 0xc8, 0xbe, 0x19, 0xb2, 0xd7, 0xc7}} }, + { (uint64_t)20000000000000ull, {{0x2, 0xfa, 0x35, 0x3a, 0xa8, 0x4e, 0xa8, 0xc4, 0x4c, 0x80, 0x23, 0x6, 0x5d, 0x79, 0x41, 0x60, 0x6b, 0x1f, 0xa5, 0xc2, 0x64, 0xdc, 0xcf, 0x46, 0xdc, 0x64, 0x94, 0xeb, 0xe9, 0x60, 0x6f, 0x20}} }, + { (uint64_t)30000000000000ull, {{0x87, 0x5, 0xd, 0xab, 0xf5, 0xb2, 0x3e, 0x8b, 0x79, 0x81, 0x3f, 0x4e, 0xd7, 0x6a, 0xa4, 0xad, 0xd2, 0x25, 0xdd, 0x2a, 0x50, 0x89, 0xaf, 0x6, 0x7d, 0xa7, 0x7c, 0xcb, 0x6e, 0xc5, 0x59, 0x46}} }, + { (uint64_t)40000000000000ull, {{0xaa, 0xe6, 0xb2, 0xc8, 0xa2, 0x9e, 0x4d, 0xbc, 0x63, 0x76, 0xc1, 0x72, 0x5, 0xfb, 0x2, 0x85, 0xe7, 0xd7, 0xd3, 0x25, 0x32, 0x3c, 0xd5, 0x26, 0xf, 0x98, 0xad, 0xff, 0xf7, 0xd4, 0xd4, 0xfb}} }, + { (uint64_t)50000000000000ull, {{0x9d, 0x79, 0x28, 0x82, 0x12, 0xa1, 0xe2, 0x3c, 0x9, 0x9f, 0xb2, 0xd8, 0xf0, 0xd0, 0xdb, 0xd3, 0xc2, 0xec, 0xd7, 0x58, 0xb9, 0xe6, 0xb5, 0xb4, 0xf2, 0x90, 0x60, 0x7, 0x9f, 0x19, 0x66, 0x9f}} }, + { (uint64_t)60000000000000ull, {{0x18, 0x90, 0x10, 0x6f, 0x1b, 0x97, 0xbc, 0x2d, 0xa, 0xe3, 0x96, 0xe5, 0xe5, 0x5e, 0xbf, 0xcc, 0x8e, 0xf6, 0x91, 0x7f, 0xb1, 0x96, 0xcb, 0x2b, 0x1e, 0x80, 0x25, 0x5d, 0x54, 0xb6, 0x87, 0x10}} }, + { (uint64_t)70000000000000ull, {{0x57, 0xd3, 0x4e, 0xf7, 0x54, 0x3b, 0xe4, 0x7b, 0x7b, 0xf4, 0x97, 0xce, 0x4a, 0x17, 0x6e, 0x78, 0xc6, 0xd6, 0x5c, 0xd3, 0x27, 0xf6, 0x4b, 0xa7, 0x5c, 0x27, 0xd1, 0x57, 0xb3, 0x37, 0x12, 0x5d}} }, + { (uint64_t)80000000000000ull, {{0x5e, 0xcb, 0x10, 0x15, 0x4b, 0x96, 0xca, 0xb5, 0x5e, 0x9, 0x46, 0x83, 0xf8, 0xdb, 0xff, 0x7f, 0x56, 0x63, 0x5f, 0xa6, 0x64, 0x97, 0xee, 0x9e, 0x24, 0xe, 0x83, 0x63, 0x7c, 0x7c, 0x87, 0x72}} }, + { (uint64_t)90000000000000ull, {{0x42, 0x32, 0x69, 0x98, 0x51, 0x30, 0xf1, 0x66, 0x51, 0x6a, 0x5b, 0xa8, 0x61, 0x9, 0x6d, 0x72, 0xec, 0xcc, 0x67, 0xad, 0xab, 0xa4, 0x5e, 0xb3, 0x73, 0x9a, 0xe, 0xbc, 0x61, 0xa3, 0x20, 0xae}} }, + { (uint64_t)100000000000000ull, {{0xa8, 0xd1, 0x60, 0x95, 0x91, 0x49, 0x8f, 0xa7, 0xc2, 0x94, 0x27, 0xad, 0x89, 0x31, 0xaf, 0x36, 0xc5, 0x2d, 0xc9, 0x7b, 0x4a, 0x11, 0xe7, 0x47, 0xa9, 0x56, 0xc2, 0x8c, 0x42, 0x54, 0xcf, 0xd4}} }, + { (uint64_t)200000000000000ull, {{0x23, 0x14, 0x49, 00, 0xa8, 0x66, 0xe8, 0xc1, 0xbf, 0x40, 0x98, 0xda, 0xa9, 0x48, 0xb9, 0x86, 0xf3, 0x84, 0xe, 0x5a, 0x7d, 0x21, 0x5e, 0xf0, 0xd5, 0x64, 0xef, 0xd8, 0xbe, 0xc6, 0x83, 0x15}} }, + { (uint64_t)300000000000000ull, {{0x6a, 0x51, 0x47, 0x3c, 0x86, 0xed, 0xad, 0x53, 0x51, 0x4b, 0x3f, 0x95, 0x97, 0xed, 0x21, 0xae, 00, 0x81, 0x51, 0xa0, 0x9e, 0x43, 0xad, 0xdd, 0x45, 0xd1, 0x74, 0x63, 0xc5, 0x34, 0x3, 0x97}} }, + { (uint64_t)400000000000000ull, {{0x8, 0xbd, 0xd4, 0xc3, 0xe4, 0x53, 0x1b, 0x29, 0x7a, 0x70, 00, 0x1e, 0xb8, 0xa4, 0xf1, 0x98, 0xdc, 0x3b, 0xd4, 0xf1, 0xf5, 0x60, 0x9a, 0xda, 0x98, 0xf6, 0xd9, 0x5f, 0x9a, 0x1a, 0x30, 0x2e}} }, + { (uint64_t)500000000000000ull, {{0x97, 0x55, 0x70, 0xea, 0x12, 0xde, 0x5a, 0xf5, 0xc5, 0x36, 0xbd, 0xb6, 0x83, 0x54, 0xfb, 0xc8, 0x32, 0x21, 0x50, 0xfc, 0x56, 0x83, 0x7c, 0x4b, 0x78, 0xa9, 0x85, 0x76, 0x5d, 0x2a, 0x70, 0x99}} }, + { (uint64_t)600000000000000ull, {{0xa7, 0xa6, 0x39, 0x93, 0x41, 0xcb, 0x4d, 0x67, 0x76, 0xcd, 0x94, 0xd, 0x1d, 0x6a, 0xb0, 0xac, 0xa, 0xbf, 0x56, 0x93, 0x6a, 0x35, 0x31, 0xdf, 0xe9, 0x6c, 0x23, 0x69, 0x97, 0x8e, 0x49, 0xfa}} }, + { (uint64_t)700000000000000ull, {{0x55, 0x9, 0x3e, 0x5e, 0xeb, 0xca, 0x3, 0x88, 0x48, 0xdc, 0x99, 0x7e, 0x31, 0x95, 0xec, 0xc5, 0x8f, 0xb4, 0xa5, 0x71, 0xb9, 0x52, 0x56, 0xc0, 0xff, 0x49, 0xbe, 0xd0, 0xf1, 0x65, 0x22, 0xbd}} }, + { (uint64_t)800000000000000ull, {{0xbb, 0xc6, 0x18, 0x2, 0x24, 0xaf, 0xd3, 0x38, 0xa6, 0xf4, 0xa0, 0x6b, 0x11, 0x98, 0x40, 0x68, 0xeb, 0x36, 0x35, 0xe7, 0xe5, 0x47, 0x66, 0x69, 0x78, 0x83, 0xaf, 0xbd, 0xce, 0xad, 0x2f, 0x31}} }, + { (uint64_t)900000000000000ull, {{0x61, 0x3a, 0xa1, 0x2c, 0xc0, 0xa1, 0x9b, 0xc8, 0x43, 0x63, 0x50, 0xbb, 0xc0, 0xf6, 0x16, 0x32, 0x6e, 0x64, 0x85, 0x83, 0x33, 0x4a, 0x32, 0x65, 0x16, 0x29, 0xe9, 0x5, 0xc5, 0x20, 0x62, 0x69}} }, + { (uint64_t)1000000000000000ull, {{0x52, 0xdd, 0xf8, 0x81, 0x13, 0xa0, 0xfc, 0xf2, 0x12, 0x90, 0x95, 0xc6, 0x18, 0x91, 0xbe, 0x88, 0x5c, 0x9, 0x30, 0x8, 0xeb, 0xc4, 0x65, 0xc, 0xb0, 0xee, 0xa5, 0x60, 0xcd, 0x4d, 0x75, 0x1b}} }, + { (uint64_t)2000000000000000ull, {{0x75, 0xbd, 0xfc, 0x35, 0xa6, 0xdf, 0x76, 0xe5, 0x98, 0x8e, 0xd9, 0xe3, 0x10, 0xa5, 0x89, 0x16, 0xae, 0xf0, 0xc5, 0xf0, 0x5b, 0x89, 0x22, 0xea, 0xae, 0x2c, 0xf9, 0x8f, 0x58, 0x42, 0x3c, 0xe3}} }, + { (uint64_t)3000000000000000ull, {{0x88, 0x98, 0x93, 0xe8, 0x7d, 0x56, 0x9f, 0x14, 0xb2, 0x48, 0xd1, 0xed, 0x93, 0xe8, 0xce, 0x60, 0xbb, 0xe3, 0x73, 0x69, 0xb0, 0xd6, 0xc7, 0xa1, 0x86, 0x89, 0x33, 0xd3, 0xc3, 0xda, 0x9a, 0x72}} }, + { (uint64_t)4000000000000000ull, {{0x88, 0x3e, 0xf3, 0x4b, 0xa2, 0xc1, 0x91, 0xf4, 0x9d, 0x3c, 0xc6, 0xad, 0xa0, 0xaf, 0xf1, 0xcf, 0xb1, 0x77, 0xbd, 0x9e, 0xd4, 0xb3, 0xa5, 0x37, 0x84, 0xb7, 0xf1, 0x62, 0x9b, 0xed, 0x17, 0x41}} }, + { (uint64_t)5000000000000000ull, {{0xa2, 0x90, 0x7c, 0x39, 0x84, 0xb1, 0x4a, 0xb1, 0xf4, 0xda, 0x58, 0xc2, 0xc8, 0x2d, 0x6b, 0x24, 0xf1, 0x29, 0x49, 0x9, 0x75, 0xfc, 0x4a, 0x33, 0x3d, 0x25, 0xa1, 0xf9, 0x2b, 0xc4, 0x32, 0xb6}} }, + { (uint64_t)6000000000000000ull, {{0xa0, 0x7d, 0x9f, 0x18, 0x95, 0x1f, 0xf2, 0x32, 0xcf, 0x4e, 0xc0, 0xee, 0x2f, 0xbc, 0xc3, 0xe1, 0x1b, 0x2c, 0xaf, 0xc9, 0x57, 0x65, 0x82, 0x10, 0x38, 0x1e, 0x3e, 0xe4, 0xed, 0xec, 0x2e, 0x7a}} }, + { (uint64_t)7000000000000000ull, {{0x66, 0x80, 0x21, 0xd5, 0xde, 0x8c, 0xa4, 0xc1, 0x8f, 0x5a, 0x74, 0xf2, 0x78, 0x69, 0xc4, 0xd6, 0xd4, 0x93, 0xa3, 0x30, 0x39, 0x3c, 0xf0, 0x26, 0x41, 0xff, 0xa8, 0x56, 0x7b, 0xa5, 0x36, 0x20}} }, + { (uint64_t)8000000000000000ull, {{0xe0, 0x48, 0x7a, 0xc4, 0x5a, 0x82, 0x59, 0xe3, 0xe5, 0xf2, 0xd9, 0xb8, 0xf6, 0xb8, 0xfa, 0x26, 0x9a, 0x63, 0x49, 0x71, 0xa2, 0xf7, 0xc2, 0x1a, 0x54, 0x17, 0x76, 0x81, 0xeb, 0x2, 0xbd, 0x4a}} }, + { (uint64_t)9000000000000000ull, {{0x98, 0x92, 0x6a, 0x3a, 0xf0, 0x5b, 0xf4, 0xa9, 0x8d, 0xf9, 0xf6, 0x4a, 0xe7, 0xb9, 0xda, 0x45, 0xa7, 0x6, 0xc3, 0xf8, 0x39, 0x5e, 0x47, 0x1f, 0x96, 0xed, 0x3c, 0x6, 0x6, 0xbe, 0xbb, 0x71}} }, + { (uint64_t)10000000000000000ull, {{0x80, 0xad, 0xb7, 0xd, 0x46, 0xf6, 0x3a, 0x75, 0x64, 0xa3, 0xf6, 0x71, 0xd9, 0xba, 0x95, 0x71, 0xb7, 0xf7, 0x95, 0xa9, 0x63, 0x38, 0x2a, 0x4d, 0x9f, 0xaf, 0x2d, 0x54, 0xf6, 0xc6, 0x84, 0x29}} }, + { (uint64_t)20000000000000000ull, {{0xae, 0xbd, 0x97, 0x42, 0x1f, 0x3f, 0xca, 0xe8, 0x95, 0x18, 0x60, 0xe6, 0xd9, 0xd1, 0xf3, 0xec, 0x59, 0x73, 0xa2, 0xf7, 0x66, 0x88, 0x4b, 0xfe, 0x17, 0x50, 0x79, 0x51, 0xe4, 0x62, 0xc6, 0x63}} }, + { (uint64_t)30000000000000000ull, {{0x61, 0x2, 0x6c, 0x84, 0x2a, 0x6a, 0x22, 0x25, 0x74, 0x6b, 0x19, 0x6b, 0x56, 0x89, 0xe1, 0x18, 0xf5, 0x41, 0x34, 0x15, 0x98, 0x1d, 0x7, 0x73, 0x62, 0xb2, 0xe7, 0xb9, 0xac, 0xa5, 0x28, 0x16}} }, + { (uint64_t)40000000000000000ull, {{0x52, 0x54, 0xb5, 0x78, 0xe8, 0x57, 0x9a, 0x27, 0x3b, 0x89, 0x8e, 0x65, 0x9d, 0xd3, 0xe1, 0xa1, 0xcf, 0xba, 0x12, 0x47, 0x26, 0x64, 0xbd, 0x4e, 0x7f, 0x9a, 0x13, 0xb1, 0xfc, 0xee, 0x2, 0x93}} }, + { (uint64_t)50000000000000000ull, {{0x7b, 0x2a, 0xb, 00, 0xcf, 0xdc, 0xa9, 0x51, 0x46, 0xcf, 0x80, 0x95, 0xdd, 0x2b, 0x82, 0x90, 0x91, 0xb0, 0xf2, 0xd5, 0xbb, 0xc, 0x33, 0x82, 0x2d, 0x8b, 0x43, 0x42, 0x69, 0xcd, 0x2a, 0x42}} }, + { (uint64_t)60000000000000000ull, {{0x21, 0x57, 0x4f, 0xed, 0x15, 0x1a, 0x2f, 0x9f, 0x64, 0xa4, 0x5b, 0xe2, 0x8a, 0x3a, 0xf5, 0x88, 0xe9, 0xf2, 0xd1, 0x71, 0x35, 0xa3, 0x53, 0x7f, 0x7, 0xfd, 0x6a, 0xef, 0xa2, 0x9f, 0x2, 0xaf}} }, + { (uint64_t)70000000000000000ull, {{0x1a, 0xf2, 0x41, 0xe1, 0x38, 0x27, 0x98, 0x29, 0xac, 0x6a, 0xe6, 0x2f, 0xf, 0x33, 0x20, 0x4b, 0xb2, 0x8a, 0xfd, 0x6, 0x5c, 0x42, 0x59, 0x3b, 0xdc, 0x79, 0x14, 0x85, 0x97, 0x5b, 0x26, 0x95}} }, + { (uint64_t)80000000000000000ull, {{0xa8, 0xc8, 0xb8, 0x7b, 0x51, 0x2d, 0xef, 0x9b, 0x5e, 0x50, 0xe, 0xb4, 0x98, 0xaf, 0x86, 0xaa, 0xd2, 0x46, 0x4a, 0xea, 0xe7, 0x6d, 0xb1, 0xf6, 0x5d, 0x23, 0x26, 0xce, 0x90, 0x26, 0xec, 0x69}} }, + { (uint64_t)90000000000000000ull, {{0x3d, 0x78, 0x73, 0x63, 0x95, 0xf1, 0xd7, 0xde, 0x8e, 0x16, 0xc0, 0xb5, 0xa9, 0x9f, 0x4d, 0xc4, 0xeb, 0x8f, 0x22, 0xac, 0xc1, 0x5b, 0x21, 0x42, 0x44, 0x1d, 0xbd, 0x8d, 0x2c, 0x31, 0xb9, 0xce}} }, + { (uint64_t)100000000000000000ull, {{0x27, 0x27, 0xd4, 0x93, 0x2f, 0x98, 0x39, 0xe4, 0x3b, 0x6b, 0xf5, 0xfb, 0x29, 0xa3, 0xbe, 0x4c, 0x9, 0xb, 0x6e, 0xb9, 0x31, 00, 0xbb, 0x92, 0x58, 0x1a, 0xdb, 0x8d, 0xd2, 0xb6, 0x61, 0x54}} }, + { (uint64_t)200000000000000000ull, {{0xae, 0x96, 0x78, 0x2e, 0xf2, 0xc4, 0xdf, 0x7d, 0x2e, 0x4, 0xcc, 0xf9, 0xef, 0x76, 0x23, 0x7f, 0x17, 0xc, 0x97, 0x3, 0xb4, 0x92, 0xc0, 0x78, 0x52, 0x6e, 0xb1, 0xf6, 0x85, 0x3d, 0xb1, 0x33}} }, + { (uint64_t)300000000000000000ull, {{0x17, 0x43, 0xfe, 0xab, 0x12, 0xad, 0xe5, 0xfe, 0x12, 0x53, 0x22, 0x27, 0x2f, 0xd1, 0x40, 0x6b, 0x74, 0xe8, 0x19, 0x70, 0x32, 0x68, 0x46, 0x22, 0xee, 0x79, 0xab, 0xcd, 0x94, 0x93, 0x66, 0x4c}} }, + { (uint64_t)400000000000000000ull, {{0x7, 0x9b, 0xf2, 0xa9, 0x6e, 0x16, 0x6e, 0xf9, 0xe6, 0xb2, 0x23, 0x1d, 0xb9, 0x85, 0x8b, 0x99, 0x98, 0x7f, 0x49, 0x33, 0x87, 0xde, 0xeb, 0xd5, 0x17, 0x48, 0x54, 0x9a, 0xd, 0xf7, 0xdc, 0x44}} }, + { (uint64_t)500000000000000000ull, {{0xca, 0xba, 0x97, 0x98, 0x51, 0x6d, 0xad, 0x3, 0x38, 0xd0, 0x6e, 0x10, 0x6d, 0x76, 0xa2, 0x1, 0x93, 0x7a, 0xce, 0x4c, 0x91, 0x53, 0x9e, 0x61, 0x7d, 0x89, 0x28, 0x73, 0x6, 0xa3, 0x92, 0xb1}} }, + { (uint64_t)600000000000000000ull, {{0x6b, 0x8, 0x7f, 0x48, 0xb3, 0xd7, 0xaa, 0xc9, 0x57, 0xc4, 0x52, 0xe5, 0x1a, 0x18, 0xd7, 0x26, 0xb, 0xf8, 0xc8, 0x56, 0xc4, 0xc7, 0x1e, 0x48, 0xf6, 0x49, 0xae, 00, 0x4a, 0xf6, 0x8f, 0x13}} }, + { (uint64_t)700000000000000000ull, {{0x9e, 0xed, 0x8b, 0x23, 0x1f, 0x79, 0x4c, 0x46, 0x5c, 0xbe, 0x88, 0x40, 0xd0, 0xf1, 0x6f, 0x7b, 0x9f, 0x9c, 0x6e, 0xb4, 0x9c, 0x20, 0x7d, 0xe9, 0xd8, 0x55, 0x11, 0x83, 0xd0, 0xc7, 0x6e, 0x43}} }, + { (uint64_t)800000000000000000ull, {{0x58, 0x4a, 0x78, 0x93, 0x13, 0x7e, 0xbd, 0x2, 0x8b, 0xa7, 0x59, 0x82, 0xc3, 0x39, 0xb7, 0x66, 0xaa, 0xda, 0xad, 0xf9, 0x14, 0x50, 0xf9, 0x40, 0x7d, 0x2a, 0x97, 0xd7, 0xf6, 0xb1, 0x93, 0x5e}} }, + { (uint64_t)900000000000000000ull, {{0x7, 0xce, 0x54, 0xb1, 0x18, 0x26, 0xa1, 0x75, 0x23, 0x13, 0x55, 0x1a, 00, 0x20, 0xfd, 0x79, 0x8a, 00, 0x9e, 0x20, 0xcd, 0xb2, 0x40, 0x1d, 0x52, 0x51, 0xc1, 0x55, 0x8e, 0xea, 0xd2, 0x6c}} }, + { (uint64_t)1000000000000000000ull, {{0x39, 0x80, 0x7f, 0x3d, 0xce, 0xb0, 0xa6, 0xfe, 0x34, 0xa7, 0xa1, 0xed, 0xc6, 0x9b, 0x78, 0xff, 0xbe, 0xd5, 0xa7, 0x8c, 0x6c, 0x87, 0x5d, 0xda, 0x96, 0x69, 0xdb, 0xb2, 0x95, 0x70, 0xf0, 0xf4}} }, + { (uint64_t)2000000000000000000ull, {{0xda, 0x74, 00, 0x86, 0xf1, 0x5c, 0xe8, 0x21, 0xe9, 0xd, 0x50, 0xaf, 0xcf, 0x80, 0x9c, 0x7e, 0x18, 0x51, 0x90, 0x1b, 0xa3, 0x5f, 0x9f, 0x63, 0x78, 0xd6, 0x40, 0x7c, 0xb9, 0xc7, 0xa2, 0x75}} }, + { (uint64_t)3000000000000000000ull, {{0x7, 0xa1, 0x75, 0x63, 0xae, 0xf5, 0xcf, 0xd0, 0x36, 0xfa, 0x64, 0xd4, 0xb1, 0x97, 0xa9, 0x51, 0xc0, 0xd2, 0x87, 0x2b, 0xd, 0xb6, 0xf9, 0xbe, 0x47, 0xe6, 0x7c, 0xa6, 0xb5, 0x35, 0xe2, 0x6e}} }, + { (uint64_t)4000000000000000000ull, {{0xe3, 0x49, 0xf7, 0xeb, 0xe5, 0x11, 0x39, 0xfe, 0xd5, 0x69, 0x40, 0x37, 0xd1, 0x14, 0xb7, 0xbd, 0x45, 0xdd, 0xa, 0x6a, 0xf0, 0x4b, 0x62, 0xec, 0xa4, 0xd8, 0xcd, 0x55, 0x2a, 0x14, 0xe3, 0xfb}} }, + { (uint64_t)5000000000000000000ull, {{0x8d, 0x59, 0x7e, 0xa9, 0xf5, 0x79, 0x9a, 0x4d, 0x15, 0x3d, 0x82, 0xd6, 0xf7, 0xbe, 0xa0, 0x2e, 0x52, 0x40, 0xa2, 0xc8, 0x9b, 0x4, 0x1e, 0x6, 0x2f, 0x37, 0xbc, 0x7b, 0x82, 0xa0, 0xac, 0x55}} }, + { (uint64_t)6000000000000000000ull, {{0xa3, 0x43, 0xa7, 0xe1, 0x14, 0x4d, 0x33, 0x50, 0xf, 0x3e, 0xfd, 0x38, 0x15, 0x82, 0xdd, 0xc5, 0xd0, 0x18, 0x3e, 0x5d, 0xcf, 0x8a, 0xfa, 0x64, 0xbb, 0x67, 0x6c, 0x97, 0x3e, 0x3d, 0x1a, 0xb1}} }, + { (uint64_t)7000000000000000000ull, {{0x89, 0xe9, 0x3e, 0xe9, 0xf2, 0x4d, 0x72, 0x61, 0xe5, 0x44, 0xca, 0x8f, 0x9, 0xa7, 0x40, 0x4e, 0xe3, 0xa9, 0xe, 0xe2, 0x50, 0x7d, 0xda, 0xcf, 0x41, 0x2a, 0x58, 0xc, 0x9, 0x65, 0x1c, 0x53}} }, + { (uint64_t)8000000000000000000ull, {{0xc5, 0x94, 0x10, 0x81, 0x54, 0x69, 0xf4, 0x59, 0xd1, 0x5a, 0x6f, 0xe3, 0xf2, 0xa1, 0x1b, 0xa6, 0x31, 0x12, 0xfa, 0xaa, 0xc5, 0x3d, 0xbc, 0x52, 0x5d, 0x3c, 0xfa, 0xb1, 0xfa, 0x9c, 0x3d, 0xdb}} }, + { (uint64_t)9000000000000000000ull, {{0x9d, 0xe7, 0xcb, 0xb, 0x8d, 0x7b, 0xac, 0x47, 0xff, 0xd3, 0x93, 0x1b, 0xcd, 0x82, 0xcd, 0xd5, 0x35, 0xc, 0x29, 0x34, 0xb1, 0x6e, 0xb, 0x64, 0x32, 0xab, 0xf7, 0xcb, 0x4b, 0x5c, 0x37, 0x6d}} }, + { (uint64_t)10000000000000000000ull, {{0x65, 0x8d, 0x1, 0x37, 0x6d, 0x18, 0x63, 0xe7, 0x7b, 0x9, 0x6f, 0x98, 0xe6, 0xe5, 0x13, 0xc2, 0x4, 0x10, 0xf5, 0xc7, 0xfb, 0x18, 0xa6, 0xe5, 0x9a, 0x52, 0x66, 0x84, 0x5c, 0xd9, 0xb1, 0xe3}} }, +}; + namespace rct { //Various key initialization functions @@ -117,8 +295,7 @@ namespace rct { //generates C =aG + bH from b, a is given.. void genC(key & C, const key & a, xmr_amount amount) { - key bH = scalarmultH(d2h(amount)); - addKeys1(C, a, bH); + addKeys2(C, a, d2h(amount), rct::H); } //generates a <secret , public> / Pedersen commitment to the amount @@ -143,16 +320,22 @@ namespace rct { } key zeroCommit(xmr_amount amount) { + const zero_commitment *begin = zero_commitments; + const zero_commitment *end = zero_commitments + sizeof(zero_commitments) / sizeof(zero_commitments[0]); + const zero_commitment value{amount, rct::zero()}; + const auto it = std::lower_bound(begin, end, value, [](const zero_commitment &e0, const zero_commitment &e1){ return e0.amount < e1.amount; }); + if (it != end && it->amount == amount) + { + return it->commitment; + } key am = d2h(amount); key bH = scalarmultH(am); return addKeys(G, bH); } key commit(xmr_amount amount, const key &mask) { - key c = scalarmultBase(mask); - key am = d2h(amount); - key bH = scalarmultH(am); - addKeys(c, c, bH); + key c; + genC(c, mask, amount); return c; } @@ -487,18 +670,58 @@ namespace rct { //Elliptic Curve Diffie Helman: encodes and decodes the amount b and mask a // where C= aG + bH - void ecdhEncode(ecdhTuple & unmasked, const key & sharedSec) { - key sharedSec1 = hash_to_scalar(sharedSec); - key sharedSec2 = hash_to_scalar(sharedSec1); + static key ecdhHash(const key &k) + { + char data[38]; + rct::key hash; + memcpy(data, "amount", 6); + memcpy(data + 6, &k, sizeof(k)); + cn_fast_hash(hash, data, sizeof(data)); + return hash; + } + static void xor8(key &v, const key &k) + { + for (int i = 0; i < 8; ++i) + v.bytes[i] ^= k.bytes[i]; + } + key genCommitmentMask(const key &sk) + { + char data[15 + sizeof(key)]; + memcpy(data, "commitment_mask", 15); + memcpy(data + 15, &sk, sizeof(sk)); + key scalar; + hash_to_scalar(scalar, data, sizeof(data)); + return scalar; + } + + void ecdhEncode(ecdhTuple & unmasked, const key & sharedSec, bool v2) { //encode - sc_add(unmasked.mask.bytes, unmasked.mask.bytes, sharedSec1.bytes); - sc_add(unmasked.amount.bytes, unmasked.amount.bytes, sharedSec2.bytes); + if (v2) + { + unmasked.mask = zero(); + xor8(unmasked.amount, ecdhHash(sharedSec)); + } + else + { + key sharedSec1 = hash_to_scalar(sharedSec); + key sharedSec2 = hash_to_scalar(sharedSec1); + sc_add(unmasked.mask.bytes, unmasked.mask.bytes, sharedSec1.bytes); + sc_add(unmasked.amount.bytes, unmasked.amount.bytes, sharedSec2.bytes); + } } - void ecdhDecode(ecdhTuple & masked, const key & sharedSec) { - key sharedSec1 = hash_to_scalar(sharedSec); - key sharedSec2 = hash_to_scalar(sharedSec1); + void ecdhDecode(ecdhTuple & masked, const key & sharedSec, bool v2) { //decode - sc_sub(masked.mask.bytes, masked.mask.bytes, sharedSec1.bytes); - sc_sub(masked.amount.bytes, masked.amount.bytes, sharedSec2.bytes); + if (v2) + { + masked.mask = genCommitmentMask(sharedSec); + xor8(masked.amount, ecdhHash(sharedSec)); + } + else + { + key sharedSec1 = hash_to_scalar(sharedSec); + key sharedSec2 = hash_to_scalar(sharedSec1); + sc_sub(masked.mask.bytes, masked.mask.bytes, sharedSec1.bytes); + sc_sub(masked.amount.bytes, masked.amount.bytes, sharedSec2.bytes); + } } } diff --git a/src/ringct/rctOps.h b/src/ringct/rctOps.h index 60e920b3a..dd6d87593 100644 --- a/src/ringct/rctOps.h +++ b/src/ringct/rctOps.h @@ -182,7 +182,8 @@ namespace rct { //Elliptic Curve Diffie Helman: encodes and decodes the amount b and mask a // where C= aG + bH - void ecdhEncode(ecdhTuple & unmasked, const key & sharedSec); - void ecdhDecode(ecdhTuple & masked, const key & sharedSec); + key genCommitmentMask(const key &sk); + void ecdhEncode(ecdhTuple & unmasked, const key & sharedSec, bool v2); + void ecdhDecode(ecdhTuple & masked, const key & sharedSec, bool v2); } #endif /* RCTOPS_H */ diff --git a/src/ringct/rctSigs.cpp b/src/ringct/rctSigs.cpp index 0d1789a38..81bec487c 100644 --- a/src/ringct/rctSigs.cpp +++ b/src/ringct/rctSigs.cpp @@ -44,19 +44,47 @@ using namespace std; #define CHECK_AND_ASSERT_MES_L1(expr, ret, message) {if(!(expr)) {MCERROR("verify", message); return ret;}} -namespace rct { - Bulletproof proveRangeBulletproof(key &C, key &mask, uint64_t amount) +namespace +{ + rct::Bulletproof make_dummy_bulletproof(const std::vector<uint64_t> &outamounts, rct::keyV &C, rct::keyV &masks) { - mask = rct::skGen(); - Bulletproof proof = bulletproof_PROVE(amount, mask); - CHECK_AND_ASSERT_THROW_MES(proof.V.size() == 1, "V has not exactly one element"); - C = proof.V[0]; - return proof; + const size_t n_outs = outamounts.size(); + const rct::key I = rct::identity(); + size_t nrl = 0; + while ((1u << nrl) < n_outs) + ++nrl; + nrl += 6; + + C.resize(n_outs); + masks.resize(n_outs); + for (size_t i = 0; i < n_outs; ++i) + { + masks[i] = I; + rct::key sv8, sv; + sv = rct::zero(); + sv.bytes[0] = outamounts[i] & 255; + sv.bytes[1] = (outamounts[i] >> 8) & 255; + sv.bytes[2] = (outamounts[i] >> 16) & 255; + sv.bytes[3] = (outamounts[i] >> 24) & 255; + sv.bytes[4] = (outamounts[i] >> 32) & 255; + sv.bytes[5] = (outamounts[i] >> 40) & 255; + sv.bytes[6] = (outamounts[i] >> 48) & 255; + sv.bytes[7] = (outamounts[i] >> 56) & 255; + sc_mul(sv8.bytes, sv.bytes, rct::INV_EIGHT.bytes); + rct::addKeys2(C[i], rct::INV_EIGHT, sv8, rct::H); + } + + return rct::Bulletproof{rct::keyV(n_outs, I), I, I, I, I, I, I, rct::keyV(nrl, I), rct::keyV(nrl, I), I, I, I}; } +} - Bulletproof proveRangeBulletproof(keyV &C, keyV &masks, const std::vector<uint64_t> &amounts) +namespace rct { + Bulletproof proveRangeBulletproof(keyV &C, keyV &masks, const std::vector<uint64_t> &amounts, epee::span<const key> sk) { - masks = rct::skvGen(amounts.size()); + CHECK_AND_ASSERT_THROW_MES(amounts.size() == sk.size(), "Invalid amounts/sk sizes"); + masks.resize(amounts.size()); + for (size_t i = 0; i < masks.size(); ++i) + masks[i] = genCommitmentMask(sk[i]); Bulletproof proof = bulletproof_PROVE(amounts, masks); CHECK_AND_ASSERT_THROW_MES(proof.V.size() == amounts.size(), "V does not have the expected size"); C = proof.V; @@ -391,7 +419,7 @@ namespace rct { hashes.push_back(hash2rct(h)); keyV kv; - if (rv.type == RCTTypeBulletproof) + if (rv.type == RCTTypeBulletproof || rv.type == RCTTypeBulletproof2) { kv.reserve((6*2+9) * rv.p.bulletproofs.size()); for (const auto &p: rv.p.bulletproofs) @@ -440,7 +468,7 @@ namespace rct { // this shows that sum inputs = sum outputs //Ver: // verifies the above sig is created corretly - mgSig proveRctMG(const key &message, const ctkeyM & pubs, const ctkeyV & inSk, const ctkeyV &outSk, const ctkeyV & outPk, const multisig_kLRki *kLRki, key *mscout, unsigned int index, key txnFeeKey, hw::device &hwdev) { + mgSig proveRctMG(const key &message, const ctkeyM & pubs, const ctkeyV & inSk, const ctkeyV &outSk, const ctkeyV & outPk, const multisig_kLRki *kLRki, key *mscout, unsigned int index, const key &txnFeeKey, hw::device &hwdev) { mgSig mg; //setup vars size_t cols = pubs.size(); @@ -530,7 +558,7 @@ namespace rct { // this shows that sum inputs = sum outputs //Ver: // verifies the above sig is created corretly - bool verRctMG(const mgSig &mg, const ctkeyM & pubs, const ctkeyV & outPk, key txnFeeKey, const key &message) { + bool verRctMG(const mgSig &mg, const ctkeyM & pubs, const ctkeyV & outPk, const key &txnFeeKey, const key &message) { PERF_TIMER(verRctMG); //setup vars size_t cols = pubs.size(); @@ -580,10 +608,19 @@ namespace rct { keyV tmp(rows + 1); size_t i; keyM M(cols, tmp); + ge_p3 Cp3; + CHECK_AND_ASSERT_MES_L1(ge_frombytes_vartime(&Cp3, C.bytes) == 0, false, "point conv failed"); + ge_cached Ccached; + ge_p3_to_cached(&Ccached, &Cp3); + ge_p1p1 p1; //create the matrix to mg sig for (i = 0; i < cols; i++) { M[i][0] = pubs[i].dest; - subKeys(M[i][1], pubs[i].mask, C); + ge_p3 p3; + CHECK_AND_ASSERT_MES_L1(ge_frombytes_vartime(&p3, pubs[i].mask.bytes) == 0, false, "point conv failed"); + ge_sub(&p1, &p3, &Ccached); + ge_p1p1_to_p3(&p3, &p1); + ge_p3_tobytes(M[i][1].bytes, &p3); } //DP(C); return MLSAG_Ver(message, M, mg, rows); @@ -652,7 +689,7 @@ namespace rct { // must know the destination private key to find the correct amount, else will return a random number // Note: For txn fees, the last index in the amounts vector should contain that // Thus the amounts vector will be "one" longer than the destinations vectort - rctSig genRct(const key &message, const ctkeyV & inSk, const keyV & destinations, const vector<xmr_amount> & amounts, const ctkeyM &mixRing, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, unsigned int index, ctkeyV &outSk, hw::device &hwdev) { + rctSig genRct(const key &message, const ctkeyV & inSk, const keyV & destinations, const vector<xmr_amount> & amounts, const ctkeyM &mixRing, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, unsigned int index, ctkeyV &outSk, const RCTConfig &rct_config, hw::device &hwdev) { CHECK_AND_ASSERT_THROW_MES(amounts.size() == destinations.size() || amounts.size() == destinations.size() + 1, "Different number of amounts/destinations"); CHECK_AND_ASSERT_THROW_MES(amount_keys.size() == destinations.size(), "Different number of amount_keys/destinations"); CHECK_AND_ASSERT_THROW_MES(index < mixRing.size(), "Bad index into mixRing"); @@ -682,7 +719,7 @@ namespace rct { //mask amount and mask rv.ecdhInfo[i].mask = copy(outSk[i].mask); rv.ecdhInfo[i].amount = d2h(amounts[i]); - hwdev.ecdhEncode(rv.ecdhInfo[i], amount_keys[i]); + hwdev.ecdhEncode(rv.ecdhInfo[i], amount_keys[i], rv.type == RCTTypeBulletproof2); } //set txn fee @@ -703,18 +740,18 @@ namespace rct { return rv; } - rctSig genRct(const key &message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const vector<xmr_amount> & amounts, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, const int mixin, hw::device &hwdev) { + rctSig genRct(const key &message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const vector<xmr_amount> & amounts, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, const int mixin, const RCTConfig &rct_config, hw::device &hwdev) { unsigned int index; ctkeyM mixRing; ctkeyV outSk; tie(mixRing, index) = populateFromBlockchain(inPk, mixin); - return genRct(message, inSk, destinations, amounts, mixRing, amount_keys, kLRki, msout, index, outSk, hwdev); + return genRct(message, inSk, destinations, amounts, mixRing, amount_keys, kLRki, msout, index, outSk, rct_config, hwdev); } //RCT simple //for post-rct only - rctSig genRctSimple(const key &message, const ctkeyV & inSk, const keyV & destinations, const vector<xmr_amount> &inamounts, const vector<xmr_amount> &outamounts, xmr_amount txnFee, const ctkeyM & mixRing, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, const std::vector<unsigned int> & index, ctkeyV &outSk, RangeProofType range_proof_type, hw::device &hwdev) { - const bool bulletproof = range_proof_type != RangeProofBorromean; + rctSig genRctSimple(const key &message, const ctkeyV & inSk, const keyV & destinations, const vector<xmr_amount> &inamounts, const vector<xmr_amount> &outamounts, xmr_amount txnFee, const ctkeyM & mixRing, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, const std::vector<unsigned int> & index, ctkeyV &outSk, const RCTConfig &rct_config, hw::device &hwdev) { + const bool bulletproof = rct_config.range_proof_type != RangeProofBorromean; CHECK_AND_ASSERT_THROW_MES(inamounts.size() > 0, "Empty inamounts"); CHECK_AND_ASSERT_THROW_MES(inamounts.size() == inSk.size(), "Different number of inamounts/inSk"); CHECK_AND_ASSERT_THROW_MES(outamounts.size() == destinations.size(), "Different number of amounts/destinations"); @@ -730,7 +767,7 @@ namespace rct { } rctSig rv; - rv.type = bulletproof ? RCTTypeBulletproof : RCTTypeSimple; + rv.type = bulletproof ? (rct_config.bp_version == 0 || rct_config.bp_version >= 2 ? RCTTypeBulletproof2 : RCTTypeBulletproof) : RCTTypeSimple; rv.message = message; rv.outPk.resize(destinations.size()); if (!bulletproof) @@ -759,13 +796,22 @@ namespace rct { std::vector<uint64_t> proof_amounts; size_t n_amounts = outamounts.size(); size_t amounts_proved = 0; - if (range_proof_type == RangeProofPaddedBulletproof) + if (rct_config.range_proof_type == RangeProofPaddedBulletproof) { rct::keyV C, masks; - rv.p.bulletproofs.push_back(proveRangeBulletproof(C, masks, outamounts)); - #ifdef DBG - CHECK_AND_ASSERT_THROW_MES(verBulletproof(rv.p.bulletproofs.back()), "verBulletproof failed on newly created proof"); - #endif + if (hwdev.get_mode() == hw::device::TRANSACTION_CREATE_FAKE) + { + // use a fake bulletproof for speed + rv.p.bulletproofs.push_back(make_dummy_bulletproof(outamounts, C, masks)); + } + else + { + const epee::span<const key> keys{&amount_keys[0], amount_keys.size()}; + rv.p.bulletproofs.push_back(proveRangeBulletproof(C, masks, outamounts, keys)); + #ifdef DBG + CHECK_AND_ASSERT_THROW_MES(verBulletproof(rv.p.bulletproofs.back()), "verBulletproof failed on newly created proof"); + #endif + } for (i = 0; i < outamounts.size(); ++i) { rv.outPk[i].mask = rct::scalarmult8(C[i]); @@ -775,17 +821,26 @@ namespace rct { else while (amounts_proved < n_amounts) { size_t batch_size = 1; - if (range_proof_type == RangeProofMultiOutputBulletproof) + if (rct_config.range_proof_type == RangeProofMultiOutputBulletproof) while (batch_size * 2 + amounts_proved <= n_amounts && batch_size * 2 <= BULLETPROOF_MAX_OUTPUTS) batch_size *= 2; rct::keyV C, masks; std::vector<uint64_t> batch_amounts(batch_size); for (i = 0; i < batch_size; ++i) batch_amounts[i] = outamounts[i + amounts_proved]; - rv.p.bulletproofs.push_back(proveRangeBulletproof(C, masks, batch_amounts)); - #ifdef DBG - CHECK_AND_ASSERT_THROW_MES(verBulletproof(rv.p.bulletproofs.back()), "verBulletproof failed on newly created proof"); - #endif + if (hwdev.get_mode() == hw::device::TRANSACTION_CREATE_FAKE) + { + // use a fake bulletproof for speed + rv.p.bulletproofs.push_back(make_dummy_bulletproof(batch_amounts, C, masks)); + } + else + { + const epee::span<const key> keys{&amount_keys[amounts_proved], batch_size}; + rv.p.bulletproofs.push_back(proveRangeBulletproof(C, masks, batch_amounts, keys)); + #ifdef DBG + CHECK_AND_ASSERT_THROW_MES(verBulletproof(rv.p.bulletproofs.back()), "verBulletproof failed on newly created proof"); + #endif + } for (i = 0; i < batch_size; ++i) { rv.outPk[i + amounts_proved].mask = rct::scalarmult8(C[i]); @@ -803,7 +858,7 @@ namespace rct { //mask amount and mask rv.ecdhInfo[i].mask = copy(outSk[i].mask); rv.ecdhInfo[i].amount = d2h(outamounts[i]); - hwdev.ecdhEncode(rv.ecdhInfo[i], amount_keys[i]); + hwdev.ecdhEncode(rv.ecdhInfo[i], amount_keys[i], rv.type == RCTTypeBulletproof2); } //set txn fee @@ -821,7 +876,6 @@ namespace rct { sc_add(sumpouts.bytes, a[i].bytes, sumpouts.bytes); genC(pseudoOuts[i], a[i], inamounts[i]); } - rv.mixRing = mixRing; sc_sub(a[i].bytes, sumout.bytes, sumpouts.bytes); genC(pseudoOuts[i], a[i], inamounts[i]); DP(pseudoOuts[i]); @@ -835,7 +889,7 @@ namespace rct { return rv; } - rctSig genRctSimple(const key &message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const vector<xmr_amount> &inamounts, const vector<xmr_amount> &outamounts, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, xmr_amount txnFee, unsigned int mixin, hw::device &hwdev) { + rctSig genRctSimple(const key &message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const vector<xmr_amount> &inamounts, const vector<xmr_amount> &outamounts, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, xmr_amount txnFee, unsigned int mixin, const RCTConfig &rct_config, hw::device &hwdev) { std::vector<unsigned int> index; index.resize(inPk.size()); ctkeyM mixRing; @@ -845,7 +899,7 @@ namespace rct { mixRing[i].resize(mixin+1); index[i] = populateFromBlockchainSimple(mixRing[i], inPk[i], mixin); } - return genRctSimple(message, inSk, destinations, inamounts, outamounts, txnFee, mixRing, amount_keys, kLRki, msout, index, outSk, RangeProofBorromean, hwdev); + return genRctSimple(message, inSk, destinations, inamounts, outamounts, txnFee, mixRing, amount_keys, kLRki, msout, index, outSk, rct_config, hwdev); } //RingCT protocol @@ -935,7 +989,8 @@ namespace rct { { CHECK_AND_ASSERT_MES(rvp, false, "rctSig pointer is NULL"); const rctSig &rv = *rvp; - CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof, false, "verRctSemanticsSimple called on non simple rctSig"); + CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof || rv.type == RCTTypeBulletproof2, + false, "verRctSemanticsSimple called on non simple rctSig"); const bool bulletproof = is_rct_bulletproof(rv.type); if (bulletproof) { @@ -1034,7 +1089,8 @@ namespace rct { { PERF_TIMER(verRctNonSemanticsSimple); - CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof, false, "verRctNonSemanticsSimple called on non simple rctSig"); + CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof || rv.type == RCTTypeBulletproof2, + false, "verRctNonSemanticsSimple called on non simple rctSig"); const bool bulletproof = is_rct_bulletproof(rv.type); // semantics check is early, and mixRing/MGs aren't resolved yet if (bulletproof) @@ -1100,7 +1156,7 @@ namespace rct { //mask amount and mask ecdhTuple ecdh_info = rv.ecdhInfo[i]; - hwdev.ecdhDecode(ecdh_info, sk); + hwdev.ecdhDecode(ecdh_info, sk, rv.type == RCTTypeBulletproof2); mask = ecdh_info.mask; key amount = ecdh_info.amount; key C = rv.outPk[i].mask; @@ -1124,13 +1180,13 @@ namespace rct { } xmr_amount decodeRctSimple(const rctSig & rv, const key & sk, unsigned int i, key &mask, hw::device &hwdev) { - CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof, false, "decodeRct called on non simple rctSig"); + CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof || rv.type == RCTTypeBulletproof2, false, "decodeRct called on non simple rctSig"); CHECK_AND_ASSERT_THROW_MES(i < rv.ecdhInfo.size(), "Bad index"); CHECK_AND_ASSERT_THROW_MES(rv.outPk.size() == rv.ecdhInfo.size(), "Mismatched sizes of rv.outPk and rv.ecdhInfo"); //mask amount and mask ecdhTuple ecdh_info = rv.ecdhInfo[i]; - hwdev.ecdhDecode(ecdh_info, sk); + hwdev.ecdhDecode(ecdh_info, sk, rv.type == RCTTypeBulletproof2); mask = ecdh_info.mask; key amount = ecdh_info.amount; key C = rv.outPk[i].mask; @@ -1154,7 +1210,7 @@ namespace rct { } bool signMultisig(rctSig &rv, const std::vector<unsigned int> &indices, const keyV &k, const multisig_out &msout, const key &secret_key) { - CHECK_AND_ASSERT_MES(rv.type == RCTTypeFull || rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof, + CHECK_AND_ASSERT_MES(rv.type == RCTTypeFull || rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof || rv.type == RCTTypeBulletproof2, false, "unsupported rct type"); CHECK_AND_ASSERT_MES(indices.size() == k.size(), false, "Mismatched k/indices sizes"); CHECK_AND_ASSERT_MES(k.size() == rv.p.MGs.size(), false, "Mismatched k/MGs size"); diff --git a/src/ringct/rctSigs.h b/src/ringct/rctSigs.h index ae8bb91d7..9227eab1e 100644 --- a/src/ringct/rctSigs.h +++ b/src/ringct/rctSigs.h @@ -96,9 +96,9 @@ namespace rct { // this shows that sum inputs = sum outputs //Ver: // verifies the above sig is created corretly - mgSig proveRctMG(const ctkeyM & pubs, const ctkeyV & inSk, const keyV &outMasks, const ctkeyV & outPk, const multisig_kLRki *kLRki, key *mscout, unsigned int index, key txnFee, const key &message, hw::device &hwdev); + mgSig proveRctMG(const ctkeyM & pubs, const ctkeyV & inSk, const keyV &outMasks, const ctkeyV & outPk, const multisig_kLRki *kLRki, key *mscout, unsigned int index, const key &txnFee, const key &message, hw::device &hwdev); mgSig proveRctMGSimple(const key & message, const ctkeyV & pubs, const ctkey & inSk, const key &a , const key &Cout, const multisig_kLRki *kLRki, key *mscout, unsigned int index, hw::device &hwdev); - bool verRctMG(const mgSig &mg, const ctkeyM & pubs, const ctkeyV & outPk, key txnFee, const key &message); + bool verRctMG(const mgSig &mg, const ctkeyM & pubs, const ctkeyV & outPk, const key &txnFee, const key &message); bool verRctMGSimple(const key &message, const mgSig &mg, const ctkeyV & pubs, const key & C); //These functions get keys from blockchain @@ -119,10 +119,10 @@ namespace rct { //decodeRct: (c.f. https://eprint.iacr.org/2015/1098 section 5.1.1) // uses the attached ecdh info to find the amounts represented by each output commitment // must know the destination private key to find the correct amount, else will return a random number - rctSig genRct(const key &message, const ctkeyV & inSk, const keyV & destinations, const std::vector<xmr_amount> & amounts, const ctkeyM &mixRing, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, unsigned int index, ctkeyV &outSk, hw::device &hwdev); - rctSig genRct(const key &message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const std::vector<xmr_amount> & amounts, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, const int mixin, hw::device &hwdev); - rctSig genRctSimple(const key & message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const std::vector<xmr_amount> & inamounts, const std::vector<xmr_amount> & outamounts, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, xmr_amount txnFee, unsigned int mixin, hw::device &hwdev); - rctSig genRctSimple(const key & message, const ctkeyV & inSk, const keyV & destinations, const std::vector<xmr_amount> & inamounts, const std::vector<xmr_amount> & outamounts, xmr_amount txnFee, const ctkeyM & mixRing, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, const std::vector<unsigned int> & index, ctkeyV &outSk, RangeProofType range_proof_type, hw::device &hwdev); + rctSig genRct(const key &message, const ctkeyV & inSk, const keyV & destinations, const std::vector<xmr_amount> & amounts, const ctkeyM &mixRing, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, unsigned int index, ctkeyV &outSk, const RCTConfig &rct_config, hw::device &hwdev); + rctSig genRct(const key &message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const std::vector<xmr_amount> & amounts, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, const int mixin, const RCTConfig &rct_config, hw::device &hwdev); + rctSig genRctSimple(const key & message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const std::vector<xmr_amount> & inamounts, const std::vector<xmr_amount> & outamounts, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, xmr_amount txnFee, unsigned int mixin, const RCTConfig &rct_config, hw::device &hwdev); + rctSig genRctSimple(const key & message, const ctkeyV & inSk, const keyV & destinations, const std::vector<xmr_amount> & inamounts, const std::vector<xmr_amount> & outamounts, xmr_amount txnFee, const ctkeyM & mixRing, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, const std::vector<unsigned int> & index, ctkeyV &outSk, const RCTConfig &rct_config, hw::device &hwdev); bool verRct(const rctSig & rv, bool semantics); static inline bool verRct(const rctSig & rv) { return verRct(rv, true) && verRct(rv, false); } bool verRctSemanticsSimple(const rctSig & rv); @@ -133,7 +133,7 @@ namespace rct { xmr_amount decodeRct(const rctSig & rv, const key & sk, unsigned int i, hw::device &hwdev); xmr_amount decodeRctSimple(const rctSig & rv, const key & sk, unsigned int i, key & mask, hw::device &hwdev); xmr_amount decodeRctSimple(const rctSig & rv, const key & sk, unsigned int i, hw::device &hwdev); - + key get_pre_mlsag_hash(const rctSig &rv, hw::device &hwdev); bool signMultisig(rctSig &rv, const std::vector<unsigned int> &indices, const keyV &k, const multisig_out &msout, const key &secret_key); } #endif /* RCTSIGS_H */ diff --git a/src/ringct/rctTypes.cpp b/src/ringct/rctTypes.cpp index 90ed65df0..f01e683cb 100644 --- a/src/ringct/rctTypes.cpp +++ b/src/ringct/rctTypes.cpp @@ -217,6 +217,7 @@ namespace rct { { case RCTTypeSimple: case RCTTypeBulletproof: + case RCTTypeBulletproof2: return true; default: return false; @@ -228,6 +229,7 @@ namespace rct { switch (type) { case RCTTypeBulletproof: + case RCTTypeBulletproof2: return true; default: return false; diff --git a/src/ringct/rctTypes.h b/src/ringct/rctTypes.h index 18290637b..50d0f4d91 100644 --- a/src/ringct/rctTypes.h +++ b/src/ringct/rctTypes.h @@ -120,17 +120,14 @@ namespace rct { // If the pedersen commitment to an amount is C = aG + bH, // "mask" contains a 32 byte key a // "amount" contains a hex representation (in 32 bytes) of a 64 bit number - // "senderPk" is not the senders actual public key, but a one-time public key generated for // the purpose of the ECDH exchange struct ecdhTuple { key mask; key amount; - key senderPk; BEGIN_SERIALIZE_OBJECT() - FIELD(mask) + FIELD(mask) // not saved from v2 BPs FIELD(amount) - // FIELD(senderPk) // not serialized, as we do not use it in monero currently END_SERIALIZE() }; @@ -187,7 +184,8 @@ namespace rct { rct::keyV L, R; rct::key a, b, t; - Bulletproof() {} + Bulletproof(): + A({}), S({}), T1({}), T2({}), taux({}), mu({}), a({}), b({}), t({}) {} Bulletproof(const rct::key &V, const rct::key &A, const rct::key &S, const rct::key &T1, const rct::key &T2, const rct::key &taux, const rct::key &mu, const rct::keyV &L, const rct::keyV &R, const rct::key &a, const rct::key &b, const rct::key &t): V({V}), A(A), S(S), T1(T1), T2(T2), taux(taux), mu(mu), L(L), R(R), a(a), b(b), t(t) {} Bulletproof(const rct::keyV &V, const rct::key &A, const rct::key &S, const rct::key &T1, const rct::key &T2, const rct::key &taux, const rct::key &mu, const rct::keyV &L, const rct::keyV &R, const rct::key &a, const rct::key &b, const rct::key &t): @@ -230,8 +228,13 @@ namespace rct { RCTTypeFull = 1, RCTTypeSimple = 2, RCTTypeBulletproof = 3, + RCTTypeBulletproof2 = 4, }; enum RangeProofType { RangeProofBorromean, RangeProofBulletproof, RangeProofMultiOutputBulletproof, RangeProofPaddedBulletproof }; + struct RCTConfig { + RangeProofType range_proof_type; + int bp_version; + }; struct rctSigBase { uint8_t type; key message; @@ -248,7 +251,7 @@ namespace rct { FIELD(type) if (type == RCTTypeNull) return true; - if (type != RCTTypeFull && type != RCTTypeSimple && type != RCTTypeBulletproof) + if (type != RCTTypeFull && type != RCTTypeSimple && type != RCTTypeBulletproof && type != RCTTypeBulletproof2) return false; VARINT_FIELD(txnFee) // inputs/outputs not saved, only here for serialization help @@ -277,7 +280,19 @@ namespace rct { return false; for (size_t i = 0; i < outputs; ++i) { - FIELDS(ecdhInfo[i]) + if (type == RCTTypeBulletproof2) + { + ar.begin_object(); + if (!typename Archive<W>::is_saving()) + memset(ecdhInfo[i].amount.bytes, 0, sizeof(ecdhInfo[i].amount.bytes)); + crypto::hash8 &amount = (crypto::hash8&)ecdhInfo[i].amount; + FIELD(amount); + ar.end_object(); + } + else + { + FIELDS(ecdhInfo[i]) + } if (outputs - i > 1) ar.delimit_array(); } @@ -309,12 +324,15 @@ namespace rct { { if (type == RCTTypeNull) return true; - if (type != RCTTypeFull && type != RCTTypeSimple && type != RCTTypeBulletproof) + if (type != RCTTypeFull && type != RCTTypeSimple && type != RCTTypeBulletproof && type != RCTTypeBulletproof2) return false; - if (type == RCTTypeBulletproof) + if (type == RCTTypeBulletproof || type == RCTTypeBulletproof2) { uint32_t nbp = bulletproofs.size(); - FIELD(nbp) + if (type == RCTTypeBulletproof2) + VARINT_FIELD(nbp) + else + FIELD(nbp) ar.tag("bp"); ar.begin_array(); if (nbp > outputs) @@ -350,7 +368,7 @@ namespace rct { ar.begin_array(); // we keep a byte for size of MGs, because we don't know whether this is // a simple or full rct signature, and it's starting to annoy the hell out of me - size_t mg_elements = (type == RCTTypeSimple || type == RCTTypeBulletproof) ? inputs : 1; + size_t mg_elements = (type == RCTTypeSimple || type == RCTTypeBulletproof || type == RCTTypeBulletproof2) ? inputs : 1; PREPARE_CUSTOM_VECTOR_SERIALIZATION(mg_elements, MGs); if (MGs.size() != mg_elements) return false; @@ -368,7 +386,7 @@ namespace rct { for (size_t j = 0; j < mixin + 1; ++j) { ar.begin_array(); - size_t mg_ss2_elements = ((type == RCTTypeSimple || type == RCTTypeBulletproof) ? 1 : inputs) + 1; + size_t mg_ss2_elements = ((type == RCTTypeSimple || type == RCTTypeBulletproof || type == RCTTypeBulletproof2) ? 1 : inputs) + 1; PREPARE_CUSTOM_VECTOR_SERIALIZATION(mg_ss2_elements, MGs[i].ss[j]); if (MGs[i].ss[j].size() != mg_ss2_elements) return false; @@ -394,7 +412,7 @@ namespace rct { ar.delimit_array(); } ar.end_array(); - if (type == RCTTypeBulletproof) + if (type == RCTTypeBulletproof || type == RCTTypeBulletproof2) { ar.tag("pseudoOuts"); ar.begin_array(); @@ -418,12 +436,12 @@ namespace rct { keyV& get_pseudo_outs() { - return type == RCTTypeBulletproof ? p.pseudoOuts : pseudoOuts; + return type == RCTTypeBulletproof || type == RCTTypeBulletproof2 ? p.pseudoOuts : pseudoOuts; } keyV const& get_pseudo_outs() const { - return type == RCTTypeBulletproof ? p.pseudoOuts : pseudoOuts; + return type == RCTTypeBulletproof || type == RCTTypeBulletproof2 ? p.pseudoOuts : pseudoOuts; } }; diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt index 4f8f96524..d2c4a33cb 100644 --- a/src/rpc/CMakeLists.txt +++ b/src/rpc/CMakeLists.txt @@ -31,6 +31,7 @@ set(rpc_base_sources set(rpc_sources core_rpc_server.cpp + rpc_handler.cpp instanciations) set(daemon_messages_sources @@ -45,7 +46,8 @@ set(daemon_rpc_server_sources set(rpc_base_headers rpc_args.h) -set(rpc_headers) +set(rpc_headers + rpc_handler.cpp) set(daemon_rpc_server_headers) @@ -63,7 +65,6 @@ set(daemon_rpc_server_private_headers message.h daemon_messages.h daemon_handler.h - rpc_handler.h zmq_server.h) @@ -111,6 +112,7 @@ target_link_libraries(rpc common cryptonote_core cryptonote_protocol + version ${Boost_REGEX_LIBRARY} ${Boost_THREAD_LIBRARY} PRIVATE @@ -120,6 +122,7 @@ target_link_libraries(daemon_messages LINK_PRIVATE cryptonote_core cryptonote_protocol + version serialization ${EXTRA_LIBRARIES}) @@ -128,6 +131,7 @@ target_link_libraries(daemon_rpc_server rpc cryptonote_core cryptonote_protocol + version daemon_messages serialization ${Boost_CHRONO_LIBRARY} diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 55ee66a79..08674a06e 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -45,6 +45,7 @@ using namespace epee; #include "storages/http_abstract_invoke.h" #include "crypto/hash.h" #include "rpc/rpc_args.h" +#include "rpc/rpc_handler.h" #include "core_rpc_server_error_codes.h" #include "p2p/net_node.h" #include "version.h" @@ -90,12 +91,10 @@ namespace cryptonote bool core_rpc_server::init( const boost::program_options::variables_map& vm , const bool restricted - , const network_type nettype , const std::string& port ) { m_restricted = restricted; - m_nettype = nettype; m_net_server.set_threads_prefix("RPC"); auto rpc_config = cryptonote::rpc_args::process(vm); @@ -183,17 +182,19 @@ namespace cryptonote res.target = m_core.get_blockchain_storage().get_difficulty_target(); res.tx_count = m_core.get_blockchain_storage().get_total_transactions() - res.height; //without coinbase res.tx_pool_size = m_core.get_pool_transactions_count(); - res.alt_blocks_count = m_core.get_blockchain_storage().get_alternative_blocks_count(); - uint64_t total_conn = m_p2p.get_connections_count(); - res.outgoing_connections_count = m_p2p.get_outgoing_connections_count(); - res.incoming_connections_count = total_conn - res.outgoing_connections_count; - res.rpc_connections_count = get_connections_count(); - res.white_peerlist_size = m_p2p.get_peerlist_manager().get_white_peers_count(); - res.grey_peerlist_size = m_p2p.get_peerlist_manager().get_gray_peers_count(); - res.mainnet = m_nettype == MAINNET; - res.testnet = m_nettype == TESTNET; - res.stagenet = m_nettype == STAGENET; - res.nettype = m_nettype == MAINNET ? "mainnet" : m_nettype == TESTNET ? "testnet" : m_nettype == STAGENET ? "stagenet" : "fakechain"; + res.alt_blocks_count = m_restricted ? 0 : m_core.get_blockchain_storage().get_alternative_blocks_count(); + uint64_t total_conn = m_restricted ? 0 : m_p2p.get_connections_count(); + res.outgoing_connections_count = m_restricted ? 0 : m_p2p.get_outgoing_connections_count(); + res.incoming_connections_count = m_restricted ? 0 : (total_conn - res.outgoing_connections_count); + res.rpc_connections_count = m_restricted ? 0 : get_connections_count(); + res.white_peerlist_size = m_restricted ? 0 : m_p2p.get_peerlist_manager().get_white_peers_count(); + res.grey_peerlist_size = m_restricted ? 0 : m_p2p.get_peerlist_manager().get_gray_peers_count(); + + cryptonote::network_type net_type = nettype(); + res.mainnet = net_type == MAINNET; + res.testnet = net_type == TESTNET; + res.stagenet = net_type == STAGENET; + res.nettype = net_type == MAINNET ? "mainnet" : net_type == TESTNET ? "testnet" : net_type == STAGENET ? "stagenet" : "fakechain"; res.cumulative_difficulty = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.height - 1); res.block_size_limit = res.block_weight_limit = m_core.get_blockchain_storage().get_current_cumulative_block_weight_limit(); res.block_size_median = res.block_weight_median = m_core.get_blockchain_storage().get_current_cumulative_block_weight_median(); @@ -201,34 +202,30 @@ namespace cryptonote res.start_time = m_restricted ? 0 : (uint64_t)m_core.get_start_time(); res.free_space = m_restricted ? std::numeric_limits<uint64_t>::max() : m_core.get_free_space(); res.offline = m_core.offline(); - res.bootstrap_daemon_address = m_bootstrap_daemon_address; - res.height_without_bootstrap = res.height; + res.bootstrap_daemon_address = m_restricted ? "" : m_bootstrap_daemon_address; + res.height_without_bootstrap = m_restricted ? 0 : res.height; + if (m_restricted) + res.was_bootstrap_ever_used = false; + else { boost::shared_lock<boost::shared_mutex> lock(m_bootstrap_daemon_mutex); res.was_bootstrap_ever_used = m_was_bootstrap_ever_used; } - res.database_size = m_core.get_blockchain_storage().get_db().get_database_size(); - res.update_available = m_core.is_update_available(); + res.database_size = m_restricted ? 0 : m_core.get_blockchain_storage().get_db().get_database_size(); + res.update_available = m_restricted ? false : m_core.is_update_available(); + res.version = m_restricted ? "" : MONERO_VERSION; return true; } //------------------------------------------------------------------------------------------------------------------------------ - static cryptonote::blobdata get_pruned_tx_blob(cryptonote::transaction &tx) - { - std::stringstream ss; - binary_archive<true> ba(ss); - bool r = tx.serialize_base(ba); - CHECK_AND_ASSERT_MES(r, cryptonote::blobdata(), "Failed to serialize rct signatures base"); - return ss.str(); - } - //------------------------------------------------------------------------------------------------------------------------------ - static cryptonote::blobdata get_pruned_tx_json(cryptonote::transaction &tx) - { - std::stringstream ss; - json_archive<true> ar(ss); - bool r = tx.serialize_base(ar); - CHECK_AND_ASSERT_MES(r, cryptonote::blobdata(), "Failed to serialize rct signatures base"); - return ss.str(); - } + class pruned_transaction { + transaction& tx; + public: + pruned_transaction(transaction& tx) : tx(tx) {} + BEGIN_SERIALIZE_OBJECT() + bool r = tx.serialize_base(ar); + if (!r) return false; + END_SERIALIZE() + }; //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_get_blocks(const COMMAND_RPC_GET_BLOCKS_FAST::request& req, COMMAND_RPC_GET_BLOCKS_FAST::response& res) { @@ -255,19 +252,11 @@ namespace cryptonote pruned_size += bd.first.first.size(); unpruned_size += bd.first.first.size(); res.output_indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices()); - res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices()); - if (!req.no_miner_tx) - { - bool r = m_core.get_tx_outputs_gindexs(bd.first.second, res.output_indices.back().indices.back().indices); - if (!r) - { - res.status = "Failed"; - return false; - } - } ntxes += bd.second.size(); + res.output_indices.back().indices.reserve(1 + bd.second.size()); + if (req.no_miner_tx) + res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices()); res.blocks.back().txs.reserve(bd.second.size()); - res.output_indices.back().indices.reserve(bd.second.size()); for (std::vector<std::pair<crypto::hash, cryptonote::blobdata>>::iterator i = bd.second.begin(); i != bd.second.end(); ++i) { unpruned_size += i->second.size(); @@ -275,14 +264,25 @@ namespace cryptonote i->second.clear(); i->second.shrink_to_fit(); pruned_size += res.blocks.back().txs.back().size(); + } - res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices()); - bool r = m_core.get_tx_outputs_gindexs(i->first, res.output_indices.back().indices.back().indices); + const size_t n_txes_to_lookup = bd.second.size() + (req.no_miner_tx ? 0 : 1); + if (n_txes_to_lookup > 0) + { + std::vector<std::vector<uint64_t>> indices; + bool r = m_core.get_tx_outputs_gindexs(req.no_miner_tx ? bd.second.front().first : bd.first.second, n_txes_to_lookup, indices); if (!r) { res.status = "Failed"; return false; } + if (indices.size() != n_txes_to_lookup || res.output_indices.back().indices.size() != (req.no_miner_tx ? 1 : 0)) + { + res.status = "Failed"; + return false; + } + for (size_t i = 0; i < indices.size(); ++i) + res.output_indices.back().indices.push_back({std::move(indices[i])}); } } @@ -485,8 +485,8 @@ namespace cryptonote vh.push_back(*reinterpret_cast<const crypto::hash*>(b.data())); } std::vector<crypto::hash> missed_txs; - std::vector<transaction> txs; - bool r = m_core.get_transactions(vh, txs, missed_txs); + std::vector<std::tuple<crypto::hash, cryptonote::blobdata, crypto::hash, cryptonote::blobdata>> txs; + bool r = m_core.get_split_transactions_blobs(vh, txs, missed_txs); if(!r) { res.status = "Failed"; @@ -506,7 +506,7 @@ namespace cryptonote if(r) { // sort to match original request - std::vector<transaction> sorted_txs; + std::vector<std::tuple<crypto::hash, cryptonote::blobdata, crypto::hash, cryptonote::blobdata>> sorted_txs; std::vector<tx_info>::const_iterator i; unsigned txs_processed = 0; for (const crypto::hash &h: vh) @@ -519,7 +519,7 @@ namespace cryptonote return true; } // core returns the ones it finds in the right order - if (get_transaction_hash(txs[txs_processed]) != h) + if (std::get<0>(txs[txs_processed]) != h) { res.status = "Failed: tx hash mismatch"; return true; @@ -535,7 +535,16 @@ namespace cryptonote res.status = "Failed to parse and validate tx from blob"; return true; } - sorted_txs.push_back(tx); + std::stringstream ss; + binary_archive<true> ba(ss); + bool r = const_cast<cryptonote::transaction&>(tx).serialize_base(ba); + if (!r) + { + res.status = "Failed to serialize transaction base"; + return true; + } + const cryptonote::blobdata pruned = ss.str(); + sorted_txs.push_back(std::make_tuple(h, pruned, get_transaction_prunable_hash(tx), std::string(i->tx_blob, pruned.size()))); missed_txs.erase(std::find(missed_txs.begin(), missed_txs.end(), h)); pool_tx_hashes.insert(h); const std::string hash_string = epee::string_tools::pod_to_hex(h); @@ -564,10 +573,36 @@ namespace cryptonote crypto::hash tx_hash = *vhi++; e.tx_hash = *txhi++; - blobdata blob = req.prune ? get_pruned_tx_blob(tx) : t_serializable_object_to_blob(tx); - e.as_hex = string_tools::buff_to_hex_nodelimer(blob); - if (req.decode_as_json) - e.as_json = req.prune ? get_pruned_tx_json(tx) : obj_to_json_str(tx); + e.prunable_hash = epee::string_tools::pod_to_hex(std::get<2>(tx)); + if (req.split || req.prune || std::get<3>(tx).empty()) + { + e.pruned_as_hex = string_tools::buff_to_hex_nodelimer(std::get<1>(tx)); + if (!req.prune) + e.prunable_as_hex = string_tools::buff_to_hex_nodelimer(std::get<3>(tx)); + } + else + { + cryptonote::blobdata tx_data; + if (req.prune) + tx_data = std::get<1>(tx); + else + tx_data = std::get<1>(tx) + std::get<3>(tx); + e.as_hex = string_tools::buff_to_hex_nodelimer(tx_data); + if (req.decode_as_json && !tx_data.empty()) + { + cryptonote::transaction t; + if (cryptonote::parse_and_validate_tx_from_blob(tx_data, t)) + { + if (req.prune) + { + pruned_transaction pruned_tx{t}; + e.as_json = obj_to_json_str(pruned_tx); + } + else + e.as_json = obj_to_json_str(t); + } + } + } e.in_pool = pool_tx_hashes.find(tx_hash) != pool_tx_hashes.end(); if (e.in_pool) { @@ -705,31 +740,31 @@ namespace cryptonote if(!m_core.handle_incoming_tx(tx_blob, tvc, false, false, req.do_not_relay) || tvc.m_verifivation_failed) { res.status = "Failed"; - res.reason = ""; + std::string reason = ""; if ((res.low_mixin = tvc.m_low_mixin)) - add_reason(res.reason, "ring size too small"); + add_reason(reason, "bad ring size"); if ((res.double_spend = tvc.m_double_spend)) - add_reason(res.reason, "double spend"); + add_reason(reason, "double spend"); if ((res.invalid_input = tvc.m_invalid_input)) - add_reason(res.reason, "invalid input"); + add_reason(reason, "invalid input"); if ((res.invalid_output = tvc.m_invalid_output)) - add_reason(res.reason, "invalid output"); + add_reason(reason, "invalid output"); if ((res.too_big = tvc.m_too_big)) - add_reason(res.reason, "too big"); + add_reason(reason, "too big"); if ((res.overspend = tvc.m_overspend)) - add_reason(res.reason, "overspend"); + add_reason(reason, "overspend"); if ((res.fee_too_low = tvc.m_fee_too_low)) - add_reason(res.reason, "fee too low"); + add_reason(reason, "fee too low"); if ((res.not_rct = tvc.m_not_rct)) - add_reason(res.reason, "tx is not ringct"); - const std::string punctuation = res.reason.empty() ? "" : ": "; + add_reason(reason, "tx is not ringct"); + const std::string punctuation = reason.empty() ? "" : ": "; if (tvc.m_verifivation_failed) { - LOG_PRINT_L0("[on_send_raw_tx]: tx verification failed" << punctuation << res.reason); + LOG_PRINT_L0("[on_send_raw_tx]: tx verification failed" << punctuation << reason); } else { - LOG_PRINT_L0("[on_send_raw_tx]: Failed to process tx" << punctuation << res.reason); + LOG_PRINT_L0("[on_send_raw_tx]: Failed to process tx" << punctuation << reason); } return true; } @@ -756,7 +791,7 @@ namespace cryptonote PERF_TIMER(on_start_mining); CHECK_CORE_READY(); cryptonote::address_parse_info info; - if(!get_account_address_from_str(info, m_nettype, req.miner_address)) + if(!get_account_address_from_str(info, nettype(), req.miner_address)) { res.status = "Failed, wrong address"; LOG_PRINT_L0(res.status); @@ -808,7 +843,14 @@ namespace cryptonote bool core_rpc_server::on_stop_mining(const COMMAND_RPC_STOP_MINING::request& req, COMMAND_RPC_STOP_MINING::response& res) { PERF_TIMER(on_stop_mining); - if(!m_core.get_miner().stop()) + cryptonote::miner &miner= m_core.get_miner(); + if(!miner.is_mining()) + { + res.status = "Mining never started"; + LOG_PRINT_L0(res.status); + return true; + } + if(!miner.stop()) { res.status = "Failed, mining not stopped"; LOG_PRINT_L0(res.status); @@ -830,7 +872,7 @@ namespace cryptonote res.speed = lMiner.get_speed(); res.threads_count = lMiner.get_threads_count(); const account_public_address& lMiningAdr = lMiner.get_mining_address(); - res.address = get_account_address_as_str(m_nettype, false, lMiningAdr); + res.address = get_account_address_as_str(nettype(), false, lMiningAdr); } res.status = CORE_RPC_STATUS_OK; @@ -852,27 +894,28 @@ namespace cryptonote bool core_rpc_server::on_get_peer_list(const COMMAND_RPC_GET_PEER_LIST::request& req, COMMAND_RPC_GET_PEER_LIST::response& res) { PERF_TIMER(on_get_peer_list); - std::list<nodetool::peerlist_entry> white_list; - std::list<nodetool::peerlist_entry> gray_list; + std::vector<nodetool::peerlist_entry> white_list; + std::vector<nodetool::peerlist_entry> gray_list; m_p2p.get_peerlist_manager().get_peerlist_full(gray_list, white_list); - + res.white_list.reserve(white_list.size()); for (auto & entry : white_list) { if (entry.adr.get_type_id() == epee::net_utils::ipv4_network_address::ID) res.white_list.emplace_back(entry.id, entry.adr.as<epee::net_utils::ipv4_network_address>().ip(), - entry.adr.as<epee::net_utils::ipv4_network_address>().port(), entry.last_seen); + entry.adr.as<epee::net_utils::ipv4_network_address>().port(), entry.last_seen, entry.pruning_seed); else - res.white_list.emplace_back(entry.id, entry.adr.str(), entry.last_seen); + res.white_list.emplace_back(entry.id, entry.adr.str(), entry.last_seen, entry.pruning_seed); } + res.gray_list.reserve(gray_list.size()); for (auto & entry : gray_list) { if (entry.adr.get_type_id() == epee::net_utils::ipv4_network_address::ID) res.gray_list.emplace_back(entry.id, entry.adr.as<epee::net_utils::ipv4_network_address>().ip(), - entry.adr.as<epee::net_utils::ipv4_network_address>().port(), entry.last_seen); + entry.adr.as<epee::net_utils::ipv4_network_address>().port(), entry.last_seen, entry.pruning_seed); else - res.gray_list.emplace_back(entry.id, entry.adr.str(), entry.last_seen); + res.gray_list.emplace_back(entry.id, entry.adr.str(), entry.last_seen, entry.pruning_seed); } res.status = CORE_RPC_STATUS_OK; @@ -1017,7 +1060,7 @@ namespace cryptonote if(m_core.get_current_blockchain_height() <= h) { error_resp.code = CORE_RPC_ERROR_CODE_TOO_BIG_HEIGHT; - error_resp.message = std::string("Too big height: ") + std::to_string(h) + ", current blockchain height = " + std::to_string(m_core.get_current_blockchain_height()); + error_resp.message = std::string("Requested block height: ") + std::to_string(h) + " greater than current top block height: " + std::to_string(m_core.get_current_blockchain_height() - 1); } res = string_tools::pod_to_hex(m_core.get_block_id_by_height(h)); return true; @@ -1025,7 +1068,7 @@ namespace cryptonote //------------------------------------------------------------------------------------------------------------------------------ // equivalent of strstr, but with arbitrary bytes (ie, NULs) // This does not differentiate between "not found" and "found at offset 0" - uint64_t slow_memmem(const void* start_buff, size_t buflen,const void* pat,size_t patlen) + size_t slow_memmem(const void* start_buff, size_t buflen,const void* pat,size_t patlen) { const void* buf = start_buff; const void* end=(const char*)buf+buflen; @@ -1063,7 +1106,7 @@ namespace cryptonote cryptonote::address_parse_info info; - if(!req.wallet_address.size() || !cryptonote::get_account_address_from_str(info, m_nettype, req.wallet_address)) + if(!req.wallet_address.size() || !cryptonote::get_account_address_from_str(info, nettype(), req.wallet_address)) { error_resp.code = CORE_RPC_ERROR_CODE_WRONG_WALLET_ADDRESS; error_resp.message = "Failed to parse wallet address"; @@ -1076,7 +1119,7 @@ namespace cryptonote return false; } - block b = AUTO_VAL_INIT(b); + block b; cryptonote::blobdata blob_reserve; blob_reserve.resize(req.reserve_size, 0); if(!m_core.get_block_template(b, info.address, res.difficulty, res.height, res.expected_reward, blob_reserve)) @@ -1147,7 +1190,7 @@ namespace cryptonote // Fixing of high orphan issue for most pools // Thanks Boolberry! - block b = AUTO_VAL_INIT(b); + block b; if(!parse_and_validate_block_from_blob(blockblob, b)) { error_resp.code = CORE_RPC_ERROR_CODE_WRONG_BLOCKBLOB; @@ -1215,7 +1258,7 @@ namespace cryptonote error_resp.message = "Wrong block blob"; return false; } - block b = AUTO_VAL_INIT(b); + block b; if(!parse_and_validate_block_from_blob(blockblob, b)) { error_resp.code = CORE_RPC_ERROR_CODE_WRONG_BLOCKBLOB; @@ -1464,7 +1507,7 @@ namespace cryptonote if(m_core.get_current_blockchain_height() <= req.height) { error_resp.code = CORE_RPC_ERROR_CODE_TOO_BIG_HEIGHT; - error_resp.message = std::string("Too big height: ") + std::to_string(req.height) + ", current blockchain height = " + std::to_string(m_core.get_current_blockchain_height()); + error_resp.message = std::string("Requested block height: ") + std::to_string(req.height) + " greater than current top block height: " + std::to_string(m_core.get_current_blockchain_height() - 1); return false; } crypto::hash block_hash = m_core.get_block_id_by_height(req.height); @@ -1509,7 +1552,7 @@ namespace cryptonote if(m_core.get_current_blockchain_height() <= req.height) { error_resp.code = CORE_RPC_ERROR_CODE_TOO_BIG_HEIGHT; - error_resp.message = std::string("Too big height: ") + std::to_string(req.height) + ", current blockchain height = " + std::to_string(m_core.get_current_blockchain_height()); + error_resp.message = std::string("Requested block height: ") + std::to_string(req.height) + " greater than current top block height: " + std::to_string(m_core.get_current_blockchain_height() - 1); return false; } block_hash = m_core.get_block_id_by_height(req.height); @@ -1582,32 +1625,39 @@ namespace cryptonote res.target = m_core.get_blockchain_storage().get_current_hard_fork_version() < 2 ? DIFFICULTY_TARGET_V1 : DIFFICULTY_TARGET_V2; res.tx_count = m_core.get_blockchain_storage().get_total_transactions() - res.height; //without coinbase res.tx_pool_size = m_core.get_pool_transactions_count(); - res.alt_blocks_count = m_core.get_blockchain_storage().get_alternative_blocks_count(); - uint64_t total_conn = m_p2p.get_connections_count(); - res.outgoing_connections_count = m_p2p.get_outgoing_connections_count(); - res.incoming_connections_count = total_conn - res.outgoing_connections_count; - res.rpc_connections_count = get_connections_count(); - res.white_peerlist_size = m_p2p.get_peerlist_manager().get_white_peers_count(); - res.grey_peerlist_size = m_p2p.get_peerlist_manager().get_gray_peers_count(); - res.mainnet = m_nettype == MAINNET; - res.testnet = m_nettype == TESTNET; - res.stagenet = m_nettype == STAGENET; - res.nettype = m_nettype == MAINNET ? "mainnet" : m_nettype == TESTNET ? "testnet" : m_nettype == STAGENET ? "stagenet" : "fakechain"; + res.alt_blocks_count = m_restricted ? 0 : m_core.get_blockchain_storage().get_alternative_blocks_count(); + uint64_t total_conn = m_restricted ? 0 : m_p2p.get_connections_count(); + res.outgoing_connections_count = m_restricted ? 0 : m_p2p.get_outgoing_connections_count(); + res.incoming_connections_count = m_restricted ? 0 : (total_conn - res.outgoing_connections_count); + res.rpc_connections_count = m_restricted ? 0 : get_connections_count(); + res.white_peerlist_size = m_restricted ? 0 : m_p2p.get_peerlist_manager().get_white_peers_count(); + res.grey_peerlist_size = m_restricted ? 0 : m_p2p.get_peerlist_manager().get_gray_peers_count(); + + cryptonote::network_type net_type = nettype(); + res.mainnet = net_type == MAINNET; + res.testnet = net_type == TESTNET; + res.stagenet = net_type == STAGENET; + res.nettype = net_type == MAINNET ? "mainnet" : net_type == TESTNET ? "testnet" : net_type == STAGENET ? "stagenet" : "fakechain"; + res.cumulative_difficulty = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.height - 1); res.block_size_limit = res.block_weight_limit = m_core.get_blockchain_storage().get_current_cumulative_block_weight_limit(); res.block_size_median = res.block_weight_median = m_core.get_blockchain_storage().get_current_cumulative_block_weight_median(); res.status = CORE_RPC_STATUS_OK; - res.start_time = (uint64_t)m_core.get_start_time(); + res.start_time = m_restricted ? 0 : (uint64_t)m_core.get_start_time(); res.free_space = m_restricted ? std::numeric_limits<uint64_t>::max() : m_core.get_free_space(); res.offline = m_core.offline(); - res.bootstrap_daemon_address = m_bootstrap_daemon_address; - res.height_without_bootstrap = res.height; + res.bootstrap_daemon_address = m_restricted ? "" : m_bootstrap_daemon_address; + res.height_without_bootstrap = m_restricted ? 0 : res.height; + if (m_restricted) + res.was_bootstrap_ever_used = false; + else { boost::shared_lock<boost::shared_mutex> lock(m_bootstrap_daemon_mutex); res.was_bootstrap_ever_used = m_was_bootstrap_ever_used; } - res.database_size = m_core.get_blockchain_storage().get_db().get_database_size(); - res.update_available = m_core.is_update_available(); + res.database_size = m_restricted ? 0 : m_core.get_blockchain_storage().get_db().get_database_size(); + res.update_available = m_restricted ? false : m_core.is_update_available(); + res.version = m_restricted ? "" : MONERO_VERSION; return true; } //------------------------------------------------------------------------------------------------------------------------------ @@ -2020,6 +2070,18 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_pop_blocks(const COMMAND_RPC_POP_BLOCKS::request& req, COMMAND_RPC_POP_BLOCKS::response& res) + { + PERF_TIMER(on_pop_blocks); + + m_core.get_blockchain_storage().pop_blocks(req.nblocks); + + res.height = m_core.get_current_blockchain_height(); + res.status = CORE_RPC_STATUS_OK; + + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_relay_tx(const COMMAND_RPC_RELAY_TX::request& req, COMMAND_RPC_RELAY_TX::response& res, epee::json_rpc::error& error_resp) { PERF_TIMER(on_relay_tx); @@ -2074,6 +2136,7 @@ namespace cryptonote m_core.get_blockchain_top(res.height, top_hash); ++res.height; // turn top block height into blockchain height res.target_height = m_core.get_target_blockchain_height(); + res.next_needed_pruning_seed = m_p2p.get_payload_object().get_next_needed_pruning_stripe().second; for (const auto &c: m_p2p.get_payload_object().get_connections()) res.peers.push_back({c}); @@ -2088,6 +2151,7 @@ namespace cryptonote res.spans.push_back({span.start_block_height, span.nblocks, span_connection_id, (uint32_t)(span.rate + 0.5f), speed, span.size, address}); return true; }); + res.overview = block_queue.get_overview(res.height); res.status = CORE_RPC_STATUS_OK; return true; @@ -2124,68 +2188,85 @@ namespace cryptonote const uint64_t req_to_height = req.to_height ? req.to_height : (m_core.get_current_blockchain_height() - 1); for (uint64_t amount: req.amounts) { - static struct D - { - boost::mutex mutex; - std::vector<uint64_t> cached_distribution; - uint64_t cached_from, cached_to, cached_start_height, cached_base; - bool cached; - D(): cached_from(0), cached_to(0), cached_start_height(0), cached_base(0), cached(false) {} - } d; - boost::unique_lock<boost::mutex> lock(d.mutex); - - if (d.cached && amount == 0 && d.cached_from == req.from_height && d.cached_to == req_to_height) - { - res.distributions.push_back({amount, d.cached_start_height, req.binary, d.cached_distribution, d.cached_base}); - if (!req.cumulative) - { - auto &distribution = res.distributions.back().distribution; - for (size_t n = distribution.size() - 1; n > 0; --n) - distribution[n] -= distribution[n-1]; - distribution[0] -= d.cached_base; - } - continue; - } - - std::vector<uint64_t> distribution; - uint64_t start_height, base; - if (!m_core.get_output_distribution(amount, req.from_height, req_to_height, start_height, distribution, base)) + auto data = rpc::RpcHandler::get_output_distribution([this](uint64_t amount, uint64_t from, uint64_t to, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) { return m_core.get_output_distribution(amount, from, to, start_height, distribution, base); }, amount, req.from_height, req_to_height, req.cumulative); + if (!data) { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; - error_resp.message = "Failed to get rct distribution"; + error_resp.message = "Failed to get output distribution"; return false; } - if (req_to_height > 0 && req_to_height >= req.from_height) - { - uint64_t offset = std::max(req.from_height, start_height); - if (offset <= req_to_height && req_to_height - offset + 1 < distribution.size()) - distribution.resize(req_to_height - offset + 1); - } - if (amount == 0) - { - d.cached_from = req.from_height; - d.cached_to = req_to_height; - d.cached_distribution = distribution; - d.cached_start_height = start_height; - d.cached_base = base; - d.cached = true; - } + res.distributions.push_back({std::move(*data), amount, "", req.binary, req.compress}); + } + } + catch (const std::exception &e) + { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = "Failed to get output distribution"; + return false; + } + + res.status = CORE_RPC_STATUS_OK; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_get_output_distribution_bin(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res) + { + PERF_TIMER(on_get_output_distribution_bin); + + bool r; + if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_OUTPUT_DISTRIBUTION>(invoke_http_mode::BIN, "/get_output_distribution.bin", req, res, r)) + return r; + + res.status = "Failed"; - if (!req.cumulative) + if (!req.binary) + { + res.status = "Binary only call"; + return false; + } + try + { + // 0 is placeholder for the whole chain + const uint64_t req_to_height = req.to_height ? req.to_height : (m_core.get_current_blockchain_height() - 1); + for (uint64_t amount: req.amounts) + { + auto data = rpc::RpcHandler::get_output_distribution([this](uint64_t amount, uint64_t from, uint64_t to, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) { return m_core.get_output_distribution(amount, from, to, start_height, distribution, base); }, amount, req.from_height, req_to_height, req.cumulative); + if (!data) { - for (size_t n = distribution.size() - 1; n > 0; --n) - distribution[n] -= distribution[n-1]; - distribution[0] -= base; + res.status = "Failed to get output distribution"; + return false; } - res.distributions.push_back({amount, start_height, req.binary, std::move(distribution), base}); + res.distributions.push_back({std::move(*data), amount, "", req.binary, req.compress}); + } + } + catch (const std::exception &e) + { + res.status = "Failed to get output distribution"; + return false; + } + + res.status = CORE_RPC_STATUS_OK; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_prune_blockchain(const COMMAND_RPC_PRUNE_BLOCKCHAIN::request& req, COMMAND_RPC_PRUNE_BLOCKCHAIN::response& res, epee::json_rpc::error& error_resp) + { + try + { + if (!(req.check ? m_core.check_blockchain_pruning() : m_core.prune_blockchain())) + { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = req.check ? "Failed to check blockchain pruning" : "Failed to prune blockchain"; + return false; } + res.pruning_seed = m_core.get_blockchain_pruning_seed(); } catch (const std::exception &e) { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; - error_resp.message = "Failed to get output distribution"; + error_resp.message = "Failed to prune blockchain"; return false; } diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index 3ba882b23..83a7cfe27 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -70,10 +70,9 @@ namespace cryptonote bool init( const boost::program_options::variables_map& vm, const bool restricted, - const network_type nettype, const std::string& port ); - network_type nettype() const { return m_nettype; } + network_type nettype() const { return m_core.get_nettype(); } CHAIN_HTTP_TO_MAP2(connection_context); //forward http requests to uri map @@ -117,6 +116,8 @@ namespace cryptonote MAP_URI_AUTO_JON2_IF("/stop_save_graph", on_stop_save_graph, COMMAND_RPC_STOP_SAVE_GRAPH, !m_restricted) MAP_URI_AUTO_JON2("/get_outs", on_get_outs, COMMAND_RPC_GET_OUTPUTS) MAP_URI_AUTO_JON2_IF("/update", on_update, COMMAND_RPC_UPDATE, !m_restricted) + MAP_URI_AUTO_BIN2("/get_output_distribution.bin", on_get_output_distribution_bin, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION) + MAP_URI_AUTO_JON2_IF("/pop_blocks", on_pop_blocks, COMMAND_RPC_POP_BLOCKS, !m_restricted) BEGIN_JSON_RPC_MAP("/json_rpc") MAP_JON_RPC("get_block_count", on_getblockcount, COMMAND_RPC_GETBLOCKCOUNT) MAP_JON_RPC("getblockcount", on_getblockcount, COMMAND_RPC_GETBLOCKCOUNT) @@ -152,6 +153,7 @@ namespace cryptonote MAP_JON_RPC_WE_IF("sync_info", on_sync_info, COMMAND_RPC_SYNC_INFO, !m_restricted) MAP_JON_RPC_WE("get_txpool_backlog", on_get_txpool_backlog, COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG) MAP_JON_RPC_WE("get_output_distribution", on_get_output_distribution, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION) + MAP_JON_RPC_WE_IF("prune_blockchain", on_prune_blockchain, COMMAND_RPC_PRUNE_BLOCKCHAIN, !m_restricted) END_JSON_RPC_MAP() END_URI_MAP2() @@ -187,6 +189,8 @@ namespace cryptonote bool on_start_save_graph(const COMMAND_RPC_START_SAVE_GRAPH::request& req, COMMAND_RPC_START_SAVE_GRAPH::response& res); bool on_stop_save_graph(const COMMAND_RPC_STOP_SAVE_GRAPH::request& req, COMMAND_RPC_STOP_SAVE_GRAPH::response& res); bool on_update(const COMMAND_RPC_UPDATE::request& req, COMMAND_RPC_UPDATE::response& res); + bool on_get_output_distribution_bin(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res); + bool on_pop_blocks(const COMMAND_RPC_POP_BLOCKS::request& req, COMMAND_RPC_POP_BLOCKS::response& res); //json_rpc bool on_getblockcount(const COMMAND_RPC_GETBLOCKCOUNT::request& req, COMMAND_RPC_GETBLOCKCOUNT::response& res); @@ -214,6 +218,7 @@ namespace cryptonote bool on_sync_info(const COMMAND_RPC_SYNC_INFO::request& req, COMMAND_RPC_SYNC_INFO::response& res, epee::json_rpc::error& error_resp); bool on_get_txpool_backlog(const COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::response& res, epee::json_rpc::error& error_resp); bool on_get_output_distribution(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res, epee::json_rpc::error& error_resp); + bool on_prune_blockchain(const COMMAND_RPC_PRUNE_BLOCKCHAIN::request& req, COMMAND_RPC_PRUNE_BLOCKCHAIN::response& res, epee::json_rpc::error& error_resp); //----------------------- private: diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index 3b654d4cb..dfad5d6a7 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -33,6 +33,41 @@ #include "cryptonote_basic/cryptonote_basic.h" #include "cryptonote_basic/difficulty.h" #include "crypto/hash.h" +#include "rpc/rpc_handler.h" +#include "common/varint.h" +#include "common/perf_timer.h" + +namespace +{ + template<typename T> + std::string compress_integer_array(const std::vector<T> &v) + { + std::string s; + s.resize(v.size() * (sizeof(T) * 8 / 7 + 1)); + char *ptr = (char*)s.data(); + for (const T &t: v) + tools::write_varint(ptr, t); + s.resize(ptr - s.data()); + return s; + } + + template<typename T> + std::vector<T> decompress_integer_array(const std::string &s) + { + std::vector<T> v; + v.reserve(s.size()); + int read = 0; + const std::string::const_iterator end = s.end(); + for (std::string::const_iterator i = s.begin(); i != end; std::advance(i, read)) + { + T t; + read = tools::read_varint(std::string::const_iterator(i), s.end(), t); + CHECK_AND_ASSERT_THROW_MES(read > 0 && read <= 256, "Error decompressing data"); + v.push_back(t); + } + return v; + } +} namespace cryptonote { @@ -49,7 +84,7 @@ namespace cryptonote // advance which version they will stop working with // Don't go over 32767 for any of these #define CORE_RPC_VERSION_MAJOR 2 -#define CORE_RPC_VERSION_MINOR 1 +#define CORE_RPC_VERSION_MINOR 3 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR) @@ -566,11 +601,13 @@ namespace cryptonote std::vector<std::string> txs_hashes; bool decode_as_json; bool prune; + bool split; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(txs_hashes) KV_SERIALIZE(decode_as_json) KV_SERIALIZE_OPT(prune, false) + KV_SERIALIZE_OPT(split, false) END_KV_SERIALIZE_MAP() }; @@ -578,6 +615,9 @@ namespace cryptonote { std::string tx_hash; std::string as_hex; + std::string pruned_as_hex; + std::string prunable_as_hex; + std::string prunable_hash; std::string as_json; bool in_pool; bool double_spend_seen; @@ -588,6 +628,9 @@ namespace cryptonote BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tx_hash) KV_SERIALIZE(as_hex) + KV_SERIALIZE(pruned_as_hex) + KV_SERIALIZE(prunable_as_hex) + KV_SERIALIZE(prunable_hash) KV_SERIALIZE(as_json) KV_SERIALIZE(in_pool) KV_SERIALIZE(double_spend_seen) @@ -696,9 +739,11 @@ namespace cryptonote struct request { std::vector<get_outputs_out> outputs; + bool get_txid; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(outputs) + KV_SERIALIZE_OPT(get_txid, true) END_KV_SERIALIZE_MAP() }; @@ -782,9 +827,6 @@ namespace cryptonote std::string tx_as_hex; bool do_not_relay; - request() {} - explicit request(const transaction &); - BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tx_as_hex) KV_SERIALIZE_OPT(do_not_relay, false) @@ -894,6 +936,7 @@ namespace cryptonote bool was_bootstrap_ever_used; uint64_t database_size; bool update_available; + std::string version; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(status) @@ -928,6 +971,7 @@ namespace cryptonote KV_SERIALIZE(was_bootstrap_ever_used) KV_SERIALIZE(database_size) KV_SERIALIZE(update_available) + KV_SERIALIZE(version) END_KV_SERIALIZE_MAP() }; }; @@ -1275,14 +1319,15 @@ namespace cryptonote uint32_t ip; uint16_t port; uint64_t last_seen; + uint32_t pruning_seed; peer() = default; - peer(uint64_t id, const std::string &host, uint64_t last_seen) - : id(id), host(host), ip(0), port(0), last_seen(last_seen) + peer(uint64_t id, const std::string &host, uint64_t last_seen, uint32_t pruning_seed) + : id(id), host(host), ip(0), port(0), last_seen(last_seen), pruning_seed(pruning_seed) {} - peer(uint64_t id, uint32_t ip, uint16_t port, uint64_t last_seen) - : id(id), host(std::to_string(ip)), ip(ip), port(port), last_seen(last_seen) + peer(uint64_t id, uint32_t ip, uint16_t port, uint64_t last_seen, uint32_t pruning_seed) + : id(id), host(std::to_string(ip)), ip(ip), port(port), last_seen(last_seen), pruning_seed(pruning_seed) {} BEGIN_KV_SERIALIZE_MAP() @@ -1291,6 +1336,7 @@ namespace cryptonote KV_SERIALIZE(ip) KV_SERIALIZE(port) KV_SERIALIZE(last_seen) + KV_SERIALIZE_OPT(pruning_seed, (uint32_t)0) END_KV_SERIALIZE_MAP() }; @@ -2202,15 +2248,19 @@ namespace cryptonote std::string status; uint64_t height; uint64_t target_height; + uint32_t next_needed_pruning_seed; std::list<peer> peers; std::list<span> spans; + std::string overview; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(status) KV_SERIALIZE(height) KV_SERIALIZE(target_height) + KV_SERIALIZE(next_needed_pruning_seed) KV_SERIALIZE(peers) KV_SERIALIZE(spans) + KV_SERIALIZE(overview) END_KV_SERIALIZE_MAP() }; }; @@ -2224,6 +2274,7 @@ namespace cryptonote uint64_t to_height; bool cumulative; bool binary; + bool compress; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(amounts) @@ -2231,26 +2282,49 @@ namespace cryptonote KV_SERIALIZE_OPT(to_height, (uint64_t)0) KV_SERIALIZE_OPT(cumulative, false) KV_SERIALIZE_OPT(binary, true) + KV_SERIALIZE_OPT(compress, false) END_KV_SERIALIZE_MAP() }; struct distribution { + rpc::output_distribution_data data; uint64_t amount; - uint64_t start_height; + std::string compressed_data; bool binary; - std::vector<uint64_t> distribution; - uint64_t base; + bool compress; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(amount) - KV_SERIALIZE(start_height) + KV_SERIALIZE_N(data.start_height, "start_height") KV_SERIALIZE(binary) + KV_SERIALIZE(compress) if (this_ref.binary) - KV_SERIALIZE_CONTAINER_POD_AS_BLOB(distribution) + { + if (is_store) + { + if (this_ref.compress) + { + const_cast<std::string&>(this_ref.compressed_data) = compress_integer_array(this_ref.data.distribution); + KV_SERIALIZE(compressed_data) + } + else + KV_SERIALIZE_CONTAINER_POD_AS_BLOB_N(data.distribution, "distribution") + } + else + { + if (this_ref.compress) + { + KV_SERIALIZE(compressed_data) + const_cast<std::vector<uint64_t>&>(this_ref.data.distribution) = decompress_integer_array<uint64_t>(this_ref.compressed_data); + } + else + KV_SERIALIZE_CONTAINER_POD_AS_BLOB_N(data.distribution, "distribution") + } + } else - KV_SERIALIZE(distribution) - KV_SERIALIZE(base) + KV_SERIALIZE_N(data.distribution, "distribution") + KV_SERIALIZE_N(data.base, "base") END_KV_SERIALIZE_MAP() }; @@ -2268,4 +2342,50 @@ namespace cryptonote }; }; + struct COMMAND_RPC_POP_BLOCKS + { + struct request + { + uint64_t nblocks; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(nblocks); + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string status; + uint64_t height; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + KV_SERIALIZE(height) + END_KV_SERIALIZE_MAP() + }; + }; + + struct COMMAND_RPC_PRUNE_BLOCKCHAIN + { + struct request + { + bool check; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_OPT(check, false) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + uint32_t pruning_seed; + std::string status; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + KV_SERIALIZE(pruning_seed) + END_KV_SERIALIZE_MAP() + }; + }; + } diff --git a/src/rpc/daemon_handler.cpp b/src/rpc/daemon_handler.cpp index 9d3b09b68..e2885dbb5 100644 --- a/src/rpc/daemon_handler.cpp +++ b/src/rpc/daemon_handler.cpp @@ -34,6 +34,7 @@ #include "cryptonote_basic/cryptonote_format_utils.h" #include "cryptonote_basic/blobdatatype.h" #include "ringct/rctSigs.h" +#include "version.h" namespace cryptonote { @@ -437,6 +438,7 @@ namespace rpc res.info.block_size_limit = res.info.block_weight_limit = m_core.get_blockchain_storage().get_current_cumulative_block_weight_limit(); res.info.block_size_median = res.info.block_weight_median = m_core.get_blockchain_storage().get_current_cumulative_block_weight_median(); res.info.start_time = (uint64_t)m_core.get_start_time(); + res.info.version = MONERO_VERSION; res.status = Message::STATUS_OK; res.error_details = ""; @@ -724,12 +726,53 @@ namespace rpc res.status = Message::STATUS_OK; } - void DaemonHandler::handle(const GetPerKBFeeEstimate::Request& req, GetPerKBFeeEstimate::Response& res) + void DaemonHandler::handle(const GetFeeEstimate::Request& req, GetFeeEstimate::Response& res) { - res.estimated_fee_per_kb = m_core.get_blockchain_storage().get_dynamic_base_fee_estimate(req.num_grace_blocks); + res.hard_fork_version = m_core.get_blockchain_storage().get_current_hard_fork_version(); + res.estimated_base_fee = m_core.get_blockchain_storage().get_dynamic_base_fee_estimate(req.num_grace_blocks); + + if (res.hard_fork_version < HF_VERSION_PER_BYTE_FEE) + { + res.size_scale = 1024; // per KiB fee + res.fee_mask = 1; + } + else + { + res.size_scale = 1; // per byte fee + res.fee_mask = Blockchain::get_fee_quantization_mask(); + } res.status = Message::STATUS_OK; } + void DaemonHandler::handle(const GetOutputDistribution::Request& req, GetOutputDistribution::Response& res) + { + try + { + res.distributions.reserve(req.amounts.size()); + + const uint64_t req_to_height = req.to_height ? req.to_height : (m_core.get_current_blockchain_height() - 1); + for (std::uint64_t amount : req.amounts) + { + auto data = rpc::RpcHandler::get_output_distribution([this](uint64_t amount, uint64_t from, uint64_t to, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) { return m_core.get_output_distribution(amount, from, to, start_height, distribution, base); }, amount, req.from_height, req_to_height, req.cumulative); + if (!data) + { + res.distributions.clear(); + res.status = Message::STATUS_FAILED; + res.error_details = "Failed to get output distribution"; + return; + } + res.distributions.push_back(output_distribution{std::move(*data), amount, req.cumulative}); + } + res.status = Message::STATUS_OK; + } + catch (const std::exception& e) + { + res.distributions.clear(); + res.status = Message::STATUS_FAILED; + res.error_details = e.what(); + } + } + bool DaemonHandler::getBlockHeaderByHash(const crypto::hash& hash_in, cryptonote::rpc::BlockHeaderResponse& header) { block b; @@ -804,7 +847,8 @@ namespace rpc REQ_RESP_TYPES_MACRO(request_type, GetOutputHistogram, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, GetOutputKeys, req_json, resp_message, handle); REQ_RESP_TYPES_MACRO(request_type, GetRPCVersion, req_json, resp_message, handle); - REQ_RESP_TYPES_MACRO(request_type, GetPerKBFeeEstimate, req_json, resp_message, handle); + REQ_RESP_TYPES_MACRO(request_type, GetFeeEstimate, req_json, resp_message, handle); + REQ_RESP_TYPES_MACRO(request_type, GetOutputDistribution, req_json, resp_message, handle); // if none of the request types matches if (resp_message == NULL) diff --git a/src/rpc/daemon_handler.h b/src/rpc/daemon_handler.h index 5f9687511..2c8ac3867 100644 --- a/src/rpc/daemon_handler.h +++ b/src/rpc/daemon_handler.h @@ -126,7 +126,9 @@ class DaemonHandler : public RpcHandler void handle(const GetRPCVersion::Request& req, GetRPCVersion::Response& res); - void handle(const GetPerKBFeeEstimate::Request& req, GetPerKBFeeEstimate::Response& res); + void handle(const GetFeeEstimate::Request& req, GetFeeEstimate::Response& res); + + void handle(const GetOutputDistribution::Request& req, GetOutputDistribution::Response& res); std::string handle(const std::string& request); diff --git a/src/rpc/daemon_messages.cpp b/src/rpc/daemon_messages.cpp index 56f6f6a8c..7c7442014 100644 --- a/src/rpc/daemon_messages.cpp +++ b/src/rpc/daemon_messages.cpp @@ -59,7 +59,8 @@ const char* const HardForkInfo::name = "hard_fork_info"; const char* const GetOutputHistogram::name = "get_output_histogram"; const char* const GetOutputKeys::name = "get_output_keys"; const char* const GetRPCVersion::name = "get_rpc_version"; -const char* const GetPerKBFeeEstimate::name = "get_dynamic_per_kb_fee_estimate"; +const char* const GetFeeEstimate::name = "get_dynamic_fee_estimate"; +const char* const GetOutputDistribution::name = "get_output_distribution"; @@ -176,8 +177,6 @@ rapidjson::Value GetTransactions::Request::toJson(rapidjson::Document& doc) cons { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, tx_hashes, tx_hashes); return val; @@ -192,8 +191,6 @@ rapidjson::Value GetTransactions::Response::toJson(rapidjson::Document& doc) con { rapidjson::Value val(rapidjson::kObjectType); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, txs, txs); INSERT_INTO_JSON_OBJECT(val, doc, missed_hashes, missed_hashes); @@ -211,8 +208,6 @@ rapidjson::Value KeyImagesSpent::Request::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, key_images, key_images); return val; @@ -227,8 +222,6 @@ rapidjson::Value KeyImagesSpent::Response::toJson(rapidjson::Document& doc) cons { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, spent_status, spent_status); return val; @@ -244,8 +237,6 @@ rapidjson::Value GetTxGlobalOutputIndices::Request::toJson(rapidjson::Document& { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, tx_hash, tx_hash); return val; @@ -260,8 +251,6 @@ rapidjson::Value GetTxGlobalOutputIndices::Response::toJson(rapidjson::Document& { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, output_indices, output_indices); return val; @@ -276,8 +265,6 @@ rapidjson::Value SendRawTx::Request::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, tx, tx); INSERT_INTO_JSON_OBJECT(val, doc, relay, relay); @@ -294,8 +281,6 @@ rapidjson::Value SendRawTx::Response::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, relayed, relayed); return val; @@ -311,8 +296,6 @@ rapidjson::Value StartMining::Request::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, miner_address, miner_address); INSERT_INTO_JSON_OBJECT(val, doc, threads_count, threads_count); INSERT_INTO_JSON_OBJECT(val, doc, do_background_mining, do_background_mining); @@ -371,8 +354,6 @@ rapidjson::Value MiningStatus::Response::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, active, active); INSERT_INTO_JSON_OBJECT(val, doc, speed, speed); INSERT_INTO_JSON_OBJECT(val, doc, threads_count, threads_count); @@ -405,8 +386,6 @@ rapidjson::Value GetInfo::Response::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, info, info); return val; @@ -422,8 +401,6 @@ rapidjson::Value SaveBC::Request::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - return val; } @@ -435,8 +412,6 @@ rapidjson::Value SaveBC::Response::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - return val; } @@ -449,8 +424,6 @@ rapidjson::Value GetBlockHash::Request::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, height, height); return val; @@ -465,8 +438,6 @@ rapidjson::Value GetBlockHash::Response::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, hash, hash); return val; @@ -482,8 +453,6 @@ rapidjson::Value GetLastBlockHeader::Request::toJson(rapidjson::Document& doc) c { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - return val; } @@ -495,8 +464,6 @@ rapidjson::Value GetLastBlockHeader::Response::toJson(rapidjson::Document& doc) { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, header, header); return val; @@ -512,8 +479,6 @@ rapidjson::Value GetBlockHeaderByHash::Request::toJson(rapidjson::Document& doc) { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, hash, hash); return val; @@ -528,8 +493,6 @@ rapidjson::Value GetBlockHeaderByHash::Response::toJson(rapidjson::Document& doc { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, header, header); return val; @@ -545,8 +508,6 @@ rapidjson::Value GetBlockHeaderByHeight::Request::toJson(rapidjson::Document& do { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, height, height); return val; @@ -561,8 +522,6 @@ rapidjson::Value GetBlockHeaderByHeight::Response::toJson(rapidjson::Document& d { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, header, header); return val; @@ -578,8 +537,6 @@ rapidjson::Value GetBlockHeadersByHeight::Request::toJson(rapidjson::Document& d { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, heights, heights); return val; @@ -594,8 +551,6 @@ rapidjson::Value GetBlockHeadersByHeight::Response::toJson(rapidjson::Document& { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, headers, headers); return val; @@ -611,8 +566,6 @@ rapidjson::Value GetPeerList::Request::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - return val; } @@ -624,8 +577,6 @@ rapidjson::Value GetPeerList::Response::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, white_list, white_list); INSERT_INTO_JSON_OBJECT(val, doc, gray_list, gray_list); @@ -678,8 +629,6 @@ rapidjson::Value GetTransactionPool::Response::toJson(rapidjson::Document& doc) { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, transactions, transactions); INSERT_INTO_JSON_OBJECT(val, doc, key_images, key_images); @@ -697,8 +646,6 @@ rapidjson::Value HardForkInfo::Request::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, version, version); return val; @@ -713,8 +660,6 @@ rapidjson::Value HardForkInfo::Response::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, info, info); return val; @@ -730,8 +675,6 @@ rapidjson::Value GetOutputHistogram::Request::toJson(rapidjson::Document& doc) c { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, amounts, amounts); INSERT_INTO_JSON_OBJECT(val, doc, min_count, min_count); INSERT_INTO_JSON_OBJECT(val, doc, max_count, max_count); @@ -754,8 +697,6 @@ rapidjson::Value GetOutputHistogram::Response::toJson(rapidjson::Document& doc) { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, histogram, histogram); return val; @@ -771,8 +712,6 @@ rapidjson::Value GetOutputKeys::Request::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, outputs, outputs); return val; @@ -787,8 +726,6 @@ rapidjson::Value GetOutputKeys::Response::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, keys, keys); return val; @@ -813,8 +750,6 @@ rapidjson::Value GetRPCVersion::Response::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, version, version); return val; @@ -825,38 +760,75 @@ void GetRPCVersion::Response::fromJson(rapidjson::Value& val) GET_FROM_JSON_OBJECT(val, version, version); } -rapidjson::Value GetPerKBFeeEstimate::Request::toJson(rapidjson::Document& doc) const +rapidjson::Value GetFeeEstimate::Request::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); - INSERT_INTO_JSON_OBJECT(val, doc, num_grace_blocks, num_grace_blocks); return val; } -void GetPerKBFeeEstimate::Request::fromJson(rapidjson::Value& val) +void GetFeeEstimate::Request::fromJson(rapidjson::Value& val) { GET_FROM_JSON_OBJECT(val, num_grace_blocks, num_grace_blocks); } -rapidjson::Value GetPerKBFeeEstimate::Response::toJson(rapidjson::Document& doc) const +rapidjson::Value GetFeeEstimate::Response::toJson(rapidjson::Document& doc) const { auto val = Message::toJson(doc); - auto& al = doc.GetAllocator(); + INSERT_INTO_JSON_OBJECT(val, doc, estimated_base_fee, estimated_base_fee); + INSERT_INTO_JSON_OBJECT(val, doc, fee_mask, fee_mask); + INSERT_INTO_JSON_OBJECT(val, doc, size_scale, size_scale); + INSERT_INTO_JSON_OBJECT(val, doc, hard_fork_version, hard_fork_version); + + return val; +} + +void GetFeeEstimate::Response::fromJson(rapidjson::Value& val) +{ + GET_FROM_JSON_OBJECT(val, estimated_base_fee, estimated_base_fee); + GET_FROM_JSON_OBJECT(val, fee_mask, fee_mask); + GET_FROM_JSON_OBJECT(val, size_scale, size_scale); + GET_FROM_JSON_OBJECT(val, hard_fork_version, hard_fork_version); +} + +rapidjson::Value GetOutputDistribution::Request::toJson(rapidjson::Document& doc) const +{ + auto val = Message::toJson(doc); - INSERT_INTO_JSON_OBJECT(val, doc, estimated_fee_per_kb, estimated_fee_per_kb); + INSERT_INTO_JSON_OBJECT(val, doc, amounts, amounts); + INSERT_INTO_JSON_OBJECT(val, doc, from_height, from_height); + INSERT_INTO_JSON_OBJECT(val, doc, to_height, to_height); + INSERT_INTO_JSON_OBJECT(val, doc, cumulative, cumulative); return val; } -void GetPerKBFeeEstimate::Response::fromJson(rapidjson::Value& val) +void GetOutputDistribution::Request::fromJson(rapidjson::Value& val) { - GET_FROM_JSON_OBJECT(val, estimated_fee_per_kb, estimated_fee_per_kb); + GET_FROM_JSON_OBJECT(val, amounts, amounts); + GET_FROM_JSON_OBJECT(val, from_height, from_height); + GET_FROM_JSON_OBJECT(val, to_height, to_height); + GET_FROM_JSON_OBJECT(val, cumulative, cumulative); } +rapidjson::Value GetOutputDistribution::Response::toJson(rapidjson::Document& doc) const +{ + auto val = Message::toJson(doc); + + INSERT_INTO_JSON_OBJECT(val, doc, status, status); + INSERT_INTO_JSON_OBJECT(val, doc, distributions, distributions); + + return val; +} + +void GetOutputDistribution::Response::fromJson(rapidjson::Value& val) +{ + GET_FROM_JSON_OBJECT(val, status, status); + GET_FROM_JSON_OBJECT(val, distributions, distributions); +} } // namespace rpc diff --git a/src/rpc/daemon_messages.h b/src/rpc/daemon_messages.h index be3138e3b..d2014247c 100644 --- a/src/rpc/daemon_messages.h +++ b/src/rpc/daemon_messages.h @@ -28,6 +28,9 @@ #pragma once +#include <unordered_map> +#include <vector> + #include "message.h" #include "cryptonote_protocol/cryptonote_protocol_defs.h" #include "rpc/message_data_structs.h" @@ -62,12 +65,10 @@ class classname \ #define END_RPC_MESSAGE_RESPONSE }; #define END_RPC_MESSAGE_CLASS }; -#define COMMA() , - // NOTE: when using a type with multiple template parameters, // replace any comma in the template specifier with the macro // above, or the preprocessor will eat the comma in a bad way. -#define RPC_MESSAGE_MEMBER(type, name) type name = type{} +#define RPC_MESSAGE_MEMBER(type, name) type name = {} namespace cryptonote @@ -118,7 +119,8 @@ BEGIN_RPC_MESSAGE_CLASS(GetTransactions); RPC_MESSAGE_MEMBER(std::vector<crypto::hash>, tx_hashes); END_RPC_MESSAGE_REQUEST; BEGIN_RPC_MESSAGE_RESPONSE; - RPC_MESSAGE_MEMBER(std::unordered_map<crypto::hash COMMA() cryptonote::rpc::transaction_info>, txs); + using txes_map = std::unordered_map<crypto::hash, transaction_info>; + RPC_MESSAGE_MEMBER(txes_map, txs); RPC_MESSAGE_MEMBER(std::vector<crypto::hash>, missed_hashes); END_RPC_MESSAGE_RESPONSE; END_RPC_MESSAGE_CLASS; @@ -407,12 +409,27 @@ BEGIN_RPC_MESSAGE_CLASS(GetRPCVersion); END_RPC_MESSAGE_RESPONSE; END_RPC_MESSAGE_CLASS; -BEGIN_RPC_MESSAGE_CLASS(GetPerKBFeeEstimate); +BEGIN_RPC_MESSAGE_CLASS(GetFeeEstimate); BEGIN_RPC_MESSAGE_REQUEST; RPC_MESSAGE_MEMBER(uint64_t, num_grace_blocks); END_RPC_MESSAGE_REQUEST; BEGIN_RPC_MESSAGE_RESPONSE; - RPC_MESSAGE_MEMBER(uint64_t, estimated_fee_per_kb); + RPC_MESSAGE_MEMBER(uint64_t, estimated_base_fee); + RPC_MESSAGE_MEMBER(uint64_t, fee_mask); + RPC_MESSAGE_MEMBER(uint32_t, size_scale); + RPC_MESSAGE_MEMBER(uint8_t, hard_fork_version); + END_RPC_MESSAGE_RESPONSE; +END_RPC_MESSAGE_CLASS; + +BEGIN_RPC_MESSAGE_CLASS(GetOutputDistribution); + BEGIN_RPC_MESSAGE_REQUEST; + RPC_MESSAGE_MEMBER(std::vector<uint64_t>, amounts); + RPC_MESSAGE_MEMBER(uint64_t, from_height); + RPC_MESSAGE_MEMBER(uint64_t, to_height); + RPC_MESSAGE_MEMBER(bool, cumulative); + END_RPC_MESSAGE_REQUEST; + BEGIN_RPC_MESSAGE_RESPONSE; + RPC_MESSAGE_MEMBER(std::vector<output_distribution>, distributions); END_RPC_MESSAGE_RESPONSE; END_RPC_MESSAGE_CLASS; diff --git a/src/rpc/message.h b/src/rpc/message.h index 16b8e92fc..56087b998 100644 --- a/src/rpc/message.h +++ b/src/rpc/message.h @@ -65,7 +65,7 @@ namespace rpc static const char* STATUS_BAD_REQUEST; static const char* STATUS_BAD_JSON; - Message() : status(STATUS_OK) { } + Message() : status(STATUS_OK), rpc_version(0) { } virtual ~Message() { } diff --git a/src/rpc/message_data_structs.h b/src/rpc/message_data_structs.h index cf15ade1c..73cf28cec 100644 --- a/src/rpc/message_data_structs.h +++ b/src/rpc/message_data_structs.h @@ -31,6 +31,7 @@ #include "crypto/hash.h" #include "cryptonote_basic/cryptonote_basic.h" #include "ringct/rctSigs.h" +#include "rpc/rpc_handler.h" #include <unordered_map> #include <vector> @@ -78,6 +79,7 @@ namespace rpc uint32_t ip; uint16_t port; uint64_t last_seen; + uint32_t pruning_seed; }; struct tx_in_pool @@ -190,8 +192,15 @@ namespace rpc uint64_t block_size_median; uint64_t block_weight_median; uint64_t start_time; + std::string version; }; + struct output_distribution + { + output_distribution_data data; + uint64_t amount; + bool cumulative; + }; } // namespace rpc } // namespace cryptonote diff --git a/src/rpc/rpc_handler.cpp b/src/rpc/rpc_handler.cpp new file mode 100644 index 000000000..e0a81c70f --- /dev/null +++ b/src/rpc/rpc_handler.cpp @@ -0,0 +1,86 @@ + +#include <algorithm> +#include <boost/thread/locks.hpp> +#include <boost/thread/mutex.hpp> + +#include "cryptonote_core/cryptonote_core.h" + +namespace cryptonote +{ +namespace rpc +{ + namespace + { + output_distribution_data + process_distribution(bool cumulative, std::uint64_t start_height, std::vector<std::uint64_t> distribution, std::uint64_t base) + { + if (!cumulative && !distribution.empty()) + { + for (std::size_t n = distribution.size() - 1; 0 < n; --n) + distribution[n] -= distribution[n - 1]; + distribution[0] -= base; + } + + return {std::move(distribution), start_height, base}; + } + } + + boost::optional<output_distribution_data> + RpcHandler::get_output_distribution(const std::function<bool(uint64_t, uint64_t, uint64_t, uint64_t&, std::vector<uint64_t>&, uint64_t&)> &f, uint64_t amount, uint64_t from_height, uint64_t to_height, bool cumulative) + { + static struct D + { + boost::mutex mutex; + std::vector<std::uint64_t> cached_distribution; + std::uint64_t cached_from, cached_to, cached_start_height, cached_base; + bool cached; + D(): cached_from(0), cached_to(0), cached_start_height(0), cached_base(0), cached(false) {} + } d; + const boost::unique_lock<boost::mutex> lock(d.mutex); + + if (d.cached && amount == 0 && d.cached_from == from_height && d.cached_to == to_height) + return process_distribution(cumulative, d.cached_start_height, d.cached_distribution, d.cached_base); + + std::vector<std::uint64_t> distribution; + std::uint64_t start_height, base; + + // see if we can extend the cache - a common case + if (d.cached && amount == 0 && d.cached_from == from_height && to_height > d.cached_to) + { + std::vector<std::uint64_t> new_distribution; + if (!f(amount, d.cached_to + 1, to_height, start_height, new_distribution, base)) + return boost::none; + distribution = d.cached_distribution; + distribution.reserve(distribution.size() + new_distribution.size()); + for (const auto &e: new_distribution) + distribution.push_back(e); + start_height = d.cached_start_height; + base = d.cached_base; + } + else + { + if (!f(amount, from_height, to_height, start_height, distribution, base)) + return boost::none; + } + + if (to_height > 0 && to_height >= from_height) + { + const std::uint64_t offset = std::max(from_height, start_height); + if (offset <= to_height && to_height - offset + 1 < distribution.size()) + distribution.resize(to_height - offset + 1); + } + + if (amount == 0) + { + d.cached_from = from_height; + d.cached_to = to_height; + d.cached_distribution = distribution; + d.cached_start_height = start_height; + d.cached_base = base; + d.cached = true; + } + + return process_distribution(cumulative, start_height, std::move(distribution), base); + } +} // rpc +} // cryptonote diff --git a/src/rpc/rpc_handler.h b/src/rpc/rpc_handler.h index 64bade5a8..e0d520408 100644 --- a/src/rpc/rpc_handler.h +++ b/src/rpc/rpc_handler.h @@ -28,24 +28,35 @@ #pragma once +#include <boost/optional/optional.hpp> +#include <cstdint> #include <string> +#include <vector> namespace cryptonote { +class core; namespace rpc { +struct output_distribution_data +{ + std::vector<std::uint64_t> distribution; + std::uint64_t start_height; + std::uint64_t base; +}; class RpcHandler { public: + RpcHandler() { } + virtual ~RpcHandler() { } virtual std::string handle(const std::string& request) = 0; - RpcHandler() { } - - virtual ~RpcHandler() { } + static boost::optional<output_distribution_data> + get_output_distribution(const std::function<bool(uint64_t, uint64_t, uint64_t, uint64_t&, std::vector<uint64_t>&, uint64_t&)> &f, uint64_t amount, uint64_t from_height, uint64_t to_height, bool cumulative); }; diff --git a/src/rpc/zmq_server.cpp b/src/rpc/zmq_server.cpp index edd3e6669..a2ff76668 100644 --- a/src/rpc/zmq_server.cpp +++ b/src/rpc/zmq_server.cpp @@ -27,7 +27,6 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "zmq_server.h" -#include <boost/chrono/chrono.hpp> namespace cryptonote { diff --git a/src/serialization/json_archive.h b/src/serialization/json_archive.h index f906b5d3b..04436c21c 100644 --- a/src/serialization/json_archive.h +++ b/src/serialization/json_archive.h @@ -113,7 +113,7 @@ struct json_archive; template <> struct json_archive<true> : public json_archive_base<std::ostream, true> { - json_archive(stream_type &s, bool indent = false) : base_type(s, indent) { } + json_archive(stream_type &s, bool indent = false) : base_type(s, indent), inner_array_size_(0) { } template<typename T> static auto promote_to_printable_integer_type(T v) -> decltype(+v) diff --git a/src/serialization/json_object.cpp b/src/serialization/json_object.cpp index 89a1dbd23..ee4fa4a19 100644 --- a/src/serialization/json_object.cpp +++ b/src/serialization/json_object.cpp @@ -734,6 +734,7 @@ void toJsonValue(rapidjson::Document& doc, const cryptonote::rpc::peer& peer, ra INSERT_INTO_JSON_OBJECT(val, doc, ip, peer.ip); INSERT_INTO_JSON_OBJECT(val, doc, port, peer.port); INSERT_INTO_JSON_OBJECT(val, doc, last_seen, peer.last_seen); + INSERT_INTO_JSON_OBJECT(val, doc, pruning_seed, peer.pruning_seed); } @@ -748,6 +749,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::peer& peer) GET_FROM_JSON_OBJECT(val, peer.ip, ip); GET_FROM_JSON_OBJECT(val, peer.port, port); GET_FROM_JSON_OBJECT(val, peer.last_seen, last_seen); + GET_FROM_JSON_OBJECT(val, peer.pruning_seed, pruning_seed); } void toJsonValue(rapidjson::Document& doc, const cryptonote::rpc::tx_in_pool& tx, rapidjson::Value& val) @@ -1192,7 +1194,9 @@ void toJsonValue(rapidjson::Document& doc, const cryptonote::rpc::DaemonInfo& in INSERT_INTO_JSON_OBJECT(val, doc, incoming_connections_count, info.incoming_connections_count); INSERT_INTO_JSON_OBJECT(val, doc, white_peerlist_size, info.white_peerlist_size); INSERT_INTO_JSON_OBJECT(val, doc, grey_peerlist_size, info.grey_peerlist_size); + INSERT_INTO_JSON_OBJECT(val, doc, mainnet, info.mainnet); INSERT_INTO_JSON_OBJECT(val, doc, testnet, info.testnet); + INSERT_INTO_JSON_OBJECT(val, doc, stagenet, info.stagenet); INSERT_INTO_JSON_OBJECT(val, doc, nettype, info.nettype); INSERT_INTO_JSON_OBJECT(val, doc, top_block_hash, info.top_block_hash); INSERT_INTO_JSON_OBJECT(val, doc, cumulative_difficulty, info.cumulative_difficulty); @@ -1221,7 +1225,9 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::DaemonInfo& inf GET_FROM_JSON_OBJECT(val, info.incoming_connections_count, incoming_connections_count); GET_FROM_JSON_OBJECT(val, info.white_peerlist_size, white_peerlist_size); GET_FROM_JSON_OBJECT(val, info.grey_peerlist_size, grey_peerlist_size); + GET_FROM_JSON_OBJECT(val, info.mainnet, mainnet); GET_FROM_JSON_OBJECT(val, info.testnet, testnet); + GET_FROM_JSON_OBJECT(val, info.stagenet, stagenet); GET_FROM_JSON_OBJECT(val, info.nettype, nettype); GET_FROM_JSON_OBJECT(val, info.top_block_hash, top_block_hash); GET_FROM_JSON_OBJECT(val, info.cumulative_difficulty, cumulative_difficulty); @@ -1232,6 +1238,29 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::DaemonInfo& inf GET_FROM_JSON_OBJECT(val, info.start_time, start_time); } +void toJsonValue(rapidjson::Document& doc, const cryptonote::rpc::output_distribution& dist, rapidjson::Value& val) +{ + val.SetObject(); + + INSERT_INTO_JSON_OBJECT(val, doc, distribution, dist.data.distribution); + INSERT_INTO_JSON_OBJECT(val, doc, amount, dist.amount); + INSERT_INTO_JSON_OBJECT(val, doc, start_height, dist.data.start_height); + INSERT_INTO_JSON_OBJECT(val, doc, base, dist.data.base); +} + +void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::output_distribution& dist) +{ + if (!val.IsObject()) + { + throw WRONG_TYPE("json object"); + } + + GET_FROM_JSON_OBJECT(val, dist.data.distribution, distribution); + GET_FROM_JSON_OBJECT(val, dist.amount, amount); + GET_FROM_JSON_OBJECT(val, dist.data.start_height, start_height); + GET_FROM_JSON_OBJECT(val, dist.data.base, base); +} + } // namespace json } // namespace cryptonote diff --git a/src/serialization/json_object.h b/src/serialization/json_object.h index da3351fe3..b6384ca0d 100644 --- a/src/serialization/json_object.h +++ b/src/serialization/json_object.h @@ -281,6 +281,9 @@ void fromJsonValue(const rapidjson::Value& val, rct::mgSig& sig); void toJsonValue(rapidjson::Document& doc, const cryptonote::rpc::DaemonInfo& info, rapidjson::Value& val); void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::DaemonInfo& info); +void toJsonValue(rapidjson::Document& doc, const cryptonote::rpc::output_distribution& dist, rapidjson::Value& val); +void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::output_distribution& dist); + template <typename Map> typename std::enable_if<sfinae::is_map_like<Map>::value, void>::type toJsonValue(rapidjson::Document& doc, const Map& map, rapidjson::Value& val); diff --git a/src/simplewallet/CMakeLists.txt b/src/simplewallet/CMakeLists.txt index c31cdebde..e292f85dd 100644 --- a/src/simplewallet/CMakeLists.txt +++ b/src/simplewallet/CMakeLists.txt @@ -53,6 +53,7 @@ target_link_libraries(simplewallet ${Boost_CHRONO_LIBRARY} ${Boost_PROGRAM_OPTIONS_LIBRARY} ${Boost_FILESYSTEM_LIBRARY} + ${Boost_LOCALE_LIBRARY} ${ICU_LIBRARIES} ${Boost_THREAD_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index 18b596662..3cbfb760b 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -43,6 +43,7 @@ #include <boost/algorithm/string.hpp> #include <boost/format.hpp> #include <boost/regex.hpp> +#include <boost/range/adaptor/transformed.hpp> #include "include_base_utils.h" #include "common/i18n.h" #include "common/command_line.h" @@ -64,6 +65,7 @@ #include "wallet/wallet_args.h" #include "version.h" #include <stdexcept> +#include "wallet/message_store.h" #ifdef WIN32 #include <boost/locale.hpp> @@ -101,12 +103,24 @@ typedef cryptonote::simple_wallet sw; m_auto_refresh_enabled.store(auto_refresh_enabled, std::memory_order_relaxed); \ }) -#define SCOPED_WALLET_UNLOCK() \ +#define SCOPED_WALLET_UNLOCK_ON_BAD_PASSWORD(code) \ LOCK_IDLE_SCOPE(); \ boost::optional<tools::password_container> pwd_container = boost::none; \ - if (m_wallet->ask_password() && !m_wallet->watch_only() && !(pwd_container = get_and_verify_password())) { return true; } \ + if (m_wallet->ask_password() && !(pwd_container = get_and_verify_password())) { code; } \ tools::wallet_keys_unlocker unlocker(*m_wallet, pwd_container); +#define SCOPED_WALLET_UNLOCK() SCOPED_WALLET_UNLOCK_ON_BAD_PASSWORD(return true;) + +#define PRINT_USAGE(usage_help) fail_msg_writer() << boost::format(tr("usage: %s")) % usage_help; + +#define LONG_PAYMENT_ID_SUPPORT_CHECK() \ + do { \ + if (!m_long_payment_id_support) { \ + fail_msg_writer() << tr("Long payment IDs are obsolete. Use --long-payment-id-support if you really must use one."); \ + return true; \ + } \ + } while(0) + enum TransferType { Transfer, TransferLocked, @@ -130,19 +144,111 @@ namespace const command_line::arg_descriptor<bool> arg_non_deterministic = {"non-deterministic", sw::tr("Generate non-deterministic view and spend keys"), false}; const command_line::arg_descriptor<bool> arg_allow_mismatched_daemon_version = {"allow-mismatched-daemon-version", sw::tr("Allow communicating with a daemon that uses a different RPC version"), false}; const command_line::arg_descriptor<uint64_t> arg_restore_height = {"restore-height", sw::tr("Restore from specific blockchain height"), 0}; + const command_line::arg_descriptor<std::string> arg_restore_date = {"restore-date", sw::tr("Restore from estimated blockchain height on specified date"), ""}; const command_line::arg_descriptor<bool> arg_do_not_relay = {"do-not-relay", sw::tr("The newly created transaction will not be relayed to the monero network"), false}; const command_line::arg_descriptor<bool> arg_create_address_file = {"create-address-file", sw::tr("Create an address file for new wallets"), false}; const command_line::arg_descriptor<std::string> arg_subaddress_lookahead = {"subaddress-lookahead", tools::wallet2::tr("Set subaddress lookahead sizes to <major>:<minor>"), ""}; const command_line::arg_descriptor<bool> arg_use_english_language_names = {"use-english-language-names", sw::tr("Display English language names"), false}; + const command_line::arg_descriptor<bool> arg_long_payment_id_support = {"long-payment-id-support", sw::tr("Support obsolete long (unencrypted) payment ids"), false}; const command_line::arg_descriptor< std::vector<std::string> > arg_command = {"command", ""}; - std::string input_line(const std::string& prompt) + const char* USAGE_START_MINING("start_mining [<number_of_threads>] [bg_mining] [ignore_battery]"); + const char* USAGE_SET_DAEMON("set_daemon <host>[:<port>] [trusted|untrusted]"); + const char* USAGE_SHOW_BALANCE("balance [detail]"); + const char* USAGE_INCOMING_TRANSFERS("incoming_transfers [available|unavailable] [verbose] [uses] [index=<N1>[,<N2>[,...]]]"); + const char* USAGE_PAYMENTS("payments <PID_1> [<PID_2> ... <PID_N>]"); + const char* USAGE_PAYMENT_ID("payment_id"); + const char* USAGE_TRANSFER("transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] (<URI> | <address> <amount>) [<payment_id>]"); + const char* USAGE_LOCKED_TRANSFER("locked_transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] (<URI> | <addr> <amount>) <lockblocks> [<payment_id (obsolete)>]"); + const char* USAGE_LOCKED_SWEEP_ALL("locked_sweep_all [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> <lockblocks> [<payment_id (obsolete)>]"); + const char* USAGE_SWEEP_ALL("sweep_all [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] [outputs=<N>] <address> [<payment_id (obsolete)>]"); + const char* USAGE_SWEEP_BELOW("sweep_below <amount_threshold> [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> [<payment_id (obsolete)>]"); + const char* USAGE_SWEEP_SINGLE("sweep_single [<priority>] [<ring_size>] [outputs=<N>] <key_image> <address> [<payment_id (obsolete)>]"); + const char* USAGE_DONATE("donate [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <amount> [<payment_id (obsolete)>]"); + const char* USAGE_SIGN_TRANSFER("sign_transfer [export_raw]"); + const char* USAGE_SET_LOG("set_log <level>|{+,-,}<categories>"); + const char* USAGE_ACCOUNT("account\n" + " account new <label text with white spaces allowed>\n" + " account switch <index> \n" + " account label <index> <label text with white spaces allowed>\n" + " account tag <tag_name> <account_index_1> [<account_index_2> ...]\n" + " account untag <account_index_1> [<account_index_2> ...]\n" + " account tag_description <tag_name> <description>"); + const char* USAGE_ADDRESS("address [ new <label text with white spaces allowed> | all | <index_min> [<index_max>] | label <index> <label text with white spaces allowed>]"); + const char* USAGE_INTEGRATED_ADDRESS("integrated_address [<payment_id> | <address>]"); + const char* USAGE_ADDRESS_BOOK("address_book [(add ((<address> [pid <id>])|<integrated address>) [<description possibly with whitespaces>])|(delete <index>)]"); + const char* USAGE_SET_VARIABLE("set <option> [<value>]"); + const char* USAGE_GET_TX_KEY("get_tx_key <txid>"); + const char* USAGE_SET_TX_KEY("set_tx_key <txid> <tx_key>"); + const char* USAGE_CHECK_TX_KEY("check_tx_key <txid> <txkey> <address>"); + const char* USAGE_GET_TX_PROOF("get_tx_proof <txid> <address> [<message>]"); + const char* USAGE_CHECK_TX_PROOF("check_tx_proof <txid> <address> <signature_file> [<message>]"); + const char* USAGE_GET_SPEND_PROOF("get_spend_proof <txid> [<message>]"); + const char* USAGE_CHECK_SPEND_PROOF("check_spend_proof <txid> <signature_file> [<message>]"); + const char* USAGE_GET_RESERVE_PROOF("get_reserve_proof (all|<amount>) [<message>]"); + const char* USAGE_CHECK_RESERVE_PROOF("check_reserve_proof <address> <signature_file> [<message>]"); + const char* USAGE_SHOW_TRANSFERS("show_transfers [in|out|pending|failed|pool|coinbase] [index=<N1>[,<N2>,...]] [<min_height> [<max_height>]]"); + const char* USAGE_UNSPENT_OUTPUTS("unspent_outputs [index=<N1>[,<N2>,...]] [<min_amount> [<max_amount>]]"); + const char* USAGE_RESCAN_BC("rescan_bc [hard]"); + const char* USAGE_SET_TX_NOTE("set_tx_note <txid> [free text note]"); + const char* USAGE_GET_TX_NOTE("get_tx_note <txid>"); + const char* USAGE_GET_DESCRIPTION("get_description"); + const char* USAGE_SET_DESCRIPTION("set_description [free text note]"); + const char* USAGE_SIGN("sign <filename>"); + const char* USAGE_VERIFY("verify <filename> <address> <signature>"); + const char* USAGE_EXPORT_KEY_IMAGES("export_key_images <filename>"); + const char* USAGE_IMPORT_KEY_IMAGES("import_key_images <filename>"); + const char* USAGE_HW_KEY_IMAGES_SYNC("hw_key_images_sync"); + const char* USAGE_HW_RECONNECT("hw_reconnect"); + const char* USAGE_EXPORT_OUTPUTS("export_outputs <filename>"); + const char* USAGE_IMPORT_OUTPUTS("import_outputs <filename>"); + const char* USAGE_SHOW_TRANSFER("show_transfer <txid>"); + const char* USAGE_MAKE_MULTISIG("make_multisig <threshold> <string1> [<string>...]"); + const char* USAGE_FINALIZE_MULTISIG("finalize_multisig <string> [<string>...]"); + const char* USAGE_EXCHANGE_MULTISIG_KEYS("exchange_multisig_keys <string> [<string>...]"); + const char* USAGE_EXPORT_MULTISIG_INFO("export_multisig_info <filename>"); + const char* USAGE_IMPORT_MULTISIG_INFO("import_multisig_info <filename> [<filename>...]"); + const char* USAGE_SIGN_MULTISIG("sign_multisig <filename>"); + const char* USAGE_SUBMIT_MULTISIG("submit_multisig <filename>"); + const char* USAGE_EXPORT_RAW_MULTISIG_TX("export_raw_multisig_tx <filename>"); + const char* USAGE_MMS("mms [<subcommand> [<subcommand_parameters>]]"); + const char* USAGE_MMS_INIT("mms init <required_signers>/<authorized_signers> <own_label> <own_transport_address>"); + const char* USAGE_MMS_INFO("mms info"); + const char* USAGE_MMS_SIGNER("mms signer [<number> <label> [<transport_address> [<monero_address>]]]"); + const char* USAGE_MMS_LIST("mms list"); + const char* USAGE_MMS_NEXT("mms next [sync]"); + const char* USAGE_MMS_SYNC("mms sync"); + const char* USAGE_MMS_TRANSFER("mms transfer <transfer_command_arguments>"); + const char* USAGE_MMS_DELETE("mms delete (<message_id> | all)"); + const char* USAGE_MMS_SEND("mms send [<message_id>]"); + const char* USAGE_MMS_RECEIVE("mms receive"); + const char* USAGE_MMS_EXPORT("mms export <message_id>"); + const char* USAGE_MMS_NOTE("mms note [<label> <text>]"); + const char* USAGE_MMS_SHOW("mms show <message_id>"); + const char* USAGE_MMS_SET("mms set <option_name> [<option_value>]"); + const char* USAGE_MMS_SEND_SIGNER_CONFIG("mms send_signer_config"); + const char* USAGE_MMS_START_AUTO_CONFIG("mms start_auto_config [<label> <label> ...]"); + const char* USAGE_MMS_STOP_AUTO_CONFIG("mms stop_auto_config"); + const char* USAGE_MMS_AUTO_CONFIG("mms auto_config <auto_config_token>"); + const char* USAGE_PRINT_RING("print_ring <key_image> | <txid>"); + const char* USAGE_SET_RING("set_ring <filename> | ( <key_image> absolute|relative <index> [<index>...] )"); + const char* USAGE_SAVE_KNOWN_RINGS("save_known_rings"); + const char* USAGE_MARK_OUTPUT_SPENT("mark_output_spent <amount>/<offset> | <filename> [add]"); + const char* USAGE_MARK_OUTPUT_UNSPENT("mark_output_unspent <amount>/<offset>"); + const char* USAGE_IS_OUTPUT_SPENT("is_output_spent <amount>/<offset>"); + const char* USAGE_VERSION("version"); + const char* USAGE_HELP("help [<command>]"); + + std::string input_line(const std::string& prompt, bool yesno = false) { #ifdef HAVE_READLINE rdln::suspend_readline pause_readline; #endif std::cout << prompt; + if (yesno) + std::cout << " (Y/Yes/N/No)"; + std::cout << ": " << std::flush; std::string buf; #ifdef _WIN32 @@ -154,12 +260,12 @@ namespace return epee::string_tools::trim(buf); } - epee::wipeable_string input_secure_line(const std::string& prompt) + epee::wipeable_string input_secure_line(const char *prompt) { #ifdef HAVE_READLINE rdln::suspend_readline pause_readline; #endif - auto pwd_container = tools::password_container::prompt(false, prompt.c_str(), false); + auto pwd_container = tools::password_container::prompt(false, prompt, false); if (!pwd_container) { MERROR("Failed to read secure line"); @@ -332,10 +438,10 @@ namespace << ", " << dnssec_str << std::endl << sw::tr(" Monero Address = ") << addresses[0] << std::endl - << sw::tr("Is this OK? (Y/n) ") + << sw::tr("Is this OK?") ; // prompt the user for confirmation given the dns query and dnssec status - std::string confirm_dns_ok = input_line(prompt.str()); + std::string confirm_dns_ok = input_line(prompt.str(), true); if (std::cin.eof()) { return {}; @@ -447,7 +553,7 @@ namespace } catch (const tools::error::tx_rejected& e) { - fail_msg_writer() << (boost::format(sw::tr("transaction %s was rejected by daemon with status: ")) % get_transaction_hash(e.tx())) << e.status(); + fail_msg_writer() << (boost::format(sw::tr("transaction %s was rejected by daemon")) % get_transaction_hash(e.tx())); std::string reason = e.reason(); if (!reason.empty()) fail_msg_writer() << sw::tr("Reason: ") << reason; @@ -503,7 +609,7 @@ namespace fail_msg_writer() << boost::format(sw::tr("File %s likely stores wallet private keys! Use a different file name.")) % filename; return false; } - return command_line::is_yes(input_line((boost::format(sw::tr("File %s already exists. Are you sure to overwrite it? (Y/Yes/N/No): ")) % filename).str())); + return command_line::is_yes(input_line((boost::format(sw::tr("File %s already exists. Are you sure to overwrite it?")) % filename).str(), true)); } return true; } @@ -580,12 +686,12 @@ std::string simple_wallet::get_command_usage(const std::vector<std::string> &arg bool simple_wallet::viewkey(const std::vector<std::string> &args/* = std::vector<std::string>()*/) { - SCOPED_WALLET_UNLOCK(); // don't log PAUSE_READLINE(); if (m_wallet->key_on_device()) { std::cout << "secret: On device. Not available" << std::endl; } else { + SCOPED_WALLET_UNLOCK(); printf("secret: "); print_secret_key(m_wallet->get_account().get_keys().m_view_secret_key); putchar('\n'); @@ -602,12 +708,12 @@ bool simple_wallet::spendkey(const std::vector<std::string> &args/* = std::vecto fail_msg_writer() << tr("wallet is watch-only and has no spend key"); return true; } - SCOPED_WALLET_UNLOCK(); // don't log PAUSE_READLINE(); if (m_wallet->key_on_device()) { std::cout << "secret: On device. Not available" << std::endl; } else { + SCOPED_WALLET_UNLOCK(); printf("secret: "); print_secret_key(m_wallet->get_account().get_keys().m_spend_secret_key); putchar('\n'); @@ -634,8 +740,6 @@ bool simple_wallet::print_seed(bool encrypted) return true; } - SCOPED_WALLET_UNLOCK(); - multisig = m_wallet->multisig(&ready); if (multisig) { @@ -645,7 +749,10 @@ bool simple_wallet::print_seed(bool encrypted) return true; } } - else if (!m_wallet->is_deterministic()) + + SCOPED_WALLET_UNLOCK(); + + if (!multisig && !m_wallet->is_deterministic()) { fail_msg_writer() << tr("wallet is non-deterministic and has no seed"); return true; @@ -766,10 +873,12 @@ bool simple_wallet::change_password(const std::vector<std::string> &args) bool simple_wallet::payment_id(const std::vector<std::string> &args/* = std::vector<std::string>()*/) { + LONG_PAYMENT_ID_SUPPORT_CHECK(); + crypto::hash payment_id; if (args.size() > 0) { - fail_msg_writer() << tr("usage: payment_id"); + PRINT_USAGE(USAGE_PAYMENT_ID); return true; } payment_id = crypto::rand<crypto::hash>(); @@ -780,10 +889,7 @@ bool simple_wallet::payment_id(const std::vector<std::string> &args/* = std::vec bool simple_wallet::print_fee_info(const std::vector<std::string> &args/* = std::vector<std::string>()*/) { if (!try_connect_to_daemon()) - { - fail_msg_writer() << tr("Cannot connect to daemon"); return true; - } const bool per_byte = m_wallet->use_fork_rules(HF_VERSION_PER_BYTE_FEE); const uint64_t base_fee = m_wallet->get_base_fee(); const char *base = per_byte ? "byte" : "kB"; @@ -837,65 +943,83 @@ bool simple_wallet::print_fee_info(const std::vector<std::string> &args/* = std: bool simple_wallet::prepare_multisig(const std::vector<std::string> &args) { + prepare_multisig_main(args, false); + return true; +} + +bool simple_wallet::prepare_multisig_main(const std::vector<std::string> &args, bool called_by_mms) +{ if (m_wallet->key_on_device()) { fail_msg_writer() << tr("command not supported by HW wallet"); - return true; + return false; } if (m_wallet->multisig()) { fail_msg_writer() << tr("This wallet is already multisig"); - return true; + return false; } if (m_wallet->watch_only()) { fail_msg_writer() << tr("wallet is watch-only and cannot be made multisig"); - return true; + return false; } if(m_wallet->get_num_transfer_details()) { fail_msg_writer() << tr("This wallet has been used before, please use a new wallet to create a multisig wallet"); - return true; + return false; } - SCOPED_WALLET_UNLOCK(); + SCOPED_WALLET_UNLOCK_ON_BAD_PASSWORD(return false;); std::string multisig_info = m_wallet->get_multisig_info(); success_msg_writer() << multisig_info; success_msg_writer() << tr("Send this multisig info to all other participants, then use make_multisig <threshold> <info1> [<info2>...] with others' multisig info"); success_msg_writer() << tr("This includes the PRIVATE view key, so needs to be disclosed only to that multisig wallet's participants "); + + if (called_by_mms) + { + get_message_store().process_wallet_created_data(get_multisig_wallet_state(), mms::message_type::key_set, multisig_info); + } + return true; } bool simple_wallet::make_multisig(const std::vector<std::string> &args) { + make_multisig_main(args, false); + return true; +} + +bool simple_wallet::make_multisig_main(const std::vector<std::string> &args, bool called_by_mms) +{ if (m_wallet->key_on_device()) { fail_msg_writer() << tr("command not supported by HW wallet"); - return true; + return false; } if (m_wallet->multisig()) { fail_msg_writer() << tr("This wallet is already multisig"); - return true; + return false; } if (m_wallet->watch_only()) { fail_msg_writer() << tr("wallet is watch-only and cannot be made multisig"); - return true; + return false; } if(m_wallet->get_num_transfer_details()) { fail_msg_writer() << tr("This wallet has been used before, please use a new wallet to create a multisig wallet"); - return true; + return false; } if (args.size() < 2) { - fail_msg_writer() << tr("usage: make_multisig <threshold> <multisiginfo1> [<multisiginfo2>...]"); - return true; + PRINT_USAGE(USAGE_MAKE_MULTISIG); + return false; } // parse threshold @@ -903,14 +1027,14 @@ bool simple_wallet::make_multisig(const std::vector<std::string> &args) if (!string_tools::get_xtype_from_string(threshold, args[0])) { fail_msg_writer() << tr("Invalid threshold"); - return true; + return false; } const auto orig_pwd_container = get_and_verify_password(); if(orig_pwd_container == boost::none) { fail_msg_writer() << tr("Your original password was incorrect."); - return true; + return false; } LOCK_IDLE_SCOPE(); @@ -925,20 +1049,24 @@ bool simple_wallet::make_multisig(const std::vector<std::string> &args) success_msg_writer() << tr("Another step is needed"); success_msg_writer() << multisig_extra_info; success_msg_writer() << tr("Send this multisig info to all other participants, then use exchange_multisig_keys <info1> [<info2>...] with others' multisig info"); + if (called_by_mms) + { + get_message_store().process_wallet_created_data(get_multisig_wallet_state(), mms::message_type::additional_key_set, multisig_extra_info); + } return true; } } catch (const std::exception &e) { fail_msg_writer() << tr("Error creating multisig: ") << e.what(); - return true; + return false; } uint32_t total; if (!m_wallet->multisig(NULL, &threshold, &total)) { fail_msg_writer() << tr("Error creating multisig: new wallet is not multisig"); - return true; + return false; } success_msg_writer() << std::to_string(threshold) << "/" << total << tr(" multisig address: ") << m_wallet->get_account().get_public_address_str(m_wallet->nettype()); @@ -977,7 +1105,7 @@ bool simple_wallet::finalize_multisig(const std::vector<std::string> &args) if (args.size() < 2) { - fail_msg_writer() << tr("usage: finalize_multisig <multisiginfo1> [<multisiginfo2>...]"); + PRINT_USAGE(USAGE_FINALIZE_MULTISIG); return true; } @@ -998,35 +1126,41 @@ bool simple_wallet::finalize_multisig(const std::vector<std::string> &args) return true; } -bool simple_wallet::exchange_multisig_keys(const std::vector<std::string> &args) { +bool simple_wallet::exchange_multisig_keys(const std::vector<std::string> &args) +{ + exchange_multisig_keys_main(args, false); + return true; +} + +bool simple_wallet::exchange_multisig_keys_main(const std::vector<std::string> &args, bool called_by_mms) { bool ready; if (m_wallet->key_on_device()) { fail_msg_writer() << tr("command not supported by HW wallet"); - return true; + return false; } if (!m_wallet->multisig(&ready)) { fail_msg_writer() << tr("This wallet is not multisig"); - return true; + return false; } if (ready) { fail_msg_writer() << tr("This wallet is already finalized"); - return true; + return false; } const auto orig_pwd_container = get_and_verify_password(); if(orig_pwd_container == boost::none) { fail_msg_writer() << tr("Your original password was incorrect."); - return true; + return false; } if (args.size() < 2) { - fail_msg_writer() << tr("usage: exchange_multisig_keys <multisiginfo1> [<multisiginfo2>...]"); - return true; + PRINT_USAGE(USAGE_EXCHANGE_MULTISIG_KEYS); + return false; } try @@ -1037,17 +1171,22 @@ bool simple_wallet::exchange_multisig_keys(const std::vector<std::string> &args) message_writer() << tr("Another step is needed"); message_writer() << multisig_extra_info; message_writer() << tr("Send this multisig info to all other participants, then use exchange_multisig_keys <info1> [<info2>...] with others' multisig info"); + if (called_by_mms) + { + get_message_store().process_wallet_created_data(get_multisig_wallet_state(), mms::message_type::additional_key_set, multisig_extra_info); + } return true; } else { uint32_t threshold, total; m_wallet->multisig(NULL, &threshold, &total); success_msg_writer() << tr("Multisig wallet has been successfully created. Current wallet type: ") << threshold << "/" << total; + success_msg_writer() << tr("Multisig address: ") << m_wallet->get_account().get_public_address_str(m_wallet->nettype()); } } catch (const std::exception &e) { fail_msg_writer() << tr("Failed to perform multisig keys exchange: ") << e.what(); - return true; + return false; } return true; @@ -1055,49 +1194,63 @@ bool simple_wallet::exchange_multisig_keys(const std::vector<std::string> &args) bool simple_wallet::export_multisig(const std::vector<std::string> &args) { + export_multisig_main(args, false); + return true; +} + +bool simple_wallet::export_multisig_main(const std::vector<std::string> &args, bool called_by_mms) +{ bool ready; if (m_wallet->key_on_device()) { fail_msg_writer() << tr("command not supported by HW wallet"); - return true; + return false; } if (!m_wallet->multisig(&ready)) { fail_msg_writer() << tr("This wallet is not multisig"); - return true; + return false; } if (!ready) { fail_msg_writer() << tr("This multisig wallet is not yet finalized"); - return true; + return false; } if (args.size() != 1) { - fail_msg_writer() << tr("usage: export_multisig_info <filename>"); - return true; + PRINT_USAGE(USAGE_EXPORT_MULTISIG_INFO); + return false; } - SCOPED_WALLET_UNLOCK(); - const std::string filename = args[0]; - if (m_wallet->confirm_export_overwrite() && !check_file_overwrite(filename)) + if (!called_by_mms && m_wallet->confirm_export_overwrite() && !check_file_overwrite(filename)) return true; + + SCOPED_WALLET_UNLOCK_ON_BAD_PASSWORD(return false;); + try { cryptonote::blobdata ciphertext = m_wallet->export_multisig(); - bool r = epee::file_io_utils::save_string_to_file(filename, ciphertext); - if (!r) + if (called_by_mms) { - fail_msg_writer() << tr("failed to save file ") << filename; - return true; + get_message_store().process_wallet_created_data(get_multisig_wallet_state(), mms::message_type::multisig_sync_data, ciphertext); + } + else + { + bool r = epee::file_io_utils::save_string_to_file(filename, ciphertext); + if (!r) + { + fail_msg_writer() << tr("failed to save file ") << filename; + return false; + } } } catch (const std::exception &e) { LOG_ERROR("Error exporting multisig info: " << e.what()); fail_msg_writer() << tr("Error exporting multisig info: ") << e.what(); - return true; + return false; } success_msg_writer() << tr("Multisig info exported to ") << filename; @@ -1106,45 +1259,58 @@ bool simple_wallet::export_multisig(const std::vector<std::string> &args) bool simple_wallet::import_multisig(const std::vector<std::string> &args) { + import_multisig_main(args, false); + return true; +} + +bool simple_wallet::import_multisig_main(const std::vector<std::string> &args, bool called_by_mms) +{ bool ready; uint32_t threshold, total; if (m_wallet->key_on_device()) { fail_msg_writer() << tr("command not supported by HW wallet"); - return true; + return false; } if (!m_wallet->multisig(&ready, &threshold, &total)) { fail_msg_writer() << tr("This wallet is not multisig"); - return true; + return false; } if (!ready) { fail_msg_writer() << tr("This multisig wallet is not yet finalized"); - return true; + return false; } if (args.size() < threshold - 1) { - fail_msg_writer() << tr("usage: import_multisig_info <filename1> [<filename2>...] - one for each other participant"); - return true; + PRINT_USAGE(USAGE_IMPORT_MULTISIG_INFO); + return false; } - SCOPED_WALLET_UNLOCK(); - std::vector<cryptonote::blobdata> info; for (size_t n = 0; n < args.size(); ++n) { - const std::string filename = args[n]; - std::string data; - bool r = epee::file_io_utils::load_file_to_string(filename, data); - if (!r) + if (called_by_mms) { - fail_msg_writer() << tr("failed to read file ") << filename; - return true; + info.push_back(args[n]); + } + else + { + const std::string &filename = args[n]; + std::string data; + bool r = epee::file_io_utils::load_file_to_string(filename, data); + if (!r) + { + fail_msg_writer() << tr("failed to read file ") << filename; + return false; + } + info.push_back(std::move(data)); } - info.push_back(std::move(data)); } + SCOPED_WALLET_UNLOCK_ON_BAD_PASSWORD(return false;); + // all read and parsed, actually import try { @@ -1158,7 +1324,7 @@ bool simple_wallet::import_multisig(const std::vector<std::string> &args) catch (const std::exception &e) { fail_msg_writer() << tr("Failed to import multisig info: ") << e.what(); - return true; + return false; } if (m_wallet->is_trusted_daemon()) { @@ -1169,11 +1335,13 @@ bool simple_wallet::import_multisig(const std::vector<std::string> &args) catch (const std::exception &e) { message_writer() << tr("Failed to update spent status after importing multisig info: ") << e.what(); + return false; } } else { message_writer() << tr("Untrusted daemon, spent status may be incorrect. Use a trusted daemon and run \"rescan_spent\""); + return false; } return true; } @@ -1186,51 +1354,93 @@ bool simple_wallet::accept_loaded_tx(const tools::wallet2::multisig_tx_set &txs) bool simple_wallet::sign_multisig(const std::vector<std::string> &args) { + sign_multisig_main(args, false); + return true; +} + +bool simple_wallet::sign_multisig_main(const std::vector<std::string> &args, bool called_by_mms) +{ bool ready; if (m_wallet->key_on_device()) { fail_msg_writer() << tr("command not supported by HW wallet"); - return true; + return false; } if(!m_wallet->multisig(&ready)) { fail_msg_writer() << tr("This is not a multisig wallet"); - return true; + return false; } if (!ready) { fail_msg_writer() << tr("This multisig wallet is not yet finalized"); - return true; + return false; } if (args.size() != 1) { - fail_msg_writer() << tr("usage: sign_multisig <filename>"); - return true; + PRINT_USAGE(USAGE_SIGN_MULTISIG); + return false; } - SCOPED_WALLET_UNLOCK(); + SCOPED_WALLET_UNLOCK_ON_BAD_PASSWORD(return false;); std::string filename = args[0]; std::vector<crypto::hash> txids; uint32_t signers = 0; try { - bool r = m_wallet->sign_multisig_tx_from_file(filename, txids, [&](const tools::wallet2::multisig_tx_set &tx){ signers = tx.m_signers.size(); return accept_loaded_tx(tx); }); - if (!r) + if (called_by_mms) { - fail_msg_writer() << tr("Failed to sign multisig transaction"); - return true; + tools::wallet2::multisig_tx_set exported_txs; + std::string ciphertext; + bool r = m_wallet->load_multisig_tx(args[0], exported_txs, [&](const tools::wallet2::multisig_tx_set &tx){ signers = tx.m_signers.size(); return accept_loaded_tx(tx); }); + if (r) + { + r = m_wallet->sign_multisig_tx(exported_txs, txids); + } + if (r) + { + ciphertext = m_wallet->save_multisig_tx(exported_txs); + if (ciphertext.empty()) + { + r = false; + } + } + if (r) + { + mms::message_type message_type = mms::message_type::fully_signed_tx; + if (txids.empty()) + { + message_type = mms::message_type::partially_signed_tx; + } + get_message_store().process_wallet_created_data(get_multisig_wallet_state(), message_type, ciphertext); + filename = "MMS"; // for the messages below + } + else + { + fail_msg_writer() << tr("Failed to sign multisig transaction"); + return false; + } + } + else + { + bool r = m_wallet->sign_multisig_tx_from_file(filename, txids, [&](const tools::wallet2::multisig_tx_set &tx){ signers = tx.m_signers.size(); return accept_loaded_tx(tx); }); + if (!r) + { + fail_msg_writer() << tr("Failed to sign multisig transaction"); + return false; + } } } catch (const tools::error::multisig_export_needed& e) { fail_msg_writer() << tr("Multisig error: ") << e.what(); - return true; + return false; } catch (const std::exception &e) { fail_msg_writer() << tr("Failed to sign multisig transaction: ") << e.what(); - return true; + return false; } if (txids.empty()) @@ -1259,49 +1469,67 @@ bool simple_wallet::sign_multisig(const std::vector<std::string> &args) bool simple_wallet::submit_multisig(const std::vector<std::string> &args) { + submit_multisig_main(args, false); + return true; +} + +bool simple_wallet::submit_multisig_main(const std::vector<std::string> &args, bool called_by_mms) +{ bool ready; uint32_t threshold; if (m_wallet->key_on_device()) { fail_msg_writer() << tr("command not supported by HW wallet"); - return true; + return false; } if (!m_wallet->multisig(&ready, &threshold)) { fail_msg_writer() << tr("This is not a multisig wallet"); - return true; + return false; } if (!ready) { fail_msg_writer() << tr("This multisig wallet is not yet finalized"); - return true; + return false; } if (args.size() != 1) { - fail_msg_writer() << tr("usage: submit_multisig <filename>"); - return true; + PRINT_USAGE(USAGE_SUBMIT_MULTISIG); + return false; } - SCOPED_WALLET_UNLOCK(); - if (!try_connect_to_daemon()) - return true; + return false; + + SCOPED_WALLET_UNLOCK_ON_BAD_PASSWORD(return false;); std::string filename = args[0]; try { tools::wallet2::multisig_tx_set txs; - bool r = m_wallet->load_multisig_tx_from_file(filename, txs, [&](const tools::wallet2::multisig_tx_set &tx){ return accept_loaded_tx(tx); }); - if (!r) + if (called_by_mms) { - fail_msg_writer() << tr("Failed to load multisig transaction from file"); - return true; + bool r = m_wallet->load_multisig_tx(args[0], txs, [&](const tools::wallet2::multisig_tx_set &tx){ return accept_loaded_tx(tx); }); + if (!r) + { + fail_msg_writer() << tr("Failed to load multisig transaction from MMS"); + return false; + } + } + else + { + bool r = m_wallet->load_multisig_tx_from_file(filename, txs, [&](const tools::wallet2::multisig_tx_set &tx){ return accept_loaded_tx(tx); }); + if (!r) + { + fail_msg_writer() << tr("Failed to load multisig transaction from file"); + return false; + } } if (txs.m_signers.size() < threshold) { fail_msg_writer() << (boost::format(tr("Multisig transaction signed by only %u signers, needs %u more signatures")) % txs.m_signers.size() % (threshold - txs.m_signers.size())).str(); - return true; + return false; } // actually commit the transactions @@ -1320,6 +1548,7 @@ bool simple_wallet::submit_multisig(const std::vector<std::string> &args) { LOG_ERROR("unknown error"); fail_msg_writer() << tr("unknown error"); + return false; } return true; @@ -1346,15 +1575,16 @@ bool simple_wallet::export_raw_multisig(const std::vector<std::string> &args) } if (args.size() != 1) { - fail_msg_writer() << tr("usage: export_raw_multisig <filename>"); + PRINT_USAGE(USAGE_EXPORT_RAW_MULTISIG_TX); return true; } - SCOPED_WALLET_UNLOCK(); - std::string filename = args[0]; if (m_wallet->confirm_export_overwrite() && !check_file_overwrite(filename)) return true; + + SCOPED_WALLET_UNLOCK(); + try { tools::wallet2::multisig_tx_set txs; @@ -1408,7 +1638,7 @@ bool simple_wallet::print_ring(const std::vector<std::string> &args) crypto::hash txid; if (args.size() != 1) { - fail_msg_writer() << tr("usage: print_ring <key_image|txid>"); + PRINT_USAGE(USAGE_PRINT_RING); return true; } @@ -1565,7 +1795,7 @@ bool simple_wallet::set_ring(const std::vector<std::string> &args) if (args.size() < 3) { - fail_msg_writer() << tr("usage: set_ring <filename> | ( <key_image> absolute|relative <index> [<index>...] )"); + PRINT_USAGE(USAGE_SET_RING); return true; } @@ -1640,7 +1870,7 @@ bool simple_wallet::blackball(const std::vector<std::string> &args) uint64_t amount = std::numeric_limits<uint64_t>::max(), offset, num_offsets; if (args.size() == 0) { - fail_msg_writer() << tr("usage: mark_output_spent <amount>/<offset> | <filename> [add]"); + PRINT_USAGE(USAGE_MARK_OUTPUT_SPENT); return true; } @@ -1729,7 +1959,7 @@ bool simple_wallet::unblackball(const std::vector<std::string> &args) std::pair<uint64_t, uint64_t> output; if (args.size() != 1) { - fail_msg_writer() << tr("usage: mark_output_unspent <amount>/<offset>"); + PRINT_USAGE(USAGE_MARK_OUTPUT_UNSPENT); return true; } @@ -1756,7 +1986,7 @@ bool simple_wallet::blackballed(const std::vector<std::string> &args) std::pair<uint64_t, uint64_t> output; if (args.size() != 1) { - fail_msg_writer() << tr("usage: is_output_spent <amount>/<offset>"); + PRINT_USAGE(USAGE_IS_OUTPUT_SPENT); return true; } @@ -1801,6 +2031,27 @@ bool simple_wallet::version(const std::vector<std::string> &args) return true; } +bool simple_wallet::cold_sign_tx(const std::vector<tools::wallet2::pending_tx>& ptx_vector, tools::wallet2::signed_tx_set &exported_txs, std::vector<cryptonote::address_parse_info> &dsts_info, std::function<bool(const tools::wallet2::signed_tx_set &)> accept_func) +{ + std::vector<std::string> tx_aux; + + message_writer(console_color_white, false) << tr("Please confirm the transaction on the device"); + + m_wallet->cold_sign_tx(ptx_vector, exported_txs, dsts_info, tx_aux); + + if (accept_func && !accept_func(exported_txs)) + { + MERROR("Transactions rejected by callback"); + return false; + } + + // aux info + m_wallet->cold_tx_aux_import(exported_txs.ptx, tx_aux); + + // import key images + return m_wallet->import_key_images(exported_txs.key_images); +} + bool simple_wallet::set_always_confirm_transfers(const std::vector<std::string> &args/* = std::vector<std::string>()*/) { const auto pwd_container = get_and_verify_password(); @@ -1986,6 +2237,8 @@ bool simple_wallet::set_refresh_type(const std::vector<std::string> &args/* = st bool simple_wallet::set_confirm_missing_payment_id(const std::vector<std::string> &args/* = std::vector<std::string>()*/) { + LONG_PAYMENT_ID_SUPPORT_CHECK(); + const auto pwd_container = get_and_verify_password(); if (pwd_container) { @@ -2253,12 +2506,58 @@ bool simple_wallet::set_ignore_fractional_outputs(const std::vector<std::string> return true; } +bool simple_wallet::set_track_uses(const std::vector<std::string> &args/* = std::vector<std::string>()*/) +{ + const auto pwd_container = get_and_verify_password(); + if (pwd_container) + { + parse_bool_and_use(args[1], [&](bool r) { + m_wallet->track_uses(r); + m_wallet->rewrite(m_wallet_file, pwd_container->password()); + }); + } + return true; +} + +bool simple_wallet::set_device_name(const std::vector<std::string> &args/* = std::vector<std::string>()*/) +{ + const auto pwd_container = get_and_verify_password(); + if (pwd_container) + { + if (args.size() == 0){ + fail_msg_writer() << tr("Device name not specified"); + return true; + } + + m_wallet->device_name(args[0]); + bool r = false; + try { + r = m_wallet->reconnect_device(); + if (!r){ + fail_msg_writer() << tr("Device reconnect failed"); + } + + } catch(const std::exception & e){ + MWARNING("Device reconnect failed: " << e.what()); + fail_msg_writer() << tr("Device reconnect failed: ") << e.what(); + } + + } + return true; +} + bool simple_wallet::help(const std::vector<std::string> &args/* = std::vector<std::string>()*/) { if(args.empty()) { success_msg_writer() << get_commands_str(); } + else if ((args.size() == 2) && (args.front() == "mms")) + { + // Little hack to be able to do "help mms <subcommand>" + std::vector<std::string> mms_args(1, args.front() + " " + args.back()); + success_msg_writer() << get_command_usage(mms_args); + } else { success_msg_writer() << get_command_usage(args); @@ -2277,14 +2576,14 @@ simple_wallet::simple_wallet() { m_cmd_binder.set_handler("start_mining", boost::bind(&simple_wallet::start_mining, this, _1), - tr("start_mining [<number_of_threads>] [bg_mining] [ignore_battery]"), + tr(USAGE_START_MINING), tr("Start mining in the daemon (bg_mining and ignore_battery are optional booleans).")); m_cmd_binder.set_handler("stop_mining", boost::bind(&simple_wallet::stop_mining, this, _1), tr("Stop mining in the daemon.")); m_cmd_binder.set_handler("set_daemon", boost::bind(&simple_wallet::set_daemon, this, _1), - tr("set_daemon <host>[:<port>] [trusted|untrusted]"), + tr(USAGE_SET_DAEMON), tr("Set another daemon to connect to.")); m_cmd_binder.set_handler("save_bc", boost::bind(&simple_wallet::save_bc, this, _1), @@ -2294,68 +2593,64 @@ simple_wallet::simple_wallet() tr("Synchronize the transactions and balance.")); m_cmd_binder.set_handler("balance", boost::bind(&simple_wallet::show_balance, this, _1), - tr("balance [detail]"), + tr(USAGE_SHOW_BALANCE), tr("Show the wallet's balance of the currently selected account.")); m_cmd_binder.set_handler("incoming_transfers", boost::bind(&simple_wallet::show_incoming_transfers, this, _1), - tr("incoming_transfers [available|unavailable] [verbose] [index=<N1>[,<N2>[,...]]]"), - tr("Show the incoming transfers, all or filtered by availability and address index.")); + tr(USAGE_INCOMING_TRANSFERS), + tr("Show the incoming transfers, all or filtered by availability and address index.\n\n" + "Output format:\n" + "Amount, Spent(\"T\"|\"F\"), \"locked\"|\"unlocked\", RingCT, Global Index, Transaction Hash, Address Index, [Public Key, Key Image] ")); m_cmd_binder.set_handler("payments", boost::bind(&simple_wallet::show_payments, this, _1), - tr("payments <PID_1> [<PID_2> ... <PID_N>]"), + tr(USAGE_PAYMENTS), tr("Show the payments for the given payment IDs.")); m_cmd_binder.set_handler("bc_height", boost::bind(&simple_wallet::show_blockchain_height, this, _1), tr("Show the blockchain height.")); m_cmd_binder.set_handler("transfer", boost::bind(&simple_wallet::transfer, this, _1), - tr("transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] (<URI> | <address> <amount>) [<payment_id>]"), + tr(USAGE_TRANSFER), tr("Transfer <amount> to <address>. If the parameter \"index=<N1>[,<N2>,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. <priority> is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability. Multiple payments can be made at once by adding URI_2 or <address_2> <amount_2> etcetera (before the payment ID, if it's included)")); m_cmd_binder.set_handler("locked_transfer", boost::bind(&simple_wallet::locked_transfer, this, _1), - tr("locked_transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] (<URI> | <addr> <amount>) <lockblocks> [<payment_id>]"), + tr(USAGE_LOCKED_TRANSFER), tr("Transfer <amount> to <address> and lock it for <lockblocks> (max. 1000000). If the parameter \"index=<N1>[,<N2>,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. <priority> is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability. Multiple payments can be made at once by adding URI_2 or <address_2> <amount_2> etcetera (before the payment ID, if it's included)")); m_cmd_binder.set_handler("locked_sweep_all", boost::bind(&simple_wallet::locked_sweep_all, this, _1), - tr("locked_sweep_all [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> <lockblocks> [<payment_id>]"), + tr(USAGE_LOCKED_SWEEP_ALL), tr("Send all unlocked balance to an address and lock it for <lockblocks> (max. 1000000). If the parameter \"index<N1>[,<N2>,...]\" is specified, the wallet sweeps outputs received by those address indices. If omitted, the wallet randomly chooses an address index to be used. <priority> is the priority of the sweep. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability.")); m_cmd_binder.set_handler("sweep_unmixable", boost::bind(&simple_wallet::sweep_unmixable, this, _1), tr("Send all unmixable outputs to yourself with ring_size 1")); m_cmd_binder.set_handler("sweep_all", boost::bind(&simple_wallet::sweep_all, this, _1), - tr("sweep_all [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] [outputs=<N>] <address> [<payment_id>]"), + tr(USAGE_SWEEP_ALL), tr("Send all unlocked balance to an address. If the parameter \"index<N1>[,<N2>,...]\" is specified, the wallet sweeps outputs received by those address indices. If omitted, the wallet randomly chooses an address index to be used. If the parameter \"outputs=<N>\" is specified and N > 0, wallet splits the transaction into N even outputs.")); m_cmd_binder.set_handler("sweep_below", boost::bind(&simple_wallet::sweep_below, this, _1), - tr("sweep_below <amount_threshold> [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> [<payment_id>]"), + tr(USAGE_SWEEP_BELOW), tr("Send all unlocked outputs below the threshold to an address.")); m_cmd_binder.set_handler("sweep_single", boost::bind(&simple_wallet::sweep_single, this, _1), - tr("sweep_single [<priority>] [<ring_size>] [outputs=<N>] <key_image> <address> [<payment_id>]"), + tr(USAGE_SWEEP_SINGLE), tr("Send a single output of the given key image to an address without change.")); m_cmd_binder.set_handler("donate", boost::bind(&simple_wallet::donate, this, _1), - tr("donate [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <amount> [<payment_id>]"), + tr(USAGE_DONATE), tr("Donate <amount> to the development team (donate.getmonero.org).")); m_cmd_binder.set_handler("sign_transfer", boost::bind(&simple_wallet::sign_transfer, this, _1), - tr("sign_transfer [export_raw]"), + tr(USAGE_SIGN_TRANSFER), tr("Sign a transaction from a file. If the parameter \"export_raw\" is specified, transaction raw hex data suitable for the daemon RPC /sendrawtransaction is exported.")); m_cmd_binder.set_handler("submit_transfer", boost::bind(&simple_wallet::submit_transfer, this, _1), tr("Submit a signed transaction from a file.")); m_cmd_binder.set_handler("set_log", boost::bind(&simple_wallet::set_log, this, _1), - tr("set_log <level>|{+,-,}<categories>"), + tr(USAGE_SET_LOG), tr("Change the current log detail (level must be <0-4>).")); m_cmd_binder.set_handler("account", boost::bind(&simple_wallet::account, this, _1), - tr("account\n" - " account new <label text with white spaces allowed>\n" - " account switch <index> \n" - " account label <index> <label text with white spaces allowed>\n" - " account tag <tag_name> <account_index_1> [<account_index_2> ...]\n" - " account untag <account_index_1> [<account_index_2> ...]\n" - " account tag_description <tag_name> <description>"), + tr(USAGE_ACCOUNT), tr("If no arguments are specified, the wallet shows all the existing accounts along with their balances.\n" "If the \"new\" argument is specified, the wallet creates a new account with its label initialized by the provided label text (which can be empty).\n" "If the \"switch\" argument is specified, the wallet switches to the account specified by <index>.\n" @@ -2365,15 +2660,15 @@ simple_wallet::simple_wallet() "If the \"tag_description\" argument is specified, the tag <tag_name> is assigned an arbitrary text <description>.")); m_cmd_binder.set_handler("address", boost::bind(&simple_wallet::print_address, this, _1), - tr("address [ new <label text with white spaces allowed> | all | <index_min> [<index_max>] | label <index> <label text with white spaces allowed>]"), + tr(USAGE_ADDRESS), tr("If no arguments are specified or <index> is specified, the wallet shows the default or specified address. If \"all\" is specified, the wallet shows all the existing addresses in the currently selected account. If \"new \" is specified, the wallet creates a new address with the provided label text (which can be empty). If \"label\" is specified, the wallet sets the label of the address specified by <index> to the provided label text.")); m_cmd_binder.set_handler("integrated_address", boost::bind(&simple_wallet::print_integrated_address, this, _1), - tr("integrated_address [<payment_id> | <address>]"), + tr(USAGE_INTEGRATED_ADDRESS), tr("Encode a payment ID into an integrated address for the current wallet public address (no argument uses a random payment ID), or decode an integrated address to standard address and payment ID")); m_cmd_binder.set_handler("address_book", boost::bind(&simple_wallet::address_book, this, _1), - tr("address_book [(add ((<address> [pid <id>])|<integrated address>) [<description possibly with whitespaces>])|(delete <index>)]"), + tr(USAGE_ADDRESS_BOOK), tr("Print all entries in the address book, optionally adding/deleting an entry to/from it.")); m_cmd_binder.set_handler("save", boost::bind(&simple_wallet::save, this, _1), @@ -2392,7 +2687,7 @@ simple_wallet::simple_wallet() tr("Display the Electrum-style mnemonic seed")); m_cmd_binder.set_handler("set", boost::bind(&simple_wallet::set_variable, this, _1), - tr("set <option> [<value>]"), + tr(USAGE_SET_VARIABLE), tr("Available options:\n " "seed language\n " " Set the wallet's seed language.\n " @@ -2445,67 +2740,81 @@ simple_wallet::simple_wallet() tr("Rescan the blockchain for spent outputs.")); m_cmd_binder.set_handler("get_tx_key", boost::bind(&simple_wallet::get_tx_key, this, _1), - tr("get_tx_key <txid>"), + tr(USAGE_GET_TX_KEY), tr("Get the transaction key (r) for a given <txid>.")); m_cmd_binder.set_handler("set_tx_key", boost::bind(&simple_wallet::set_tx_key, this, _1), - tr("set_tx_key <txid> <tx_key>"), + tr(USAGE_SET_TX_KEY), tr("Set the transaction key (r) for a given <txid> in case the tx was made by some other device or 3rd party wallet.")); m_cmd_binder.set_handler("check_tx_key", boost::bind(&simple_wallet::check_tx_key, this, _1), - tr("check_tx_key <txid> <txkey> <address>"), + tr(USAGE_CHECK_TX_KEY), tr("Check the amount going to <address> in <txid>.")); m_cmd_binder.set_handler("get_tx_proof", boost::bind(&simple_wallet::get_tx_proof, this, _1), - tr("get_tx_proof <txid> <address> [<message>]"), + tr(USAGE_GET_TX_PROOF), tr("Generate a signature proving funds sent to <address> in <txid>, optionally with a challenge string <message>, using either the transaction secret key (when <address> is not your wallet's address) or the view secret key (otherwise), which does not disclose the secret key.")); m_cmd_binder.set_handler("check_tx_proof", boost::bind(&simple_wallet::check_tx_proof, this, _1), - tr("check_tx_proof <txid> <address> <signature_file> [<message>]"), + tr(USAGE_CHECK_TX_PROOF), tr("Check the proof for funds going to <address> in <txid> with the challenge string <message> if any.")); m_cmd_binder.set_handler("get_spend_proof", boost::bind(&simple_wallet::get_spend_proof, this, _1), - tr("get_spend_proof <txid> [<message>]"), + tr(USAGE_GET_SPEND_PROOF), tr("Generate a signature proving that you generated <txid> using the spend secret key, optionally with a challenge string <message>.")); m_cmd_binder.set_handler("check_spend_proof", boost::bind(&simple_wallet::check_spend_proof, this, _1), - tr("check_spend_proof <txid> <signature_file> [<message>]"), + tr(USAGE_CHECK_SPEND_PROOF), tr("Check a signature proving that the signer generated <txid>, optionally with a challenge string <message>.")); m_cmd_binder.set_handler("get_reserve_proof", boost::bind(&simple_wallet::get_reserve_proof, this, _1), - tr("get_reserve_proof (all|<amount>) [<message>]"), + tr(USAGE_GET_RESERVE_PROOF), tr("Generate a signature proving that you own at least this much, optionally with a challenge string <message>.\n" "If 'all' is specified, you prove the entire sum of all of your existing accounts' balances.\n" "Otherwise, you prove the reserve of the smallest possible amount above <amount> available in your current account.")); m_cmd_binder.set_handler("check_reserve_proof", boost::bind(&simple_wallet::check_reserve_proof, this, _1), - tr("check_reserve_proof <address> <signature_file> [<message>]"), + tr(USAGE_CHECK_RESERVE_PROOF), tr("Check a signature proving that the owner of <address> holds at least this much, optionally with a challenge string <message>.")); m_cmd_binder.set_handler("show_transfers", boost::bind(&simple_wallet::show_transfers, this, _1), - tr("show_transfers [in|out|pending|failed|pool|coinbase] [index=<N1>[,<N2>,...]] [<min_height> [<max_height>]]"), - tr("Show the incoming/outgoing transfers within an optional height range.")); + tr(USAGE_SHOW_TRANSFERS), + // Seemingly broken formatting to compensate for the backslash before the quotes. + tr("Show the incoming/outgoing transfers within an optional height range.\n\n" + "Output format:\n" + "In or Coinbase: Block Number, \"block\"|\"in\", Time, Amount, Transaction Hash, Payment ID, Subaddress Index, \"-\", Note\n" + "Out: Block Number, \"out\", Time, Amount*, Transaction Hash, Payment ID, Fee, Destinations, Input addresses**, \"-\", Note\n" + "Pool: \"pool\", \"in\", Time, Amount, Transaction Hash, Payment Id, Subaddress Index, \"-\", Note, Double Spend Note\n" + "Pending or Failed: \"failed\"|\"pending\", \"out\", Time, Amount*, Transaction Hash, Payment ID, Fee, Input addresses**, \"-\", Note\n\n" + "* Excluding change and fee.\n" + "** Set of address indices used as inputs in this transfer.")); + m_cmd_binder.set_handler("export_transfers", + boost::bind(&simple_wallet::export_transfers, this, _1), + tr("export_transfers [in|out|all|pending|failed|coinbase] [index=<N1>[,<N2>,...]] [<min_height> [<max_height>]] [output=<filepath>]"), + tr("Export to CSV the incoming/outgoing transfers within an optional height range.")); m_cmd_binder.set_handler("unspent_outputs", boost::bind(&simple_wallet::unspent_outputs, this, _1), - tr("unspent_outputs [index=<N1>[,<N2>,...]] [<min_amount> [<max_amount>]]"), + tr(USAGE_UNSPENT_OUTPUTS), tr("Show the unspent outputs of a specified address within an optional amount range.")); m_cmd_binder.set_handler("rescan_bc", boost::bind(&simple_wallet::rescan_blockchain, this, _1), - tr("Rescan the blockchain from scratch, losing any information which can not be recovered from the blockchain itself.")); + tr(USAGE_RESCAN_BC), + tr("Rescan the blockchain from scratch. If \"hard\" is specified, you will lose any information which can not be recovered from the blockchain itself.")); m_cmd_binder.set_handler("set_tx_note", boost::bind(&simple_wallet::set_tx_note, this, _1), - tr("set_tx_note <txid> [free text note]"), + tr(USAGE_SET_TX_NOTE), tr("Set an arbitrary string note for a <txid>.")); m_cmd_binder.set_handler("get_tx_note", boost::bind(&simple_wallet::get_tx_note, this, _1), - tr("get_tx_note <txid>"), + tr(USAGE_GET_TX_NOTE), tr("Get a string note for a txid.")); m_cmd_binder.set_handler("set_description", boost::bind(&simple_wallet::set_description, this, _1), - tr("set_description [free text note]"), + tr(USAGE_SET_DESCRIPTION), tr("Set an arbitrary description for the wallet.")); m_cmd_binder.set_handler("get_description", boost::bind(&simple_wallet::get_description, this, _1), + tr(USAGE_GET_DESCRIPTION), tr("Get the description of the wallet.")); m_cmd_binder.set_handler("status", boost::bind(&simple_wallet::status, this, _1), @@ -2515,109 +2824,199 @@ simple_wallet::simple_wallet() tr("Show the wallet's information.")); m_cmd_binder.set_handler("sign", boost::bind(&simple_wallet::sign, this, _1), - tr("sign <file>"), + tr(USAGE_SIGN), tr("Sign the contents of a file.")); m_cmd_binder.set_handler("verify", boost::bind(&simple_wallet::verify, this, _1), - tr("verify <filename> <address> <signature>"), + tr(USAGE_VERIFY), tr("Verify a signature on the contents of a file.")); m_cmd_binder.set_handler("export_key_images", boost::bind(&simple_wallet::export_key_images, this, _1), - tr("export_key_images <file>"), - tr("Export a signed set of key images to a <file>.")); + tr(USAGE_EXPORT_KEY_IMAGES), + tr("Export a signed set of key images to a <filename>.")); m_cmd_binder.set_handler("import_key_images", boost::bind(&simple_wallet::import_key_images, this, _1), - tr("import_key_images <file>"), + tr(USAGE_IMPORT_KEY_IMAGES), tr("Import a signed key images list and verify their spent status.")); + m_cmd_binder.set_handler("hw_key_images_sync", + boost::bind(&simple_wallet::hw_key_images_sync, this, _1), + tr(USAGE_HW_KEY_IMAGES_SYNC), + tr("Synchronizes key images with the hw wallet.")); m_cmd_binder.set_handler("hw_reconnect", boost::bind(&simple_wallet::hw_reconnect, this, _1), - tr("hw_reconnect"), + tr(USAGE_HW_RECONNECT), tr("Attempts to reconnect HW wallet.")); m_cmd_binder.set_handler("export_outputs", boost::bind(&simple_wallet::export_outputs, this, _1), - tr("export_outputs <file>"), + tr(USAGE_EXPORT_OUTPUTS), tr("Export a set of outputs owned by this wallet.")); m_cmd_binder.set_handler("import_outputs", boost::bind(&simple_wallet::import_outputs, this, _1), - tr("import_outputs <file>"), + tr(USAGE_IMPORT_OUTPUTS), tr("Import a set of outputs owned by this wallet.")); m_cmd_binder.set_handler("show_transfer", boost::bind(&simple_wallet::show_transfer, this, _1), - tr("show_transfer <txid>"), + tr(USAGE_SHOW_TRANSFER), tr("Show information about a transfer to/from this address.")); m_cmd_binder.set_handler("password", boost::bind(&simple_wallet::change_password, this, _1), tr("Change the wallet's password.")); m_cmd_binder.set_handler("payment_id", boost::bind(&simple_wallet::payment_id, this, _1), - tr("Generate a new random full size payment id. These will be unencrypted on the blockchain, see integrated_address for encrypted short payment ids.")); + tr(USAGE_PAYMENT_ID), + tr("Generate a new random full size payment id (obsolete). These will be unencrypted on the blockchain, see integrated_address for encrypted short payment ids.")); m_cmd_binder.set_handler("fee", boost::bind(&simple_wallet::print_fee_info, this, _1), tr("Print the information about the current fee and transaction backlog.")); m_cmd_binder.set_handler("prepare_multisig", boost::bind(&simple_wallet::prepare_multisig, this, _1), tr("Export data needed to create a multisig wallet")); m_cmd_binder.set_handler("make_multisig", boost::bind(&simple_wallet::make_multisig, this, _1), - tr("make_multisig <threshold> <string1> [<string>...]"), + tr(USAGE_MAKE_MULTISIG), tr("Turn this wallet into a multisig wallet")); m_cmd_binder.set_handler("finalize_multisig", boost::bind(&simple_wallet::finalize_multisig, this, _1), - tr("finalize_multisig <string> [<string>...]"), + tr(USAGE_FINALIZE_MULTISIG), tr("Turn this wallet into a multisig wallet, extra step for N-1/N wallets")); m_cmd_binder.set_handler("exchange_multisig_keys", boost::bind(&simple_wallet::exchange_multisig_keys, this, _1), - tr("exchange_multisig_keys <string> [<string>...]"), + tr(USAGE_EXCHANGE_MULTISIG_KEYS), tr("Performs extra multisig keys exchange rounds. Needed for arbitrary M/N multisig wallets")); m_cmd_binder.set_handler("export_multisig_info", boost::bind(&simple_wallet::export_multisig, this, _1), - tr("export_multisig_info <filename>"), + tr(USAGE_EXPORT_MULTISIG_INFO), tr("Export multisig info for other participants")); m_cmd_binder.set_handler("import_multisig_info", boost::bind(&simple_wallet::import_multisig, this, _1), - tr("import_multisig_info <filename> [<filename>...]"), + tr(USAGE_IMPORT_MULTISIG_INFO), tr("Import multisig info from other participants")); m_cmd_binder.set_handler("sign_multisig", boost::bind(&simple_wallet::sign_multisig, this, _1), - tr("sign_multisig <filename>"), + tr(USAGE_SIGN_MULTISIG), tr("Sign a multisig transaction from a file")); m_cmd_binder.set_handler("submit_multisig", boost::bind(&simple_wallet::submit_multisig, this, _1), - tr("submit_multisig <filename>"), + tr(USAGE_SUBMIT_MULTISIG), tr("Submit a signed multisig transaction from a file")); m_cmd_binder.set_handler("export_raw_multisig_tx", boost::bind(&simple_wallet::export_raw_multisig, this, _1), - tr("export_raw_multisig_tx <filename>"), + tr(USAGE_EXPORT_RAW_MULTISIG_TX), tr("Export a signed multisig transaction to a file")); + m_cmd_binder.set_handler("mms", + boost::bind(&simple_wallet::mms, this, _1), + tr(USAGE_MMS), + tr("Interface with the MMS (Multisig Messaging System)\n" + "<subcommand> is one of:\n" + " init, info, signer, list, next, sync, transfer, delete, send, receive, export, note, show, set, help\n" + " send_signer_config, start_auto_config, stop_auto_config, auto_config\n" + "Get help about a subcommand with: help mms <subcommand>, or mms help <subcommand>")); + m_cmd_binder.set_handler("mms init", + boost::bind(&simple_wallet::mms, this, _1), + tr(USAGE_MMS_INIT), + tr("Initialize and configure the MMS for M/N = number of required signers/number of authorized signers multisig")); + m_cmd_binder.set_handler("mms info", + boost::bind(&simple_wallet::mms, this, _1), + tr(USAGE_MMS_INFO), + tr("Display current MMS configuration")); + m_cmd_binder.set_handler("mms signer", + boost::bind(&simple_wallet::mms, this, _1), + tr(USAGE_MMS_SIGNER), + tr("Set or modify authorized signer info (single-word label, transport address, Monero address), or list all signers")); + m_cmd_binder.set_handler("mms list", + boost::bind(&simple_wallet::mms, this, _1), + tr(USAGE_MMS_LIST), + tr("List all messages")); + m_cmd_binder.set_handler("mms next", + boost::bind(&simple_wallet::mms, this, _1), + tr(USAGE_MMS_NEXT), + tr("Evaluate the next possible multisig-related action(s) according to wallet state, and execute or offer for choice\n" + "By using 'sync' processing of waiting messages with multisig sync info can be forced regardless of wallet state")); + m_cmd_binder.set_handler("mms sync", + boost::bind(&simple_wallet::mms, this, _1), + tr(USAGE_MMS_SYNC), + tr("Force generation of multisig sync info regardless of wallet state, to recover from special situations like \"stale data\" errors")); + m_cmd_binder.set_handler("mms transfer", + boost::bind(&simple_wallet::mms, this, _1), + tr(USAGE_MMS_TRANSFER), + tr("Initiate transfer with MMS support; arguments identical to normal 'transfer' command arguments, for info see there")); + m_cmd_binder.set_handler("mms delete", + boost::bind(&simple_wallet::mms, this, _1), + tr(USAGE_MMS_DELETE), + tr("Delete a single message by giving its id, or delete all messages by using 'all'")); + m_cmd_binder.set_handler("mms send", + boost::bind(&simple_wallet::mms, this, _1), + tr(USAGE_MMS_SEND), + tr("Send a single message by giving its id, or send all waiting messages")); + m_cmd_binder.set_handler("mms receive", + boost::bind(&simple_wallet::mms, this, _1), + tr(USAGE_MMS_RECEIVE), + tr("Check right away for new messages to receive")); + m_cmd_binder.set_handler("mms export", + boost::bind(&simple_wallet::mms, this, _1), + tr(USAGE_MMS_EXPORT), + tr("Write the content of a message to a file \"mms_message_content\"")); + m_cmd_binder.set_handler("mms note", + boost::bind(&simple_wallet::mms, this, _1), + tr(USAGE_MMS_NOTE), + tr("Send a one-line message to an authorized signer, identified by its label, or show any waiting unread notes")); + m_cmd_binder.set_handler("mms show", + boost::bind(&simple_wallet::mms, this, _1), + tr(USAGE_MMS_SHOW), + tr("Show detailed info about a single message")); + m_cmd_binder.set_handler("mms set", + boost::bind(&simple_wallet::mms, this, _1), + tr(USAGE_MMS_SET), + tr("Available options:\n " + "auto-send <1|0>\n " + " Whether to automatically send newly generated messages right away.\n ")); + m_cmd_binder.set_handler("mms send_message_config", + boost::bind(&simple_wallet::mms, this, _1), + tr(USAGE_MMS_SEND_SIGNER_CONFIG), + tr("Send completed signer config to all other authorized signers")); + m_cmd_binder.set_handler("mms start_auto_config", + boost::bind(&simple_wallet::mms, this, _1), + tr(USAGE_MMS_START_AUTO_CONFIG), + tr("Start auto-config at the auto-config manager's wallet by issuing auto-config tokens and optionally set others' labels")); + m_cmd_binder.set_handler("mms stop_auto_config", + boost::bind(&simple_wallet::mms, this, _1), + tr(USAGE_MMS_STOP_AUTO_CONFIG), + tr("Delete any auto-config tokens and abort a auto-config process")); + m_cmd_binder.set_handler("mms auto_config", + boost::bind(&simple_wallet::mms, this, _1), + tr(USAGE_MMS_AUTO_CONFIG), + tr("Start auto-config by using the token received from the auto-config manager")); m_cmd_binder.set_handler("print_ring", boost::bind(&simple_wallet::print_ring, this, _1), - tr("print_ring <key_image> | <txid>"), - tr("Print the ring(s) used to spend a given key image or transaction (if the ring size is > 1)")); + tr(USAGE_PRINT_RING), + tr("Print the ring(s) used to spend a given key image or transaction (if the ring size is > 1)\n\n" + "Output format:\n" + "Key Image, \"absolute\", list of rings")); m_cmd_binder.set_handler("set_ring", boost::bind(&simple_wallet::set_ring, this, _1), - tr("set_ring <filename> | ( <key_image> absolute|relative <index> [<index>...] )"), + tr(USAGE_SET_RING), tr("Set the ring used for a given key image, so it can be reused in a fork")); m_cmd_binder.set_handler("save_known_rings", boost::bind(&simple_wallet::save_known_rings, this, _1), - tr("save_known_rings"), + tr(USAGE_SAVE_KNOWN_RINGS), tr("Save known rings to the shared rings database")); m_cmd_binder.set_handler("mark_output_spent", boost::bind(&simple_wallet::blackball, this, _1), - tr("mark_output_spent <amount>/<offset> | <filename> [add]"), + tr(USAGE_MARK_OUTPUT_SPENT), tr("Mark output(s) as spent so they never get selected as fake outputs in a ring")); m_cmd_binder.set_handler("mark_output_unspent", boost::bind(&simple_wallet::unblackball, this, _1), - tr("mark_output_unspent <amount>/<offset>"), + tr(USAGE_MARK_OUTPUT_UNSPENT), tr("Marks an output as unspent so it may get selected as a fake output in a ring")); m_cmd_binder.set_handler("is_output_spent", boost::bind(&simple_wallet::blackballed, this, _1), - tr("is_output_spent <amount>/<offset>"), + tr(USAGE_IS_OUTPUT_SPENT), tr("Checks whether an output is marked as spent")); m_cmd_binder.set_handler("version", boost::bind(&simple_wallet::version, this, _1), - tr("version"), + tr(USAGE_VERSION), tr("Returns version information")); m_cmd_binder.set_handler("help", boost::bind(&simple_wallet::help, this, _1), - tr("help [<command>]"), + tr(USAGE_HELP), tr("Show the help section or the documentation about a <command>.")); } //---------------------------------------------------------------------------------------------------- @@ -2664,6 +3063,7 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args) success_msg_writer() << "subaddress-lookahead = " << lookahead.first << ":" << lookahead.second; success_msg_writer() << "segregation-height = " << m_wallet->segregation_height(); success_msg_writer() << "ignore-fractional-outputs = " << m_wallet->ignore_fractional_outputs(); + success_msg_writer() << "track-uses = " << m_wallet->track_uses(); success_msg_writer() << "device_name = " << m_wallet->device_name(); return true; } @@ -2720,6 +3120,8 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args) CHECK_SIMPLE_VARIABLE("subaddress-lookahead", set_subaddress_lookahead, tr("<major>:<minor>")); CHECK_SIMPLE_VARIABLE("segregation-height", set_segregation_height, tr("unsigned integer")); CHECK_SIMPLE_VARIABLE("ignore-fractional-outputs", set_ignore_fractional_outputs, tr("0 or 1")); + CHECK_SIMPLE_VARIABLE("track-uses", set_track_uses, tr("0 or 1")); + CHECK_SIMPLE_VARIABLE("device-name", set_device_name, tr("<device_name[:device_spec]>")); } fail_msg_writer() << tr("set: unrecognized argument(s)"); return true; @@ -2730,7 +3132,7 @@ bool simple_wallet::set_log(const std::vector<std::string> &args) { if(args.size() > 1) { - fail_msg_writer() << tr("usage: set_log <log_level_number_0-4> | <categories>"); + PRINT_USAGE(USAGE_SET_LOG); return true; } if(!args.empty()) @@ -2740,7 +3142,7 @@ bool simple_wallet::set_log(const std::vector<std::string> &args) { if(4 < level) { - fail_msg_writer() << tr("wrong number range, use: set_log <log_level_number_0-4> | <categories>"); + fail_msg_writer() << boost::format(tr("wrong number range, use: %s")) % USAGE_SET_LOG; return true; } mlog_set_log_level(level); @@ -2768,9 +3170,9 @@ bool simple_wallet::ask_wallet_create_if_needed() LOG_PRINT_L3("User asked to specify wallet file name."); wallet_path = input_line( tr(m_restoring ? "Specify a new wallet file name for your restored wallet (e.g., MyWallet).\n" - "Wallet file name (or Ctrl-C to quit): " : + "Wallet file name (or Ctrl-C to quit)" : "Specify wallet file name (e.g., MyWallet). If the wallet doesn't exist, it will be created.\n" - "Wallet file name (or Ctrl-C to quit): ") + "Wallet file name (or Ctrl-C to quit)") ); if(std::cin.eof()) { @@ -2817,7 +3219,7 @@ bool simple_wallet::ask_wallet_create_if_needed() if (!m_restoring) { message_writer() << tr("No wallet found with that name. Confirm creation of new wallet named: ") << wallet_path; - confirm_creation = input_line(tr("(Y/Yes/N/No): ")); + confirm_creation = input_line("", true); if(std::cin.eof()) { LOG_ERROR("Unexpected std::cin.eof() - Exited simple_wallet::ask_wallet_create_if_needed()"); @@ -2845,9 +3247,9 @@ bool simple_wallet::ask_wallet_create_if_needed() */ void simple_wallet::print_seed(const epee::wipeable_string &seed) { - success_msg_writer(true) << "\n" << tr("NOTE: the following 25 words can be used to recover access to your wallet. " + success_msg_writer(true) << "\n" << boost::format(tr("NOTE: the following %s can be used to recover access to your wallet. " "Write them down and store them somewhere safe and secure. Please do not store them in " - "your email or on file storage services outside of your immediate control.\n"); + "your email or on file storage services outside of your immediate control.\n")) % (m_wallet->multisig() ? tr("string") : tr("25 words")); // don't log int space_index = 0; size_t len = seed.size(); @@ -2876,6 +3278,28 @@ static bool might_be_partial_seed(const epee::wipeable_string &words) return seed.size() < 24; } //---------------------------------------------------------------------------------------------------- +static bool datestr_to_int(const std::string &heightstr, uint16_t &year, uint8_t &month, uint8_t &day) +{ + if (heightstr.size() != 10 || heightstr[4] != '-' || heightstr[7] != '-') + { + fail_msg_writer() << tr("date format must be YYYY-MM-DD"); + return false; + } + try + { + year = boost::lexical_cast<uint16_t>(heightstr.substr(0,4)); + // lexical_cast<uint8_t> won't work because uint8_t is treated as character type + month = boost::lexical_cast<uint16_t>(heightstr.substr(5,2)); + day = boost::lexical_cast<uint16_t>(heightstr.substr(8,2)); + } + catch (const boost::bad_lexical_cast &) + { + fail_msg_writer() << tr("bad height parameter: ") << heightstr; + return false; + } + return true; +} +//---------------------------------------------------------------------------------------------------- bool simple_wallet::init(const boost::program_options::variables_map& vm) { epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){ @@ -3004,7 +3428,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) { m_wallet_file = m_generate_from_view_key; // parse address - std::string address_string = input_line("Standard address: "); + std::string address_string = input_line("Standard address"); if (std::cin.eof()) return false; if (address_string.empty()) { @@ -3024,7 +3448,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) } // parse view secret key - epee::wipeable_string viewkey_string = input_secure_line("Secret view key: "); + epee::wipeable_string viewkey_string = input_secure_line("Secret view key"); if (std::cin.eof()) return false; if (viewkey_string.empty()) { @@ -3059,7 +3483,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) { m_wallet_file = m_generate_from_spend_key; // parse spend secret key - epee::wipeable_string spendkey_string = input_secure_line("Secret spend key: "); + epee::wipeable_string spendkey_string = input_secure_line("Secret spend key"); if (std::cin.eof()) return false; if (spendkey_string.empty()) { @@ -3079,7 +3503,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) { m_wallet_file = m_generate_from_keys; // parse address - std::string address_string = input_line("Standard address: "); + std::string address_string = input_line("Standard address"); if (std::cin.eof()) return false; if (address_string.empty()) { @@ -3099,7 +3523,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) } // parse spend secret key - epee::wipeable_string spendkey_string = input_secure_line("Secret spend key: "); + epee::wipeable_string spendkey_string = input_secure_line("Secret spend key"); if (std::cin.eof()) return false; if (spendkey_string.empty()) { @@ -3114,7 +3538,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) } // parse view secret key - epee::wipeable_string viewkey_string = input_secure_line("Secret view key: "); + epee::wipeable_string viewkey_string = input_secure_line("Secret view key"); if (std::cin.eof()) return false; if (viewkey_string.empty()) { @@ -3161,7 +3585,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) unsigned int multisig_n; // parse multisig type - std::string multisig_type_string = input_line("Multisig type (input as M/N with M <= N and M > 1): "); + std::string multisig_type_string = input_line("Multisig type (input as M/N with M <= N and M > 1)"); if (std::cin.eof()) return false; if (multisig_type_string.empty()) @@ -3187,7 +3611,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) message_writer() << boost::format(tr("Generating master wallet from %u of %u multisig wallet keys")) % multisig_m % multisig_n; // parse multisig address - std::string address_string = input_line("Multisig wallet address: "); + std::string address_string = input_line("Multisig wallet address"); if (std::cin.eof()) return false; if (address_string.empty()) { @@ -3202,7 +3626,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) } // parse secret view key - epee::wipeable_string viewkey_string = input_secure_line("Secret view key: "); + epee::wipeable_string viewkey_string = input_secure_line("Secret view key"); if (std::cin.eof()) return false; if (viewkey_string.empty()) @@ -3241,7 +3665,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) // get N secret spend keys from user for(unsigned int i=0; i<multisig_n; ++i) { - spendkey_string = input_secure_line(tr((boost::format(tr("Secret spend key (%u of %u):")) % (i+1) % multisig_m).str().c_str())); + spendkey_string = input_secure_line(tr((boost::format(tr("Secret spend key (%u of %u)")) % (i+1) % multisig_m).str().c_str())); if (std::cin.eof()) return false; if (spendkey_string.empty()) @@ -3314,15 +3738,15 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) if(m_wallet->get_refresh_from_block_height() == 0) { { tools::scoped_message_writer wrt = tools::msg_writer(); - wrt << tr("No restore height is specified."); - wrt << tr("Assumed you are creating a new account, restore will be done from current estimated blockchain height."); - wrt << tr("Use --restore-height if you want to restore an already setup account from a specific height"); + wrt << tr("No restore height is specified.") << " "; + wrt << tr("Assumed you are creating a new account, restore will be done from current estimated blockchain height.") << " "; + wrt << tr("Use --restore-height or --restore-date if you want to restore an already setup account from a specific height."); } - std::string confirm = input_line(tr("Is this okay? (Y/Yes/N/No): ")); + std::string confirm = input_line(tr("Is this okay?"), true); if (std::cin.eof() || !command_line::is_yes(confirm)) CHECK_AND_ASSERT_MES(false, false, tr("account creation aborted")); - m_wallet->set_refresh_from_block_height(m_wallet->estimate_blockchain_height()-1); + m_wallet->set_refresh_from_block_height(m_wallet->estimate_blockchain_height()); m_wallet->explicit_refresh_from_block_height(true); m_restore_height = m_wallet->get_refresh_from_block_height(); } @@ -3345,7 +3769,26 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) if (m_restoring && m_generate_from_json.empty() && m_generate_from_device.empty()) { - m_wallet->explicit_refresh_from_block_height(!command_line::is_arg_defaulted(vm, arg_restore_height)); + m_wallet->explicit_refresh_from_block_height(!(command_line::is_arg_defaulted(vm, arg_restore_height) || + command_line::is_arg_defaulted(vm, arg_restore_date))); + if (command_line::is_arg_defaulted(vm, arg_restore_height) && !command_line::is_arg_defaulted(vm, arg_restore_date)) + { + uint16_t year; + uint8_t month; + uint8_t day; + if (!datestr_to_int(m_restore_date, year, month, day)) + return false; + try + { + m_restore_height = m_wallet->get_blockchain_height_by_date(year, month, day); + success_msg_writer() << tr("Restore height is: ") << m_restore_height; + } + catch (const std::runtime_error& e) + { + fail_msg_writer() << e.what(); + return false; + } + } } if (!m_wallet->explicit_refresh_from_block_height() && m_restoring) { @@ -3355,9 +3798,9 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) { std::string heightstr; if (!connected || version < MAKE_CORE_RPC_VERSION(1, 6)) - heightstr = input_line("Restore from specific blockchain height (optional, default 0): "); + heightstr = input_line("Restore from specific blockchain height (optional, default 0)"); else - heightstr = input_line("Restore from specific blockchain height (optional, default 0),\nor alternatively from specific date (YYYY-MM-DD): "); + heightstr = input_line("Restore from specific blockchain height (optional, default 0),\nor alternatively from specific date (YYYY-MM-DD)"); if (std::cin.eof()) return false; if (heightstr.empty()) @@ -3377,23 +3820,16 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) fail_msg_writer() << tr("bad m_restore_height parameter: ") << heightstr; continue; } - if (heightstr.size() != 10 || heightstr[4] != '-' || heightstr[7] != '-') - { - fail_msg_writer() << tr("date format must be YYYY-MM-DD"); - continue; - } uint16_t year; uint8_t month; // 1, 2, ..., 12 uint8_t day; // 1, 2, ..., 31 try { - year = boost::lexical_cast<uint16_t>(heightstr.substr(0,4)); - // lexical_cast<uint8_t> won't work because uint8_t is treated as character type - month = boost::lexical_cast<uint16_t>(heightstr.substr(5,2)); - day = boost::lexical_cast<uint16_t>(heightstr.substr(8,2)); + if (!datestr_to_int(heightstr, year, month, day)) + return false; m_restore_height = m_wallet->get_blockchain_height_by_date(year, month, day); success_msg_writer() << tr("Restore height is: ") << m_restore_height; - std::string confirm = input_line(tr("Is this okay? (Y/Yes/N/No): ")); + std::string confirm = input_line(tr("Is this okay?"), true); if (std::cin.eof()) return false; if(command_line::is_yes(confirm)) @@ -3416,7 +3852,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) if (m_restore_height >= estimate_height) { success_msg_writer() << tr("Restore height ") << m_restore_height << (" is not yet reached. The current estimated height is ") << estimate_height; - std::string confirm = input_line(tr("Still apply restore height? (Y/Yes/N/No): ")); + std::string confirm = input_line(tr("Still apply restore height?"), true); if (std::cin.eof() || command_line::is_no(confirm)) m_restore_height = 0; } @@ -3477,9 +3913,11 @@ bool simple_wallet::handle_command_line(const boost::program_options::variables_ m_non_deterministic = command_line::get_arg(vm, arg_non_deterministic); m_allow_mismatched_daemon_version = command_line::get_arg(vm, arg_allow_mismatched_daemon_version); m_restore_height = command_line::get_arg(vm, arg_restore_height); + m_restore_date = command_line::get_arg(vm, arg_restore_date); m_do_not_relay = command_line::get_arg(vm, arg_do_not_relay); m_subaddress_lookahead = command_line::get_arg(vm, arg_subaddress_lookahead); m_use_english_language_names = command_line::get_arg(vm, arg_use_english_language_names); + m_long_payment_id_support = command_line::get_arg(vm, arg_long_payment_id_support); m_restoring = !m_generate_from_view_key.empty() || !m_generate_from_spend_key.empty() || !m_generate_from_keys.empty() || @@ -3489,6 +3927,14 @@ bool simple_wallet::handle_command_line(const boost::program_options::variables_ m_restore_deterministic_wallet || m_restore_multisig_wallet; + if (!command_line::is_arg_defaulted(vm, arg_restore_date)) + { + uint16_t year; + uint8_t month, day; + if (!datestr_to_int(m_restore_date, year, month, day)) + return false; + } + return true; } //---------------------------------------------------------------------------------------------------- @@ -3539,7 +3985,7 @@ std::string simple_wallet::get_mnemonic_language() } while (language_number < 0) { - language_choice = input_line(tr("Enter the number corresponding to the language of your choice: ")); + language_choice = input_line(tr("Enter the number corresponding to the language of your choice")); if (std::cin.eof()) return std::string(); try @@ -3719,6 +4165,7 @@ boost::optional<epee::wipeable_string> simple_wallet::new_wallet(const boost::pr { auto rc = tools::wallet2::make_new(vm, false, password_prompter); m_wallet = std::move(rc.first); + m_wallet->callback(this); if (!m_wallet) { return {}; @@ -3736,9 +4183,11 @@ boost::optional<epee::wipeable_string> simple_wallet::new_wallet(const boost::pr m_wallet->set_refresh_from_block_height(m_restore_height); auto device_desc = tools::wallet2::device_name_option(vm); + auto device_derivation_path = tools::wallet2::device_derivation_path_option(vm); try { bool create_address_file = command_line::get_arg(vm, arg_create_address_file); + m_wallet->device_derivation_path(device_derivation_path); m_wallet->restore(m_wallet_file, std::move(rc.second).password(), device_desc.empty() ? "Ledger" : device_desc, create_address_file); message_writer(console_color_white, true) << tr("Generated new wallet on hw device: ") << m_wallet->get_account().get_public_address_str(m_wallet->nettype()); @@ -3826,7 +4275,7 @@ bool simple_wallet::open_wallet(const boost::program_options::variables_map& vm) epee::wipeable_string password; try { - auto rc = tools::wallet2::make_from_file(vm, false, m_wallet_file, password_prompter); + auto rc = tools::wallet2::make_from_file(vm, false, "", password_prompter); m_wallet = std::move(rc.first); password = std::move(std::move(rc.second).password()); if (!m_wallet) @@ -3834,6 +4283,8 @@ bool simple_wallet::open_wallet(const boost::program_options::variables_map& vm) return false; } + m_wallet->callback(this); + m_wallet->load(m_wallet_file, password); std::string prefix; bool ready; uint32_t threshold, total; @@ -4031,7 +4482,7 @@ bool simple_wallet::start_mining(const std::vector<std::string>& args) if (!ok) { - fail_msg_writer() << tr("invalid arguments. Please use start_mining [<number_of_threads>] [do_bg_mining] [ignore_battery]"); + PRINT_USAGE(USAGE_START_MINING); return true; } @@ -4073,7 +4524,7 @@ bool simple_wallet::set_daemon(const std::vector<std::string>& args) if (args.size() < 1) { - fail_msg_writer() << tr("missing daemon URL argument"); + PRINT_USAGE(USAGE_SET_DAEMON); return true; } @@ -4173,14 +4624,10 @@ void simple_wallet::on_money_received(uint64_t height, const crypto::hash &txid, tx_extra_nonce extra_nonce; if (find_tx_extra_field_by_type(tx_extra_fields, extra_nonce)) { - crypto::hash8 payment_id8 = crypto::null_hash8; crypto::hash payment_id = crypto::null_hash; - if (get_encrypted_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id8)) - message_writer() << - tr("NOTE: this transaction uses an encrypted payment ID: consider using subaddresses instead"); - else if (get_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id)) + if (get_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id)) message_writer(console_color_red, false) << - tr("WARNING: this transaction uses an unencrypted payment ID: consider using subaddresses instead"); + (m_long_payment_id_support ? tr("WARNING: this transaction uses an unencrypted payment ID: consider using subaddresses instead.") : tr("WARNING: this transaction uses an unencrypted payment ID: these are obsolete. Support will be withdrawn in the future. Use subaddresses instead.")); } } if (m_auto_refresh_refreshing) @@ -4237,15 +4684,70 @@ boost::optional<epee::wipeable_string> simple_wallet::on_get_password(const char return pwd_container->password(); } //---------------------------------------------------------------------------------------------------- -bool simple_wallet::refresh_main(uint64_t start_height, bool reset, bool is_init) +void simple_wallet::on_button_request() +{ + message_writer(console_color_white, false) << tr("Device requires attention"); +} +//---------------------------------------------------------------------------------------------------- +void simple_wallet::on_pin_request(epee::wipeable_string & pin) +{ +#ifdef HAVE_READLINE + rdln::suspend_readline pause_readline; +#endif + std::string msg = tr("Enter device PIN"); + auto pwd_container = tools::password_container::prompt(false, msg.c_str()); + THROW_WALLET_EXCEPTION_IF(!pwd_container, tools::error::password_entry_failed, tr("Failed to read device PIN")); + pin = pwd_container->password(); +} +//---------------------------------------------------------------------------------------------------- +void simple_wallet::on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) +{ + if (on_device){ + message_writer(console_color_white, true) << tr("Please enter the device passphrase on the device"); + return; + } + +#ifdef HAVE_READLINE + rdln::suspend_readline pause_readline; +#endif + std::string msg = tr("Enter device passphrase"); + auto pwd_container = tools::password_container::prompt(false, msg.c_str()); + THROW_WALLET_EXCEPTION_IF(!pwd_container, tools::error::password_entry_failed, tr("Failed to read device passphrase")); + passphrase = pwd_container->password(); +} +//---------------------------------------------------------------------------------------------------- +void simple_wallet::on_refresh_finished(uint64_t start_height, uint64_t fetched_blocks, bool is_init, bool received_money) +{ + // Key image sync after the first refresh + if (!m_wallet->get_account().get_device().has_tx_cold_sign()) { + return; + } + + if (!received_money || m_wallet->get_device_last_key_image_sync() != 0) { + return; + } + + // Finished first refresh for HW device and money received -> KI sync + message_writer() << "\n" << tr("The first refresh has finished for the HW-based wallet with received money. hw_key_images_sync is needed. "); + + std::string accepted = input_line(tr("Do you want to do it now? (Y/Yes/N/No): ")); + if (std::cin.eof() || !command_line::is_yes(accepted)) { + message_writer(console_color_red, false) << tr("hw_key_images_sync skipped. Run command manually before a transfer."); + return; + } + + key_images_sync_intern(); +} +//---------------------------------------------------------------------------------------------------- +bool simple_wallet::refresh_main(uint64_t start_height, enum ResetType reset, bool is_init) { if (!try_connect_to_daemon(is_init)) return true; LOCK_IDLE_SCOPE(); - if (reset) - m_wallet->rescan_blockchain(false); + if (reset != ResetNone) + m_wallet->rescan_blockchain(reset == ResetHard, false); #ifdef HAVE_READLINE rdln::suspend_readline pause_readline; @@ -4254,13 +4756,14 @@ bool simple_wallet::refresh_main(uint64_t start_height, bool reset, bool is_init message_writer() << tr("Starting refresh..."); uint64_t fetched_blocks = 0; + bool received_money = false; bool ok = false; std::ostringstream ss; try { m_in_manual_refresh.store(true, std::memory_order_relaxed); epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){m_in_manual_refresh.store(false, std::memory_order_relaxed);}); - m_wallet->refresh(m_wallet->is_trusted_daemon(), start_height, fetched_blocks); + m_wallet->refresh(m_wallet->is_trusted_daemon(), start_height, fetched_blocks, received_money); ok = true; // Clear line "Height xxx of xxx" std::cout << "\r \r"; @@ -4268,6 +4771,7 @@ bool simple_wallet::refresh_main(uint64_t start_height, bool reset, bool is_init if (is_init) print_accounts(); show_balance_unlocked(); + on_refresh_finished(start_height, fetched_blocks, is_init, received_money); } catch (const tools::error::daemon_busy&) { @@ -4324,7 +4828,7 @@ bool simple_wallet::refresh(const std::vector<std::string>& args) start_height = 0; } } - return refresh_main(start_height, false); + return refresh_main(start_height, ResetNone); } //---------------------------------------------------------------------------------------------------- bool simple_wallet::show_balance_unlocked(bool detailed) @@ -4361,7 +4865,7 @@ bool simple_wallet::show_balance(const std::vector<std::string>& args/* = std::v { if (args.size() > 1 || (args.size() == 1 && args[0] != "detail")) { - fail_msg_writer() << tr("usage: balance [detail]"); + PRINT_USAGE(USAGE_SHOW_BALANCE); return true; } LOCK_IDLE_SCOPE(); @@ -4373,7 +4877,7 @@ bool simple_wallet::show_incoming_transfers(const std::vector<std::string>& args { if (args.size() > 3) { - fail_msg_writer() << tr("usage: incoming_transfers [available|unavailable] [verbose] [index=<N>]"); + PRINT_USAGE(USAGE_INCOMING_TRANSFERS); return true; } auto local_args = args; @@ -4382,6 +4886,7 @@ bool simple_wallet::show_incoming_transfers(const std::vector<std::string>& args bool filter = false; bool available = false; bool verbose = false; + bool uses = false; if (local_args.size() > 0) { if (local_args[0] == "available") @@ -4397,12 +4902,22 @@ bool simple_wallet::show_incoming_transfers(const std::vector<std::string>& args local_args.erase(local_args.begin()); } } - if (local_args.size() > 0 && local_args[0] == "verbose") + while (local_args.size() > 0) { - verbose = true; + if (local_args[0] == "verbose") + verbose = true; + else if (local_args[0] == "uses") + uses = true; + else + { + fail_msg_writer() << tr("Invalid keyword: ") << local_args.front(); + break; + } local_args.erase(local_args.begin()); } + const uint64_t blockchain_height = m_wallet->get_blockchain_current_height(); + PAUSE_READLINE(); std::set<uint32_t> subaddr_indices; @@ -4415,14 +4930,14 @@ bool simple_wallet::show_incoming_transfers(const std::vector<std::string>& args if (local_args.size() > 0) { - fail_msg_writer() << tr("usage: incoming_transfers [available|unavailable] [verbose] [index=<N>]"); + PRINT_USAGE(USAGE_INCOMING_TRANSFERS); return true; } tools::wallet2::transfer_container transfers; m_wallet->get_transfers(transfers); - bool transfers_found = false; + size_t transfers_found = 0; for (const auto& td : transfers) { if (!filter || available != td.m_spent) @@ -4435,11 +4950,17 @@ bool simple_wallet::show_incoming_transfers(const std::vector<std::string>& args if (verbose) verbose_string = (boost::format("%68s%68s") % tr("pubkey") % tr("key image")).str(); message_writer() << boost::format("%21s%8s%12s%8s%16s%68s%16s%s") % tr("amount") % tr("spent") % tr("unlocked") % tr("ringct") % tr("global index") % tr("tx id") % tr("addr index") % verbose_string; - transfers_found = true; } - std::string verbose_string; + std::string extra_string; if (verbose) - verbose_string = (boost::format("%68s%68s") % td.get_public_key() % (td.m_key_image_known ? epee::string_tools::pod_to_hex(td.m_key_image) : td.m_key_image_partial ? (epee::string_tools::pod_to_hex(td.m_key_image) + "/p") : std::string(64, '?'))).str(); + extra_string += (boost::format("%68s%68s") % td.get_public_key() % (td.m_key_image_known ? epee::string_tools::pod_to_hex(td.m_key_image) : td.m_key_image_partial ? (epee::string_tools::pod_to_hex(td.m_key_image) + "/p") : std::string(64, '?'))).str(); + if (uses) + { + std::vector<uint64_t> heights; + for (const auto &e: td.m_uses) heights.push_back(e.first); + const std::pair<std::string, std::string> line = show_outputs_line(heights, blockchain_height, td.m_spent_height); + extra_string += tr("Heights: ") + line.first + "\n" + line.second; + } message_writer(td.m_spent ? console_color_magenta : console_color_green, false) << boost::format("%21s%8s%12s%8s%16u%68s%16u%s") % print_money(td.amount()) % @@ -4449,7 +4970,8 @@ bool simple_wallet::show_incoming_transfers(const std::vector<std::string>& args td.m_global_output_index % td.m_txid % td.m_subaddr_index.minor % - verbose_string; + extra_string; + ++transfers_found; } } @@ -4468,6 +4990,10 @@ bool simple_wallet::show_incoming_transfers(const std::vector<std::string>& args success_msg_writer() << tr("No incoming unavailable transfers"); } } + else + { + success_msg_writer() << boost::format("Found %u/%u transfers") % transfers_found % transfers.size(); + } return true; } @@ -4476,7 +5002,7 @@ bool simple_wallet::show_payments(const std::vector<std::string> &args) { if(args.empty()) { - fail_msg_writer() << tr("expected at least one payment ID"); + PRINT_USAGE(USAGE_PAYMENTS); return true; } @@ -4596,14 +5122,38 @@ bool simple_wallet::rescan_spent(const std::vector<std::string> &args) return true; } //---------------------------------------------------------------------------------------------------- +std::pair<std::string, std::string> simple_wallet::show_outputs_line(const std::vector<uint64_t> &heights, uint64_t blockchain_height, uint64_t highlight_height) const +{ + std::stringstream ostr; + + for (uint64_t h: heights) + blockchain_height = std::max(blockchain_height, h); + + for (size_t j = 0; j < heights.size(); ++j) + ostr << (heights[j] == highlight_height ? " *" : " ") << heights[j]; + + // visualize the distribution, using the code by moneroexamples onion-monero-viewer + const uint64_t resolution = 79; + std::string ring_str(resolution, '_'); + for (size_t j = 0; j < heights.size(); ++j) + { + uint64_t pos = (heights[j] * resolution) / blockchain_height; + ring_str[pos] = 'o'; + } + if (highlight_height < blockchain_height) + { + uint64_t pos = (highlight_height * resolution) / blockchain_height; + ring_str[pos] = '*'; + } + + return std::make_pair(ostr.str(), ring_str); +} +//---------------------------------------------------------------------------------------------------- bool simple_wallet::print_ring_members(const std::vector<tools::wallet2::pending_tx>& ptx_vector, std::ostream& ostr) { uint32_t version; if (!try_connect_to_daemon(false, &version)) - { - fail_msg_writer() << tr("failed to connect to the daemon"); return false; - } // available for RPC version 1.4 or higher if (version < MAKE_CORE_RPC_VERSION(1, 4)) return true; @@ -4669,21 +5219,18 @@ bool simple_wallet::print_ring_members(const std::vector<tools::wallet2::pending } } ostr << tr("\nOriginating block heights: "); - for (size_t j = 0; j < absolute_offsets.size(); ++j) - ostr << tr(j == source.real_output ? " *" : " ") << res.outs[j].height; spent_key_height[i] = res.outs[source.real_output].height; spent_key_txid [i] = res.outs[source.real_output].txid; - // visualize the distribution, using the code by moneroexamples onion-monero-viewer - const uint64_t resolution = 79; - std::string ring_str(resolution, '_'); + std::vector<uint64_t> heights(absolute_offsets.size(), 0); + uint64_t highlight_height = std::numeric_limits<uint64_t>::max(); for (size_t j = 0; j < absolute_offsets.size(); ++j) { - uint64_t pos = (res.outs[j].height * resolution) / blockchain_height; - ring_str[pos] = 'o'; + heights[j] = res.outs[j].height; + if (j == source.real_output) + highlight_height = heights[j]; } - uint64_t pos = (res.outs[source.real_output].height * resolution) / blockchain_height; - ring_str[pos] = '*'; - ostr << tr("\n|") << ring_str << tr("|\n"); + std::pair<std::string, std::string> ring_str = show_outputs_line(heights, highlight_height); + ostr << ring_str.first << tr("\n|") << ring_str.second << tr("|\n"); } // warn if rings contain keys originating from the same tx or temporally very close block heights bool are_keys_from_same_tx = false; @@ -4709,13 +5256,11 @@ bool simple_wallet::print_ring_members(const std::vector<tools::wallet2::pending return true; } //---------------------------------------------------------------------------------------------------- -bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::string> &args_) +bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::string> &args_, bool called_by_mms) { // "transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> <amount> [<payment_id>]" if (!try_connect_to_daemon()) - return true; - - SCOPED_WALLET_UNLOCK(); + return false; std::vector<std::string> local_args = args_; @@ -4723,7 +5268,7 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri if (local_args.size() > 0 && local_args[0].substr(0, 6) == "index=") { if (!parse_subaddress_indices(local_args[0], subaddr_indices)) - return true; + return false; local_args.erase(local_args.begin()); } @@ -4745,7 +5290,7 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri else if (ring_size == 0) { fail_msg_writer() << tr("Ring size must not be 0"); - return true; + return false; } else { @@ -4757,19 +5302,19 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri if (adjusted_fake_outs_count > fake_outs_count) { fail_msg_writer() << (boost::format(tr("ring size %u is too small, minimum is %u")) % (fake_outs_count+1) % (adjusted_fake_outs_count+1)).str(); - return true; + return false; } if (adjusted_fake_outs_count < fake_outs_count) { fail_msg_writer() << (boost::format(tr("ring size %u is too large, maximum is %u")) % (fake_outs_count+1) % (adjusted_fake_outs_count+1)).str(); - return true; + return false; } const size_t min_args = (transfer_type == TransferLocked) ? 2 : 1; if(local_args.size() < min_args) { fail_msg_writer() << tr("wrong number of arguments"); - return true; + return false; } std::vector<uint8_t> extra; @@ -4781,6 +5326,8 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri bool r = true; if (tools::wallet2::parse_long_payment_id(payment_id_str, payment_id)) { + LONG_PAYMENT_ID_SUPPORT_CHECK(); + std::string extra_nonce; set_payment_id_to_tx_extra_nonce(extra_nonce, payment_id); r = add_extra_nonce_to_tx_extra(extra, extra_nonce); @@ -4788,23 +5335,10 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri payment_id_seen = true; message_writer() << tr("Unencrypted payment IDs are bad for privacy: ask the recipient to use subaddresses instead"); } - else - { - crypto::hash8 payment_id8; - if (tools::wallet2::parse_short_payment_id(payment_id_str, payment_id8)) - { - std::string extra_nonce; - set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, payment_id8); - r = add_extra_nonce_to_tx_extra(extra, extra_nonce); - local_args.pop_back(); - payment_id_seen = true; - } - } - if(!r) { fail_msg_writer() << tr("payment id failed to encode"); - return true; + return false; } } @@ -4818,22 +5352,24 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri catch (const std::exception &e) { fail_msg_writer() << tr("bad locked_blocks parameter:") << " " << local_args.back(); - return true; + return false; } if (locked_blocks > 1000000) { fail_msg_writer() << tr("Locked blocks too high, max 1000000 (Ëœ4 yrs)"); - return true; + return false; } local_args.pop_back(); } + vector<cryptonote::address_parse_info> dsts_info; vector<cryptonote::tx_destination_entry> dsts; size_t num_subaddresses = 0; for (size_t i = 0; i < local_args.size(); ) { + dsts_info.emplace_back(); + cryptonote::address_parse_info & info = dsts_info.back(); cryptonote::tx_destination_entry de; - cryptonote::address_parse_info info; bool r = true; // check for a URI @@ -4849,11 +5385,12 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri if (!tools::wallet2::parse_short_payment_id(payment_id_uri, info.payment_id)) { fail_msg_writer() << tr("failed to parse short payment ID from URI"); - return true; + return false; } info.has_payment_id = true; } de.amount = amount; + de.original = local_args[i]; ++i; } else if (i + 1 < local_args.size()) @@ -4864,8 +5401,9 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri { fail_msg_writer() << tr("amount is wrong: ") << local_args[i] << ' ' << local_args[i + 1] << ", " << tr("expected number from 0 to ") << print_money(std::numeric_limits<uint64_t>::max()); - return true; + return false; } + de.original = local_args[i]; i += 2; } else @@ -4874,16 +5412,17 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri fail_msg_writer() << tr("Invalid last argument: ") << local_args.back() << ": " << error; else fail_msg_writer() << tr("Invalid last argument: ") << local_args.back(); - return true; + return false; } if (!r) { fail_msg_writer() << tr("failed to parse address"); - return true; + return false; } de.addr = info.address; de.is_subaddress = info.is_subaddress; + de.is_integrated = info.has_payment_id; num_subaddresses += info.is_subaddress; if (info.has_payment_id || !payment_id_uri.empty()) @@ -4891,7 +5430,7 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri if (payment_id_seen) { fail_msg_writer() << tr("a single transaction cannot use more than one payment id"); - return true; + return false; } crypto::hash payment_id; @@ -4902,19 +5441,20 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri } else if (tools::wallet2::parse_payment_id(payment_id_uri, payment_id)) { + LONG_PAYMENT_ID_SUPPORT_CHECK(); set_payment_id_to_tx_extra_nonce(extra_nonce, payment_id); message_writer() << tr("Unencrypted payment IDs are bad for privacy: ask the recipient to use subaddresses instead"); } else { fail_msg_writer() << tr("failed to parse payment id, though it was detected"); - return true; + return false; } bool r = add_extra_nonce_to_tx_extra(extra, extra_nonce); if(!r) { fail_msg_writer() << tr("failed to set up payment id, though it was decoded correctly"); - return true; + return false; } payment_id_seen = true; } @@ -4923,19 +5463,21 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri } // prompt is there is no payment id and confirmation is required - if (!payment_id_seen && m_wallet->confirm_missing_payment_id() && dsts.size() > num_subaddresses) + if (m_long_payment_id_support && !payment_id_seen && m_wallet->confirm_missing_payment_id() && dsts.size() > num_subaddresses) { - std::string accepted = input_line(tr("No payment id is included with this transaction. Is this okay? (Y/Yes/N/No): ")); + std::string accepted = input_line(tr("No payment id is included with this transaction. Is this okay?"), true); if (std::cin.eof()) - return true; + return false; if (!command_line::is_yes(accepted)) { fail_msg_writer() << tr("transaction cancelled."); - return true; + return false; } } + SCOPED_WALLET_UNLOCK_ON_BAD_PASSWORD(return false;); + try { // figure out what tx will be necessary @@ -4949,7 +5491,7 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri if (!err.empty()) { fail_msg_writer() << tr("failed to get blockchain height: ") << err; - return true; + return false; } unlock_block = bc_height + locked_blocks; ptx_vector = m_wallet->create_transactions_2(dsts, fake_outs_count, unlock_block /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices); @@ -4965,7 +5507,7 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri if (ptx_vector.empty()) { fail_msg_writer() << tr("No outputs found, or daemon is not ready"); - return true; + return false; } // if we need to check for backlog, check the worst case tx @@ -4987,30 +5529,30 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri std::vector<std::pair<uint64_t, uint64_t>> nblocks = m_wallet->estimate_backlog({std::make_pair(worst_fee_per_byte, worst_fee_per_byte)}); if (nblocks.size() != 1) { - prompt << "Internal error checking for backlog. " << tr("Is this okay anyway? (Y/Yes/N/No): "); + prompt << "Internal error checking for backlog. " << tr("Is this okay anyway?"); } else { if (nblocks[0].first > m_wallet->get_confirm_backlog_threshold()) - prompt << (boost::format(tr("There is currently a %u block backlog at that fee level. Is this okay? (Y/Yes/N/No): ")) % nblocks[0].first).str(); + prompt << (boost::format(tr("There is currently a %u block backlog at that fee level. Is this okay?")) % nblocks[0].first).str(); } } catch (const std::exception &e) { - prompt << tr("Failed to check for backlog: ") << e.what() << ENDL << tr("Is this okay anyway? (Y/Yes/N/No): "); + prompt << tr("Failed to check for backlog: ") << e.what() << ENDL << tr("Is this okay anyway?"); } std::string prompt_str = prompt.str(); if (!prompt_str.empty()) { - std::string accepted = input_line(prompt_str); + std::string accepted = input_line(prompt_str, true); if (std::cin.eof()) - return true; + return false; if (!command_line::is_yes(accepted)) { fail_msg_writer() << tr("transaction cancelled."); - return true; + return false; } } } @@ -5070,7 +5612,7 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri if (m_wallet->print_ring_members()) { if (!print_ring_members(ptx_vector, prompt)) - return true; + return false; } bool default_ring_size = true; for (const auto &ptx: ptx_vector) @@ -5089,38 +5631,73 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri { prompt << tr("WARNING: this is a non default ring size, which may harm your privacy. Default is recommended."); } - prompt << ENDL << tr("Is this okay? (Y/Yes/N/No): "); + prompt << ENDL << tr("Is this okay?"); - std::string accepted = input_line(prompt.str()); + std::string accepted = input_line(prompt.str(), true); if (std::cin.eof()) - return true; + return false; if (!command_line::is_yes(accepted)) { fail_msg_writer() << tr("transaction cancelled."); - return true; + return false; } } // actually commit the transactions - if (m_wallet->multisig()) + if (m_wallet->multisig() && called_by_mms) + { + std::string ciphertext = m_wallet->save_multisig_tx(ptx_vector); + if (!ciphertext.empty()) + { + get_message_store().process_wallet_created_data(get_multisig_wallet_state(), mms::message_type::partially_signed_tx, ciphertext); + success_msg_writer(true) << tr("Unsigned transaction(s) successfully written to MMS"); + } + } + else if (m_wallet->multisig()) { bool r = m_wallet->save_multisig_tx(ptx_vector, "multisig_monero_tx"); if (!r) { fail_msg_writer() << tr("Failed to write transaction(s) to file"); + return false; } else { success_msg_writer(true) << tr("Unsigned transaction(s) successfully written to file: ") << "multisig_monero_tx"; } } + else if (m_wallet->get_account().get_device().has_tx_cold_sign()) + { + try + { + tools::wallet2::signed_tx_set signed_tx; + if (!cold_sign_tx(ptx_vector, signed_tx, dsts_info, [&](const tools::wallet2::signed_tx_set &tx){ return accept_loaded_tx(tx); })){ + fail_msg_writer() << tr("Failed to cold sign transaction with HW wallet"); + return false; + } + + commit_or_save(signed_tx.ptx, m_do_not_relay); + } + catch (const std::exception& e) + { + handle_transfer_exception(std::current_exception(), m_wallet->is_trusted_daemon()); + return false; + } + catch (...) + { + LOG_ERROR("Unknown error"); + fail_msg_writer() << tr("unknown error"); + return false; + } + } else if (m_wallet->watch_only()) { bool r = m_wallet->save_tx(ptx_vector, "unsigned_monero_tx"); if (!r) { fail_msg_writer() << tr("Failed to write transaction(s) to file"); + return false; } else { @@ -5135,11 +5712,13 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri catch (const std::exception &e) { handle_transfer_exception(std::current_exception(), m_wallet->is_trusted_daemon()); + return false; } catch (...) { LOG_ERROR("unknown error"); fail_msg_writer() << tr("unknown error"); + return false; } return true; @@ -5147,12 +5726,14 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri //---------------------------------------------------------------------------------------------------- bool simple_wallet::transfer(const std::vector<std::string> &args_) { - return transfer_main(Transfer, args_); + transfer_main(Transfer, args_, false); + return true; } //---------------------------------------------------------------------------------------------------- bool simple_wallet::locked_transfer(const std::vector<std::string> &args_) { - return transfer_main(TransferLocked, args_); + transfer_main(TransferLocked, args_, false); + return true; } //---------------------------------------------------------------------------------------------------- bool simple_wallet::locked_sweep_all(const std::vector<std::string> &args_) @@ -5190,17 +5771,17 @@ bool simple_wallet::sweep_unmixable(const std::vector<std::string> &args_) std::string prompt_str = tr("Sweeping ") + print_money(total_unmixable); if (ptx_vector.size() > 1) { - prompt_str = (boost::format(tr("Sweeping %s in %llu transactions for a total fee of %s. Is this okay? (Y/Yes/N/No): ")) % + prompt_str = (boost::format(tr("Sweeping %s in %llu transactions for a total fee of %s. Is this okay?")) % print_money(total_unmixable) % ((unsigned long long)ptx_vector.size()) % print_money(total_fee)).str(); } else { - prompt_str = (boost::format(tr("Sweeping %s for a total fee of %s. Is this okay? (Y/Yes/N/No): ")) % + prompt_str = (boost::format(tr("Sweeping %s for a total fee of %s. Is this okay?")) % print_money(total_unmixable) % print_money(total_fee)).str(); } - std::string accepted = input_line(prompt_str); + std::string accepted = input_line(prompt_str, true); if (std::cin.eof()) return true; if (!command_line::is_yes(accepted)) @@ -5243,7 +5824,7 @@ bool simple_wallet::sweep_unmixable(const std::vector<std::string> &args_) catch (const tools::error::not_enough_unlocked_money& e) { fail_msg_writer() << tr("Not enough money in unlocked balance"); - std::string accepted = input_line((boost::format(tr("Discarding %s of unmixable outputs that cannot be spent, which can be undone by \"rescan_spent\". Is this okay? (Y/Yes/N/No): ")) % print_money(e.available())).str()); + std::string accepted = input_line((boost::format(tr("Discarding %s of unmixable outputs that cannot be spent, which can be undone by \"rescan_spent\". Is this okay?")) % print_money(e.available())).str(), true); if (std::cin.eof()) return true; if (command_line::is_yes(accepted)) @@ -5271,7 +5852,14 @@ bool simple_wallet::sweep_main(uint64_t below, bool locked, const std::vector<st { auto print_usage = [below]() { - fail_msg_writer() << boost::format(tr("usage: %s [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] [outputs=<N>] <address> [<payment_id>]")) % (below ? "sweep_below" : "sweep_all"); + if (below) + { + PRINT_USAGE(USAGE_SWEEP_BELOW); + } + else + { + PRINT_USAGE(USAGE_SWEEP_ALL); + } }; if (args_.size() == 0) { @@ -5398,23 +5986,13 @@ bool simple_wallet::sweep_main(uint64_t below, bool locked, const std::vector<st bool r = tools::wallet2::parse_long_payment_id(payment_id_str, payment_id); if(r) { + LONG_PAYMENT_ID_SUPPORT_CHECK(); + std::string extra_nonce; set_payment_id_to_tx_extra_nonce(extra_nonce, payment_id); r = add_extra_nonce_to_tx_extra(extra, extra_nonce); payment_id_seen = true; } - else - { - crypto::hash8 payment_id8; - r = tools::wallet2::parse_short_payment_id(payment_id_str, payment_id8); - if(r) - { - std::string extra_nonce; - set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, payment_id8); - r = add_extra_nonce_to_tx_extra(extra, extra_nonce); - payment_id_seen = true; - } - } if(!r && local_args.size() == 3) { @@ -5454,9 +6032,9 @@ bool simple_wallet::sweep_main(uint64_t below, bool locked, const std::vector<st } // prompt is there is no payment id and confirmation is required - if (!payment_id_seen && m_wallet->confirm_missing_payment_id() && !info.is_subaddress) + if (m_long_payment_id_support && !payment_id_seen && m_wallet->confirm_missing_payment_id() && !info.is_subaddress) { - std::string accepted = input_line(tr("No payment id is included with this transaction. Is this okay? (Y/Yes/N/No): ")); + std::string accepted = input_line(tr("No payment id is included with this transaction. Is this okay?"), true); if (std::cin.eof()) return true; if (!command_line::is_yes(accepted)) @@ -5504,17 +6082,17 @@ bool simple_wallet::sweep_main(uint64_t below, bool locked, const std::vector<st if (m_wallet->print_ring_members() && !print_ring_members(ptx_vector, prompt)) return true; if (ptx_vector.size() > 1) { - prompt << boost::format(tr("Sweeping %s in %llu transactions for a total fee of %s. Is this okay? (Y/Yes/N/No): ")) % + prompt << boost::format(tr("Sweeping %s in %llu transactions for a total fee of %s. Is this okay?")) % print_money(total_sent) % ((unsigned long long)ptx_vector.size()) % print_money(total_fee); } else { - prompt << boost::format(tr("Sweeping %s for a total fee of %s. Is this okay? (Y/Yes/N/No): ")) % + prompt << boost::format(tr("Sweeping %s for a total fee of %s. Is this okay?")) % print_money(total_sent) % print_money(total_fee); } - std::string accepted = input_line(prompt.str()); + std::string accepted = input_line(prompt.str(), true); if (std::cin.eof()) return true; if (!command_line::is_yes(accepted)) @@ -5537,6 +6115,31 @@ bool simple_wallet::sweep_main(uint64_t below, bool locked, const std::vector<st success_msg_writer(true) << tr("Unsigned transaction(s) successfully written to file: ") << "multisig_monero_tx"; } } + else if (m_wallet->get_account().get_device().has_tx_cold_sign()) + { + try + { + tools::wallet2::signed_tx_set signed_tx; + std::vector<cryptonote::address_parse_info> dsts_info; + dsts_info.push_back(info); + + if (!cold_sign_tx(ptx_vector, signed_tx, dsts_info, [&](const tools::wallet2::signed_tx_set &tx){ return accept_loaded_tx(tx); })){ + fail_msg_writer() << tr("Failed to cold sign transaction with HW wallet"); + return true; + } + + commit_or_save(signed_tx.ptx, m_do_not_relay); + } + catch (const std::exception& e) + { + handle_transfer_exception(std::current_exception(), m_wallet->is_trusted_daemon()); + } + catch (...) + { + LOG_ERROR("Unknown error"); + fail_msg_writer() << tr("unknown error"); + } + } else if (m_wallet->watch_only()) { bool r = m_wallet->save_tx(ptx_vector, "unsigned_monero_tx"); @@ -5569,7 +6172,6 @@ bool simple_wallet::sweep_main(uint64_t below, bool locked, const std::vector<st //---------------------------------------------------------------------------------------------------- bool simple_wallet::sweep_single(const std::vector<std::string> &args_) { - SCOPED_WALLET_UNLOCK(); if (!try_connect_to_daemon()) return true; @@ -5641,12 +6243,9 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) std::string extra_nonce; if (tools::wallet2::parse_long_payment_id(local_args.back(), payment_id)) { + LONG_PAYMENT_ID_SUPPORT_CHECK(); set_payment_id_to_tx_extra_nonce(extra_nonce, payment_id); } - else if(tools::wallet2::parse_short_payment_id(local_args.back(), payment_id8)) - { - set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, payment_id8); - } else { fail_msg_writer() << tr("failed to parse Payment ID"); @@ -5665,7 +6264,7 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) if (local_args.size() != 2) { - fail_msg_writer() << tr("usage: sweep_single [<priority>] [<ring_size>] [outputs=<N>] <key_image> <address> [<payment_id>]"); + PRINT_USAGE(USAGE_SWEEP_SINGLE); return true; } @@ -5702,9 +6301,9 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) } // prompt if there is no payment id and confirmation is required - if (!payment_id_seen && m_wallet->confirm_missing_payment_id() && !info.is_subaddress) + if (m_long_payment_id_support && !payment_id_seen && m_wallet->confirm_missing_payment_id() && !info.is_subaddress) { - std::string accepted = input_line(tr("No payment id is included with this transaction. Is this okay? (Y/Yes/N/No): ")); + std::string accepted = input_line(tr("No payment id is included with this transaction. Is this okay?"), true); if (std::cin.eof()) return true; if (!command_line::is_yes(accepted)) @@ -5717,6 +6316,8 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) } } + SCOPED_WALLET_UNLOCK(); + try { // figure out what tx will be necessary @@ -5744,10 +6345,10 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) std::ostringstream prompt; if (!print_ring_members(ptx_vector, prompt)) return true; - prompt << boost::format(tr("Sweeping %s for a total fee of %s. Is this okay? (Y/Yes/N/No): ")) % + prompt << boost::format(tr("Sweeping %s for a total fee of %s. Is this okay?")) % print_money(total_sent) % print_money(total_fee); - std::string accepted = input_line(prompt.str()); + std::string accepted = input_line(prompt.str(), true); if (std::cin.eof()) return true; if (!command_line::is_yes(accepted)) @@ -5824,16 +6425,10 @@ bool simple_wallet::sweep_below(const std::vector<std::string> &args_) //---------------------------------------------------------------------------------------------------- bool simple_wallet::donate(const std::vector<std::string> &args_) { - if(m_wallet->nettype() != cryptonote::MAINNET) - { - fail_msg_writer() << tr("donations are not enabled on the testnet or on the stagenet"); - return true; - } - std::vector<std::string> local_args = args_; if(local_args.empty() || local_args.size() > 5) { - fail_msg_writer() << tr("usage: donate [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <amount> [<payment_id>]"); + PRINT_USAGE(USAGE_DONATE); return true; } std::string amount_str; @@ -5848,14 +6443,43 @@ bool simple_wallet::donate(const std::vector<std::string> &args_) local_args.pop_back(); } // get amount and pop - amount_str = local_args.back(); - local_args.pop_back(); + uint64_t amount; + bool ok = cryptonote::parse_amount(amount, local_args.back()); + if (ok && amount != 0) + { + amount_str = local_args.back(); + local_args.pop_back(); + } + else + { + fail_msg_writer() << tr("amount is wrong: ") << local_args.back() << ", " << tr("expected number from 0 to ") << print_money(std::numeric_limits<uint64_t>::max()); + return true; + } // push back address, amount, payment id - local_args.push_back(MONERO_DONATION_ADDR); + std::string address_str; + if (m_wallet->nettype() != cryptonote::MAINNET) + { + // if not mainnet, convert donation address string to the relevant network type + address_parse_info info; + if (!cryptonote::get_account_address_from_str(info, cryptonote::MAINNET, MONERO_DONATION_ADDR)) + { + fail_msg_writer() << tr("Failed to parse donation address: ") << MONERO_DONATION_ADDR; + return true; + } + address_str = cryptonote::get_account_address_as_str(m_wallet->nettype(), info.is_subaddress, info.address); + } + else + { + address_str = MONERO_DONATION_ADDR; + } + local_args.push_back(address_str); local_args.push_back(amount_str); if (!payment_id_str.empty()) local_args.push_back(payment_id_str); - message_writer() << (boost::format(tr("Donating %s %s to The Monero Project (donate.getmonero.org or %s).")) % amount_str % cryptonote::get_unit(cryptonote::get_default_decimal_point()) % MONERO_DONATION_ADDR).str(); + if (m_wallet->nettype() == cryptonote::MAINNET) + message_writer() << (boost::format(tr("Donating %s %s to The Monero Project (donate.getmonero.org or %s).")) % amount_str % cryptonote::get_unit(cryptonote::get_default_decimal_point()) % MONERO_DONATION_ADDR).str(); + else + message_writer() << (boost::format(tr("Donating %s %s to %s.")) % amount_str % cryptonote::get_unit(cryptonote::get_default_decimal_point()) % address_str).str(); transfer(local_args); return true; } @@ -5885,14 +6509,29 @@ bool simple_wallet::accept_loaded_tx(const std::function<size_t()> get_num_txes, { if (!payment_id_string.empty()) payment_id_string += ", "; - payment_id_string = std::string("encrypted payment ID ") + epee::string_tools::pod_to_hex(payment_id8); - has_encrypted_payment_id = true; + + // if none of the addresses are integrated addresses, it's a dummy one + bool is_dummy = true; + for (const auto &e: cd.dests) + if (e.is_integrated) + is_dummy = false; + + if (is_dummy) + { + payment_id_string += std::string("dummy encrypted payment ID"); + } + else + { + payment_id_string += std::string("encrypted payment ID ") + epee::string_tools::pod_to_hex(payment_id8); + has_encrypted_payment_id = true; + } } else if (get_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id)) { if (!payment_id_string.empty()) payment_id_string += ", "; - payment_id_string = std::string("unencrypted payment ID ") + epee::string_tools::pod_to_hex(payment_id); + payment_id_string += std::string("unencrypted payment ID ") + epee::string_tools::pod_to_hex(payment_id); + payment_id_string += " (OBSOLETE)"; } } } @@ -5988,15 +6627,15 @@ bool simple_wallet::accept_loaded_tx(const std::function<size_t()> get_num_txes, change_string += tr("no change"); uint64_t fee = amount - amount_to_dests; - std::string prompt_str = (boost::format(tr("Loaded %lu transactions, for %s, fee %s, %s, %s, with min ring size %lu, %s. %sIs this okay? (Y/Yes/N/No): ")) % (unsigned long)get_num_txes() % print_money(amount) % print_money(fee) % dest_string % change_string % (unsigned long)min_ring_size % payment_id_string % extra_message).str(); - return command_line::is_yes(input_line(prompt_str)); + std::string prompt_str = (boost::format(tr("Loaded %lu transactions, for %s, fee %s, %s, %s, with min ring size %lu, %s. %sIs this okay?")) % (unsigned long)get_num_txes() % print_money(amount) % print_money(fee) % dest_string % change_string % (unsigned long)min_ring_size % payment_id_string % extra_message).str(); + return command_line::is_yes(input_line(prompt_str, true)); } //---------------------------------------------------------------------------------------------------- bool simple_wallet::accept_loaded_tx(const tools::wallet2::unsigned_tx_set &txs) { std::string extra_message; - if (!txs.transfers.empty()) - extra_message = (boost::format("%u outputs to import. ") % (unsigned)txs.transfers.size()).str(); + if (!txs.transfers.second.empty()) + extra_message = (boost::format("%u outputs to import. ") % (unsigned)txs.transfers.second.size()).str(); return accept_loaded_tx([&txs](){return txs.txes.size();}, [&txs](size_t n)->const tools::wallet2::tx_construction_data&{return txs.txes[n];}, extra_message); } //---------------------------------------------------------------------------------------------------- @@ -6027,7 +6666,7 @@ bool simple_wallet::sign_transfer(const std::vector<std::string> &args_) } if (args_.size() > 1 || (args_.size() == 1 && args_[0] != "export_raw")) { - fail_msg_writer() << tr("usage: sign_transfer [export_raw]"); + PRINT_USAGE(USAGE_SIGN_TRANSFER); return true; } @@ -6117,7 +6756,7 @@ bool simple_wallet::get_tx_key(const std::vector<std::string> &args_) return true; } if(local_args.size() != 1) { - fail_msg_writer() << tr("usage: get_tx_key <txid>"); + PRINT_USAGE(USAGE_GET_TX_KEY); return true; } @@ -6153,7 +6792,7 @@ bool simple_wallet::set_tx_key(const std::vector<std::string> &args_) std::vector<std::string> local_args = args_; if(local_args.size() != 2) { - fail_msg_writer() << tr("usage: set_tx_key <txid> <tx_key>"); + PRINT_USAGE(USAGE_SET_TX_KEY); return true; } @@ -6215,7 +6854,7 @@ bool simple_wallet::get_tx_proof(const std::vector<std::string> &args) } if (args.size() != 2 && args.size() != 3) { - fail_msg_writer() << tr("usage: get_tx_proof <txid> <address> [<message>]"); + PRINT_USAGE(USAGE_GET_TX_PROOF); return true; } @@ -6256,7 +6895,7 @@ bool simple_wallet::check_tx_key(const std::vector<std::string> &args_) std::vector<std::string> local_args = args_; if(local_args.size() != 3) { - fail_msg_writer() << tr("usage: check_tx_key <txid> <txkey> <address>"); + PRINT_USAGE(USAGE_CHECK_TX_KEY); return true; } @@ -6342,7 +6981,7 @@ bool simple_wallet::check_tx_key(const std::vector<std::string> &args_) bool simple_wallet::check_tx_proof(const std::vector<std::string> &args) { if(args.size() != 3 && args.size() != 4) { - fail_msg_writer() << tr("usage: check_tx_proof <txid> <address> <signature_file> [<message>]"); + PRINT_USAGE(USAGE_CHECK_TX_PROOF); return true; } @@ -6425,7 +7064,7 @@ bool simple_wallet::get_spend_proof(const std::vector<std::string> &args) return true; } if(args.size() != 1 && args.size() != 2) { - fail_msg_writer() << tr("usage: get_spend_proof <txid> [<message>]"); + PRINT_USAGE(USAGE_GET_SPEND_PROOF); return true; } @@ -6443,10 +7082,7 @@ bool simple_wallet::get_spend_proof(const std::vector<std::string> &args) } if (!try_connect_to_daemon()) - { - fail_msg_writer() << tr("failed to connect to the daemon"); return true; - } SCOPED_WALLET_UNLOCK(); @@ -6469,7 +7105,7 @@ bool simple_wallet::get_spend_proof(const std::vector<std::string> &args) bool simple_wallet::check_spend_proof(const std::vector<std::string> &args) { if(args.size() != 2 && args.size() != 3) { - fail_msg_writer() << tr("usage: check_spend_proof <txid> <signature_file> [<message>]"); + PRINT_USAGE(USAGE_CHECK_SPEND_PROOF); return true; } @@ -6481,10 +7117,7 @@ bool simple_wallet::check_spend_proof(const std::vector<std::string> &args) } if (!try_connect_to_daemon()) - { - fail_msg_writer() << tr("failed to connect to the daemon"); return true; - } std::string sig_str; if (!epee::file_io_utils::load_file_to_string(args[1], sig_str)) @@ -6515,7 +7148,7 @@ bool simple_wallet::get_reserve_proof(const std::vector<std::string> &args) return true; } if(args.size() != 1 && args.size() != 2) { - fail_msg_writer() << tr("usage: get_reserve_proof (all|<amount>) [<message>]"); + PRINT_USAGE(USAGE_GET_RESERVE_PROOF); return true; } @@ -6538,10 +7171,7 @@ bool simple_wallet::get_reserve_proof(const std::vector<std::string> &args) } if (!try_connect_to_daemon()) - { - fail_msg_writer() << tr("failed to connect to the daemon"); return true; - } SCOPED_WALLET_UNLOCK(); @@ -6564,15 +7194,12 @@ bool simple_wallet::get_reserve_proof(const std::vector<std::string> &args) bool simple_wallet::check_reserve_proof(const std::vector<std::string> &args) { if(args.size() != 2 && args.size() != 3) { - fail_msg_writer() << tr("usage: check_reserve_proof <address> <signature_file> [<message>]"); + PRINT_USAGE(USAGE_CHECK_RESERVE_PROOF); return true; } if (!try_connect_to_daemon()) - { - fail_msg_writer() << tr("failed to connect to the daemon"); return true; - } cryptonote::address_parse_info info; if(!cryptonote::get_account_address_from_str_or_url(info, m_wallet->nettype(), args[0], oa_prompter)) @@ -6614,24 +7241,6 @@ bool simple_wallet::check_reserve_proof(const std::vector<std::string> &args) return true; } //---------------------------------------------------------------------------------------------------- -static std::string get_human_readable_timestamp(uint64_t ts) -{ - char buffer[64]; - if (ts < 1234567890) - return "<unknown>"; - time_t tt = ts; - struct tm tm; -#ifdef WIN32 - gmtime_s(&tm, &tt); -#else - gmtime_r(&tt, &tm); -#endif - uint64_t now = time(NULL); - uint64_t diff = ts > now ? ts - now : now - ts; - strftime(buffer, sizeof(buffer), "%Y-%m-%d %H:%M:%S", &tm); - return std::string(buffer); -} -//---------------------------------------------------------------------------------------------------- static std::string get_human_readable_timespan(std::chrono::seconds seconds) { uint64_t ts = seconds.count(); @@ -6648,9 +7257,9 @@ static std::string get_human_readable_timespan(std::chrono::seconds seconds) return sw::tr("a long time"); } //---------------------------------------------------------------------------------------------------- -bool simple_wallet::show_transfers(const std::vector<std::string> &args_) +// mutates local_args as it parses and consumes arguments +bool simple_wallet::get_transfers(std::vector<std::string>& local_args, std::vector<transfer_view>& transfers) { - std::vector<std::string> local_args = args_; bool in = true; bool out = true; bool pending = true; @@ -6659,15 +7268,7 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_) bool coinbase = true; uint64_t min_height = 0; uint64_t max_height = (uint64_t)-1; - boost::optional<uint32_t> subaddr_index; - - if(local_args.size() > 4) { - fail_msg_writer() << tr("usage: show_transfers [in|out|all|pending|failed|coinbase] [index=<N1>[,<N2>,...]] [<min_height> [<max_height>]]"); - return true; - } - LOCK_IDLE_SCOPE(); - // optional in/out selector if (local_args.size() > 0) { if (local_args[0] == "in" || local_args[0] == "incoming") { @@ -6705,38 +7306,34 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_) if (local_args.size() > 0 && local_args[0].substr(0, 6) == "index=") { if (!parse_subaddress_indices(local_args[0], subaddr_indices)) - return true; + return false; local_args.erase(local_args.begin()); } // min height - if (local_args.size() > 0) { + if (local_args.size() > 0 && local_args[0].find('=') == std::string::npos) { try { min_height = boost::lexical_cast<uint64_t>(local_args[0]); } catch (const boost::bad_lexical_cast &) { fail_msg_writer() << tr("bad min_height parameter:") << " " << local_args[0]; - return true; + return false; } local_args.erase(local_args.begin()); } // max height - if (local_args.size() > 0) { + if (local_args.size() > 0 && local_args[0].find('=') == std::string::npos) { try { max_height = boost::lexical_cast<uint64_t>(local_args[0]); } catch (const boost::bad_lexical_cast &) { fail_msg_writer() << tr("bad max_height parameter:") << " " << local_args[0]; - return true; + return false; } local_args.erase(local_args.begin()); } - std::multimap<uint64_t, std::tuple<epee::console_colors, std::string, std::string>> output; - - PAUSE_READLINE(); - if (in || coinbase) { std::list<std::pair<crypto::hash, tools::wallet2::payment_details>> payments; m_wallet->get_payments(payments, min_height, max_height, m_current_subaddress_account, subaddr_indices); @@ -6748,23 +7345,27 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_) if (payment_id.substr(16).find_first_not_of('0') == std::string::npos) payment_id = payment_id.substr(0,16); std::string note = m_wallet->get_tx_note(pd.m_tx_hash); + std::string destination = m_wallet->get_subaddress_as_str({m_current_subaddress_account, pd.m_subaddr_index.minor}); const std::string type = pd.m_coinbase ? tr("block") : tr("in"); - output.insert(std::make_pair(pd.m_block_height, std::make_tuple(epee::console_color_green, type, (boost::format("%25.25s %20.20s %s %s %d %s %s") % get_human_readable_timestamp(pd.m_timestamp) % print_money(pd.m_amount) % string_tools::pod_to_hex(pd.m_tx_hash) % payment_id % pd.m_subaddr_index.minor % "-" % note).str()))); + const bool unlocked = m_wallet->is_transfer_unlocked(pd.m_unlock_time, pd.m_block_height); + transfers.push_back({ + type, + pd.m_block_height, + pd.m_timestamp, + type, + true, + pd.m_amount, + pd.m_tx_hash, + payment_id, + 0, + {{destination, pd.m_amount}}, + {pd.m_subaddr_index.minor}, + note, + (unlocked) ? "unlocked" : "locked" + }); } } - auto print_subaddr_indices = [](const std::set<uint32_t>& indices) - { - stringstream ss; - bool first = true; - for (uint32_t i : indices) - { - ss << (first ? "" : ",") << i; - first = false; - } - return ss.str(); - }; - if (out) { std::list<std::pair<crypto::hash, tools::wallet2::confirmed_transfer_details>> payments; m_wallet->get_payments_out(payments, min_height, max_height, m_current_subaddress_account, subaddr_indices); @@ -6772,27 +7373,32 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_) const tools::wallet2::confirmed_transfer_details &pd = i->second; uint64_t change = pd.m_change == (uint64_t)-1 ? 0 : pd.m_change; // change may not be known uint64_t fee = pd.m_amount_in - pd.m_amount_out; - std::string dests; + std::vector<std::pair<std::string, uint64_t>> destinations; for (const auto &d: pd.m_dests) { - if (!dests.empty()) - dests += ", "; - dests += get_account_address_as_str(m_wallet->nettype(), d.is_subaddress, d.addr) + ": " + print_money(d.amount); + destinations.push_back({get_account_address_as_str(m_wallet->nettype(), d.is_subaddress, d.addr), d.amount}); } std::string payment_id = string_tools::pod_to_hex(i->second.m_payment_id); if (payment_id.substr(16).find_first_not_of('0') == std::string::npos) payment_id = payment_id.substr(0,16); std::string note = m_wallet->get_tx_note(i->first); - output.insert(std::make_pair(pd.m_block_height, std::make_tuple(epee::console_color_magenta, tr("out"), (boost::format("%25.25s %20.20s %s %s %14.14s %s %s - %s") % get_human_readable_timestamp(pd.m_timestamp) % print_money(pd.m_amount_in - change - fee) % string_tools::pod_to_hex(i->first) % payment_id % print_money(fee) % dests % print_subaddr_indices(pd.m_subaddr_indices) % note).str()))); + transfers.push_back({ + "out", + pd.m_block_height, + pd.m_timestamp, + "out", + true, + pd.m_amount_in - change - fee, + i->first, + payment_id, + fee, + destinations, + pd.m_subaddr_indices, + note, + "-" + }); } } - // print in and out sorted by height - for (std::multimap<uint64_t, std::tuple<epee::console_colors, std::string, std::string>>::const_iterator i = output.begin(); i != output.end(); ++i) { - message_writer(std::get<0>(i->second), false) << - boost::format("%8.8llu %6.6s %s") % - ((unsigned long long)i->first) % std::get<1>(i->second) % std::get<2>(i->second); - } - if (pool) { try { @@ -6808,10 +7414,25 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_) if (payment_id.substr(16).find_first_not_of('0') == std::string::npos) payment_id = payment_id.substr(0,16); std::string note = m_wallet->get_tx_note(pd.m_tx_hash); + std::string destination = m_wallet->get_subaddress_as_str({m_current_subaddress_account, pd.m_subaddr_index.minor}); std::string double_spend_note; if (i->second.m_double_spend_seen) double_spend_note = tr("[Double spend seen on the network: this transaction may or may not end up being mined] "); - message_writer() << (boost::format("%8.8s %6.6s %25.25s %20.20s %s %s %d %s %s%s") % "pool" % "in" % get_human_readable_timestamp(pd.m_timestamp) % print_money(pd.m_amount) % string_tools::pod_to_hex(pd.m_tx_hash) % payment_id % pd.m_subaddr_index.minor % "-" % note % double_spend_note).str(); + transfers.push_back({ + "pool", + "pool", + pd.m_timestamp, + "in", + false, + pd.m_amount, + pd.m_tx_hash, + payment_id, + 0, + {{destination, pd.m_amount}}, + {pd.m_subaddr_index.minor}, + note + double_spend_note, + "locked" + }); } } catch (const std::exception& e) @@ -6828,25 +7449,193 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_) const tools::wallet2::unconfirmed_transfer_details &pd = i->second; uint64_t amount = pd.m_amount_in; uint64_t fee = amount - pd.m_amount_out; + std::vector<std::pair<std::string, uint64_t>> destinations; + for (const auto &d: pd.m_dests) { + destinations.push_back({get_account_address_as_str(m_wallet->nettype(), d.is_subaddress, d.addr), d.amount}); + } std::string payment_id = string_tools::pod_to_hex(i->second.m_payment_id); if (payment_id.substr(16).find_first_not_of('0') == std::string::npos) payment_id = payment_id.substr(0,16); std::string note = m_wallet->get_tx_note(i->first); bool is_failed = pd.m_state == tools::wallet2::unconfirmed_transfer_details::failed; if ((failed && is_failed) || (!is_failed && pending)) { - message_writer() << (boost::format("%8.8s %6.6s %25.25s %20.20s %s %s %14.14s %s - %s") % (is_failed ? tr("failed") : tr("pending")) % tr("out") % get_human_readable_timestamp(pd.m_timestamp) % print_money(amount - pd.m_change - fee) % string_tools::pod_to_hex(i->first) % payment_id % print_money(fee) % print_subaddr_indices(pd.m_subaddr_indices) % note).str(); + transfers.push_back({ + (is_failed ? "failed" : "pending"), + (is_failed ? "failed" : "pending"), + pd.m_timestamp, + "out", + false, + amount - pd.m_change - fee, + i->first, + payment_id, + fee, + destinations, + pd.m_subaddr_indices, + note, + "-" + }); + } + } + } + // sort by block, then by timestamp (unconfirmed last) + std::sort(transfers.begin(), transfers.end(), [](const transfer_view& a, const transfer_view& b) -> bool { + if (a.confirmed && !b.confirmed) + return true; + if (a.block == b.block) + return a.timestamp < b.timestamp; + return a.block < b.block; + }); + + return true; +} +//---------------------------------------------------------------------------------------------------- +bool simple_wallet::show_transfers(const std::vector<std::string> &args_) +{ + std::vector<std::string> local_args = args_; + + if(local_args.size() > 4) { + fail_msg_writer() << tr("usage: show_transfers [in|out|all|pending|failed|coinbase] [index=<N1>[,<N2>,...]] [<min_height> [<max_height>]]"); + return true; + } + + LOCK_IDLE_SCOPE(); + + std::vector<transfer_view> all_transfers; + + if (!get_transfers(local_args, all_transfers)) + return true; + + PAUSE_READLINE(); + + for (const auto& transfer : all_transfers) + { + const auto color = transfer.type == "failed" ? console_color_red : transfer.confirmed ? ((transfer.direction == "in" || transfer.direction == "block") ? console_color_green : console_color_magenta) : console_color_default; + + std::string destinations = "-"; + if (!transfer.outputs.empty()) + { + destinations = ""; + for (const auto& output : transfer.outputs) + { + if (!destinations.empty()) + destinations += ", "; + destinations += (transfer.direction == "in" ? output.first.substr(0, 6) : output.first) + ":" + print_money(output.second); } } + + auto formatter = boost::format("%8.8llu %6.6s %8.8s %25.25s %20.20s %s %s %14.14s %s %s - %s"); + + message_writer(color, false) << formatter + % transfer.block + % transfer.direction + % transfer.unlocked + % tools::get_human_readable_timestamp(transfer.timestamp) + % print_money(transfer.amount) + % string_tools::pod_to_hex(transfer.hash) + % transfer.payment_id + % print_money(transfer.fee) + % destinations + % boost::algorithm::join(transfer.index | boost::adaptors::transformed([](uint32_t i) { return std::to_string(i); }), ", ") + % transfer.note; } return true; } //---------------------------------------------------------------------------------------------------- +bool simple_wallet::export_transfers(const std::vector<std::string>& args_) +{ + std::vector<std::string> local_args = args_; + + if(local_args.size() > 5) { + fail_msg_writer() << tr("usage: export_transfers [in|out|all|pending|failed|coinbase] [index=<N1>[,<N2>,...]] [<min_height> [<max_height>]] [output=<path>]"); + return true; + } + + LOCK_IDLE_SCOPE(); + + std::vector<transfer_view> all_transfers; + + // might consumes arguments in local_args + if (!get_transfers(local_args, all_transfers)) + return true; + + // output filename + std::string filename = (boost::format("output%u.csv") % m_current_subaddress_account).str(); + if (local_args.size() > 0 && local_args[0].substr(0, 7) == "output=") + { + filename = local_args[0].substr(7, -1); + local_args.erase(local_args.begin()); + } + + std::ofstream file(filename); + + // header + file << + boost::format("%8.8s,%9.9s,%8.8s,%25.25s,%20.20s,%20.20s,%64.64s,%16.16s,%14.14s,%100.100s,%20.20s,%s,%s") % + tr("block") % tr("direction") % tr("unlocked") % tr("timestamp") % tr("amount") % tr("running balance") % tr("hash") % tr("payment ID") % tr("fee") % tr("destination") % tr("amount") % tr("index") % tr("note") + << std::endl; + + uint64_t running_balance = 0; + auto formatter = boost::format("%8.8llu,%9.9s,%8.8s,%25.25s,%20.20s,%20.20s,%64.64s,%16.16s,%14.14s,%100.100s,%20.20s,\"%s\",%s"); + + for (const auto& transfer : all_transfers) + { + // ignore unconfirmed transfers in running balance + if (transfer.confirmed) + { + if (transfer.direction == "in" || transfer.direction == "block") + running_balance += transfer.amount; + else + running_balance -= transfer.amount + transfer.fee; + } + + file << formatter + % transfer.block + % transfer.direction + % transfer.unlocked + % tools::get_human_readable_timestamp(transfer.timestamp) + % print_money(transfer.amount) + % print_money(running_balance) + % string_tools::pod_to_hex(transfer.hash) + % transfer.payment_id + % print_money(transfer.fee) + % (transfer.outputs.size() ? transfer.outputs[0].first : "-") + % (transfer.outputs.size() ? print_money(transfer.outputs[0].second) : "") + % boost::algorithm::join(transfer.index | boost::adaptors::transformed([](uint32_t i) { return std::to_string(i); }), ", ") + % transfer.note + << std::endl; + + for (size_t i = 1; i < transfer.outputs.size(); ++i) + { + file << formatter + % "" + % "" + % "" + % "" + % "" + % "" + % "" + % "" + % "" + % transfer.outputs[i].first + % print_money(transfer.outputs[i].second) + % "" + % "" + << std::endl; + } + } + file.close(); + + success_msg_writer() << tr("CSV exported to ") << filename; + + return true; +} +//---------------------------------------------------------------------------------------------------- bool simple_wallet::unspent_outputs(const std::vector<std::string> &args_) { if(args_.size() > 3) { - fail_msg_writer() << tr("usage: unspent_outputs [index=<N1>[,<N2>,...]] [<min_amount> [<max_amount>]]"); + PRINT_USAGE(USAGE_UNSPENT_OUTPUTS); return true; } auto local_args = args_; @@ -6982,15 +7771,45 @@ bool simple_wallet::unspent_outputs(const std::vector<std::string> &args_) //---------------------------------------------------------------------------------------------------- bool simple_wallet::rescan_blockchain(const std::vector<std::string> &args_) { - message_writer() << tr("Warning: this will lose any information which can not be recovered from the blockchain."); - message_writer() << tr("This includes destination addresses, tx secret keys, tx notes, etc"); - std::string confirm = input_line(tr("Rescan anyway ? (Y/Yes/N/No): ")); - if(!std::cin.eof()) + bool hard = false; + if (!args_.empty()) { - if (!command_line::is_yes(confirm)) + if (args_[0] != "hard") + { + PRINT_USAGE(USAGE_RESCAN_BC); return true; + } + hard = true; + } + + if (hard) + { + message_writer() << tr("Warning: this will lose any information which can not be recovered from the blockchain."); + message_writer() << tr("This includes destination addresses, tx secret keys, tx notes, etc"); + std::string confirm = input_line(tr("Rescan anyway?"), true); + if(!std::cin.eof()) + { + if (!command_line::is_yes(confirm)) + return true; + } + } + return refresh_main(0, hard ? ResetHard : ResetSoft, true); +} +//---------------------------------------------------------------------------------------------------- +void simple_wallet::check_for_messages() +{ + try + { + std::vector<mms::message> new_messages; + bool new_message = get_message_store().check_for_messages(get_multisig_wallet_state(), new_messages); + if (new_message) + { + message_writer(console_color_magenta, true) << tr("MMS received new message"); + list_mms_messages(new_messages); + m_cmd_binder.print_prompt(); + } } - return refresh_main(0, true); + catch(...) {} } //---------------------------------------------------------------------------------------------------- void simple_wallet::wallet_idle_thread() @@ -7008,13 +7827,22 @@ void simple_wallet::wallet_idle_thread() try { uint64_t fetched_blocks; + bool received_money; if (try_connect_to_daemon(true)) - m_wallet->refresh(m_wallet->is_trusted_daemon(), 0, fetched_blocks); + m_wallet->refresh(m_wallet->is_trusted_daemon(), 0, fetched_blocks, received_money, false); // don't check the pool in background mode } catch(...) {} m_auto_refresh_refreshing = false; } + // Check for new MMS messages; + // For simplicity auto message check is ALSO controlled by "m_auto_refresh_enabled" and has no + // separate thread either; thread syncing is tricky enough with only this one idle thread here + if (m_auto_refresh_enabled && get_message_store().get_active()) + { + check_for_messages(); + } + if (!m_idle_run.load(std::memory_order_relaxed)) break; m_idle_cond.wait_for(lock, boost::chrono::seconds(90)); @@ -7038,7 +7866,7 @@ bool simple_wallet::run() // check and display warning, but go on anyway try_connect_to_daemon(); - refresh_main(0, false, true); + refresh_main(0, ResetNone, true); m_auto_refresh_enabled = m_wallet->auto_refresh(); m_idle_thread = boost::thread([&]{wallet_idle_thread();}); @@ -7050,12 +7878,6 @@ bool simple_wallet::run() void simple_wallet::stop() { m_cmd_binder.stop_handling(); - - m_idle_run.store(false, std::memory_order_relaxed); - m_wallet->stop(); - // make the background refresh thread quit - boost::unique_lock<boost::mutex> lock(m_idle_mutex); - m_idle_cond.notify_one(); } //---------------------------------------------------------------------------------------------------- bool simple_wallet::account(const std::vector<std::string> &args/* = std::vector<std::string>()*/) @@ -7200,14 +8022,7 @@ bool simple_wallet::account(const std::vector<std::string> &args/* = std::vector } else { - fail_msg_writer() << tr("usage:\n" - " account\n" - " account new <label text with white spaces allowed>\n" - " account switch <index>\n" - " account label <index> <label text with white spaces allowed>\n" - " account tag <tag_name> <account_index_1> [<account_index_2> ...]\n" - " account untag <account_index_1> [<account_index_2> ...]\n" - " account tag_description <tag_name> <description>"); + PRINT_USAGE(USAGE_ACCOUNT); } return true; } @@ -7362,7 +8177,7 @@ bool simple_wallet::print_address(const std::vector<std::string> &args/* = std:: } else { - fail_msg_writer() << tr("usage: address [ new <label text with white spaces allowed> | all | <index_min> [<index_max>] | label <index> <label text with white spaces allowed> ]"); + PRINT_USAGE(USAGE_ADDRESS); } return true; @@ -7373,7 +8188,7 @@ bool simple_wallet::print_integrated_address(const std::vector<std::string> &arg crypto::hash8 payment_id; if (args.size() > 1) { - fail_msg_writer() << tr("usage: integrated_address [payment ID]"); + PRINT_USAGE(USAGE_INTEGRATED_ADDRESS); return true; } if (args.size() == 0) @@ -7425,7 +8240,7 @@ bool simple_wallet::address_book(const std::vector<std::string> &args/* = std::v } else if (args.size() == 1 || (args[0] != "add" && args[0] != "delete")) { - fail_msg_writer() << tr("usage: address_book [(add (<address> [pid <long or short payment id>])|<integrated address> [<description possibly with whitespaces>])|(delete <index>)]"); + PRINT_USAGE(USAGE_ADDRESS_BOOK); return true; } else if (args[0] == "add") @@ -7446,12 +8261,13 @@ bool simple_wallet::address_book(const std::vector<std::string> &args/* = std::v { if (tools::wallet2::parse_long_payment_id(args[3], payment_id)) { + LONG_PAYMENT_ID_SUPPORT_CHECK(); description_start += 2; } else if (tools::wallet2::parse_short_payment_id(args[3], info.payment_id)) { - memcpy(payment_id.data, info.payment_id.data, 8); - description_start += 2; + fail_msg_writer() << tr("Short payment IDs are to be used within an integrated address only"); + return true; } else { @@ -7489,7 +8305,7 @@ bool simple_wallet::address_book(const std::vector<std::string> &args/* = std::v auto& row = address_book[i]; success_msg_writer() << tr("Index: ") << i; success_msg_writer() << tr("Address: ") << get_account_address_as_str(m_wallet->nettype(), row.m_is_subaddress, row.m_address); - success_msg_writer() << tr("Payment ID: ") << row.m_payment_id; + success_msg_writer() << tr("Payment ID: ") << row.m_payment_id << " (OBSOLETE)"; success_msg_writer() << tr("Description: ") << row.m_description << "\n"; } } @@ -7500,7 +8316,7 @@ bool simple_wallet::set_tx_note(const std::vector<std::string> &args) { if (args.size() == 0) { - fail_msg_writer() << tr("usage: set_tx_note [txid] free text note"); + PRINT_USAGE(USAGE_SET_TX_NOTE); return true; } @@ -7528,7 +8344,7 @@ bool simple_wallet::get_tx_note(const std::vector<std::string> &args) { if (args.size() != 1) { - fail_msg_writer() << tr("usage: get_tx_note [txid]"); + PRINT_USAGE(USAGE_GET_TX_NOTE); return true; } @@ -7569,7 +8385,7 @@ bool simple_wallet::get_description(const std::vector<std::string> &args) { if (args.size() != 0) { - fail_msg_writer() << tr("usage: get_description"); + PRINT_USAGE(USAGE_GET_DESCRIPTION); return true; } @@ -7642,7 +8458,7 @@ bool simple_wallet::sign(const std::vector<std::string> &args) } if (args.size() != 1) { - fail_msg_writer() << tr("usage: sign <filename>"); + PRINT_USAGE(USAGE_SIGN); return true; } if (m_wallet->watch_only()) @@ -7656,7 +8472,6 @@ bool simple_wallet::sign(const std::vector<std::string> &args) return true; } - SCOPED_WALLET_UNLOCK(); std::string filename = args[0]; std::string data; bool r = epee::file_io_utils::load_file_to_string(filename, data); @@ -7665,6 +8480,9 @@ bool simple_wallet::sign(const std::vector<std::string> &args) fail_msg_writer() << tr("failed to read file ") << filename; return true; } + + SCOPED_WALLET_UNLOCK(); + std::string signature = m_wallet->sign(data); success_msg_writer() << signature; return true; @@ -7674,7 +8492,7 @@ bool simple_wallet::verify(const std::vector<std::string> &args) { if (args.size() != 3) { - fail_msg_writer() << tr("usage: verify <filename> <address> <signature>"); + PRINT_USAGE(USAGE_VERIFY); return true; } std::string filename = args[0]; @@ -7717,7 +8535,7 @@ bool simple_wallet::export_key_images(const std::vector<std::string> &args) } if (args.size() != 1) { - fail_msg_writer() << tr("usage: export_key_images <filename>"); + PRINT_USAGE(USAGE_EXPORT_KEY_IMAGES); return true; } if (m_wallet->watch_only()) @@ -7726,11 +8544,12 @@ bool simple_wallet::export_key_images(const std::vector<std::string> &args) return true; } - SCOPED_WALLET_UNLOCK(); std::string filename = args[0]; if (m_wallet->confirm_export_overwrite() && !check_file_overwrite(filename)) return true; + SCOPED_WALLET_UNLOCK(); + try { if (!m_wallet->export_key_images(filename)) @@ -7765,7 +8584,7 @@ bool simple_wallet::import_key_images(const std::vector<std::string> &args) if (args.size() != 1) { - fail_msg_writer() << tr("usage: import_key_images <filename>"); + PRINT_USAGE(USAGE_IMPORT_KEY_IMAGES); return true; } std::string filename = args[0]; @@ -7792,6 +8611,52 @@ bool simple_wallet::import_key_images(const std::vector<std::string> &args) return true; } //---------------------------------------------------------------------------------------------------- +bool simple_wallet::hw_key_images_sync(const std::vector<std::string> &args) +{ + if (!m_wallet->key_on_device()) + { + fail_msg_writer() << tr("command only supported by HW wallet"); + return true; + } + if (!m_wallet->get_account().get_device().has_ki_cold_sync()) + { + fail_msg_writer() << tr("hw wallet does not support cold KI sync"); + return true; + } + + LOCK_IDLE_SCOPE(); + key_images_sync_intern(); + return true; +} +//---------------------------------------------------------------------------------------------------- +void simple_wallet::key_images_sync_intern(){ + try + { + message_writer(console_color_white, false) << tr("Please confirm the key image sync on the device"); + + uint64_t spent = 0, unspent = 0; + uint64_t height = m_wallet->cold_key_image_sync(spent, unspent); + if (height > 0) + { + success_msg_writer() << tr("Key images synchronized to height ") << height; + if (!m_wallet->is_trusted_daemon()) + { + message_writer() << tr("Running untrusted daemon, cannot determine which transaction output is spent. Use a trusted daemon with --trusted-daemon and run rescan_spent"); + } else + { + success_msg_writer() << print_money(spent) << tr(" spent, ") << print_money(unspent) << tr(" unspent"); + } + } + else { + fail_msg_writer() << tr("Failed to import key images"); + } + } + catch (const std::exception &e) + { + fail_msg_writer() << tr("Failed to import key images: ") << e.what(); + } +} +//---------------------------------------------------------------------------------------------------- bool simple_wallet::hw_reconnect(const std::vector<std::string> &args) { if (!m_wallet->key_on_device()) @@ -7826,15 +8691,16 @@ bool simple_wallet::export_outputs(const std::vector<std::string> &args) } if (args.size() != 1) { - fail_msg_writer() << tr("usage: export_outputs <filename>"); + PRINT_USAGE(USAGE_EXPORT_OUTPUTS); return true; } - SCOPED_WALLET_UNLOCK(); std::string filename = args[0]; if (m_wallet->confirm_export_overwrite() && !check_file_overwrite(filename)) return true; + SCOPED_WALLET_UNLOCK(); + try { std::string data = m_wallet->export_outputs_to_str(); @@ -7865,7 +8731,7 @@ bool simple_wallet::import_outputs(const std::vector<std::string> &args) } if (args.size() != 1) { - fail_msg_writer() << tr("usage: import_outputs <filename>"); + PRINT_USAGE(USAGE_IMPORT_OUTPUTS); return true; } std::string filename = args[0]; @@ -7897,7 +8763,7 @@ bool simple_wallet::show_transfer(const std::vector<std::string> &args) { if (args.size() != 1) { - fail_msg_writer() << tr("usage: show_transfer <txid>"); + PRINT_USAGE(USAGE_SHOW_TRANSFER); return true; } @@ -7922,7 +8788,7 @@ bool simple_wallet::show_transfer(const std::vector<std::string> &args) success_msg_writer() << "Incoming transaction found"; success_msg_writer() << "txid: " << txid; success_msg_writer() << "Height: " << pd.m_block_height; - success_msg_writer() << "Timestamp: " << get_human_readable_timestamp(pd.m_timestamp); + success_msg_writer() << "Timestamp: " << tools::get_human_readable_timestamp(pd.m_timestamp); success_msg_writer() << "Amount: " << print_money(pd.m_amount); success_msg_writer() << "Payment ID: " << payment_id; if (pd.m_unlock_time < CRYPTONOTE_MAX_BLOCK_NUMBER) @@ -7972,7 +8838,7 @@ bool simple_wallet::show_transfer(const std::vector<std::string> &args) success_msg_writer() << "Outgoing transaction found"; success_msg_writer() << "txid: " << txid; success_msg_writer() << "Height: " << pd.m_block_height; - success_msg_writer() << "Timestamp: " << get_human_readable_timestamp(pd.m_timestamp); + success_msg_writer() << "Timestamp: " << tools::get_human_readable_timestamp(pd.m_timestamp); success_msg_writer() << "Amount: " << print_money(pd.m_amount_in - change - fee); success_msg_writer() << "Payment ID: " << payment_id; success_msg_writer() << "Change: " << print_money(change); @@ -7997,7 +8863,7 @@ bool simple_wallet::show_transfer(const std::vector<std::string> &args) payment_id = payment_id.substr(0,16); success_msg_writer() << "Unconfirmed incoming transaction found in the txpool"; success_msg_writer() << "txid: " << txid; - success_msg_writer() << "Timestamp: " << get_human_readable_timestamp(pd.m_timestamp); + success_msg_writer() << "Timestamp: " << tools::get_human_readable_timestamp(pd.m_timestamp); success_msg_writer() << "Amount: " << print_money(pd.m_amount); success_msg_writer() << "Payment ID: " << payment_id; success_msg_writer() << "Address index: " << pd.m_subaddr_index.minor; @@ -8028,7 +8894,7 @@ bool simple_wallet::show_transfer(const std::vector<std::string> &args) success_msg_writer() << (is_failed ? "Failed" : "Pending") << " outgoing transaction found"; success_msg_writer() << "txid: " << txid; - success_msg_writer() << "Timestamp: " << get_human_readable_timestamp(pd.m_timestamp); + success_msg_writer() << "Timestamp: " << tools::get_human_readable_timestamp(pd.m_timestamp); success_msg_writer() << "Amount: " << print_money(amount - pd.m_change - fee); success_msg_writer() << "Payment ID: " << payment_id; success_msg_writer() << "Change: " << print_money(pd.m_change); @@ -8117,10 +8983,12 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_params, arg_electrum_seed ); command_line::add_arg(desc_params, arg_allow_mismatched_daemon_version); command_line::add_arg(desc_params, arg_restore_height); + command_line::add_arg(desc_params, arg_restore_date); command_line::add_arg(desc_params, arg_do_not_relay); command_line::add_arg(desc_params, arg_create_address_file); command_line::add_arg(desc_params, arg_subaddress_lookahead); command_line::add_arg(desc_params, arg_use_english_language_names); + command_line::add_arg(desc_params, arg_long_payment_id_support); po::positional_options_description positional_options; positional_options.add(arg_command.name, -1); @@ -8129,7 +8997,7 @@ int main(int argc, char* argv[]) bool should_terminate = false; std::tie(vm, should_terminate) = wallet_args::main( argc, argv, - "monero-wallet-cli [--wallet-file=<file>|--generate-new-wallet=<file>] [<COMMAND>]", + "monero-wallet-cli [--wallet-file=<filename>|--generate-new-wallet=<filename>] [<COMMAND>]", sw::tr("This is the command line monero wallet. It needs to connect to a monero\ndaemon to work correctly.\nWARNING: Do not reuse your Monero keys on another fork, UNLESS this fork has key reuse mitigations built in. Doing so will harm your privacy."), desc_params, positional_options, @@ -8188,3 +9056,1010 @@ int main(int argc, char* argv[]) return 0; CATCH_ENTRY_L0("main", 1); } + +// MMS --------------------------------------------------------------------------------------------------- + +// Access to the message store, or more exactly to the list of the messages that can be changed +// by the idle thread, is guarded by the same mutex-based mechanism as access to the wallet +// as a whole and thus e.g. uses the "LOCK_IDLE_SCOPE" macro. This is a little over-cautious, but +// simple and safe. Care has to be taken however where MMS methods call other simplewallet methods +// that use "LOCK_IDLE_SCOPE" as this cannot be nested! + +// Methods for commands like "export_multisig_info" usually read data from file(s) or write data +// to files. The MMS calls now those methods as well, to produce data for messages and to process data +// from messages. As it would be quite inconvenient for the MMS to write data for such methods to files +// first or get data out of result files after the call, those methods detect a call from the MMS and +// expect data as arguments instead of files and give back data by calling 'process_wallet_created_data'. + +bool simple_wallet::user_confirms(const std::string &question) +{ + std::string answer = input_line(question + tr(" (Y/Yes/N/No): ")); + return !std::cin.eof() && command_line::is_yes(answer); +} + +bool simple_wallet::get_number_from_arg(const std::string &arg, uint32_t &number, const uint32_t lower_bound, const uint32_t upper_bound) +{ + bool valid = false; + try + { + number = boost::lexical_cast<uint32_t>(arg); + valid = (number >= lower_bound) && (number <= upper_bound); + } + catch(const boost::bad_lexical_cast &) + { + } + return valid; +} + +bool simple_wallet::choose_mms_processing(const std::vector<mms::processing_data> &data_list, uint32_t &choice) +{ + size_t choices = data_list.size(); + if (choices == 1) + { + choice = 0; + return true; + } + mms::message_store& ms = m_wallet->get_message_store(); + message_writer() << tr("Choose processing:"); + std::string text; + for (size_t i = 0; i < choices; ++i) + { + const mms::processing_data &data = data_list[i]; + text = std::to_string(i+1) + ": "; + switch (data.processing) + { + case mms::message_processing::sign_tx: + text += tr("Sign tx"); + break; + case mms::message_processing::send_tx: + { + mms::message m; + ms.get_message_by_id(data.message_ids[0], m); + if (m.type == mms::message_type::fully_signed_tx) + { + text += tr("Send the tx for submission to "); + } + else + { + text += tr("Send the tx for signing to "); + } + mms::authorized_signer signer = ms.get_signer(data.receiving_signer_index); + text += ms.signer_to_string(signer, 50); + break; + } + case mms::message_processing::submit_tx: + text += tr("Submit tx"); + break; + default: + text += tr("unknown"); + break; + } + message_writer() << text; + } + + std::string line = input_line(tr("Choice: ")); + if (std::cin.eof() || line.empty()) + { + return false; + } + bool choice_ok = get_number_from_arg(line, choice, 1, choices); + if (choice_ok) + { + choice--; + } + else + { + fail_msg_writer() << tr("Wrong choice"); + } + return choice_ok; +} + +void simple_wallet::list_mms_messages(const std::vector<mms::message> &messages) +{ + message_writer() << boost::format("%4s %-4s %-30s %-21s %7s %3s %-15s %-40s") % tr("Id") % tr("I/O") % tr("Authorized Signer") + % tr("Message Type") % tr("Height") % tr("R") % tr("Message State") % tr("Since"); + mms::message_store& ms = m_wallet->get_message_store(); + uint64_t now = (uint64_t)time(NULL); + for (size_t i = 0; i < messages.size(); ++i) + { + const mms::message &m = messages[i]; + const mms::authorized_signer &signer = ms.get_signer(m.signer_index); + bool highlight = (m.state == mms::message_state::ready_to_send) || (m.state == mms::message_state::waiting); + message_writer(m.direction == mms::message_direction::out ? console_color_green : console_color_magenta, highlight) << + boost::format("%4s %-4s %-30s %-21s %7s %3s %-15s %-40s") % + m.id % + ms.message_direction_to_string(m.direction) % + ms.signer_to_string(signer, 30) % + ms.message_type_to_string(m.type) % + m.wallet_height % + m.round % + ms.message_state_to_string(m.state) % + (tools::get_human_readable_timestamp(m.modified) + ", " + get_human_readable_timespan(std::chrono::seconds(now - m.modified)) + tr(" ago")); + } +} + +void simple_wallet::list_signers(const std::vector<mms::authorized_signer> &signers) +{ + message_writer() << boost::format("%2s %-20s %-s") % tr("#") % tr("Label") % tr("Transport Address"); + message_writer() << boost::format("%2s %-20s %-s") % "" % tr("Auto-Config Token") % tr("Monero Address"); + for (size_t i = 0; i < signers.size(); ++i) + { + const mms::authorized_signer &signer = signers[i]; + std::string label = signer.label.empty() ? tr("<not set>") : signer.label; + std::string monero_address; + if (signer.monero_address_known) + { + monero_address = get_account_address_as_str(m_wallet->nettype(), false, signer.monero_address); + } + else + { + monero_address = tr("<not set>"); + } + std::string transport_address = signer.transport_address.empty() ? tr("<not set>") : signer.transport_address; + message_writer() << boost::format("%2s %-20s %-s") % (i + 1) % label % transport_address; + message_writer() << boost::format("%2s %-20s %-s") % "" % signer.auto_config_token % monero_address; + message_writer() << ""; + } +} + +void simple_wallet::add_signer_config_messages() +{ + mms::message_store& ms = m_wallet->get_message_store(); + std::string signer_config; + ms.get_signer_config(signer_config); + + const std::vector<mms::authorized_signer> signers = ms.get_all_signers(); + mms::multisig_wallet_state state = get_multisig_wallet_state(); + uint32_t num_authorized_signers = ms.get_num_authorized_signers(); + for (uint32_t i = 1 /* without me */; i < num_authorized_signers; ++i) + { + ms.add_message(state, i, mms::message_type::signer_config, mms::message_direction::out, signer_config); + } +} + +void simple_wallet::show_message(const mms::message &m) +{ + mms::message_store& ms = m_wallet->get_message_store(); + const mms::authorized_signer &signer = ms.get_signer(m.signer_index); + bool display_content; + std::string sanitized_text; + switch (m.type) + { + case mms::message_type::key_set: + case mms::message_type::additional_key_set: + case mms::message_type::note: + display_content = true; + ms.get_sanitized_message_text(m, sanitized_text); + break; + default: + display_content = false; + } + uint64_t now = (uint64_t)time(NULL); + message_writer() << ""; + message_writer() << tr("Message ") << m.id; + message_writer() << tr("In/out: ") << ms.message_direction_to_string(m.direction); + message_writer() << tr("Type: ") << ms.message_type_to_string(m.type); + message_writer() << tr("State: ") << boost::format(tr("%s since %s, %s ago")) % + ms.message_state_to_string(m.state) % tools::get_human_readable_timestamp(m.modified) % get_human_readable_timespan(std::chrono::seconds(now - m.modified)); + if (m.sent == 0) + { + message_writer() << tr("Sent: Never"); + } + else + { + message_writer() << boost::format(tr("Sent: %s, %s ago")) % + tools::get_human_readable_timestamp(m.sent) % get_human_readable_timespan(std::chrono::seconds(now - m.sent)); + } + message_writer() << tr("Authorized signer: ") << ms.signer_to_string(signer, 100); + message_writer() << tr("Content size: ") << m.content.length() << tr(" bytes"); + message_writer() << tr("Content: ") << (display_content ? sanitized_text : tr("(binary data)")); + + if (m.type == mms::message_type::note) + { + // Showing a note and read its text is "processing" it: Set the state accordingly + // which will also delete it from Bitmessage as a side effect + // (Without this little "twist" it would never change the state, and never get deleted) + ms.set_message_processed_or_sent(m.id); + } +} + +void simple_wallet::ask_send_all_ready_messages() +{ + mms::message_store& ms = m_wallet->get_message_store(); + std::vector<mms::message> ready_messages; + const std::vector<mms::message> &messages = ms.get_all_messages(); + for (size_t i = 0; i < messages.size(); ++i) + { + const mms::message &m = messages[i]; + if (m.state == mms::message_state::ready_to_send) + { + ready_messages.push_back(m); + } + } + if (ready_messages.size() != 0) + { + list_mms_messages(ready_messages); + bool send = ms.get_auto_send(); + if (!send) + { + send = user_confirms(tr("Send these messages now?")); + } + if (send) + { + mms::multisig_wallet_state state = get_multisig_wallet_state(); + for (size_t i = 0; i < ready_messages.size(); ++i) + { + ms.send_message(state, ready_messages[i].id); + ms.set_message_processed_or_sent(ready_messages[i].id); + } + success_msg_writer() << tr("Queued for sending."); + } + } +} + +bool simple_wallet::get_message_from_arg(const std::string &arg, mms::message &m) +{ + mms::message_store& ms = m_wallet->get_message_store(); + bool valid_id = false; + uint32_t id; + try + { + id = (uint32_t)boost::lexical_cast<uint32_t>(arg); + valid_id = ms.get_message_by_id(id, m); + } + catch (const boost::bad_lexical_cast &) + { + } + if (!valid_id) + { + fail_msg_writer() << tr("Invalid message id"); + } + return valid_id; +} + +void simple_wallet::mms_init(const std::vector<std::string> &args) +{ + if (args.size() != 3) + { + fail_msg_writer() << tr("usage: mms init <required_signers>/<authorized_signers> <own_label> <own_transport_address>"); + return; + } + mms::message_store& ms = m_wallet->get_message_store(); + if (ms.get_active()) + { + if (!user_confirms(tr("The MMS is already initialized. Re-initialize by deleting all signer info and messages?"))) + { + return; + } + } + uint32_t num_required_signers; + uint32_t num_authorized_signers; + const std::string &mn = args[0]; + std::vector<std::string> numbers; + boost::split(numbers, mn, boost::is_any_of("/")); + bool mn_ok = (numbers.size() == 2) + && get_number_from_arg(numbers[1], num_authorized_signers, 2, 100) + && get_number_from_arg(numbers[0], num_required_signers, 2, num_authorized_signers); + if (!mn_ok) + { + fail_msg_writer() << tr("Error in the number of required signers and/or authorized signers"); + return; + } + LOCK_IDLE_SCOPE(); + ms.init(get_multisig_wallet_state(), args[1], args[2], num_authorized_signers, num_required_signers); +} + +void simple_wallet::mms_info(const std::vector<std::string> &args) +{ + mms::message_store& ms = m_wallet->get_message_store(); + if (ms.get_active()) + { + message_writer() << boost::format("The MMS is active for %s/%s multisig.") + % ms.get_num_required_signers() % ms.get_num_authorized_signers(); + } + else + { + message_writer() << tr("The MMS is not active."); + } +} + +void simple_wallet::mms_signer(const std::vector<std::string> &args) +{ + mms::message_store& ms = m_wallet->get_message_store(); + const std::vector<mms::authorized_signer> &signers = ms.get_all_signers(); + if (args.size() == 0) + { + // Without further parameters list all defined signers + list_signers(signers); + return; + } + + uint32_t index; + bool index_valid = get_number_from_arg(args[0], index, 1, ms.get_num_authorized_signers()); + if (index_valid) + { + index--; + } + else + { + fail_msg_writer() << tr("Invalid signer number ") + args[0]; + return; + } + if ((args.size() < 2) || (args.size() > 4)) + { + fail_msg_writer() << tr("mms signer [<number> <label> [<transport_address> [<monero_address>]]]"); + return; + } + + boost::optional<string> label = args[1]; + boost::optional<string> transport_address; + if (args.size() >= 3) + { + transport_address = args[2]; + } + boost::optional<cryptonote::account_public_address> monero_address; + LOCK_IDLE_SCOPE(); + mms::multisig_wallet_state state = get_multisig_wallet_state(); + if (args.size() == 4) + { + cryptonote::address_parse_info info; + bool ok = cryptonote::get_account_address_from_str_or_url(info, m_wallet->nettype(), args[3], oa_prompter); + if (!ok) + { + fail_msg_writer() << tr("Invalid Monero address"); + return; + } + monero_address = info.address; + const std::vector<mms::message> &messages = ms.get_all_messages(); + if ((messages.size() > 0) || state.multisig) + { + fail_msg_writer() << tr("Wallet state does not allow changing Monero addresses anymore"); + return; + } + } + ms.set_signer(state, index, label, transport_address, monero_address); +} + +void simple_wallet::mms_list(const std::vector<std::string> &args) +{ + mms::message_store& ms = m_wallet->get_message_store(); + if (args.size() != 0) + { + fail_msg_writer() << tr("Usage: mms list"); + return; + } + LOCK_IDLE_SCOPE(); + const std::vector<mms::message> &messages = ms.get_all_messages(); + list_mms_messages(messages); +} + +void simple_wallet::mms_next(const std::vector<std::string> &args) +{ + mms::message_store& ms = m_wallet->get_message_store(); + if ((args.size() > 1) || ((args.size() == 1) && (args[0] != "sync"))) + { + fail_msg_writer() << tr("Usage: mms next [sync]"); + return; + } + bool avail = false; + std::vector<mms::processing_data> data_list; + bool force_sync = false; + uint32_t choice = 0; + { + LOCK_IDLE_SCOPE(); + if ((args.size() == 1) && (args[0] == "sync")) + { + // Force the MMS to process any waiting sync info although on its own it would just ignore + // those messages because no need to process them can be seen + force_sync = true; + } + string wait_reason; + { + avail = ms.get_processable_messages(get_multisig_wallet_state(), force_sync, data_list, wait_reason); + } + if (avail) + { + avail = choose_mms_processing(data_list, choice); + } + else if (!wait_reason.empty()) + { + message_writer() << tr("No next step: ") << wait_reason; + } + } + if (avail) + { + mms::processing_data data = data_list[choice]; + bool command_successful = false; + switch(data.processing) + { + case mms::message_processing::prepare_multisig: + message_writer() << tr("prepare_multisig"); + command_successful = prepare_multisig_main(std::vector<std::string>(), true); + break; + + case mms::message_processing::make_multisig: + { + message_writer() << tr("make_multisig"); + size_t number_of_key_sets = data.message_ids.size(); + std::vector<std::string> sig_args(number_of_key_sets + 1); + sig_args[0] = std::to_string(ms.get_num_required_signers()); + for (size_t i = 0; i < number_of_key_sets; ++i) + { + mms::message m = ms.get_message_by_id(data.message_ids[i]); + sig_args[i+1] = m.content; + } + command_successful = make_multisig_main(sig_args, true); + break; + } + + case mms::message_processing::exchange_multisig_keys: + { + message_writer() << tr("exchange_multisig_keys"); + size_t number_of_key_sets = data.message_ids.size(); + // Other than "make_multisig" only the key sets as parameters, no num_required_signers + std::vector<std::string> sig_args(number_of_key_sets); + for (size_t i = 0; i < number_of_key_sets; ++i) + { + mms::message m = ms.get_message_by_id(data.message_ids[i]); + sig_args[i] = m.content; + } + command_successful = exchange_multisig_keys_main(sig_args, true); + break; + } + + case mms::message_processing::create_sync_data: + { + message_writer() << tr("export_multisig_info"); + std::vector<std::string> export_args; + export_args.push_back("MMS"); // dummy filename + command_successful = export_multisig_main(export_args, true); + break; + } + + case mms::message_processing::process_sync_data: + { + message_writer() << tr("import_multisig_info"); + std::vector<std::string> import_args; + for (size_t i = 0; i < data.message_ids.size(); ++i) + { + mms::message m = ms.get_message_by_id(data.message_ids[i]); + import_args.push_back(m.content); + } + command_successful = import_multisig_main(import_args, true); + break; + } + + case mms::message_processing::sign_tx: + { + message_writer() << tr("sign_multisig"); + std::vector<std::string> sign_args; + mms::message m = ms.get_message_by_id(data.message_ids[0]); + sign_args.push_back(m.content); + command_successful = sign_multisig_main(sign_args, true); + break; + } + + case mms::message_processing::submit_tx: + { + message_writer() << tr("submit_multisig"); + std::vector<std::string> submit_args; + mms::message m = ms.get_message_by_id(data.message_ids[0]); + submit_args.push_back(m.content); + command_successful = submit_multisig_main(submit_args, true); + break; + } + + case mms::message_processing::send_tx: + { + message_writer() << tr("Send tx"); + mms::message m = ms.get_message_by_id(data.message_ids[0]); + LOCK_IDLE_SCOPE(); + ms.add_message(get_multisig_wallet_state(), data.receiving_signer_index, m.type, mms::message_direction::out, + m.content); + command_successful = true; + break; + } + + case mms::message_processing::process_signer_config: + { + message_writer() << tr("Process signer config"); + LOCK_IDLE_SCOPE(); + mms::message m = ms.get_message_by_id(data.message_ids[0]); + mms::authorized_signer me = ms.get_signer(0); + mms::multisig_wallet_state state = get_multisig_wallet_state(); + if (!me.auto_config_running) + { + // If no auto-config is running, the config sent may be unsolicited or problematic + // so show what arrived and ask for confirmation before taking it in + std::vector<mms::authorized_signer> signers; + ms.unpack_signer_config(state, m.content, signers); + list_signers(signers); + if (!user_confirms(tr("Replace current signer config with the one displayed above?"))) + { + break; + } + } + ms.process_signer_config(state, m.content); + ms.stop_auto_config(); + list_signers(ms.get_all_signers()); + command_successful = true; + break; + } + + case mms::message_processing::process_auto_config_data: + { + message_writer() << tr("Process auto config data"); + LOCK_IDLE_SCOPE(); + for (size_t i = 0; i < data.message_ids.size(); ++i) + { + ms.process_auto_config_data_message(data.message_ids[i]); + } + ms.stop_auto_config(); + list_signers(ms.get_all_signers()); + add_signer_config_messages(); + command_successful = true; + break; + } + + default: + message_writer() << tr("Nothing ready to process"); + break; + } + + if (command_successful) + { + { + LOCK_IDLE_SCOPE(); + ms.set_messages_processed(data); + ask_send_all_ready_messages(); + } + } + } +} + +void simple_wallet::mms_sync(const std::vector<std::string> &args) +{ + mms::message_store& ms = m_wallet->get_message_store(); + if (args.size() != 0) + { + fail_msg_writer() << tr("Usage: mms sync"); + return; + } + // Force the start of a new sync round, for exceptional cases where something went wrong + // Can e.g. solve the problem "This signature was made with stale data" after trying to + // create 2 transactions in a row somehow + // Code is identical to the code for 'message_processing::create_sync_data' + message_writer() << tr("export_multisig_info"); + std::vector<std::string> export_args; + export_args.push_back("MMS"); // dummy filename + export_multisig_main(export_args, true); + ask_send_all_ready_messages(); +} + +void simple_wallet::mms_transfer(const std::vector<std::string> &args) +{ + // It's too complicated to check any arguments here, just let 'transfer_main' do the whole job + transfer_main(Transfer, args, true); +} + +void simple_wallet::mms_delete(const std::vector<std::string> &args) +{ + if (args.size() != 1) + { + fail_msg_writer() << tr("Usage: mms delete (<message_id> | all)"); + return; + } + LOCK_IDLE_SCOPE(); + mms::message_store& ms = m_wallet->get_message_store(); + if (args[0] == "all") + { + if (user_confirms(tr("Delete all messages?"))) + { + ms.delete_all_messages(); + } + } + else + { + mms::message m; + bool valid_id = get_message_from_arg(args[0], m); + if (valid_id) + { + // If only a single message and not all delete even if unsent / unprocessed + ms.delete_message(m.id); + } + } +} + +void simple_wallet::mms_send(const std::vector<std::string> &args) +{ + if (args.size() == 0) + { + ask_send_all_ready_messages(); + return; + } + else if (args.size() != 1) + { + fail_msg_writer() << tr("Usage: mms send [<message_id>]"); + return; + } + LOCK_IDLE_SCOPE(); + mms::message_store& ms = m_wallet->get_message_store(); + mms::message m; + bool valid_id = get_message_from_arg(args[0], m); + if (valid_id) + { + ms.send_message(get_multisig_wallet_state(), m.id); + } +} + +void simple_wallet::mms_receive(const std::vector<std::string> &args) +{ + if (args.size() != 0) + { + fail_msg_writer() << tr("Usage: mms receive"); + return; + } + std::vector<mms::message> new_messages; + LOCK_IDLE_SCOPE(); + mms::message_store& ms = m_wallet->get_message_store(); + bool avail = ms.check_for_messages(get_multisig_wallet_state(), new_messages); + if (avail) + { + list_mms_messages(new_messages); + } +} + +void simple_wallet::mms_export(const std::vector<std::string> &args) +{ + if (args.size() != 1) + { + fail_msg_writer() << tr("Usage: mms export <message_id>"); + return; + } + LOCK_IDLE_SCOPE(); + mms::message_store& ms = m_wallet->get_message_store(); + mms::message m; + bool valid_id = get_message_from_arg(args[0], m); + if (valid_id) + { + const std::string filename = "mms_message_content"; + if (epee::file_io_utils::save_string_to_file(filename, m.content)) + { + success_msg_writer() << tr("Message content saved to: ") << filename; + } + else + { + fail_msg_writer() << tr("Failed to to save message content"); + } + } +} + +void simple_wallet::mms_note(const std::vector<std::string> &args) +{ + mms::message_store& ms = m_wallet->get_message_store(); + if (args.size() == 0) + { + LOCK_IDLE_SCOPE(); + const std::vector<mms::message> &messages = ms.get_all_messages(); + for (size_t i = 0; i < messages.size(); ++i) + { + const mms::message &m = messages[i]; + if ((m.type == mms::message_type::note) && (m.state == mms::message_state::waiting)) + { + show_message(m); + } + } + return; + } + if (args.size() < 2) + { + fail_msg_writer() << tr("Usage: mms note [<label> <text>]"); + return; + } + uint32_t signer_index; + bool found = ms.get_signer_index_by_label(args[0], signer_index); + if (!found) + { + fail_msg_writer() << tr("No signer found with label ") << args[0]; + return; + } + std::string note = ""; + for (size_t n = 1; n < args.size(); ++n) + { + if (n > 1) + { + note += " "; + } + note += args[n]; + } + LOCK_IDLE_SCOPE(); + ms.add_message(get_multisig_wallet_state(), signer_index, mms::message_type::note, + mms::message_direction::out, note); + ask_send_all_ready_messages(); +} + +void simple_wallet::mms_show(const std::vector<std::string> &args) +{ + if (args.size() != 1) + { + fail_msg_writer() << tr("Usage: mms show <message_id>"); + return; + } + LOCK_IDLE_SCOPE(); + mms::message_store& ms = m_wallet->get_message_store(); + mms::message m; + bool valid_id = get_message_from_arg(args[0], m); + if (valid_id) + { + show_message(m); + } +} + +void simple_wallet::mms_set(const std::vector<std::string> &args) +{ + bool set = args.size() == 2; + bool query = args.size() == 1; + if (!set && !query) + { + fail_msg_writer() << tr("Usage: mms set <option_name> [<option_value>]"); + return; + } + mms::message_store& ms = m_wallet->get_message_store(); + LOCK_IDLE_SCOPE(); + if (args[0] == "auto-send") + { + if (set) + { + bool result; + bool ok = parse_bool(args[1], result); + if (ok) + { + ms.set_auto_send(result); + } + else + { + fail_msg_writer() << tr("Wrong option value"); + } + } + else + { + message_writer() << (ms.get_auto_send() ? tr("Auto-send is on") : tr("Auto-send is off")); + } + } + else + { + fail_msg_writer() << tr("Unknown option"); + } +} + +void simple_wallet::mms_help(const std::vector<std::string> &args) +{ + if (args.size() > 1) + { + fail_msg_writer() << tr("Usage: mms help [<subcommand>]"); + return; + } + std::vector<std::string> help_args; + help_args.push_back("mms"); + if (args.size() == 1) + { + help_args.push_back(args[0]); + } + help(help_args); +} + +void simple_wallet::mms_send_signer_config(const std::vector<std::string> &args) +{ + if (args.size() != 0) + { + fail_msg_writer() << tr("Usage: mms send_signer_config"); + return; + } + mms::message_store& ms = m_wallet->get_message_store(); + if (!ms.signer_config_complete()) + { + fail_msg_writer() << tr("Signer config not yet complete"); + return; + } + LOCK_IDLE_SCOPE(); + add_signer_config_messages(); + ask_send_all_ready_messages(); +} + +void simple_wallet::mms_start_auto_config(const std::vector<std::string> &args) +{ + mms::message_store& ms = m_wallet->get_message_store(); + uint32_t other_signers = ms.get_num_authorized_signers() - 1; + size_t args_size = args.size(); + if ((args_size != 0) && (args_size != other_signers)) + { + fail_msg_writer() << tr("Usage: mms start_auto_config [<label> <label> ...]"); + return; + } + if ((args_size == 0) && !ms.signer_labels_complete()) + { + fail_msg_writer() << tr("There are signers without a label set. Complete labels before auto-config or specify them as parameters here."); + return; + } + mms::authorized_signer me = ms.get_signer(0); + if (me.auto_config_running) + { + if (!user_confirms(tr("Auto-config is already running. Cancel and restart?"))) + { + return; + } + } + LOCK_IDLE_SCOPE(); + mms::multisig_wallet_state state = get_multisig_wallet_state(); + if (args_size != 0) + { + // Set (or overwrite) all the labels except "me" from the arguments + for (uint32_t i = 1; i < (other_signers + 1); ++i) + { + ms.set_signer(state, i, args[i - 1], boost::none, boost::none); + } + } + ms.start_auto_config(state); + // List the signers to show the generated auto-config tokens + list_signers(ms.get_all_signers()); +} + +void simple_wallet::mms_stop_auto_config(const std::vector<std::string> &args) +{ + if (args.size() != 0) + { + fail_msg_writer() << tr("Usage: mms stop_auto_config"); + return; + } + if (!user_confirms(tr("Delete any auto-config tokens and stop auto-config?"))) + { + return; + } + mms::message_store& ms = m_wallet->get_message_store(); + LOCK_IDLE_SCOPE(); + ms.stop_auto_config(); +} + +void simple_wallet::mms_auto_config(const std::vector<std::string> &args) +{ + if (args.size() != 1) + { + fail_msg_writer() << tr("Usage: mms auto_config <auto_config_token>"); + return; + } + mms::message_store& ms = m_wallet->get_message_store(); + std::string adjusted_token; + if (!ms.check_auto_config_token(args[0], adjusted_token)) + { + fail_msg_writer() << tr("Invalid auto-config token"); + return; + } + mms::authorized_signer me = ms.get_signer(0); + if (me.auto_config_running) + { + if (!user_confirms(tr("Auto-config already running. Cancel and restart?"))) + { + return; + } + } + LOCK_IDLE_SCOPE(); + ms.add_auto_config_data_message(get_multisig_wallet_state(), adjusted_token); + ask_send_all_ready_messages(); +} + +bool simple_wallet::mms(const std::vector<std::string> &args) +{ + try + { + mms::message_store& ms = m_wallet->get_message_store(); + if (args.size() == 0) + { + mms_info(args); + return true; + } + + const std::string &sub_command = args[0]; + std::vector<std::string> mms_args = args; + mms_args.erase(mms_args.begin()); + + if (sub_command == "init") + { + mms_init(mms_args); + return true; + } + if (!ms.get_active()) + { + fail_msg_writer() << tr("The MMS is not active. Activate using the \"mms init\" command"); + return true; + } + else if (sub_command == "info") + { + mms_info(mms_args); + } + else if (sub_command == "signer") + { + mms_signer(mms_args); + } + else if (sub_command == "list") + { + mms_list(mms_args); + } + else if (sub_command == "next") + { + mms_next(mms_args); + } + else if (sub_command == "sync") + { + mms_sync(mms_args); + } + else if (sub_command == "transfer") + { + mms_transfer(mms_args); + } + else if (sub_command == "delete") + { + mms_delete(mms_args); + } + else if (sub_command == "send") + { + mms_send(mms_args); + } + else if (sub_command == "receive") + { + mms_receive(mms_args); + } + else if (sub_command == "export") + { + mms_export(mms_args); + } + else if (sub_command == "note") + { + mms_note(mms_args); + } + else if (sub_command == "show") + { + mms_show(mms_args); + } + else if (sub_command == "set") + { + mms_set(mms_args); + } + else if (sub_command == "help") + { + mms_help(mms_args); + } + else if (sub_command == "send_signer_config") + { + mms_send_signer_config(mms_args); + } + else if (sub_command == "start_auto_config") + { + mms_start_auto_config(mms_args); + } + else if (sub_command == "stop_auto_config") + { + mms_stop_auto_config(mms_args); + } + else if (sub_command == "auto_config") + { + mms_auto_config(mms_args); + } + else + { + fail_msg_writer() << tr("Invalid MMS subcommand"); + } + } + catch (const tools::error::no_connection_to_daemon &e) + { + fail_msg_writer() << tr("Error in MMS command: ") << e.what() << " " << e.request(); + } + catch (const std::exception &e) + { + fail_msg_writer() << tr("Error in MMS command: ") << e.what(); + PRINT_USAGE(USAGE_MMS); + return true; + } + return true; +} +// End MMS ------------------------------------------------------------------------------------------------ + diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h index 39b715b73..c3dc16d96 100644 --- a/src/simplewallet/simplewallet.h +++ b/src/simplewallet/simplewallet.h @@ -83,6 +83,9 @@ namespace cryptonote std::string get_commands_str(); std::string get_command_usage(const std::vector<std::string> &args); private: + + enum ResetType { ResetNone, ResetSoft, ResetHard }; + bool handle_command_line(const boost::program_options::variables_map& vm); bool run_console_handler(); @@ -139,6 +142,8 @@ namespace cryptonote bool set_subaddress_lookahead(const std::vector<std::string> &args = std::vector<std::string>()); bool set_segregation_height(const std::vector<std::string> &args = std::vector<std::string>()); bool set_ignore_fractional_outputs(const std::vector<std::string> &args = std::vector<std::string>()); + bool set_track_uses(const std::vector<std::string> &args = std::vector<std::string>()); + bool set_device_name(const std::vector<std::string> &args = std::vector<std::string>()); bool help(const std::vector<std::string> &args = std::vector<std::string>()); bool start_mining(const std::vector<std::string> &args); bool stop_mining(const std::vector<std::string> &args); @@ -150,7 +155,7 @@ namespace cryptonote bool show_incoming_transfers(const std::vector<std::string> &args); bool show_payments(const std::vector<std::string> &args); bool show_blockchain_height(const std::vector<std::string> &args); - bool transfer_main(int transfer_type, const std::vector<std::string> &args); + bool transfer_main(int transfer_type, const std::vector<std::string> &args, bool called_by_mms); bool transfer(const std::vector<std::string> &args); bool locked_transfer(const std::vector<std::string> &args); bool locked_sweep_all(const std::vector<std::string> &args); @@ -186,9 +191,10 @@ namespace cryptonote bool get_reserve_proof(const std::vector<std::string> &args); bool check_reserve_proof(const std::vector<std::string> &args); bool show_transfers(const std::vector<std::string> &args); + bool export_transfers(const std::vector<std::string> &args); bool unspent_outputs(const std::vector<std::string> &args); bool rescan_blockchain(const std::vector<std::string> &args); - bool refresh_main(uint64_t start_height, bool reset = false, bool is_init = false); + bool refresh_main(uint64_t start_height, ResetType reset, bool is_init = false); bool set_tx_note(const std::vector<std::string> &args); bool get_tx_note(const std::vector<std::string> &args); bool set_description(const std::vector<std::string> &args); @@ -200,6 +206,7 @@ namespace cryptonote bool verify(const std::vector<std::string> &args); bool export_key_images(const std::vector<std::string> &args); bool import_key_images(const std::vector<std::string> &args); + bool hw_key_images_sync(const std::vector<std::string> &args); bool hw_reconnect(const std::vector<std::string> &args); bool export_outputs(const std::vector<std::string> &args); bool import_outputs(const std::vector<std::string> &args); @@ -208,15 +215,23 @@ namespace cryptonote bool payment_id(const std::vector<std::string> &args); bool print_fee_info(const std::vector<std::string> &args); bool prepare_multisig(const std::vector<std::string>& args); + bool prepare_multisig_main(const std::vector<std::string>& args, bool called_by_mms); bool make_multisig(const std::vector<std::string>& args); + bool make_multisig_main(const std::vector<std::string>& args, bool called_by_mms); bool finalize_multisig(const std::vector<std::string> &args); bool exchange_multisig_keys(const std::vector<std::string> &args); + bool exchange_multisig_keys_main(const std::vector<std::string> &args, bool called_by_mms); bool export_multisig(const std::vector<std::string>& args); + bool export_multisig_main(const std::vector<std::string>& args, bool called_by_mms); bool import_multisig(const std::vector<std::string>& args); + bool import_multisig_main(const std::vector<std::string>& args, bool called_by_mms); bool accept_loaded_tx(const tools::wallet2::multisig_tx_set &txs); bool sign_multisig(const std::vector<std::string>& args); + bool sign_multisig_main(const std::vector<std::string>& args, bool called_by_mms); bool submit_multisig(const std::vector<std::string>& args); + bool submit_multisig_main(const std::vector<std::string>& args, bool called_by_mms); bool export_raw_multisig(const std::vector<std::string>& args); + bool mms(const std::vector<std::string>& args); bool print_ring(const std::vector<std::string>& args); bool set_ring(const std::vector<std::string>& args); bool save_known_rings(const std::vector<std::string>& args); @@ -224,6 +239,7 @@ namespace cryptonote bool unblackball(const std::vector<std::string>& args); bool blackballed(const std::vector<std::string>& args); bool version(const std::vector<std::string>& args); + bool cold_sign_tx(const std::vector<tools::wallet2::pending_tx>& ptx_vector, tools::wallet2::signed_tx_set &exported_txs, std::vector<cryptonote::address_parse_info> &dsts_info, std::function<bool(const tools::wallet2::signed_tx_set &)> accept_func); uint64_t get_daemon_blockchain_height(std::string& err); bool try_connect_to_daemon(bool silent = false, uint32_t* version = nullptr); @@ -234,6 +250,27 @@ namespace cryptonote bool print_ring_members(const std::vector<tools::wallet2::pending_tx>& ptx_vector, std::ostream& ostr); std::string get_prompt() const; bool print_seed(bool encrypted); + void key_images_sync_intern(); + void on_refresh_finished(uint64_t start_height, uint64_t fetched_blocks, bool is_init, bool received_money); + std::pair<std::string, std::string> show_outputs_line(const std::vector<uint64_t> &heights, uint64_t blockchain_height, uint64_t highlight_height = std::numeric_limits<uint64_t>::max()) const; + + struct transfer_view + { + std::string type; + boost::variant<uint64_t, std::string> block; + uint64_t timestamp; + std::string direction; + bool confirmed; + uint64_t amount; + crypto::hash hash; + std::string payment_id; + uint64_t fee; + std::vector<std::pair<std::string, uint64_t>> outputs; + std::set<uint32_t> index; + std::string note; + std::string unlocked; + }; + bool get_transfers(std::vector<std::string>& args_, std::vector<transfer_view>& transfers); /*! * \brief Prints the seed with a nice message @@ -263,6 +300,9 @@ namespace cryptonote virtual void on_money_spent(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& in_tx, uint64_t amount, const cryptonote::transaction& spend_tx, const cryptonote::subaddress_index& subaddr_index); virtual void on_skip_transaction(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx); virtual boost::optional<epee::wipeable_string> on_get_password(const char *reason); + virtual void on_button_request(); + virtual void on_pin_request(epee::wipeable_string & pin); + virtual void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase); //---------------------------------------------------------- friend class refresh_progress_reporter_t; @@ -330,6 +370,7 @@ namespace cryptonote std::string m_mnemonic_language; std::string m_import_path; std::string m_subaddress_lookahead; + std::string m_restore_date; // optional - converted to m_restore_height epee::wipeable_string m_electrum_seed; // electrum-style seed parameter @@ -357,5 +398,42 @@ namespace cryptonote bool m_auto_refresh_refreshing; std::atomic<bool> m_in_manual_refresh; uint32_t m_current_subaddress_account; + + bool m_long_payment_id_support; + + // MMS + mms::message_store& get_message_store() const { return m_wallet->get_message_store(); }; + mms::multisig_wallet_state get_multisig_wallet_state() const { return m_wallet->get_multisig_wallet_state(); }; + bool mms_active() const { return get_message_store().get_active(); }; + bool choose_mms_processing(const std::vector<mms::processing_data> &data_list, uint32_t &choice); + void list_mms_messages(const std::vector<mms::message> &messages); + void list_signers(const std::vector<mms::authorized_signer> &signers); + void add_signer_config_messages(); + void show_message(const mms::message &m); + void ask_send_all_ready_messages(); + void check_for_messages(); + bool user_confirms(const std::string &question); + bool get_message_from_arg(const std::string &arg, mms::message &m); + bool get_number_from_arg(const std::string &arg, uint32_t &number, const uint32_t lower_bound, const uint32_t upper_bound); + + void mms_init(const std::vector<std::string> &args); + void mms_info(const std::vector<std::string> &args); + void mms_signer(const std::vector<std::string> &args); + void mms_list(const std::vector<std::string> &args); + void mms_next(const std::vector<std::string> &args); + void mms_sync(const std::vector<std::string> &args); + void mms_transfer(const std::vector<std::string> &args); + void mms_delete(const std::vector<std::string> &args); + void mms_send(const std::vector<std::string> &args); + void mms_receive(const std::vector<std::string> &args); + void mms_export(const std::vector<std::string> &args); + void mms_note(const std::vector<std::string> &args); + void mms_show(const std::vector<std::string> &args); + void mms_set(const std::vector<std::string> &args); + void mms_help(const std::vector<std::string> &args); + void mms_send_signer_config(const std::vector<std::string> &args); + void mms_start_auto_config(const std::vector<std::string> &args); + void mms_stop_auto_config(const std::vector<std::string> &args); + void mms_auto_config(const std::vector<std::string> &args); }; } diff --git a/src/wallet/CMakeLists.txt b/src/wallet/CMakeLists.txt index be10b9f62..2991f75c5 100644 --- a/src/wallet/CMakeLists.txt +++ b/src/wallet/CMakeLists.txt @@ -34,7 +34,10 @@ set(wallet_sources wallet2.cpp wallet_args.cpp ringdb.cpp - node_rpc_proxy.cpp) + node_rpc_proxy.cpp + message_store.cpp + message_transporter.cpp +) set(wallet_private_headers wallet2.h @@ -44,7 +47,9 @@ set(wallet_private_headers wallet_rpc_server_commands_defs.h wallet_rpc_server_error_codes.h ringdb.h - node_rpc_proxy.h) + node_rpc_proxy.h + message_store.h + message_transporter.h) monero_private_headers(wallet ${wallet_private_headers}) @@ -57,6 +62,7 @@ target_link_libraries(wallet common cryptonote_core mnemonics + device_trezor ${LMDB_LIBRARY} ${Boost_CHRONO_LIBRARY} ${Boost_SERIALIZATION_LIBRARY} @@ -119,7 +125,8 @@ if (BUILD_GUI_DEPS) ringct ringct_basic checkpoints - version) + version + device_trezor) foreach(lib ${libs_to_merge}) list(APPEND objlibs $<TARGET_OBJECTS:obj_${lib}>) # matches naming convention in src/CMakeLists.txt @@ -132,6 +139,12 @@ if (BUILD_GUI_DEPS) endif() install(TARGETS wallet_merged ARCHIVE DESTINATION ${lib_folder}) + + install(FILES ${TREZOR_DEP_LIBS} + DESTINATION ${lib_folder}) + file(WRITE "trezor_link_flags.txt" ${TREZOR_DEP_LINKER}) + install(FILES "trezor_link_flags.txt" + DESTINATION ${lib_folder}) endif() add_subdirectory(api) diff --git a/src/wallet/api/wallet.cpp b/src/wallet/api/wallet.cpp index 1b4370c36..935a8d51c 100644 --- a/src/wallet/api/wallet.cpp +++ b/src/wallet/api/wallet.cpp @@ -381,6 +381,7 @@ WalletImpl::WalletImpl(NetworkType nettype, uint64_t kdf_rounds) , m_synchronized(false) , m_rebuildWalletCache(false) , m_is_connected(false) + , m_refreshShouldRescan(false) { m_wallet.reset(new tools::wallet2(static_cast<cryptonote::network_type>(nettype), kdf_rounds, true)); m_history.reset(new TransactionHistoryImpl(this)); @@ -506,7 +507,7 @@ bool WalletImpl::createWatchOnly(const std::string &path, const std::string &pas auto key_images = m_wallet->export_key_images(); uint64_t spent = 0; uint64_t unspent = 0; - view_wallet->import_key_images(key_images,spent,unspent,false); + view_wallet->import_key_images(key_images.second, key_images.first, spent, unspent, false); clearStatus(); } catch (const std::exception &e) { LOG_ERROR("Error creating view only wallet: " << e.what()); @@ -941,6 +942,12 @@ uint64_t WalletImpl::approximateBlockChainHeight() const { return m_wallet->get_approximate_blockchain_height(); } + +uint64_t WalletImpl::estimateBlockChainHeight() const +{ + return m_wallet->estimate_blockchain_height(); +} + uint64_t WalletImpl::daemonBlockChainHeight() const { if(m_wallet->light_wallet()) { @@ -1011,6 +1018,20 @@ void WalletImpl::refreshAsync() m_refreshCV.notify_one(); } +bool WalletImpl::rescanBlockchain() +{ + clearStatus(); + m_refreshShouldRescan = true; + doRefresh(); + return status() == Status_Ok; +} + +void WalletImpl::rescanBlockchainAsync() +{ + m_refreshShouldRescan = true; + refreshAsync(); +} + void WalletImpl::setAutoRefreshInterval(int millis) { if (millis > MAX_REFRESH_INTERVAL_MILLIS) { @@ -1036,8 +1057,8 @@ UnsignedTransaction *WalletImpl::loadUnsignedTx(const std::string &unsigned_file // Check tx data and construct confirmation message std::string extra_message; - if (!transaction->m_unsigned_tx_set.transfers.empty()) - extra_message = (boost::format("%u outputs to import. ") % (unsigned)transaction->m_unsigned_tx_set.transfers.size()).str(); + if (!transaction->m_unsigned_tx_set.transfers.second.empty()) + extra_message = (boost::format("%u outputs to import. ") % (unsigned)transaction->m_unsigned_tx_set.transfers.second.size()).str(); transaction->checkLoadedTx([&transaction](){return transaction->m_unsigned_tx_set.txes.size();}, [&transaction](size_t n)->const tools::wallet2::tx_construction_data&{return transaction->m_unsigned_tx_set.txes[n];}, extra_message); setStatus(transaction->status(), transaction->errorString()); @@ -1134,7 +1155,7 @@ std::string WalletImpl::getSubaddressLabel(uint32_t accountIndex, uint32_t addre } catch (const std::exception &e) { - LOG_ERROR("Error getting subaddress label: ") << e.what(); + LOG_ERROR("Error getting subaddress label: " << e.what()); setStatusError(string(tr("Failed to get subaddress label: ")) + e.what()); return ""; } @@ -1147,7 +1168,7 @@ void WalletImpl::setSubaddressLabel(uint32_t accountIndex, uint32_t addressIndex } catch (const std::exception &e) { - LOG_ERROR("Error setting subaddress label: ") << e.what(); + LOG_ERROR("Error setting subaddress label: " << e.what()); setStatusError(string(tr("Failed to set subaddress label: ")) + e.what()); } } @@ -1164,7 +1185,7 @@ string WalletImpl::getMultisigInfo() const { clearStatus(); return m_wallet->get_multisig_info(); } catch (const exception& e) { - LOG_ERROR("Error on generating multisig info: ") << e.what(); + LOG_ERROR("Error on generating multisig info: " << e.what()); setStatusError(string(tr("Failed to get multisig info: ")) + e.what()); } @@ -1181,7 +1202,7 @@ string WalletImpl::makeMultisig(const vector<string>& info, uint32_t threshold) return m_wallet->make_multisig(epee::wipeable_string(m_password), info, threshold); } catch (const exception& e) { - LOG_ERROR("Error on making multisig wallet: ") << e.what(); + LOG_ERROR("Error on making multisig wallet: " << e.what()); setStatusError(string(tr("Failed to make multisig: ")) + e.what()); } @@ -1195,7 +1216,7 @@ std::string WalletImpl::exchangeMultisigKeys(const std::vector<std::string> &inf return m_wallet->exchange_multisig_keys(epee::wipeable_string(m_password), info); } catch (const exception& e) { - LOG_ERROR("Error on exchanging multisig keys: ") << e.what(); + LOG_ERROR("Error on exchanging multisig keys: " << e.what()); setStatusError(string(tr("Failed to make multisig: ")) + e.what()); } @@ -1213,7 +1234,7 @@ bool WalletImpl::finalizeMultisig(const vector<string>& extraMultisigInfo) { setStatusError(tr("Failed to finalize multisig wallet creation")); } catch (const exception& e) { - LOG_ERROR("Error on finalizing multisig wallet creation: ") << e.what(); + LOG_ERROR("Error on finalizing multisig wallet creation: " << e.what()); setStatusError(string(tr("Failed to finalize multisig wallet creation: ")) + e.what()); } @@ -1229,7 +1250,7 @@ bool WalletImpl::exportMultisigImages(string& images) { images = epee::string_tools::buff_to_hex_nodelimer(blob); return true; } catch (const exception& e) { - LOG_ERROR("Error on exporting multisig images: ") << e.what(); + LOG_ERROR("Error on exporting multisig images: " << e.what()); setStatusError(string(tr("Failed to export multisig images: ")) + e.what()); } @@ -1257,7 +1278,7 @@ size_t WalletImpl::importMultisigImages(const vector<string>& images) { return m_wallet->import_multisig(blobs); } catch (const exception& e) { - LOG_ERROR("Error on importing multisig images: ") << e.what(); + LOG_ERROR("Error on importing multisig images: " << e.what()); setStatusError(string(tr("Failed to import multisig images: ")) + e.what()); } @@ -1271,7 +1292,7 @@ bool WalletImpl::hasMultisigPartialKeyImages() const { return m_wallet->has_multisig_partial_key_images(); } catch (const exception& e) { - LOG_ERROR("Error on checking for partial multisig key images: ") << e.what(); + LOG_ERROR("Error on checking for partial multisig key images: " << e.what()); setStatusError(string(tr("Failed to check for partial multisig key images: ")) + e.what()); } @@ -1299,7 +1320,7 @@ PendingTransaction* WalletImpl::restoreMultisigTransaction(const string& signDat return ptx; } catch (exception& e) { - LOG_ERROR("Error on restoring multisig transaction: ") << e.what(); + LOG_ERROR("Error on restoring multisig transaction: " << e.what()); setStatusError(string(tr("Failed to restore multisig transaction: ")) + e.what()); } @@ -1386,9 +1407,11 @@ PendingTransaction *WalletImpl::createTransaction(const string &dst_addr, const if (amount) { vector<cryptonote::tx_destination_entry> dsts; cryptonote::tx_destination_entry de; + de.original = dst_addr; de.addr = info.address; de.amount = *amount; de.is_subaddress = info.is_subaddress; + de.is_integrated = info.has_payment_id; dsts.push_back(de); transaction->m_pending_tx = m_wallet->create_transactions_2(dsts, fake_outs_count, 0 /* unlock_time */, adjusted_priority, @@ -1984,6 +2007,7 @@ void WalletImpl::refreshThreadFunc() LOG_PRINT_L3(__FUNCTION__ << ": refresh lock acquired..."); LOG_PRINT_L3(__FUNCTION__ << ": m_refreshEnabled: " << m_refreshEnabled); LOG_PRINT_L3(__FUNCTION__ << ": m_status: " << status()); + LOG_PRINT_L3(__FUNCTION__ << ": m_refreshShouldRescan: " << m_refreshShouldRescan); if (m_refreshEnabled) { LOG_PRINT_L3(__FUNCTION__ << ": refreshing..."); doRefresh(); @@ -1994,12 +2018,16 @@ void WalletImpl::refreshThreadFunc() void WalletImpl::doRefresh() { + bool rescan = m_refreshShouldRescan.exchange(false); // synchronizing async and sync refresh calls boost::lock_guard<boost::mutex> guarg(m_refreshMutex2); - try { + do try { + LOG_PRINT_L3(__FUNCTION__ << ": doRefresh, rescan = "<<rescan); // Syncing daemon and refreshing wallet simultaneously is very resource intensive. // Disable refresh if wallet is disconnected or daemon isn't synced. if (m_wallet->light_wallet() || daemonSynced()) { + if(rescan) + m_wallet->rescan_blockchain(false); m_wallet->refresh(trustedDaemon()); if (!m_synchronized) { m_synchronized = true; @@ -2016,7 +2044,9 @@ void WalletImpl::doRefresh() } } catch (const std::exception &e) { setStatusError(e.what()); - } + break; + }while(!rescan && (rescan=m_refreshShouldRescan.exchange(false))); // repeat if not rescanned and rescan was requested + if (m_wallet2Callback->getListener()) { m_wallet2Callback->getListener()->refreshed(); } diff --git a/src/wallet/api/wallet.h b/src/wallet/api/wallet.h index 6d343888b..55240d64f 100644 --- a/src/wallet/api/wallet.h +++ b/src/wallet/api/wallet.h @@ -109,11 +109,14 @@ public: uint64_t unlockedBalance(uint32_t accountIndex = 0) const override; uint64_t blockChainHeight() const override; uint64_t approximateBlockChainHeight() const override; + uint64_t estimateBlockChainHeight() const override; uint64_t daemonBlockChainHeight() const override; uint64_t daemonBlockChainTargetHeight() const override; bool synchronized() const override; bool refresh() override; void refreshAsync() override; + bool rescanBlockchain() override; + void rescanBlockchainAsync() override; void setAutoRefreshInterval(int millis) override; int autoRefreshInterval() const override; void setRefreshFromBlockHeight(uint64_t refresh_from_block_height) override; @@ -232,6 +235,7 @@ private: std::atomic<bool> m_refreshEnabled; std::atomic<bool> m_refreshThreadDone; std::atomic<int> m_refreshIntervalMillis; + std::atomic<bool> m_refreshShouldRescan; // synchronizing refresh loop; boost::mutex m_refreshMutex; diff --git a/src/wallet/api/wallet2_api.h b/src/wallet/api/wallet2_api.h index ec1a84877..5c301974f 100644 --- a/src/wallet/api/wallet2_api.h +++ b/src/wallet/api/wallet2_api.h @@ -575,6 +575,12 @@ struct Wallet virtual uint64_t approximateBlockChainHeight() const = 0; /** + * @brief estimateBlockChainHeight - returns estimate blockchain height. More accurate than approximateBlockChainHeight, + * uses daemon height and falls back to calculation from date/time + * @return + **/ + virtual uint64_t estimateBlockChainHeight() const = 0; + /** * @brief daemonBlockChainHeight - returns daemon blockchain height * @return 0 - in case error communicating with the daemon. * status() will return Status_Error and errorString() will return verbose error description @@ -644,6 +650,17 @@ struct Wallet virtual void refreshAsync() = 0; /** + * @brief rescanBlockchain - rescans the wallet, updating transactions from daemon + * @return - true if refreshed successfully; + */ + virtual bool rescanBlockchain() = 0; + + /** + * @brief rescanBlockchainAsync - rescans wallet asynchronously, starting from genesys + */ + virtual void rescanBlockchainAsync() = 0; + + /** * @brief setAutoRefreshInterval - setup interval for automatic refresh. * @param seconds - interval in millis. if zero or less than zero - automatic refresh disabled; */ diff --git a/src/wallet/api/wallet_manager.cpp b/src/wallet/api/wallet_manager.cpp index 5b262f1b7..89fe01c0d 100644 --- a/src/wallet/api/wallet_manager.cpp +++ b/src/wallet/api/wallet_manager.cpp @@ -127,6 +127,8 @@ Wallet *WalletManagerImpl::createWalletFromDevice(const std::string &path, WalletImpl * wallet = new WalletImpl(nettype, kdf_rounds); if(restoreHeight > 0){ wallet->setRefreshFromBlockHeight(restoreHeight); + } else { + wallet->setRefreshFromBlockHeight(wallet->estimateBlockChainHeight()); } auto lookahead = tools::parse_subaddress_lookahead(subaddressLookahead); if (lookahead) diff --git a/src/wallet/message_store.cpp b/src/wallet/message_store.cpp new file mode 100644 index 000000000..7381005c1 --- /dev/null +++ b/src/wallet/message_store.cpp @@ -0,0 +1,1445 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "message_store.h" +#include <boost/archive/portable_binary_oarchive.hpp> +#include <boost/archive/portable_binary_iarchive.hpp> +#include <boost/format.hpp> +#include <boost/algorithm/string.hpp> +#include <fstream> +#include <sstream> +#include "file_io_utils.h" +#include "storages/http_abstract_invoke.h" +#include "wallet_errors.h" +#include "serialization/binary_utils.h" +#include "common/base58.h" +#include "common/util.h" +#include "string_tools.h" + + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "wallet.mms" + +namespace mms +{ + +message_store::message_store() +{ + m_active = false; + m_auto_send = false; + m_next_message_id = 1; + m_num_authorized_signers = 0; + m_num_required_signers = 0; + m_nettype = cryptonote::network_type::UNDEFINED; + m_run = true; +} + +namespace +{ + // MMS options handling mirrors what "wallet2" is doing for its options, on-demand init and all + // It's not very clean to initialize Bitmessage-specific options here, but going one level further + // down still into "message_transporter" for that is a little bit too much + struct options + { + const command_line::arg_descriptor<std::string> bitmessage_address = {"bitmessage-address", mms::message_store::tr("Use PyBitmessage instance at URL <arg>"), "http://localhost:8442/"}; + const command_line::arg_descriptor<std::string> bitmessage_login = {"bitmessage-login", mms::message_store::tr("Specify <arg> as username:password for PyBitmessage API"), "username:password"}; + }; +} + +void message_store::init_options(boost::program_options::options_description& desc_params) +{ + const options opts{}; + command_line::add_arg(desc_params, opts.bitmessage_address); + command_line::add_arg(desc_params, opts.bitmessage_login); +} + +void message_store::init(const multisig_wallet_state &state, const std::string &own_label, + const std::string &own_transport_address, uint32_t num_authorized_signers, uint32_t num_required_signers) +{ + m_num_authorized_signers = num_authorized_signers; + m_num_required_signers = num_required_signers; + m_signers.clear(); + m_messages.clear(); + m_next_message_id = 1; + + // The vector "m_signers" gets here once the required number of elements, one for each authorized signer, + // and is never changed again. The rest of the code relies on "size(m_signers) == m_num_authorized_signers" + // without further checks. + authorized_signer signer; + for (uint32_t i = 0; i < m_num_authorized_signers; ++i) + { + signer.me = signer.index == 0; // Strict convention: The very first signer is fixed as / must be "me" + m_signers.push_back(signer); + signer.index++; + } + + set_signer(state, 0, own_label, own_transport_address, state.address); + + m_nettype = state.nettype; + set_active(true); + m_filename = state.mms_file; + save(state); +} + +void message_store::set_options(const boost::program_options::variables_map& vm) +{ + const options opts{}; + std::string bitmessage_address = command_line::get_arg(vm, opts.bitmessage_address); + epee::wipeable_string bitmessage_login = command_line::get_arg(vm, opts.bitmessage_login); + set_options(bitmessage_address, bitmessage_login); +} + +void message_store::set_options(const std::string &bitmessage_address, const epee::wipeable_string &bitmessage_login) +{ + m_transporter.set_options(bitmessage_address, bitmessage_login); +} + +void message_store::set_signer(const multisig_wallet_state &state, + uint32_t index, + const boost::optional<std::string> &label, + const boost::optional<std::string> &transport_address, + const boost::optional<cryptonote::account_public_address> monero_address) +{ + THROW_WALLET_EXCEPTION_IF(index >= m_num_authorized_signers, tools::error::wallet_internal_error, "Invalid signer index " + std::to_string(index)); + authorized_signer &m = m_signers[index]; + if (label) + { + m.label = label.get(); + } + if (transport_address) + { + m.transport_address = transport_address.get(); + } + if (monero_address) + { + m.monero_address_known = true; + m.monero_address = monero_address.get(); + } + // Save to minimize the chance to loose that info (at least while in beta) + save(state); +} + +const authorized_signer &message_store::get_signer(uint32_t index) const +{ + THROW_WALLET_EXCEPTION_IF(index >= m_num_authorized_signers, tools::error::wallet_internal_error, "Invalid signer index " + std::to_string(index)); + return m_signers[index]; +} + +bool message_store::signer_config_complete() const +{ + for (uint32_t i = 0; i < m_num_authorized_signers; ++i) + { + const authorized_signer &m = m_signers[i]; + if (m.label.empty() || m.transport_address.empty() || !m.monero_address_known) + { + return false; + } + } + return true; +} + +// Check if all signers have a label set (as it's a requirement for starting auto-config +// by the "manager") +bool message_store::signer_labels_complete() const +{ + for (uint32_t i = 0; i < m_num_authorized_signers; ++i) + { + const authorized_signer &m = m_signers[i]; + if (m.label.empty()) + { + return false; + } + } + return true; +} + +void message_store::get_signer_config(std::string &signer_config) +{ + std::stringstream oss; + boost::archive::portable_binary_oarchive ar(oss); + ar << m_signers; + signer_config = oss.str(); +} + +void message_store::unpack_signer_config(const multisig_wallet_state &state, const std::string &signer_config, + std::vector<authorized_signer> &signers) +{ + try + { + std::stringstream iss; + iss << signer_config; + boost::archive::portable_binary_iarchive ar(iss); + ar >> signers; + } + catch (...) + { + THROW_WALLET_EXCEPTION_IF(true, tools::error::wallet_internal_error, "Invalid structure of signer config"); + } + uint32_t num_signers = (uint32_t)signers.size(); + THROW_WALLET_EXCEPTION_IF(num_signers != m_num_authorized_signers, tools::error::wallet_internal_error, "Wrong number of signers in config: " + std::to_string(num_signers)); +} + +void message_store::process_signer_config(const multisig_wallet_state &state, const std::string &signer_config) +{ + // The signers in "signer_config" and the resident wallet signers are matched not by label, but + // by Monero address, and ALL labels will be set from "signer_config", even the "me" label. + // In the auto-config process as implemented now the auto-config manager is responsible for defining + // the labels, and right at the end of the process ALL wallets use the SAME labels. The idea behind this + // is preventing problems like duplicate labels and confusion (Bob choosing a label "IamAliceHonest"). + // (Of course signers are free to re-define any labels they don't like AFTER auto-config.) + // + // Usually this method will be called with only the "me" signer defined in the wallet, and may + // produce unexpected behaviour if that wallet contains additional signers that have nothing to do with + // those arriving in "signer_config". + std::vector<authorized_signer> signers; + unpack_signer_config(state, signer_config, signers); + + uint32_t new_index = 1; + for (uint32_t i = 0; i < m_num_authorized_signers; ++i) + { + const authorized_signer &m = signers[i]; + uint32_t index; + uint32_t take_index; + bool found = get_signer_index_by_monero_address(m.monero_address, index); + if (found) + { + // Redefine existing (probably "me", under usual circumstances) + take_index = index; + } + else + { + // Add new; neglect that we may erroneously overwrite already defined signers + // (but protect "me") + take_index = new_index; + if ((new_index + 1) < m_num_authorized_signers) + { + new_index++; + } + } + authorized_signer &modify = m_signers[take_index]; + modify.label = m.label; // ALWAYS set label, see comments above + if (!modify.me) + { + modify.transport_address = m.transport_address; + modify.monero_address_known = m.monero_address_known; + if (m.monero_address_known) + { + modify.monero_address = m.monero_address; + } + } + } + save(state); +} + +void message_store::start_auto_config(const multisig_wallet_state &state) +{ + for (uint32_t i = 0; i < m_num_authorized_signers; ++i) + { + authorized_signer &m = m_signers[i]; + if (!m.me) + { + setup_signer_for_auto_config(i, create_auto_config_token(), true); + } + m.auto_config_running = true; + } + save(state); +} + +// Check auto-config token string and convert to standardized form; +// Try to make it as foolproof as possible, with built-in tolerance to make up for +// errors in transmission that still leave the token recognizable. +bool message_store::check_auto_config_token(const std::string &raw_token, + std::string &adjusted_token) const +{ + std::string prefix(AUTO_CONFIG_TOKEN_PREFIX); + uint32_t num_hex_digits = (AUTO_CONFIG_TOKEN_BYTES + 1) * 2; + uint32_t full_length = num_hex_digits + prefix.length(); + uint32_t raw_length = raw_token.length(); + std::string hex_digits; + + if (raw_length == full_length) + { + // Prefix must be there; accept it in any casing + std::string raw_prefix(raw_token.substr(0, 3)); + boost::algorithm::to_lower(raw_prefix); + if (raw_prefix != prefix) + { + return false; + } + hex_digits = raw_token.substr(3); + } + else if (raw_length == num_hex_digits) + { + // Accept the token without the prefix if it's otherwise ok + hex_digits = raw_token; + } + else + { + return false; + } + + // Convert to strict lowercase and correct any common misspellings + boost::algorithm::to_lower(hex_digits); + std::replace(hex_digits.begin(), hex_digits.end(), 'o', '0'); + std::replace(hex_digits.begin(), hex_digits.end(), 'i', '1'); + std::replace(hex_digits.begin(), hex_digits.end(), 'l', '1'); + + // Now it must be correct hex with correct checksum, no further tolerance possible + std::string token_bytes; + if (!epee::string_tools::parse_hexstr_to_binbuff(hex_digits, token_bytes)) + { + return false; + } + const crypto::hash &hash = crypto::cn_fast_hash(token_bytes.data(), token_bytes.size() - 1); + if (token_bytes[AUTO_CONFIG_TOKEN_BYTES] != hash.data[0]) + { + return false; + } + adjusted_token = prefix + hex_digits; + return true; +} + +// Create a new auto-config token with prefix, random 8-hex digits plus 2 checksum digits +std::string message_store::create_auto_config_token() +{ + unsigned char random[AUTO_CONFIG_TOKEN_BYTES]; + crypto::rand(AUTO_CONFIG_TOKEN_BYTES, random); + std::string token_bytes; + token_bytes.append((char *)random, AUTO_CONFIG_TOKEN_BYTES); + + // Add a checksum because technically ANY four bytes are a valid token, and without a checksum we would send + // auto-config messages "to nowhere" after the slightest typo without knowing it + const crypto::hash &hash = crypto::cn_fast_hash(token_bytes.data(), token_bytes.size()); + token_bytes += hash.data[0]; + std::string prefix(AUTO_CONFIG_TOKEN_PREFIX); + return prefix + epee::string_tools::buff_to_hex_nodelimer(token_bytes); +} + +// Add a message for sending "me" address data to the auto-config transport address +// that can be derived from the token and activate auto-config +size_t message_store::add_auto_config_data_message(const multisig_wallet_state &state, + const std::string &auto_config_token) +{ + authorized_signer &me = m_signers[0]; + me.auto_config_token = auto_config_token; + setup_signer_for_auto_config(0, auto_config_token, false); + me.auto_config_running = true; + + auto_config_data data; + data.label = me.label; + data.transport_address = me.transport_address; + data.monero_address = me.monero_address; + + std::stringstream oss; + boost::archive::portable_binary_oarchive ar(oss); + ar << data; + + return add_message(state, 0, message_type::auto_config_data, message_direction::out, oss.str()); +} + +// Process a single message with auto-config data, destined for "message.signer_index" +void message_store::process_auto_config_data_message(uint32_t id) +{ + // "auto_config_data" contains the label that the auto-config data sender uses for "me", but that's + // more for completeness' sake, and right now it's not used. In general, the auto-config manager + // decides/defines the labels, and right after completing auto-config ALL wallets use the SAME labels. + + const message &m = get_message_ref_by_id(id); + + auto_config_data data; + try + { + std::stringstream iss; + iss << m.content; + boost::archive::portable_binary_iarchive ar(iss); + ar >> data; + } + catch (...) + { + THROW_WALLET_EXCEPTION_IF(true, tools::error::wallet_internal_error, "Invalid structure of auto config data"); + } + + authorized_signer &signer = m_signers[m.signer_index]; + // "signer.label" does NOT change, see comment above + signer.transport_address = data.transport_address; + signer.monero_address_known = true; + signer.monero_address = data.monero_address; + signer.auto_config_running = false; +} + +void message_store::stop_auto_config() +{ + for (uint32_t i = 0; i < m_num_authorized_signers; ++i) + { + authorized_signer &m = m_signers[i]; + if (!m.me && !m.auto_config_transport_address.empty()) + { + // Try to delete those "unused API" addresses in PyBitmessage, especially since + // it seems it's not possible to delete them interactively, only to "disable" them + m_transporter.delete_transport_address(m.auto_config_transport_address); + } + m.auto_config_token.clear(); + m.auto_config_public_key = crypto::null_pkey; + m.auto_config_secret_key = crypto::null_skey; + m.auto_config_transport_address.clear(); + m.auto_config_running = false; + } +} + +void message_store::setup_signer_for_auto_config(uint32_t index, const std::string token, bool receiving) +{ + // It may be a little strange to hash the textual hex digits of the auto config token into + // 32 bytes and turn that into a Monero public/secret key pair, instead of doing something + // much less complicated like directly using the underlying random 40 bits as key for a + // symmetric cipher, but everything is there already for encrypting and decrypting messages + // with such key pairs, and furthermore it would be trivial to use tokens with a different + // number of bytes. + // + // In the wallet of the auto-config manager each signer except "me" gets set its own + // auto-config parameters. In the wallet of somebody using the token to send auto-config + // data the auto-config parameters are stored in the "me" signer and taken from there + // to send that data. + THROW_WALLET_EXCEPTION_IF(index >= m_num_authorized_signers, tools::error::wallet_internal_error, "Invalid signer index " + std::to_string(index)); + authorized_signer &m = m_signers[index]; + m.auto_config_token = token; + crypto::hash_to_scalar(token.data(), token.size(), m.auto_config_secret_key); + crypto::secret_key_to_public_key(m.auto_config_secret_key, m.auto_config_public_key); + if (receiving) + { + m.auto_config_transport_address = m_transporter.derive_and_receive_transport_address(m.auto_config_token); + } + else + { + m.auto_config_transport_address = m_transporter.derive_transport_address(m.auto_config_token); + } +} + +bool message_store::get_signer_index_by_monero_address(const cryptonote::account_public_address &monero_address, uint32_t &index) const +{ + for (uint32_t i = 0; i < m_num_authorized_signers; ++i) + { + const authorized_signer &m = m_signers[i]; + if (m.monero_address == monero_address) + { + index = m.index; + return true; + } + } + MWARNING("No authorized signer with Monero address " << account_address_to_string(monero_address)); + return false; +} + +bool message_store::get_signer_index_by_label(const std::string label, uint32_t &index) const +{ + for (uint32_t i = 0; i < m_num_authorized_signers; ++i) + { + const authorized_signer &m = m_signers[i]; + if (m.label == label) + { + index = m.index; + return true; + } + } + MWARNING("No authorized signer with label " << label); + return false; +} + +void message_store::process_wallet_created_data(const multisig_wallet_state &state, message_type type, const std::string &content) +{ + switch(type) + { + case message_type::key_set: + // Result of a "prepare_multisig" command in the wallet + // Send the key set to all other signers + case message_type::additional_key_set: + // Result of a "make_multisig" command or a "exchange_multisig_keys" in the wallet in case of M/N multisig + // Send the additional key set to all other signers + case message_type::multisig_sync_data: + // Result of a "export_multisig_info" command in the wallet + // Send the sync data to all other signers + for (uint32_t i = 1; i < m_num_authorized_signers; ++i) + { + add_message(state, i, type, message_direction::out, content); + } + break; + + case message_type::partially_signed_tx: + // Result of a "transfer" command in the wallet, or a "sign_multisig" command + // that did not yet result in the minimum number of signatures required + // Create a message "from me to me" as a container for the tx data + if (m_num_required_signers == 1) + { + // Probably rare, but possible: The 1 signature is already enough, correct the type + // Easier to correct here than asking all callers to detect this rare special case + type = message_type::fully_signed_tx; + } + add_message(state, 0, type, message_direction::in, content); + break; + + case message_type::fully_signed_tx: + add_message(state, 0, type, message_direction::in, content); + break; + + default: + THROW_WALLET_EXCEPTION(tools::error::wallet_internal_error, "Illegal message type " + std::to_string((uint32_t)type)); + break; + } +} + +size_t message_store::add_message(const multisig_wallet_state &state, + uint32_t signer_index, message_type type, message_direction direction, + const std::string &content) +{ + message m; + m.id = m_next_message_id++; + m.type = type; + m.direction = direction; + m.content = content; + m.created = (uint64_t)time(NULL); + m.modified = m.created; + m.sent = 0; + m.signer_index = signer_index; + if (direction == message_direction::out) + { + m.state = message_state::ready_to_send; + } + else + { + m.state = message_state::waiting; + }; + m.wallet_height = (uint32_t)state.num_transfer_details; + if (m.type == message_type::additional_key_set) + { + m.round = state.multisig_rounds_passed; + } + else + { + m.round = 0; + } + m.signature_count = 0; // Future expansion for signature counting when signing txs + m.hash = crypto::null_hash; + m_messages.push_back(m); + + // Save for every new message right away (at least while in beta) + save(state); + + MINFO(boost::format("Added %s message %s for signer %s of type %s") + % message_direction_to_string(direction) % m.id % signer_index % message_type_to_string(type)); + return m_messages.size() - 1; +} + +// Get the index of the message with id "id", return false if not found +bool message_store::get_message_index_by_id(uint32_t id, size_t &index) const +{ + for (size_t i = 0; i < m_messages.size(); ++i) + { + if (m_messages[i].id == id) + { + index = i; + return true; + } + } + MWARNING("No message found with an id of " << id); + return false; +} + +// Get the index of the message with id "id" that must exist +size_t message_store::get_message_index_by_id(uint32_t id) const +{ + size_t index; + bool found = get_message_index_by_id(id, index); + THROW_WALLET_EXCEPTION_IF(!found, tools::error::wallet_internal_error, "Invalid message id " + std::to_string(id)); + return index; +} + +// Get the modifiable message with id "id" that must exist; private/internal use! +message& message_store::get_message_ref_by_id(uint32_t id) +{ + return m_messages[get_message_index_by_id(id)]; +} + +// Get the message with id "id", return false if not found +// This version of the method allows to check whether id is valid without triggering an error +bool message_store::get_message_by_id(uint32_t id, message &m) const +{ + size_t index; + bool found = get_message_index_by_id(id, index); + if (found) + { + m = m_messages[index]; + } + return found; +} + +// Get the message with id "id" that must exist +message message_store::get_message_by_id(uint32_t id) const +{ + message m; + bool found = get_message_by_id(id, m); + THROW_WALLET_EXCEPTION_IF(!found, tools::error::wallet_internal_error, "Invalid message id " + std::to_string(id)); + return m; +} + +bool message_store::any_message_of_type(message_type type, message_direction direction) const +{ + for (size_t i = 0; i < m_messages.size(); ++i) + { + if ((m_messages[i].type == type) && (m_messages[i].direction == direction)) + { + return true; + } + } + return false; +} + +bool message_store::any_message_with_hash(const crypto::hash &hash) const +{ + for (size_t i = 0; i < m_messages.size(); ++i) + { + if (m_messages[i].hash == hash) + { + return true; + } + } + return false; +} + +// Count the ids in the vector that are set i.e. not 0, while ignoring index 0 +// Mostly used to check whether we have a message for each authorized signer except me, +// with the signer index used as index into 'ids'; the element at index 0, for me, +// is ignored, to make constant subtractions of 1 for indices when filling the +// vector unnecessary +size_t message_store::get_other_signers_id_count(const std::vector<uint32_t> &ids) const +{ + size_t count = 0; + for (size_t i = 1 /* and not 0 */; i < ids.size(); ++i) + { + if (ids[i] != 0) + { + count++; + } + } + return count; +} + +// Is in every element of vector 'ids' (except at index 0) a message id i.e. not 0? +bool message_store::message_ids_complete(const std::vector<uint32_t> &ids) const +{ + return get_other_signers_id_count(ids) == (ids.size() - 1); +} + +void message_store::delete_message(uint32_t id) +{ + delete_transport_message(id); + size_t index = get_message_index_by_id(id); + m_messages.erase(m_messages.begin() + index); +} + +void message_store::delete_all_messages() +{ + for (size_t i = 0; i < m_messages.size(); ++i) + { + delete_transport_message(m_messages[i].id); + } + m_messages.clear(); +} + +// Make a message text, which is "attacker controlled data", reasonably safe to display +// This is mostly geared towards the safe display of notes sent by "mms note" with a "mms show" command +void message_store::get_sanitized_message_text(const message &m, std::string &sanitized_text) const +{ + sanitized_text.clear(); + + // Restrict the size to fend of DOS-style attacks with heaps of data + size_t length = std::min(m.content.length(), (size_t)1000); + + for (size_t i = 0; i < length; ++i) + { + char c = m.content[i]; + if ((int)c < 32) + { + // Strip out any controls, especially ESC for getting rid of potentially dangerous + // ANSI escape sequences that a console window might interpret + c = ' '; + } + else if ((c == '<') || (c == '>')) + { + // Make XML or HTML impossible that e.g. might contain scripts that Qt might execute + // when displayed in the GUI wallet + c = ' '; + } + sanitized_text += c; + } +} + +void message_store::write_to_file(const multisig_wallet_state &state, const std::string &filename) +{ + std::stringstream oss; + boost::archive::portable_binary_oarchive ar(oss); + ar << *this; + std::string buf = oss.str(); + + crypto::chacha_key key; + crypto::generate_chacha_key(&state.view_secret_key, sizeof(crypto::secret_key), key, 1); + + file_data write_file_data = boost::value_initialized<file_data>(); + write_file_data.magic_string = "MMS"; + write_file_data.file_version = 0; + write_file_data.iv = crypto::rand<crypto::chacha_iv>(); + std::string encrypted_data; + encrypted_data.resize(buf.size()); + crypto::chacha20(buf.data(), buf.size(), key, write_file_data.iv, &encrypted_data[0]); + write_file_data.encrypted_data = encrypted_data; + + std::stringstream file_oss; + boost::archive::portable_binary_oarchive file_ar(file_oss); + file_ar << write_file_data; + + bool success = epee::file_io_utils::save_string_to_file(filename, file_oss.str()); + THROW_WALLET_EXCEPTION_IF(!success, tools::error::file_save_error, filename); +} + +void message_store::read_from_file(const multisig_wallet_state &state, const std::string &filename) +{ + boost::system::error_code ignored_ec; + bool file_exists = boost::filesystem::exists(filename, ignored_ec); + if (!file_exists) + { + // Simply do nothing if the file is not there; allows e.g. easy recovery + // from problems with the MMS by deleting the file + MERROR("No message store file found: " << filename); + return; + } + + std::string buf; + bool success = epee::file_io_utils::load_file_to_string(filename, buf); + THROW_WALLET_EXCEPTION_IF(!success, tools::error::file_read_error, filename); + + file_data read_file_data; + try + { + std::stringstream iss; + iss << buf; + boost::archive::portable_binary_iarchive ar(iss); + ar >> read_file_data; + } + catch (const std::exception &e) + { + MERROR("MMS file " << filename << " has bad structure <iv,encrypted_data>: " << e.what()); + THROW_WALLET_EXCEPTION_IF(true, tools::error::file_read_error, filename); + } + + crypto::chacha_key key; + crypto::generate_chacha_key(&state.view_secret_key, sizeof(crypto::secret_key), key, 1); + std::string decrypted_data; + decrypted_data.resize(read_file_data.encrypted_data.size()); + crypto::chacha20(read_file_data.encrypted_data.data(), read_file_data.encrypted_data.size(), key, read_file_data.iv, &decrypted_data[0]); + + try + { + std::stringstream iss; + iss << decrypted_data; + boost::archive::portable_binary_iarchive ar(iss); + ar >> *this; + } + catch (const std::exception &e) + { + MERROR("MMS file " << filename << " has bad structure: " << e.what()); + THROW_WALLET_EXCEPTION_IF(true, tools::error::file_read_error, filename); + } + + m_filename = filename; +} + +// Save to the same file this message store was loaded from +// Called after changes deemed "important", to make it less probable to lose messages in case of +// a crash; a better and long-term solution would of course be to use LMDB ... +void message_store::save(const multisig_wallet_state &state) +{ + if (!m_filename.empty()) + { + write_to_file(state, m_filename); + } +} + +bool message_store::get_processable_messages(const multisig_wallet_state &state, + bool force_sync, std::vector<processing_data> &data_list, std::string &wait_reason) +{ + uint32_t wallet_height = (uint32_t)state.num_transfer_details; + data_list.clear(); + wait_reason.clear(); + // In all scans over all messages looking for complete sets (1 message for each signer), + // if there are duplicates, the OLDEST of them is taken. This may not play a role with + // any of the current message types, but may with future ones, and it's probably a good + // idea to have a clear and somewhat defensive strategy. + + std::vector<uint32_t> auto_config_messages(m_num_authorized_signers, 0); + bool any_auto_config = false; + + for (size_t i = 0; i < m_messages.size(); ++i) + { + message &m = m_messages[i]; + if ((m.type == message_type::auto_config_data) && (m.state == message_state::waiting)) + { + if (auto_config_messages[m.signer_index] == 0) + { + auto_config_messages[m.signer_index] = m.id; + any_auto_config = true; + } + // else duplicate auto config data, ignore + } + } + + if (any_auto_config) + { + bool auto_config_complete = message_ids_complete(auto_config_messages); + if (auto_config_complete) + { + processing_data data; + data.processing = message_processing::process_auto_config_data; + data.message_ids = auto_config_messages; + data.message_ids.erase(data.message_ids.begin()); + data_list.push_back(data); + return true; + } + else + { + wait_reason = tr("Auto-config cannot proceed because auto config data from other signers is not complete"); + return false; + // With ANY auto config data present but not complete refuse to check for any + // other processing. Manually delete those messages to abort such an auto config + // phase if needed. + } + } + + // Any signer config that arrived will be processed right away, regardless of other things that may wait + for (size_t i = 0; i < m_messages.size(); ++i) + { + message &m = m_messages[i]; + if ((m.type == message_type::signer_config) && (m.state == message_state::waiting)) + { + processing_data data; + data.processing = message_processing::process_signer_config; + data.message_ids.push_back(m.id); + data_list.push_back(data); + return true; + } + } + + // ALL of the following processings depend on the signer info being complete + if (!signer_config_complete()) + { + wait_reason = tr("The signer config is not complete."); + return false; + } + + if (!state.multisig) + { + + if (!any_message_of_type(message_type::key_set, message_direction::out)) + { + // With the own key set not yet ready we must do "prepare_multisig" first; + // Key sets from other signers may be here already, but if we process them now + // the wallet will go multisig too early: we can't produce our own key set any more! + processing_data data; + data.processing = message_processing::prepare_multisig; + data_list.push_back(data); + return true; + } + + // Ids of key set messages per signer index, to check completeness + // Naturally, does not care about the order of the messages and is trivial to secure against + // key sets that were received more than once + // With full M/N multisig now possible consider only key sets of the right round, i.e. + // with not yet multisig the only possible round 0 + std::vector<uint32_t> key_set_messages(m_num_authorized_signers, 0); + + for (size_t i = 0; i < m_messages.size(); ++i) + { + message &m = m_messages[i]; + if ((m.type == message_type::key_set) && (m.state == message_state::waiting) + && (m.round == 0)) + { + if (key_set_messages[m.signer_index] == 0) + { + key_set_messages[m.signer_index] = m.id; + } + // else duplicate key set, ignore + } + } + + bool key_sets_complete = message_ids_complete(key_set_messages); + if (key_sets_complete) + { + // Nothing else can be ready to process earlier than this, ignore everything else and give back + processing_data data; + data.processing = message_processing::make_multisig; + data.message_ids = key_set_messages; + data.message_ids.erase(data.message_ids.begin()); + data_list.push_back(data); + return true; + } + else + { + wait_reason = tr("Wallet can't go multisig because key sets from other signers are missing or not complete."); + return false; + } + } + + if (state.multisig && !state.multisig_is_ready) + { + // In the case of M/N multisig the call 'wallet2::multisig' returns already true + // after "make_multisig" but with calls to "exchange_multisig_keys" still needed, and + // sets the parameter 'ready' to false to document this particular "in-between" state. + // So what may be possible here, with all necessary messages present, is a call to + // "exchange_multisig_keys". + // Consider only messages belonging to the next round to do, which has the number + // "state.multisig_rounds_passed". + std::vector<uint32_t> additional_key_set_messages(m_num_authorized_signers, 0); + + for (size_t i = 0; i < m_messages.size(); ++i) + { + message &m = m_messages[i]; + if ((m.type == message_type::additional_key_set) && (m.state == message_state::waiting) + && (m.round == state.multisig_rounds_passed)) + { + if (additional_key_set_messages[m.signer_index] == 0) + { + additional_key_set_messages[m.signer_index] = m.id; + } + // else duplicate key set, ignore + } + } + + bool key_sets_complete = message_ids_complete(additional_key_set_messages); + if (key_sets_complete) + { + processing_data data; + data.processing = message_processing::exchange_multisig_keys; + data.message_ids = additional_key_set_messages; + data.message_ids.erase(data.message_ids.begin()); + data_list.push_back(data); + return true; + } + else + { + wait_reason = tr("Wallet can't start another key exchange round because key sets from other signers are missing or not complete."); + return false; + } + } + + // Properly exchanging multisig sync data is easiest and most transparent + // for the user if a wallet sends its own data first and processes any received + // sync data afterwards so that's the order that the MMS enforces here. + // (Technically, it seems to work also the other way round.) + // + // To check whether a NEW round of syncing is necessary the MMS works with a + // "wallet state": new state means new syncing needed. + // + // The MMS monitors the "wallet state" by recording "wallet heights" as + // numbers of transfers present in a wallet at the time of message creation. While + // not watertight, this quite simple scheme should already suffice to trigger + // and orchestrate a sensible exchange of sync data. + if (state.has_multisig_partial_key_images || force_sync) + { + // Sync is necessary and not yet completed: Processing of transactions + // will only be possible again once properly synced + // Check first whether we generated already OUR sync info; take note of + // any processable sync info from other signers on the way in case we need it + bool own_sync_data_created = false; + std::vector<uint32_t> sync_messages(m_num_authorized_signers, 0); + for (size_t i = 0; i < m_messages.size(); ++i) + { + message &m = m_messages[i]; + if ((m.type == message_type::multisig_sync_data) && (force_sync || (m.wallet_height == wallet_height))) + // It's data for the same "round" of syncing, on the same "wallet height", therefore relevant + // With "force_sync" take ANY waiting sync data, maybe it will work out + { + if (m.direction == message_direction::out) + { + own_sync_data_created = true; + // Ignore whether sent already or not, and assume as complete if several other signers there + } + else if ((m.direction == message_direction::in) && (m.state == message_state::waiting)) + { + if (sync_messages[m.signer_index] == 0) + { + sync_messages[m.signer_index] = m.id; + } + // else duplicate sync message, ignore + } + } + } + if (!own_sync_data_created) + { + // As explained above, creating sync data BEFORE processing such data from + // other signers reliably works, so insist on that here + processing_data data; + data.processing = message_processing::create_sync_data; + data_list.push_back(data); + return true; + } + uint32_t id_count = (uint32_t)get_other_signers_id_count(sync_messages); + // Do we have sync data from ALL other signers? + bool all_sync_data = id_count == (m_num_authorized_signers - 1); + // Do we have just ENOUGH sync data to have a minimal viable sync set? + // In cases like 2/3 multisig we don't need messages from ALL other signers, only + // from enough of them i.e. num_required_signers minus 1 messages + bool enough_sync_data = id_count >= (m_num_required_signers - 1); + bool sync = false; + wait_reason = tr("Syncing not done because multisig sync data from other signers are missing or not complete."); + if (all_sync_data) + { + sync = true; + } + else if (enough_sync_data) + { + if (force_sync) + { + sync = true; + } + else + { + // Don't sync, but give a hint how this minimal set COULD be synced if really wanted + wait_reason += (boost::format("\nUse \"mms next sync\" if you want to sync with just %s out of %s authorized signers and transact just with them") + % (m_num_required_signers - 1) % (m_num_authorized_signers - 1)).str(); + } + } + if (sync) + { + processing_data data; + data.processing = message_processing::process_sync_data; + for (size_t i = 0; i < sync_messages.size(); ++i) + { + uint32_t id = sync_messages[i]; + if (id != 0) + { + data.message_ids.push_back(id); + } + } + data_list.push_back(data); + return true; + } + else + { + // We can't proceed to any transactions until we have synced; "wait_reason" already set above + return false; + } + } + + bool waiting_found = false; + bool note_found = false; + bool sync_data_found = false; + for (size_t i = 0; i < m_messages.size(); ++i) + { + message &m = m_messages[i]; + if (m.state == message_state::waiting) + { + waiting_found = true; + switch (m.type) + { + case message_type::fully_signed_tx: + { + // We can either submit it ourselves, or send it to any other signer for submission + processing_data data; + data.processing = message_processing::submit_tx; + data.message_ids.push_back(m.id); + data_list.push_back(data); + + data.processing = message_processing::send_tx; + for (uint32_t j = 1; j < m_num_authorized_signers; ++j) + { + data.receiving_signer_index = j; + data_list.push_back(data); + } + return true; + } + + case message_type::partially_signed_tx: + { + if (m.signer_index == 0) + { + // We started this ourselves, or signed it but with still signatures missing: + // We can send it to any other signer for signing / further signing + // In principle it does not make sense to send it back to somebody who + // already signed, but the MMS does not / not yet keep track of that, + // because that would be somewhat complicated. + processing_data data; + data.processing = message_processing::send_tx; + data.message_ids.push_back(m.id); + for (uint32_t j = 1; j < m_num_authorized_signers; ++j) + { + data.receiving_signer_index = j; + data_list.push_back(data); + } + return true; + } + else + { + // Somebody else sent this to us: We can sign it + // It would be possible to just pass it on, but that's not directly supported here + processing_data data; + data.processing = message_processing::sign_tx; + data.message_ids.push_back(m.id); + data_list.push_back(data); + return true; + } + } + + case message_type::note: + note_found = true; + break; + + case message_type::multisig_sync_data: + sync_data_found = true; + break; + + default: + break; + } + } + } + if (waiting_found) + { + wait_reason = tr("There are waiting messages, but nothing is ready to process under normal circumstances"); + if (sync_data_found) + { + wait_reason += tr("\nUse \"mms next sync\" if you want to force processing of the waiting sync data"); + } + if (note_found) + { + wait_reason += tr("\nUse \"mms note\" to display the waiting notes"); + } + } + else + { + wait_reason = tr("There are no messages waiting to be processed."); + } + + return false; +} + +void message_store::set_messages_processed(const processing_data &data) +{ + for (size_t i = 0; i < data.message_ids.size(); ++i) + { + set_message_processed_or_sent(data.message_ids[i]); + } +} + +void message_store::set_message_processed_or_sent(uint32_t id) +{ + message &m = get_message_ref_by_id(id); + if (m.state == message_state::waiting) + { + // So far a fairly cautious and conservative strategy: Only delete from Bitmessage + // when fully processed (and e.g. not already after reception and writing into + // the message store file) + delete_transport_message(id); + m.state = message_state::processed; + } + else if (m.state == message_state::ready_to_send) + { + m.state = message_state::sent; + } + m.modified = (uint64_t)time(NULL); +} + +void message_store::encrypt(crypto::public_key public_key, const std::string &plaintext, + std::string &ciphertext, crypto::public_key &encryption_public_key, crypto::chacha_iv &iv) +{ + crypto::secret_key encryption_secret_key; + crypto::generate_keys(encryption_public_key, encryption_secret_key); + + crypto::key_derivation derivation; + bool success = crypto::generate_key_derivation(public_key, encryption_secret_key, derivation); + THROW_WALLET_EXCEPTION_IF(!success, tools::error::wallet_internal_error, "Failed to generate key derivation for message encryption"); + + crypto::chacha_key chacha_key; + crypto::generate_chacha_key(&derivation, sizeof(derivation), chacha_key, 1); + iv = crypto::rand<crypto::chacha_iv>(); + ciphertext.resize(plaintext.size()); + crypto::chacha20(plaintext.data(), plaintext.size(), chacha_key, iv, &ciphertext[0]); +} + +void message_store::decrypt(const std::string &ciphertext, const crypto::public_key &encryption_public_key, const crypto::chacha_iv &iv, + const crypto::secret_key &view_secret_key, std::string &plaintext) +{ + crypto::key_derivation derivation; + bool success = crypto::generate_key_derivation(encryption_public_key, view_secret_key, derivation); + THROW_WALLET_EXCEPTION_IF(!success, tools::error::wallet_internal_error, "Failed to generate key derivation for message decryption"); + crypto::chacha_key chacha_key; + crypto::generate_chacha_key(&derivation, sizeof(derivation), chacha_key, 1); + plaintext.resize(ciphertext.size()); + crypto::chacha20(ciphertext.data(), ciphertext.size(), chacha_key, iv, &plaintext[0]); +} + +void message_store::send_message(const multisig_wallet_state &state, uint32_t id) +{ + message &m = get_message_ref_by_id(id); + const authorized_signer &me = m_signers[0]; + const authorized_signer &receiver = m_signers[m.signer_index]; + transport_message dm; + crypto::public_key public_key; + + dm.timestamp = (uint64_t)time(NULL); + dm.subject = "MMS V0 " + tools::get_human_readable_timestamp(dm.timestamp); + dm.source_transport_address = me.transport_address; + dm.source_monero_address = me.monero_address; + if (m.type == message_type::auto_config_data) + { + // Encrypt with the public key derived from the auto-config token, and send to the + // transport address likewise derived from that token + public_key = me.auto_config_public_key; + dm.destination_transport_address = me.auto_config_transport_address; + // The destination Monero address is not yet known + memset(&dm.destination_monero_address, 0, sizeof(cryptonote::account_public_address)); + } + else + { + // Encrypt with the receiver's view public key + public_key = receiver.monero_address.m_view_public_key; + const authorized_signer &receiver = m_signers[m.signer_index]; + dm.destination_monero_address = receiver.monero_address; + dm.destination_transport_address = receiver.transport_address; + } + encrypt(public_key, m.content, dm.content, dm.encryption_public_key, dm.iv); + dm.type = (uint32_t)m.type; + dm.hash = crypto::cn_fast_hash(dm.content.data(), dm.content.size()); + dm.round = m.round; + + crypto::generate_signature(dm.hash, me.monero_address.m_view_public_key, state.view_secret_key, dm.signature); + + m_transporter.send_message(dm); + + m.state=message_state::sent; + m.sent= (uint64_t)time(NULL); +} + +bool message_store::check_for_messages(const multisig_wallet_state &state, std::vector<message> &messages) +{ + m_run.store(true, std::memory_order_relaxed); + const authorized_signer &me = m_signers[0]; + std::vector<std::string> destinations; + destinations.push_back(me.transport_address); + for (uint32_t i = 1; i < m_num_authorized_signers; ++i) + { + const authorized_signer &m = m_signers[i]; + if (m.auto_config_running) + { + destinations.push_back(m.auto_config_transport_address); + } + } + std::vector<transport_message> transport_messages; + bool r = m_transporter.receive_messages(destinations, transport_messages); + if (!m_run.load(std::memory_order_relaxed)) + { + // Stop was called, don't waste time processing the messages + // (but once started processing them, don't react to stop request anymore, avoid receiving them "partially)" + return false; + } + + bool new_messages = false; + for (size_t i = 0; i < transport_messages.size(); ++i) + { + transport_message &rm = transport_messages[i]; + if (any_message_with_hash(rm.hash)) + { + // Already seen, do not take again + } + else + { + uint32_t sender_index; + bool take = false; + message_type type = static_cast<message_type>(rm.type); + crypto::secret_key decrypt_key = state.view_secret_key; + if (type == message_type::auto_config_data) + { + // Find out which signer sent it by checking which auto config transport address + // the message was sent to + for (uint32_t i = 1; i < m_num_authorized_signers; ++i) + { + const authorized_signer &m = m_signers[i]; + if (m.auto_config_transport_address == rm.destination_transport_address) + { + take = true; + sender_index = i; + decrypt_key = m.auto_config_secret_key; + break; + } + } + } + else if (type == message_type::signer_config) + { + // Typically we can't check yet whether we know the sender, so take from any + // and pretend it's from "me" because we might have nothing else yet + take = true; + sender_index = 0; + } + else + { + // Only accept from senders that are known as signer here, otherwise just ignore + take = get_signer_index_by_monero_address(rm.source_monero_address, sender_index); + } + if (take && (type != message_type::auto_config_data)) + { + // If the destination address is known, check it as well; this additional filter + // allows using the same transport address for multiple signers + take = rm.destination_monero_address == me.monero_address; + } + if (take) + { + crypto::hash actual_hash = crypto::cn_fast_hash(rm.content.data(), rm.content.size()); + THROW_WALLET_EXCEPTION_IF(actual_hash != rm.hash, tools::error::wallet_internal_error, "Message hash mismatch"); + + bool signature_valid = crypto::check_signature(actual_hash, rm.source_monero_address.m_view_public_key, rm.signature); + THROW_WALLET_EXCEPTION_IF(!signature_valid, tools::error::wallet_internal_error, "Message signature not valid"); + + std::string plaintext; + decrypt(rm.content, rm.encryption_public_key, rm.iv, decrypt_key, plaintext); + size_t index = add_message(state, sender_index, (message_type)rm.type, message_direction::in, plaintext); + message &m = m_messages[index]; + m.hash = rm.hash; + m.transport_id = rm.transport_id; + m.sent = rm.timestamp; + m.round = rm.round; + m.signature_count = rm.signature_count; + messages.push_back(m); + new_messages = true; + } + } + } + return new_messages; +} + +void message_store::delete_transport_message(uint32_t id) +{ + const message &m = get_message_by_id(id); + if (!m.transport_id.empty()) + { + m_transporter.delete_message(m.transport_id); + } +} + +std::string message_store::account_address_to_string(const cryptonote::account_public_address &account_address) const +{ + return get_account_address_as_str(m_nettype, false, account_address); +} + +const char* message_store::message_type_to_string(message_type type) +{ + switch (type) + { + case message_type::key_set: + return tr("key set"); + case message_type::additional_key_set: + return tr("additional key set"); + case message_type::multisig_sync_data: + return tr("multisig sync data"); + case message_type::partially_signed_tx: + return tr("partially signed tx"); + case message_type::fully_signed_tx: + return tr("fully signed tx"); + case message_type::note: + return tr("note"); + case message_type::signer_config: + return tr("signer config"); + case message_type::auto_config_data: + return tr("auto-config data"); + default: + return tr("unknown message type"); + } +} + +const char* message_store::message_direction_to_string(message_direction direction) +{ + switch (direction) + { + case message_direction::in: + return tr("in"); + case message_direction::out: + return tr("out"); + default: + return tr("unknown message direction"); + } +} + +const char* message_store::message_state_to_string(message_state state) +{ + switch (state) + { + case message_state::ready_to_send: + return tr("ready to send"); + case message_state::sent: + return tr("sent"); + case message_state::waiting: + return tr("waiting"); + case message_state::processed: + return tr("processed"); + case message_state::cancelled: + return tr("cancelled"); + default: + return tr("unknown message state"); + } +} + +// Convert a signer to string suitable for a column in a list, with 'max_width' +// Format: label: transport_address +std::string message_store::signer_to_string(const authorized_signer &signer, uint32_t max_width) +{ + std::string s = ""; + s.reserve(max_width); + uint32_t avail = max_width; + uint32_t label_len = signer.label.length(); + if (label_len > avail) + { + s.append(signer.label.substr(0, avail - 2)); + s.append(".."); + return s; + } + s.append(signer.label); + avail -= label_len; + uint32_t transport_addr_len = signer.transport_address.length(); + if ((transport_addr_len > 0) && (avail > 10)) + { + s.append(": "); + avail -= 2; + if (transport_addr_len <= avail) + { + s.append(signer.transport_address); + } + else + { + s.append(signer.transport_address.substr(0, avail-2)); + s.append(".."); + } + } + return s; +} + +} diff --git a/src/wallet/message_store.h b/src/wallet/message_store.h new file mode 100644 index 000000000..637bd29a1 --- /dev/null +++ b/src/wallet/message_store.h @@ -0,0 +1,421 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <cstdlib> +#include <string> +#include <vector> +#include "crypto/hash.h" +#include <boost/serialization/vector.hpp> +#include <boost/program_options/variables_map.hpp> +#include <boost/program_options/options_description.hpp> +#include <boost/optional/optional.hpp> +#include "serialization/serialization.h" +#include "cryptonote_basic/cryptonote_boost_serialization.h" +#include "cryptonote_basic/account_boost_serialization.h" +#include "cryptonote_basic/cryptonote_basic.h" +#include "common/i18n.h" +#include "common/command_line.h" +#include "wipeable_string.h" +#include "message_transporter.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "wallet.mms" +#define AUTO_CONFIG_TOKEN_BYTES 4 +#define AUTO_CONFIG_TOKEN_PREFIX "mms" + +namespace mms +{ + enum class message_type + { + key_set, + additional_key_set, + multisig_sync_data, + partially_signed_tx, + fully_signed_tx, + note, + signer_config, + auto_config_data + }; + + enum class message_direction + { + in, + out + }; + + enum class message_state + { + ready_to_send, + sent, + + waiting, + processed, + + cancelled + }; + + enum class message_processing + { + prepare_multisig, + make_multisig, + exchange_multisig_keys, + create_sync_data, + process_sync_data, + sign_tx, + send_tx, + submit_tx, + process_signer_config, + process_auto_config_data + }; + + struct message + { + uint32_t id; + message_type type; + message_direction direction; + std::string content; + uint64_t created; + uint64_t modified; + uint64_t sent; + uint32_t signer_index; + crypto::hash hash; + message_state state; + uint32_t wallet_height; + uint32_t round; + uint32_t signature_count; + std::string transport_id; + }; + // "wallet_height" (for lack of a short name that would describe what it is about) + // is the number of transfers present in the wallet at the time of message + // construction; used to coordinate generation of sync info (which depends + // on the content of the wallet at time of generation) + + struct authorized_signer + { + std::string label; + std::string transport_address; + bool monero_address_known; + cryptonote::account_public_address monero_address; + bool me; + uint32_t index; + std::string auto_config_token; + crypto::public_key auto_config_public_key; + crypto::secret_key auto_config_secret_key; + std::string auto_config_transport_address; + bool auto_config_running; + + authorized_signer() + { + monero_address_known = false; + memset(&monero_address, 0, sizeof(cryptonote::account_public_address)); + me = false; + index = 0; + auto_config_public_key = crypto::null_pkey; + auto_config_secret_key = crypto::null_skey; + auto_config_running = false; + }; + }; + + struct processing_data + { + message_processing processing; + std::vector<uint32_t> message_ids; + uint32_t receiving_signer_index = 0; + }; + + struct file_transport_message + { + cryptonote::account_public_address sender_address; + crypto::chacha_iv iv; + crypto::public_key encryption_public_key; + message internal_message; + }; + + struct auto_config_data + { + std::string label; + std::string transport_address; + cryptonote::account_public_address monero_address; + }; + + // Overal .mms file structure, with the "message_store" object serialized to and + // encrypted in "encrypted_data" + struct file_data + { + std::string magic_string; + uint32_t file_version; + crypto::chacha_iv iv; + std::string encrypted_data; + }; + + // The following struct provides info about the current state of a "wallet2" object + // at the time of a "message_store" method call that those methods need. See on the + // one hand a first parameter of this type for several of those methods, and on the + // other hand the method "wallet2::get_multisig_wallet_state" which clients like the + // CLI wallet can use to get that info. + // + // Note that in the case of a wallet that is already multisig "address" is NOT the + // multisig address, but the "original" wallet address at creation time. Likewise + // "view_secret_key" is the original view secret key then. + // + // This struct definition is here and not in "wallet2.h" to avoid circular imports. + struct multisig_wallet_state + { + cryptonote::account_public_address address; + cryptonote::network_type nettype; + crypto::secret_key view_secret_key; + bool multisig; + bool multisig_is_ready; + bool has_multisig_partial_key_images; + uint32_t multisig_rounds_passed; + size_t num_transfer_details; + std::string mms_file; + }; + + class message_store + { + public: + message_store(); + // Initialize and start to use the MMS, set the first signer, this wallet itself + // Filename, if not null and not empty, is used to create the ".mms" file + // reset it if already used, with deletion of all signers and messages + void init(const multisig_wallet_state &state, const std::string &own_label, + const std::string &own_transport_address, uint32_t num_authorized_signers, uint32_t num_required_signers); + void set_active(bool active) { m_active = active; }; + void set_auto_send(bool auto_send) { m_auto_send = auto_send; }; + void set_options(const boost::program_options::variables_map& vm); + void set_options(const std::string &bitmessage_address, const epee::wipeable_string &bitmessage_login); + bool get_active() const { return m_active; }; + bool get_auto_send() const { return m_auto_send; }; + uint32_t get_num_required_signers() const { return m_num_required_signers; }; + uint32_t get_num_authorized_signers() const { return m_num_authorized_signers; }; + + void set_signer(const multisig_wallet_state &state, + uint32_t index, + const boost::optional<std::string> &label, + const boost::optional<std::string> &transport_address, + const boost::optional<cryptonote::account_public_address> monero_address); + + const authorized_signer &get_signer(uint32_t index) const; + bool get_signer_index_by_monero_address(const cryptonote::account_public_address &monero_address, uint32_t &index) const; + bool get_signer_index_by_label(const std::string label, uint32_t &index) const; + const std::vector<authorized_signer> &get_all_signers() const { return m_signers; }; + bool signer_config_complete() const; + bool signer_labels_complete() const; + void get_signer_config(std::string &signer_config); + void unpack_signer_config(const multisig_wallet_state &state, const std::string &signer_config, + std::vector<authorized_signer> &signers); + void process_signer_config(const multisig_wallet_state &state, const std::string &signer_config); + + void start_auto_config(const multisig_wallet_state &state); + bool check_auto_config_token(const std::string &raw_token, + std::string &adjusted_token) const; + size_t add_auto_config_data_message(const multisig_wallet_state &state, + const std::string &auto_config_token); + void process_auto_config_data_message(uint32_t id); + void stop_auto_config(); + + // Process data just created by "me" i.e. the own local wallet, e.g. as the result of a "prepare_multisig" command + // Creates the resulting messages to the right signers + void process_wallet_created_data(const multisig_wallet_state &state, message_type type, const std::string &content); + + // Go through all the messages, look at the "ready to process" ones, and check whether any single one + // or any group of them can be processed, because they are processable as single messages (like a tx + // that is fully signed and thus ready for submit to the net) or because they form a complete group + // (e.g. key sets from all authorized signers to make the wallet multisig). If there are multiple + // candidates, e.g. in 2/3 multisig sending to one OR the other signer to sign, there will be more + // than 1 element in 'data' for the user to choose. If nothing is ready "false" is returned. + // The method mostly ignores the order in which the messages were received because messages may be delayed + // (e.g. sync data from a signer arrives AFTER a transaction to submit) or because message time stamps + // may be wrong so it's not possible to order them reliably. + // Messages also may be ready by themselves but the wallet not yet ready for them (e.g. sync data already + // arriving when the wallet is not yet multisig because key sets were delayed or were lost altogether.) + // If nothing is ready 'wait_reason' may contain further info about the reason why. + bool get_processable_messages(const multisig_wallet_state &state, + bool force_sync, + std::vector<processing_data> &data_list, + std::string &wait_reason); + void set_messages_processed(const processing_data &data); + + size_t add_message(const multisig_wallet_state &state, + uint32_t signer_index, message_type type, message_direction direction, + const std::string &content); + const std::vector<message> &get_all_messages() const { return m_messages; }; + bool get_message_by_id(uint32_t id, message &m) const; + message get_message_by_id(uint32_t id) const; + void set_message_processed_or_sent(uint32_t id); + void delete_message(uint32_t id); + void delete_all_messages(); + void get_sanitized_message_text(const message &m, std::string &sanitized_text) const; + + void send_message(const multisig_wallet_state &state, uint32_t id); + bool check_for_messages(const multisig_wallet_state &state, std::vector<message> &messages); + void stop() { m_run.store(false, std::memory_order_relaxed); m_transporter.stop(); } + + void write_to_file(const multisig_wallet_state &state, const std::string &filename); + void read_from_file(const multisig_wallet_state &state, const std::string &filename); + + template <class t_archive> + inline void serialize(t_archive &a, const unsigned int ver) + { + a & m_active; + a & m_num_authorized_signers; + a & m_nettype; + a & m_num_required_signers; + a & m_signers; + a & m_messages; + a & m_next_message_id; + a & m_auto_send; + } + + static const char* message_type_to_string(message_type type); + static const char* message_direction_to_string(message_direction direction); + static const char* message_state_to_string(message_state state); + std::string signer_to_string(const authorized_signer &signer, uint32_t max_width); + + static const char *tr(const char *str) { return i18n_translate(str, "tools::mms"); } + static void init_options(boost::program_options::options_description& desc_params); + + private: + bool m_active; + uint32_t m_num_authorized_signers; + uint32_t m_num_required_signers; + bool m_auto_send; + cryptonote::network_type m_nettype; + std::vector<authorized_signer> m_signers; + std::vector<message> m_messages; + uint32_t m_next_message_id; + std::string m_filename; + message_transporter m_transporter; + std::atomic<bool> m_run; + + bool get_message_index_by_id(uint32_t id, size_t &index) const; + size_t get_message_index_by_id(uint32_t id) const; + message& get_message_ref_by_id(uint32_t id); + bool any_message_of_type(message_type type, message_direction direction) const; + bool any_message_with_hash(const crypto::hash &hash) const; + size_t get_other_signers_id_count(const std::vector<uint32_t> &ids) const; + bool message_ids_complete(const std::vector<uint32_t> &ids) const; + void encrypt(crypto::public_key public_key, const std::string &plaintext, + std::string &ciphertext, crypto::public_key &encryption_public_key, crypto::chacha_iv &iv); + void decrypt(const std::string &ciphertext, const crypto::public_key &encryption_public_key, const crypto::chacha_iv &iv, + const crypto::secret_key &view_secret_key, std::string &plaintext); + std::string create_auto_config_token(); + void setup_signer_for_auto_config(uint32_t index, const std::string token, bool receiving); + void delete_transport_message(uint32_t id); + std::string account_address_to_string(const cryptonote::account_public_address &account_address) const; + void save(const multisig_wallet_state &state); + }; +} + +BOOST_CLASS_VERSION(mms::file_data, 0) +BOOST_CLASS_VERSION(mms::message_store, 0) +BOOST_CLASS_VERSION(mms::message, 0) +BOOST_CLASS_VERSION(mms::file_transport_message, 0) +BOOST_CLASS_VERSION(mms::authorized_signer, 1) +BOOST_CLASS_VERSION(mms::auto_config_data, 0) + +namespace boost +{ + namespace serialization + { + template <class Archive> + inline void serialize(Archive &a, mms::file_data &x, const boost::serialization::version_type ver) + { + a & x.magic_string; + a & x.file_version; + a & x.iv; + a & x.encrypted_data; + } + + template <class Archive> + inline void serialize(Archive &a, mms::message &x, const boost::serialization::version_type ver) + { + a & x.id; + a & x.type; + a & x.direction; + a & x.content; + a & x.created; + a & x.modified; + a & x.sent; + a & x.signer_index; + a & x.hash; + a & x.state; + a & x.wallet_height; + a & x.round; + a & x.signature_count; + a & x.transport_id; + } + + template <class Archive> + inline void serialize(Archive &a, mms::authorized_signer &x, const boost::serialization::version_type ver) + { + a & x.label; + a & x.transport_address; + a & x.monero_address_known; + a & x.monero_address; + a & x.me; + a & x.index; + if (ver < 1) + { + return; + } + a & x.auto_config_token; + a & x.auto_config_public_key; + a & x.auto_config_secret_key; + a & x.auto_config_transport_address; + a & x.auto_config_running; + } + + template <class Archive> + inline void serialize(Archive &a, mms::auto_config_data &x, const boost::serialization::version_type ver) + { + a & x.label; + a & x.transport_address; + a & x.monero_address; + } + + template <class Archive> + inline void serialize(Archive &a, mms::file_transport_message &x, const boost::serialization::version_type ver) + { + a & x.sender_address; + a & x.iv; + a & x.encryption_public_key; + a & x.internal_message; + } + + template <class Archive> + inline void serialize(Archive &a, crypto::chacha_iv &x, const boost::serialization::version_type ver) + { + a & x.data; + } + + } +} diff --git a/src/wallet/message_transporter.cpp b/src/wallet/message_transporter.cpp new file mode 100644 index 000000000..eafd13d3b --- /dev/null +++ b/src/wallet/message_transporter.cpp @@ -0,0 +1,317 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "message_transporter.h" +#include "string_coding.h" +#include <boost/format.hpp> +#include "wallet_errors.h" +#include "net/http_client.h" +#include "net/net_parse_helpers.h" +#include <algorithm> + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "wallet.mms" +#define PYBITMESSAGE_DEFAULT_API_PORT 8442 + +namespace mms +{ + +namespace bitmessage_rpc +{ + + struct message_info + { + uint32_t encodingType; + std::string toAddress; + uint32_t read; + std::string msgid; + std::string message; + std::string fromAddress; + std::string receivedTime; + std::string subject; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(encodingType) + KV_SERIALIZE(toAddress) + KV_SERIALIZE(read) + KV_SERIALIZE(msgid) + KV_SERIALIZE(message); + KV_SERIALIZE(fromAddress) + KV_SERIALIZE(receivedTime) + KV_SERIALIZE(subject) + END_KV_SERIALIZE_MAP() + }; + + struct inbox_messages_response + { + std::vector<message_info> inboxMessages; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(inboxMessages) + END_KV_SERIALIZE_MAP() + }; + +} + +message_transporter::message_transporter() +{ + m_run = true; +} + +void message_transporter::set_options(const std::string &bitmessage_address, const epee::wipeable_string &bitmessage_login) +{ + m_bitmessage_url = bitmessage_address; + epee::net_utils::http::url_content address_parts{}; + epee::net_utils::parse_url(m_bitmessage_url, address_parts); + if (address_parts.port == 0) + { + address_parts.port = PYBITMESSAGE_DEFAULT_API_PORT; + } + m_bitmessage_login = bitmessage_login; + + m_http_client.set_server(address_parts.host, std::to_string(address_parts.port), boost::none); +} + +bool message_transporter::receive_messages(const std::vector<std::string> &destination_transport_addresses, + std::vector<transport_message> &messages) +{ + // The message body of the Bitmessage message is basically the transport message, as JSON (and nothing more). + // Weeding out other, non-MMS messages is done in a simple way: If it deserializes without error, it's an MMS message + // That JSON is Base64-encoded by the MMS because the Monero epee JSON serializer does not escape anything and happily + // includes even 0 (NUL) in strings, which might confuse Bitmessage or at least display confusingly in the client. + // There is yet another Base64-encoding of course as part of the Bitmessage API for the message body parameter + // The Bitmessage API call "getAllInboxMessages" gives back a JSON array with all the messages (despite using + // XML-RPC for the calls, and not JSON-RPC ...) + m_run.store(true, std::memory_order_relaxed); + std::string request; + start_xml_rpc_cmd(request, "getAllInboxMessages"); + end_xml_rpc_cmd(request); + std::string answer; + post_request(request, answer); + + std::string json = get_str_between_tags(answer, "<string>", "</string>"); + bitmessage_rpc::inbox_messages_response bitmessage_res; + epee::serialization::load_t_from_json(bitmessage_res, json); + size_t size = bitmessage_res.inboxMessages.size(); + messages.clear(); + + for (size_t i = 0; i < size; ++i) + { + if (!m_run.load(std::memory_order_relaxed)) + { + // Stop was called, don't waste time processing any more messages + return false; + } + const bitmessage_rpc::message_info &message_info = bitmessage_res.inboxMessages[i]; + if (std::find(destination_transport_addresses.begin(), destination_transport_addresses.end(), message_info.toAddress) != destination_transport_addresses.end()) + { + transport_message message; + bool is_mms_message = false; + try + { + // First Base64-decoding: The message body is Base64 in the Bitmessage API + std::string message_body = epee::string_encoding::base64_decode(message_info.message); + // Second Base64-decoding: The MMS uses Base64 to hide non-textual data in its JSON from Bitmessage + json = epee::string_encoding::base64_decode(message_body); + epee::serialization::load_t_from_json(message, json); + is_mms_message = true; + } + catch(const std::exception& e) + { + } + if (is_mms_message) + { + message.transport_id = message_info.msgid; + messages.push_back(message); + } + } + } + + return true; +} + +bool message_transporter::send_message(const transport_message &message) +{ + // <toAddress> <fromAddress> <subject> <message> [encodingType [TTL]] + std::string request; + start_xml_rpc_cmd(request, "sendMessage"); + add_xml_rpc_string_param(request, message.destination_transport_address); + add_xml_rpc_string_param(request, message.source_transport_address); + add_xml_rpc_base64_param(request, message.subject); + std::string json = epee::serialization::store_t_to_json(message); + std::string message_body = epee::string_encoding::base64_encode(json); // See comment in "receive_message" about reason for (double-)Base64 encoding + add_xml_rpc_base64_param(request, message_body); + add_xml_rpc_integer_param(request, 2); + end_xml_rpc_cmd(request); + std::string answer; + post_request(request, answer); + return true; +} + +bool message_transporter::delete_message(const std::string &transport_id) +{ + std::string request; + start_xml_rpc_cmd(request, "trashMessage"); + add_xml_rpc_string_param(request, transport_id); + end_xml_rpc_cmd(request); + std::string answer; + post_request(request, answer); + return true; +} + +// Deterministically derive a transport / Bitmessage address from 'seed' (the 10-hex-digits +// auto-config token will be used), but do not set it up for receiving in PyBitmessage as +// well, because it's possible the address will only ever be used to SEND auto-config data +std::string message_transporter::derive_transport_address(const std::string &seed) +{ + std::string request; + start_xml_rpc_cmd(request, "getDeterministicAddress"); + add_xml_rpc_base64_param(request, seed); + add_xml_rpc_integer_param(request, 4); // addressVersionNumber + add_xml_rpc_integer_param(request, 1); // streamNumber + end_xml_rpc_cmd(request); + std::string answer; + post_request(request, answer); + std::string address = get_str_between_tags(answer, "<string>", "</string>"); + return address; +} + +// Derive a transport address and configure it for receiving in PyBitmessage, typically +// for receiving auto-config messages by the wallet of the auto-config organizer +std::string message_transporter::derive_and_receive_transport_address(const std::string &seed) +{ + // We need to call both "get_deterministic_address" AND "createDeterministicAddresses" + // because we won't get back the address from the latter call if it exists already + std::string address = derive_transport_address(seed); + + std::string request; + start_xml_rpc_cmd(request, "createDeterministicAddresses"); + add_xml_rpc_base64_param(request, seed); + add_xml_rpc_integer_param(request, 1); // numberOfAddresses + add_xml_rpc_integer_param(request, 4); // addressVersionNumber + end_xml_rpc_cmd(request); + std::string answer; + post_request(request, answer); + + return address; +} + +bool message_transporter::delete_transport_address(const std::string &transport_address) +{ + std::string request; + start_xml_rpc_cmd(request, "deleteAddress"); + add_xml_rpc_string_param(request, transport_address); + end_xml_rpc_cmd(request); + std::string answer; + return post_request(request, answer); +} + +bool message_transporter::post_request(const std::string &request, std::string &answer) +{ + // Somehow things do not work out if one tries to connect "m_http_client" to Bitmessage + // and keep it connected over the course of several calls. But with a new connection per + // call and disconnecting after the call there is no problem (despite perhaps a small + // slowdown) + epee::net_utils::http::fields_list additional_params; + + // Basic access authentication according to RFC 7617 (which the epee HTTP classes do not seem to support?) + // "m_bitmessage_login" just contains what is needed here, "user:password" + std::string auth_string = epee::string_encoding::base64_encode((const unsigned char*)m_bitmessage_login.data(), m_bitmessage_login.size()); + auth_string.insert(0, "Basic "); + additional_params.push_back(std::make_pair("Authorization", auth_string)); + + additional_params.push_back(std::make_pair("Content-Type", "application/xml; charset=utf-8")); + const epee::net_utils::http::http_response_info* response = NULL; + std::chrono::milliseconds timeout = std::chrono::seconds(15); + bool r = m_http_client.invoke("/", "POST", request, timeout, std::addressof(response), std::move(additional_params)); + if (r) + { + answer = response->m_body; + } + else + { + LOG_ERROR("POST request to Bitmessage failed: " << request.substr(0, 300)); + THROW_WALLET_EXCEPTION(tools::error::no_connection_to_bitmessage, m_bitmessage_url); + } + m_http_client.disconnect(); // see comment above + std::string string_value = get_str_between_tags(answer, "<string>", "</string>"); + if ((string_value.find("API Error") == 0) || (string_value.find("RPC ") == 0)) + { + THROW_WALLET_EXCEPTION(tools::error::bitmessage_api_error, string_value); + } + + return r; +} + +// Pick some string between two delimiters +// When parsing the XML returned by PyBitmessage, don't bother to fully parse it but as a little hack rely on the +// fact that e.g. a single string returned will be, however deeply nested in "<params><param><value>...", delivered +// between the very first "<string>" and "</string>" tags to be found in the XML +std::string message_transporter::get_str_between_tags(const std::string &s, const std::string &start_delim, const std::string &stop_delim) +{ + size_t first_delim_pos = s.find(start_delim); + if (first_delim_pos != std::string::npos) + { + size_t end_pos_of_first_delim = first_delim_pos + start_delim.length(); + size_t last_delim_pos = s.find(stop_delim); + if (last_delim_pos != std::string::npos) + { + return s.substr(end_pos_of_first_delim, last_delim_pos - end_pos_of_first_delim); + } + } + return std::string(); +} + +void message_transporter::start_xml_rpc_cmd(std::string &xml, const std::string &method_name) +{ + xml = (boost::format("<?xml version=\"1.0\"?><methodCall><methodName>%s</methodName><params>") % method_name).str(); +} + +void message_transporter::add_xml_rpc_string_param(std::string &xml, const std::string ¶m) +{ + xml += (boost::format("<param><value><string>%s</string></value></param>") % param).str(); +} + +void message_transporter::add_xml_rpc_base64_param(std::string &xml, const std::string ¶m) +{ + // Bitmessage expects some arguments Base64-encoded, but it wants them as parameters of type "string", not "base64" that is also part of XML-RPC + std::string encoded_param = epee::string_encoding::base64_encode(param); + xml += (boost::format("<param><value><string>%s</string></value></param>") % encoded_param).str(); +} + +void message_transporter::add_xml_rpc_integer_param(std::string &xml, const int32_t ¶m) +{ + xml += (boost::format("<param><value><int>%i</int></value></param>") % param).str(); +} + +void message_transporter::end_xml_rpc_cmd(std::string &xml) +{ + xml += "</params></methodCall>"; +} + +} diff --git a/src/wallet/message_transporter.h b/src/wallet/message_transporter.h new file mode 100644 index 000000000..8291311ce --- /dev/null +++ b/src/wallet/message_transporter.h @@ -0,0 +1,113 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once +#include "serialization/keyvalue_serialization.h" +#include "cryptonote_basic/cryptonote_basic.h" +#include "cryptonote_basic/cryptonote_boost_serialization.h" +#include "cryptonote_basic/account_boost_serialization.h" +#include "cryptonote_basic/cryptonote_basic.h" +#include "net/http_server_impl_base.h" +#include "net/http_client.h" +#include "common/util.h" +#include "wipeable_string.h" +#include "serialization/keyvalue_serialization.h" +#include <vector> + +namespace mms +{ + +struct transport_message +{ + cryptonote::account_public_address source_monero_address; + std::string source_transport_address; + cryptonote::account_public_address destination_monero_address; + std::string destination_transport_address; + crypto::chacha_iv iv; + crypto::public_key encryption_public_key; + uint64_t timestamp; + uint32_t type; + std::string subject; + std::string content; + crypto::hash hash; + crypto::signature signature; + uint32_t round; + uint32_t signature_count; + std::string transport_id; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(source_monero_address) + KV_SERIALIZE(source_transport_address) + KV_SERIALIZE(destination_monero_address) + KV_SERIALIZE(destination_transport_address) + KV_SERIALIZE_VAL_POD_AS_BLOB(iv) + KV_SERIALIZE_VAL_POD_AS_BLOB(encryption_public_key) + KV_SERIALIZE(timestamp) + KV_SERIALIZE(type) + KV_SERIALIZE(subject) + KV_SERIALIZE(content) + KV_SERIALIZE_VAL_POD_AS_BLOB(hash) + KV_SERIALIZE_VAL_POD_AS_BLOB(signature) + KV_SERIALIZE(round) + KV_SERIALIZE(signature_count) + KV_SERIALIZE(transport_id) + END_KV_SERIALIZE_MAP() +}; + +class message_transporter +{ +public: + message_transporter(); + void set_options(const std::string &bitmessage_address, const epee::wipeable_string &bitmessage_login); + bool send_message(const transport_message &message); + bool receive_messages(const std::vector<std::string> &destination_transport_addresses, + std::vector<transport_message> &messages); + bool delete_message(const std::string &transport_id); + void stop() { m_run.store(false, std::memory_order_relaxed); } + std::string derive_transport_address(const std::string &seed); + std::string derive_and_receive_transport_address(const std::string &seed); + bool delete_transport_address(const std::string &transport_address); + +private: + epee::net_utils::http::http_simple_client m_http_client; + std::string m_bitmessage_url; + epee::wipeable_string m_bitmessage_login; + std::atomic<bool> m_run; + + bool post_request(const std::string &request, std::string &answer); + static std::string get_str_between_tags(const std::string &s, const std::string &start_delim, const std::string &stop_delim); + + static void start_xml_rpc_cmd(std::string &xml, const std::string &method_name); + static void add_xml_rpc_string_param(std::string &xml, const std::string ¶m); + static void add_xml_rpc_base64_param(std::string &xml, const std::string ¶m); + static void add_xml_rpc_integer_param(std::string &xml, const int32_t ¶m); + static void end_xml_rpc_cmd(std::string &xml); + +}; + +} diff --git a/src/wallet/node_rpc_proxy.cpp b/src/wallet/node_rpc_proxy.cpp index 346c052b5..605531e59 100644 --- a/src/wallet/node_rpc_proxy.cpp +++ b/src/wallet/node_rpc_proxy.cpp @@ -28,7 +28,6 @@ #include "node_rpc_proxy.h" #include "rpc/core_rpc_server_commands_defs.h" -#include "common/json_util.h" #include "storages/http_abstract_invoke.h" using namespace epee; diff --git a/src/wallet/ringdb.cpp b/src/wallet/ringdb.cpp index f562d6c06..b69022af4 100644 --- a/src/wallet/ringdb.cpp +++ b/src/wallet/ringdb.cpp @@ -30,6 +30,7 @@ #include <boost/algorithm/string.hpp> #include <boost/range/adaptor/transformed.hpp> #include <boost/filesystem.hpp> +#include "common/util.h" #include "misc_log_ex.h" #include "misc_language.h" #include "wallet_errors.h" diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 6b3a8533e..7a7d0ad64 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -67,10 +67,14 @@ using namespace epee; #include "common/json_util.h" #include "memwipe.h" #include "common/base58.h" +#include "common/combinator.h" #include "common/dns_utils.h" #include "common/notify.h" +#include "common/perf_timer.h" #include "ringct/rctSigs.h" #include "ringdb.h" +#include "device/device_cold.hpp" +#include "device_trezor/device_trezor.hpp" extern "C" { @@ -110,11 +114,11 @@ using namespace cryptonote; #define SUBADDRESS_LOOKAHEAD_MAJOR 50 #define SUBADDRESS_LOOKAHEAD_MINOR 200 -#define KEY_IMAGE_EXPORT_FILE_MAGIC "Monero key image export\002" +#define KEY_IMAGE_EXPORT_FILE_MAGIC "Monero key image export\003" #define MULTISIG_EXPORT_FILE_MAGIC "Monero multisig export\001" -#define OUTPUT_EXPORT_FILE_MAGIC "Monero output export\003" +#define OUTPUT_EXPORT_FILE_MAGIC "Monero output export\004" #define SEGREGATION_FORK_HEIGHT 99999999 #define TESTNET_SEGREGATION_FORK_HEIGHT 99999999 @@ -123,6 +127,8 @@ using namespace cryptonote; #define FIRST_REFRESH_GRANULARITY 1024 +#define GAMMA_PICK_HALF_WINDOW 5 + static const std::string MULTISIG_SIGNATURE_MAGIC = "SigMultisigPkV1"; static const std::string MULTISIG_EXTRA_INFO_MAGIC = "MultisigxV1"; @@ -172,6 +178,51 @@ namespace return public_keys; } + + bool keys_intersect(const std::unordered_set<crypto::public_key>& s1, const std::unordered_set<crypto::public_key>& s2) + { + if (s1.empty() || s2.empty()) + return false; + + for (const auto& e: s1) + { + if (s2.find(e) != s2.end()) + return true; + } + + return false; + } + + void add_reason(std::string &reasons, const char *reason) + { + if (!reasons.empty()) + reasons += ", "; + reasons += reason; + } + + std::string get_text_reason(const cryptonote::COMMAND_RPC_SEND_RAW_TX::response &res) + { + std::string reason; + if (res.low_mixin) + add_reason(reason, "bad ring size"); + if (res.double_spend) + add_reason(reason, "double spend"); + if (res.invalid_input) + add_reason(reason, "invalid input"); + if (res.invalid_output) + add_reason(reason, "invalid output"); + if (res.too_big) + add_reason(reason, "too big"); + if (res.overspend) + add_reason(reason, "overspend"); + if (res.fee_too_low) + add_reason(reason, "fee too low"); + if (res.not_rct) + add_reason(reason, "tx is not ringct"); + if (res.not_relayed) + add_reason(reason, "tx was not relayed"); + return reason; + } } namespace @@ -202,10 +253,11 @@ struct options { }; const command_line::arg_descriptor<uint64_t> kdf_rounds = {"kdf-rounds", tools::wallet2::tr("Number of rounds for the key derivation function"), 1}; const command_line::arg_descriptor<std::string> hw_device = {"hw-device", tools::wallet2::tr("HW device to use"), ""}; + const command_line::arg_descriptor<std::string> hw_device_derivation_path = {"hw-device-deriv-path", tools::wallet2::tr("HW device wallet derivation path (e.g., SLIP-10)"), ""}; const command_line::arg_descriptor<std::string> tx_notify = { "tx-notify" , "Run a program for each new incoming transaction, '%s' will be replaced by the transaction hash" , "" }; }; -void do_prepare_file_names(const std::string& file_path, std::string& keys_file, std::string& wallet_file) +void do_prepare_file_names(const std::string& file_path, std::string& keys_file, std::string& wallet_file, std::string &mms_file) { keys_file = file_path; wallet_file = file_path; @@ -217,6 +269,7 @@ void do_prepare_file_names(const std::string& file_path, std::string& keys_file, {//provided wallet file name keys_file += ".keys"; } + mms_file = file_path + ".mms"; } uint64_t calculate_fee(uint64_t fee_per_kb, size_t bytes, uint64_t fee_multiplier) @@ -254,6 +307,7 @@ std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variabl auto daemon_host = command_line::get_arg(vm, opts.daemon_host); auto daemon_port = command_line::get_arg(vm, opts.daemon_port); auto device_name = command_line::get_arg(vm, opts.hw_device); + auto device_derivation_path = command_line::get_arg(vm, opts.hw_device_derivation_path); THROW_WALLET_EXCEPTION_IF(!daemon_address.empty() && !daemon_host.empty() && 0 != daemon_port, tools::error::wallet_internal_error, tools::wallet2::tr("can't specify daemon host or port more than once")); @@ -308,7 +362,9 @@ std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variabl wallet->init(std::move(daemon_address), std::move(login), 0, false, *trusted_daemon); boost::filesystem::path ringdb_path = command_line::get_arg(vm, opts.shared_ringdb_dir); wallet->set_ring_database(ringdb_path.string()); + wallet->get_message_store().set_options(vm); wallet->device_name(device_name); + wallet->device_derivation_path(device_derivation_path); try { @@ -558,19 +614,6 @@ std::pair<std::unique_ptr<tools::wallet2>, tools::password_container> generate_f return {nullptr, tools::password_container{}}; } -static void throw_on_rpc_response_error(const boost::optional<std::string> &status, const char *method) -{ - // no error - if (!status) - return; - - // empty string -> not connection - THROW_WALLET_EXCEPTION_IF(status->empty(), tools::error::no_connection_to_daemon, method); - - THROW_WALLET_EXCEPTION_IF(*status == CORE_RPC_STATUS_BUSY, tools::error::daemon_busy, method); - THROW_WALLET_EXCEPTION_IF(*status != CORE_RPC_STATUS_OK, tools::error::wallet_generic_rpc_error, method, *status); -} - std::string strjoin(const std::vector<size_t> &V, const char *sep) { std::stringstream ss; @@ -766,6 +809,48 @@ uint32_t get_subaddress_clamped_sum(uint32_t idx, uint32_t extra) return idx + extra; } +static void setup_shim(hw::wallet_shim * shim, tools::wallet2 * wallet) +{ + shim->get_tx_pub_key_from_received_outs = boost::bind(&tools::wallet2::get_tx_pub_key_from_received_outs, wallet, _1); +} + +bool get_pruned_tx(const cryptonote::COMMAND_RPC_GET_TRANSACTIONS::entry &entry, cryptonote::transaction &tx, crypto::hash &tx_hash) +{ + cryptonote::blobdata bd; + + // easy case if we have the whole tx + if (!entry.as_hex.empty() || (!entry.prunable_as_hex.empty() && !entry.pruned_as_hex.empty())) + { + CHECK_AND_ASSERT_MES(epee::string_tools::parse_hexstr_to_binbuff(entry.as_hex.empty() ? entry.pruned_as_hex + entry.prunable_as_hex : entry.as_hex, bd), false, "Failed to parse tx data"); + CHECK_AND_ASSERT_MES(cryptonote::parse_and_validate_tx_from_blob(bd, tx), false, "Invalid tx data"); + tx_hash = cryptonote::get_transaction_hash(tx); + // if the hash was given, check it matches + CHECK_AND_ASSERT_MES(entry.tx_hash.empty() || epee::string_tools::pod_to_hex(tx_hash) == entry.tx_hash, false, + "Response claims a different hash than the data yields"); + return true; + } + // case of a pruned tx with its prunable data hash + if (!entry.pruned_as_hex.empty() && !entry.prunable_hash.empty()) + { + crypto::hash ph; + CHECK_AND_ASSERT_MES(epee::string_tools::hex_to_pod(entry.prunable_hash, ph), false, "Failed to parse prunable hash"); + CHECK_AND_ASSERT_MES(epee::string_tools::parse_hexstr_to_binbuff(entry.pruned_as_hex, bd), false, "Failed to parse pruned data"); + CHECK_AND_ASSERT_MES(parse_and_validate_tx_base_from_blob(bd, tx), false, "Invalid base tx data"); + // only v2 txes can calculate their txid after pruned + if (bd[0] > 1) + { + tx_hash = cryptonote::get_pruned_transaction_hash(tx, ph); + } + else + { + // for v1, we trust the dameon + CHECK_AND_ASSERT_MES(epee::string_tools::hex_to_pod(entry.tx_hash, tx_hash), false, "Failed to parse tx hash"); + } + return true; + } + return false; +} + //----------------------------------------------------------------- } //namespace @@ -805,12 +890,36 @@ wallet_keys_unlocker::~wallet_keys_unlocker() { if (!locked) return; - w.encrypt_keys(key); + try { w.encrypt_keys(key); } + catch (...) + { + MERROR("Failed to re-encrypt wallet keys"); + // do not propagate through dtor, we'd crash + } +} + +void wallet_device_callback::on_button_request() +{ + if (wallet) + wallet->on_button_request(); +} + +void wallet_device_callback::on_pin_request(epee::wipeable_string & pin) +{ + if (wallet) + wallet->on_pin_request(pin); +} + +void wallet_device_callback::on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) +{ + if (wallet) + wallet->on_passphrase_request(on_device, passphrase); } wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended): m_multisig_rescan_info(NULL), m_multisig_rescan_k(NULL), + m_upper_transaction_weight_limit(0), m_run(true), m_callback(0), m_trusted_daemon(false), @@ -840,10 +949,15 @@ wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended): m_key_reuse_mitigation2(true), m_segregation_height(0), m_ignore_fractional_outputs(true), + m_track_uses(false), m_is_initialized(false), m_kdf_rounds(kdf_rounds), is_old_file_format(false), + m_watch_only(false), + m_multisig(false), + m_multisig_threshold(0), m_node_rpc_proxy(m_http_client, m_daemon_rpc_mutex), + m_account_public_address{crypto::null_pkey, crypto::null_pkey}, m_subaddress_lookahead_major(SUBADDRESS_LOOKAHEAD_MAJOR), m_subaddress_lookahead_minor(SUBADDRESS_LOOKAHEAD_MINOR), m_light_wallet(false), @@ -852,12 +966,16 @@ wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended): m_light_wallet_connected(false), m_light_wallet_balance(0), m_light_wallet_unlocked_balance(0), + m_original_keys_available(false), + m_message_store(), m_key_device_type(hw::device::device_type::SOFTWARE), m_ring_history_saved(false), m_ringdb(), m_last_block_reward(0), m_encrypt_keys_after_refresh(boost::none), - m_unattended(unattended) + m_unattended(unattended), + m_devices_registered(false), + m_device_last_key_image_sync(0) { } @@ -880,6 +998,11 @@ std::string wallet2::device_name_option(const boost::program_options::variables_ return command_line::get_arg(vm, options().hw_device); } +std::string wallet2::device_derivation_path_option(const boost::program_options::variables_map &vm) +{ + return command_line::get_arg(vm, options().hw_device_derivation_path); +} + void wallet2::init_options(boost::program_options::options_description& desc_params) { const options opts{}; @@ -895,7 +1018,9 @@ void wallet2::init_options(boost::program_options::options_description& desc_par command_line::add_arg(desc_params, opts.stagenet); command_line::add_arg(desc_params, opts.shared_ringdb_dir); command_line::add_arg(desc_params, opts.kdf_rounds); + mms::message_store::init_options(desc_params); command_line::add_arg(desc_params, opts.hw_device); + command_line::add_arg(desc_params, opts.hw_device_derivation_path); command_line::add_arg(desc_params, opts.tx_notify); } @@ -915,7 +1040,7 @@ std::pair<std::unique_ptr<wallet2>, password_container> wallet2::make_from_file( return {nullptr, password_container{}}; } auto wallet = make_basic(vm, unattended, opts, password_prompter); - if (wallet) + if (wallet && !wallet_file.empty()) { wallet->load(wallet_file, pwd->password()); } @@ -1054,17 +1179,20 @@ bool wallet2::get_multisig_seed(epee::wipeable_string& seed, const epee::wipeabl bool wallet2::reconnect_device() { bool r = true; - hw::device &hwdev = hw::get_device(m_device_name); + hw::device &hwdev = lookup_device(m_device_name); hwdev.set_name(m_device_name); + hwdev.set_network_type(m_nettype); + hwdev.set_derivation_path(m_device_derivation_path); + hwdev.set_callback(get_device_callback()); r = hwdev.init(); if (!r){ - LOG_PRINT_L2("Could not init device"); + MERROR("Could not init device"); return false; } r = hwdev.connect(); if (!r){ - LOG_PRINT_L2("Could not connect to the device"); + MERROR("Could not connect to the device"); return false; } @@ -1281,6 +1409,7 @@ static uint64_t decodeRct(const rct::rctSig & rv, const crypto::key_derivation & { case rct::RCTTypeSimple: case rct::RCTTypeBulletproof: + case rct::RCTTypeBulletproof2: return rct::decodeRctSimple(rv, rct::sk2rct(scalar1), i, mask, hwdev); case rct::RCTTypeFull: return rct::decodeRct(rv, rct::sk2rct(scalar1), i, mask, hwdev); @@ -1296,7 +1425,7 @@ static uint64_t decodeRct(const rct::rctSig & rv, const crypto::key_derivation & } } //---------------------------------------------------------------------------------------------------- -void wallet2::scan_output(const cryptonote::transaction &tx, const crypto::public_key &tx_pub_key, size_t i, tx_scan_info_t &tx_scan_info, int &num_vouts_received, std::unordered_map<cryptonote::subaddress_index, uint64_t> &tx_money_got_in_outs, std::vector<size_t> &outs) +void wallet2::scan_output(const cryptonote::transaction &tx, const crypto::public_key &tx_pub_key, size_t i, tx_scan_info_t &tx_scan_info, int &num_vouts_received, std::unordered_map<cryptonote::subaddress_index, uint64_t> &tx_money_got_in_outs, std::vector<size_t> &outs, bool pool) { THROW_WALLET_EXCEPTION_IF(i >= tx.vout.size(), error::wallet_internal_error, "Invalid vout index"); @@ -1307,7 +1436,7 @@ void wallet2::scan_output(const cryptonote::transaction &tx, const crypto::publi CRITICAL_REGION_LOCAL(password_lock); if (!m_encrypt_keys_after_refresh) { - boost::optional<epee::wipeable_string> pwd = m_callback->on_get_password("output received"); + boost::optional<epee::wipeable_string> pwd = m_callback->on_get_password(pool ? "output found in pool" : "output received"); THROW_WALLET_EXCEPTION_IF(!pwd, error::password_needed, tr("Password is needed to compute key image for incoming monero")); THROW_WALLET_EXCEPTION_IF(!verify_password(*pwd), error::password_needed, tr("Invalid password: password is needed to compute key image for incoming monero")); decrypt_keys(*pwd); @@ -1341,14 +1470,12 @@ void wallet2::scan_output(const cryptonote::transaction &tx, const crypto::publi //---------------------------------------------------------------------------------------------------- void wallet2::cache_tx_data(const cryptonote::transaction& tx, const crypto::hash &txid, tx_cache_data &tx_cache_data) const { - const cryptonote::account_keys& keys = m_account.get_keys(); - if(!parse_tx_extra(tx.extra, tx_cache_data.tx_extra_fields)) { // Extra may only be partially parsed, it's OK if tx_extra_fields contains public key LOG_PRINT_L0("Transaction extra has unsupported format: " << txid); - tx_cache_data.tx_extra_fields.clear(); - return; + if (tx_cache_data.tx_extra_fields.empty()) + return; } // Don't try to extract tx public key if tx has no ouputs @@ -1378,8 +1505,9 @@ void wallet2::cache_tx_data(const cryptonote::transaction& tx, const crypto::has } } //---------------------------------------------------------------------------------------------------- -void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector<uint64_t> &o_indices, uint64_t height, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen, const tx_cache_data &tx_cache_data) +void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector<uint64_t> &o_indices, uint64_t height, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen, const tx_cache_data &tx_cache_data, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache) { + PERF_TIMER(process_new_transaction); // In this function, tx (probably) only contains the base information // (that is, the prunable stuff may or may not be included) if (!miner_tx && !pool) @@ -1512,7 +1640,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote if (tx_scan_info[i].received) { hwdev.conceal_derivation(tx_scan_info[i].received->derivation, tx_pub_key, additional_tx_pub_keys.data, derivation, additional_derivations); - scan_output(tx, tx_pub_key, i, tx_scan_info[i], num_vouts_received, tx_money_got_in_outs, outs); + scan_output(tx, tx_pub_key, i, tx_scan_info[i], num_vouts_received, tx_money_got_in_outs, outs, pool); } } } @@ -1535,7 +1663,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote if (tx_scan_info[i].received) { hwdev.conceal_derivation(tx_scan_info[i].received->derivation, tx_pub_key, additional_tx_pub_keys.data, derivation, additional_derivations); - scan_output(tx, tx_pub_key, i, tx_scan_info[i], num_vouts_received, tx_money_got_in_outs, outs); + scan_output(tx, tx_pub_key, i, tx_scan_info[i], num_vouts_received, tx_money_got_in_outs, outs, pool); } } } @@ -1551,7 +1679,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote boost::unique_lock<hw::device> hwdev_lock (hwdev); hwdev.set_mode(hw::device::NONE); hwdev.conceal_derivation(tx_scan_info[i].received->derivation, tx_pub_key, additional_tx_pub_keys.data, derivation, additional_derivations); - scan_output(tx, tx_pub_key, i, tx_scan_info[i], num_vouts_received, tx_money_got_in_outs, outs); + scan_output(tx, tx_pub_key, i, tx_scan_info[i], num_vouts_received, tx_money_got_in_outs, outs, pool); } } } @@ -1591,6 +1719,25 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote td.m_txid = txid; td.m_key_image = tx_scan_info[o].ki; td.m_key_image_known = !m_watch_only && !m_multisig; + if (!td.m_key_image_known) + { + // we might have cold signed, and have a mapping to key images + std::unordered_map<crypto::public_key, crypto::key_image>::const_iterator i = m_cold_key_images.find(tx_scan_info[o].in_ephemeral.pub); + if (i != m_cold_key_images.end()) + { + td.m_key_image = i->second; + td.m_key_image_known = true; + } + } + if (m_watch_only) + { + // for view wallets, that flag means "we want to request it" + td.m_key_image_request = true; + } + else + { + td.m_key_image_request = false; + } td.m_key_image_partial = m_multisig; td.m_amount = amount; td.m_pk_index = pk_index - 1; @@ -1612,9 +1759,11 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote td.m_rct = false; } set_unspent(m_transfers.size()-1); - if (!m_multisig && !m_watch_only) + if (td.m_key_image_known) m_key_images[td.m_key_image] = m_transfers.size()-1; m_pub_keys[tx_scan_info[o].in_ephemeral.pub] = m_transfers.size()-1; + if (output_tracker_cache) + (*output_tracker_cache)[std::make_pair(tx.vout[o].amount, td.m_global_output_index)] = m_transfers.size() - 1; if (m_multisig) { THROW_WALLET_EXCEPTION_IF(!m_multisig_rescan_k && m_multisig_rescan_info, @@ -1680,6 +1829,8 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote td.m_mask = rct::identity(); td.m_rct = false; } + if (output_tracker_cache) + (*output_tracker_cache)[std::make_pair(tx.vout[o].amount, td.m_global_output_index)] = kit->second; if (m_multisig) { THROW_WALLET_EXCEPTION_IF(!m_multisig_rescan_k && m_multisig_rescan_info, @@ -1711,11 +1862,12 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote { if(in.type() != typeid(cryptonote::txin_to_key)) continue; - auto it = m_key_images.find(boost::get<cryptonote::txin_to_key>(in).k_image); + const cryptonote::txin_to_key &in_to_key = boost::get<cryptonote::txin_to_key>(in); + auto it = m_key_images.find(in_to_key.k_image); if(it != m_key_images.end()) { transfer_details& td = m_transfers[it->second]; - uint64_t amount = boost::get<cryptonote::txin_to_key>(in).amount; + uint64_t amount = in_to_key.amount; if (amount > 0) { if(amount != td.amount()) @@ -1746,6 +1898,34 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote m_callback->on_money_spent(height, txid, tx, amount, tx, td.m_subaddr_index); } } + + if (!pool && m_track_uses) + { + PERF_TIMER(track_uses); + const uint64_t amount = in_to_key.amount; + std::vector<uint64_t> offsets = cryptonote::relative_output_offsets_to_absolute(in_to_key.key_offsets); + if (output_tracker_cache) + { + for (uint64_t offset: offsets) + { + const std::map<std::pair<uint64_t, uint64_t>, size_t>::const_iterator i = output_tracker_cache->find(std::make_pair(amount, offset)); + if (i != output_tracker_cache->end()) + { + size_t idx = i->second; + THROW_WALLET_EXCEPTION_IF(idx >= m_transfers.size(), error::wallet_internal_error, "Output tracker cache index out of range"); + m_transfers[idx].m_uses.push_back(std::make_pair(height, txid)); + } + } + } + else for (transfer_details &td: m_transfers) + { + if (amount != in_to_key.amount) + continue; + for (uint64_t offset: offsets) + if (offset == td.m_global_output_index) + td.m_uses.push_back(std::make_pair(height, txid)); + } + } } uint64_t fee = miner_tx ? 0 : tx.version == 1 ? tx_money_spent_in_ins - get_outs_money_amount(tx) : tx.rct_signatures.txnFee; @@ -1913,6 +2093,7 @@ void wallet2::process_outgoing(const crypto::hash &txid, const cryptonote::trans entry.first->second.m_subaddr_indices = subaddr_indices; } + entry.first->second.m_rings.clear(); for (const auto &in: tx.vin) { if (in.type() != typeid(cryptonote::txin_to_key)) @@ -1927,7 +2108,7 @@ void wallet2::process_outgoing(const crypto::hash &txid, const cryptonote::trans add_rings(tx); } //---------------------------------------------------------------------------------------------------- -void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const parsed_block &parsed_block, const crypto::hash& bl_id, uint64_t height, const std::vector<tx_cache_data> &tx_cache_data, size_t tx_cache_data_offset) +void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const parsed_block &parsed_block, const crypto::hash& bl_id, uint64_t height, const std::vector<tx_cache_data> &tx_cache_data, size_t tx_cache_data_offset, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache) { THROW_WALLET_EXCEPTION_IF(bche.txs.size() + 1 != parsed_block.o_indices.indices.size(), error::wallet_internal_error, "block transactions=" + std::to_string(bche.txs.size()) + @@ -1940,7 +2121,7 @@ void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cry { TIME_MEASURE_START(miner_tx_handle_time); if (m_refresh_type != RefreshNoCoinbase) - process_new_transaction(get_transaction_hash(b.miner_tx), b.miner_tx, parsed_block.o_indices.indices[0].indices, height, b.timestamp, true, false, false, tx_cache_data[tx_cache_data_offset]); + process_new_transaction(get_transaction_hash(b.miner_tx), b.miner_tx, parsed_block.o_indices.indices[0].indices, height, b.timestamp, true, false, false, tx_cache_data[tx_cache_data_offset], output_tracker_cache); ++tx_cache_data_offset; TIME_MEASURE_FINISH(miner_tx_handle_time); @@ -1949,7 +2130,7 @@ void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cry THROW_WALLET_EXCEPTION_IF(bche.txs.size() != parsed_block.txes.size(), error::wallet_internal_error, "Wrong amount of transactions for block"); for (size_t idx = 0; idx < b.tx_hashes.size(); ++idx) { - process_new_transaction(b.tx_hashes[idx], parsed_block.txes[idx], parsed_block.o_indices.indices[idx+1].indices, height, b.timestamp, false, false, false, tx_cache_data[tx_cache_data_offset++]); + process_new_transaction(b.tx_hashes[idx], parsed_block.txes[idx], parsed_block.o_indices.indices[idx+1].indices, height, b.timestamp, false, false, false, tx_cache_data[tx_cache_data_offset++], output_tracker_cache); } TIME_MEASURE_FINISH(txs_handle_time); m_last_block_reward = cryptonote::get_outs_money_amount(b.miner_tx); @@ -2019,7 +2200,7 @@ void wallet2::pull_blocks(uint64_t start_height, uint64_t &blocks_start_height, m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "getblocks.bin"); THROW_WALLET_EXCEPTION_IF(res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "getblocks.bin"); - THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::get_blocks_error, res.status); + THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::get_blocks_error, get_rpc_status(res.status)); THROW_WALLET_EXCEPTION_IF(res.blocks.size() != res.output_indices.size(), error::wallet_internal_error, "mismatched blocks (" + boost::lexical_cast<std::string>(res.blocks.size()) + ") and output_indices (" + boost::lexical_cast<std::string>(res.output_indices.size()) + ") sizes from daemon"); @@ -2041,13 +2222,13 @@ void wallet2::pull_hashes(uint64_t start_height, uint64_t &blocks_start_height, m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "gethashes.bin"); THROW_WALLET_EXCEPTION_IF(res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "gethashes.bin"); - THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::get_hashes_error, res.status); + THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::get_hashes_error, get_rpc_status(res.status)); blocks_start_height = res.start_height; hashes = std::move(res.m_block_ids); } //---------------------------------------------------------------------------------------------------- -void wallet2::process_parsed_blocks(uint64_t start_height, const std::vector<cryptonote::block_complete_entry> &blocks, const std::vector<parsed_block> &parsed_blocks, uint64_t& blocks_added) +void wallet2::process_parsed_blocks(uint64_t start_height, const std::vector<cryptonote::block_complete_entry> &blocks, const std::vector<parsed_block> &parsed_blocks, uint64_t& blocks_added, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache) { size_t current_index = start_height; blocks_added = 0; @@ -2086,7 +2267,6 @@ void wallet2::process_parsed_blocks(uint64_t start_height, const std::vector<cry const cryptonote::account_keys &keys = m_account.get_keys(); auto gender = [&](wallet2::is_out_data &iod) { - boost::unique_lock<hw::device> hwdev_lock(hwdev); if (!hwdev.generate_key_derivation(iod.pkey, keys.m_view_secret_key, iod.derivation)) { MWARNING("Failed to generate key derivation from tx pubkey, skipping"); @@ -2095,12 +2275,16 @@ void wallet2::process_parsed_blocks(uint64_t start_height, const std::vector<cry } }; - for (auto &slot: tx_cache_data) + for (size_t i = 0; i < tx_cache_data.size(); ++i) { - for (auto &iod: slot.primary) - tpool.submit(&waiter, [&gender, &iod]() { gender(iod); }, true); - for (auto &iod: slot.additional) - tpool.submit(&waiter, [&gender, &iod]() { gender(iod); }, true); + tpool.submit(&waiter, [&hwdev, &gender, &tx_cache_data, i]() { + auto &slot = tx_cache_data[i]; + boost::unique_lock<hw::device> hwdev_lock(hwdev); + for (auto &iod: slot.primary) + gender(iod); + for (auto &iod: slot.additional) + gender(iod); + }, true); } waiter.wait(&tpool); @@ -2132,7 +2316,7 @@ void wallet2::process_parsed_blocks(uint64_t start_height, const std::vector<cry { THROW_WALLET_EXCEPTION_IF(txidx >= tx_cache_data.size(), error::wallet_internal_error, "txidx out of range"); const size_t n_vouts = m_refresh_type == RefreshType::RefreshOptimizeCoinbase ? 1 : parsed_blocks[i].block.miner_tx.vout.size(); - tpool.submit(&waiter, [&, i, txidx](){ geniod(parsed_blocks[i].block.miner_tx, n_vouts, txidx); }, true); + tpool.submit(&waiter, [&, i, n_vouts, txidx](){ geniod(parsed_blocks[i].block.miner_tx, n_vouts, txidx); }, true); } ++txidx; for (size_t j = 0; j < parsed_blocks[i].txes.size(); ++j) @@ -2154,7 +2338,7 @@ void wallet2::process_parsed_blocks(uint64_t start_height, const std::vector<cry if(current_index >= m_blockchain.size()) { - process_new_blockchain_entry(bl, blocks[i], parsed_blocks[i], bl_id, current_index, tx_cache_data, tx_cache_data_offset); + process_new_blockchain_entry(bl, blocks[i], parsed_blocks[i], bl_id, current_index, tx_cache_data, tx_cache_data_offset, output_tracker_cache); ++blocks_added; } else if(bl_id != m_blockchain[current_index]) @@ -2166,7 +2350,7 @@ void wallet2::process_parsed_blocks(uint64_t start_height, const std::vector<cry string_tools::pod_to_hex(m_blockchain[current_index])); detach_blockchain(current_index); - process_new_blockchain_entry(bl, blocks[i], parsed_blocks[i], bl_id, current_index, tx_cache_data, tx_cache_data_offset); + process_new_blockchain_entry(bl, blocks[i], parsed_blocks[i], bl_id, current_index, tx_cache_data, tx_cache_data_offset, output_tracker_cache); } else { @@ -2200,11 +2384,10 @@ void wallet2::pull_and_parse_next_blocks(uint64_t start_height, uint64_t &blocks THROW_WALLET_EXCEPTION_IF(prev_blocks.size() != prev_parsed_blocks.size(), error::wallet_internal_error, "size mismatch"); // prepend the last 3 blocks, should be enough to guard against a block or two's reorg - std::vector<parsed_block>::const_reverse_iterator i = prev_parsed_blocks.rbegin(); - for (size_t n = 0; n < std::min((size_t)3, prev_parsed_blocks.size()); ++n) + auto s = std::next(prev_parsed_blocks.rbegin(), std::min((size_t)3, prev_parsed_blocks.size())).base(); + for (; s != prev_parsed_blocks.end(); ++s) { - short_chain_history.push_front(i->hash); - ++i; + short_chain_history.push_front(s->hash); } // pull the new blocks @@ -2284,7 +2467,7 @@ void wallet2::remove_obsolete_pool_txs(const std::vector<crypto::hash> &tx_hashe //---------------------------------------------------------------------------------------------------- void wallet2::update_pool_state(bool refreshed) { - MDEBUG("update_pool_state start"); + MTRACE("update_pool_state start"); auto keys_reencryptor = epee::misc_utils::create_scope_leave_handler([&, this]() { if (m_encrypt_keys_after_refresh) @@ -2303,7 +2486,7 @@ void wallet2::update_pool_state(bool refreshed) THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "get_transaction_pool_hashes.bin"); THROW_WALLET_EXCEPTION_IF(res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_transaction_pool_hashes.bin"); THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::get_tx_pool_error); - MDEBUG("update_pool_state got pool"); + MTRACE("update_pool_state got pool"); // remove any pending tx that's not in the pool std::unordered_map<crypto::hash, wallet2::unconfirmed_transfer_details>::iterator it = m_unconfirmed_txs.begin(); @@ -2360,7 +2543,7 @@ void wallet2::update_pool_state(bool refreshed) } } } - MDEBUG("update_pool_state done first loop"); + MTRACE("update_pool_state done first loop"); // remove pool txes to us that aren't in the pool anymore // but only if we just refreshed, so that the tx can go in @@ -2369,7 +2552,7 @@ void wallet2::update_pool_state(bool refreshed) if (refreshed) remove_obsolete_pool_txs(res.tx_hashes); - MDEBUG("update_pool_state done second loop"); + MTRACE("update_pool_state done second loop"); // gather txids of new pool txes to us std::vector<std::pair<crypto::hash, bool>> txids; @@ -2443,7 +2626,7 @@ void wallet2::update_pool_state(bool refreshed) req.txs_hashes.push_back(epee::string_tools::pod_to_hex(p.first)); MDEBUG("asking for " << txids.size() << " transactions"); req.decode_as_json = false; - req.prune = false; + req.prune = true; m_daemon_rpc_mutex.lock(); bool r = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); @@ -2458,11 +2641,10 @@ void wallet2::update_pool_state(bool refreshed) { cryptonote::transaction tx; cryptonote::blobdata bd; - crypto::hash tx_hash, tx_prefix_hash; - if (epee::string_tools::parse_hexstr_to_binbuff(tx_entry.as_hex, bd)) + crypto::hash tx_hash; + + if (get_pruned_tx(tx_entry, tx, tx_hash)) { - if (cryptonote::parse_and_validate_tx_from_blob(bd, tx, tx_hash, tx_prefix_hash)) - { const std::vector<std::pair<crypto::hash, bool>>::const_iterator i = std::find_if(txids.begin(), txids.end(), [tx_hash](const std::pair<crypto::hash, bool> &e) { return e.first == tx_hash; }); if (i != txids.end()) @@ -2479,11 +2661,6 @@ void wallet2::update_pool_state(bool refreshed) { MERROR("Got txid " << tx_hash << " which we did not ask for"); } - } - else - { - LOG_PRINT_L0("failed to validate transaction from daemon"); - } } else { @@ -2503,10 +2680,10 @@ void wallet2::update_pool_state(bool refreshed) } else { - LOG_PRINT_L0("Error calling gettransactions daemon RPC: r " << r << ", status " << res.status); + LOG_PRINT_L0("Error calling gettransactions daemon RPC: r " << r << ", status " << get_rpc_status(res.status)); } } - MDEBUG("update_pool_state end"); + MTRACE("update_pool_state end"); } //---------------------------------------------------------------------------------------------------- void wallet2::fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, bool force) @@ -2599,7 +2776,18 @@ bool wallet2::delete_address_book_row(std::size_t row_id) { } //---------------------------------------------------------------------------------------------------- -void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched, bool& received_money) +std::shared_ptr<std::map<std::pair<uint64_t, uint64_t>, size_t>> wallet2::create_output_tracker_cache() const +{ + std::shared_ptr<std::map<std::pair<uint64_t, uint64_t>, size_t>> cache{new std::map<std::pair<uint64_t, uint64_t>, size_t>()}; + for (size_t i = 0; i < m_transfers.size(); ++i) + { + const transfer_details &td = m_transfers[i]; + (*cache)[std::make_pair(td.is_rct() ? 0 : td.amount(), td.m_global_output_index)] = i; + } + return cache; +} +//---------------------------------------------------------------------------------------------------- +void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched, bool& received_money, bool check_pool) { if(m_light_wallet) { @@ -2645,6 +2833,7 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo std::vector<cryptonote::block_complete_entry> blocks; std::vector<parsed_block> parsed_blocks; bool refreshed = false; + std::shared_ptr<std::map<std::pair<uint64_t, uint64_t>, size_t>> output_tracker_cache; // pull the first set of blocks get_short_chain_history(short_chain_history, (m_first_refresh_done || trusted_daemon) ? 1 : FIRST_REFRESH_GRANULARITY); @@ -2679,13 +2868,16 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo bool first = true; while(m_run.load(std::memory_order_relaxed)) { + uint64_t next_blocks_start_height; + std::vector<cryptonote::block_complete_entry> next_blocks; + std::vector<parsed_block> next_parsed_blocks; + bool error; try { // pull the next set of blocks while we're processing the current one - uint64_t next_blocks_start_height; - std::vector<cryptonote::block_complete_entry> next_blocks; - std::vector<parsed_block> next_parsed_blocks; - bool error = false; + error = false; + next_blocks.clear(); + next_parsed_blocks.clear(); added_blocks = 0; if (!first && blocks.empty()) { @@ -2698,7 +2890,7 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo { try { - process_parsed_blocks(blocks_start_height, blocks, parsed_blocks, added_blocks); + process_parsed_blocks(blocks_start_height, blocks, parsed_blocks, added_blocks, output_tracker_cache.get()); } catch (const tools::error::out_of_hashchain_bounds_error&) { @@ -2714,7 +2906,7 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo short_chain_history.clear(); get_short_chain_history(short_chain_history); fast_refresh(stop_height, blocks_start_height, short_chain_history, true); - THROW_WALLET_EXCEPTION_IF(m_blockchain.size() != stop_height, error::wallet_internal_error, "Unexpected hashchain size"); + THROW_WALLET_EXCEPTION_IF((m_blockchain.size() == stop_height || (m_blockchain.size() == 1 && stop_height == 0) ? false : true), error::wallet_internal_error, "Unexpected hashchain size"); THROW_WALLET_EXCEPTION_IF(m_blockchain.offset() != 0, error::wallet_internal_error, "Unexpected hashchain offset"); for (const auto &h: tip) m_blockchain.push_back(h); @@ -2723,6 +2915,11 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo start_height = stop_height; throw std::runtime_error(""); // loop again } + catch (const std::exception &e) + { + MERROR("Error parsing blocks: " << e.what()); + error = true; + } blocks_fetched += added_blocks; } waiter.wait(&tpool); @@ -2741,6 +2938,11 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo throw std::runtime_error("proxy exception in refresh thread"); } + // if we've got at least 10 blocks to refresh, assume we're starting + // a long refresh, and setup a tracking output cache if we need to + if (m_track_uses && !output_tracker_cache && next_blocks.size() >= 10) + output_tracker_cache = create_output_tracker_cache(); + // switch to the new blocks from the daemon blocks_start_height = next_blocks_start_height; blocks = std::move(next_blocks); @@ -2775,7 +2977,7 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo try { // If stop() is called we don't need to check pending transactions - if(m_run.load(std::memory_order_relaxed)) + if (check_pool && m_run.load(std::memory_order_relaxed)) update_pool_state(refreshed); } catch (...) @@ -2835,10 +3037,11 @@ bool wallet2::get_rct_distribution(uint64_t &start_height, std::vector<uint64_t> cryptonote::COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response res = AUTO_VAL_INIT(res); req.amounts.push_back(0); req.from_height = 0; - req.cumulative = true; + req.cumulative = false; req.binary = true; + req.compress = true; m_daemon_rpc_mutex.lock(); - bool r = net_utils::invoke_http_json_rpc("/json_rpc", "get_output_distribution", req, res, m_http_client, rpc_timeout); + bool r = net_utils::invoke_http_bin("/get_output_distribution.bin", req, res, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); if (!r) { @@ -2865,8 +3068,10 @@ bool wallet2::get_rct_distribution(uint64_t &start_height, std::vector<uint64_t> MWARNING("Failed to request output distribution: results are not for amount 0"); return false; } - start_height = res.distributions[0].start_height; - distribution = std::move(res.distributions[0].distribution); + for (size_t i = 1; i < res.distributions[0].data.distribution.size(); ++i) + res.distributions[0].data.distribution[i] += res.distributions[0].data.distribution[i-1]; + start_height = res.distributions[0].data.start_height; + distribution = std::move(res.distributions[0].data.distribution); return true; } //---------------------------------------------------------------------------------------------------- @@ -2938,6 +3143,7 @@ bool wallet2::deinit() { m_is_initialized=false; unlock_keys_file(); + m_account.deinit(); return true; } //---------------------------------------------------------------------------------------------------- @@ -2959,6 +3165,7 @@ bool wallet2::clear() m_subaddresses.clear(); m_subaddress_labels.clear(); m_multisig_rounds_passed = 0; + m_device_last_key_image_sync = 0; return true; } @@ -3108,18 +3315,39 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable value2.SetInt(m_ignore_fractional_outputs ? 1 : 0); json.AddMember("ignore_fractional_outputs", value2, json.GetAllocator()); + value2.SetInt(m_track_uses ? 1 : 0); + json.AddMember("track_uses", value2, json.GetAllocator()); + value2.SetUint(m_subaddress_lookahead_major); json.AddMember("subaddress_lookahead_major", value2, json.GetAllocator()); value2.SetUint(m_subaddress_lookahead_minor); json.AddMember("subaddress_lookahead_minor", value2, json.GetAllocator()); + value2.SetInt(m_original_keys_available ? 1 : 0); + json.AddMember("original_keys_available", value2, json.GetAllocator()); + value2.SetUint(1); json.AddMember("encrypted_secret_keys", value2, json.GetAllocator()); value.SetString(m_device_name.c_str(), m_device_name.size()); json.AddMember("device_name", value, json.GetAllocator()); + value.SetString(m_device_derivation_path.c_str(), m_device_derivation_path.size()); + json.AddMember("device_derivation_path", value, json.GetAllocator()); + + std::string original_address; + std::string original_view_secret_key; + if (m_original_keys_available) + { + original_address = get_account_address_as_str(m_nettype, false, m_original_address); + value.SetString(original_address.c_str(), original_address.length()); + json.AddMember("original_address", value, json.GetAllocator()); + original_view_secret_key = epee::string_tools::pod_to_hex(m_original_view_secret_key); + value.SetString(original_view_secret_key.c_str(), original_view_secret_key.length()); + json.AddMember("original_view_secret_key", value, json.GetAllocator()); + } + // Serialize the JSON object rapidjson::StringBuffer buffer; rapidjson::Writer<rapidjson::StringBuffer> writer(buffer); @@ -3127,20 +3355,28 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable account_data = buffer.GetString(); // Encrypt the entire JSON object. - crypto::generate_chacha_key(password.data(), password.size(), key, m_kdf_rounds); std::string cipher; cipher.resize(account_data.size()); keys_file_data.iv = crypto::rand<crypto::chacha_iv>(); crypto::chacha20(account_data.data(), account_data.size(), key, keys_file_data.iv, &cipher[0]); keys_file_data.account_data = cipher; - unlock_keys_file(); + std::string tmp_file_name = keys_file_name + ".new"; std::string buf; r = ::serialization::dump_binary(keys_file_data, buf); - r = r && epee::file_io_utils::save_string_to_file(keys_file_name, buf); //and never touch wallet_keys_file again, only read - CHECK_AND_ASSERT_MES(r, false, "failed to generate wallet keys file " << keys_file_name); + r = r && epee::file_io_utils::save_string_to_file(tmp_file_name, buf); + CHECK_AND_ASSERT_MES(r, false, "failed to generate wallet keys file " << tmp_file_name); + + unlock_keys_file(); + std::error_code e = tools::replace_file(tmp_file_name, keys_file_name); lock_keys_file(); + if (e) { + boost::filesystem::remove(tmp_file_name); + LOG_ERROR("failed to update wallet keys file " << keys_file_name); + return false; + } + return true; } //---------------------------------------------------------------------------------------------------- @@ -3228,9 +3464,12 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ m_key_reuse_mitigation2 = true; m_segregation_height = 0; m_ignore_fractional_outputs = true; + m_track_uses = false; m_subaddress_lookahead_major = SUBADDRESS_LOOKAHEAD_MAJOR; m_subaddress_lookahead_minor = SUBADDRESS_LOOKAHEAD_MINOR; + m_original_keys_available = false; m_device_name = ""; + m_device_derivation_path = ""; m_key_device_type = hw::device::device_type::SOFTWARE; encrypted_secret_keys = false; } @@ -3378,6 +3617,8 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ m_segregation_height = field_segregation_height; GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, ignore_fractional_outputs, int, Int, false, true); m_ignore_fractional_outputs = field_ignore_fractional_outputs; + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, track_uses, int, Int, false, false); + m_track_uses = field_track_uses; GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, subaddress_lookahead_major, uint32_t, Uint, false, SUBADDRESS_LOOKAHEAD_MAJOR); m_subaddress_lookahead_major = field_subaddress_lookahead_major; GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, subaddress_lookahead_minor, uint32_t, Uint, false, SUBADDRESS_LOOKAHEAD_MINOR); @@ -3398,6 +3639,38 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ m_device_name = m_key_device_type == hw::device::device_type::LEDGER ? "Ledger" : "default"; } } + + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, device_derivation_path, std::string, String, false, std::string()); + m_device_derivation_path = field_device_derivation_path; + + if (json.HasMember("original_keys_available")) + { + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, original_keys_available, int, Int, false, false); + m_original_keys_available = field_original_keys_available; + if (m_original_keys_available) + { + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, original_address, std::string, String, true, std::string()); + address_parse_info info; + bool ok = get_account_address_from_str(info, m_nettype, field_original_address); + if (!ok) + { + LOG_ERROR("Failed to parse original_address from JSON"); + return false; + } + m_original_address = info.address; + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, original_view_secret_key, std::string, String, true, std::string()); + ok = epee::string_tools::hex_to_pod(field_original_view_secret_key, m_original_view_secret_key); + if (!ok) + { + LOG_ERROR("Failed to parse original_view_secret_key from JSON"); + return false; + } + } + } + else + { + m_original_keys_available = false; + } } else { @@ -3407,13 +3680,22 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ r = epee::serialization::load_t_from_binary(m_account, account_data); THROW_WALLET_EXCEPTION_IF(!r, error::invalid_password); - if (m_key_device_type == hw::device::device_type::LEDGER) { + if (m_key_device_type == hw::device::device_type::LEDGER || m_key_device_type == hw::device::device_type::TREZOR) { LOG_PRINT_L0("Account on device. Initing device..."); - hw::device &hwdev = hw::get_device(m_device_name); - hwdev.set_name(m_device_name); - hwdev.init(); - hwdev.connect(); + hw::device &hwdev = lookup_device(m_device_name); + THROW_WALLET_EXCEPTION_IF(!hwdev.set_name(m_device_name), error::wallet_internal_error, "Could not set device name " + m_device_name); + hwdev.set_network_type(m_nettype); + hwdev.set_derivation_path(m_device_derivation_path); + hwdev.set_callback(get_device_callback()); + THROW_WALLET_EXCEPTION_IF(!hwdev.init(), error::wallet_internal_error, "Could not initialize the device " + m_device_name); + THROW_WALLET_EXCEPTION_IF(!hwdev.connect(), error::wallet_internal_error, "Could not connect to the device " + m_device_name); m_account.set_device(hwdev); + + account_public_address device_account_public_address; + THROW_WALLET_EXCEPTION_IF(!hwdev.get_public_address(device_account_public_address), error::wallet_internal_error, "Cannot get a device address"); + THROW_WALLET_EXCEPTION_IF(device_account_public_address != m_account.get_keys().m_account_address, error::wallet_internal_error, "Device wallet does not match wallet address. " + "Device address: " + cryptonote::get_account_address_as_str(m_nettype, false, device_account_public_address) + + ", wallet address: " + m_account.get_public_address_str(m_nettype)); LOG_PRINT_L0("Device inited..."); } else if (key_on_device()) { THROW_WALLET_EXCEPTION(error::wallet_internal_error, "hardware device not supported"); @@ -3444,7 +3726,7 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_ const cryptonote::account_keys& keys = m_account.get_keys(); hw::device &hwdev = m_account.get_device(); r = r && hwdev.verify_keys(keys.m_view_secret_key, keys.m_account_address.m_view_public_key); - if(!m_watch_only && !m_multisig) + if(!m_watch_only && !m_multisig && hwdev.device_protocol() != hw::device::PROTOCOL_COLD) r = r && hwdev.verify_keys(keys.m_spend_secret_key, keys.m_account_address.m_spend_public_key); THROW_WALLET_EXCEPTION_IF(!r, error::invalid_password); @@ -3468,7 +3750,7 @@ bool wallet2::verify_password(const epee::wipeable_string& password) { // this temporary unlocking is necessary for Windows (otherwise the file couldn't be loaded). unlock_keys_file(); - bool r = verify_password(m_keys_file, password, m_watch_only || m_multisig, m_account.get_device(), m_kdf_rounds); + bool r = verify_password(m_keys_file, password, m_account.get_device().device_protocol() == hw::device::PROTOCOL_COLD || m_watch_only || m_multisig, m_account.get_device(), m_kdf_rounds); lock_keys_file(); return r; } @@ -3715,6 +3997,10 @@ void wallet2::generate(const std::string& wallet_, const epee::wipeable_string& m_key_device_type = hw::device::device_type::SOFTWARE; setup_keys(password); + // Not possible to restore a multisig wallet that is able to activate the MMS + // (because the original keys are not (yet) part of the restore info) + m_original_keys_available = false; + create_keys_file(wallet_, false, password, m_nettype != MAINNET || create_address_file); setup_new_blockchain(); @@ -3752,6 +4038,7 @@ crypto::secret_key wallet2::generate(const std::string& wallet_, const epee::wip m_multisig = false; m_multisig_threshold = 0; m_multisig_signers.clear(); + m_original_keys_available = false; m_key_device_type = hw::device::device_type::SOFTWARE; setup_keys(password); @@ -3840,6 +4127,7 @@ void wallet2::generate(const std::string& wallet_, const epee::wipeable_string& m_multisig = false; m_multisig_threshold = 0; m_multisig_signers.clear(); + m_original_keys_available = false; m_key_device_type = hw::device::device_type::SOFTWARE; setup_keys(password); @@ -3880,6 +4168,7 @@ void wallet2::generate(const std::string& wallet_, const epee::wipeable_string& m_multisig = false; m_multisig_threshold = 0; m_multisig_signers.clear(); + m_original_keys_available = false; m_key_device_type = hw::device::device_type::SOFTWARE; setup_keys(password); @@ -3908,8 +4197,11 @@ void wallet2::restore(const std::string& wallet_, const epee::wipeable_string& p THROW_WALLET_EXCEPTION_IF(boost::filesystem::exists(m_keys_file, ignored_ec), error::file_exists, m_keys_file); } - auto &hwdev = hw::get_device(device_name); + auto &hwdev = lookup_device(device_name); hwdev.set_name(device_name); + hwdev.set_network_type(m_nettype); + hwdev.set_derivation_path(m_device_derivation_path); + hwdev.set_callback(get_device_callback()); m_account.create_from_device(hwdev); m_key_device_type = m_account.get_device().get_type(); @@ -3918,6 +4210,7 @@ void wallet2::restore(const std::string& wallet_, const epee::wipeable_string& p m_multisig = false; m_multisig_threshold = 0; m_multisig_signers.clear(); + m_original_keys_available = false; setup_keys(password); m_device_name = device_name; @@ -4030,6 +4323,15 @@ std::string wallet2::make_multisig(const epee::wipeable_string &password, m_multisig_derivations = derivations; } } + + if (!m_original_keys_available) + { + // Save the original i.e. non-multisig keys so the MMS can continue to use them to encrypt and decrypt messages + // (making a wallet multisig overwrites those keys, see account_base::make_multisig) + m_original_address = m_account.get_keys().m_account_address; + m_original_view_secret_key = m_account.get_keys().m_view_secret_key; + m_original_keys_available = true; + } clear(); MINFO("Creating view key..."); @@ -4257,8 +4559,25 @@ std::string wallet2::make_multisig(const epee::wipeable_string &password, return make_multisig(password, secret_keys, public_keys, threshold); } -bool wallet2::finalize_multisig(const epee::wipeable_string &password, std::unordered_set<crypto::public_key> pkeys, std::vector<crypto::public_key> signers) +bool wallet2::finalize_multisig(const epee::wipeable_string &password, const std::unordered_set<crypto::public_key> &pkeys, std::vector<crypto::public_key> signers) { + bool ready; + uint32_t threshold, total; + if (!multisig(&ready, &threshold, &total)) + { + MERROR("This is not a multisig wallet"); + return false; + } + if (ready) + { + MERROR("This multisig wallet is already finalized"); + return false; + } + if (threshold + 1 != total) + { + MERROR("finalize_multisig should only be used for N-1/N wallets, use exchange_multisig_keys instead"); + return false; + } exchange_multisig_keys(password, pkeys, signers); return true; } @@ -4463,8 +4782,8 @@ void wallet2::write_watch_only_wallet(const std::string& wallet_name, const epee //---------------------------------------------------------------------------------------------------- void wallet2::wallet_exists(const std::string& file_path, bool& keys_file_exists, bool& wallet_file_exists) { - std::string keys_file, wallet_file; - do_prepare_file_names(file_path, keys_file, wallet_file); + std::string keys_file, wallet_file, mms_file; + do_prepare_file_names(file_path, keys_file, wallet_file, mms_file); boost::system::error_code ignore; keys_file_exists = boost::filesystem::exists(keys_file, ignore); @@ -4518,7 +4837,7 @@ bool wallet2::parse_payment_id(const std::string& payment_id_str, crypto::hash& //---------------------------------------------------------------------------------------------------- bool wallet2::prepare_file_names(const std::string& file_path) { - do_prepare_file_names(file_path, m_keys_file, m_wallet_file); + do_prepare_file_names(file_path, m_keys_file, m_wallet_file, m_mms_file); return true; } //---------------------------------------------------------------------------------------------------- @@ -4711,6 +5030,8 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass { MERROR("Failed to save rings, will try again next time"); } + + m_message_store.read_from_file(get_multisig_wallet_state(), m_mms_file); } //---------------------------------------------------------------------------------------------------- void wallet2::trim_hashchain() @@ -4816,6 +5137,7 @@ void wallet2::store_to(const std::string &path, const epee::wipeable_string &pas const std::string old_file = m_wallet_file; const std::string old_keys_file = m_keys_file; const std::string old_address_file = m_wallet_file + ".address.txt"; + const std::string old_mms_file = m_mms_file; // save keys to the new file // if we here, main wallet file is saved and we only need to save keys and address files @@ -4845,6 +5167,14 @@ void wallet2::store_to(const std::string &path, const epee::wipeable_string &pas if (!r) { LOG_ERROR("error removing file: " << old_address_file); } + // remove old message store file + if (boost::filesystem::exists(old_mms_file)) + { + r = boost::filesystem::remove(old_mms_file); + if (!r) { + LOG_ERROR("error removing file: " << old_mms_file); + } + } } else { // save to new file #ifdef WIN32 @@ -4870,6 +5200,14 @@ void wallet2::store_to(const std::string &path, const epee::wipeable_string &pas std::error_code e = tools::replace_file(new_file, m_wallet_file); THROW_WALLET_EXCEPTION_IF(e, error::file_save_error, m_wallet_file, e); } + + if (m_message_store.get_active()) + { + // While the "m_message_store" object of course always exist, a file for the message + // store should only exist if the MMS is really active + m_message_store.write_to_file(get_multisig_wallet_state(), m_mms_file); + } + } //---------------------------------------------------------------------------------------------------- uint64_t wallet2::balance(uint32_t index_major) const @@ -5039,7 +5377,7 @@ void wallet2::rescan_spent() m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "is_key_image_spent"); THROW_WALLET_EXCEPTION_IF(daemon_resp.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "is_key_image_spent"); - THROW_WALLET_EXCEPTION_IF(daemon_resp.status != CORE_RPC_STATUS_OK, error::is_key_image_spent_error, daemon_resp.status); + THROW_WALLET_EXCEPTION_IF(daemon_resp.status != CORE_RPC_STATUS_OK, error::is_key_image_spent_error, get_rpc_status(daemon_resp.status)); THROW_WALLET_EXCEPTION_IF(daemon_resp.spent_status.size() != n_outputs, error::wallet_internal_error, "daemon returned wrong response for is_key_image_spent, wrong amounts count = " + std::to_string(daemon_resp.spent_status.size()) + ", expected " + std::to_string(n_outputs)); @@ -5071,11 +5409,31 @@ void wallet2::rescan_spent() } } //---------------------------------------------------------------------------------------------------- -void wallet2::rescan_blockchain(bool refresh) +void wallet2::rescan_blockchain(bool hard, bool refresh) { - clear(); - - setup_new_blockchain(); + if(hard) + { + clear(); + setup_new_blockchain(); + } + else + { + m_blockchain.clear(); + m_transfers.clear(); + m_key_images.clear(); + m_pub_keys.clear(); + m_unconfirmed_txs.clear(); + m_payments.clear(); + m_confirmed_txs.clear(); + m_unconfirmed_payments.clear(); + m_scanned_pool_txs[0].clear(); + m_scanned_pool_txs[1].clear(); + + cryptonote::block b; + generate_genesis(b); + m_blockchain.push_back(get_block_hash(b)); + m_last_block_reward = cryptonote::get_outs_money_amount(b.miner_tx); + } if (refresh) this->refresh(false); @@ -5352,7 +5710,7 @@ void wallet2::commit_tx(pending_tx& ptx) m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "submit_raw_tx"); // MyMonero and OpenMonero use different status strings - THROW_WALLET_EXCEPTION_IF(ores.status != "OK" && ores.status != "success" , error::tx_rejected, ptx.tx, ores.status, ores.error); + THROW_WALLET_EXCEPTION_IF(ores.status != "OK" && ores.status != "success" , error::tx_rejected, ptx.tx, get_rpc_status(ores.status), ores.error); } else { @@ -5366,7 +5724,7 @@ void wallet2::commit_tx(pending_tx& ptx) m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "sendrawtransaction"); THROW_WALLET_EXCEPTION_IF(daemon_send_resp.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "sendrawtransaction"); - THROW_WALLET_EXCEPTION_IF(daemon_send_resp.status != CORE_RPC_STATUS_OK, error::tx_rejected, ptx.tx, daemon_send_resp.status, daemon_send_resp.reason); + THROW_WALLET_EXCEPTION_IF(daemon_send_resp.status != CORE_RPC_STATUS_OK, error::tx_rejected, ptx.tx, get_rpc_status(daemon_send_resp.status), get_text_reason(daemon_send_resp)); // sanity checks for (size_t idx: ptx.selected_transfers) { @@ -5442,7 +5800,7 @@ std::string wallet2::dump_tx_to_str(const std::vector<pending_tx> &ptx_vector) c txs.txes.push_back(get_construction_data_with_decrypted_short_payment_id(tx, m_account.get_device())); } - txs.transfers = m_transfers; + txs.transfers = export_outputs(); // save as binary std::ostringstream oss; boost::archive::portable_binary_oarchive ar(oss); @@ -5563,15 +5921,16 @@ bool wallet2::sign_tx(unsigned_tx_set &exported_txs, std::vector<wallet2::pendin LOG_PRINT_L1(" " << (n+1) << ": " << sd.sources.size() << " inputs, ring size " << sd.sources[0].outputs.size()); signed_txes.ptx.push_back(pending_tx()); tools::wallet2::pending_tx &ptx = signed_txes.ptx.back(); - rct::RangeProofType range_proof_type = rct::RangeProofBorromean; + rct::RCTConfig rct_config = { rct::RangeProofBorromean, 0 }; if (sd.use_bulletproofs) { - range_proof_type = rct::RangeProofPaddedBulletproof; + rct_config.range_proof_type = rct::RangeProofPaddedBulletproof; + rct_config.bp_version = use_fork_rules(HF_VERSION_SMALLER_BP, -10) ? 2 : 1; } crypto::secret_key tx_key; std::vector<crypto::secret_key> additional_tx_keys; rct::multisig_out msout; - bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sd.sources, sd.splitted_dsts, sd.change_dts.addr, sd.extra, ptx.tx, sd.unlock_time, tx_key, additional_tx_keys, sd.use_rct, range_proof_type, m_multisig ? &msout : NULL); + bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sd.sources, sd.splitted_dsts, sd.change_dts.addr, sd.extra, ptx.tx, sd.unlock_time, tx_key, additional_tx_keys, sd.use_rct, rct_config, m_multisig ? &msout : NULL); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sd.sources, sd.splitted_dsts, sd.unlock_time, m_nettype); // we don't test tx size, because we don't know the current limit, due to not having a blockchain, // and it's a bit pointless to fail there anyway, since it'd be a (good) guess only. We sign anyway, @@ -5610,6 +5969,65 @@ bool wallet2::sign_tx(unsigned_tx_set &exported_txs, std::vector<wallet2::pendin ptx.construction_data = sd; txs.push_back(ptx); + + // add tx keys only to ptx + txs.back().tx_key = tx_key; + txs.back().additional_tx_keys = additional_tx_keys; + } + + // add key image mapping for these txes + const account_keys &keys = get_account().get_keys(); + hw::device &hwdev = m_account.get_device(); + for (size_t n = 0; n < exported_txs.txes.size(); ++n) + { + const cryptonote::transaction &tx = signed_txes.ptx[n].tx; + + crypto::key_derivation derivation; + std::vector<crypto::key_derivation> additional_derivations; + + // compute public keys from out secret keys + crypto::public_key tx_pub_key; + crypto::secret_key_to_public_key(txs[n].tx_key, tx_pub_key); + std::vector<crypto::public_key> additional_tx_pub_keys; + for (const crypto::secret_key &skey: txs[n].additional_tx_keys) + { + additional_tx_pub_keys.resize(additional_tx_pub_keys.size() + 1); + crypto::secret_key_to_public_key(skey, additional_tx_pub_keys.back()); + } + + // compute derivations + hwdev.set_mode(hw::device::TRANSACTION_PARSE); + if (!hwdev.generate_key_derivation(tx_pub_key, keys.m_view_secret_key, derivation)) + { + MWARNING("Failed to generate key derivation from tx pubkey in " << cryptonote::get_transaction_hash(tx) << ", skipping"); + static_assert(sizeof(derivation) == sizeof(rct::key), "Mismatched sizes of key_derivation and rct::key"); + memcpy(&derivation, rct::identity().bytes, sizeof(derivation)); + } + for (size_t i = 0; i < additional_tx_pub_keys.size(); ++i) + { + additional_derivations.push_back({}); + if (!hwdev.generate_key_derivation(additional_tx_pub_keys[i], keys.m_view_secret_key, additional_derivations.back())) + { + MWARNING("Failed to generate key derivation from additional tx pubkey in " << cryptonote::get_transaction_hash(tx) << ", skipping"); + memcpy(&additional_derivations.back(), rct::identity().bytes, sizeof(crypto::key_derivation)); + } + } + + for (size_t i = 0; i < tx.vout.size(); ++i) + { + if (tx.vout[i].target.type() != typeid(cryptonote::txout_to_key)) + continue; + const cryptonote::txout_to_key &out = boost::get<cryptonote::txout_to_key>(tx.vout[i].target); + // if this output is back to this wallet, we can calculate its key image already + if (!is_out_to_acc_precomp(m_subaddresses, out.key, derivation, additional_derivations, i, hwdev)) + continue; + crypto::key_image ki; + cryptonote::keypair in_ephemeral; + if (generate_key_image_helper(keys, m_subaddresses, out.key, tx_pub_key, additional_tx_pub_keys, i, in_ephemeral, ki, hwdev)) + signed_txes.tx_key_images[out.key] = ki; + else + MERROR("Failed to calculate key image"); + } } // add key images @@ -5771,22 +6189,12 @@ bool wallet2::parse_tx_from_str(const std::string &signed_tx_st, std::vector<too } // import key images - if (signed_txs.key_images.size() > m_transfers.size()) - { - LOG_PRINT_L1("More key images returned that we know outputs for"); - return false; - } - for (size_t i = 0; i < signed_txs.key_images.size(); ++i) - { - transfer_details &td = m_transfers[i]; - if (td.m_key_image_known && !td.m_key_image_partial && td.m_key_image != signed_txs.key_images[i]) - LOG_PRINT_L0("WARNING: imported key image differs from previously known key image at index " << i << ": trusting imported one"); - td.m_key_image = signed_txs.key_images[i]; - m_key_images[m_transfers[i].m_key_image] = i; - td.m_key_image_known = true; - td.m_key_image_partial = false; - m_pub_keys[m_transfers[i].get_public_key()] = i; - } + bool r = import_key_images(signed_txs.key_images); + if (!r) return false; + + // remember key images for this tx, for when we get those txes from the blockchain + for (const auto &e: signed_txs.tx_key_images) + m_cold_key_images.insert(e); ptx = signed_txs.ptx; @@ -5984,15 +6392,13 @@ bool wallet2::sign_multisig_tx(multisig_tx_set &exported_txs, std::vector<crypto cryptonote::transaction tx; rct::multisig_out msout = ptx.multisig_sigs.front().msout; auto sources = sd.sources; - rct::RangeProofType range_proof_type = rct::RangeProofBorromean; + rct::RCTConfig rct_config = { rct::RangeProofBorromean, 0 }; if (sd.use_bulletproofs) { - range_proof_type = rct::RangeProofBulletproof; - for (const rct::Bulletproof &proof: ptx.tx.rct_signatures.p.bulletproofs) - if (proof.V.size() > 1) - range_proof_type = rct::RangeProofPaddedBulletproof; + rct_config.range_proof_type = rct::RangeProofPaddedBulletproof; + rct_config.bp_version = use_fork_rules(HF_VERSION_SMALLER_BP, -10) ? 2 : 1; } - bool r = cryptonote::construct_tx_with_tx_key(m_account.get_keys(), m_subaddresses, sources, sd.splitted_dsts, ptx.change_dts.addr, sd.extra, tx, sd.unlock_time, ptx.tx_key, ptx.additional_tx_keys, sd.use_rct, range_proof_type, &msout, false); + bool r = cryptonote::construct_tx_with_tx_key(m_account.get_keys(), m_subaddresses, sources, sd.splitted_dsts, ptx.change_dts.addr, sd.extra, tx, sd.unlock_time, ptx.tx_key, ptx.additional_tx_keys, sd.use_rct, rct_config, &msout, false); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sd.sources, sd.splitted_dsts, sd.unlock_time, m_nettype); THROW_WALLET_EXCEPTION_IF(get_transaction_prefix_hash (tx) != get_transaction_prefix_hash(ptx.tx), @@ -6005,7 +6411,7 @@ bool wallet2::sign_multisig_tx(multisig_tx_set &exported_txs, std::vector<crypto for (auto &sig: ptx.multisig_sigs) { - if (sig.ignore != local_signer) + if (sig.ignore.find(local_signer) == sig.ignore.end()) { ptx.tx.rct_signatures = sig.sigs; @@ -6039,7 +6445,7 @@ bool wallet2::sign_multisig_tx(multisig_tx_set &exported_txs, std::vector<crypto bool found = false; for (const auto &sig: ptx.multisig_sigs) { - if (sig.ignore != local_signer && exported_txs.m_signers.find(sig.ignore) == exported_txs.m_signers.end()) + if (sig.ignore.find(local_signer) == sig.ignore.end() && !keys_intersect(sig.ignore, exported_txs.m_signers)) { THROW_WALLET_EXCEPTION_IF(found, error::wallet_internal_error, "More than one transaction is final"); ptx.tx.rct_signatures = sig.sigs; @@ -6270,7 +6676,7 @@ uint32_t wallet2::adjust_priority(uint32_t priority) m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "getblockheadersrange"); THROW_WALLET_EXCEPTION_IF(getbh_res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "getblockheadersrange"); - THROW_WALLET_EXCEPTION_IF(getbh_res.status != CORE_RPC_STATUS_OK, error::get_blocks_error, getbh_res.status); + THROW_WALLET_EXCEPTION_IF(getbh_res.status != CORE_RPC_STATUS_OK, error::get_blocks_error, get_rpc_status(getbh_res.status)); if (getbh_res.headers.size() != N) { MERROR("Bad blockheaders size"); @@ -6336,6 +6742,19 @@ crypto::chacha_key wallet2::get_ringdb_key() return *m_ringdb_key; } +void wallet2::register_devices(){ + hw::trezor::register_all(); +} + +hw::device& wallet2::lookup_device(const std::string & device_descriptor){ + if (!m_devices_registered){ + m_devices_registered = true; + register_devices(); + } + + return hw::get_device(device_descriptor); +} + bool wallet2::add_rings(const crypto::chacha_key &key, const cryptonote::transaction_prefix &tx) { if (!m_ringdb) @@ -6429,11 +6848,12 @@ bool wallet2::find_and_save_rings(bool force) MDEBUG("Found " << std::to_string(txs_hashes.size()) << " transactions"); // get those transactions from the daemon + auto it = txs_hashes.begin(); static const size_t SLICE_SIZE = 200; for (size_t slice = 0; slice < txs_hashes.size(); slice += SLICE_SIZE) { req.decode_as_json = false; - req.prune = false; + req.prune = true; req.txs_hashes.clear(); size_t ntxes = slice + SLICE_SIZE > txs_hashes.size() ? txs_hashes.size() - slice : SLICE_SIZE; for (size_t s = slice; s < slice + ntxes; ++s) @@ -6452,19 +6872,15 @@ bool wallet2::find_and_save_rings(bool force) MDEBUG("Scanning " << res.txs.size() << " transactions"); THROW_WALLET_EXCEPTION_IF(slice + res.txs.size() > txs_hashes.size(), error::wallet_internal_error, "Unexpected tx array size"); - auto it = req.txs_hashes.begin(); for (size_t i = 0; i < res.txs.size(); ++i, ++it) { const auto &tx_info = res.txs[i]; - THROW_WALLET_EXCEPTION_IF(tx_info.tx_hash != epee::string_tools::pod_to_hex(txs_hashes[slice + i]), error::wallet_internal_error, "Wrong txid received"); - THROW_WALLET_EXCEPTION_IF(tx_info.tx_hash != *it, error::wallet_internal_error, "Wrong txid received"); - cryptonote::blobdata bd; - THROW_WALLET_EXCEPTION_IF(!epee::string_tools::parse_hexstr_to_binbuff(tx_info.as_hex, bd), error::wallet_internal_error, "failed to parse tx from hexstr"); - cryptonote::transaction tx; - crypto::hash tx_hash, tx_prefix_hash; - THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(bd, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, "failed to parse tx from blob"); - THROW_WALLET_EXCEPTION_IF(epee::string_tools::pod_to_hex(tx_hash) != tx_info.tx_hash, error::wallet_internal_error, "txid mismatch"); - THROW_WALLET_EXCEPTION_IF(!add_rings(get_ringdb_key(), tx), error::wallet_internal_error, "Failed to save ring"); + cryptonote::transaction tx; + crypto::hash tx_hash; + THROW_WALLET_EXCEPTION_IF(!get_pruned_tx(tx_info, tx, tx_hash), error::wallet_internal_error, + "Failed to get transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(!(tx_hash == *it), error::wallet_internal_error, "Wrong txid received"); + THROW_WALLET_EXCEPTION_IF(!add_rings(get_ringdb_key(), tx), error::wallet_internal_error, "Failed to save ring"); } } @@ -6683,15 +7099,21 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> uint64_t rct_start_height; std::vector<uint64_t> rct_offsets; bool has_rct = false; + uint64_t max_rct_index = 0; for (size_t idx: selected_transfers) if (m_transfers[idx].is_rct()) - { has_rct = true; break; } + { + has_rct = true; + max_rct_index = std::max(max_rct_index, m_transfers[idx].m_global_output_index); + } const bool has_rct_distribution = has_rct && get_rct_distribution(rct_start_height, rct_offsets); if (has_rct_distribution) { // check we're clear enough of rct start, to avoid corner cases below THROW_WALLET_EXCEPTION_IF(rct_offsets.size() <= CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE, error::get_output_distribution, "Not enough rct outputs"); + THROW_WALLET_EXCEPTION_IF(rct_offsets.back() <= max_rct_index, + error::get_output_distribution, "Daemon reports suspicious number of rct outputs"); } // get histogram for the amounts we need @@ -6713,7 +7135,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "transfer_selected"); THROW_WALLET_EXCEPTION_IF(resp_t.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_output_histogram"); - THROW_WALLET_EXCEPTION_IF(resp_t.status != CORE_RPC_STATUS_OK, error::get_histogram_error, resp_t.status); + THROW_WALLET_EXCEPTION_IF(resp_t.status != CORE_RPC_STATUS_OK, error::get_histogram_error, get_rpc_status(resp_t.status)); } // if we want to segregate fake outs pre or post fork, get distribution @@ -6736,7 +7158,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "transfer_selected"); THROW_WALLET_EXCEPTION_IF(resp_t.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_output_distribution"); - THROW_WALLET_EXCEPTION_IF(resp_t.status != CORE_RPC_STATUS_OK, error::get_output_distribution, resp_t.status); + THROW_WALLET_EXCEPTION_IF(resp_t.status != CORE_RPC_STATUS_OK, error::get_output_distribution, get_rpc_status(resp_t.status)); // check we got all data for(size_t idx: selected_transfers) @@ -6747,13 +7169,13 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> { if (d.amount == amount) { - THROW_WALLET_EXCEPTION_IF(d.start_height > segregation_fork_height, error::get_output_distribution, "Distribution start_height too high"); - THROW_WALLET_EXCEPTION_IF(segregation_fork_height - d.start_height >= d.distribution.size(), error::get_output_distribution, "Distribution size too small"); - THROW_WALLET_EXCEPTION_IF(segregation_fork_height - RECENT_OUTPUT_BLOCKS - d.start_height >= d.distribution.size(), error::get_output_distribution, "Distribution size too small"); + THROW_WALLET_EXCEPTION_IF(d.data.start_height > segregation_fork_height, error::get_output_distribution, "Distribution start_height too high"); + THROW_WALLET_EXCEPTION_IF(segregation_fork_height - d.data.start_height >= d.data.distribution.size(), error::get_output_distribution, "Distribution size too small"); + THROW_WALLET_EXCEPTION_IF(segregation_fork_height - RECENT_OUTPUT_BLOCKS - d.data.start_height >= d.data.distribution.size(), error::get_output_distribution, "Distribution size too small"); THROW_WALLET_EXCEPTION_IF(segregation_fork_height <= RECENT_OUTPUT_BLOCKS, error::wallet_internal_error, "Fork height too low"); - THROW_WALLET_EXCEPTION_IF(segregation_fork_height - RECENT_OUTPUT_BLOCKS < d.start_height, error::get_output_distribution, "Bad start height"); - uint64_t till_fork = d.distribution[segregation_fork_height - d.start_height]; - uint64_t recent = till_fork - d.distribution[segregation_fork_height - RECENT_OUTPUT_BLOCKS - d.start_height]; + THROW_WALLET_EXCEPTION_IF(segregation_fork_height - RECENT_OUTPUT_BLOCKS < d.data.start_height, error::get_output_distribution, "Bad start height"); + uint64_t till_fork = d.data.distribution[segregation_fork_height - d.data.start_height]; + uint64_t recent = till_fork - d.data.distribution[segregation_fork_height - RECENT_OUTPUT_BLOCKS - d.data.start_height]; segregation_limit[amount] = std::make_pair(till_fork, recent); found = true; break; @@ -6782,23 +7204,49 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> //static const double shape = m_testnet ? 17.02 : 17.28; static const double scale = 1/1.61; std::gamma_distribution<double> gamma(shape, scale); + THROW_WALLET_EXCEPTION_IF(rct_offsets.size() <= CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE, error::wallet_internal_error, "Bad offset calculation"); + uint64_t last_usable_block = rct_offsets.size() - 1; auto pick_gamma = [&]() { double x = gamma(engine); x = exp(x); uint64_t block_offset = x / DIFFICULTY_TARGET_V2; // this assumes constant target over the whole rct range - if (block_offset >= rct_offsets.size() - 1) + if (block_offset > last_usable_block - CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE) return std::numeric_limits<uint64_t>::max(); // bad pick - block_offset = rct_offsets.size() - 2 - block_offset; - THROW_WALLET_EXCEPTION_IF(block_offset >= rct_offsets.size() - 1, error::wallet_internal_error, "Bad offset calculation"); - THROW_WALLET_EXCEPTION_IF(rct_offsets[block_offset + 1] < rct_offsets[block_offset], + block_offset = last_usable_block - block_offset; + THROW_WALLET_EXCEPTION_IF(block_offset > last_usable_block, error::wallet_internal_error, "Bad offset calculation"); + THROW_WALLET_EXCEPTION_IF(block_offset > 0 && rct_offsets[block_offset] < rct_offsets[block_offset - 1], error::get_output_distribution, "Decreasing offsets in rct distribution: " + - std::to_string(block_offset) + ": " + std::to_string(rct_offsets[block_offset]) + ", " + - std::to_string(block_offset + 1) + ": " + std::to_string(rct_offsets[block_offset + 1])); - uint64_t n_rct = rct_offsets[block_offset + 1] - rct_offsets[block_offset]; + std::to_string(block_offset - 1) + ": " + std::to_string(rct_offsets[block_offset - 1]) + ", " + + std::to_string(block_offset) + ": " + std::to_string(rct_offsets[block_offset])); + uint64_t first_block_offset = block_offset, last_block_offset = block_offset; + for (size_t half_window = 0; half_window <= GAMMA_PICK_HALF_WINDOW; ++half_window) + { + // end when we have a non empty block + uint64_t cum0 = first_block_offset > 0 ? rct_offsets[first_block_offset] - rct_offsets[first_block_offset - 1] : rct_offsets[0]; + if (cum0 > 1) + break; + uint64_t cum1 = last_block_offset > 0 ? rct_offsets[last_block_offset] - rct_offsets[last_block_offset - 1] : rct_offsets[0]; + if (cum1 > 1) + break; + if (first_block_offset == 0 && last_block_offset >= last_usable_block) + break; + // expand up to bounds + if (first_block_offset > 0) + --first_block_offset; + else + return std::numeric_limits<uint64_t>::max(); // bad pick + if (last_block_offset < last_usable_block - CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE) + ++last_block_offset; + else + return std::numeric_limits<uint64_t>::max(); // bad pick + } + const uint64_t first_rct = first_block_offset == 0 ? 0 : rct_offsets[first_block_offset - 1]; + const uint64_t n_rct = rct_offsets[last_block_offset] - first_rct; if (n_rct == 0) return rct_offsets[block_offset] ? rct_offsets[block_offset] - 1 : 0; - return rct_offsets[block_offset] + crypto::rand<uint64_t>() % n_rct; + MDEBUG("Picking 1/" << n_rct << " in " << (last_block_offset - first_block_offset + 1) << " blocks centered around " << block_offset + rct_start_height); + return first_rct + crypto::rand<uint64_t>() % n_rct; }; size_t num_selected_transfers = 0; @@ -7103,12 +7551,13 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> } // get the keys for those + req.get_txid = false; m_daemon_rpc_mutex.lock(); bool r = epee::net_utils::invoke_http_bin("/get_outs.bin", req, daemon_resp, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "get_outs.bin"); THROW_WALLET_EXCEPTION_IF(daemon_resp.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_outs.bin"); - THROW_WALLET_EXCEPTION_IF(daemon_resp.status != CORE_RPC_STATUS_OK, error::get_outs_error, daemon_resp.status); + THROW_WALLET_EXCEPTION_IF(daemon_resp.status != CORE_RPC_STATUS_OK, error::get_outs_error, get_rpc_status(daemon_resp.status)); THROW_WALLET_EXCEPTION_IF(daemon_resp.outs.size() != req.outputs.size(), error::wallet_internal_error, "daemon returned wrong response for get_outs.bin, wrong amounts count = " + std::to_string(daemon_resp.outs.size()) + ", expected " + std::to_string(req.outputs.size())); @@ -7137,6 +7586,9 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> break; } } + bool use_histogram = amount != 0 || !has_rct_distribution; + if (!use_histogram) + num_outs = rct_offsets[rct_offsets.size() - CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE]; // make sure the real outputs we asked for are really included, along // with the correct key and mask: this guards against an active attack @@ -7229,6 +7681,20 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> outs.push_back(v); } } + + // save those outs in the ringdb for reuse + for (size_t i = 0; i < selected_transfers.size(); ++i) + { + const size_t idx = selected_transfers[i]; + THROW_WALLET_EXCEPTION_IF(idx >= m_transfers.size(), error::wallet_internal_error, "selected_transfers entry out of range"); + const transfer_details &td = m_transfers[idx]; + std::vector<uint64_t> ring; + ring.reserve(outs[i].size()); + for (const auto &e: outs[i]) + ring.push_back(std::get<0>(e)); + if (!set_ring(td.m_key_image, ring, false)) + MERROR("Failed to set ring for " << td.m_key_image); + } } template<typename T> @@ -7344,7 +7810,7 @@ void wallet2::transfer_selected(const std::vector<cryptonote::tx_destination_ent std::vector<crypto::secret_key> additional_tx_keys; rct::multisig_out msout; LOG_PRINT_L2("constructing tx"); - bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sources, splitted_dsts, change_dts.addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, rct::RangeProofBulletproof, m_multisig ? &msout : NULL); + bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sources, splitted_dsts, change_dts.addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, {}, m_multisig ? &msout : NULL); LOG_PRINT_L2("constructed tx, r="<<r); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sources, splitted_dsts, unlock_time, m_nettype); THROW_WALLET_EXCEPTION_IF(upper_transaction_weight_limit <= get_transaction_weight(tx), error::tx_too_big, tx, upper_transaction_weight_limit); @@ -7393,7 +7859,7 @@ void wallet2::transfer_selected(const std::vector<cryptonote::tx_destination_ent void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry> dsts, const std::vector<size_t>& selected_transfers, size_t fake_outputs_count, std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, - uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx &ptx, rct::RangeProofType range_proof_type) + uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx &ptx, const rct::RCTConfig &rct_config) { using namespace cryptonote; // throw if attempting a transaction with no destinations @@ -7417,30 +7883,56 @@ void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry // if this is a multisig wallet, create a list of multisig signers we can use std::deque<crypto::public_key> multisig_signers; size_t n_multisig_txes = 0; + std::vector<std::unordered_set<crypto::public_key>> ignore_sets; if (m_multisig && !m_transfers.empty()) { const crypto::public_key local_signer = get_multisig_signer_public_key(); size_t n_available_signers = 1; + + // At this step we need to define set of participants available for signature, + // i.e. those of them who exchanged with multisig info's for (const crypto::public_key &signer: m_multisig_signers) { if (signer == local_signer) continue; - multisig_signers.push_front(signer); for (const auto &i: m_transfers[0].m_multisig_info) { if (i.m_signer == signer) { - multisig_signers.pop_front(); multisig_signers.push_back(signer); ++n_available_signers; break; } } } - multisig_signers.push_back(local_signer); + // n_available_signers includes the transaction creator, but multisig_signers doesn't MDEBUG("We can use " << n_available_signers << "/" << m_multisig_signers.size() << " other signers"); - THROW_WALLET_EXCEPTION_IF(n_available_signers+1 < m_multisig_threshold, error::multisig_import_needed); - n_multisig_txes = n_available_signers == m_multisig_signers.size() ? m_multisig_threshold : 1; + THROW_WALLET_EXCEPTION_IF(n_available_signers < m_multisig_threshold, error::multisig_import_needed); + if (n_available_signers > m_multisig_threshold) + { + // If there more potential signers (those who exchanged with multisig info) + // than threshold needed some of them should be skipped since we don't know + // who will sign tx and who won't. Hence we don't contribute their LR pairs to the signature. + + // We create as many transactions as many combinations of excluded signers may be. + // For example, if we have 2/4 wallet and wallets are: A, B, C and D. Let A be + // transaction creator, so we need just 1 signature from set of B, C, D. + // Using "excluding" logic here we have to exclude 2-of-3 wallets. Combinations go as follows: + // BC, BD, and CD. We save these sets to use later and counting the number of required txs. + tools::Combinator<crypto::public_key> c(std::vector<crypto::public_key>(multisig_signers.begin(), multisig_signers.end())); + auto ignore_combinations = c.combine(multisig_signers.size() + 1 - m_multisig_threshold); + for (const auto& combination: ignore_combinations) + { + ignore_sets.push_back(std::unordered_set<crypto::public_key>(combination.begin(), combination.end())); + } + + n_multisig_txes = ignore_sets.size(); + } + else + { + // If we have exact count of signers just to fit in threshold we don't exclude anyone and create 1 transaction + n_multisig_txes = 1; + } MDEBUG("We will create " << n_multisig_txes << " txes"); } @@ -7508,8 +8000,8 @@ void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry src.mask = td.m_mask; if (m_multisig) { - crypto::public_key ignore = m_multisig_threshold == m_multisig_signers.size() ? crypto::null_pkey : multisig_signers.front(); - src.multisig_kLRki = get_multisig_composite_kLRki(idx, ignore, used_L, used_L); + auto ignore_set = ignore_sets.empty() ? std::unordered_set<crypto::public_key>() : ignore_sets.front(); + src.multisig_kLRki = get_multisig_composite_kLRki(idx, ignore_set, used_L, used_L); } else src.multisig_kLRki = rct::multisig_kLRki({rct::zero(), rct::zero(), rct::zero(), rct::zero()}); @@ -7549,7 +8041,7 @@ void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry rct::multisig_out msout; LOG_PRINT_L2("constructing tx"); auto sources_copy = sources; - bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sources, splitted_dsts, change_dts.addr, extra, tx, unlock_time, tx_key, additional_tx_keys, true, range_proof_type, m_multisig ? &msout : NULL); + bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sources, splitted_dsts, change_dts.addr, extra, tx, unlock_time, tx_key, additional_tx_keys, true, rct_config, m_multisig ? &msout : NULL); LOG_PRINT_L2("constructed tx, r="<<r); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sources, dsts, unlock_time, m_nettype); THROW_WALLET_EXCEPTION_IF(upper_transaction_weight_limit <= get_transaction_weight(tx), error::tx_too_big, tx, upper_transaction_weight_limit); @@ -7571,7 +8063,7 @@ void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry std::vector<tools::wallet2::multisig_sig> multisig_sigs; if (m_multisig) { - crypto::public_key ignore = m_multisig_threshold == m_multisig_signers.size() ? crypto::null_pkey : multisig_signers.front(); + auto ignore = ignore_sets.empty() ? std::unordered_set<crypto::public_key>() : ignore_sets.front(); multisig_sigs.push_back({tx.rct_signatures, ignore, used_L, std::unordered_set<crypto::public_key>(), msout}); if (m_multisig_threshold < m_multisig_signers.size()) @@ -7579,7 +8071,7 @@ void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry const crypto::hash prefix_hash = cryptonote::get_transaction_prefix_hash(tx); // create the other versions, one for every other participant (the first one's already done above) - for (size_t signer_index = 1; signer_index < n_multisig_txes; ++signer_index) + for (size_t ignore_index = 1; ignore_index < ignore_sets.size(); ++ignore_index) { std::unordered_set<rct::key> new_used_L; size_t src_idx = 0; @@ -7587,19 +8079,19 @@ void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry for(size_t idx: selected_transfers) { cryptonote::tx_source_entry& src = sources_copy[src_idx]; - src.multisig_kLRki = get_multisig_composite_kLRki(idx, multisig_signers[signer_index], used_L, new_used_L); + src.multisig_kLRki = get_multisig_composite_kLRki(idx, ignore_sets[ignore_index], used_L, new_used_L); ++src_idx; } LOG_PRINT_L2("Creating supplementary multisig transaction"); cryptonote::transaction ms_tx; auto sources_copy_copy = sources_copy; - bool r = cryptonote::construct_tx_with_tx_key(m_account.get_keys(), m_subaddresses, sources_copy_copy, splitted_dsts, change_dts.addr, extra, ms_tx, unlock_time,tx_key, additional_tx_keys, true, range_proof_type, &msout, false); + bool r = cryptonote::construct_tx_with_tx_key(m_account.get_keys(), m_subaddresses, sources_copy_copy, splitted_dsts, change_dts.addr, extra, ms_tx, unlock_time,tx_key, additional_tx_keys, true, rct_config, &msout, false); LOG_PRINT_L2("constructed tx, r="<<r); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sources, splitted_dsts, unlock_time, m_nettype); THROW_WALLET_EXCEPTION_IF(upper_transaction_weight_limit <= get_transaction_weight(tx), error::tx_too_big, tx, upper_transaction_weight_limit); THROW_WALLET_EXCEPTION_IF(cryptonote::get_transaction_prefix_hash(ms_tx) != prefix_hash, error::wallet_internal_error, "Multisig txes do not share prefix"); - multisig_sigs.push_back({ms_tx.rct_signatures, multisig_signers[signer_index], new_used_L, std::unordered_set<crypto::public_key>(), msout}); + multisig_sigs.push_back({ms_tx.rct_signatures, ignore_sets[ignore_index], new_used_L, std::unordered_set<crypto::public_key>(), msout}); ms_tx.rct_signatures = tx.rct_signatures; THROW_WALLET_EXCEPTION_IF(cryptonote::get_transaction_hash(ms_tx) != cryptonote::get_transaction_hash(tx), error::wallet_internal_error, "Multisig txes differ by more than the signatures"); @@ -7900,6 +8392,7 @@ void wallet2::light_wallet_get_unspent_outs() td.m_key_image = unspent_key_image; td.m_key_image_known = !m_watch_only && !m_multisig; + td.m_key_image_request = false; td.m_key_image_partial = m_multisig; td.m_amount = o.amount; td.m_pk_index = 0; @@ -8004,7 +8497,6 @@ void wallet2::light_wallet_get_address_txs() // for balance calculation uint64_t wallet_total_sent = 0; - uint64_t wallet_total_unlocked_sent = 0; // txs in pool std::vector<crypto::hash> pool_txs; @@ -8271,15 +8763,16 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp TX() : weight(0), needed_fee(0) {} - void add(const account_public_address &addr, bool is_subaddress, uint64_t amount, unsigned int original_output_index, bool merge_destinations) { + void add(const cryptonote::tx_destination_entry &de, uint64_t amount, unsigned int original_output_index, bool merge_destinations) { if (merge_destinations) { std::vector<cryptonote::tx_destination_entry>::iterator i; - i = std::find_if(dsts.begin(), dsts.end(), [&](const cryptonote::tx_destination_entry &d) { return !memcmp (&d.addr, &addr, sizeof(addr)); }); + i = std::find_if(dsts.begin(), dsts.end(), [&](const cryptonote::tx_destination_entry &d) { return !memcmp (&d.addr, &de.addr, sizeof(de.addr)); }); if (i == dsts.end()) { - dsts.push_back(tx_destination_entry(0,addr,is_subaddress)); + dsts.push_back(de); i = dsts.end() - 1; + i->amount = 0; } i->amount += amount; } @@ -8288,8 +8781,11 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp THROW_WALLET_EXCEPTION_IF(original_output_index > dsts.size(), error::wallet_internal_error, std::string("original_output_index too large: ") + std::to_string(original_output_index) + " > " + std::to_string(dsts.size())); if (original_output_index == dsts.size()) - dsts.push_back(tx_destination_entry(0,addr,is_subaddress)); - THROW_WALLET_EXCEPTION_IF(memcmp(&dsts[original_output_index].addr, &addr, sizeof(addr)), error::wallet_internal_error, "Mismatched destination address"); + { + dsts.push_back(de); + dsts.back().amount = 0; + } + THROW_WALLET_EXCEPTION_IF(memcmp(&dsts[original_output_index].addr, &de.addr, sizeof(de.addr)), error::wallet_internal_error, "Mismatched destination address"); dsts[original_output_index].amount += amount; } } @@ -8301,7 +8797,10 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp const bool use_per_byte_fee = use_fork_rules(HF_VERSION_PER_BYTE_FEE, 0); const bool use_rct = use_fork_rules(4, 0); const bool bulletproof = use_fork_rules(get_bulletproof_fork(), 0); - const rct::RangeProofType range_proof_type = bulletproof ? rct::RangeProofPaddedBulletproof : rct::RangeProofBorromean; + const rct::RCTConfig rct_config { + bulletproof ? rct::RangeProofPaddedBulletproof : rct::RangeProofBorromean, + bulletproof ? (use_fork_rules(HF_VERSION_SMALLER_BP, -10) ? 2 : 1) : 0 + }; const uint64_t base_fee = get_base_fee(); const uint64_t fee_multiplier = get_fee_multiplier(priority, get_fee_algorithm()); @@ -8336,7 +8835,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp // early out if we know we can't make it anyway // we could also check for being within FEE_PER_KB, but if the fee calculation // ever changes, this might be missed, so let this go through - const uint64_t min_fee = (fee_multiplier * base_fee * estimate_tx_size(use_rct, 1, fake_outs_count, 2, extra.size(), bulletproof)) / 1024; + const uint64_t min_fee = (fee_multiplier * base_fee * estimate_tx_size(use_rct, 1, fake_outs_count, 2, extra.size(), bulletproof)); uint64_t balance_subtotal = 0; uint64_t unlocked_balance_subtotal = 0; for (uint32_t index_minor : subaddr_indices) @@ -8404,12 +8903,8 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp } } - // shuffle & sort output indices + // sort output indices { - std::random_device rd; - std::mt19937 g(rd()); - std::shuffle(unused_transfers_indices_per_subaddr.begin(), unused_transfers_indices_per_subaddr.end(), g); - std::shuffle(unused_dust_indices_per_subaddr.begin(), unused_dust_indices_per_subaddr.end(), g); auto sort_predicate = [&unlocked_balance_per_subaddr] (const std::pair<uint32_t, std::vector<size_t>>& x, const std::pair<uint32_t, std::vector<size_t>>& y) { return unlocked_balance_per_subaddr[x.first] > unlocked_balance_per_subaddr[y.first]; @@ -8561,7 +9056,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp // we can fully pay that destination LOG_PRINT_L2("We can fully pay " << get_account_address_as_str(m_nettype, dsts[0].is_subaddress, dsts[0].addr) << " for " << print_money(dsts[0].amount)); - tx.add(dsts[0].addr, dsts[0].is_subaddress, dsts[0].amount, original_output_index, m_merge_destinations); + tx.add(dsts[0], dsts[0].amount, original_output_index, m_merge_destinations); available_amount -= dsts[0].amount; dsts[0].amount = 0; pop_index(dsts, 0); @@ -8572,7 +9067,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp // we can partially fill that destination LOG_PRINT_L2("We can partially pay " << get_account_address_as_str(m_nettype, dsts[0].is_subaddress, dsts[0].addr) << " for " << print_money(available_amount) << "/" << print_money(dsts[0].amount)); - tx.add(dsts[0].addr, dsts[0].is_subaddress, available_amount, original_output_index, m_merge_destinations); + tx.add(dsts[0], available_amount, original_output_index, m_merge_destinations); dsts[0].amount -= available_amount; available_amount = 0; } @@ -8601,7 +9096,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp cryptonote::transaction test_tx; pending_tx test_ptx; - needed_fee = estimate_fee(use_per_byte_fee, use_rct ,tx.selected_transfers.size(), fake_outs_count, tx.dsts.size()+1, extra.size(), bulletproof, base_fee, fee_multiplier, fee_multiplier); + needed_fee = estimate_fee(use_per_byte_fee, use_rct ,tx.selected_transfers.size(), fake_outs_count, tx.dsts.size()+1, extra.size(), bulletproof, base_fee, fee_multiplier, fee_quantization_mask); uint64_t inputs = 0, outputs = needed_fee; for (size_t idx: tx.selected_transfers) inputs += m_transfers[idx].amount(); @@ -8618,7 +9113,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp tx.selected_transfers.size() << " inputs"); if (use_rct) transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, - test_tx, test_ptx, range_proof_type); + test_tx, test_ptx, rct_config); else transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); @@ -8661,7 +9156,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp while (needed_fee > test_ptx.fee) { if (use_rct) transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, - test_tx, test_ptx, range_proof_type); + test_tx, test_ptx, rct_config); else transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); @@ -8734,7 +9229,7 @@ skip_tx: extra, /* const std::vector<uint8_t>& extra, */ test_tx, /* OUT cryptonote::transaction& tx, */ test_ptx, /* OUT cryptonote::transaction& tx, */ - range_proof_type); + rct_config); } else { transfer_selected(tx.dsts, tx.selected_transfers, @@ -8874,7 +9369,10 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton const bool use_per_byte_fee = use_fork_rules(HF_VERSION_PER_BYTE_FEE); const bool use_rct = fake_outs_count > 0 && use_fork_rules(4, 0); const bool bulletproof = use_fork_rules(get_bulletproof_fork(), 0); - const rct::RangeProofType range_proof_type = bulletproof ? rct::RangeProofPaddedBulletproof : rct::RangeProofBorromean; + const rct::RCTConfig rct_config { + bulletproof ? rct::RangeProofPaddedBulletproof : rct::RangeProofBorromean, + bulletproof ? (use_fork_rules(HF_VERSION_SMALLER_BP, -10) ? 2 : 1) : 0, + }; const uint64_t base_fee = get_base_fee(); const uint64_t fee_multiplier = get_fee_multiplier(priority, get_fee_algorithm()); const uint64_t fee_quantization_mask = get_fee_quantization_mask(); @@ -8950,7 +9448,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton tx.selected_transfers.size() << " outputs"); if (use_rct) transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, - test_tx, test_ptx, range_proof_type); + test_tx, test_ptx, rct_config); else transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); @@ -8987,7 +9485,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton } if (use_rct) transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, - test_tx, test_ptx, range_proof_type); + test_tx, test_ptx, rct_config); else transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); @@ -9026,7 +9524,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton pending_tx test_ptx; if (use_rct) { transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, tx.outs, unlock_time, tx.needed_fee, extra, - test_tx, test_ptx, range_proof_type); + test_tx, test_ptx, rct_config); } else { transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, tx.outs, unlock_time, tx.needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); @@ -9055,6 +9553,64 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton return ptx_vector; } //---------------------------------------------------------------------------------------------------- +void wallet2::cold_tx_aux_import(const std::vector<pending_tx> & ptx, const std::vector<std::string> & tx_device_aux) +{ + CHECK_AND_ASSERT_THROW_MES(ptx.size() == tx_device_aux.size(), "TX aux has invalid size"); + for (size_t i = 0; i < ptx.size(); ++i){ + crypto::hash txid; + txid = get_transaction_hash(ptx[i].tx); + set_tx_device_aux(txid, tx_device_aux[i]); + } +} +//---------------------------------------------------------------------------------------------------- +void wallet2::cold_sign_tx(const std::vector<pending_tx>& ptx_vector, signed_tx_set &exported_txs, std::vector<cryptonote::address_parse_info> &dsts_info, std::vector<std::string> & tx_device_aux) +{ + auto & hwdev = get_account().get_device(); + if (!hwdev.has_tx_cold_sign()){ + throw std::invalid_argument("Device does not support cold sign protocol"); + } + + unsigned_tx_set txs; + for (auto &tx: ptx_vector) + { + txs.txes.push_back(get_construction_data_with_decrypted_short_payment_id(tx, m_account.get_device())); + } + txs.transfers = std::make_pair(0, m_transfers); + + auto dev_cold = dynamic_cast<::hw::device_cold*>(&hwdev); + CHECK_AND_ASSERT_THROW_MES(dev_cold, "Device does not implement cold signing interface"); + + hw::tx_aux_data aux_data; + hw::wallet_shim wallet_shim; + setup_shim(&wallet_shim, this); + aux_data.tx_recipients = dsts_info; + dev_cold->tx_sign(&wallet_shim, txs, exported_txs, aux_data); + tx_device_aux = aux_data.tx_device_aux; + + MDEBUG("Signed tx data from hw: " << exported_txs.ptx.size() << " transactions"); + for (auto &c_ptx: exported_txs.ptx) LOG_PRINT_L0(cryptonote::obj_to_json_str(c_ptx.tx)); +} +//---------------------------------------------------------------------------------------------------- +uint64_t wallet2::cold_key_image_sync(uint64_t &spent, uint64_t &unspent) { + auto & hwdev = get_account().get_device(); + CHECK_AND_ASSERT_THROW_MES(hwdev.has_ki_cold_sync(), "Device does not support cold ki sync protocol"); + + auto dev_cold = dynamic_cast<::hw::device_cold*>(&hwdev); + CHECK_AND_ASSERT_THROW_MES(dev_cold, "Device does not implement cold signing interface"); + + std::vector<std::pair<crypto::key_image, crypto::signature>> ski; + hw::wallet_shim wallet_shim; + setup_shim(&wallet_shim, this); + + dev_cold->ki_sync(&wallet_shim, m_transfers, ski); + + // Call COMMAND_RPC_IS_KEY_IMAGE_SPENT only if daemon is trusted. + uint64_t import_res = import_key_images(ski, 0, spent, unspent, is_trusted_daemon()); + m_device_last_key_image_sync = time(NULL); + + return import_res; +} +//---------------------------------------------------------------------------------------------------- void wallet2::get_hard_fork_info(uint8_t version, uint64_t &earliest_height) const { boost::optional<std::string> result = m_node_rpc_proxy.get_earliest_height(version, earliest_height); @@ -9263,7 +9819,7 @@ void wallet2::set_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_ COMMAND_RPC_GET_TRANSACTIONS::request req = AUTO_VAL_INIT(req); req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); req.decode_as_json = false; - req.prune = false; + req.prune = true; COMMAND_RPC_GET_TRANSACTIONS::response res = AUTO_VAL_INIT(res); bool r; { @@ -9276,11 +9832,10 @@ void wallet2::set_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_ THROW_WALLET_EXCEPTION_IF(res.txs.size() != 1, error::wallet_internal_error, "daemon returned wrong response for gettransactions, wrong txs count = " + std::to_string(res.txs.size()) + ", expected 1"); - cryptonote::blobdata bd; - THROW_WALLET_EXCEPTION_IF(!epee::string_tools::parse_hexstr_to_binbuff(res.txs[0].as_hex, bd), error::wallet_internal_error, "failed to parse tx from hexstr"); cryptonote::transaction tx; - crypto::hash tx_hash, tx_prefix_hash; - THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(bd, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, "failed to parse tx from blob"); + crypto::hash tx_hash; + THROW_WALLET_EXCEPTION_IF(!get_pruned_tx(res.txs[0], tx, tx_hash), error::wallet_internal_error, + "Failed to get transaction from daemon"); THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, "txid mismatch"); std::vector<tx_extra_field> tx_extra_fields; THROW_WALLET_EXCEPTION_IF(!parse_tx_extra(tx.extra, tx_extra_fields), error::wallet_internal_error, "Transaction extra has unsupported format"); @@ -9314,7 +9869,7 @@ std::string wallet2::get_spend_proof(const crypto::hash &txid, const std::string COMMAND_RPC_GET_TRANSACTIONS::request req = AUTO_VAL_INIT(req); req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); req.decode_as_json = false; - req.prune = false; + req.prune = true; COMMAND_RPC_GET_TRANSACTIONS::response res = AUTO_VAL_INIT(res); bool r; { @@ -9327,12 +9882,10 @@ std::string wallet2::get_spend_proof(const crypto::hash &txid, const std::string THROW_WALLET_EXCEPTION_IF(res.txs.size() != 1, error::wallet_internal_error, "daemon returned wrong response for gettransactions, wrong txs count = " + std::to_string(res.txs.size()) + ", expected 1"); - cryptonote::blobdata bd; - THROW_WALLET_EXCEPTION_IF(!epee::string_tools::parse_hexstr_to_binbuff(res.txs[0].as_hex, bd), error::wallet_internal_error, "failed to parse tx from hexstr"); + cryptonote::transaction tx; - crypto::hash tx_hash, tx_prefix_hash; - THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(bd, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, "failed to parse tx from blob"); - THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, "txid mismatch"); + crypto::hash tx_hash; + THROW_WALLET_EXCEPTION_IF(!get_pruned_tx(res.txs[0], tx, tx_hash), error::wallet_internal_error, "Failed to get tx from daemon"); std::vector<std::vector<crypto::signature>> signatures; @@ -9434,7 +9987,7 @@ bool wallet2::check_spend_proof(const crypto::hash &txid, const std::string &mes COMMAND_RPC_GET_TRANSACTIONS::request req = AUTO_VAL_INIT(req); req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); req.decode_as_json = false; - req.prune = false; + req.prune = true; COMMAND_RPC_GET_TRANSACTIONS::response res = AUTO_VAL_INIT(res); bool r; { @@ -9447,12 +10000,10 @@ bool wallet2::check_spend_proof(const crypto::hash &txid, const std::string &mes THROW_WALLET_EXCEPTION_IF(res.txs.size() != 1, error::wallet_internal_error, "daemon returned wrong response for gettransactions, wrong txs count = " + std::to_string(res.txs.size()) + ", expected 1"); - cryptonote::blobdata bd; - THROW_WALLET_EXCEPTION_IF(!epee::string_tools::parse_hexstr_to_binbuff(res.txs[0].as_hex, bd), error::wallet_internal_error, "failed to parse tx from hexstr"); + cryptonote::transaction tx; - crypto::hash tx_hash, tx_prefix_hash; - THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(bd, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, "failed to parse tx from blob"); - THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, "txid mismatch"); + crypto::hash tx_hash; + THROW_WALLET_EXCEPTION_IF(!get_pruned_tx(res.txs[0], tx, tx_hash), error::wallet_internal_error, "failed to get tx from daemon"); // check signature size size_t num_sigs = 0; @@ -9559,24 +10110,30 @@ void wallet2::check_tx_key_helper(const crypto::hash &txid, const crypto::key_de COMMAND_RPC_GET_TRANSACTIONS::response res; req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); req.decode_as_json = false; - req.prune = false; + req.prune = true; m_daemon_rpc_mutex.lock(); bool ok = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client); m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1), error::wallet_internal_error, "Failed to get transaction from daemon"); - cryptonote::blobdata tx_data; + cryptonote::transaction tx; + crypto::hash tx_hash; if (res.txs.size() == 1) - ok = string_tools::parse_hexstr_to_binbuff(res.txs.front().as_hex, tx_data); + { + ok = get_pruned_tx(res.txs.front(), tx, tx_hash); + THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + } else + { + cryptonote::blobdata tx_data; + crypto::hash tx_prefix_hash; ok = string_tools::parse_hexstr_to_binbuff(res.txs_as_hex.front(), tx_data); - THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx, tx_hash, tx_prefix_hash), + error::wallet_internal_error, "Failed to validate transaction from daemon"); + } - crypto::hash tx_hash, tx_prefix_hash; - cryptonote::transaction tx; - THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, - "Failed to validate transaction from daemon"); THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, "Failed to get the right transaction from daemon"); THROW_WALLET_EXCEPTION_IF(!additional_derivations.empty() && additional_derivations.size() != tx.vout.size(), error::wallet_internal_error, @@ -9615,7 +10172,7 @@ void wallet2::check_tx_key_helper(const crypto::hash &txid, const crypto::key_de crypto::secret_key scalar1; hwdev.derivation_to_scalar(found_derivation, n, scalar1); rct::ecdhTuple ecdh_info = tx.rct_signatures.ecdhInfo[n]; - hwdev.ecdhDecode(ecdh_info, rct::sk2rct(scalar1)); + hwdev.ecdhDecode(ecdh_info, rct::sk2rct(scalar1), tx.rct_signatures.type == rct::RCTTypeBulletproof2); const rct::key C = tx.rct_signatures.outPk[n].mask; rct::key Ctmp; THROW_WALLET_EXCEPTION_IF(sc_check(ecdh_info.mask.bytes) != 0, error::wallet_internal_error, "Bad ECDH input mask"); @@ -9699,24 +10256,30 @@ std::string wallet2::get_tx_proof(const crypto::hash &txid, const cryptonote::ac COMMAND_RPC_GET_TRANSACTIONS::response res; req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); req.decode_as_json = false; - req.prune = false; + req.prune = true; m_daemon_rpc_mutex.lock(); bool ok = net_utils::invoke_http_json("/gettransactions", req, res, m_http_client); m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1), error::wallet_internal_error, "Failed to get transaction from daemon"); - cryptonote::blobdata tx_data; + cryptonote::transaction tx; + crypto::hash tx_hash; if (res.txs.size() == 1) - ok = string_tools::parse_hexstr_to_binbuff(res.txs.front().as_hex, tx_data); + { + ok = get_pruned_tx(res.txs.front(), tx, tx_hash); + THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + } else + { + cryptonote::blobdata tx_data; + crypto::hash tx_prefix_hash; ok = string_tools::parse_hexstr_to_binbuff(res.txs_as_hex.front(), tx_data); - THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx, tx_hash, tx_prefix_hash), + error::wallet_internal_error, "Failed to validate transaction from daemon"); + } - crypto::hash tx_hash, tx_prefix_hash; - cryptonote::transaction tx; - THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, - "Failed to validate transaction from daemon"); THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, "Failed to get the right transaction from daemon"); crypto::public_key tx_pub_key = get_tx_pub_key_from_extra(tx); @@ -9811,24 +10374,30 @@ bool wallet2::check_tx_proof(const crypto::hash &txid, const cryptonote::account COMMAND_RPC_GET_TRANSACTIONS::response res; req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); req.decode_as_json = false; - req.prune = false; + req.prune = true; m_daemon_rpc_mutex.lock(); bool ok = net_utils::invoke_http_json("/gettransactions", req, res, m_http_client); m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1), error::wallet_internal_error, "Failed to get transaction from daemon"); - cryptonote::blobdata tx_data; + cryptonote::transaction tx; + crypto::hash tx_hash; if (res.txs.size() == 1) - ok = string_tools::parse_hexstr_to_binbuff(res.txs.front().as_hex, tx_data); + { + ok = get_pruned_tx(res.txs.front(), tx, tx_hash); + THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + } else + { + cryptonote::blobdata tx_data; + crypto::hash tx_prefix_hash; ok = string_tools::parse_hexstr_to_binbuff(res.txs_as_hex.front(), tx_data); - THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx, tx_hash, tx_prefix_hash), + error::wallet_internal_error, "Failed to validate transaction from daemon"); + } - crypto::hash tx_hash, tx_prefix_hash; - cryptonote::transaction tx; - THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, - "Failed to validate transaction from daemon"); THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, "Failed to get the right transaction from daemon"); crypto::public_key tx_pub_key = get_tx_pub_key_from_extra(tx); @@ -10047,7 +10616,7 @@ bool wallet2::check_reserve_proof(const cryptonote::account_public_address &addr for (size_t i = 0; i < proofs.size(); ++i) gettx_req.txs_hashes.push_back(epee::string_tools::pod_to_hex(proofs[i].txid)); gettx_req.decode_as_json = false; - gettx_req.prune = false; + gettx_req.prune = true; m_daemon_rpc_mutex.lock(); bool ok = net_utils::invoke_http_json("/gettransactions", gettx_req, gettx_res, m_http_client); m_daemon_rpc_mutex.unlock(); @@ -10071,14 +10640,11 @@ bool wallet2::check_reserve_proof(const cryptonote::account_public_address &addr const reserve_proof_entry& proof = proofs[i]; THROW_WALLET_EXCEPTION_IF(gettx_res.txs[i].in_pool, error::wallet_internal_error, "Tx is unconfirmed"); - cryptonote::blobdata tx_data; - ok = string_tools::parse_hexstr_to_binbuff(gettx_res.txs[i].as_hex, tx_data); + cryptonote::transaction tx; + crypto::hash tx_hash; + ok = get_pruned_tx(gettx_res.txs[i], tx, tx_hash); THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); - crypto::hash tx_hash, tx_prefix_hash; - cryptonote::transaction tx; - THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, - "Failed to validate transaction from daemon"); THROW_WALLET_EXCEPTION_IF(tx_hash != proof.txid, error::wallet_internal_error, "Failed to get the right transaction from daemon"); THROW_WALLET_EXCEPTION_IF(proof.index_in_tx >= tx.vout.size(), error::wallet_internal_error, "index_in_tx is out of bound"); @@ -10120,7 +10686,7 @@ bool wallet2::check_reserve_proof(const cryptonote::account_public_address &addr crypto::secret_key shared_secret; crypto::derivation_to_scalar(derivation, proof.index_in_tx, shared_secret); rct::ecdhTuple ecdh_info = tx.rct_signatures.ecdhInfo[proof.index_in_tx]; - rct::ecdhDecode(ecdh_info, rct::sk2rct(shared_secret)); + rct::ecdhDecode(ecdh_info, rct::sk2rct(shared_secret), tx.rct_signatures.type == rct::RCTTypeBulletproof2); amount = rct::h2d(ecdh_info.amount); } total += amount; @@ -10159,7 +10725,10 @@ uint64_t wallet2::get_daemon_blockchain_height(string &err) const boost::optional<std::string> result = m_node_rpc_proxy.get_height(height); if (result) { - err = *result; + if (m_trusted_daemon) + err = *result; + else + err = "daemon error"; return 0; } @@ -10174,7 +10743,10 @@ uint64_t wallet2::get_daemon_blockchain_target_height(string &err) const auto result = m_node_rpc_proxy.get_target_height(target_height); if (result && *result != CORE_RPC_STATUS_OK) { - err= *result; + if (m_trusted_daemon) + err = *result; + else + err = "daemon error"; return 0; } return target_height; @@ -10211,6 +10783,19 @@ std::string wallet2::get_tx_note(const crypto::hash &txid) const return i->second; } +void wallet2::set_tx_device_aux(const crypto::hash &txid, const std::string &aux) +{ + m_tx_device[txid] = aux; +} + +std::string wallet2::get_tx_device_aux(const crypto::hash &txid) const +{ + std::unordered_map<crypto::hash, std::string>::const_iterator i = m_tx_device.find(txid); + if (i == m_tx_device.end()) + return std::string(); + return i->second; +} + void wallet2::set_attribute(const std::string &key, const std::string &value) { m_attributes[key] = value; @@ -10254,7 +10839,7 @@ const std::pair<std::map<std::string, std::string>, std::vector<std::string>>& w return m_account_tags; } -void wallet2::set_account_tag(const std::set<uint32_t> account_indices, const std::string& tag) +void wallet2::set_account_tag(const std::set<uint32_t> &account_indices, const std::string& tag) { for (uint32_t account_index : account_indices) { @@ -10401,36 +10986,50 @@ crypto::public_key wallet2::get_tx_pub_key_from_received_outs(const tools::walle bool wallet2::export_key_images(const std::string &filename) const { - std::vector<std::pair<crypto::key_image, crypto::signature>> ski = export_key_images(); + PERF_TIMER(export_key_images); + std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> ski = export_key_images(); std::string magic(KEY_IMAGE_EXPORT_FILE_MAGIC, strlen(KEY_IMAGE_EXPORT_FILE_MAGIC)); const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address; + const uint32_t offset = ski.first; std::string data; + data.reserve(4 + ski.second.size() * (sizeof(crypto::key_image) + sizeof(crypto::signature)) + 2 * sizeof(crypto::public_key)); + data.resize(4); + data[0] = offset & 0xff; + data[1] = (offset >> 8) & 0xff; + data[2] = (offset >> 16) & 0xff; + data[3] = (offset >> 24) & 0xff; data += std::string((const char *)&keys.m_spend_public_key, sizeof(crypto::public_key)); data += std::string((const char *)&keys.m_view_public_key, sizeof(crypto::public_key)); - for (const auto &i: ski) + for (const auto &i: ski.second) { data += std::string((const char *)&i.first, sizeof(crypto::key_image)); data += std::string((const char *)&i.second, sizeof(crypto::signature)); } // encrypt data, keep magic plaintext + PERF_TIMER(export_key_images_encrypt); std::string ciphertext = encrypt_with_view_secret_key(data); return epee::file_io_utils::save_string_to_file(filename, magic + ciphertext); } //---------------------------------------------------------------------------------------------------- -std::vector<std::pair<crypto::key_image, crypto::signature>> wallet2::export_key_images() const +std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> wallet2::export_key_images(bool all) const { + PERF_TIMER(export_key_images_raw); std::vector<std::pair<crypto::key_image, crypto::signature>> ski; - ski.reserve(m_transfers.size()); - for (size_t n = 0; n < m_transfers.size(); ++n) + size_t offset = 0; + if (!all) { - const transfer_details &td = m_transfers[n]; + while (offset < m_transfers.size() && !m_transfers[offset].m_key_image_request) + ++offset; + } - crypto::hash hash; - crypto::cn_fast_hash(&td.m_key_image, sizeof(td.m_key_image), hash); + ski.reserve(m_transfers.size() - offset); + for (size_t n = offset; n < m_transfers.size(); ++n) + { + const transfer_details &td = m_transfers[n]; // get ephemeral public key const cryptonote::tx_out &out = td.m_tx.vout[td.m_internal_output_index]; @@ -10469,11 +11068,12 @@ std::vector<std::pair<crypto::key_image, crypto::signature>> wallet2::export_key ski.push_back(std::make_pair(td.m_key_image, signature)); } - return ski; + return std::make_pair(offset, ski); } uint64_t wallet2::import_key_images(const std::string &filename, uint64_t &spent, uint64_t &unspent) { + PERF_TIMER(import_key_images_fsu); std::string data; bool r = epee::file_io_utils::load_file_to_string(filename, data); @@ -10487,6 +11087,7 @@ uint64_t wallet2::import_key_images(const std::string &filename, uint64_t &spent try { + PERF_TIMER(import_key_images_decrypt); data = decrypt_with_view_secret_key(std::string(data, magiclen)); } catch (const std::exception &e) @@ -10494,15 +11095,17 @@ uint64_t wallet2::import_key_images(const std::string &filename, uint64_t &spent THROW_WALLET_EXCEPTION(error::wallet_internal_error, std::string("Failed to decrypt ") + filename + ": " + e.what()); } - const size_t headerlen = 2 * sizeof(crypto::public_key); + const size_t headerlen = 4 + 2 * sizeof(crypto::public_key); THROW_WALLET_EXCEPTION_IF(data.size() < headerlen, error::wallet_internal_error, std::string("Bad data size from file ") + filename); - const crypto::public_key &public_spend_key = *(const crypto::public_key*)&data[0]; - const crypto::public_key &public_view_key = *(const crypto::public_key*)&data[sizeof(crypto::public_key)]; + const uint32_t offset = (uint8_t)data[0] | (((uint8_t)data[1]) << 8) | (((uint8_t)data[2]) << 16) | (((uint8_t)data[3]) << 24); + const crypto::public_key &public_spend_key = *(const crypto::public_key*)&data[4]; + const crypto::public_key &public_view_key = *(const crypto::public_key*)&data[4 + sizeof(crypto::public_key)]; const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address; if (public_spend_key != keys.m_spend_public_key || public_view_key != keys.m_view_public_key) { THROW_WALLET_EXCEPTION(error::wallet_internal_error, std::string( "Key images from ") + filename + " are for a different account"); } + THROW_WALLET_EXCEPTION_IF(offset > m_transfers.size(), error::wallet_internal_error, "Offset larger than known outputs"); const size_t record_size = sizeof(crypto::key_image) + sizeof(crypto::signature); THROW_WALLET_EXCEPTION_IF((data.size() - headerlen) % record_size, @@ -10519,28 +11122,33 @@ uint64_t wallet2::import_key_images(const std::string &filename, uint64_t &spent ski.push_back(std::make_pair(key_image, signature)); } - return import_key_images(ski, spent, unspent); + return import_key_images(ski, offset, spent, unspent); } //---------------------------------------------------------------------------------------------------- -uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_image, crypto::signature>> &signed_key_images, uint64_t &spent, uint64_t &unspent, bool check_spent) +uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_image, crypto::signature>> &signed_key_images, size_t offset, uint64_t &spent, uint64_t &unspent, bool check_spent) { + PERF_TIMER(import_key_images_lots); COMMAND_RPC_IS_KEY_IMAGE_SPENT::request req = AUTO_VAL_INIT(req); COMMAND_RPC_IS_KEY_IMAGE_SPENT::response daemon_resp = AUTO_VAL_INIT(daemon_resp); - THROW_WALLET_EXCEPTION_IF(signed_key_images.size() > m_transfers.size(), error::wallet_internal_error, + THROW_WALLET_EXCEPTION_IF(offset > m_transfers.size(), error::wallet_internal_error, "Offset larger than known outputs"); + THROW_WALLET_EXCEPTION_IF(signed_key_images.size() > m_transfers.size() - offset, error::wallet_internal_error, "The blockchain is out of date compared to the signed key images"); - if (signed_key_images.empty()) + if (signed_key_images.empty() && offset == 0) { spent = 0; unspent = 0; return 0; } + req.key_images.reserve(signed_key_images.size()); + + PERF_TIMER_START(import_key_images_A); for (size_t n = 0; n < signed_key_images.size(); ++n) { - const transfer_details &td = m_transfers[n]; + const transfer_details &td = m_transfers[n + offset]; const crypto::key_image &key_image = signed_key_images[n].first; const crypto::signature &signature = signed_key_images[n].second; @@ -10551,30 +11159,37 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag const cryptonote::txout_to_key &o = boost::get<cryptonote::txout_to_key>(out.target); const crypto::public_key pkey = o.key; - std::vector<const crypto::public_key*> pkeys; - pkeys.push_back(&pkey); - THROW_WALLET_EXCEPTION_IF(!(rct::scalarmultKey(rct::ki2rct(key_image), rct::curveOrder()) == rct::identity()), - error::wallet_internal_error, "Key image out of validity domain: input " + boost::lexical_cast<std::string>(n) + "/" - + boost::lexical_cast<std::string>(signed_key_images.size()) + ", key image " + epee::string_tools::pod_to_hex(key_image)); - - THROW_WALLET_EXCEPTION_IF(!crypto::check_ring_signature((const crypto::hash&)key_image, key_image, pkeys, &signature), - error::wallet_internal_error, "Signature check failed: input " + boost::lexical_cast<std::string>(n) + "/" - + boost::lexical_cast<std::string>(signed_key_images.size()) + ", key image " + epee::string_tools::pod_to_hex(key_image) - + ", signature " + epee::string_tools::pod_to_hex(signature) + ", pubkey " + epee::string_tools::pod_to_hex(*pkeys[0])); + if (!td.m_key_image_known || !(key_image == td.m_key_image)) + { + std::vector<const crypto::public_key*> pkeys; + pkeys.push_back(&pkey); + THROW_WALLET_EXCEPTION_IF(!(rct::scalarmultKey(rct::ki2rct(key_image), rct::curveOrder()) == rct::identity()), + error::wallet_internal_error, "Key image out of validity domain: input " + boost::lexical_cast<std::string>(n + offset) + "/" + + boost::lexical_cast<std::string>(signed_key_images.size()) + ", key image " + epee::string_tools::pod_to_hex(key_image)); + THROW_WALLET_EXCEPTION_IF(!crypto::check_ring_signature((const crypto::hash&)key_image, key_image, pkeys, &signature), + error::signature_check_failed, boost::lexical_cast<std::string>(n + offset) + "/" + + boost::lexical_cast<std::string>(signed_key_images.size()) + ", key image " + epee::string_tools::pod_to_hex(key_image) + + ", signature " + epee::string_tools::pod_to_hex(signature) + ", pubkey " + epee::string_tools::pod_to_hex(*pkeys[0])); + } req.key_images.push_back(epee::string_tools::pod_to_hex(key_image)); } + PERF_TIMER_STOP(import_key_images_A); + PERF_TIMER_START(import_key_images_B); for (size_t n = 0; n < signed_key_images.size(); ++n) { - m_transfers[n].m_key_image = signed_key_images[n].first; - m_key_images[m_transfers[n].m_key_image] = n; - m_transfers[n].m_key_image_known = true; - m_transfers[n].m_key_image_partial = false; + m_transfers[n + offset].m_key_image = signed_key_images[n].first; + m_key_images[m_transfers[n + offset].m_key_image] = n + offset; + m_transfers[n + offset].m_key_image_known = true; + m_transfers[n + offset].m_key_image_request = false; + m_transfers[n + offset].m_key_image_partial = false; } + PERF_TIMER_STOP(import_key_images_B); if(check_spent) { + PERF_TIMER(import_key_images_RPC); m_daemon_rpc_mutex.lock(); bool r = epee::net_utils::invoke_http_json("/is_key_image_spent", req, daemon_resp, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); @@ -10586,7 +11201,7 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag std::to_string(daemon_resp.spent_status.size()) + ", expected " + std::to_string(signed_key_images.size())); for (size_t n = 0; n < daemon_resp.spent_status.size(); ++n) { - transfer_details &td = m_transfers[n]; + transfer_details &td = m_transfers[n + offset]; td.m_spent = daemon_resp.spent_status[n] != COMMAND_RPC_IS_KEY_IMAGE_SPENT::UNSPENT; } } @@ -10597,6 +11212,7 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag // was created by sweep_all, so we can't know the spent height and other detailed info. std::unordered_map<crypto::key_image, crypto::hash> spent_key_images; + PERF_TIMER_START(import_key_images_C); for (const transfer_details &td: m_transfers) { for (const cryptonote::txin_v& in : td.m_tx.vin) @@ -10605,10 +11221,12 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag spent_key_images.insert(std::make_pair(boost::get<cryptonote::txin_to_key>(in).k_image, td.m_txid)); } } + PERF_TIMER_STOP(import_key_images_C); + PERF_TIMER_START(import_key_images_D); for(size_t i = 0; i < signed_key_images.size(); ++i) { - transfer_details &td = m_transfers[i]; + const transfer_details &td = m_transfers[i + offset]; uint64_t amount = td.amount(); if (td.m_spent) spent += amount; @@ -10626,6 +11244,8 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag spent_txids.insert(skii->second); } } + PERF_TIMER_STOP(import_key_images_D); + MDEBUG("Total: " << print_money(spent) << " spent, " << print_money(unspent) << " unspent"); if (check_spent) @@ -10634,9 +11254,13 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag COMMAND_RPC_GET_TRANSACTIONS::request gettxs_req; COMMAND_RPC_GET_TRANSACTIONS::response gettxs_res; gettxs_req.decode_as_json = false; - gettxs_req.prune = false; + gettxs_req.prune = true; + gettxs_req.txs_hashes.reserve(spent_txids.size()); for (const crypto::hash& spent_txid : spent_txids) gettxs_req.txs_hashes.push_back(epee::string_tools::pod_to_hex(spent_txid)); + + + PERF_TIMER_START(import_key_images_E); m_daemon_rpc_mutex.lock(); bool r = epee::net_utils::invoke_http_json("/gettransactions", gettxs_req, gettxs_res, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); @@ -10644,21 +11268,22 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag THROW_WALLET_EXCEPTION_IF(gettxs_res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "gettransactions"); THROW_WALLET_EXCEPTION_IF(gettxs_res.txs.size() != spent_txids.size(), error::wallet_internal_error, "daemon returned wrong response for gettransactions, wrong count = " + std::to_string(gettxs_res.txs.size()) + ", expected " + std::to_string(spent_txids.size())); + PERF_TIMER_STOP(import_key_images_E); // process each outgoing tx + PERF_TIMER_START(import_key_images_F); auto spent_txid = spent_txids.begin(); hw::device &hwdev = m_account.get_device(); + auto it = spent_txids.begin(); for (const COMMAND_RPC_GET_TRANSACTIONS::entry& e : gettxs_res.txs) { THROW_WALLET_EXCEPTION_IF(e.in_pool, error::wallet_internal_error, "spent tx isn't supposed to be in txpool"); - // parse tx - cryptonote::blobdata bd; - THROW_WALLET_EXCEPTION_IF(!epee::string_tools::parse_hexstr_to_binbuff(e.as_hex, bd), error::wallet_internal_error, "parse_hexstr_to_binbuff failed"); cryptonote::transaction spent_tx; - crypto::hash spnet_txid_parsed, spent_txid_prefix; - THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(bd, spent_tx, spnet_txid_parsed, spent_txid_prefix), error::wallet_internal_error, "parse_and_validate_tx_from_blob failed"); - THROW_WALLET_EXCEPTION_IF(*spent_txid != spnet_txid_parsed, error::wallet_internal_error, "parsed txid mismatch"); + crypto::hash spnet_txid_parsed; + THROW_WALLET_EXCEPTION_IF(!get_pruned_tx(e, spent_tx, spnet_txid_parsed), error::wallet_internal_error, "Failed to get tx from daemon"); + THROW_WALLET_EXCEPTION_IF(!(spnet_txid_parsed == *it), error::wallet_internal_error, "parsed txid mismatch"); + ++it; // get received (change) amount uint64_t tx_money_got_in_outs = 0; @@ -10741,7 +11366,9 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag ++spent_txid; } + PERF_TIMER_STOP(import_key_images_F); + PERF_TIMER_START(import_key_images_G); for (size_t n : swept_transfers) { const transfer_details& td = m_transfers[n]; @@ -10752,10 +11379,35 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag const crypto::hash &spent_txid = crypto::null_hash; // spent txid is unknown m_confirmed_txs.insert(std::make_pair(spent_txid, pd)); } + PERF_TIMER_STOP(import_key_images_G); } return m_transfers[signed_key_images.size() - 1].m_block_height; } + +bool wallet2::import_key_images(std::vector<crypto::key_image> key_images) +{ + if (key_images.size() > m_transfers.size()) + { + LOG_PRINT_L1("More key images returned that we know outputs for"); + return false; + } + for (size_t i = 0; i < key_images.size(); ++i) + { + transfer_details &td = m_transfers[i]; + if (td.m_key_image_known && !td.m_key_image_partial && td.m_key_image != key_images[i]) + LOG_PRINT_L0("WARNING: imported key image differs from previously known key image at index " << i << ": trusting imported one"); + td.m_key_image = key_images[i]; + m_key_images[m_transfers[i].m_key_image] = i; + td.m_key_image_known = true; + td.m_key_image_request = false; + td.m_key_image_partial = false; + m_pub_keys[m_transfers[i].get_public_key()] = i; + } + + return true; +} + wallet2::payment_container wallet2::export_payments() const { payment_container payments; @@ -10814,50 +11466,87 @@ void wallet2::import_blockchain(const std::tuple<size_t, crypto::hash, std::vect m_last_block_reward = cryptonote::get_outs_money_amount(genesis.miner_tx); } //---------------------------------------------------------------------------------------------------- -std::vector<tools::wallet2::transfer_details> wallet2::export_outputs() const +std::pair<size_t, std::vector<tools::wallet2::transfer_details>> wallet2::export_outputs() const { + PERF_TIMER(export_outputs); std::vector<tools::wallet2::transfer_details> outs; - outs.reserve(m_transfers.size()); - for (size_t n = 0; n < m_transfers.size(); ++n) + size_t offset = 0; + while (offset < m_transfers.size() && (m_transfers[offset].m_key_image_known && !m_transfers[offset].m_key_image_request)) + ++offset; + + outs.reserve(m_transfers.size() - offset); + for (size_t n = offset; n < m_transfers.size(); ++n) { const transfer_details &td = m_transfers[n]; outs.push_back(td); } - return outs; + return std::make_pair(offset, outs); } //---------------------------------------------------------------------------------------------------- std::string wallet2::export_outputs_to_str() const { - std::vector<tools::wallet2::transfer_details> outs = export_outputs(); + PERF_TIMER(export_outputs_to_str); std::stringstream oss; boost::archive::portable_binary_oarchive ar(oss); - ar << outs; + const auto& outputs = export_outputs(); + ar << outputs; std::string magic(OUTPUT_EXPORT_FILE_MAGIC, strlen(OUTPUT_EXPORT_FILE_MAGIC)); const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address; std::string header; header += std::string((const char *)&keys.m_spend_public_key, sizeof(crypto::public_key)); header += std::string((const char *)&keys.m_view_public_key, sizeof(crypto::public_key)); + PERF_TIMER(export_outputs_encryption); std::string ciphertext = encrypt_with_view_secret_key(header + oss.str()); return magic + ciphertext; } //---------------------------------------------------------------------------------------------------- -size_t wallet2::import_outputs(const std::vector<tools::wallet2::transfer_details> &outputs) +size_t wallet2::import_outputs(const std::pair<size_t, std::vector<tools::wallet2::transfer_details>> &outputs) { - m_transfers.clear(); - m_transfers.reserve(outputs.size()); - for (size_t i = 0; i < outputs.size(); ++i) + PERF_TIMER(import_outputs); + + THROW_WALLET_EXCEPTION_IF(outputs.first > m_transfers.size(), error::wallet_internal_error, + "Imported outputs omit more outputs that we know of"); + + const size_t offset = outputs.first; + const size_t original_size = m_transfers.size(); + m_transfers.resize(offset + outputs.second.size()); + for (size_t i = 0; i < offset; ++i) + m_transfers[i].m_key_image_request = false; + for (size_t i = 0; i < outputs.second.size(); ++i) { - transfer_details td = outputs[i]; + transfer_details td = outputs.second[i]; + + // skip those we've already imported, or which have different data + if (i + offset < original_size) + { + // compare the data used to create the key image below + const transfer_details &org_td = m_transfers[i + offset]; + if (!org_td.m_key_image_known) + goto process; +#define CMPF(f) if (!(td.f == org_td.f)) goto process + CMPF(m_txid); + CMPF(m_key_image); + CMPF(m_internal_output_index); +#undef CMPF + if (!(get_transaction_prefix_hash(td.m_tx) == get_transaction_prefix_hash(org_td.m_tx))) + goto process; + + // copy anyway, since the comparison does not include ancillary fields which may have changed + m_transfers[i + offset] = std::move(td); + continue; + } + +process: // the hot wallet wouldn't have known about key images (except if we already exported them) cryptonote::keypair in_ephemeral; - THROW_WALLET_EXCEPTION_IF(td.m_tx.vout.empty(), error::wallet_internal_error, "tx with no outputs at index " + boost::lexical_cast<std::string>(i)); + THROW_WALLET_EXCEPTION_IF(td.m_tx.vout.empty(), error::wallet_internal_error, "tx with no outputs at index " + boost::lexical_cast<std::string>(i + offset)); crypto::public_key tx_pub_key = get_tx_pub_key_from_received_outs(td); const std::vector<crypto::public_key> additional_tx_pub_keys = get_additional_tx_pub_keys_from_extra(td.m_tx); @@ -10868,13 +11557,14 @@ size_t wallet2::import_outputs(const std::vector<tools::wallet2::transfer_detail THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "Failed to generate key image"); expand_subaddresses(td.m_subaddr_index); td.m_key_image_known = true; + td.m_key_image_request = true; td.m_key_image_partial = false; THROW_WALLET_EXCEPTION_IF(in_ephemeral.pub != out_key, - error::wallet_internal_error, "key_image generated ephemeral public key not matched with output_key at index " + boost::lexical_cast<std::string>(i)); + error::wallet_internal_error, "key_image generated ephemeral public key not matched with output_key at index " + boost::lexical_cast<std::string>(i + offset)); - m_key_images[td.m_key_image] = m_transfers.size(); - m_pub_keys[td.get_public_key()] = m_transfers.size(); - m_transfers.push_back(std::move(td)); + m_key_images[td.m_key_image] = i + offset; + m_pub_keys[td.get_public_key()] = i + offset; + m_transfers[i + offset] = std::move(td); } return m_transfers.size(); @@ -10882,6 +11572,7 @@ size_t wallet2::import_outputs(const std::vector<tools::wallet2::transfer_detail //---------------------------------------------------------------------------------------------------- size_t wallet2::import_outputs_from_str(const std::string &outputs_st) { + PERF_TIMER(import_outputs_from_str); std::string data = outputs_st; const size_t magiclen = strlen(OUTPUT_EXPORT_FILE_MAGIC); if (data.size() < magiclen || memcmp(data.data(), OUTPUT_EXPORT_FILE_MAGIC, magiclen)) @@ -10891,6 +11582,7 @@ size_t wallet2::import_outputs_from_str(const std::string &outputs_st) try { + PERF_TIMER(import_outputs_decrypt); data = decrypt_with_view_secret_key(std::string(data, magiclen)); } catch (const std::exception &e) @@ -10917,7 +11609,7 @@ size_t wallet2::import_outputs_from_str(const std::string &outputs_st) std::string body(data, headerlen); std::stringstream iss; iss << body; - std::vector<tools::wallet2::transfer_details> outputs; + std::pair<size_t, std::vector<tools::wallet2::transfer_details>> outputs; try { boost::archive::portable_binary_iarchive ar(iss); @@ -10996,7 +11688,7 @@ rct::multisig_kLRki wallet2::get_multisig_kLRki(size_t n, const rct::key &k) con return kLRki; } //---------------------------------------------------------------------------------------------------- -rct::multisig_kLRki wallet2::get_multisig_composite_kLRki(size_t n, const crypto::public_key &ignore, std::unordered_set<rct::key> &used_L, std::unordered_set<rct::key> &new_used_L) const +rct::multisig_kLRki wallet2::get_multisig_composite_kLRki(size_t n, const std::unordered_set<crypto::public_key> &ignore_set, std::unordered_set<rct::key> &used_L, std::unordered_set<rct::key> &new_used_L) const { CHECK_AND_ASSERT_THROW_MES(n < m_transfers.size(), "Bad transfer index"); @@ -11007,8 +11699,9 @@ rct::multisig_kLRki wallet2::get_multisig_composite_kLRki(size_t n, const crypto size_t n_signers_used = 1; for (const auto &p: m_transfers[n].m_multisig_info) { - if (p.m_signer == ignore) + if (ignore_set.find(p.m_signer) != ignore_set.end()) continue; + for (const auto &lr: p.m_LR) { if (used_L.find(lr.m_L) != used_L.end()) @@ -11053,7 +11746,6 @@ cryptonote::blobdata wallet2::export_multisig() for (size_t n = 0; n < m_transfers.size(); ++n) { transfer_details &td = m_transfers[n]; - const std::vector<crypto::public_key> additional_tx_pub_keys = get_additional_tx_pub_keys_from_extra(td.m_tx); crypto::key_image ki; td.m_multisig_k.clear(); info[n].m_LR.clear(); @@ -11067,7 +11759,10 @@ cryptonote::blobdata wallet2::export_multisig() info[n].m_partial_key_images.push_back(ki); } - size_t nlr = m_multisig_threshold < m_multisig_signers.size() ? m_multisig_threshold - 1 : 1; + // Wallet tries to create as many transactions as many signers combinations. We calculate the maximum number here as follows: + // if we have 2/4 wallet with signers: A, B, C, D and A is a transaction creator it will need to pick up 1 signer from 3 wallets left. + // That means counting combinations for excluding 2-of-3 wallets (k = total signers count - threshold, n = total signers count - 1). + size_t nlr = tools::combinations_count(m_multisig_signers.size() - m_multisig_threshold, m_multisig_signers.size() - 1); for (size_t m = 0; m < nlr; ++m) { td.m_multisig_k.push_back(rct::skGen()); @@ -11082,7 +11777,6 @@ cryptonote::blobdata wallet2::export_multisig() boost::archive::portable_binary_oarchive ar(oss); ar << info; - std::string magic(MULTISIG_EXPORT_FILE_MAGIC, strlen(MULTISIG_EXPORT_FILE_MAGIC)); const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address; std::string header; header += std::string((const char *)&keys.m_spend_public_key, sizeof(crypto::public_key)); @@ -11109,6 +11803,7 @@ void wallet2::update_multisig_rescan_info(const std::vector<std::vector<rct::key m_key_images.erase(td.m_key_image); td.m_key_image = get_multisig_composite_key_image(n); td.m_key_image_known = true; + td.m_key_image_request = false; td.m_key_image_partial = false; td.m_multisig_k = multisig_k[n]; m_key_images[td.m_key_image] = n; @@ -11491,7 +12186,7 @@ uint64_t wallet2::get_blockchain_height_by_date(uint16_t year, uint8_t month, ui else if (res.status == CORE_RPC_STATUS_BUSY) oss << "daemon is busy"; else - oss << res.status; + oss << get_rpc_status(res.status); throw std::runtime_error(oss.str()); } cryptonote::block blk_min, blk_mid, blk_max; @@ -11662,4 +12357,75 @@ uint64_t wallet2::get_segregation_fork_height() const void wallet2::generate_genesis(cryptonote::block& b) const { cryptonote::generate_genesis_block(b, get_config(m_nettype).GENESIS_TX, get_config(m_nettype).GENESIS_NONCE); } +//---------------------------------------------------------------------------------------------------- +mms::multisig_wallet_state wallet2::get_multisig_wallet_state() const +{ + mms::multisig_wallet_state state; + state.nettype = m_nettype; + state.multisig = multisig(&state.multisig_is_ready); + state.has_multisig_partial_key_images = has_multisig_partial_key_images(); + state.multisig_rounds_passed = m_multisig_rounds_passed; + state.num_transfer_details = m_transfers.size(); + if (state.multisig) + { + THROW_WALLET_EXCEPTION_IF(!m_original_keys_available, error::wallet_internal_error, "MMS use not possible because own original Monero address not available"); + state.address = m_original_address; + state.view_secret_key = m_original_view_secret_key; + } + else + { + state.address = m_account.get_keys().m_account_address; + state.view_secret_key = m_account.get_keys().m_view_secret_key; + } + state.mms_file=m_mms_file; + return state; +} +//---------------------------------------------------------------------------------------------------- +wallet_device_callback * wallet2::get_device_callback() +{ + if (!m_device_callback){ + m_device_callback.reset(new wallet_device_callback(this)); + } + return m_device_callback.get(); +}//---------------------------------------------------------------------------------------------------- +void wallet2::on_button_request() +{ + if (0 != m_callback) + m_callback->on_button_request(); +} +//---------------------------------------------------------------------------------------------------- +void wallet2::on_pin_request(epee::wipeable_string & pin) +{ + if (0 != m_callback) + m_callback->on_pin_request(pin); +} +//---------------------------------------------------------------------------------------------------- +void wallet2::on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) +{ + if (0 != m_callback) + m_callback->on_passphrase_request(on_device, passphrase); +} +//---------------------------------------------------------------------------------------------------- +std::string wallet2::get_rpc_status(const std::string &s) const +{ + if (m_trusted_daemon) + return s; + return "<error>"; +} +//---------------------------------------------------------------------------------------------------- +void wallet2::throw_on_rpc_response_error(const boost::optional<std::string> &status, const char *method) const +{ + // no error + if (!status) + return; + + MERROR("RPC error: " << method << ": status " << *status); + + // empty string -> not connection + THROW_WALLET_EXCEPTION_IF(status->empty(), tools::error::no_connection_to_daemon, method); + + THROW_WALLET_EXCEPTION_IF(*status == CORE_RPC_STATUS_BUSY, tools::error::daemon_busy, method); + THROW_WALLET_EXCEPTION_IF(*status != CORE_RPC_STATUS_OK, tools::error::wallet_generic_rpc_error, method, m_trusted_daemon ? *status : "daemon error"); +} + } diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 680196f01..3b2dd6076 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -37,6 +37,7 @@ #include <boost/serialization/list.hpp> #include <boost/serialization/vector.hpp> #include <boost/serialization/deque.hpp> +#include <boost/thread/lock_guard.hpp> #include <atomic> #include "include_base_utils.h" @@ -49,15 +50,18 @@ #include "cryptonote_basic/cryptonote_format_utils.h" #include "cryptonote_core/cryptonote_tx_utils.h" #include "common/unordered_containers_boost_serialization.h" +#include "common/util.h" #include "crypto/chacha.h" #include "crypto/hash.h" #include "ringct/rctTypes.h" #include "ringct/rctOps.h" #include "checkpoints/checkpoints.h" +#include "serialization/pair.h" #include "wallet_errors.h" #include "common/password.h" #include "node_rpc_proxy.h" +#include "message_store.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "wallet.wallet2" @@ -97,11 +101,26 @@ namespace tools virtual void on_lw_money_received(uint64_t height, const crypto::hash &txid, uint64_t amount) {} virtual void on_lw_unconfirmed_money_received(uint64_t height, const crypto::hash &txid, uint64_t amount) {} virtual void on_lw_money_spent(uint64_t height, const crypto::hash &txid, uint64_t amount) {} + // Device callbacks + virtual void on_button_request() {} + virtual void on_pin_request(epee::wipeable_string & pin) {} + virtual void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) {} // Common callbacks virtual void on_pool_tx_removed(const crypto::hash &txid) {} virtual ~i_wallet2_callback() {} }; + class wallet_device_callback : public hw::i_device_callback + { + public: + wallet_device_callback(wallet2 * wallet): wallet(wallet) {}; + void on_button_request() override; + void on_pin_request(epee::wipeable_string & pin) override; + void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) override; + private: + wallet2 * wallet; + }; + struct tx_dust_policy { uint64_t dust_threshold; @@ -153,6 +172,7 @@ namespace tools { friend class ::Serialization_portability_wallet_Test; friend class wallet_keys_unlocker; + friend class wallet_device_callback; public: static constexpr const std::chrono::seconds rpc_timeout = std::chrono::minutes(3) + std::chrono::seconds(30); @@ -174,6 +194,7 @@ namespace tools static bool has_testnet_option(const boost::program_options::variables_map& vm); static bool has_stagenet_option(const boost::program_options::variables_map& vm); static std::string device_name_option(const boost::program_options::variables_map& vm); + static std::string device_derivation_path_option(const boost::program_options::variables_map &vm); static void init_options(boost::program_options::options_description& desc_params); //! Uses stdin and stdout. Returns a wallet2 if no errors. @@ -229,7 +250,7 @@ namespace tools bool error; boost::optional<cryptonote::subaddress_receive_info> received; - tx_scan_info_t(): money_transfered(0), error(true) {} + tx_scan_info_t(): amount(0), money_transfered(0), error(true) {} }; struct transfer_details @@ -246,11 +267,13 @@ namespace tools uint64_t m_amount; bool m_rct; bool m_key_image_known; + bool m_key_image_request; // view wallets: we want to request it; cold wallets: it was requested size_t m_pk_index; cryptonote::subaddress_index m_subaddr_index; bool m_key_image_partial; std::vector<rct::key> m_multisig_k; std::vector<multisig_info> m_multisig_info; // one per other participant + std::vector<std::pair<uint64_t, crypto::hash>> m_uses; bool is_rct() const { return m_rct; } uint64_t amount() const { return m_amount; } @@ -269,11 +292,13 @@ namespace tools FIELD(m_amount) FIELD(m_rct) FIELD(m_key_image_known) + FIELD(m_key_image_request) FIELD(m_pk_index) FIELD(m_subaddr_index) FIELD(m_key_image_partial) FIELD(m_multisig_k) FIELD(m_multisig_info) + FIELD(m_uses) END_SERIALIZE() }; @@ -371,7 +396,7 @@ namespace tools struct multisig_sig { rct::rctSig sigs; - crypto::public_key ignore; + std::unordered_set<crypto::public_key> ignore; std::unordered_set<rct::key> used_L; std::unordered_set<crypto::public_key> signing_keys; rct::multisig_out msout; @@ -416,13 +441,14 @@ namespace tools struct unsigned_tx_set { std::vector<tx_construction_data> txes; - wallet2::transfer_container transfers; + std::pair<size_t, wallet2::transfer_container> transfers; }; struct signed_tx_set { std::vector<pending_tx> ptx; std::vector<crypto::key_image> key_images; + std::unordered_map<crypto::public_key, crypto::key_image> tx_key_images; }; struct multisig_tx_set @@ -589,7 +615,7 @@ namespace tools /*! * \brief Finalizes creation of a multisig wallet */ - bool finalize_multisig(const epee::wipeable_string &password, std::unordered_set<crypto::public_key> pkeys, std::vector<crypto::public_key> signers); + bool finalize_multisig(const epee::wipeable_string &password, const std::unordered_set<crypto::public_key> &pkeys, std::vector<crypto::public_key> signers); /*! * Get a packaged multisig information string */ @@ -652,7 +678,7 @@ namespace tools bool init(std::string daemon_address = "http://localhost:8080", boost::optional<epee::net_utils::http::login> daemon_login = boost::none, uint64_t upper_transaction_weight_limit = 0, bool ssl = false, bool trusted_daemon = false); - void stop() { m_run.store(false, std::memory_order_relaxed); } + void stop() { m_run.store(false, std::memory_order_relaxed); m_message_store.stop(); } i_wallet2_callback* callback() const { return m_callback; } void callback(i_wallet2_callback* callback) { m_callback = callback; } @@ -707,7 +733,7 @@ namespace tools bool is_deprecated() const; void refresh(bool trusted_daemon); void refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched); - void refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched, bool& received_money); + void refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched, bool& received_money, bool check_pool = true); bool refresh(bool trusted_daemon, uint64_t & blocks_fetched, bool& received_money, bool& ok); void set_refresh_type(RefreshType refresh_type) { m_refresh_type = refresh_type; } @@ -738,7 +764,7 @@ namespace tools uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, cryptonote::transaction& tx, pending_tx &ptx); void transfer_selected_rct(std::vector<cryptonote::tx_destination_entry> dsts, const std::vector<size_t>& selected_transfers, size_t fake_outputs_count, std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, - uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx &ptx, rct::RangeProofType range_proof_type); + uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx &ptx, const rct::RCTConfig &rct_config); void commit_tx(pending_tx& ptx_vector); void commit_tx(std::vector<pending_tx>& ptx_vector); @@ -764,6 +790,9 @@ namespace tools std::vector<wallet2::pending_tx> create_transactions_all(uint64_t below, const cryptonote::account_public_address &address, bool is_subaddress, const size_t outputs, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices); std::vector<wallet2::pending_tx> create_transactions_single(const crypto::key_image &ki, const cryptonote::account_public_address &address, bool is_subaddress, const size_t outputs, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra); std::vector<wallet2::pending_tx> create_transactions_from(const cryptonote::account_public_address &address, bool is_subaddress, const size_t outputs, std::vector<size_t> unused_transfers_indices, std::vector<size_t> unused_dust_indices, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra); + void cold_tx_aux_import(const std::vector<pending_tx>& ptx, const std::vector<std::string>& tx_device_aux); + void cold_sign_tx(const std::vector<pending_tx>& ptx_vector, signed_tx_set &exported_txs, std::vector<cryptonote::address_parse_info> &dsts_info, std::vector<std::string> & tx_device_aux); + uint64_t cold_key_image_sync(uint64_t &spent, uint64_t &unspent); bool load_multisig_tx(cryptonote::blobdata blob, multisig_tx_set &exported_txs, std::function<bool(const multisig_tx_set&)> accept_func = NULL); bool load_multisig_tx_from_file(const std::string &filename, multisig_tx_set &exported_txs, std::function<bool(const multisig_tx_set&)> accept_func = NULL); bool sign_multisig_tx_from_file(const std::string &filename, std::vector<crypto::hash> &txids, std::function<bool(const multisig_tx_set&)> accept_func); @@ -782,11 +811,12 @@ namespace tools uint64_t get_blockchain_current_height() const { return m_light_wallet_blockchain_height ? m_light_wallet_blockchain_height : m_blockchain.size(); } void rescan_spent(); - void rescan_blockchain(bool refresh = true); + void rescan_blockchain(bool hard, bool refresh = true); bool is_transfer_unlocked(const transfer_details& td) const; bool is_transfer_unlocked(uint64_t unlock_time, uint64_t block_height) const; uint64_t get_last_block_reward() const { return m_last_block_reward; } + uint64_t get_device_last_key_image_sync() const { return m_device_last_key_image_sync; } template <class t_archive> inline void serialize(t_archive &a, const unsigned int ver) @@ -892,6 +922,15 @@ namespace tools if(ver < 25) return; a & m_last_block_reward; + if(ver < 26) + return; + a & m_tx_device; + if(ver < 27) + return; + a & m_device_last_key_image_sync; + if(ver < 28) + return; + a & m_cold_key_images; } /*! @@ -951,8 +990,12 @@ namespace tools void ignore_fractional_outputs(bool value) { m_ignore_fractional_outputs = value; } bool confirm_non_default_ring_size() const { return m_confirm_non_default_ring_size; } void confirm_non_default_ring_size(bool always) { m_confirm_non_default_ring_size = always; } + bool track_uses() const { return m_track_uses; } + void track_uses(bool value) { m_track_uses = value; } const std::string & device_name() const { return m_device_name; } void device_name(const std::string & device_name) { m_device_name = device_name; } + const std::string & device_derivation_path() const { return m_device_derivation_path; } + void device_derivation_path(const std::string &device_derivation_path) { m_device_derivation_path = device_derivation_path; } bool get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys) const; void set_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys); @@ -1020,6 +1063,9 @@ namespace tools void set_tx_note(const crypto::hash &txid, const std::string ¬e); std::string get_tx_note(const crypto::hash &txid) const; + void set_tx_device_aux(const crypto::hash &txid, const std::string &aux); + std::string get_tx_device_aux(const crypto::hash &txid) const; + void set_description(const std::string &description); std::string get_description() const; @@ -1033,7 +1079,7 @@ namespace tools * \param account_indices Indices of accounts. * \param tag Tag's name. If empty, the accounts become untagged. */ - void set_account_tag(const std::set<uint32_t> account_indices, const std::string& tag); + void set_account_tag(const std::set<uint32_t> &account_indices, const std::string& tag); /*! * \brief Set the label of the given tag. * \param tag Tag's name (which must be non-empty). @@ -1061,9 +1107,9 @@ namespace tools bool verify_with_public_key(const std::string &data, const crypto::public_key &public_key, const std::string &signature) const; // Import/Export wallet data - std::vector<tools::wallet2::transfer_details> export_outputs() const; + std::pair<size_t, std::vector<tools::wallet2::transfer_details>> export_outputs() const; std::string export_outputs_to_str() const; - size_t import_outputs(const std::vector<tools::wallet2::transfer_details> &outputs); + size_t import_outputs(const std::pair<size_t, std::vector<tools::wallet2::transfer_details>> &outputs); size_t import_outputs_from_str(const std::string &outputs_st); payment_container export_payments() const; void import_payments(const payment_container &payments); @@ -1071,9 +1117,11 @@ namespace tools std::tuple<size_t, crypto::hash, std::vector<crypto::hash>> export_blockchain() const; void import_blockchain(const std::tuple<size_t, crypto::hash, std::vector<crypto::hash>> &bc); bool export_key_images(const std::string &filename) const; - std::vector<std::pair<crypto::key_image, crypto::signature>> export_key_images() const; - uint64_t import_key_images(const std::vector<std::pair<crypto::key_image, crypto::signature>> &signed_key_images, uint64_t &spent, uint64_t &unspent, bool check_spent = true); + std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> export_key_images(bool all = false) const; + uint64_t import_key_images(const std::vector<std::pair<crypto::key_image, crypto::signature>> &signed_key_images, size_t offset, uint64_t &spent, uint64_t &unspent, bool check_spent = true); uint64_t import_key_images(const std::string &filename, uint64_t &spent, uint64_t &unspent); + bool import_key_images(std::vector<crypto::key_image> key_images); + crypto::public_key get_tx_pub_key_from_received_outs(const tools::wallet2::transfer_details &td) const; void update_pool_state(bool refreshed = false); void remove_obsolete_pool_txs(const std::vector<crypto::hash> &tx_hashes); @@ -1179,6 +1227,11 @@ namespace tools bool unblackball_output(const std::pair<uint64_t, uint64_t> &output); bool is_output_blackballed(const std::pair<uint64_t, uint64_t> &output) const; + // MMS ------------------------------------------------------------------------------------------------- + mms::message_store& get_message_store() { return m_message_store; }; + const mms::message_store& get_message_store() const { return m_message_store; }; + mms::multisig_wallet_state get_multisig_wallet_state() const; + bool lock_keys_file(); bool unlock_keys_file(); bool is_keys_file_locked() const; @@ -1187,6 +1240,8 @@ namespace tools void set_tx_notify(const std::shared_ptr<tools::Notify> ¬ify) { m_tx_notify = notify; } + bool is_tx_spendtime_unlocked(uint64_t unlock_time, uint64_t block_height) const; + private: /*! * \brief Stores wallet information to wallet file. @@ -1202,17 +1257,16 @@ namespace tools * \param password Password of wallet file */ bool load_keys(const std::string& keys_file_name, const epee::wipeable_string& password); - void process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector<uint64_t> &o_indices, uint64_t height, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen, const tx_cache_data &tx_cache_data); - void process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const parsed_block &parsed_block, const crypto::hash& bl_id, uint64_t height, const std::vector<tx_cache_data> &tx_cache_data, size_t tx_cache_data_offset); + void process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector<uint64_t> &o_indices, uint64_t height, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen, const tx_cache_data &tx_cache_data, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL); + void process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const parsed_block &parsed_block, const crypto::hash& bl_id, uint64_t height, const std::vector<tx_cache_data> &tx_cache_data, size_t tx_cache_data_offset, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL); void detach_blockchain(uint64_t height); void get_short_chain_history(std::list<crypto::hash>& ids, uint64_t granularity = 1) const; - bool is_tx_spendtime_unlocked(uint64_t unlock_time, uint64_t block_height) const; bool clear(); void pull_blocks(uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices); void pull_hashes(uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<crypto::hash> &hashes); void fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, bool force = false); void pull_and_parse_next_blocks(uint64_t start_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, const std::vector<cryptonote::block_complete_entry> &prev_blocks, const std::vector<parsed_block> &prev_parsed_blocks, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<parsed_block> &parsed_blocks, bool &error); - void process_parsed_blocks(uint64_t start_height, const std::vector<cryptonote::block_complete_entry> &blocks, const std::vector<parsed_block> &parsed_blocks, uint64_t& blocks_added); + void process_parsed_blocks(uint64_t start_height, const std::vector<cryptonote::block_complete_entry> &blocks, const std::vector<parsed_block> &parsed_blocks, uint64_t& blocks_added, std::map<std::pair<uint64_t, uint64_t>, size_t> *output_tracker_cache = NULL); uint64_t select_transfers(uint64_t needed_money, std::vector<size_t> unused_transfers_indices, std::vector<size_t>& selected_transfers) const; bool prepare_file_names(const std::string& file_path); void process_unconfirmed(const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t height); @@ -1236,13 +1290,12 @@ namespace tools void set_unspent(size_t idx); void get_outs(std::vector<std::vector<get_outs_entry>> &outs, const std::vector<size_t> &selected_transfers, size_t fake_outputs_count); bool tx_add_fake_output(std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, uint64_t global_index, const crypto::public_key& tx_public_key, const rct::key& mask, uint64_t real_index, bool unlocked) const; - crypto::public_key get_tx_pub_key_from_received_outs(const tools::wallet2::transfer_details &td) const; bool should_pick_a_second_output(bool use_rct, size_t n_transfers, const std::vector<size_t> &unused_transfers_indices, const std::vector<size_t> &unused_dust_indices) const; std::vector<size_t> get_only_rct(const std::vector<size_t> &unused_dust_indices, const std::vector<size_t> &unused_transfers_indices) const; - void scan_output(const cryptonote::transaction &tx, const crypto::public_key &tx_pub_key, size_t i, tx_scan_info_t &tx_scan_info, int &num_vouts_received, std::unordered_map<cryptonote::subaddress_index, uint64_t> &tx_money_got_in_outs, std::vector<size_t> &outs); + void scan_output(const cryptonote::transaction &tx, const crypto::public_key &tx_pub_key, size_t i, tx_scan_info_t &tx_scan_info, int &num_vouts_received, std::unordered_map<cryptonote::subaddress_index, uint64_t> &tx_money_got_in_outs, std::vector<size_t> &outs, bool pool); void trim_hashchain(); crypto::key_image get_multisig_composite_key_image(size_t n) const; - rct::multisig_kLRki get_multisig_composite_kLRki(size_t n, const crypto::public_key &ignore, std::unordered_set<rct::key> &used_L, std::unordered_set<rct::key> &new_used_L) const; + rct::multisig_kLRki get_multisig_composite_kLRki(size_t n, const std::unordered_set<crypto::public_key> &ignore_set, std::unordered_set<rct::key> &used_L, std::unordered_set<rct::key> &new_used_L) const; rct::multisig_kLRki get_multisig_kLRki(size_t n, const rct::key &k) const; rct::key get_multisig_k(size_t idx, const std::unordered_set<rct::key> &used_L) const; void update_multisig_rescan_info(const std::vector<std::vector<rct::key>> &multisig_k, const std::vector<std::vector<tools::wallet2::multisig_info>> &info, size_t n); @@ -1253,6 +1306,9 @@ namespace tools crypto::chacha_key get_ringdb_key(); void setup_keys(const epee::wipeable_string &password); + void register_devices(); + hw::device& lookup_device(const std::string & device_descriptor); + bool get_rct_distribution(uint64_t &start_height, std::vector<uint64_t> &distribution); uint64_t get_segregation_fork_height() const; @@ -1264,15 +1320,25 @@ namespace tools std::unordered_set<crypto::public_key> &pkeys) const; void cache_tx_data(const cryptonote::transaction& tx, const crypto::hash &txid, tx_cache_data &tx_cache_data) const; + std::shared_ptr<std::map<std::pair<uint64_t, uint64_t>, size_t>> create_output_tracker_cache() const; void setup_new_blockchain(); void create_keys_file(const std::string &wallet_, bool watch_only, const epee::wipeable_string &password, bool create_address_file); + wallet_device_callback * get_device_callback(); + void on_button_request(); + void on_pin_request(epee::wipeable_string & pin); + void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase); + + std::string get_rpc_status(const std::string &s) const; + void throw_on_rpc_response_error(const boost::optional<std::string> &status, const char *method) const; + cryptonote::account_base m_account; boost::optional<epee::net_utils::http::login> m_daemon_login; std::string m_daemon_address; std::string m_wallet_file; std::string m_keys_file; + std::string m_mms_file; epee::net_utils::http::http_simple_client m_http_client; hashchain m_blockchain; std::unordered_map<crypto::hash, unconfirmed_transfer_details> m_unconfirmed_txs; @@ -1296,6 +1362,7 @@ namespace tools uint64_t m_upper_transaction_weight_limit; //TODO: auto-calc this value or request from daemon, now use some fixed value const std::vector<std::vector<tools::wallet2::multisig_info>> *m_multisig_rescan_info; const std::vector<std::vector<rct::key>> *m_multisig_rescan_k; + std::unordered_map<crypto::public_key, crypto::key_image> m_cold_key_images; std::atomic<bool> m_run; @@ -1341,11 +1408,17 @@ namespace tools bool m_key_reuse_mitigation2; uint64_t m_segregation_height; bool m_ignore_fractional_outputs; + bool m_track_uses; bool m_is_initialized; NodeRPCProxy m_node_rpc_proxy; std::unordered_set<crypto::hash> m_scanned_pool_txs[2]; size_t m_subaddress_lookahead_major, m_subaddress_lookahead_minor; std::string m_device_name; + std::string m_device_derivation_path; + uint64_t m_device_last_key_image_sync; + + // Aux transaction data from device + std::unordered_map<crypto::hash, std::string> m_tx_device; // Light wallet bool m_light_wallet; /* sends view key to daemon for scanning */ @@ -1368,17 +1441,24 @@ namespace tools uint64_t m_last_block_reward; std::unique_ptr<tools::file_locker> m_keys_file_locker; + + mms::message_store m_message_store; + bool m_original_keys_available; + cryptonote::account_public_address m_original_address; + crypto::secret_key m_original_view_secret_key; crypto::chacha_key m_cache_key; boost::optional<epee::wipeable_string> m_encrypt_keys_after_refresh; bool m_unattended; + bool m_devices_registered; std::shared_ptr<tools::Notify> m_tx_notify; + std::unique_ptr<wallet_device_callback> m_device_callback; }; } -BOOST_CLASS_VERSION(tools::wallet2, 25) -BOOST_CLASS_VERSION(tools::wallet2::transfer_details, 9) +BOOST_CLASS_VERSION(tools::wallet2, 28) +BOOST_CLASS_VERSION(tools::wallet2::transfer_details, 11) BOOST_CLASS_VERSION(tools::wallet2::multisig_info, 1) BOOST_CLASS_VERSION(tools::wallet2::multisig_info::LR, 0) BOOST_CLASS_VERSION(tools::wallet2::multisig_tx_set, 1) @@ -1389,7 +1469,7 @@ BOOST_CLASS_VERSION(tools::wallet2::confirmed_transfer_details, 6) BOOST_CLASS_VERSION(tools::wallet2::address_book_row, 17) BOOST_CLASS_VERSION(tools::wallet2::reserve_proof_entry, 0) BOOST_CLASS_VERSION(tools::wallet2::unsigned_tx_set, 0) -BOOST_CLASS_VERSION(tools::wallet2::signed_tx_set, 0) +BOOST_CLASS_VERSION(tools::wallet2::signed_tx_set, 1) BOOST_CLASS_VERSION(tools::wallet2::tx_construction_data, 3) BOOST_CLASS_VERSION(tools::wallet2::pending_tx, 3) BOOST_CLASS_VERSION(tools::wallet2::multisig_sig, 0) @@ -1436,6 +1516,10 @@ namespace boost x.m_multisig_k.clear(); x.m_multisig_info.clear(); } + if (ver < 10) + { + x.m_key_image_request = false; + } } template <class Archive> @@ -1517,6 +1601,15 @@ namespace boost a & x.m_multisig_info; a & x.m_multisig_k; a & x.m_key_image_partial; + if (ver < 10) + { + initialize_transfer_details(a, x, ver); + return; + } + a & x.m_key_image_request; + if (ver < 11) + return; + a & x.m_uses; } template <class Archive> @@ -1713,6 +1806,9 @@ namespace boost { a & x.ptx; a & x.key_images; + if (ver < 1) + return; + a & x.tx_key_images; } template <class Archive> diff --git a/src/wallet/wallet_args.cpp b/src/wallet/wallet_args.cpp index 95a4e0ad6..b9d0a6a75 100644 --- a/src/wallet/wallet_args.cpp +++ b/src/wallet/wallet_args.cpp @@ -211,6 +211,14 @@ namespace wallet_args Print(print) << boost::format(wallet_args::tr("Logging to %s")) % log_path; + const ssize_t lockable_memory = tools::get_lockable_memory(); + if (lockable_memory >= 0 && lockable_memory < 256 * 4096) // 256 pages -> at least 256 secret keys and other such small/medium objects + Print(print) << tr("WARNING: You may not have a high enough lockable memory limit") +#ifdef ELPP_OS_UNIX + << ", " << tr("see ulimit -l") +#endif + ; + return {std::move(vm), should_terminate}; } } diff --git a/src/wallet/wallet_errors.h b/src/wallet/wallet_errors.h index bc518d04a..35862bda1 100644 --- a/src/wallet/wallet_errors.h +++ b/src/wallet/wallet_errors.h @@ -72,6 +72,7 @@ namespace tools // tx_parse_error // get_tx_pool_error // out_of_hashchain_bounds_error + // signature_check_failed // transfer_error * // get_outs_general_error // not_enough_unlocked_money @@ -218,6 +219,14 @@ namespace tools } }; //---------------------------------------------------------------------------------------------------- + struct password_entry_failed : public wallet_runtime_error + { + explicit password_entry_failed(std::string&& loc, const std::string &msg = "Password entry failed") + : wallet_runtime_error(std::move(loc), msg) + { + } + }; + //---------------------------------------------------------------------------------------------------- const char* const file_error_messages[] = { "file already exists", "file not found", @@ -418,6 +427,14 @@ namespace tools std::string to_string() const { return refresh_error::to_string(); } }; //---------------------------------------------------------------------------------------------------- + struct signature_check_failed : public wallet_logic_error + { + explicit signature_check_failed(std::string&& loc, const std::string& message) + : wallet_logic_error(std::move(loc), "Signature check failed " + message) + { + } + }; + //---------------------------------------------------------------------------------------------------- struct transfer_error : public wallet_logic_error { protected: @@ -804,6 +821,31 @@ namespace tools std::string m_wallet_file; }; //---------------------------------------------------------------------------------------------------- + struct mms_error : public wallet_logic_error + { + protected: + explicit mms_error(std::string&& loc, const std::string& message) + : wallet_logic_error(std::move(loc), message) + { + } + }; + //---------------------------------------------------------------------------------------------------- + struct no_connection_to_bitmessage : public mms_error + { + explicit no_connection_to_bitmessage(std::string&& loc, const std::string& address) + : mms_error(std::move(loc), "no connection to PyBitmessage at address " + address) + { + } + }; + //---------------------------------------------------------------------------------------------------- + struct bitmessage_api_error : public mms_error + { + explicit bitmessage_api_error(std::string&& loc, const std::string& error_string) + : mms_error(std::move(loc), "PyBitmessage returned " + error_string) + { + } + }; + //---------------------------------------------------------------------------------------------------- #if !defined(_MSC_VER) diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index 1b63d65b6..5dc6ac0e8 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -644,9 +644,11 @@ namespace tools return false; } + de.original = it->address; de.addr = info.address; de.is_subaddress = info.is_subaddress; de.amount = it->amount; + de.is_integrated = info.has_payment_id; dsts.push_back(de); if (info.has_payment_id) @@ -981,6 +983,12 @@ namespace tools for (auto &ptx: ptxs) { res.tx_hash_list.push_back(epee::string_tools::pod_to_hex(cryptonote::get_transaction_hash(ptx.tx))); + if (req.get_tx_keys) + { + res.tx_key_list.push_back(epee::string_tools::pod_to_hex(ptx.tx_key)); + for (const crypto::secret_key& additional_tx_key : ptx.additional_tx_keys) + res.tx_key_list.back() += epee::string_tools::pod_to_hex(additional_tx_key); + } } if (req.export_raw) @@ -994,6 +1002,171 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_describe_transfer(const wallet_rpc::COMMAND_RPC_DESCRIBE_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_DESCRIBE_TRANSFER::response& res, epee::json_rpc::error& er) + { + if (!m_wallet) return not_open(er); + if (m_restricted) + { + er.code = WALLET_RPC_ERROR_CODE_DENIED; + er.message = "Command unavailable in restricted mode."; + return false; + } + if (m_wallet->key_on_device()) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "command not supported by HW wallet"; + return false; + } + if(m_wallet->watch_only()) + { + er.code = WALLET_RPC_ERROR_CODE_WATCH_ONLY; + er.message = "command not supported by watch-only wallet"; + return false; + } + + tools::wallet2::unsigned_tx_set exported_txs; + try + { + cryptonote::blobdata blob; + if (!epee::string_tools::parse_hexstr_to_binbuff(req.unsigned_txset, blob)) + { + er.code = WALLET_RPC_ERROR_CODE_BAD_HEX; + er.message = "Failed to parse hex."; + return false; + } + if(!m_wallet->parse_unsigned_tx_from_str(blob, exported_txs)) + { + er.code = WALLET_RPC_ERROR_CODE_BAD_UNSIGNED_TX_DATA; + er.message = "cannot load unsigned_txset"; + return false; + } + } + catch (const std::exception &e) + { + er.code = WALLET_RPC_ERROR_CODE_BAD_UNSIGNED_TX_DATA; + er.message = "failed to parse unsigned transfers: " + std::string(e.what()); + return false; + } + + std::vector<tools::wallet2::pending_tx> ptx; + try + { + // gather info to ask the user + std::unordered_map<cryptonote::account_public_address, std::pair<std::string, uint64_t>> dests; + int first_known_non_zero_change_index = -1; + for (size_t n = 0; n < exported_txs.txes.size(); ++n) + { + const tools::wallet2::tx_construction_data &cd = exported_txs.txes[n]; + res.desc.push_back({0, 0, std::numeric_limits<uint32_t>::max(), 0, {}, "", 0, "", 0, 0, ""}); + wallet_rpc::COMMAND_RPC_DESCRIBE_TRANSFER::transfer_description &desc = res.desc.back(); + + std::vector<cryptonote::tx_extra_field> tx_extra_fields; + bool has_encrypted_payment_id = false; + crypto::hash8 payment_id8 = crypto::null_hash8; + if (cryptonote::parse_tx_extra(cd.extra, tx_extra_fields)) + { + cryptonote::tx_extra_nonce extra_nonce; + if (find_tx_extra_field_by_type(tx_extra_fields, extra_nonce)) + { + crypto::hash payment_id; + if(cryptonote::get_encrypted_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id8)) + { + desc.payment_id = epee::string_tools::pod_to_hex(payment_id8); + has_encrypted_payment_id = true; + } + else if (cryptonote::get_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id)) + { + desc.payment_id = epee::string_tools::pod_to_hex(payment_id); + } + } + } + + for (size_t s = 0; s < cd.sources.size(); ++s) + { + desc.amount_in += cd.sources[s].amount; + size_t ring_size = cd.sources[s].outputs.size(); + if (ring_size < desc.ring_size) + desc.ring_size = ring_size; + } + for (size_t d = 0; d < cd.splitted_dsts.size(); ++d) + { + const cryptonote::tx_destination_entry &entry = cd.splitted_dsts[d]; + std::string address = cryptonote::get_account_address_as_str(m_wallet->nettype(), entry.is_subaddress, entry.addr); + if (has_encrypted_payment_id && !entry.is_subaddress) + address = cryptonote::get_account_integrated_address_as_str(m_wallet->nettype(), entry.addr, payment_id8); + auto i = dests.find(entry.addr); + if (i == dests.end()) + dests.insert(std::make_pair(entry.addr, std::make_pair(address, entry.amount))); + else + i->second.second += entry.amount; + desc.amount_out += entry.amount; + } + if (cd.change_dts.amount > 0) + { + auto it = dests.find(cd.change_dts.addr); + if (it == dests.end()) + { + er.code = WALLET_RPC_ERROR_CODE_BAD_UNSIGNED_TX_DATA; + er.message = "Claimed change does not go to a paid address"; + return false; + } + if (it->second.second < cd.change_dts.amount) + { + er.code = WALLET_RPC_ERROR_CODE_BAD_UNSIGNED_TX_DATA; + er.message = "Claimed change is larger than payment to the change address"; + return false; + } + if (cd.change_dts.amount > 0) + { + if (first_known_non_zero_change_index == -1) + first_known_non_zero_change_index = n; + const tools::wallet2::tx_construction_data &cdn = exported_txs.txes[first_known_non_zero_change_index]; + if (memcmp(&cd.change_dts.addr, &cdn.change_dts.addr, sizeof(cd.change_dts.addr))) + { + er.code = WALLET_RPC_ERROR_CODE_BAD_UNSIGNED_TX_DATA; + er.message = "Change goes to more than one address"; + return false; + } + } + desc.change_amount += cd.change_dts.amount; + it->second.second -= cd.change_dts.amount; + if (it->second.second == 0) + dests.erase(cd.change_dts.addr); + } + + size_t n_dummy_outputs = 0; + for (auto i = dests.begin(); i != dests.end(); ) + { + if (i->second.second > 0) + { + desc.recipients.push_back({i->second.first, i->second.second}); + } + else + ++desc.dummy_outputs; + ++i; + } + + if (desc.change_amount > 0) + { + const tools::wallet2::tx_construction_data &cd0 = exported_txs.txes[0]; + desc.change_address = get_account_address_as_str(m_wallet->nettype(), cd0.subaddr_account > 0, cd0.change_dts.addr); + } + + desc.fee = desc.amount_in - desc.amount_out; + desc.unlock_time = cd.unlock_time; + desc.extra = epee::to_hex::string({cd.extra.data(), cd.extra.size()}); + } + } + catch (const std::exception &e) + { + er.code = WALLET_RPC_ERROR_CODE_BAD_UNSIGNED_TX_DATA; + er.message = "failed to parse unsigned transfers"; + return false; + } + + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_submit_transfer(const wallet_rpc::COMMAND_RPC_SUBMIT_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_SUBMIT_TRANSFER::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); @@ -1575,10 +1748,42 @@ namespace tools if (req.key_type.compare("mnemonic") == 0) { epee::wipeable_string seed; - if (!m_wallet->get_seed(seed)) + bool ready; + if (m_wallet->multisig(&ready)) { + if (!ready) + { + er.code = WALLET_RPC_ERROR_CODE_NOT_MULTISIG; + er.message = "This wallet is multisig, but not yet finalized"; + return false; + } + if (!m_wallet->get_multisig_seed(seed)) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Failed to get multisig seed."; + return false; + } + } + else + { + if (m_wallet->watch_only()) + { + er.code = WALLET_RPC_ERROR_CODE_WATCH_ONLY; + er.message = "The wallet is watch-only. Cannot display seed."; + return false; + } + if (!m_wallet->is_deterministic()) + { + er.code = WALLET_RPC_ERROR_CODE_NON_DETERMINISTIC; er.message = "The wallet is non-deterministic. Cannot display seed."; return false; + } + if (!m_wallet->get_seed(seed)) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Failed to get seed."; + return false; + } } res.key = std::string(seed.data(), seed.size()); // send to the network, then wipe RAM :D } @@ -1613,7 +1818,7 @@ namespace tools try { - m_wallet->rescan_blockchain(); + m_wallet->rescan_blockchain(req.hard); } catch (const std::exception& e) { @@ -2292,12 +2497,13 @@ namespace tools if (!m_wallet) return not_open(er); try { - std::vector<std::pair<crypto::key_image, crypto::signature>> ski = m_wallet->export_key_images(); - res.signed_key_images.resize(ski.size()); - for (size_t n = 0; n < ski.size(); ++n) + std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> ski = m_wallet->export_key_images(req.all); + res.offset = ski.first; + res.signed_key_images.resize(ski.second.size()); + for (size_t n = 0; n < ski.second.size(); ++n) { - res.signed_key_images[n].key_image = epee::string_tools::pod_to_hex(ski[n].first); - res.signed_key_images[n].signature = epee::string_tools::pod_to_hex(ski[n].second); + res.signed_key_images[n].key_image = epee::string_tools::pod_to_hex(ski.second[n].first); + res.signed_key_images[n].signature = epee::string_tools::pod_to_hex(ski.second[n].second); } } @@ -2350,7 +2556,7 @@ namespace tools ski[n].second = *reinterpret_cast<const crypto::signature*>(bd.data()); } uint64_t spent = 0, unspent = 0; - uint64_t height = m_wallet->import_key_images(ski, spent, unspent); + uint64_t height = m_wallet->import_key_images(ski, req.offset, spent, unspent); res.spent = spent; res.unspent = unspent; res.height = height; @@ -2687,7 +2893,8 @@ namespace tools cryptonote::COMMAND_RPC_GET_HEIGHT::response hres; hres.height = 0; bool r = wal->invoke_http_json("/getheight", hreq, hres); - wal->set_refresh_from_block_height(hres.height); + if (r) + wal->set_refresh_from_block_height(hres.height); crypto::secret_key dummy_key; try { wal->generate(wallet_file, req.password, dummy_key, false, false); @@ -2905,6 +3112,11 @@ namespace tools er.code = WALLET_RPC_ERROR_CODE_ADDRESS_INDEX_OUT_OF_BOUNDS; er.message = e.what(); } + catch (const error::signature_check_failed& e) + { + er.code = WALLET_RPC_ERROR_CODE_WRONG_SIGNATURE; + er.message = e.what(); + } catch (const std::exception& e) { er.code = default_error_code; @@ -2917,6 +3129,200 @@ namespace tools } } //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_restore_deterministic_wallet(const wallet_rpc::COMMAND_RPC_RESTORE_DETERMINISTIC_WALLET::request &req, wallet_rpc::COMMAND_RPC_RESTORE_DETERMINISTIC_WALLET::response &res, epee::json_rpc::error &er) + { + if (m_wallet_dir.empty()) + { + er.code = WALLET_RPC_ERROR_CODE_NO_WALLET_DIR; + er.message = "No wallet dir configured"; + return false; + } + + // early check for mandatory fields + if (req.filename.empty()) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "field 'filename' is mandatory. Please provide a filename to save the restored wallet to."; + return false; + } + if (req.seed.empty()) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "field 'seed' is mandatory. Please provide a seed you want to restore from."; + return false; + } + + namespace po = boost::program_options; + po::variables_map vm2; + const char *ptr = strchr(req.filename.c_str(), '/'); + #ifdef _WIN32 + if (!ptr) + ptr = strchr(req.filename.c_str(), '\\'); + if (!ptr) + ptr = strchr(req.filename.c_str(), ':'); + #endif + if (ptr) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Invalid filename"; + return false; + } + std::string wallet_file = m_wallet_dir + "/" + req.filename; + // check if wallet file already exists + if (!wallet_file.empty()) + { + try + { + boost::system::error_code ignored_ec; + THROW_WALLET_EXCEPTION_IF(boost::filesystem::exists(wallet_file, ignored_ec), error::file_exists, wallet_file); + } + catch (const std::exception &e) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Wallet already exists."; + return false; + } + } + crypto::secret_key recovery_key; + std::string old_language; + + // check the given seed + { + if (!crypto::ElectrumWords::words_to_bytes(req.seed, recovery_key, old_language)) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Electrum-style word list failed verification"; + return false; + } + } + + // process seed_offset if given + { + if (!req.seed_offset.empty()) + { + recovery_key = cryptonote::decrypt_key(recovery_key, req.seed_offset); + } + } + { + po::options_description desc("dummy"); + const command_line::arg_descriptor<std::string, true> arg_password = {"password", "password"}; + const char *argv[4]; + int argc = 3; + argv[0] = "wallet-rpc"; + argv[1] = "--password"; + argv[2] = req.password.c_str(); + argv[3] = NULL; + vm2 = *m_vm; + command_line::add_arg(desc, arg_password); + po::store(po::parse_command_line(argc, argv, desc), vm2); + } + + auto rc = tools::wallet2::make_new(vm2, true, nullptr); + std::unique_ptr<wallet2> wal; + wal = std::move(rc.first); + if (!wal) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Failed to create wallet"; + return false; + } + + epee::wipeable_string password = rc.second.password(); + + bool was_deprecated_wallet = ((old_language == crypto::ElectrumWords::old_language_name) || + crypto::ElectrumWords::get_is_old_style_seed(req.seed)); + + std::string mnemonic_language = old_language; + if (was_deprecated_wallet) + { + // The user had used an older version of the wallet with old style mnemonics. + res.was_deprecated = true; + } + + if (old_language == crypto::ElectrumWords::old_language_name) + { + if (req.language.empty()) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Wallet was using the old seed language. You need to specify a new seed language."; + return false; + } + std::vector<std::string> language_list; + std::vector<std::string> language_list_en; + crypto::ElectrumWords::get_language_list(language_list); + crypto::ElectrumWords::get_language_list(language_list_en, true); + if (std::find(language_list.begin(), language_list.end(), req.language) == language_list.end() && + std::find(language_list_en.begin(), language_list_en.end(), req.language) == language_list_en.end()) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Wallet was using the old seed language, and the specified new seed language is invalid."; + return false; + } + mnemonic_language = req.language; + } + + wal->set_seed_language(mnemonic_language); + + crypto::secret_key recovery_val; + try + { + recovery_val = wal->generate(wallet_file, std::move(rc.second).password(), recovery_key, true, false, false); + MINFO("Wallet has been restored.\n"); + } + catch (const std::exception &e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + + // // Convert the secret key back to seed + epee::wipeable_string electrum_words; + if (!crypto::ElectrumWords::bytes_to_words(recovery_val, electrum_words, mnemonic_language)) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Failed to encode seed"; + return false; + } + res.seed = electrum_words.data(); + + if (!wal) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Failed to generate wallet"; + return false; + } + + // set blockheight if given + try + { + wal->set_refresh_from_block_height(req.restore_height); + wal->rewrite(wallet_file, password); + } + catch (const std::exception &e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + + if (m_wallet) + { + try + { + m_wallet->store(); + } + catch (const std::exception &e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + delete m_wallet; + } + m_wallet = wal.release(); + res.address = m_wallet->get_account().get_public_address_str(m_wallet->nettype()); + res.info = "Wallet has been restored successfully."; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_is_multisig(const wallet_rpc::COMMAND_RPC_IS_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_IS_MULTISIG::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); diff --git a/src/wallet/wallet_rpc_server.h b/src/wallet/wallet_rpc_server.h index 35ea11902..abbbe82c5 100644 --- a/src/wallet/wallet_rpc_server.h +++ b/src/wallet/wallet_rpc_server.h @@ -87,6 +87,7 @@ namespace tools MAP_JON_RPC_WE("transfer", on_transfer, wallet_rpc::COMMAND_RPC_TRANSFER) MAP_JON_RPC_WE("transfer_split", on_transfer_split, wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT) MAP_JON_RPC_WE("sign_transfer", on_sign_transfer, wallet_rpc::COMMAND_RPC_SIGN_TRANSFER) + MAP_JON_RPC_WE("describe_transfer", on_describe_transfer, wallet_rpc::COMMAND_RPC_DESCRIBE_TRANSFER) MAP_JON_RPC_WE("submit_transfer", on_submit_transfer, wallet_rpc::COMMAND_RPC_SUBMIT_TRANSFER) MAP_JON_RPC_WE("sweep_dust", on_sweep_dust, wallet_rpc::COMMAND_RPC_SWEEP_DUST) MAP_JON_RPC_WE("sweep_unmixable", on_sweep_dust, wallet_rpc::COMMAND_RPC_SWEEP_DUST) @@ -136,6 +137,7 @@ namespace tools MAP_JON_RPC_WE("open_wallet", on_open_wallet, wallet_rpc::COMMAND_RPC_OPEN_WALLET) MAP_JON_RPC_WE("close_wallet", on_close_wallet, wallet_rpc::COMMAND_RPC_CLOSE_WALLET) MAP_JON_RPC_WE("change_wallet_password", on_change_wallet_password, wallet_rpc::COMMAND_RPC_CHANGE_WALLET_PASSWORD) + MAP_JON_RPC_WE("restore_deterministic_wallet", on_restore_deterministic_wallet, wallet_rpc::COMMAND_RPC_RESTORE_DETERMINISTIC_WALLET) MAP_JON_RPC_WE("is_multisig", on_is_multisig, wallet_rpc::COMMAND_RPC_IS_MULTISIG) MAP_JON_RPC_WE("prepare_multisig", on_prepare_multisig, wallet_rpc::COMMAND_RPC_PREPARE_MULTISIG) MAP_JON_RPC_WE("make_multisig", on_make_multisig, wallet_rpc::COMMAND_RPC_MAKE_MULTISIG) @@ -167,6 +169,7 @@ namespace tools bool on_transfer(const wallet_rpc::COMMAND_RPC_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_TRANSFER::response& res, epee::json_rpc::error& er); bool on_transfer_split(const wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT::request& req, wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT::response& res, epee::json_rpc::error& er); bool on_sign_transfer(const wallet_rpc::COMMAND_RPC_SIGN_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_SIGN_TRANSFER::response& res, epee::json_rpc::error& er); + bool on_describe_transfer(const wallet_rpc::COMMAND_RPC_DESCRIBE_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_DESCRIBE_TRANSFER::response& res, epee::json_rpc::error& er); bool on_submit_transfer(const wallet_rpc::COMMAND_RPC_SUBMIT_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_SUBMIT_TRANSFER::response& res, epee::json_rpc::error& er); bool on_sweep_dust(const wallet_rpc::COMMAND_RPC_SWEEP_DUST::request& req, wallet_rpc::COMMAND_RPC_SWEEP_DUST::response& res, epee::json_rpc::error& er); bool on_sweep_all(const wallet_rpc::COMMAND_RPC_SWEEP_ALL::request& req, wallet_rpc::COMMAND_RPC_SWEEP_ALL::response& res, epee::json_rpc::error& er); @@ -214,6 +217,7 @@ namespace tools bool on_open_wallet(const wallet_rpc::COMMAND_RPC_OPEN_WALLET::request& req, wallet_rpc::COMMAND_RPC_OPEN_WALLET::response& res, epee::json_rpc::error& er); bool on_close_wallet(const wallet_rpc::COMMAND_RPC_CLOSE_WALLET::request& req, wallet_rpc::COMMAND_RPC_CLOSE_WALLET::response& res, epee::json_rpc::error& er); bool on_change_wallet_password(const wallet_rpc::COMMAND_RPC_CHANGE_WALLET_PASSWORD::request& req, wallet_rpc::COMMAND_RPC_CHANGE_WALLET_PASSWORD::response& res, epee::json_rpc::error& er); + bool on_restore_deterministic_wallet(const wallet_rpc::COMMAND_RPC_RESTORE_DETERMINISTIC_WALLET::request& req, wallet_rpc::COMMAND_RPC_RESTORE_DETERMINISTIC_WALLET::response& res, epee::json_rpc::error& er); bool on_is_multisig(const wallet_rpc::COMMAND_RPC_IS_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_IS_MULTISIG::response& res, epee::json_rpc::error& er); bool on_prepare_multisig(const wallet_rpc::COMMAND_RPC_PREPARE_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_PREPARE_MULTISIG::response& res, epee::json_rpc::error& er); bool on_make_multisig(const wallet_rpc::COMMAND_RPC_MAKE_MULTISIG::request& req, wallet_rpc::COMMAND_RPC_MAKE_MULTISIG::response& res, epee::json_rpc::error& er); diff --git a/src/wallet/wallet_rpc_server_commands_defs.h b/src/wallet/wallet_rpc_server_commands_defs.h index 2377b69e3..f0c1a4e9d 100644 --- a/src/wallet/wallet_rpc_server_commands_defs.h +++ b/src/wallet/wallet_rpc_server_commands_defs.h @@ -47,7 +47,7 @@ // advance which version they will stop working with // Don't go over 32767 for any of these #define WALLET_RPC_VERSION_MAJOR 1 -#define WALLET_RPC_VERSION_MINOR 4 +#define WALLET_RPC_VERSION_MINOR 7 #define MAKE_WALLET_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define WALLET_RPC_VERSION MAKE_WALLET_RPC_VERSION(WALLET_RPC_VERSION_MAJOR, WALLET_RPC_VERSION_MINOR) namespace tools @@ -531,16 +531,79 @@ namespace wallet_rpc }; }; + struct COMMAND_RPC_DESCRIBE_TRANSFER + { + struct recipient + { + std::string address; + uint64_t amount; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(address) + KV_SERIALIZE(amount) + END_KV_SERIALIZE_MAP() + }; + + struct transfer_description + { + uint64_t amount_in; + uint64_t amount_out; + uint32_t ring_size; + uint64_t unlock_time; + std::list<recipient> recipients; + std::string payment_id; + uint64_t change_amount; + std::string change_address; + uint64_t fee; + uint32_t dummy_outputs; + std::string extra; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(amount_in) + KV_SERIALIZE(amount_out) + KV_SERIALIZE(ring_size) + KV_SERIALIZE(unlock_time) + KV_SERIALIZE(recipients) + KV_SERIALIZE(payment_id) + KV_SERIALIZE(change_amount) + KV_SERIALIZE(change_address) + KV_SERIALIZE(fee) + KV_SERIALIZE(dummy_outputs) + KV_SERIALIZE(extra) + END_KV_SERIALIZE_MAP() + }; + + struct request + { + std::string unsigned_txset; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(unsigned_txset) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::list<transfer_description> desc; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(desc) + END_KV_SERIALIZE_MAP() + }; + }; + struct COMMAND_RPC_SIGN_TRANSFER { struct request { std::string unsigned_txset; bool export_raw; + bool get_tx_keys; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(unsigned_txset) KV_SERIALIZE_OPT(export_raw, false) + KV_SERIALIZE_OPT(get_tx_keys, false) END_KV_SERIALIZE_MAP() }; @@ -549,11 +612,13 @@ namespace wallet_rpc std::string signed_txset; std::list<std::string> tx_hash_list; std::list<std::string> tx_raw_list; + std::list<std::string> tx_key_list; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(signed_txset) KV_SERIALIZE(tx_hash_list) KV_SERIALIZE(tx_raw_list) + KV_SERIALIZE(tx_key_list) END_KV_SERIALIZE_MAP() }; }; @@ -992,7 +1057,10 @@ namespace wallet_rpc { struct request { + bool hard; + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_OPT(hard, false); END_KV_SERIALIZE_MAP() }; @@ -1497,7 +1565,10 @@ namespace wallet_rpc { struct request { + bool all; + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_OPT(all, false); END_KV_SERIALIZE_MAP() }; @@ -1514,9 +1585,11 @@ namespace wallet_rpc struct response { + uint32_t offset; std::vector<signed_key_image> signed_key_images; BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(offset); KV_SERIALIZE(signed_key_images); END_KV_SERIALIZE_MAP() }; @@ -1537,9 +1610,11 @@ namespace wallet_rpc struct request { + uint32_t offset; std::vector<signed_key_image> signed_key_images; BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_OPT(offset, (uint32_t)0); KV_SERIALIZE(signed_key_images); END_KV_SERIALIZE_MAP() }; @@ -1859,6 +1934,43 @@ namespace wallet_rpc }; }; + struct COMMAND_RPC_RESTORE_DETERMINISTIC_WALLET + { + struct request + { + uint64_t restore_height; + std::string filename; + std::string seed; + std::string seed_offset; + std::string password; + std::string language; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_OPT(restore_height, (uint64_t)0) + KV_SERIALIZE(filename) + KV_SERIALIZE(seed) + KV_SERIALIZE(seed_offset) + KV_SERIALIZE(password) + KV_SERIALIZE(language) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string address; + std::string seed; + std::string info; + bool was_deprecated; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(address) + KV_SERIALIZE(seed) + KV_SERIALIZE(info) + KV_SERIALIZE(was_deprecated) + END_KV_SERIALIZE_MAP() + }; + }; + struct COMMAND_RPC_IS_MULTISIG { struct request diff --git a/src/wallet/wallet_rpc_server_error_codes.h b/src/wallet/wallet_rpc_server_error_codes.h index f127ae240..9b3a2847d 100644 --- a/src/wallet/wallet_rpc_server_error_codes.h +++ b/src/wallet/wallet_rpc_server_error_codes.h @@ -73,3 +73,4 @@ #define WALLET_RPC_ERROR_CODE_BAD_SIGNED_TX_DATA -40 #define WALLET_RPC_ERROR_CODE_SIGNED_SUBMISSION -41 #define WALLET_RPC_ERROR_CODE_SIGN_UNSIGNED -42 +#define WALLET_RPC_ERROR_CODE_NON_DETERMINISTIC -43 |