diff options
Diffstat (limited to 'src')
80 files changed, 4320 insertions, 1170 deletions
diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp index c25798c1e..d3eefef6e 100644 --- a/src/blockchain_db/blockchain_db.cpp +++ b/src/blockchain_db/blockchain_db.cpp @@ -267,7 +267,10 @@ void BlockchainDB::pop_block(block& blk, std::vector<transaction>& txs) for (const auto& h : boost::adaptors::reverse(blk.tx_hashes)) { - txs.push_back(get_tx(h)); + cryptonote::transaction tx; + if (!get_tx(h, tx) && !get_pruned_tx(h, tx)) + throw DB_ERROR("Failed to get pruned or unpruned transaction from the db"); + txs.push_back(std::move(tx)); remove_transaction(h); } remove_transaction(get_transaction_hash(blk.miner_tx)); @@ -280,7 +283,7 @@ bool BlockchainDB::is_open() const void BlockchainDB::remove_transaction(const crypto::hash& tx_hash) { - transaction tx = get_tx(tx_hash); + transaction tx = get_pruned_tx(tx_hash); for (const txin_v& tx_input : tx.vin) { @@ -325,6 +328,17 @@ bool BlockchainDB::get_tx(const crypto::hash& h, cryptonote::transaction &tx) co return true; } +bool BlockchainDB::get_pruned_tx(const crypto::hash& h, cryptonote::transaction &tx) const +{ + blobdata bd; + if (!get_pruned_tx_blob(h, bd)) + return false; + if (!parse_and_validate_tx_base_from_blob(bd, tx)) + throw DB_ERROR("Failed to parse transaction base from blob retrieved from the db"); + + return true; +} + transaction BlockchainDB::get_tx(const crypto::hash& h) const { transaction tx; @@ -333,6 +347,14 @@ transaction BlockchainDB::get_tx(const crypto::hash& h) const return tx; } +transaction BlockchainDB::get_pruned_tx(const crypto::hash& h) const +{ + transaction tx; + if (!get_pruned_tx(h, tx)) + throw TX_DNE(std::string("pruned tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str()); + return tx; +} + void BlockchainDB::reset_stats() { num_calls = 0; diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 9d4f59b3c..137d5958a 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -1126,6 +1126,17 @@ public: virtual transaction get_tx(const crypto::hash& h) const; /** + * @brief fetches the transaction base with the given hash + * + * If the transaction does not exist, the subclass should throw TX_DNE. + * + * @param h the hash to look for + * + * @return the transaction with the given hash + */ + virtual transaction get_pruned_tx(const crypto::hash& h) const; + + /** * @brief fetches the transaction with the given hash * * If the transaction does not exist, the subclass should return false. @@ -1137,6 +1148,17 @@ public: virtual bool get_tx(const crypto::hash& h, transaction &tx) const; /** + * @brief fetches the transaction base with the given hash + * + * If the transaction does not exist, the subclass should return false. + * + * @param h the hash to look for + * + * @return true iff the transaction was found + */ + virtual bool get_pruned_tx(const crypto::hash& h, transaction &tx) const; + + /** * @brief fetches the transaction blob with the given hash * * The subclass should return the transaction stored which has the given @@ -1165,6 +1187,21 @@ public: virtual bool get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const = 0; /** + * @brief fetches the prunable transaction blob with the given hash + * + * The subclass should return the prunable transaction stored which has the given + * hash. + * + * If the transaction does not exist, or if we do not have that prunable data, + * the subclass should return false. + * + * @param h the hash to look for + * + * @return true iff the transaction was found and we have its prunable data + */ + virtual bool get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const = 0; + + /** * @brief fetches the prunable transaction hash * * The subclass should return the hash of the prunable transaction data. @@ -1258,7 +1295,7 @@ public: * * @return the requested output data */ - virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index) const = 0; + virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index, bool include_commitmemt = true) const = 0; /** * @brief gets an output's tx hash and index @@ -1329,10 +1366,11 @@ public: * If an output cannot be found, the subclass should throw OUTPUT_DNE. * * @param tx_id a transaction ID + * @param n_txes how many txes to get data for, starting with tx_id * * @return a list of amount-specific output indices */ - virtual std::vector<uint64_t> get_tx_amount_output_indices(const uint64_t tx_id) const = 0; + virtual std::vector<std::vector<uint64_t>> get_tx_amount_output_indices(const uint64_t tx_id, size_t n_txes = 1) const = 0; /** * @brief check if a key image is stored as spent @@ -1412,6 +1450,31 @@ public: virtual void prune_outputs(uint64_t amount) = 0; /** + * @brief get the blockchain pruning seed + * @return the blockchain pruning seed + */ + virtual uint32_t get_blockchain_pruning_seed() const = 0; + + /** + * @brief prunes the blockchain + * @param pruning_seed the seed to use, 0 for default (highly recommended) + * @return success iff true + */ + virtual bool prune_blockchain(uint32_t pruning_seed = 0) = 0; + + /** + * @brief prunes recent blockchain changes as needed, iff pruning is enabled + * @return success iff true + */ + virtual bool update_pruning() = 0; + + /** + * @brief checks pruning was done correctly, iff enabled + * @return success iff true + */ + virtual bool check_pruning() = 0; + + /** * @brief runs a function over all txpool transactions * * The subclass should run the passed function for each txpool tx it has diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 77f8ade7f..8a1303be9 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -35,6 +35,7 @@ #include "string_tools.h" #include "file_io_utils.h" #include "common/util.h" +#include "common/pruning.h" #include "cryptonote_basic/cryptonote_format_utils.h" #include "crypto/crypto.h" #include "profile_tools.h" @@ -130,14 +131,20 @@ private: std::unique_ptr<char[]> data; }; -int compare_uint64(const MDB_val *a, const MDB_val *b) +} + +namespace cryptonote { - const uint64_t va = *(const uint64_t *)a->mv_data; - const uint64_t vb = *(const uint64_t *)b->mv_data; + +int BlockchainLMDB::compare_uint64(const MDB_val *a, const MDB_val *b) +{ + uint64_t va, vb; + memcpy(&va, a->mv_data, sizeof(va)); + memcpy(&vb, b->mv_data, sizeof(vb)); return (va < vb) ? -1 : va > vb; } -int compare_hash32(const MDB_val *a, const MDB_val *b) +int BlockchainLMDB::compare_hash32(const MDB_val *a, const MDB_val *b) { uint32_t *va = (uint32_t*) a->mv_data; uint32_t *vb = (uint32_t*) b->mv_data; @@ -151,13 +158,18 @@ int compare_hash32(const MDB_val *a, const MDB_val *b) return 0; } -int compare_string(const MDB_val *a, const MDB_val *b) +int BlockchainLMDB::compare_string(const MDB_val *a, const MDB_val *b) { const char *va = (const char*) a->mv_data; const char *vb = (const char*) b->mv_data; return strcmp(va, vb); } +} + +namespace +{ + /* DB schema: * * Table Key Data @@ -169,6 +181,7 @@ int compare_string(const MDB_val *a, const MDB_val *b) * txs_pruned txn ID pruned txn blob * txs_prunable txn ID prunable txn blob * txs_prunable_hash txn ID prunable txn hash + * txs_prunable_tip txn ID height * tx_indices txn hash {txn ID, metadata} * tx_outputs txn ID [txn amount output indices] * @@ -196,6 +209,7 @@ const char* const LMDB_TXS = "txs"; const char* const LMDB_TXS_PRUNED = "txs_pruned"; const char* const LMDB_TXS_PRUNABLE = "txs_prunable"; const char* const LMDB_TXS_PRUNABLE_HASH = "txs_prunable_hash"; +const char* const LMDB_TXS_PRUNABLE_TIP = "txs_prunable_tip"; const char* const LMDB_TX_INDICES = "tx_indices"; const char* const LMDB_TX_OUTPUTS = "tx_outputs"; @@ -279,11 +293,6 @@ typedef struct blk_height { uint64_t bh_height; } blk_height; -typedef struct txindex { - crypto::hash key; - tx_data_t data; -} txindex; - typedef struct pre_rct_outkey { uint64_t amount_index; uint64_t output_id; @@ -549,18 +558,18 @@ bool BlockchainLMDB::need_resize(uint64_t threshold_size) const // additional size needed. uint64_t size_used = mst.ms_psize * mei.me_last_pgno; - LOG_PRINT_L1("DB map size: " << mei.me_mapsize); - LOG_PRINT_L1("Space used: " << size_used); - LOG_PRINT_L1("Space remaining: " << mei.me_mapsize - size_used); - LOG_PRINT_L1("Size threshold: " << threshold_size); + MDEBUG("DB map size: " << mei.me_mapsize); + MDEBUG("Space used: " << size_used); + MDEBUG("Space remaining: " << mei.me_mapsize - size_used); + MDEBUG("Size threshold: " << threshold_size); float resize_percent = RESIZE_PERCENT; - LOG_PRINT_L1(boost::format("Percent used: %.04f Percent threshold: %.04f") % ((double)size_used/mei.me_mapsize) % resize_percent); + MDEBUG(boost::format("Percent used: %.04f Percent threshold: %.04f") % ((double)size_used/mei.me_mapsize) % resize_percent); if (threshold_size > 0) { if (mei.me_mapsize - size_used < threshold_size) { - LOG_PRINT_L1("Threshold met (size-based)"); + MINFO("Threshold met (size-based)"); return true; } else @@ -569,7 +578,7 @@ bool BlockchainLMDB::need_resize(uint64_t threshold_size) const if ((double)size_used / mei.me_mapsize > resize_percent) { - LOG_PRINT_L1("Threshold met (percent-based)"); + MINFO("Threshold met (percent-based)"); return true; } return false; @@ -581,7 +590,7 @@ bool BlockchainLMDB::need_resize(uint64_t threshold_size) const void BlockchainLMDB::check_and_resize_for_batch(uint64_t batch_num_blocks, uint64_t batch_bytes) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); - LOG_PRINT_L1("[" << __func__ << "] " << "checking DB size"); + MTRACE("[" << __func__ << "] " << "checking DB size"); const uint64_t min_increase_size = 512 * (1 << 20); uint64_t threshold_size = 0; uint64_t increase_size = 0; @@ -811,6 +820,7 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons CURSOR(txs_pruned) CURSOR(txs_prunable) CURSOR(txs_prunable_hash) + CURSOR(txs_prunable_tip) CURSOR(tx_indices) MDB_val_set(val_tx_id, tx_id); @@ -858,6 +868,14 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons if (result) throw0(DB_ERROR(lmdb_error("Failed to add prunable tx blob to db transaction: ", result).c_str())); + if (get_blockchain_pruning_seed()) + { + MDB_val_set(val_height, m_height); + result = mdb_cursor_put(m_cur_txs_prunable_tip, &val_tx_id, &val_height, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to add prunable tx id to db transaction: ", result).c_str())); + } + if (tx.version > 1) { MDB_val_set(val_prunable_hash, tx_prunable_hash); @@ -883,6 +901,7 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const CURSOR(txs_pruned) CURSOR(txs_prunable) CURSOR(txs_prunable_hash) + CURSOR(txs_prunable_tip) CURSOR(tx_outputs) MDB_val_set(val_h, tx_hash); @@ -898,11 +917,25 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const if (result) throw1(DB_ERROR(lmdb_error("Failed to add removal of pruned tx to db transaction: ", result).c_str())); - if ((result = mdb_cursor_get(m_cur_txs_prunable, &val_tx_id, NULL, MDB_SET))) + result = mdb_cursor_get(m_cur_txs_prunable, &val_tx_id, NULL, MDB_SET); + if (result == 0) + { + result = mdb_cursor_del(m_cur_txs_prunable, 0); + if (result) + throw1(DB_ERROR(lmdb_error("Failed to add removal of prunable tx to db transaction: ", result).c_str())); + } + else if (result != MDB_NOTFOUND) throw1(DB_ERROR(lmdb_error("Failed to locate prunable tx for removal: ", result).c_str())); - result = mdb_cursor_del(m_cur_txs_prunable, 0); - if (result) - throw1(DB_ERROR(lmdb_error("Failed to add removal of prunable tx to db transaction: ", result).c_str())); + + result = mdb_cursor_get(m_cur_txs_prunable_tip, &val_tx_id, NULL, MDB_SET); + if (result && result != MDB_NOTFOUND) + throw1(DB_ERROR(lmdb_error("Failed to locate tx id for removal: ", result).c_str())); + if (result == 0) + { + result = mdb_cursor_del(m_cur_txs_prunable_tip, 0); + if (result) + throw1(DB_ERROR(lmdb_error("Error adding removal of tx id to db transaction", result).c_str())); + } if (tx.version > 1) { @@ -1025,7 +1058,8 @@ void BlockchainLMDB::remove_tx_outputs(const uint64_t tx_id, const transaction& { LOG_PRINT_L3("BlockchainLMDB::" << __func__); - std::vector<uint64_t> amount_output_indices = get_tx_amount_output_indices(tx_id); + std::vector<std::vector<uint64_t>> amount_output_indices_set = get_tx_amount_output_indices(tx_id, 1); + const std::vector<uint64_t> &amount_output_indices = amount_output_indices_set.front(); if (amount_output_indices.empty()) { @@ -1307,6 +1341,7 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) // open necessary databases, and set properties as needed // uses macros to avoid having to change things too many places + // also change blockchain_prune.cpp to match lmdb_db_open(txn, LMDB_BLOCKS, MDB_INTEGERKEY | MDB_CREATE, m_blocks, "Failed to open db handle for m_blocks"); lmdb_db_open(txn, LMDB_BLOCK_INFO, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for m_block_info"); @@ -1315,7 +1350,9 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) lmdb_db_open(txn, LMDB_TXS, MDB_INTEGERKEY | MDB_CREATE, m_txs, "Failed to open db handle for m_txs"); lmdb_db_open(txn, LMDB_TXS_PRUNED, MDB_INTEGERKEY | MDB_CREATE, m_txs_pruned, "Failed to open db handle for m_txs_pruned"); lmdb_db_open(txn, LMDB_TXS_PRUNABLE, MDB_INTEGERKEY | MDB_CREATE, m_txs_prunable, "Failed to open db handle for m_txs_prunable"); - lmdb_db_open(txn, LMDB_TXS_PRUNABLE_HASH, MDB_INTEGERKEY | MDB_CREATE, m_txs_prunable_hash, "Failed to open db handle for m_txs_prunable_hash"); + lmdb_db_open(txn, LMDB_TXS_PRUNABLE_HASH, MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_txs_prunable_hash, "Failed to open db handle for m_txs_prunable_hash"); + if (!(mdb_flags & MDB_RDONLY)) + lmdb_db_open(txn, LMDB_TXS_PRUNABLE_TIP, MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_txs_prunable_tip, "Failed to open db handle for m_txs_prunable_tip"); lmdb_db_open(txn, LMDB_TX_INDICES, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_tx_indices, "Failed to open db handle for m_tx_indices"); lmdb_db_open(txn, LMDB_TX_OUTPUTS, MDB_INTEGERKEY | MDB_CREATE, m_tx_outputs, "Failed to open db handle for m_tx_outputs"); @@ -1343,6 +1380,10 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) mdb_set_dupsort(txn, m_output_amounts, compare_uint64); mdb_set_dupsort(txn, m_output_txs, compare_uint64); mdb_set_dupsort(txn, m_block_info, compare_uint64); + if (!(mdb_flags & MDB_RDONLY)) + mdb_set_dupsort(txn, m_txs_prunable_tip, compare_uint64); + mdb_set_compare(txn, m_txs_prunable, compare_uint64); + mdb_set_dupsort(txn, m_txs_prunable_hash, compare_uint64); mdb_set_compare(txn, m_txpool_meta, compare_hash32); mdb_set_compare(txn, m_txpool_blob, compare_hash32); @@ -1501,6 +1542,8 @@ void BlockchainLMDB::reset() throw0(DB_ERROR(lmdb_error("Failed to drop m_txs_prunable: ", result).c_str())); if (auto result = mdb_drop(txn, m_txs_prunable_hash, 0)) throw0(DB_ERROR(lmdb_error("Failed to drop m_txs_prunable_hash: ", result).c_str())); + if (auto result = mdb_drop(txn, m_txs_prunable_tip, 0)) + throw0(DB_ERROR(lmdb_error("Failed to drop m_txs_prunable_tip: ", result).c_str())); if (auto result = mdb_drop(txn, m_tx_indices, 0)) throw0(DB_ERROR(lmdb_error("Failed to drop m_tx_indices: ", result).c_str())); if (auto result = mdb_drop(txn, m_tx_outputs, 0)) @@ -1826,6 +1869,290 @@ cryptonote::blobdata BlockchainLMDB::get_txpool_tx_blob(const crypto::hash& txid return bd; } +uint32_t BlockchainLMDB::get_blockchain_pruning_seed() const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + RCURSOR(properties) + MDB_val_str(k, "pruning_seed"); + MDB_val v; + int result = mdb_cursor_get(m_cur_properties, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + return 0; + if (result) + throw0(DB_ERROR(lmdb_error("Failed to retrieve pruning seed: ", result).c_str())); + if (v.mv_size != sizeof(uint32_t)) + throw0(DB_ERROR("Failed to retrieve or create pruning seed: unexpected value size")); + uint32_t pruning_seed; + memcpy(&pruning_seed, v.mv_data, sizeof(pruning_seed)); + TXN_POSTFIX_RDONLY(); + return pruning_seed; +} + +static bool is_v1_tx(MDB_cursor *c_txs_pruned, MDB_val *tx_id) +{ + MDB_val v; + int ret = mdb_cursor_get(c_txs_pruned, tx_id, &v, MDB_SET); + if (ret) + throw0(DB_ERROR(lmdb_error("Failed to find transaction pruned data: ", ret).c_str())); + if (v.mv_size == 0) + throw0(DB_ERROR("Invalid transaction pruned data")); + return cryptonote::is_v1_tx(cryptonote::blobdata_ref{(const char*)v.mv_data, v.mv_size}); +} + +enum { prune_mode_prune, prune_mode_update, prune_mode_check }; + +bool BlockchainLMDB::prune_worker(int mode, uint32_t pruning_seed) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + const uint32_t log_stripes = tools::get_pruning_log_stripes(pruning_seed); + if (log_stripes && log_stripes != CRYPTONOTE_PRUNING_LOG_STRIPES) + throw0(DB_ERROR("Pruning seed not in range")); + pruning_seed = tools::get_pruning_stripe(pruning_seed);; + if (pruning_seed > (1ul << CRYPTONOTE_PRUNING_LOG_STRIPES)) + throw0(DB_ERROR("Pruning seed not in range")); + check_open(); + + TIME_MEASURE_START(t); + + size_t n_total_records = 0, n_prunable_records = 0, n_pruned_records = 0; + uint64_t n_bytes = 0; + + mdb_txn_safe txn; + auto result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + + MDB_stat db_stats; + if ((result = mdb_stat(txn, m_txs_prunable, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_txs_prunable: ", result).c_str())); + const size_t pages0 = db_stats.ms_branch_pages + db_stats.ms_leaf_pages + db_stats.ms_overflow_pages; + + MDB_val_str(k, "pruning_seed"); + MDB_val v; + result = mdb_get(txn, m_properties, &k, &v); + bool prune_tip_table = false; + if (result == MDB_NOTFOUND) + { + // not pruned yet + if (mode != prune_mode_prune) + { + txn.abort(); + TIME_MEASURE_FINISH(t); + MDEBUG("Pruning not enabled, nothing to do"); + return true; + } + if (pruning_seed == 0) + pruning_seed = tools::get_random_stripe(); + pruning_seed = tools::make_pruning_seed(pruning_seed, CRYPTONOTE_PRUNING_LOG_STRIPES); + v.mv_data = &pruning_seed; + v.mv_size = sizeof(pruning_seed); + result = mdb_put(txn, m_properties, &k, &v, 0); + if (result) + throw0(DB_ERROR("Failed to save pruning seed")); + prune_tip_table = false; + } + else if (result == 0) + { + // pruned already + if (v.mv_size != sizeof(uint32_t)) + throw0(DB_ERROR("Failed to retrieve or create pruning seed: unexpected value size")); + const uint32_t data = *(const uint32_t*)v.mv_data; + if (pruning_seed == 0) + pruning_seed = tools::get_pruning_stripe(data); + if (tools::get_pruning_stripe(data) != pruning_seed) + throw0(DB_ERROR("Blockchain already pruned with different seed")); + if (tools::get_pruning_log_stripes(data) != CRYPTONOTE_PRUNING_LOG_STRIPES) + throw0(DB_ERROR("Blockchain already pruned with different base")); + pruning_seed = tools::make_pruning_seed(pruning_seed, CRYPTONOTE_PRUNING_LOG_STRIPES); + prune_tip_table = (mode == prune_mode_update); + } + else + { + throw0(DB_ERROR(lmdb_error("Failed to retrieve or create pruning seed: ", result).c_str())); + } + + if (mode == prune_mode_check) + MINFO("Checking blockchain pruning..."); + else + MINFO("Pruning blockchain..."); + + MDB_cursor *c_txs_pruned, *c_txs_prunable, *c_txs_prunable_tip; + result = mdb_cursor_open(txn, m_txs_pruned, &c_txs_pruned); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_pruned: ", result).c_str())); + result = mdb_cursor_open(txn, m_txs_prunable, &c_txs_prunable); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_prunable: ", result).c_str())); + result = mdb_cursor_open(txn, m_txs_prunable_tip, &c_txs_prunable_tip); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_prunable_tip: ", result).c_str())); + const uint64_t blockchain_height = height(); + + if (prune_tip_table) + { + MDB_cursor_op op = MDB_FIRST; + while (1) + { + int ret = mdb_cursor_get(c_txs_prunable_tip, &k, &v, op); + op = MDB_NEXT; + if (ret == MDB_NOTFOUND) + break; + if (ret) + throw0(DB_ERROR(lmdb_error("Failed to enumerate transactions: ", ret).c_str())); + + uint64_t block_height; + memcpy(&block_height, v.mv_data, sizeof(block_height)); + if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS < blockchain_height) + { + ++n_total_records; + if (!tools::has_unpruned_block(block_height, blockchain_height, pruning_seed) && !is_v1_tx(c_txs_pruned, &k)) + { + ++n_prunable_records; + result = mdb_cursor_get(c_txs_prunable, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + MWARNING("Already pruned at height " << block_height << "/" << blockchain_height); + else if (result) + throw0(DB_ERROR(lmdb_error("Failed to find transaction prunable data: ", result).c_str())); + else + { + MDEBUG("Pruning at height " << block_height << "/" << blockchain_height); + ++n_pruned_records; + n_bytes += k.mv_size + v.mv_size; + result = mdb_cursor_del(c_txs_prunable, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete transaction prunable data: ", result).c_str())); + } + } + result = mdb_cursor_del(c_txs_prunable_tip, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete transaction tip data: ", result).c_str())); + } + } + } + else + { + MDB_cursor *c_tx_indices; + result = mdb_cursor_open(txn, m_tx_indices, &c_tx_indices); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for tx_indices: ", result).c_str())); + MDB_cursor_op op = MDB_FIRST; + while (1) + { + int ret = mdb_cursor_get(c_tx_indices, &k, &v, op); + op = MDB_NEXT; + if (ret == MDB_NOTFOUND) + break; + if (ret) + throw0(DB_ERROR(lmdb_error("Failed to enumerate transactions: ", ret).c_str())); + + ++n_total_records; + //const txindex *ti = (const txindex *)v.mv_data; + txindex ti; + memcpy(&ti, v.mv_data, sizeof(ti)); + const uint64_t block_height = ti.data.block_id; + if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS >= blockchain_height) + { + MDB_val_set(kp, ti.data.tx_id); + MDB_val_set(vp, block_height); + if (mode == prune_mode_check) + { + result = mdb_cursor_get(c_txs_prunable_tip, &kp, &vp, MDB_SET); + if (result && result != MDB_NOTFOUND) + throw0(DB_ERROR(lmdb_error("Error looking for transaction prunable data: ", result).c_str())); + if (result == MDB_NOTFOUND) + MERROR("Transaction not found in prunable tip table for height " << block_height << "/" << blockchain_height << + ", seed " << epee::string_tools::to_string_hex(pruning_seed)); + } + else + { + result = mdb_cursor_put(c_txs_prunable_tip, &kp, &vp, 0); + if (result && result != MDB_NOTFOUND) + throw0(DB_ERROR(lmdb_error("Error looking for transaction prunable data: ", result).c_str())); + } + } + MDB_val_set(kp, ti.data.tx_id); + if (!tools::has_unpruned_block(block_height, blockchain_height, pruning_seed) && !is_v1_tx(c_txs_pruned, &kp)) + { + result = mdb_cursor_get(c_txs_prunable, &kp, &v, MDB_SET); + if (result && result != MDB_NOTFOUND) + throw0(DB_ERROR(lmdb_error("Error looking for transaction prunable data: ", result).c_str())); + if (mode == prune_mode_check) + { + if (result != MDB_NOTFOUND) + MERROR("Prunable data found for pruned height " << block_height << "/" << blockchain_height << + ", seed " << epee::string_tools::to_string_hex(pruning_seed)); + } + else + { + ++n_prunable_records; + if (result == MDB_NOTFOUND) + MWARNING("Already pruned at height " << block_height << "/" << blockchain_height); + else + { + MDEBUG("Pruning at height " << block_height << "/" << blockchain_height); + ++n_pruned_records; + n_bytes += kp.mv_size + v.mv_size; + result = mdb_cursor_del(c_txs_prunable, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete transaction prunable data: ", result).c_str())); + } + } + } + else + { + if (mode == prune_mode_check) + { + MDB_val_set(kp, ti.data.tx_id); + result = mdb_cursor_get(c_txs_prunable, &kp, &v, MDB_SET); + if (result && result != MDB_NOTFOUND) + throw0(DB_ERROR(lmdb_error("Error looking for transaction prunable data: ", result).c_str())); + if (result == MDB_NOTFOUND) + MERROR("Prunable data not found for unpruned height " << block_height << "/" << blockchain_height << + ", seed " << epee::string_tools::to_string_hex(pruning_seed)); + } + } + } + mdb_cursor_close(c_tx_indices); + } + + if ((result = mdb_stat(txn, m_txs_prunable, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_txs_prunable: ", result).c_str())); + const size_t pages1 = db_stats.ms_branch_pages + db_stats.ms_leaf_pages + db_stats.ms_overflow_pages; + const size_t db_bytes = (pages0 - pages1) * db_stats.ms_psize; + + mdb_cursor_close(c_txs_prunable_tip); + mdb_cursor_close(c_txs_prunable); + mdb_cursor_close(c_txs_pruned); + + txn.commit(); + + TIME_MEASURE_FINISH(t); + + MINFO((mode == prune_mode_check ? "Checked" : "Pruned") << " blockchain in " << + t << " ms: " << (n_bytes/1024.0f/1024.0f) << " MB (" << db_bytes/1024.0f/1024.0f << " MB) pruned in " << + n_pruned_records << " records (" << pages0 - pages1 << "/" << pages0 << " " << db_stats.ms_psize << " byte pages), " << + n_prunable_records << "/" << n_total_records << " pruned records"); + return true; +} + +bool BlockchainLMDB::prune_blockchain(uint32_t pruning_seed) +{ + return prune_worker(prune_mode_prune, pruning_seed); +} + +bool BlockchainLMDB::update_pruning() +{ + return prune_worker(prune_mode_update, 0); +} + +bool BlockchainLMDB::check_pruning() +{ + return prune_worker(prune_mode_check, 0); +} + bool BlockchainLMDB::for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)> f, bool include_blob, bool include_unrelayed_txes) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -2427,6 +2754,36 @@ bool BlockchainLMDB::get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobd return true; } +bool BlockchainLMDB::get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &bd) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + RCURSOR(tx_indices); + RCURSOR(txs_prunable); + + MDB_val_set(v, h); + MDB_val result; + auto get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); + if (get_result == 0) + { + const txindex *tip = (const txindex *)v.mv_data; + MDB_val_set(val_tx_id, tip->data.tx_id); + get_result = mdb_cursor_get(m_cur_txs_prunable, &val_tx_id, &result, MDB_SET); + } + if (get_result == MDB_NOTFOUND) + return false; + else if (get_result) + throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx from hash", get_result).c_str())); + + bd.assign(reinterpret_cast<char*>(result.mv_data), result.mv_size); + + TXN_POSTFIX_RDONLY(); + + return true; +} + bool BlockchainLMDB::get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -2535,7 +2892,7 @@ uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) const return num_elems; } -output_data_t BlockchainLMDB::get_output_key(const uint64_t& amount, const uint64_t& index) const +output_data_t BlockchainLMDB::get_output_key(const uint64_t& amount, const uint64_t& index, bool include_commitmemt) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -2562,7 +2919,8 @@ output_data_t BlockchainLMDB::get_output_key(const uint64_t& amount, const uint6 { const pre_rct_outkey *okp = (const pre_rct_outkey *)v.mv_data; memcpy(&ret, &okp->data, sizeof(pre_rct_output_data_t));; - ret.commitment = rct::zeroCommit(amount); + if (include_commitmemt) + ret.commitment = rct::zeroCommit(amount); } TXN_POSTFIX_RDONLY(); return ret; @@ -2604,7 +2962,7 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con return indices[0]; } -std::vector<uint64_t> BlockchainLMDB::get_tx_amount_output_indices(const uint64_t tx_id) const +std::vector<std::vector<uint64_t>> BlockchainLMDB::get_tx_amount_output_indices(uint64_t tx_id, size_t n_txes) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -2613,35 +2971,40 @@ std::vector<uint64_t> BlockchainLMDB::get_tx_amount_output_indices(const uint64_ TXN_PREFIX_RDONLY(); RCURSOR(tx_outputs); - int result = 0; MDB_val_set(k_tx_id, tx_id); MDB_val v; - std::vector<uint64_t> amount_output_indices; + std::vector<std::vector<uint64_t>> amount_output_indices_set; + amount_output_indices_set.reserve(n_txes); - result = mdb_cursor_get(m_cur_tx_outputs, &k_tx_id, &v, MDB_SET); - if (result == MDB_NOTFOUND) - LOG_PRINT_L0("WARNING: Unexpected: tx has no amount indices stored in " - "tx_outputs, but it should have an empty entry even if it's a tx without " - "outputs"); - else if (result) - throw0(DB_ERROR(lmdb_error("DB error attempting to get data for tx_outputs[tx_index]", result).c_str())); + MDB_cursor_op op = MDB_SET; + while (n_txes-- > 0) + { + int result = mdb_cursor_get(m_cur_tx_outputs, &k_tx_id, &v, op); + if (result == MDB_NOTFOUND) + LOG_PRINT_L0("WARNING: Unexpected: tx has no amount indices stored in " + "tx_outputs, but it should have an empty entry even if it's a tx without " + "outputs"); + else if (result) + throw0(DB_ERROR(lmdb_error("DB error attempting to get data for tx_outputs[tx_index]", result).c_str())); - const uint64_t* indices = (const uint64_t*)v.mv_data; - int num_outputs = v.mv_size / sizeof(uint64_t); + op = MDB_NEXT; - amount_output_indices.reserve(num_outputs); - for (int i = 0; i < num_outputs; ++i) - { - // LOG_PRINT_L0("amount output index[" << 2*i << "]" << ": " << paired_indices[2*i] << " global output index: " << paired_indices[2*i+1]); - amount_output_indices.push_back(indices[i]); + const uint64_t* indices = (const uint64_t*)v.mv_data; + size_t num_outputs = v.mv_size / sizeof(uint64_t); + + amount_output_indices_set.resize(amount_output_indices_set.size() + 1); + std::vector<uint64_t> &amount_output_indices = amount_output_indices_set.back(); + amount_output_indices.reserve(num_outputs); + for (size_t i = 0; i < num_outputs; ++i) + { + amount_output_indices.push_back(indices[i]); + } } - indices = nullptr; TXN_POSTFIX_RDONLY(); - return amount_output_indices; + return amount_output_indices_set; } - bool BlockchainLMDB::has_key_image(const crypto::key_image& img) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 60797a076..c07ab8da5 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -40,6 +40,11 @@ namespace cryptonote { +typedef struct txindex { + crypto::hash key; + tx_data_t data; +} txindex; + typedef struct mdb_txn_cursors { MDB_cursor *m_txc_blocks; @@ -53,6 +58,7 @@ typedef struct mdb_txn_cursors MDB_cursor *m_txc_txs_pruned; MDB_cursor *m_txc_txs_prunable; MDB_cursor *m_txc_txs_prunable_hash; + MDB_cursor *m_txc_txs_prunable_tip; MDB_cursor *m_txc_tx_indices; MDB_cursor *m_txc_tx_outputs; @@ -62,6 +68,8 @@ typedef struct mdb_txn_cursors MDB_cursor *m_txc_txpool_blob; MDB_cursor *m_txc_hf_versions; + + MDB_cursor *m_txc_properties; } mdb_txn_cursors; #define m_cur_blocks m_cursors->m_txc_blocks @@ -73,12 +81,14 @@ typedef struct mdb_txn_cursors #define m_cur_txs_pruned m_cursors->m_txc_txs_pruned #define m_cur_txs_prunable m_cursors->m_txc_txs_prunable #define m_cur_txs_prunable_hash m_cursors->m_txc_txs_prunable_hash +#define m_cur_txs_prunable_tip m_cursors->m_txc_txs_prunable_tip #define m_cur_tx_indices m_cursors->m_txc_tx_indices #define m_cur_tx_outputs m_cursors->m_txc_tx_outputs #define m_cur_spent_keys m_cursors->m_txc_spent_keys #define m_cur_txpool_meta m_cursors->m_txc_txpool_meta #define m_cur_txpool_blob m_cursors->m_txc_txpool_blob #define m_cur_hf_versions m_cursors->m_txc_hf_versions +#define m_cur_properties m_cursors->m_txc_properties typedef struct mdb_rflags { @@ -92,12 +102,14 @@ typedef struct mdb_rflags bool m_rf_txs_pruned; bool m_rf_txs_prunable; bool m_rf_txs_prunable_hash; + bool m_rf_txs_prunable_tip; bool m_rf_tx_indices; bool m_rf_tx_outputs; bool m_rf_spent_keys; bool m_rf_txpool_meta; bool m_rf_txpool_blob; bool m_rf_hf_versions; + bool m_rf_properties; } mdb_rflags; typedef struct mdb_threadinfo @@ -232,6 +244,7 @@ public: virtual bool get_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const; virtual bool get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const; + virtual bool get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const; virtual bool get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const; virtual uint64_t get_tx_count() const; @@ -242,7 +255,7 @@ public: virtual uint64_t get_num_outputs(const uint64_t& amount) const; - virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index) const; + virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index, bool include_commitmemt) const; virtual void get_output_key(const epee::span<const uint64_t> &amounts, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs, bool allow_partial = false) const; virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const; @@ -252,7 +265,7 @@ public: virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const; virtual void get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices) const; - virtual std::vector<uint64_t> get_tx_amount_output_indices(const uint64_t tx_id) const; + virtual std::vector<std::vector<uint64_t>> get_tx_amount_output_indices(const uint64_t tx_id, size_t n_txes) const; virtual bool has_key_image(const crypto::key_image& img) const; @@ -264,6 +277,11 @@ public: virtual bool get_txpool_tx_meta(const crypto::hash& txid, txpool_tx_meta_t &meta) const; virtual bool get_txpool_tx_blob(const crypto::hash& txid, cryptonote::blobdata &bd) const; virtual cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid) const; + virtual uint32_t get_blockchain_pruning_seed() const; + virtual bool prune_blockchain(uint32_t pruning_seed = 0); + virtual bool update_pruning(); + virtual bool check_pruning(); + virtual bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)> f, bool include_blob = false, bool include_unrelayed_txes = true) const; virtual bool for_all_key_images(std::function<bool(const crypto::key_image&)>) const; @@ -309,6 +327,11 @@ public: bool get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t to_height, std::vector<uint64_t> &distribution, uint64_t &base) const; + // helper functions + static int compare_uint64(const MDB_val *a, const MDB_val *b); + static int compare_hash32(const MDB_val *a, const MDB_val *b); + static int compare_string(const MDB_val *a, const MDB_val *b); + private: void do_resize(uint64_t size_increase=0); @@ -361,6 +384,8 @@ private: inline void check_open() const; + bool prune_worker(int mode, uint32_t pruning_seed); + virtual bool is_read_only() const; virtual uint64_t get_database_size() const; @@ -393,6 +418,7 @@ private: MDB_dbi m_txs_pruned; MDB_dbi m_txs_prunable; MDB_dbi m_txs_prunable_hash; + MDB_dbi m_txs_prunable_tip; MDB_dbi m_tx_indices; MDB_dbi m_tx_outputs; diff --git a/src/blockchain_utilities/CMakeLists.txt b/src/blockchain_utilities/CMakeLists.txt index ddf575c29..df74eb695 100644 --- a/src/blockchain_utilities/CMakeLists.txt +++ b/src/blockchain_utilities/CMakeLists.txt @@ -92,6 +92,17 @@ monero_private_headers(blockchain_prune_known_spent_data +set(blockchain_prune_sources + blockchain_prune.cpp + ) + +set(blockchain_prune_private_headers) + +monero_private_headers(blockchain_prune + ${blockchain_prune_private_headers}) + + + set(blockchain_ancestry_sources blockchain_ancestry.cpp ) @@ -298,3 +309,25 @@ set_property(TARGET blockchain_prune_known_spent_data PROPERTY OUTPUT_NAME "monero-blockchain-prune-known-spent-data") install(TARGETS blockchain_prune_known_spent_data DESTINATION bin) + +monero_add_executable(blockchain_prune + ${blockchain_prune_sources} + ${blockchain_prune_private_headers}) + +set_property(TARGET blockchain_prune + PROPERTY + OUTPUT_NAME "monero-blockchain-prune") +install(TARGETS blockchain_prune DESTINATION bin) + +target_link_libraries(blockchain_prune + PRIVATE + cryptonote_core + blockchain_db + p2p + version + epee + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${Boost_THREAD_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES}) diff --git a/src/blockchain_utilities/blockchain_ancestry.cpp b/src/blockchain_utilities/blockchain_ancestry.cpp index e01a8892c..a64ce160a 100644 --- a/src/blockchain_utilities/blockchain_ancestry.cpp +++ b/src/blockchain_utilities/blockchain_ancestry.cpp @@ -51,6 +51,8 @@ using namespace epee; using namespace cryptonote; static bool stop_requested = false; +static uint64_t cached_txes = 0, cached_blocks = 0, cached_outputs = 0, total_txes = 0, total_blocks = 0, total_outputs = 0; +static bool opt_cache_outputs = false, opt_cache_txes = false, opt_cache_blocks = false; struct ancestor { @@ -137,6 +139,8 @@ struct ancestry_state_t std::unordered_map<crypto::hash, ::tx_data_t> tx_cache; std::vector<cryptonote::block> block_cache; + ancestry_state_t(): height(0) {} + template <typename t_archive> void serialize(t_archive &a, const unsigned int ver) { a & height; @@ -219,6 +223,113 @@ static std::unordered_set<ancestor> get_ancestry(const std::unordered_map<crypto return i->second; } +static bool get_block_from_height(ancestry_state_t &state, BlockchainDB *db, uint64_t height, cryptonote::block &b) +{ + ++total_blocks; + if (state.block_cache.size() > height && !state.block_cache[height].miner_tx.vin.empty()) + { + ++cached_blocks; + b = state.block_cache[height]; + return true; + } + cryptonote::blobdata bd = db->get_block_blob_from_height(height); + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return false; + } + if (opt_cache_blocks) + { + state.block_cache.resize(height + 1); + state.block_cache[height] = b; + } + return true; +} + +static bool get_transaction(ancestry_state_t &state, BlockchainDB *db, const crypto::hash &txid, ::tx_data_t &tx_data) +{ + std::unordered_map<crypto::hash, ::tx_data_t>::const_iterator i = state.tx_cache.find(txid); + ++total_txes; + if (i != state.tx_cache.end()) + { + ++cached_txes; + tx_data = i->second; + return true; + } + + cryptonote::blobdata bd; + if (!db->get_pruned_tx_blob(txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << txid << " from db"); + return false; + } + cryptonote::transaction tx; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) + { + LOG_PRINT_L0("Bad tx: " << txid); + return false; + } + tx_data = ::tx_data_t(tx); + if (opt_cache_txes) + state.tx_cache.insert(std::make_pair(txid, tx_data)); + return true; +} + +static bool get_output_txid(ancestry_state_t &state, BlockchainDB *db, uint64_t amount, uint64_t offset, crypto::hash &txid) +{ + ++total_outputs; + std::unordered_map<ancestor, crypto::hash>::const_iterator i = state.output_cache.find({amount, offset}); + if (i != state.output_cache.end()) + { + ++cached_outputs; + txid = i->second; + return true; + } + + const output_data_t od = db->get_output_key(amount, offset, false); + cryptonote::block b; + if (!get_block_from_height(state, db, od.height, b)) + return false; + + for (size_t out = 0; out < b.miner_tx.vout.size(); ++out) + { + if (b.miner_tx.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get<cryptonote::txout_to_key>(b.miner_tx.vout[out].target); + if (txout.key == od.pubkey) + { + txid = cryptonote::get_transaction_hash(b.miner_tx); + if (opt_cache_outputs) + state.output_cache.insert(std::make_pair(ancestor{amount, offset}, txid)); + return true; + } + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << cryptonote::get_transaction_hash(b.miner_tx)); + return false; + } + } + for (const crypto::hash &block_txid: b.tx_hashes) + { + ::tx_data_t tx_data3; + if (!get_transaction(state, db, block_txid, tx_data3)) + return false; + + for (size_t out = 0; out < tx_data3.vout.size(); ++out) + { + if (tx_data3.vout[out] == od.pubkey) + { + txid = block_txid; + if (opt_cache_outputs) + state.output_cache.insert(std::make_pair(ancestor{amount, offset}, txid)); + return true; + } + } + } + return false; +} + int main(int argc, char* argv[]) { TRY_ENTRY(); @@ -243,12 +354,13 @@ int main(int argc, char* argv[]) "database", available_dbs.c_str(), default_db_type }; const command_line::arg_descriptor<std::string> arg_txid = {"txid", "Get ancestry for this txid", ""}; + const command_line::arg_descriptor<std::string> arg_output = {"output", "Get ancestry for this output (amount/offset format)", ""}; const command_line::arg_descriptor<uint64_t> arg_height = {"height", "Get ancestry for all txes at this height", 0}; - const command_line::arg_descriptor<bool> arg_all = {"all", "Include the whole chain", false}; + const command_line::arg_descriptor<bool> arg_refresh = {"refresh", "Refresh the whole chain first", false}; const command_line::arg_descriptor<bool> arg_cache_outputs = {"cache-outputs", "Cache outputs (memory hungry)", false}; const command_line::arg_descriptor<bool> arg_cache_txes = {"cache-txes", "Cache txes (memory hungry)", false}; const command_line::arg_descriptor<bool> arg_cache_blocks = {"cache-blocks", "Cache blocks (memory hungry)", false}; - const command_line::arg_descriptor<bool> arg_include_coinbase = {"include-coinbase", "Including coinbase tx", false}; + const command_line::arg_descriptor<bool> arg_include_coinbase = {"include-coinbase", "Including coinbase tx in per height average", false}; const command_line::arg_descriptor<bool> arg_show_cache_stats = {"show-cache-stats", "Show cache statistics", false}; command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir); @@ -257,8 +369,9 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_cmd_sett, arg_log_level); command_line::add_arg(desc_cmd_sett, arg_database); command_line::add_arg(desc_cmd_sett, arg_txid); + command_line::add_arg(desc_cmd_sett, arg_output); command_line::add_arg(desc_cmd_sett, arg_height); - command_line::add_arg(desc_cmd_sett, arg_all); + command_line::add_arg(desc_cmd_sett, arg_refresh); command_line::add_arg(desc_cmd_sett, arg_cache_outputs); command_line::add_arg(desc_cmd_sett, arg_cache_txes); command_line::add_arg(desc_cmd_sett, arg_cache_blocks); @@ -300,20 +413,22 @@ int main(int argc, char* argv[]) bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; std::string opt_txid_string = command_line::get_arg(vm, arg_txid); + std::string opt_output_string = command_line::get_arg(vm, arg_output); uint64_t opt_height = command_line::get_arg(vm, arg_height); - bool opt_all = command_line::get_arg(vm, arg_all); - bool opt_cache_outputs = command_line::get_arg(vm, arg_cache_outputs); - bool opt_cache_txes = command_line::get_arg(vm, arg_cache_txes); - bool opt_cache_blocks = command_line::get_arg(vm, arg_cache_blocks); + bool opt_refresh = command_line::get_arg(vm, arg_refresh); + opt_cache_outputs = command_line::get_arg(vm, arg_cache_outputs); + opt_cache_txes = command_line::get_arg(vm, arg_cache_txes); + opt_cache_blocks = command_line::get_arg(vm, arg_cache_blocks); bool opt_include_coinbase = command_line::get_arg(vm, arg_include_coinbase); bool opt_show_cache_stats = command_line::get_arg(vm, arg_show_cache_stats); - if ((!opt_txid_string.empty()) + !!opt_height + !!opt_all > 1) + if ((!opt_txid_string.empty()) + !!opt_height + !opt_output_string.empty() > 1) { - std::cerr << "Only one of --txid, --height and --all can be given" << std::endl; + std::cerr << "Only one of --txid, --height, --output can be given" << std::endl; return 1; } crypto::hash opt_txid = crypto::null_hash; + uint64_t output_amount = 0, output_offset = 0; if (!opt_txid_string.empty()) { if (!epee::string_tools::hex_to_pod(opt_txid_string, opt_txid)) @@ -322,6 +437,14 @@ int main(int argc, char* argv[]) return 1; } } + else if (!opt_output_string.empty()) + { + if (sscanf(opt_output_string.c_str(), "%" SCNu64 "/%" SCNu64, &output_amount, &output_offset) != 2) + { + std::cerr << "Invalid output" << std::endl; + return 1; + } + } std::string db_type = command_line::get_arg(vm, arg_database); if (!cryptonote::blockchain_valid_db_type(db_type)) @@ -372,37 +495,36 @@ int main(int argc, char* argv[]) std::vector<crypto::hash> start_txids; - // forward method - if (opt_all) - { - uint64_t cached_txes = 0, cached_blocks = 0, cached_outputs = 0, total_txes = 0, total_blocks = 0, total_outputs = 0; - ancestry_state_t state; + ancestry_state_t state; - const std::string state_file_path = (boost::filesystem::path(opt_data_dir) / "ancestry-state.bin").string(); - LOG_PRINT_L0("Loading state data from " << state_file_path); - std::ifstream state_data_in; - state_data_in.open(state_file_path, std::ios_base::binary | std::ios_base::in); - if (!state_data_in.fail()) + const std::string state_file_path = (boost::filesystem::path(opt_data_dir) / "ancestry-state.bin").string(); + LOG_PRINT_L0("Loading state data from " << state_file_path); + std::ifstream state_data_in; + state_data_in.open(state_file_path, std::ios_base::binary | std::ios_base::in); + if (!state_data_in.fail()) + { + try { - try - { - boost::archive::portable_binary_iarchive a(state_data_in); - a >> state; - } - catch (const std::exception &e) - { - MERROR("Failed to load state data from " << state_file_path << ", restarting from scratch"); - state = ancestry_state_t(); - } - state_data_in.close(); + boost::archive::portable_binary_iarchive a(state_data_in); + a >> state; + } + catch (const std::exception &e) + { + MERROR("Failed to load state data from " << state_file_path << ", restarting from scratch"); + state = ancestry_state_t(); } + state_data_in.close(); + } - tools::signal_handler::install([](int type) { - stop_requested = true; - }); + tools::signal_handler::install([](int type) { + stop_requested = true; + }); + // forward method + const uint64_t db_height = db->height(); + if (opt_refresh) + { MINFO("Starting from height " << state.height); - const uint64_t db_height = db->height(); state.block_cache.reserve(db_height); for (uint64_t h = state.height; h < db_height; ++h) { @@ -464,113 +586,20 @@ int main(int argc, char* argv[]) { for (size_t ring = 0; ring < tx_data.vin.size(); ++ring) { - if (1) + const uint64_t amount = tx_data.vin[ring].first; + const std::vector<uint64_t> &absolute_offsets = tx_data.vin[ring].second; + for (uint64_t offset: absolute_offsets) { - const uint64_t amount = tx_data.vin[ring].first; - const std::vector<uint64_t> &absolute_offsets = tx_data.vin[ring].second; - for (uint64_t offset: absolute_offsets) + add_ancestry(state.ancestry, txid, ancestor{amount, offset}); + // find the tx which created this output + bool found = false; + crypto::hash output_txid; + if (!get_output_txid(state, db, amount, offset, output_txid)) { - const output_data_t od = db->get_output_key(amount, offset); - add_ancestry(state.ancestry, txid, ancestor{amount, offset}); - cryptonote::block b; - ++total_blocks; - if (state.block_cache.size() > od.height && !state.block_cache[od.height].miner_tx.vin.empty()) - { - ++cached_blocks; - b = state.block_cache[od.height]; - } - else - { - cryptonote::blobdata bd = db->get_block_blob_from_height(od.height); - if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) - { - LOG_PRINT_L0("Bad block from db"); - return 1; - } - if (opt_cache_blocks) - { - state.block_cache.resize(od.height + 1); - state.block_cache[od.height] = b; - } - } - // find the tx which created this output - bool found = false; - std::unordered_map<ancestor, crypto::hash>::const_iterator i = state.output_cache.find({amount, offset}); - ++total_outputs; - if (i != state.output_cache.end()) - { - ++cached_outputs; - add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, i->second)); - found = true; - } - else for (size_t out = 0; out < b.miner_tx.vout.size(); ++out) - { - if (b.miner_tx.vout[out].target.type() == typeid(cryptonote::txout_to_key)) - { - const auto &txout = boost::get<cryptonote::txout_to_key>(b.miner_tx.vout[out].target); - if (txout.key == od.pubkey) - { - found = true; - add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, cryptonote::get_transaction_hash(b.miner_tx))); - if (opt_cache_outputs) - state.output_cache.insert(std::make_pair(ancestor{amount, offset}, cryptonote::get_transaction_hash(b.miner_tx))); - break; - } - } - else - { - LOG_PRINT_L0("Bad vout type in txid " << cryptonote::get_transaction_hash(b.miner_tx)); - return 1; - } - } - for (const crypto::hash &block_txid: b.tx_hashes) - { - if (found) - break; - ::tx_data_t tx_data2; - std::unordered_map<crypto::hash, ::tx_data_t>::const_iterator i = state.tx_cache.find(block_txid); - ++total_txes; - if (i != state.tx_cache.end()) - { - ++cached_txes; - tx_data2 = i->second; - } - else - { - cryptonote::blobdata bd; - if (!db->get_pruned_tx_blob(block_txid, bd)) - { - LOG_PRINT_L0("Failed to get txid " << block_txid << " from db"); - return 1; - } - cryptonote::transaction tx; - if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) - { - LOG_PRINT_L0("Bad tx: " << block_txid); - return 1; - } - tx_data2 = ::tx_data_t(tx); - if (opt_cache_txes) - state.tx_cache.insert(std::make_pair(block_txid, tx_data2)); - } - for (size_t out = 0; out < tx_data2.vout.size(); ++out) - { - if (tx_data2.vout[out] == od.pubkey) - { - found = true; - add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, block_txid)); - if (opt_cache_outputs) - state.output_cache.insert(std::make_pair(ancestor{amount, offset}, block_txid)); - break; - } - } - } - if (!found) - { - LOG_PRINT_L0("Output originating transaction not found"); - return 1; - } + LOG_PRINT_L0("Output originating transaction not found"); + return 1; } + add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, output_txid)); } } } @@ -581,10 +610,6 @@ int main(int argc, char* argv[]) if (!txids.empty()) { std::string stats_msg; - if (opt_show_cache_stats) - stats_msg = std::string(", cache: txes ") + std::to_string(cached_txes*100./total_txes) - + ", blocks " + std::to_string(cached_blocks*100./total_blocks) + ", outputs " - + std::to_string(cached_outputs*100./total_outputs); MINFO("Height " << h << ": " << (block_ancestry_size / txids.size()) << " average over " << txids.size() << stats_msg); } state.height = h; @@ -608,14 +633,30 @@ int main(int argc, char* argv[]) } state_data_out.close(); } - - goto done; + } + else + { + if (state.height < db_height) + { + MWARNING("The state file is only built up to height " << state.height << ", but the blockchain reached height " << db_height); + MWARNING("You may want to run with --refresh if you want to get ancestry for newer data"); + } } if (!opt_txid_string.empty()) { start_txids.push_back(opt_txid); } + else if (!opt_output_string.empty()) + { + crypto::hash txid; + if (!get_output_txid(state, db, output_amount, output_offset, txid)) + { + LOG_PRINT_L0("Output not found in db"); + return 1; + } + start_txids.push_back(txid); + } else { const cryptonote::blobdata bd = db->get_block_blob_from_height(opt_height); @@ -648,108 +689,40 @@ int main(int argc, char* argv[]) const crypto::hash txid = txids.front(); txids.pop_front(); - cryptonote::blobdata bd; - if (!db->get_pruned_tx_blob(txid, bd)) - { - LOG_PRINT_L0("Failed to get txid " << txid << " from db"); - return 1; - } - cryptonote::transaction tx; - if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) - { - LOG_PRINT_L0("Bad tx: " << txid); + if (stop_requested) + goto done; + + ::tx_data_t tx_data2; + if (!get_transaction(state, db, txid, tx_data2)) return 1; - } - const bool coinbase = tx.vin.size() == 1 && tx.vin[0].type() == typeid(cryptonote::txin_gen); + + const bool coinbase = tx_data2.coinbase; if (coinbase) continue; - for (size_t ring = 0; ring < tx.vin.size(); ++ring) + for (size_t ring = 0; ring < tx_data2.vin.size(); ++ring) { - if (tx.vin[ring].type() == typeid(cryptonote::txin_to_key)) { - const cryptonote::txin_to_key &txin = boost::get<cryptonote::txin_to_key>(tx.vin[ring]); - const uint64_t amount = txin.amount; - auto absolute_offsets = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets); + const uint64_t amount = tx_data2.vin[ring].first; + auto absolute_offsets = tx_data2.vin[ring].second; for (uint64_t offset: absolute_offsets) { add_ancestor(ancestry, amount, offset); - const output_data_t od = db->get_output_key(amount, offset); - bd = db->get_block_blob_from_height(od.height); - cryptonote::block b; - if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) - { - LOG_PRINT_L0("Bad block from db"); - return 1; - } + // find the tx which created this output bool found = false; - for (size_t out = 0; out < b.miner_tx.vout.size(); ++out) - { - if (b.miner_tx.vout[out].target.type() == typeid(cryptonote::txout_to_key)) - { - const auto &txout = boost::get<cryptonote::txout_to_key>(b.miner_tx.vout[out].target); - if (txout.key == od.pubkey) - { - found = true; - txids.push_back(cryptonote::get_transaction_hash(b.miner_tx)); - MDEBUG("adding txid: " << cryptonote::get_transaction_hash(b.miner_tx)); - break; - } - } - else - { - LOG_PRINT_L0("Bad vout type in txid " << cryptonote::get_transaction_hash(b.miner_tx)); - return 1; - } - } - for (const crypto::hash &block_txid: b.tx_hashes) - { - if (found) - break; - if (!db->get_pruned_tx_blob(block_txid, bd)) - { - LOG_PRINT_L0("Failed to get txid " << block_txid << " from db"); - return 1; - } - cryptonote::transaction tx2; - if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx2)) - { - LOG_PRINT_L0("Bad tx: " << block_txid); - return 1; - } - for (size_t out = 0; out < tx2.vout.size(); ++out) - { - if (tx2.vout[out].target.type() == typeid(cryptonote::txout_to_key)) - { - const auto &txout = boost::get<cryptonote::txout_to_key>(tx2.vout[out].target); - if (txout.key == od.pubkey) - { - found = true; - txids.push_back(block_txid); - MDEBUG("adding txid: " << block_txid); - break; - } - } - else - { - LOG_PRINT_L0("Bad vout type in txid " << block_txid); - return 1; - } - } - } - if (!found) + crypto::hash output_txid; + if (!get_output_txid(state, db, amount, offset, output_txid)) { LOG_PRINT_L0("Output originating transaction not found"); return 1; } + + add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, output_txid)); + txids.push_back(output_txid); + MDEBUG("adding txid: " << output_txid); } } - else - { - LOG_PRINT_L0("Bad vin type in txid " << txid); - return 1; - } } } @@ -762,6 +735,13 @@ int main(int argc, char* argv[]) done: core_storage->deinit(); + + if (opt_show_cache_stats) + MINFO("cache: txes " << std::to_string(cached_txes*100./total_txes) + << "%, blocks " << std::to_string(cached_blocks*100./total_blocks) + << "%, outputs " << std::to_string(cached_outputs*100./total_outputs) + << "%"); + return 0; CATCH_ENTRY("Depth query error", 1); diff --git a/src/blockchain_utilities/blockchain_blackball.cpp b/src/blockchain_utilities/blockchain_blackball.cpp index 73819bd25..8b007e901 100644 --- a/src/blockchain_utilities/blockchain_blackball.cpp +++ b/src/blockchain_utilities/blockchain_blackball.cpp @@ -1131,7 +1131,7 @@ int main(int argc, char* argv[]) return 1; } - mlog_configure(mlog_get_default_log_path("monero-blockchain-find-spent-outputs.log"), true); + mlog_configure(mlog_get_default_log_path("monero-blockchain-mark-spent-outputs.log"), true); if (!command_line::is_arg_defaulted(vm, arg_log_level)) mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); else diff --git a/src/blockchain_utilities/blockchain_export.cpp b/src/blockchain_utilities/blockchain_export.cpp index 5a49f3478..5c5bc7f69 100644 --- a/src/blockchain_utilities/blockchain_export.cpp +++ b/src/blockchain_utilities/blockchain_export.cpp @@ -177,6 +177,12 @@ int main(int argc, char* argv[]) } r = core_storage->init(db, opt_testnet ? cryptonote::TESTNET : opt_stagenet ? cryptonote::STAGENET : cryptonote::MAINNET); + if (core_storage->get_blockchain_pruning_seed()) + { + LOG_PRINT_L0("Blockchain is pruned, cannot export"); + return 1; + } + CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage"); LOG_PRINT_L0("Source blockchain storage initialized OK"); LOG_PRINT_L0("Exporting blockchain raw data..."); diff --git a/src/blockchain_utilities/blockchain_prune.cpp b/src/blockchain_utilities/blockchain_prune.cpp new file mode 100644 index 000000000..8e13f2c04 --- /dev/null +++ b/src/blockchain_utilities/blockchain_prune.cpp @@ -0,0 +1,663 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <array> +#include <lmdb.h> +#include <boost/algorithm/string.hpp> +#include "common/command_line.h" +#include "common/pruning.h" +#include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_core/blockchain.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/lmdb/db_lmdb.h" +#include "blockchain_db/db_types.h" +#include "version.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "bcutil" + +#define MDB_val_set(var, val) MDB_val var = {sizeof(val), (void *)&val} + +namespace po = boost::program_options; +using namespace epee; +using namespace cryptonote; + +static std::string db_path; + +// default to fast:1 +static uint64_t records_per_sync = 128; +static const size_t slack = 512 * 1024 * 1024; + +static std::error_code replace_file(const boost::filesystem::path& replacement_name, const boost::filesystem::path& replaced_name) +{ + std::error_code ec = tools::replace_file(replacement_name.string(), replaced_name.string()); + if (ec) + MERROR("Error renaming " << replacement_name << " to " << replaced_name << ": " << ec.message()); + return ec; +} + +static void open(MDB_env *&env, const boost::filesystem::path &path, uint64_t db_flags, bool readonly) +{ + int dbr; + int flags = 0; + + if (db_flags & DBF_FAST) + flags |= MDB_NOSYNC; + if (db_flags & DBF_FASTEST) + flags |= MDB_NOSYNC | MDB_WRITEMAP | MDB_MAPASYNC; + if (readonly) + flags |= MDB_RDONLY; + + dbr = mdb_env_create(&env); + if (dbr) throw std::runtime_error("Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); + dbr = mdb_env_set_maxdbs(env, 32); + if (dbr) throw std::runtime_error("Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); + dbr = mdb_env_open(env, path.string().c_str(), flags, 0664); + if (dbr) throw std::runtime_error("Failed to open database file '" + + path.string() + "': " + std::string(mdb_strerror(dbr))); +} + +static void close(MDB_env *env) +{ + mdb_env_close(env); +} + +static void add_size(MDB_env *env, uint64_t bytes) +{ + try + { + boost::filesystem::path path(db_path); + boost::filesystem::space_info si = boost::filesystem::space(path); + if(si.available < bytes) + { + MERROR("!! WARNING: Insufficient free space to extend database !!: " << + (si.available >> 20L) << " MB available, " << (bytes >> 20L) << " MB needed"); + return; + } + } + catch(...) + { + // print something but proceed. + MWARNING("Unable to query free disk space."); + } + + MDB_envinfo mei; + mdb_env_info(env, &mei); + MDB_stat mst; + mdb_env_stat(env, &mst); + + uint64_t new_mapsize = (uint64_t)mei.me_mapsize + bytes; + new_mapsize += (new_mapsize % mst.ms_psize); + + int result = mdb_env_set_mapsize(env, new_mapsize); + if (result) + throw std::runtime_error("Failed to set new mapsize to " + std::to_string(new_mapsize) + ": " + std::string(mdb_strerror(result))); + + MGINFO("LMDB Mapsize increased." << " Old: " << mei.me_mapsize / (1024 * 1024) << "MiB" << ", New: " << new_mapsize / (1024 * 1024) << "MiB"); +} + +static void check_resize(MDB_env *env, size_t bytes) +{ + MDB_envinfo mei; + MDB_stat mst; + + mdb_env_info(env, &mei); + mdb_env_stat(env, &mst); + + uint64_t size_used = mst.ms_psize * mei.me_last_pgno; + if (size_used + bytes + slack >= mei.me_mapsize) + add_size(env, size_used + bytes + 2 * slack - mei.me_mapsize); +} + +static bool resize_point(size_t nrecords, MDB_env *env, MDB_txn **txn, size_t &bytes) +{ + if (nrecords % records_per_sync && bytes <= slack / 2) + return false; + int dbr = mdb_txn_commit(*txn); + if (dbr) throw std::runtime_error("Failed to commit txn: " + std::string(mdb_strerror(dbr))); + check_resize(env, bytes); + dbr = mdb_txn_begin(env, NULL, 0, txn); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + bytes = 0; + return true; +} + +static void copy_table(MDB_env *env0, MDB_env *env1, const char *table, unsigned int flags, unsigned int putflags, int (*cmp)(const MDB_val*, const MDB_val*)=0) +{ + MDB_dbi dbi0, dbi1; + MDB_txn *txn0, *txn1; + MDB_cursor *cur0, *cur1; + bool tx_active0 = false, tx_active1 = false; + int dbr; + + MINFO("Copying " << table); + + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){ + if (tx_active1) mdb_txn_abort(txn1); + if (tx_active0) mdb_txn_abort(txn0); + }); + + dbr = mdb_txn_begin(env0, NULL, MDB_RDONLY, &txn0); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + tx_active0 = true; + dbr = mdb_txn_begin(env1, NULL, 0, &txn1); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + tx_active1 = true; + + dbr = mdb_dbi_open(txn0, table, flags, &dbi0); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + if (cmp) + ((flags & MDB_DUPSORT) ? mdb_set_dupsort : mdb_set_compare)(txn0, dbi0, cmp); + + dbr = mdb_dbi_open(txn1, table, flags, &dbi1); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + if (cmp) + ((flags & MDB_DUPSORT) ? mdb_set_dupsort : mdb_set_compare)(txn1, dbi1, cmp); + + dbr = mdb_txn_commit(txn1); + if (dbr) throw std::runtime_error("Failed to commit txn: " + std::string(mdb_strerror(dbr))); + tx_active1 = false; + MDB_stat stats; + dbr = mdb_env_stat(env0, &stats); + if (dbr) throw std::runtime_error("Failed to stat " + std::string(table) + " LMDB table: " + std::string(mdb_strerror(dbr))); + check_resize(env1, (stats.ms_branch_pages + stats.ms_overflow_pages + stats.ms_leaf_pages) * stats.ms_psize); + dbr = mdb_txn_begin(env1, NULL, 0, &txn1); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + tx_active1 = true; + + dbr = mdb_drop(txn1, dbi1, 0); + if (dbr) throw std::runtime_error("Failed to empty " + std::string(table) + " LMDB table: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_cursor_open(txn0, dbi0, &cur0); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_open(txn1, dbi1, &cur1); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + + MDB_val k; + MDB_val v; + MDB_cursor_op op = MDB_FIRST; + size_t nrecords = 0, bytes = 0; + while (1) + { + int ret = mdb_cursor_get(cur0, &k, &v, op); + op = MDB_NEXT; + if (ret == MDB_NOTFOUND) + break; + if (ret) + throw std::runtime_error("Failed to enumerate " + std::string(table) + " records: " + std::string(mdb_strerror(ret))); + + bytes += k.mv_size + v.mv_size; + if (resize_point(++nrecords, env1, &txn1, bytes)) + { + dbr = mdb_cursor_open(txn1, dbi1, &cur1); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + } + + ret = mdb_cursor_put(cur1, &k, &v, putflags); + if (ret) + throw std::runtime_error("Failed to write " + std::string(table) + " record: " + std::string(mdb_strerror(ret))); + } + + mdb_cursor_close(cur1); + mdb_cursor_close(cur0); + mdb_txn_commit(txn1); + tx_active1 = false; + mdb_txn_commit(txn0); + tx_active0 = false; + mdb_dbi_close(env1, dbi1); + mdb_dbi_close(env0, dbi0); +} + +static bool is_v1_tx(MDB_cursor *c_txs_pruned, MDB_val *tx_id) +{ + MDB_val v; + int ret = mdb_cursor_get(c_txs_pruned, tx_id, &v, MDB_SET); + if (ret) + throw std::runtime_error("Failed to find transaction pruned data: " + std::string(mdb_strerror(ret))); + if (v.mv_size == 0) + throw std::runtime_error("Invalid transaction pruned data"); + return cryptonote::is_v1_tx(cryptonote::blobdata_ref{(const char*)v.mv_data, v.mv_size}); +} + +static void prune(MDB_env *env0, MDB_env *env1) +{ + MDB_dbi dbi0_blocks, dbi0_txs_pruned, dbi0_txs_prunable, dbi0_tx_indices, dbi1_txs_prunable, dbi1_txs_prunable_tip, dbi1_properties; + MDB_txn *txn0, *txn1; + MDB_cursor *cur0_txs_pruned, *cur0_txs_prunable, *cur0_tx_indices, *cur1_txs_prunable, *cur1_txs_prunable_tip; + bool tx_active0 = false, tx_active1 = false; + int dbr; + + MGINFO("Creating pruned txs_prunable"); + + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){ + if (tx_active1) mdb_txn_abort(txn1); + if (tx_active0) mdb_txn_abort(txn0); + }); + + dbr = mdb_txn_begin(env0, NULL, MDB_RDONLY, &txn0); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + tx_active0 = true; + dbr = mdb_txn_begin(env1, NULL, 0, &txn1); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + tx_active1 = true; + + dbr = mdb_dbi_open(txn0, "txs_pruned", MDB_INTEGERKEY, &dbi0_txs_pruned); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_compare(txn0, dbi0_txs_pruned, BlockchainLMDB::compare_uint64); + dbr = mdb_cursor_open(txn0, dbi0_txs_pruned, &cur0_txs_pruned); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_dbi_open(txn0, "txs_prunable", MDB_INTEGERKEY, &dbi0_txs_prunable); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_compare(txn0, dbi0_txs_prunable, BlockchainLMDB::compare_uint64); + dbr = mdb_cursor_open(txn0, dbi0_txs_prunable, &cur0_txs_prunable); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_dbi_open(txn0, "tx_indices", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi0_tx_indices); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_dupsort(txn0, dbi0_tx_indices, BlockchainLMDB::compare_hash32); + dbr = mdb_cursor_open(txn0, dbi0_tx_indices, &cur0_tx_indices); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_dbi_open(txn1, "txs_prunable", MDB_INTEGERKEY, &dbi1_txs_prunable); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_compare(txn1, dbi1_txs_prunable, BlockchainLMDB::compare_uint64); + dbr = mdb_cursor_open(txn1, dbi1_txs_prunable, &cur1_txs_prunable); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_dbi_open(txn1, "txs_prunable_tip", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, &dbi1_txs_prunable_tip); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + mdb_set_dupsort(txn1, dbi1_txs_prunable_tip, BlockchainLMDB::compare_uint64); + dbr = mdb_cursor_open(txn1, dbi1_txs_prunable_tip, &cur1_txs_prunable_tip); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_drop(txn1, dbi1_txs_prunable, 0); + if (dbr) throw std::runtime_error("Failed to empty LMDB table: " + std::string(mdb_strerror(dbr))); + dbr = mdb_drop(txn1, dbi1_txs_prunable_tip, 0); + if (dbr) throw std::runtime_error("Failed to empty LMDB table: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_dbi_open(txn1, "properties", 0, &dbi1_properties); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + + MDB_val k, v; + uint32_t pruning_seed = tools::make_pruning_seed(tools::get_random_stripe(), CRYPTONOTE_PRUNING_LOG_STRIPES); + static char pruning_seed_key[] = "pruning_seed"; + k.mv_data = pruning_seed_key; + k.mv_size = strlen("pruning_seed") + 1; + v.mv_data = (void*)&pruning_seed; + v.mv_size = sizeof(pruning_seed); + dbr = mdb_put(txn1, dbi1_properties, &k, &v, 0); + if (dbr) throw std::runtime_error("Failed to save pruning seed: " + std::string(mdb_strerror(dbr))); + + MDB_stat stats; + dbr = mdb_dbi_open(txn0, "blocks", 0, &dbi0_blocks); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + dbr = mdb_stat(txn0, dbi0_blocks, &stats); + if (dbr) throw std::runtime_error("Failed to query size of blocks: " + std::string(mdb_strerror(dbr))); + mdb_dbi_close(env0, dbi0_blocks); + const uint64_t blockchain_height = stats.ms_entries; + size_t nrecords = 0, bytes = 0; + + MDB_cursor_op op = MDB_FIRST; + while (1) + { + int ret = mdb_cursor_get(cur0_tx_indices, &k, &v, op); + op = MDB_NEXT; + if (ret == MDB_NOTFOUND) + break; + if (ret) throw std::runtime_error("Failed to enumerate records: " + std::string(mdb_strerror(ret))); + + const txindex *ti = (const txindex*)v.mv_data; + const uint64_t block_height = ti->data.block_id; + MDB_val_set(kk, ti->data.tx_id); + if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS >= blockchain_height) + { + MDEBUG(block_height << "/" << blockchain_height << " is in tip"); + MDB_val_set(vv, block_height); + dbr = mdb_cursor_put(cur1_txs_prunable_tip, &kk, &vv, 0); + if (dbr) throw std::runtime_error("Failed to write prunable tx tip data: " + std::string(mdb_strerror(dbr))); + bytes += kk.mv_size + vv.mv_size; + } + if (tools::has_unpruned_block(block_height, blockchain_height, pruning_seed) || is_v1_tx(cur0_txs_pruned, &kk)) + { + MDB_val vv; + dbr = mdb_cursor_get(cur0_txs_prunable, &kk, &vv, MDB_SET); + if (dbr) throw std::runtime_error("Failed to read prunable tx data: " + std::string(mdb_strerror(dbr))); + bytes += kk.mv_size + vv.mv_size; + if (resize_point(++nrecords, env1, &txn1, bytes)) + { + dbr = mdb_cursor_open(txn1, dbi1_txs_prunable, &cur1_txs_prunable); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_open(txn1, dbi1_txs_prunable_tip, &cur1_txs_prunable_tip); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + } + dbr = mdb_cursor_put(cur1_txs_prunable, &kk, &vv, 0); + if (dbr) throw std::runtime_error("Failed to write prunable tx data: " + std::string(mdb_strerror(dbr))); + } + else + { + MDEBUG("" << block_height << "/" << blockchain_height << " should be pruned, dropping"); + } + } + + mdb_cursor_close(cur1_txs_prunable_tip); + mdb_cursor_close(cur1_txs_prunable); + mdb_cursor_close(cur0_txs_prunable); + mdb_cursor_close(cur0_txs_pruned); + mdb_cursor_close(cur0_tx_indices); + mdb_txn_commit(txn1); + tx_active1 = false; + mdb_txn_commit(txn0); + tx_active0 = false; + mdb_dbi_close(env1, dbi1_properties); + mdb_dbi_close(env1, dbi1_txs_prunable_tip); + mdb_dbi_close(env1, dbi1_txs_prunable); + mdb_dbi_close(env0, dbi0_txs_prunable); + mdb_dbi_close(env0, dbi0_txs_pruned); + mdb_dbi_close(env0, dbi0_tx_indices); +} + +static bool parse_db_sync_mode(std::string db_sync_mode, uint64_t &db_flags) +{ + std::vector<std::string> options; + boost::trim(db_sync_mode); + boost::split(options, db_sync_mode, boost::is_any_of(" :")); + + for(const auto &option : options) + MDEBUG("option: " << option); + + // default to fast:async:1 + uint64_t DEFAULT_FLAGS = DBF_FAST; + + db_flags = 0; + + if(options.size() == 0) + { + // default to fast:async:1 + db_flags = DEFAULT_FLAGS; + } + + bool safemode = false; + if(options.size() >= 1) + { + if(options[0] == "safe") + { + safemode = true; + db_flags = DBF_SAFE; + } + else if(options[0] == "fast") + { + db_flags = DBF_FAST; + } + else if(options[0] == "fastest") + { + db_flags = DBF_FASTEST; + records_per_sync = 1000; // default to fastest:async:1000 + } + else + return false; + } + + if(options.size() >= 2 && !safemode) + { + char *endptr; + uint64_t bps = strtoull(options[1].c_str(), &endptr, 0); + if (*endptr != '\0') + return false; + records_per_sync = bps; + } + + return true; +} + +int main(int argc, char* argv[]) +{ + TRY_ENTRY(); + + epee::string_tools::set_module_name_and_folder(argv[0]); + + std::string default_db_type = "lmdb"; + + std::string available_dbs = cryptonote::blockchain_db_types(", "); + available_dbs = "available: " + available_dbs; + + uint32_t log_level = 0; + + tools::on_startup(); + + boost::filesystem::path output_file_path; + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""}; + const command_line::arg_descriptor<std::string> arg_database = { + "database", available_dbs.c_str(), default_db_type + }; + const command_line::arg_descriptor<std::string> arg_db_sync_mode = { + "db-sync-mode" + , "Specify sync option, using format [safe|fast|fastest]:[nrecords_per_sync]." + , "fast:1000" + }; + const command_line::arg_descriptor<bool> arg_copy_pruned_database = {"copy-pruned-database", "Copy database anyway if already pruned"}; + + command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_database); + command_line::add_arg(desc_cmd_sett, arg_db_sync_mode); + command_line::add_arg(desc_cmd_sett, arg_copy_pruned_database); + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + auto parser = po::command_line_parser(argc, argv).options(desc_options); + po::store(parser.run(), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + mlog_configure(mlog_get_default_log_path("monero-blockchain-prune.log"), true); + if (!command_line::is_arg_defaulted(vm, arg_log_level)) + mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); + else + mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str()); + + MINFO("Starting..."); + + bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); + bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); + network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; + bool opt_copy_pruned_database = command_line::get_arg(vm, arg_copy_pruned_database); + std::string data_dir = command_line::get_arg(vm, cryptonote::arg_data_dir); + while (boost::ends_with(data_dir, "/") || boost::ends_with(data_dir, "\\")) + data_dir.pop_back(); + + std::string db_type = command_line::get_arg(vm, arg_database); + if (!cryptonote::blockchain_valid_db_type(db_type)) + { + MERROR("Invalid database type: " << db_type); + return 1; + } + if (db_type != "lmdb") + { + MERROR("Unsupported database type: " << db_type << ". Only lmdb is supported"); + return 1; + } + + std::string db_sync_mode = command_line::get_arg(vm, arg_db_sync_mode); + uint64_t db_flags = 0; + if (!parse_db_sync_mode(db_sync_mode, db_flags)) + { + MERROR("Invalid db sync mode: " << db_sync_mode); + return 1; + } + + // If we wanted to use the memory pool, we would set up a fake_core. + + // Use Blockchain instead of lower-level BlockchainDB for two reasons: + // 1. Blockchain has the init() method for easy setup + // 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash() + // + // cannot match blockchain_storage setup above with just one line, + // e.g. + // Blockchain* core_storage = new Blockchain(NULL); + // because unlike blockchain_storage constructor, which takes a pointer to + // tx_memory_pool, Blockchain's constructor takes tx_memory_pool object. + MINFO("Initializing source blockchain (BlockchainDB)"); + std::array<std::unique_ptr<Blockchain>, 2> core_storage; + Blockchain *blockchain = NULL; + tx_memory_pool m_mempool(*blockchain); + boost::filesystem::path paths[2]; + bool already_pruned = false; + for (size_t n = 0; n < core_storage.size(); ++n) + { + core_storage[n].reset(new Blockchain(m_mempool)); + + BlockchainDB* db = new_db(db_type); + if (db == NULL) + { + MERROR("Attempted to use non-existent database type: " << db_type); + throw std::runtime_error("Attempting to use non-existent database type"); + } + MDEBUG("database: " << db_type); + + if (n == 1) + { + paths[1] = boost::filesystem::path(data_dir) / (db->get_db_name() + "-pruned"); + if (boost::filesystem::exists(paths[1])) + { + if (!boost::filesystem::is_directory(paths[1])) + { + MERROR("LMDB needs a directory path, but a file was passed: " << paths[1].string()); + return 1; + } + } + else + { + if (!boost::filesystem::create_directories(paths[1])) + { + MERROR("Failed to create directory: " << paths[1].string()); + return 1; + } + } + db_path = paths[1].string(); + } + else + { + paths[0] = boost::filesystem::path(data_dir) / db->get_db_name(); + } + + MINFO("Loading blockchain from folder " << paths[n] << " ..."); + + try + { + db->open(paths[n].string(), n == 0 ? DBF_RDONLY : 0); + } + catch (const std::exception& e) + { + MERROR("Error opening database: " << e.what()); + return 1; + } + r = core_storage[n]->init(db, net_type); + + std::string source_dest = n == 0 ? "source" : "pruned"; + CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize " << source_dest << " blockchain storage"); + MINFO(source_dest << " blockchain storage initialized OK"); + if (n == 0 && core_storage[0]->get_blockchain_pruning_seed()) + { + if (!opt_copy_pruned_database) + { + MERROR("Blockchain is already pruned, use --" << arg_copy_pruned_database.name << " to copy it anyway"); + return 1; + } + already_pruned = true; + } + } + core_storage[0]->deinit(); + core_storage[0].reset(NULL); + core_storage[1]->deinit(); + core_storage[1].reset(NULL); + + MINFO("Pruning..."); + MDB_env *env0 = NULL, *env1 = NULL; + open(env0, paths[0], db_flags, true); + open(env1, paths[1], db_flags, false); + copy_table(env0, env1, "blocks", MDB_INTEGERKEY, MDB_APPEND); + copy_table(env0, env1, "block_info", MDB_INTEGERKEY | MDB_DUPSORT| MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64); + copy_table(env0, env1, "block_heights", MDB_INTEGERKEY | MDB_DUPSORT| MDB_DUPFIXED, 0, BlockchainLMDB::compare_hash32); + //copy_table(env0, env1, "txs", MDB_INTEGERKEY); + copy_table(env0, env1, "txs_pruned", MDB_INTEGERKEY, MDB_APPEND); + copy_table(env0, env1, "txs_prunable_hash", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPEND); + // not copied: prunable, prunable_tip + copy_table(env0, env1, "tx_indices", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, 0, BlockchainLMDB::compare_hash32); + copy_table(env0, env1, "tx_outputs", MDB_INTEGERKEY, MDB_APPEND); + copy_table(env0, env1, "output_txs", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64); + copy_table(env0, env1, "output_amounts", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_APPENDDUP, BlockchainLMDB::compare_uint64); + copy_table(env0, env1, "spent_keys", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_NODUPDATA, BlockchainLMDB::compare_hash32); + copy_table(env0, env1, "txpool_meta", 0, MDB_NODUPDATA, BlockchainLMDB::compare_hash32); + copy_table(env0, env1, "txpool_blob", 0, MDB_NODUPDATA, BlockchainLMDB::compare_hash32); + copy_table(env0, env1, "hf_versions", MDB_INTEGERKEY, MDB_APPEND); + copy_table(env0, env1, "properties", 0, 0, BlockchainLMDB::compare_string); + if (already_pruned) + { + copy_table(env0, env1, "txs_prunable", MDB_INTEGERKEY, MDB_APPEND, BlockchainLMDB::compare_uint64); + copy_table(env0, env1, "txs_prunable_tip", MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED, MDB_NODUPDATA, BlockchainLMDB::compare_uint64); + } + else + { + prune(env0, env1); + } + close(env1); + close(env0); + + MINFO("Swapping databases, pre-pruning blockchain will be left in " << paths[0].string() + "-old and can be removed if desired"); + if (replace_file(paths[0].string(), paths[0].string() + "-old") || replace_file(paths[1].string(), paths[0].string())) + { + MERROR("Blockchain pruned OK, but renaming failed"); + return 1; + } + + MINFO("Blockchain pruned OK"); + return 0; + + CATCH_ENTRY("Pruning error", 1); +} diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 3b1eb6d23..212a1891e 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -40,6 +40,7 @@ set(common_sources notify.cpp password.cpp perf_timer.cpp + pruning.cpp spawn.cpp threadpool.cpp updates.cpp @@ -69,6 +70,7 @@ set(common_private_headers http_connection.h notify.h pod-class.h + pruning.h rpc_client.h scoped_message_writer.h unordered_containers_boost_serialization.h diff --git a/src/common/i18n.cpp b/src/common/i18n.cpp index ffe8d8b52..a32875945 100644 --- a/src/common/i18n.cpp +++ b/src/common/i18n.cpp @@ -38,6 +38,8 @@ #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "i18n" +#define MAX_LANGUAGE_SIZE 16 + static const unsigned char qm_magic[16] = {0x3c, 0xb8, 0x64, 0x18, 0xca, 0xef, 0x9c, 0x95, 0xcd, 0x21, 0x1c, 0xbf, 0x60, 0xa1, 0xbd, 0xdd}; static std::map<std::string,std::string> i18n_entries; @@ -62,7 +64,19 @@ std::string i18n_get_language() std::string language = e; language = language.substr(0, language.find(".")); + language = language.substr(0, language.find("@")); + + // check valid values + for (char c: language) + if (!strchr("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_-.@", c)) + return "en"; + std::transform(language.begin(), language.end(), language.begin(), tolower); + if (language.size() > MAX_LANGUAGE_SIZE) + { + i18n_log("Language from LANG/LC_ALL suspiciously long, defaulting to en"); + return "en"; + } return language; } diff --git a/src/common/pruning.cpp b/src/common/pruning.cpp new file mode 100644 index 000000000..442b24e4e --- /dev/null +++ b/src/common/pruning.cpp @@ -0,0 +1,116 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "cryptonote_config.h" +#include "misc_log_ex.h" +#include "crypto/crypto.h" +#include "pruning.h" + +namespace tools +{ + +uint32_t make_pruning_seed(uint32_t stripe, uint32_t log_stripes) +{ + CHECK_AND_ASSERT_THROW_MES(log_stripes <= PRUNING_SEED_LOG_STRIPES_MASK, "log_stripes out of range"); + CHECK_AND_ASSERT_THROW_MES(stripe > 0 && stripe <= (1ul << log_stripes), "stripe out of range"); + return (log_stripes << PRUNING_SEED_LOG_STRIPES_SHIFT) | ((stripe - 1) << PRUNING_SEED_STRIPE_SHIFT); +} + +bool has_unpruned_block(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed) +{ + const uint32_t stripe = get_pruning_stripe(pruning_seed); + if (stripe == 0) + return true; + const uint32_t log_stripes = get_pruning_log_stripes(pruning_seed); + uint32_t block_stripe = get_pruning_stripe(block_height, blockchain_height, log_stripes); + return block_stripe == 0 || block_stripe == stripe; +} + +uint32_t get_pruning_stripe(uint64_t block_height, uint64_t blockchain_height, uint32_t log_stripes) +{ + if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS >= blockchain_height) + return 0; + return ((block_height / CRYPTONOTE_PRUNING_STRIPE_SIZE) & (uint64_t)((1ul << log_stripes) - 1)) + 1; +} + +uint32_t get_pruning_seed(uint64_t block_height, uint64_t blockchain_height, uint32_t log_stripes) +{ + const uint32_t stripe = get_pruning_stripe(block_height, blockchain_height, log_stripes); + if (stripe == 0) + return 0; + return make_pruning_seed(stripe, log_stripes); +} + +uint64_t get_next_unpruned_block_height(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed) +{ + CHECK_AND_ASSERT_MES(block_height <= CRYPTONOTE_MAX_BLOCK_NUMBER+1, block_height, "block_height too large"); + CHECK_AND_ASSERT_MES(blockchain_height <= CRYPTONOTE_MAX_BLOCK_NUMBER+1, block_height, "blockchain_height too large"); + const uint32_t stripe = get_pruning_stripe(pruning_seed); + if (stripe == 0) + return block_height; + if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS >= blockchain_height) + return block_height; + const uint32_t seed_log_stripes = get_pruning_log_stripes(pruning_seed); + const uint64_t log_stripes = seed_log_stripes ? seed_log_stripes : CRYPTONOTE_PRUNING_LOG_STRIPES; + const uint64_t mask = (1ul << log_stripes) - 1; + const uint32_t block_pruning_stripe = ((block_height / CRYPTONOTE_PRUNING_STRIPE_SIZE) & mask) + 1; + if (block_pruning_stripe == stripe) + return block_height; + const uint64_t cycles = ((block_height / CRYPTONOTE_PRUNING_STRIPE_SIZE) >> log_stripes); + const uint64_t cycle_start = cycles + ((stripe > block_pruning_stripe) ? 0 : 1); + const uint64_t h = cycle_start * (CRYPTONOTE_PRUNING_STRIPE_SIZE << log_stripes) + (stripe - 1) * CRYPTONOTE_PRUNING_STRIPE_SIZE; + if (h + CRYPTONOTE_PRUNING_TIP_BLOCKS > blockchain_height) + return blockchain_height < CRYPTONOTE_PRUNING_TIP_BLOCKS ? 0 : blockchain_height - CRYPTONOTE_PRUNING_TIP_BLOCKS; + CHECK_AND_ASSERT_MES(h >= block_height, block_height, "h < block_height, unexpected"); + return h; +} + +uint64_t get_next_pruned_block_height(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed) +{ + const uint32_t stripe = get_pruning_stripe(pruning_seed); + if (stripe == 0) + return blockchain_height; + if (block_height + CRYPTONOTE_PRUNING_TIP_BLOCKS >= blockchain_height) + return blockchain_height; + const uint32_t seed_log_stripes = get_pruning_log_stripes(pruning_seed); + const uint64_t log_stripes = seed_log_stripes ? seed_log_stripes : CRYPTONOTE_PRUNING_LOG_STRIPES; + const uint64_t mask = (1ul << log_stripes) - 1; + const uint32_t block_pruning_seed = ((block_height / CRYPTONOTE_PRUNING_STRIPE_SIZE) & mask) + 1; + if (block_pruning_seed != stripe) + return block_height; + const uint32_t next_stripe = 1 + (block_pruning_seed & mask); + return get_next_unpruned_block_height(block_height, blockchain_height, tools::make_pruning_seed(next_stripe, log_stripes)); +} + +uint32_t get_random_stripe() +{ + return 1 + crypto::rand<uint8_t>() % (1ul << CRYPTONOTE_PRUNING_LOG_STRIPES); +} + +} + diff --git a/src/common/pruning.h b/src/common/pruning.h new file mode 100644 index 000000000..3fac3c0fa --- /dev/null +++ b/src/common/pruning.h @@ -0,0 +1,52 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <stdint.h> + +namespace tools +{ + static constexpr uint32_t PRUNING_SEED_LOG_STRIPES_SHIFT = 7; + static constexpr uint32_t PRUNING_SEED_LOG_STRIPES_MASK = 0x7; + static constexpr uint32_t PRUNING_SEED_STRIPE_SHIFT = 0; + static constexpr uint32_t PRUNING_SEED_STRIPE_MASK = 0x7f; + + constexpr inline uint32_t get_pruning_log_stripes(uint32_t pruning_seed) { return (pruning_seed >> PRUNING_SEED_LOG_STRIPES_SHIFT) & PRUNING_SEED_LOG_STRIPES_MASK; } + inline uint32_t get_pruning_stripe(uint32_t pruning_seed) { if (pruning_seed == 0) return 0; return 1 + ((pruning_seed >> PRUNING_SEED_STRIPE_SHIFT) & PRUNING_SEED_STRIPE_MASK); } + + uint32_t make_pruning_seed(uint32_t stripe, uint32_t log_stripes); + + bool has_unpruned_block(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed); + uint32_t get_pruning_stripe(uint64_t block_height, uint64_t blockchain_height, uint32_t log_stripes); + uint32_t get_pruning_seed(uint64_t block_height, uint64_t blockchain_height, uint32_t log_stripes); + uint64_t get_next_unpruned_block_height(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed); + uint64_t get_next_pruned_block_height(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed); + uint32_t get_random_stripe(); +} + diff --git a/src/common/varint.h b/src/common/varint.h index 151d13dbf..904255afc 100644 --- a/src/common/varint.h +++ b/src/common/varint.h @@ -123,6 +123,6 @@ namespace tools { */ template<typename InputIt, typename T> int read_varint(InputIt &&first, InputIt &&last, T &i) { - return read_varint<std::numeric_limits<T>::digits, InputIt, T>(std::move(first), std::move(last), i); + return read_varint<std::numeric_limits<T>::digits>(std::forward<InputIt>(first), std::forward<InputIt>(last), i); } } diff --git a/src/cryptonote_basic/blobdatatype.h b/src/cryptonote_basic/blobdatatype.h index 7d6ff0187..82484c0a8 100644 --- a/src/cryptonote_basic/blobdatatype.h +++ b/src/cryptonote_basic/blobdatatype.h @@ -30,7 +30,11 @@ #pragma once +#include <string> +#include "span.h" + namespace cryptonote { typedef std::string blobdata; + typedef epee::span<const char> blobdata_ref; } diff --git a/src/cryptonote_basic/connection_context.h b/src/cryptonote_basic/connection_context.h index eb73ab0ea..112c13049 100644 --- a/src/cryptonote_basic/connection_context.h +++ b/src/cryptonote_basic/connection_context.h @@ -40,7 +40,7 @@ namespace cryptonote struct cryptonote_connection_context: public epee::net_utils::connection_context_base { cryptonote_connection_context(): m_state(state_before_handshake), m_remote_blockchain_height(0), m_last_response_height(0), - m_last_request_time(boost::posix_time::microsec_clock::universal_time()), m_callback_request_count(0), m_last_known_hash(crypto::null_hash) {} + m_last_request_time(boost::date_time::not_a_date_time), m_callback_request_count(0), m_last_known_hash(crypto::null_hash), m_pruning_seed(0), m_anchor(false) {} enum state { @@ -59,6 +59,8 @@ namespace cryptonote boost::posix_time::ptime m_last_request_time; epee::copyable_atomic m_callback_request_count; //in debug purpose: problem with double callback rise crypto::hash m_last_known_hash; + uint32_t m_pruning_seed; + bool m_anchor; //size_t m_score; TODO: add score calculations }; @@ -81,4 +83,23 @@ namespace cryptonote } } + inline char get_protocol_state_char(cryptonote_connection_context::state s) + { + switch (s) + { + case cryptonote_connection_context::state_before_handshake: + return 'h'; + case cryptonote_connection_context::state_synchronizing: + return 's'; + case cryptonote_connection_context::state_standby: + return 'w'; + case cryptonote_connection_context::state_idle: + return 'i'; + case cryptonote_connection_context::state_normal: + return 'n'; + default: + return 'u'; + } + } + } diff --git a/src/cryptonote_basic/cryptonote_basic.h b/src/cryptonote_basic/cryptonote_basic.h index 196b20e2a..c9c783a56 100644 --- a/src/cryptonote_basic/cryptonote_basic.h +++ b/src/cryptonote_basic/cryptonote_basic.h @@ -201,9 +201,11 @@ namespace cryptonote mutable crypto::hash hash; mutable size_t blob_size; + bool pruned; + transaction(); - transaction(const transaction &t): transaction_prefix(t), hash_valid(false), blob_size_valid(false), signatures(t.signatures), rct_signatures(t.rct_signatures) { if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } } - transaction &operator=(const transaction &t) { transaction_prefix::operator=(t); set_hash_valid(false); set_blob_size_valid(false); signatures = t.signatures; rct_signatures = t.rct_signatures; if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } return *this; } + transaction(const transaction &t): transaction_prefix(t), hash_valid(false), blob_size_valid(false), signatures(t.signatures), rct_signatures(t.rct_signatures), pruned(t.pruned) { if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } } + transaction &operator=(const transaction &t) { transaction_prefix::operator=(t); set_hash_valid(false); set_blob_size_valid(false); signatures = t.signatures; rct_signatures = t.rct_signatures; if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } pruned = t.pruned; return *this; } virtual ~transaction(); void set_null(); void invalidate_hashes(); @@ -232,7 +234,7 @@ namespace cryptonote if (!signatures_not_expected && vin.size() != signatures.size()) return false; - for (size_t i = 0; i < vin.size(); ++i) + if (!pruned) for (size_t i = 0; i < vin.size(); ++i) { size_t signature_size = get_signature_size(vin[i]); if (signatures_not_expected) @@ -263,7 +265,7 @@ namespace cryptonote bool r = rct_signatures.serialize_rctsig_base(ar, vin.size(), vout.size()); if (!r || !ar.stream().good()) return false; ar.end_object(); - if (rct_signatures.type != rct::RCTTypeNull) + if (!pruned && rct_signatures.type != rct::RCTTypeNull) { ar.tag("rctsig_prunable"); ar.begin_object(); @@ -274,6 +276,8 @@ namespace cryptonote } } } + if (!typename Archive<W>::is_saving()) + pruned = false; END_SERIALIZE() template<bool W, template <bool> class Archive> @@ -295,6 +299,8 @@ namespace cryptonote ar.end_object(); } } + if (!typename Archive<W>::is_saving()) + pruned = true; return true; } @@ -322,6 +328,7 @@ namespace cryptonote rct_signatures.type = rct::RCTTypeNull; set_hash_valid(false); set_blob_size_valid(false); + pruned = false; } inline diff --git a/src/cryptonote_basic/cryptonote_boost_serialization.h b/src/cryptonote_basic/cryptonote_boost_serialization.h index 0725a2bb8..a4228b849 100644 --- a/src/cryptonote_basic/cryptonote_boost_serialization.h +++ b/src/cryptonote_basic/cryptonote_boost_serialization.h @@ -45,6 +45,8 @@ #include "ringct/rctTypes.h" #include "ringct/rctOps.h" +BOOST_CLASS_VERSION(rct::ecdhTuple, 1) + //namespace cryptonote { namespace boost { @@ -247,9 +249,19 @@ namespace boost template <class Archive> inline void serialize(Archive &a, rct::ecdhTuple &x, const boost::serialization::version_type ver) { - a & x.mask; - a & x.amount; - // a & x.senderPk; // not serialized, as we do not use it in monero currently + if (ver < 1) + { + a & x.mask; + a & x.amount; + return; + } + crypto::hash8 &amount = (crypto::hash8&)x.amount; + if (!Archive::is_saving::value) + { + memset(&x.mask, 0, sizeof(x.mask)); + memset(&x.amount, 0, sizeof(x.amount)); + } + a & amount; } template <class Archive> @@ -295,7 +307,7 @@ namespace boost a & x.type; if (x.type == rct::RCTTypeNull) return; - if (x.type != rct::RCTTypeFull && x.type != rct::RCTTypeSimple && x.type != rct::RCTTypeBulletproof) + if (x.type != rct::RCTTypeFull && x.type != rct::RCTTypeSimple && x.type != rct::RCTTypeBulletproof && x.type != rct::RCTTypeBulletproof2) throw boost::archive::archive_exception(boost::archive::archive_exception::other_exception, "Unsupported rct type"); // a & x.message; message is not serialized, as it can be reconstructed from the tx data // a & x.mixRing; mixRing is not serialized, as it can be reconstructed from the offsets @@ -323,7 +335,7 @@ namespace boost a & x.type; if (x.type == rct::RCTTypeNull) return; - if (x.type != rct::RCTTypeFull && x.type != rct::RCTTypeSimple && x.type != rct::RCTTypeBulletproof) + if (x.type != rct::RCTTypeFull && x.type != rct::RCTTypeSimple && x.type != rct::RCTTypeBulletproof && x.type != rct::RCTTypeBulletproof2) throw boost::archive::archive_exception(boost::archive::archive_exception::other_exception, "Unsupported rct type"); // a & x.message; message is not serialized, as it can be reconstructed from the tx data // a & x.mixRing; mixRing is not serialized, as it can be reconstructed from the offsets @@ -337,7 +349,7 @@ namespace boost if (x.p.rangeSigs.empty()) a & x.p.bulletproofs; a & x.p.MGs; - if (x.type == rct::RCTTypeBulletproof) + if (x.type == rct::RCTTypeBulletproof || x.type == rct::RCTTypeBulletproof2) a & x.p.pseudoOuts; } } diff --git a/src/cryptonote_basic/cryptonote_format_utils.cpp b/src/cryptonote_basic/cryptonote_format_utils.cpp index 82428f196..f6daaab95 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.cpp +++ b/src/cryptonote_basic/cryptonote_format_utils.cpp @@ -196,6 +196,7 @@ namespace cryptonote bool r = tx.serialize_base(ba); CHECK_AND_ASSERT_MES(r, false, "Failed to parse transaction from blob"); CHECK_AND_ASSERT_MES(expand_transaction_1(tx, true), false, "Failed to expand transaction data"); + tx.invalidate_hashes(); return true; } //--------------------------------------------------------------- @@ -225,6 +226,22 @@ namespace cryptonote return true; } //--------------------------------------------------------------- + bool is_v1_tx(const blobdata_ref& tx_blob) + { + uint64_t version; + const char* begin = static_cast<const char*>(tx_blob.data()); + const char* end = begin + tx_blob.size(); + int read = tools::read_varint(begin, end, version); + if (read <= 0) + throw std::runtime_error("Internal error getting transaction version"); + return version <= 1; + } + //--------------------------------------------------------------- + bool is_v1_tx(const blobdata& tx_blob) + { + return is_v1_tx(blobdata_ref{tx_blob.data(), tx_blob.size()}); + } + //--------------------------------------------------------------- bool generate_key_image_helper(const account_keys& ack, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, const crypto::public_key& out_key, const crypto::public_key& tx_public_key, const std::vector<crypto::public_key>& additional_tx_public_keys, size_t real_output_index, keypair& in_ephemeral, crypto::key_image& ki, hw::device &hwdev) { crypto::key_derivation recv_derivation = AUTO_VAL_INIT(recv_derivation); diff --git a/src/cryptonote_basic/cryptonote_format_utils.h b/src/cryptonote_basic/cryptonote_format_utils.h index 8d33b1ca4..994978c10 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.h +++ b/src/cryptonote_basic/cryptonote_format_utils.h @@ -53,6 +53,8 @@ namespace cryptonote bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx, crypto::hash& tx_hash, crypto::hash& tx_prefix_hash); bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx); bool parse_and_validate_tx_base_from_blob(const blobdata& tx_blob, transaction& tx); + bool is_v1_tx(const blobdata_ref& tx_blob); + bool is_v1_tx(const blobdata& tx_blob); template<typename T> bool find_tx_extra_field_by_type(const std::vector<tx_extra_field>& tx_extra_fields, T& field, size_t index = 0) diff --git a/src/cryptonote_basic/hardfork.cpp b/src/cryptonote_basic/hardfork.cpp index 87a394918..447d79aee 100644 --- a/src/cryptonote_basic/hardfork.cpp +++ b/src/cryptonote_basic/hardfork.cpp @@ -222,7 +222,6 @@ bool HardFork::reorganize_from_block_height(uint64_t height) if (height >= db.height()) return false; - db.set_batch_transactions(true); bool stop_batch = db.batch_start(); versions.clear(); @@ -306,6 +305,29 @@ bool HardFork::rescan_from_chain_height(uint64_t height) return rescan_from_block_height(height - 1); } +void HardFork::on_block_popped(uint64_t nblocks) +{ + CHECK_AND_ASSERT_THROW_MES(nblocks > 0, "nblocks must be greater than 0"); + + CRITICAL_REGION_LOCAL(lock); + + const uint64_t new_chain_height = db.height(); + const uint64_t old_chain_height = new_chain_height + nblocks; + uint8_t version; + uint64_t height; + for (height = old_chain_height - 1; height >= new_chain_height; --height) + { + versions.pop_back(); + version = db.get_hard_fork_version(height); + versions.push_front(version); + } + + // does not take voting into account + for (current_fork_index = heights.size() - 1; current_fork_index > 0; --current_fork_index) + if (height >= heights[current_fork_index].height) + break; +} + int HardFork::get_voted_fork_index(uint64_t height) const { CRITICAL_REGION_LOCAL(lock); diff --git a/src/cryptonote_basic/hardfork.h b/src/cryptonote_basic/hardfork.h index a63a66976..a3fc25dfa 100644 --- a/src/cryptonote_basic/hardfork.h +++ b/src/cryptonote_basic/hardfork.h @@ -150,6 +150,16 @@ namespace cryptonote bool reorganize_from_chain_height(uint64_t height); /** + * @brief called when one or more blocks are popped from the blockchain + * + * The current fork will be updated by looking up the db, + * which is much cheaper than recomputing everything + * + * @param new_chain_height the height of the chain after popping + */ + void on_block_popped(uint64_t new_chain_height); + + /** * @brief returns current state at the given time * * Based on the approximate time of the last known hard fork, diff --git a/src/cryptonote_config.h b/src/cryptonote_config.h index 496678b5e..4f652cd42 100644 --- a/src/cryptonote_config.h +++ b/src/cryptonote_config.h @@ -141,6 +141,7 @@ #define HF_VERSION_MIN_MIXIN_10 8 #define HF_VERSION_ENFORCE_RCT 6 #define HF_VERSION_PER_BYTE_FEE 8 +#define HF_VERSION_SMALLER_BP 10 #define PER_KB_FEE_QUANTIZATION_DECIMALS 8 @@ -150,6 +151,11 @@ #define BULLETPROOF_MAX_OUTPUTS 16 +#define CRYPTONOTE_PRUNING_STRIPE_SIZE 4096 // the smaller, the smoother the increase +#define CRYPTONOTE_PRUNING_LOG_STRIPES 3 // the higher, the more space saved +#define CRYPTONOTE_PRUNING_TIP_BLOCKS 5500 // the smaller, the more space saved +//#define CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED + // New constants are intended to go here namespace config { diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index bbac20eaa..9f1e2a0c0 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -53,6 +53,8 @@ #include "ringct/rctSigs.h" #include "common/perf_timer.h" #include "common/notify.h" +#include "common/varint.h" +#include "common/pruning.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "blockchain" @@ -173,7 +175,8 @@ Blockchain::Blockchain(tx_memory_pool& tx_pool) : //------------------------------------------------------------------ Blockchain::~Blockchain() { - deinit(); + try { deinit(); } + catch (const std::exception &e) { /* ignore */ } } //------------------------------------------------------------------ bool Blockchain::have_tx(const crypto::hash &id) const @@ -641,9 +644,18 @@ block Blockchain::pop_block_from_blockchain() throw; } + // make sure the hard fork object updates its current version + m_hardfork->on_block_popped(1); + // return transactions from popped block to the tx_pool + size_t pruned = 0; for (transaction& tx : popped_txs) { + if (tx.pruned) + { + ++pruned; + continue; + } if (!is_coinbase(tx)) { cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc); @@ -651,12 +663,7 @@ block Blockchain::pop_block_from_blockchain() // FIXME: HardFork // Besides the below, popping a block should also remove the last entry // in hf_versions. - // - // FIXME: HardFork - // This is not quite correct, as we really want to add the txes - // to the pool based on the version determined after all blocks - // are popped. - uint8_t version = get_current_hard_fork_version(); + uint8_t version = get_ideal_hard_fork_version(m_db->height()); // We assume that if they were in a block, the transactions are already // known to the network as a whole. However, if we had mined that block, @@ -670,6 +677,8 @@ block Blockchain::pop_block_from_blockchain() } } } + if (pruned) + MWARNING(pruned << " pruned txes could not be added back to the txpool"); m_blocks_longhash_table.clear(); m_scan_table.clear(); @@ -921,8 +930,10 @@ difficulty_type Blockchain::get_difficulty_for_next_block() //------------------------------------------------------------------ std::vector<time_t> Blockchain::get_last_block_timestamps(unsigned int blocks) const { - std::vector<time_t> timestamps(blocks); uint64_t height = m_db->height(); + if (blocks > height) + blocks = height; + std::vector<time_t> timestamps(blocks); while (blocks--) timestamps[blocks] = m_db->get_block_timestamp(height - blocks - 1); return timestamps; @@ -1199,7 +1210,7 @@ bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_bl return false; } // From hard fork 2, we allow a miner to claim less block reward than is allowed, in case a miner wants less dust - if (m_hardfork->get_current_version() < 2) + if (version < 2) { if(base_reward + fee != money_in_use) { @@ -2043,6 +2054,51 @@ bool Blockchain::get_transactions_blobs(const t_ids_container& txs_ids, t_tx_con return true; } //------------------------------------------------------------------ +size_t get_transaction_version(const cryptonote::blobdata &bd) +{ + size_t version; + const char* begin = static_cast<const char*>(bd.data()); + const char* end = begin + bd.size(); + int read = tools::read_varint(begin, end, version); + if (read <= 0) + throw std::runtime_error("Internal error getting transaction version"); + return version; +} +//------------------------------------------------------------------ +template<class t_ids_container, class t_tx_container, class t_missed_container> +bool Blockchain::get_split_transactions_blobs(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + reserve_container(txs, txs_ids.size()); + for (const auto& tx_hash : txs_ids) + { + try + { + cryptonote::blobdata tx; + if (m_db->get_pruned_tx_blob(tx_hash, tx)) + { + txs.push_back(std::make_tuple(tx_hash, std::move(tx), crypto::null_hash, cryptonote::blobdata())); + if (!is_v1_tx(std::get<1>(txs.back())) && !m_db->get_prunable_tx_hash(tx_hash, std::get<2>(txs.back()))) + { + MERROR("Prunable data hash not found for " << tx_hash); + return false; + } + if (!m_db->get_prunable_tx_blob(tx_hash, std::get<3>(txs.back()))) + std::get<3>(txs.back()).clear(); + } + else + missed_txs.push_back(tx_hash); + } + catch (const std::exception& e) + { + return false; + } + } + return true; +} +//------------------------------------------------------------------ template<class t_ids_container, class t_tx_container, class t_missed_container> bool Blockchain::get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const { @@ -2091,9 +2147,12 @@ bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qbloc m_db->block_txn_start(true); current_height = get_current_blockchain_height(); + const uint32_t pruning_seed = get_blockchain_pruning_seed(); + start_height = tools::get_next_unpruned_block_height(start_height, current_height, pruning_seed); + uint64_t stop_height = tools::get_next_pruned_block_height(start_height, current_height, pruning_seed); size_t count = 0; - hashes.reserve(std::max((size_t)(current_height - start_height), (size_t)BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT)); - for(size_t i = start_height; i < current_height && count < BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT; i++, count++) + hashes.reserve(std::min((size_t)(stop_height - start_height), (size_t)BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT)); + for(size_t i = start_height; i < stop_height && count < BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT; i++, count++) { hashes.push_back(m_db->get_block_hash_from_height(i)); } @@ -2298,7 +2357,7 @@ bool Blockchain::check_for_double_spend(const transaction& tx, key_images_contai return true; } //------------------------------------------------------------------ -bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<uint64_t>& indexs) const +bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, size_t n_txes, std::vector<std::vector<uint64_t>>& indexs) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -2308,16 +2367,25 @@ bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<u MERROR_VER("get_tx_outputs_gindexs failed to find transaction with id = " << tx_id); return false; } + indexs = m_db->get_tx_amount_output_indices(tx_index, n_txes); + CHECK_AND_ASSERT_MES(n_txes == indexs.size(), false, "Wrong indexs size"); - // get amount output indexes, currently referred to in parts as "output global indices", but they are actually specific to amounts - indexs = m_db->get_tx_amount_output_indices(tx_index); - if (indexs.empty()) + return true; +} +//------------------------------------------------------------------ +bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<uint64_t>& indexs) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + uint64_t tx_index; + if (!m_db->tx_exists(tx_id, tx_index)) { - // empty indexs is only valid if the vout is empty, which is legal but rare - cryptonote::transaction tx = m_db->get_tx(tx_id); - CHECK_AND_ASSERT_MES(tx.vout.empty(), false, "internal error: global indexes for transaction " << tx_id << " is empty, and tx vout is not"); + MERROR_VER("get_tx_outputs_gindexs failed to find transaction with id = " << tx_id); + return false; } - + std::vector<std::vector<uint64_t>> indices = m_db->get_tx_amount_output_indices(tx_index, 1); + CHECK_AND_ASSERT_MES(indices.size() == 1, false, "Wrong indices size"); + indexs = indices.front(); return true; } //------------------------------------------------------------------ @@ -2449,6 +2517,18 @@ bool Blockchain::check_tx_outputs(const transaction& tx, tx_verification_context } } + // from v10, allow bulletproofs v2 + if (hf_version < HF_VERSION_SMALLER_BP) { + if (tx.version >= 2) { + if (tx.rct_signatures.type == rct::RCTTypeBulletproof2) + { + MERROR_VER("Bulletproofs v2 are not allowed before v" << HF_VERSION_SMALLER_BP); + tvc.m_invalid_output = true; + return false; + } + } + } + return true; } //------------------------------------------------------------------ @@ -2489,7 +2569,7 @@ bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_pr } } } - else if (rv.type == rct::RCTTypeSimple || rv.type == rct::RCTTypeBulletproof) + else if (rv.type == rct::RCTTypeSimple || rv.type == rct::RCTTypeBulletproof || rv.type == rct::RCTTypeBulletproof2) { CHECK_AND_ASSERT_MES(!pubkeys.empty() && !pubkeys[0].empty(), false, "empty pubkeys"); rv.mixRing.resize(pubkeys.size()); @@ -2515,7 +2595,7 @@ bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_pr for (size_t n = 0; n < tx.vin.size(); ++n) rv.p.MGs[0].II[n] = rct::ki2rct(boost::get<txin_to_key>(tx.vin[n]).k_image); } - else if (rv.type == rct::RCTTypeSimple || rv.type == rct::RCTTypeBulletproof) + else if (rv.type == rct::RCTTypeSimple || rv.type == rct::RCTTypeBulletproof || rct::RCTTypeBulletproof2) { CHECK_AND_ASSERT_MES(rv.p.MGs.size() == tx.vin.size(), false, "Bad MGs size"); for (size_t n = 0; n < tx.vin.size(); ++n) @@ -2789,6 +2869,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, } case rct::RCTTypeSimple: case rct::RCTTypeBulletproof: + case rct::RCTTypeBulletproof2: { // check all this, either reconstructed (so should really pass), or not { @@ -3359,7 +3440,7 @@ leave: { if (memcmp(&hash, &expected_hash, sizeof(hash)) != 0) { - MERROR_VER("Block with id is INVALID: " << id); + MERROR_VER("Block with id is INVALID: " << id << ", expected " << expected_hash); bvc.m_verifivation_failed = true; goto leave; } @@ -3625,6 +3706,35 @@ leave: return true; } //------------------------------------------------------------------ +bool Blockchain::prune_blockchain(uint32_t pruning_seed) +{ + uint8_t hf_version = m_hardfork->get_current_version(); + if (hf_version < 10) + { + MERROR("Most of the network will only be ready for pruned blockchains from v10, not pruning"); + return false; + } + return m_db->prune_blockchain(pruning_seed); +} +//------------------------------------------------------------------ +bool Blockchain::update_blockchain_pruning() +{ + m_tx_pool.lock(); + epee::misc_utils::auto_scope_leave_caller unlocker = epee::misc_utils::create_scope_leave_handler([&](){m_tx_pool.unlock();}); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + return m_db->update_pruning(); +} +//------------------------------------------------------------------ +bool Blockchain::check_blockchain_pruning() +{ + m_tx_pool.lock(); + epee::misc_utils::auto_scope_leave_caller unlocker = epee::misc_utils::create_scope_leave_handler([&](){m_tx_pool.unlock();}); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + return m_db->check_pruning(); +} +//------------------------------------------------------------------ bool Blockchain::update_next_cumulative_weight_limit() { uint64_t full_reward_zone = get_min_block_weight(get_current_hard_fork_version()); @@ -3838,37 +3948,17 @@ bool Blockchain::cleanup_handle_incoming_blocks(bool force_sync) CRITICAL_REGION_END(); m_tx_pool.unlock(); + update_blockchain_pruning(); + return success; } //------------------------------------------------------------------ -void Blockchain::output_scan_worker(const uint64_t amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs, const std::vector<output_data_t> &extra_tx_map) const +void Blockchain::output_scan_worker(const uint64_t amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs) const { try { m_db->get_output_key(epee::span<const uint64_t>(&amount, 1), offsets, outputs, true); - if (outputs.size() < offsets.size()) - { - const uint64_t n_outputs = m_db->get_num_outputs(amount); - for (size_t i = outputs.size(); i < offsets.size(); ++i) - { - uint64_t idx = offsets[i]; - if (idx < n_outputs) - { - MWARNING("Index " << idx << " not found in db for amount " << amount << ", but it is less than the number of entries"); - break; - } - else if (idx < n_outputs + extra_tx_map.size()) - { - outputs.push_back(extra_tx_map[idx - n_outputs]); - } - else - { - MWARNING("missed " << amount << "/" << idx << " in " << extra_tx_map.size() << " (chain " << n_outputs << ")"); - break; - } - } - } } catch (const std::exception& e) { @@ -3983,34 +4073,6 @@ uint64_t Blockchain::prevalidate_block_hashes(uint64_t height, const std::vector // vs [k_image, output_keys] (m_scan_table). This is faster because it takes advantage of bulk queries // and is threaded if possible. The table (m_scan_table) will be used later when querying output // keys. -static bool update_output_map(std::map<uint64_t, std::vector<output_data_t>> &extra_tx_map, const transaction &tx, uint64_t height, bool miner) -{ - MTRACE("Blockchain::" << __func__); - for (size_t i = 0; i < tx.vout.size(); ++i) - { - const auto &out = tx.vout[i]; - if (out.target.type() != typeid(txout_to_key)) - continue; - const txout_to_key &out_to_key = boost::get<txout_to_key>(out.target); - rct::key commitment; - uint64_t amount = out.amount; - if (miner && tx.version == 2) - { - commitment = rct::zeroCommit(amount); - amount = 0; - } - else if (tx.version > 1) - { - CHECK_AND_ASSERT_MES(i < tx.rct_signatures.outPk.size(), false, "Invalid outPk size"); - commitment = tx.rct_signatures.outPk[i].mask; - } - else - commitment = rct::zero(); - extra_tx_map[amount].push_back(output_data_t{out_to_key.key, tx.unlock_time, height, commitment}); - } - return true; -} - bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete_entry> &blocks_entry) { MTRACE("Blockchain::" << __func__); @@ -4179,7 +4241,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete // [input] stores all absolute_offsets for each amount std::map<uint64_t, std::vector<uint64_t>> offset_map; // [output] stores all output_data_t for each absolute_offset - std::map<uint64_t, std::vector<output_data_t>> tx_map, extra_tx_map; + std::map<uint64_t, std::vector<output_data_t>> tx_map; std::vector<std::pair<cryptonote::transaction, crypto::hash>> txes(total_txs); #define SCAN_TABLE_QUIT(m) \ @@ -4196,8 +4258,6 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete if (m_cancel) return false; - if (!update_output_map(extra_tx_map, blocks[block_index].miner_tx, height + block_index, true)) - SCAN_TABLE_QUIT("Error building extra tx map."); for (const auto &tx_blob : entry.txs) { if (tx_index >= txes.size()) @@ -4256,8 +4316,6 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete offset_map[in_to_key.amount].push_back(offset); } - if (!update_output_map(extra_tx_map, tx, height + block_index, false)) - SCAN_TABLE_QUIT("Error building extra tx map."); } ++block_index; } @@ -4282,7 +4340,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete for (size_t i = 0; i < amounts.size(); i++) { uint64_t amount = amounts[i]; - tpool.submit(&waiter, boost::bind(&Blockchain::output_scan_worker, this, amount, std::cref(offset_map[amount]), std::ref(tx_map[amount]), std::cref(extra_tx_map[amount])), true); + tpool.submit(&waiter, boost::bind(&Blockchain::output_scan_worker, this, amount, std::cref(offset_map[amount]), std::ref(tx_map[amount])), true); } waiter.wait(&tpool); } @@ -4291,7 +4349,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete for (size_t i = 0; i < amounts.size(); i++) { uint64_t amount = amounts[i]; - output_scan_worker(amount, offset_map[amount], tx_map[amount], extra_tx_map[amount]); + output_scan_worker(amount, offset_map[amount], tx_map[amount]); } } @@ -4665,4 +4723,5 @@ void Blockchain::cache_block_template(const block &b, const cryptonote::account_ namespace cryptonote { template bool Blockchain::get_transactions(const std::vector<crypto::hash>&, std::vector<transaction>&, std::vector<crypto::hash>&) const; template bool Blockchain::get_transactions_blobs(const std::vector<crypto::hash>&, std::vector<cryptonote::blobdata>&, std::vector<crypto::hash>&, bool) const; +template bool Blockchain::get_split_transactions_blobs(const std::vector<crypto::hash>&, std::vector<std::tuple<crypto::hash, cryptonote::blobdata, crypto::hash, cryptonote::blobdata>>&, std::vector<crypto::hash>&) const; } diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index 67bccc6c6..4952116ac 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -521,10 +521,12 @@ namespace cryptonote * * @param tx_id the hash of the transaction to fetch indices for * @param indexs return-by-reference the global indices for the transaction's outputs + * @param n_txes how many txes in a row to get results for * * @return false if the transaction does not exist, or if no indices are found, otherwise true */ bool get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<uint64_t>& indexs) const; + bool get_tx_outputs_gindexs(const crypto::hash& tx_id, size_t n_txes, std::vector<std::vector<uint64_t>>& indexs) const; /** * @brief stores the blockchain @@ -675,6 +677,8 @@ namespace cryptonote template<class t_ids_container, class t_tx_container, class t_missed_container> bool get_transactions_blobs(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs, bool pruned = false) const; template<class t_ids_container, class t_tx_container, class t_missed_container> + bool get_split_transactions_blobs(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const; + template<class t_ids_container, class t_tx_container, class t_missed_container> bool get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const; //debug functions @@ -923,7 +927,7 @@ namespace cryptonote * @param outputs return-by-reference the outputs collected */ void output_scan_worker(const uint64_t amount,const std::vector<uint64_t> &offsets, - std::vector<output_data_t> &outputs, const std::vector<output_data_t> &extra_tx_map) const; + std::vector<output_data_t> &outputs) const; /** * @brief computes the "short" and "long" hashes for a set of blocks @@ -954,6 +958,10 @@ namespace cryptonote bool is_within_compiled_block_hash_area(uint64_t height) const; bool is_within_compiled_block_hash_area() const { return is_within_compiled_block_hash_area(m_db->height()); } uint64_t prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes); + uint32_t get_blockchain_pruning_seed() const { return m_db->get_blockchain_pruning_seed(); } + bool prune_blockchain(uint32_t pruning_seed = 0); + bool update_blockchain_pruning(); + bool check_blockchain_pruning(); void lock(); void unlock(); diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index f3249ea92..48c607ff1 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -105,6 +105,11 @@ namespace cryptonote "disable-dns-checkpoints" , "Do not retrieve checkpoints from DNS" }; + const command_line::arg_descriptor<size_t> arg_block_download_max_size = { + "block-download-max-size" + , "Set maximum size of block download queue in bytes (0 for default)" + , 0 + }; static const command_line::arg_descriptor<bool> arg_test_drop_download = { "test-drop-download" @@ -175,6 +180,11 @@ namespace cryptonote , "Run a program for each new block, '%s' will be replaced by the block hash" , "" }; + static const command_line::arg_descriptor<bool> arg_prune_blockchain = { + "prune-blockchain" + , "Prune blockchain" + , false + }; //----------------------------------------------------------------------------------------------- core::core(i_cryptonote_protocol* pprotocol): @@ -285,9 +295,11 @@ namespace cryptonote command_line::add_arg(desc, arg_test_dbg_lock_sleep); command_line::add_arg(desc, arg_offline); command_line::add_arg(desc, arg_disable_dns_checkpoints); + command_line::add_arg(desc, arg_block_download_max_size); command_line::add_arg(desc, arg_max_txpool_weight); command_line::add_arg(desc, arg_pad_transactions); command_line::add_arg(desc, arg_block_notify); + command_line::add_arg(desc, arg_prune_blockchain); miner::init_options(desc); BlockchainDB::init_options(desc); @@ -374,6 +386,11 @@ namespace cryptonote return m_blockchain_storage.get_transactions_blobs(txs_ids, txs, missed_txs); } //----------------------------------------------------------------------------------------------- + bool core::get_split_transactions_blobs(const std::vector<crypto::hash>& txs_ids, std::vector<std::tuple<crypto::hash, cryptonote::blobdata, crypto::hash, cryptonote::blobdata>>& txs, std::vector<crypto::hash>& missed_txs) const + { + return m_blockchain_storage.get_split_transactions_blobs(txs_ids, txs, missed_txs); + } + //----------------------------------------------------------------------------------------------- bool core::get_txpool_backlog(std::vector<tx_backlog_entry>& backlog) const { m_mempool.get_transaction_backlog(backlog); @@ -413,6 +430,7 @@ namespace cryptonote uint64_t blocks_threads = command_line::get_arg(vm, arg_prep_blocks_threads); std::string check_updates_string = command_line::get_arg(vm, arg_check_updates); size_t max_txpool_weight = command_line::get_arg(vm, arg_max_txpool_weight); + bool prune_blockchain = command_line::get_arg(vm, arg_prune_blockchain); boost::filesystem::path folder(m_config_folder); if (m_nettype == FAKECHAIN) @@ -607,6 +625,14 @@ namespace cryptonote r = m_miner.init(vm, m_nettype); CHECK_AND_ASSERT_MES(r, false, "Failed to initialize miner instance"); + if (prune_blockchain) + { + // display a message if the blockchain is not pruned yet + if (m_blockchain_storage.get_current_blockchain_height() > 1 && !m_blockchain_storage.get_blockchain_pruning_seed()) + MGINFO("Pruning blockchain..."); + CHECK_AND_ASSERT_MES(m_blockchain_storage.prune_blockchain(), false, "Failed to prune blockchain"); + } + return load_state_data(); } //----------------------------------------------------------------------------------------------- @@ -790,6 +816,7 @@ namespace cryptonote } break; case rct::RCTTypeBulletproof: + case rct::RCTTypeBulletproof2: if (!is_canonical_bulletproof_layout(rv.p.bulletproofs)) { MERROR_VER("Bulletproof does not have canonical form"); @@ -817,7 +844,7 @@ namespace cryptonote { if (!tx_info[n].result) continue; - if (tx_info[n].tx->rct_signatures.type != rct::RCTTypeBulletproof) + if (tx_info[n].tx->rct_signatures.type != rct::RCTTypeBulletproof && tx_info[n].tx->rct_signatures.type != rct::RCTTypeBulletproof2) continue; if (assumed_bad || !rct::verRctSemanticsSimple(tx_info[n].tx->rct_signatures)) { @@ -1220,6 +1247,11 @@ namespace cryptonote return m_blockchain_storage.get_tx_outputs_gindexs(tx_id, indexs); } //----------------------------------------------------------------------------------------------- + bool core::get_tx_outputs_gindexs(const crypto::hash& tx_id, size_t n_txes, std::vector<std::vector<uint64_t>>& indexs) const + { + return m_blockchain_storage.get_tx_outputs_gindexs(tx_id, n_txes, indexs); + } + //----------------------------------------------------------------------------------------------- void core::pause_mine() { m_miner.pause(); @@ -1496,6 +1528,7 @@ namespace cryptonote m_check_updates_interval.do_call(boost::bind(&core::check_updates, this)); m_check_disk_space_interval.do_call(boost::bind(&core::check_disk_space, this)); m_block_rate_interval.do_call(boost::bind(&core::check_block_rate, this)); + m_blockchain_pruning_interval.do_call(boost::bind(&core::update_blockchain_pruning, this)); m_miner.on_idle(); m_mempool.on_idle(); return true; @@ -1731,6 +1764,16 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- + bool core::update_blockchain_pruning() + { + return m_blockchain_storage.update_blockchain_pruning(); + } + //----------------------------------------------------------------------------------------------- + bool core::check_blockchain_pruning() + { + return m_blockchain_storage.check_blockchain_pruning(); + } + //----------------------------------------------------------------------------------------------- void core::set_target_blockchain_height(uint64_t target_blockchain_height) { m_target_blockchain_height = target_blockchain_height; @@ -1753,6 +1796,16 @@ namespace cryptonote return si.available; } //----------------------------------------------------------------------------------------------- + uint32_t core::get_blockchain_pruning_seed() const + { + return get_blockchain_storage().get_blockchain_pruning_seed(); + } + //----------------------------------------------------------------------------------------------- + bool core::prune_blockchain(uint32_t pruning_seed) + { + return get_blockchain_storage().prune_blockchain(pruning_seed); + } + //----------------------------------------------------------------------------------------------- std::time_t core::get_start_time() const { return start_time; diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index cc53fce58..4810fc891 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -62,6 +62,7 @@ namespace cryptonote extern const command_line::arg_descriptor<bool, false> arg_regtest_on; extern const command_line::arg_descriptor<difficulty_type> arg_fixed_difficulty; extern const command_line::arg_descriptor<bool> arg_offline; + extern const command_line::arg_descriptor<size_t> arg_block_download_max_size; /************************************************************************/ /* */ @@ -359,6 +360,13 @@ namespace cryptonote * * @note see Blockchain::get_transactions */ + bool get_split_transactions_blobs(const std::vector<crypto::hash>& txs_ids, std::vector<std::tuple<crypto::hash, cryptonote::blobdata, crypto::hash, cryptonote::blobdata>>& txs, std::vector<crypto::hash>& missed_txs) const; + + /** + * @copydoc Blockchain::get_transactions + * + * @note see Blockchain::get_transactions + */ bool get_transactions(const std::vector<crypto::hash>& txs_ids, std::vector<transaction>& txs, std::vector<crypto::hash>& missed_txs) const; /** @@ -534,6 +542,7 @@ namespace cryptonote * @note see Blockchain::get_tx_outputs_gindexs */ bool get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector<uint64_t>& indexs) const; + bool get_tx_outputs_gindexs(const crypto::hash& tx_id, size_t n_txes, std::vector<std::vector<uint64_t>>& indexs) const; /** * @copydoc Blockchain::get_tail_id @@ -782,6 +791,36 @@ namespace cryptonote */ bool offline() const { return m_offline; } + /** + * @brief get the blockchain pruning seed + * + * @return the blockchain pruning seed + */ + uint32_t get_blockchain_pruning_seed() const; + + /** + * @brief prune the blockchain + * + * @param pruning_seed the seed to use to prune the chain (0 for default, highly recommended) + * + * @return true iff success + */ + bool prune_blockchain(uint32_t pruning_seed = 0); + + /** + * @brief incrementally prunes blockchain + * + * @return true on success, false otherwise + */ + bool update_blockchain_pruning(); + + /** + * @brief checks the blockchain pruning if enabled + * + * @return true on success, false otherwise + */ + bool check_blockchain_pruning(); + private: /** @@ -984,6 +1023,7 @@ namespace cryptonote epee::math_helper::once_a_time_seconds<60*60*12, true> m_check_updates_interval; //!< interval for checking for new versions epee::math_helper::once_a_time_seconds<60*10, true> m_check_disk_space_interval; //!< interval for checking for disk space epee::math_helper::once_a_time_seconds<90, false> m_block_rate_interval; //!< interval for checking block rate + epee::math_helper::once_a_time_seconds<60*60*5, true> m_blockchain_pruning_interval; //!< interval for incremental blockchain pruning std::atomic<bool> m_starter_message_showed; //!< has the "daemon will sync now" message been shown? diff --git a/src/cryptonote_core/cryptonote_tx_utils.cpp b/src/cryptonote_core/cryptonote_tx_utils.cpp index f443d638c..0a04e0d38 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.cpp +++ b/src/cryptonote_core/cryptonote_tx_utils.cpp @@ -198,7 +198,7 @@ namespace cryptonote return addr.m_view_public_key; } //--------------------------------------------------------------- - bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct, rct::RangeProofType range_proof_type, rct::multisig_out *msout, bool shuffle_outs) + bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct, const rct::RCTConfig &rct_config, rct::multisig_out *msout, bool shuffle_outs) { hw::device &hwdev = sender_account_keys.get_device(); @@ -226,13 +226,15 @@ namespace cryptonote std::vector<tx_extra_field> tx_extra_fields; if (parse_tx_extra(tx.extra, tx_extra_fields)) { + bool add_dummy_payment_id = true; tx_extra_nonce extra_nonce; if (find_tx_extra_field_by_type(tx_extra_fields, extra_nonce)) { - crypto::hash8 payment_id = null_hash8; - if (get_encrypted_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id)) + crypto::hash payment_id = null_hash; + crypto::hash8 payment_id8 = null_hash8; + if (get_encrypted_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id8)) { - LOG_PRINT_L2("Encrypting payment id " << payment_id); + LOG_PRINT_L2("Encrypting payment id " << payment_id8); crypto::public_key view_key_pub = get_destination_view_key_pub(destinations, change_addr); if (view_key_pub == null_pkey) { @@ -240,28 +242,60 @@ namespace cryptonote return false; } - if (!hwdev.encrypt_payment_id(payment_id, view_key_pub, tx_key)) + if (!hwdev.encrypt_payment_id(payment_id8, view_key_pub, tx_key)) { LOG_ERROR("Failed to encrypt payment id"); return false; } std::string extra_nonce; - set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, payment_id); + set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, payment_id8); remove_field_from_tx_extra(tx.extra, typeid(tx_extra_nonce)); if (!add_extra_nonce_to_tx_extra(tx.extra, extra_nonce)) { LOG_ERROR("Failed to add encrypted payment id to tx extra"); return false; } - LOG_PRINT_L1("Encrypted payment ID: " << payment_id); + LOG_PRINT_L1("Encrypted payment ID: " << payment_id8); + add_dummy_payment_id = false; + } + else if (get_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id)) + { + add_dummy_payment_id = false; + } + } + + // we don't add one if we've got more than the usual 1 destination plus change + if (destinations.size() > 2) + add_dummy_payment_id = false; + + if (add_dummy_payment_id) + { + // if we have neither long nor short payment id, add a dummy short one, + // this should end up being the vast majority of txes as time goes on + std::string extra_nonce; + crypto::hash8 payment_id8 = null_hash8; + crypto::public_key view_key_pub = get_destination_view_key_pub(destinations, change_addr); + if (view_key_pub == null_pkey) + { + LOG_ERROR("Failed to get key to encrypt dummy payment id with"); + } + else + { + hwdev.encrypt_payment_id(payment_id8, view_key_pub, tx_key); + set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, payment_id8); + if (!add_extra_nonce_to_tx_extra(tx.extra, extra_nonce)) + { + LOG_ERROR("Failed to add dummy encrypted payment id to tx extra"); + // continue anyway + } } } } else { - LOG_ERROR("Failed to parse tx extra"); - return false; + MWARNING("Failed to parse tx extra"); + tx_extra_fields.clear(); } struct input_generation_context_data @@ -497,7 +531,7 @@ namespace cryptonote // the non-simple version is slightly smaller, but assumes all real inputs // are on the same index, so can only be used if there just one ring. - bool use_simple_rct = sources.size() > 1 || range_proof_type != rct::RangeProofBorromean; + bool use_simple_rct = sources.size() > 1 || rct_config.range_proof_type != rct::RangeProofBorromean; if (!use_simple_rct) { @@ -595,9 +629,9 @@ namespace cryptonote get_transaction_prefix_hash(tx, tx_prefix_hash); rct::ctkeyV outSk; if (use_simple_rct) - tx.rct_signatures = rct::genRctSimple(rct::hash2rct(tx_prefix_hash), inSk, destinations, inamounts, outamounts, amount_in - amount_out, mixRing, amount_keys, msout ? &kLRki : NULL, msout, index, outSk, range_proof_type, hwdev); + tx.rct_signatures = rct::genRctSimple(rct::hash2rct(tx_prefix_hash), inSk, destinations, inamounts, outamounts, amount_in - amount_out, mixRing, amount_keys, msout ? &kLRki : NULL, msout, index, outSk, rct_config, hwdev); else - tx.rct_signatures = rct::genRct(rct::hash2rct(tx_prefix_hash), inSk, destinations, outamounts, mixRing, amount_keys, msout ? &kLRki[0] : NULL, msout, sources[0].real_output, outSk, hwdev); // same index assumption + tx.rct_signatures = rct::genRct(rct::hash2rct(tx_prefix_hash), inSk, destinations, outamounts, mixRing, amount_keys, msout ? &kLRki[0] : NULL, msout, sources[0].real_output, outSk, rct_config, hwdev); // same index assumption memwipe(inSk.data(), inSk.size() * sizeof(rct::ctkey)); CHECK_AND_ASSERT_MES(tx.vout.size() == outSk.size(), false, "outSk size does not match vout"); @@ -610,7 +644,7 @@ namespace cryptonote return true; } //--------------------------------------------------------------- - bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys, bool rct, rct::RangeProofType range_proof_type, rct::multisig_out *msout) + bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys, bool rct, const rct::RCTConfig &rct_config, rct::multisig_out *msout) { hw::device &hwdev = sender_account_keys.get_device(); hwdev.open_tx(tx_key); @@ -628,7 +662,7 @@ namespace cryptonote additional_tx_keys.push_back(keypair::generate(sender_account_keys.get_device()).sec); } - bool r = construct_tx_with_tx_key(sender_account_keys, subaddresses, sources, destinations, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, rct, range_proof_type, msout); + bool r = construct_tx_with_tx_key(sender_account_keys, subaddresses, sources, destinations, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, rct, rct_config, msout); hwdev.close_tx(); return r; } @@ -640,7 +674,7 @@ namespace cryptonote crypto::secret_key tx_key; std::vector<crypto::secret_key> additional_tx_keys; std::vector<tx_destination_entry> destinations_copy = destinations; - return construct_tx_and_get_tx_key(sender_account_keys, subaddresses, sources, destinations_copy, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, rct::RangeProofBorromean, NULL); + return construct_tx_and_get_tx_key(sender_account_keys, subaddresses, sources, destinations_copy, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, { rct::RangeProofBorromean, 0}, NULL); } //--------------------------------------------------------------- bool generate_genesis_block( diff --git a/src/cryptonote_core/cryptonote_tx_utils.h b/src/cryptonote_core/cryptonote_tx_utils.h index 87edafe9d..85061668b 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.h +++ b/src/cryptonote_core/cryptonote_tx_utils.h @@ -73,25 +73,30 @@ namespace cryptonote struct tx_destination_entry { + std::string original; uint64_t amount; //money account_public_address addr; //destination address bool is_subaddress; + bool is_integrated; - tx_destination_entry() : amount(0), addr(AUTO_VAL_INIT(addr)), is_subaddress(false) { } - tx_destination_entry(uint64_t a, const account_public_address &ad, bool is_subaddress) : amount(a), addr(ad), is_subaddress(is_subaddress) { } + tx_destination_entry() : amount(0), addr(AUTO_VAL_INIT(addr)), is_subaddress(false), is_integrated(false) { } + tx_destination_entry(uint64_t a, const account_public_address &ad, bool is_subaddress) : amount(a), addr(ad), is_subaddress(is_subaddress), is_integrated(false) { } + tx_destination_entry(const std::string &o, uint64_t a, const account_public_address &ad, bool is_subaddress) : original(o), amount(a), addr(ad), is_subaddress(is_subaddress), is_integrated(false) { } BEGIN_SERIALIZE_OBJECT() + FIELD(original) VARINT_FIELD(amount) FIELD(addr) FIELD(is_subaddress) + FIELD(is_integrated) END_SERIALIZE() }; //--------------------------------------------------------------- crypto::public_key get_destination_view_key_pub(const std::vector<tx_destination_entry> &destinations, const boost::optional<cryptonote::account_public_address>& change_addr); bool construct_tx(const account_keys& sender_account_keys, std::vector<tx_source_entry> &sources, const std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time); - bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, rct::RangeProofType range_proof_type = rct::RangeProofBorromean, rct::multisig_out *msout = NULL, bool shuffle_outs = true); - bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, rct::RangeProofType range_proof_type = rct::RangeProofBorromean, rct::multisig_out *msout = NULL); + bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, const rct::RCTConfig &rct_config = { rct::RangeProofBorromean, 0 }, rct::multisig_out *msout = NULL, bool shuffle_outs = true); + bool construct_tx_and_get_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, const rct::RCTConfig &rct_config = { rct::RangeProofBorromean, 0 }, rct::multisig_out *msout = NULL); bool generate_genesis_block( block& bl @@ -102,7 +107,7 @@ namespace cryptonote } BOOST_CLASS_VERSION(cryptonote::tx_source_entry, 1) -BOOST_CLASS_VERSION(cryptonote::tx_destination_entry, 1) +BOOST_CLASS_VERSION(cryptonote::tx_destination_entry, 2) namespace boost { @@ -132,6 +137,13 @@ namespace boost if (ver < 1) return; a & x.is_subaddress; + if (ver < 2) + { + x.is_integrated = false; + return; + } + a & x.original; + a & x.is_integrated; } } } diff --git a/src/cryptonote_protocol/block_queue.cpp b/src/cryptonote_protocol/block_queue.cpp index 05f4189fb..672696944 100644 --- a/src/cryptonote_protocol/block_queue.cpp +++ b/src/cryptonote_protocol/block_queue.cpp @@ -31,8 +31,10 @@ #include <vector> #include <unordered_map> #include <boost/uuid/nil_generator.hpp> +#include <boost/uuid/uuid_io.hpp> #include "string_tools.h" #include "cryptonote_protocol_defs.h" +#include "common/pruning.h" #include "block_queue.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -59,7 +61,10 @@ void block_queue::add_blocks(uint64_t height, std::vector<cryptonote::block_comp if (has_hashes) { for (const crypto::hash &h: hashes) + { requested_hashes.insert(h); + have_blocks.insert(h); + } set_span_hashes(height, connection_id, hashes); } } @@ -89,7 +94,10 @@ void block_queue::erase_block(block_map::iterator j) { CHECK_AND_ASSERT_THROW_MES(j != blocks.end(), "Invalid iterator"); for (const crypto::hash &h: j->hashes) + { requested_hashes.erase(h); + have_blocks.erase(h); + } blocks.erase(j); } @@ -97,12 +105,10 @@ void block_queue::flush_stale_spans(const std::set<boost::uuids::uuid> &live_con { boost::unique_lock<boost::recursive_mutex> lock(mutex); block_map::iterator i = blocks.begin(); - if (i != blocks.end() && is_blockchain_placeholder(*i)) - ++i; while (i != blocks.end()) { block_map::iterator j = i++; - if (live_connections.find(j->connection_id) == live_connections.end() && j->blocks.size() == 0) + if (j->blocks.empty() && live_connections.find(j->connection_id) == live_connections.end()) { erase_block(j); } @@ -151,23 +157,56 @@ uint64_t block_queue::get_max_block_height() const return height; } +uint64_t block_queue::get_next_needed_height(uint64_t blockchain_height) const +{ + boost::unique_lock<boost::recursive_mutex> lock(mutex); + if (blocks.empty()) + return blockchain_height; + uint64_t last_needed_height = blockchain_height; + bool first = true; + for (const auto &span: blocks) + { + if (span.start_block_height + span.nblocks - 1 < blockchain_height) + continue; + if (span.start_block_height != last_needed_height || (first && span.blocks.empty())) + return last_needed_height; + last_needed_height = span.start_block_height + span.nblocks; + first = false; + } + return last_needed_height; +} + void block_queue::print() const { boost::unique_lock<boost::recursive_mutex> lock(mutex); MDEBUG("Block queue has " << blocks.size() << " spans"); for (const auto &span: blocks) - MDEBUG(" " << span.start_block_height << " - " << (span.start_block_height+span.nblocks-1) << " (" << span.nblocks << ") - " << (is_blockchain_placeholder(span) ? "blockchain" : span.blocks.empty() ? "scheduled" : "filled ") << " " << span.connection_id << " (" << ((unsigned)(span.rate*10/1024.f))/10.f << " kB/s)"); + MDEBUG(" " << span.start_block_height << " - " << (span.start_block_height+span.nblocks-1) << " (" << span.nblocks << ") - " << (span.blocks.empty() ? "scheduled" : "filled ") << " " << span.connection_id << " (" << ((unsigned)(span.rate*10/1024.f))/10.f << " kB/s)"); } -std::string block_queue::get_overview() const +std::string block_queue::get_overview(uint64_t blockchain_height) const { boost::unique_lock<boost::recursive_mutex> lock(mutex); if (blocks.empty()) return "[]"; block_map::const_iterator i = blocks.begin(); - std::string s = std::string("[") + std::to_string(i->start_block_height + i->nblocks - 1) + ":"; - while (++i != blocks.end()) - s += i->blocks.empty() ? "." : "o"; + std::string s = std::string("["); + uint64_t expected = blockchain_height; + while (i != blocks.end()) + { + if (expected > i->start_block_height) + { + s += "<"; + } + else + { + if (expected < i->start_block_height) + s += std::string(std::max((uint64_t)1, (i->start_block_height - expected) / (i->nblocks ? i->nblocks : 1)), '_'); + s += i->blocks.empty() ? "." : i->start_block_height == blockchain_height ? "m" : "o"; + expected = i->start_block_height + i->nblocks; + } + ++i; + } s += "]"; return s; } @@ -183,16 +222,31 @@ bool block_queue::requested(const crypto::hash &hash) const return requested_internal(hash); } -std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, const std::vector<crypto::hash> &block_hashes, boost::posix_time::ptime time) +bool block_queue::have(const crypto::hash &hash) const +{ + boost::unique_lock<boost::recursive_mutex> lock(mutex); + return have_blocks.find(hash) != have_blocks.end(); +} + +std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, uint32_t pruning_seed, uint64_t blockchain_height, const std::vector<crypto::hash> &block_hashes, boost::posix_time::ptime time) { boost::unique_lock<boost::recursive_mutex> lock(mutex); + MDEBUG("reserve_span: first_block_height " << first_block_height << ", last_block_height " << last_block_height + << ", max " << max_blocks << ", seed " << epee::string_tools::to_string_hex(pruning_seed) << ", blockchain_height " << + blockchain_height << ", block hashes size " << block_hashes.size()); if (last_block_height < first_block_height || max_blocks == 0) { MDEBUG("reserve_span: early out: first_block_height " << first_block_height << ", last_block_height " << last_block_height << ", max_blocks " << max_blocks); return std::make_pair(0, 0); } + if (block_hashes.size() >= last_block_height) + { + MDEBUG("reserve_span: more block hashes than fit within last_block_height: " << block_hashes.size() << " and " << last_block_height); + return std::make_pair(0, 0); + } + // skip everything we've already requested uint64_t span_start_height = last_block_height - block_hashes.size() + 1; std::vector<crypto::hash>::const_iterator i = block_hashes.begin(); while (i != block_hashes.end() && requested_internal(*i)) @@ -200,55 +254,57 @@ std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_hei ++i; ++span_start_height; } + + // if the peer's pruned for the starting block and its unpruned stripe comes next, start downloading from there + const uint32_t next_unpruned_height = tools::get_next_unpruned_block_height(span_start_height, blockchain_height, pruning_seed); + MDEBUG("reserve_span: next_unpruned_height " << next_unpruned_height << " from " << span_start_height << " and seed " + << epee::string_tools::to_string_hex(pruning_seed) << ", limit " << span_start_height + CRYPTONOTE_PRUNING_STRIPE_SIZE); + if (next_unpruned_height > span_start_height && next_unpruned_height < span_start_height + CRYPTONOTE_PRUNING_STRIPE_SIZE) + { + MDEBUG("We can download from next span: ideal height " << span_start_height << ", next unpruned height " << next_unpruned_height << + "(+" << next_unpruned_height - span_start_height << "), current seed " << pruning_seed); + span_start_height = next_unpruned_height; + } + MDEBUG("span_start_height: " <<span_start_height); + const uint64_t block_hashes_start_height = last_block_height - block_hashes.size() + 1; + if (span_start_height >= block_hashes.size() + block_hashes_start_height) + { + MDEBUG("Out of hashes, cannot reserve"); + return std::make_pair(0, 0); + } + + i = block_hashes.begin() + span_start_height - block_hashes_start_height; + while (i != block_hashes.end() && requested_internal(*i)) + { + ++i; + ++span_start_height; + } + uint64_t span_length = 0; std::vector<crypto::hash> hashes; - while (i != block_hashes.end() && span_length < max_blocks) + while (i != block_hashes.end() && span_length < max_blocks && tools::has_unpruned_block(span_start_height + span_length, blockchain_height, pruning_seed)) { hashes.push_back(*i); ++i; ++span_length; } if (span_length == 0) + { + MDEBUG("span_length 0, cannot reserve"); return std::make_pair(0, 0); + } MDEBUG("Reserving span " << span_start_height << " - " << (span_start_height + span_length - 1) << " for " << connection_id); add_blocks(span_start_height, span_length, connection_id, time); set_span_hashes(span_start_height, connection_id, hashes); return std::make_pair(span_start_height, span_length); } -bool block_queue::is_blockchain_placeholder(const span &span) const -{ - return span.connection_id == boost::uuids::nil_uuid(); -} - -std::pair<uint64_t, uint64_t> block_queue::get_start_gap_span() const -{ - boost::unique_lock<boost::recursive_mutex> lock(mutex); - if (blocks.empty()) - return std::make_pair(0, 0); - block_map::const_iterator i = blocks.begin(); - if (!is_blockchain_placeholder(*i)) - return std::make_pair(0, 0); - uint64_t current_height = i->start_block_height + i->nblocks - 1; - ++i; - if (i == blocks.end()) - return std::make_pair(0, 0); - uint64_t first_span_height = i->start_block_height; - if (first_span_height <= current_height + 1) - return std::make_pair(0, 0); - MDEBUG("Found gap at start of spans: last blockchain block height " << current_height << ", first span's block height " << first_span_height); - print(); - return std::make_pair(current_height + 1, first_span_height - current_height - 1); -} - std::pair<uint64_t, uint64_t> block_queue::get_next_span_if_scheduled(std::vector<crypto::hash> &hashes, boost::uuids::uuid &connection_id, boost::posix_time::ptime &time) const { boost::unique_lock<boost::recursive_mutex> lock(mutex); if (blocks.empty()) return std::make_pair(0, 0); block_map::const_iterator i = blocks.begin(); - if (is_blockchain_placeholder(*i)) - ++i; if (i == blocks.end()) return std::make_pair(0, 0); if (!i->blocks.empty()) @@ -259,6 +315,16 @@ std::pair<uint64_t, uint64_t> block_queue::get_next_span_if_scheduled(std::vecto return std::make_pair(i->start_block_height, i->nblocks); } +void block_queue::reset_next_span_time(boost::posix_time::ptime t) +{ + boost::unique_lock<boost::recursive_mutex> lock(mutex); + CHECK_AND_ASSERT_THROW_MES(!blocks.empty(), "No next span to reset time"); + block_map::iterator i = blocks.begin(); + CHECK_AND_ASSERT_THROW_MES(i != blocks.end(), "No next span to reset time"); + CHECK_AND_ASSERT_THROW_MES(i->blocks.empty(), "Next span is not empty"); + (boost::posix_time::ptime&)i->time = t; // sod off, time doesn't influence sorting +} + void block_queue::set_span_hashes(uint64_t start_height, const boost::uuids::uuid &connection_id, std::vector<crypto::hash> hashes) { boost::unique_lock<boost::recursive_mutex> lock(mutex); @@ -283,8 +349,6 @@ bool block_queue::get_next_span(uint64_t &height, std::vector<cryptonote::block_ if (blocks.empty()) return false; block_map::const_iterator i = blocks.begin(); - if (is_blockchain_placeholder(*i)) - ++i; for (; i != blocks.end(); ++i) { if (!filled || !i->blocks.empty()) @@ -298,19 +362,34 @@ bool block_queue::get_next_span(uint64_t &height, std::vector<cryptonote::block_ return false; } -bool block_queue::has_next_span(const boost::uuids::uuid &connection_id, bool &filled) const +bool block_queue::has_next_span(const boost::uuids::uuid &connection_id, bool &filled, boost::posix_time::ptime &time) const { boost::unique_lock<boost::recursive_mutex> lock(mutex); if (blocks.empty()) return false; block_map::const_iterator i = blocks.begin(); - if (is_blockchain_placeholder(*i)) - ++i; if (i == blocks.end()) return false; if (i->connection_id != connection_id) return false; filled = !i->blocks.empty(); + time = i->time; + return true; +} + +bool block_queue::has_next_span(uint64_t height, bool &filled, boost::posix_time::ptime &time, boost::uuids::uuid &connection_id) const +{ + boost::unique_lock<boost::recursive_mutex> lock(mutex); + if (blocks.empty()) + return false; + block_map::const_iterator i = blocks.begin(); + if (i == blocks.end()) + return false; + if (i->start_block_height > height) + return false; + filled = !i->blocks.empty(); + time = i->time; + connection_id = i->connection_id; return true; } @@ -330,8 +409,6 @@ size_t block_queue::get_num_filled_spans_prefix() const if (blocks.empty()) return 0; block_map::const_iterator i = blocks.begin(); - if (is_blockchain_placeholder(*i)) - ++i; size_t size = 0; while (i != blocks.end() && !i->blocks.empty()) { @@ -416,12 +493,35 @@ float block_queue::get_speed(const boost::uuids::uuid &connection_id) const return speed; } -bool block_queue::foreach(std::function<bool(const span&)> f, bool include_blockchain_placeholder) const +float block_queue::get_download_rate(const boost::uuids::uuid &connection_id) const +{ + boost::unique_lock<boost::recursive_mutex> lock(mutex); + float conn_rate = -1.f; + for (const auto &span: blocks) + { + if (span.blocks.empty()) + continue; + if (span.connection_id != connection_id) + continue; + // note that the average below does not average over the whole set, but over the + // previous pseudo average and the latest rate: this gives much more importance + // to the latest measurements, which is fine here + if (conn_rate < 0.f) + conn_rate = span.rate; + else + conn_rate = (conn_rate + span.rate) / 2; + } + + if (conn_rate < 0) + conn_rate = 0.0f; + MTRACE("Download rate for " << connection_id << ": " << conn_rate << " b/s"); + return conn_rate; +} + +bool block_queue::foreach(std::function<bool(const span&)> f) const { boost::unique_lock<boost::recursive_mutex> lock(mutex); block_map::const_iterator i = blocks.begin(); - if (!include_blockchain_placeholder && i != blocks.end() && is_blockchain_placeholder(*i)) - ++i; while (i != blocks.end()) if (!f(*i++)) return false; diff --git a/src/cryptonote_protocol/block_queue.h b/src/cryptonote_protocol/block_queue.h index 9cce95075..c7d3af7ac 100644 --- a/src/cryptonote_protocol/block_queue.h +++ b/src/cryptonote_protocol/block_queue.h @@ -76,22 +76,26 @@ namespace cryptonote void remove_spans(const boost::uuids::uuid &connection_id, uint64_t start_block_height); uint64_t get_max_block_height() const; void print() const; - std::string get_overview() const; - std::pair<uint64_t, uint64_t> reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, const std::vector<crypto::hash> &block_hashes, boost::posix_time::ptime time = boost::posix_time::microsec_clock::universal_time()); - bool is_blockchain_placeholder(const span &span) const; - std::pair<uint64_t, uint64_t> get_start_gap_span() const; + std::string get_overview(uint64_t blockchain_height) const; + bool has_unpruned_height(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed) const; + std::pair<uint64_t, uint64_t> reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, uint32_t pruning_seed, uint64_t blockchain_height, const std::vector<crypto::hash> &block_hashes, boost::posix_time::ptime time = boost::posix_time::microsec_clock::universal_time()); + uint64_t get_next_needed_height(uint64_t blockchain_height) const; std::pair<uint64_t, uint64_t> get_next_span_if_scheduled(std::vector<crypto::hash> &hashes, boost::uuids::uuid &connection_id, boost::posix_time::ptime &time) const; + void reset_next_span_time(boost::posix_time::ptime t = boost::posix_time::microsec_clock::universal_time()); void set_span_hashes(uint64_t start_height, const boost::uuids::uuid &connection_id, std::vector<crypto::hash> hashes); bool get_next_span(uint64_t &height, std::vector<cryptonote::block_complete_entry> &bcel, boost::uuids::uuid &connection_id, bool filled = true) const; - bool has_next_span(const boost::uuids::uuid &connection_id, bool &filled) const; + bool has_next_span(const boost::uuids::uuid &connection_id, bool &filled, boost::posix_time::ptime &time) const; + bool has_next_span(uint64_t height, bool &filled, boost::posix_time::ptime &time, boost::uuids::uuid &connection_id) const; size_t get_data_size() const; size_t get_num_filled_spans_prefix() const; size_t get_num_filled_spans() const; crypto::hash get_last_known_hash(const boost::uuids::uuid &connection_id) const; bool has_spans(const boost::uuids::uuid &connection_id) const; float get_speed(const boost::uuids::uuid &connection_id) const; - bool foreach(std::function<bool(const span&)> f, bool include_blockchain_placeholder = false) const; + float get_download_rate(const boost::uuids::uuid &connection_id) const; + bool foreach(std::function<bool(const span&)> f) const; bool requested(const crypto::hash &hash) const; + bool have(const crypto::hash &hash) const; private: void erase_block(block_map::iterator j); @@ -101,5 +105,6 @@ namespace cryptonote block_map blocks; mutable boost::recursive_mutex mutex; std::unordered_set<crypto::hash> requested_hashes; + std::unordered_set<crypto::hash> have_blocks; }; } diff --git a/src/cryptonote_protocol/cryptonote_protocol_defs.h b/src/cryptonote_protocol/cryptonote_protocol_defs.h index d5bb50930..c49371d48 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_defs.h +++ b/src/cryptonote_protocol/cryptonote_protocol_defs.h @@ -78,6 +78,8 @@ namespace cryptonote uint64_t height; + uint32_t pruning_seed; + BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(incoming) KV_SERIALIZE(localhost) @@ -100,6 +102,7 @@ namespace cryptonote KV_SERIALIZE(support_flags) KV_SERIALIZE(connection_id) KV_SERIALIZE(height) + KV_SERIALIZE(pruning_seed) END_KV_SERIALIZE_MAP() }; @@ -200,12 +203,14 @@ namespace cryptonote uint64_t cumulative_difficulty; crypto::hash top_id; uint8_t top_version; + uint32_t pruning_seed; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(current_height) KV_SERIALIZE(cumulative_difficulty) KV_SERIALIZE_VAL_POD_AS_BLOB(top_id) KV_SERIALIZE_OPT(top_version, (uint8_t)0) + KV_SERIALIZE_OPT(pruning_seed, (uint32_t)0) END_KV_SERIALIZE_MAP() }; diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.h b/src/cryptonote_protocol/cryptonote_protocol_handler.h index 618b635cc..a1bd9171c 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.h @@ -43,6 +43,7 @@ #include "cryptonote_protocol_defs.h" #include "cryptonote_protocol_handler_common.h" #include "block_queue.h" +#include "common/perf_timer.h" #include "cryptonote_basic/connection_context.h" #include "cryptonote_basic/cryptonote_stat_info.h" #include <boost/circular_buffer.hpp> @@ -109,6 +110,10 @@ namespace cryptonote const block_queue &get_block_queue() const { return m_block_queue; } void stop(); void on_connection_close(cryptonote_connection_context &context); + void set_max_out_peers(unsigned int max) { m_max_out_peers = max; } + std::string get_peers_overview() const; + std::pair<uint32_t, uint32_t> get_next_needed_pruning_stripe() const; + bool needs_new_sync_connections() const; private: //----------------- commands handlers ---------------------------------------------- int handle_notify_new_block(int command, NOTIFY_NEW_BLOCK::request& arg, cryptonote_connection_context& context); @@ -125,14 +130,17 @@ namespace cryptonote virtual bool relay_transactions(NOTIFY_NEW_TRANSACTIONS::request& arg, cryptonote_connection_context& exclude_context); //---------------------------------------------------------------------------------- //bool get_payload_sync_data(HANDSHAKE_DATA::request& hshd, cryptonote_connection_context& context); + bool should_drop_connection(cryptonote_connection_context& context, uint32_t next_stripe); bool request_missing_objects(cryptonote_connection_context& context, bool check_having_blocks, bool force_next_span = false); size_t get_synchronizing_connections_count(); bool on_connection_synchronized(); - bool should_download_next_span(cryptonote_connection_context& context) const; + bool should_download_next_span(cryptonote_connection_context& context, bool standby); void drop_connection(cryptonote_connection_context &context, bool add_fail, bool flush_all_spans); bool kick_idle_peers(); bool check_standby_peers(); int try_add_next_blocks(cryptonote_connection_context &context); + void notify_new_stripe(cryptonote_connection_context &context, uint32_t stripe); + void skip_unneeded_hashes(cryptonote_connection_context& context, bool check_block_queue) const; t_core& m_core; @@ -145,6 +153,12 @@ namespace cryptonote block_queue m_block_queue; epee::math_helper::once_a_time_seconds<30> m_idle_peer_kicker; epee::math_helper::once_a_time_milliseconds<100> m_standby_checker; + std::atomic<unsigned int> m_max_out_peers; + tools::PerformanceTimer m_sync_timer, m_add_timer; + uint64_t m_last_add_end_time; + uint64_t m_sync_spans_downloaded, m_sync_old_spans_downloaded, m_sync_bad_spans_downloaded; + uint64_t m_sync_download_chain_size, m_sync_download_objects_size; + size_t m_block_download_max_size; boost::mutex m_buffer_mutex; double get_avg_block_size(); @@ -157,7 +171,7 @@ namespace cryptonote std::string blob; epee::serialization::store_t_to_binary(arg, blob); //handler_response_blocks_now(blob.size()); // XXX - return m_p2p->invoke_notify_to_peer(t_parameter::ID, blob, context); + return m_p2p->invoke_notify_to_peer(t_parameter::ID, epee::strspan<uint8_t>(blob), context); } template<class t_parameter> @@ -166,7 +180,7 @@ namespace cryptonote LOG_PRINT_L2("[" << epee::net_utils::print_connection_context_short(exclude_context) << "] post relay " << typeid(t_parameter).name() << " -->"); std::string arg_buff; epee::serialization::store_t_to_binary(arg, arg_buff); - return m_p2p->relay_notify_to_all(t_parameter::ID, arg_buff, exclude_context); + return m_p2p->relay_notify_to_all(t_parameter::ID, epee::strspan<uint8_t>(arg_buff), exclude_context); } }; diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index 999ec5650..61a211094 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -42,17 +42,24 @@ #include "cryptonote_basic/cryptonote_format_utils.h" #include "profile_tools.h" #include "net/network_throttle-detail.hpp" +#include "common/pruning.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net.cn" #define MLOG_P2P_MESSAGE(x) MCINFO("net.p2p.msg", context << x) +#define MLOG_PEER_STATE(x) \ + MCINFO(MONERO_DEFAULT_LOG_CATEGORY, context << "[" << epee::string_tools::to_string_hex(context.m_pruning_seed) << "] state: " << x << " in state " << cryptonote::get_protocol_state_string(context.m_state)) -#define BLOCK_QUEUE_NBLOCKS_THRESHOLD 10 // chunks of N blocks +#define BLOCK_QUEUE_NSPANS_THRESHOLD 10 // chunks of N blocks #define BLOCK_QUEUE_SIZE_THRESHOLD (100*1024*1024) // MB -#define REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD (5 * 1000000) // microseconds +#define BLOCK_QUEUE_FORCE_DOWNLOAD_NEAR_BLOCKS 1000 +#define REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD_STANDBY (5 * 1000000) // microseconds +#define REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD (30 * 1000000) // microseconds #define IDLE_PEER_KICK_TIME (600 * 1000000) // microseconds #define PASSIVE_PEER_KICK_TIME (60 * 1000000) // microseconds +#define DROP_ON_SYNC_WEDGE_THRESHOLD (30 * 1000000000ull) // nanoseconds +#define LAST_ACTIVITY_STALL_THRESHOLD (2.0f) // seconds namespace cryptonote { @@ -75,6 +82,19 @@ namespace cryptonote template<class t_core> bool t_cryptonote_protocol_handler<t_core>::init(const boost::program_options::variables_map& vm) { + m_sync_timer.pause(); + m_sync_timer.reset(); + m_add_timer.pause(); + m_add_timer.reset(); + m_last_add_end_time = 0; + m_sync_spans_downloaded = 0; + m_sync_old_spans_downloaded = 0; + m_sync_bad_spans_downloaded = 0; + m_sync_download_chain_size = 0; + m_sync_download_objects_size = 0; + + m_block_download_max_size = command_line::get_arg(vm, cryptonote::arg_block_download_max_size); + return true; } //------------------------------------------------------------------------------------------------------------------------ @@ -103,9 +123,12 @@ namespace cryptonote if(context.m_state == cryptonote_connection_context::state_synchronizing) { NOTIFY_REQUEST_CHAIN::request r = boost::value_initialized<NOTIFY_REQUEST_CHAIN::request>(); + context.m_needed_objects.clear(); m_core.get_short_chain_history(r.block_ids); - LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); + handler_request_blocks_history( r.block_ids ); // change the limit(?), sleep(?) + MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); post_notify<NOTIFY_REQUEST_CHAIN>(r, context); + MLOG_PEER_STATE("requesting chain"); } else if(context.m_state == cryptonote_connection_context::state_standby) { @@ -247,6 +270,7 @@ namespace cryptonote cnx.connection_id = epee::string_tools::pod_to_hex(cntxt.m_connection_id); cnx.height = cntxt.m_remote_blockchain_height; + cnx.pruning_seed = cntxt.m_pruning_seed; connections.push_back(cnx); @@ -279,7 +303,23 @@ namespace cryptonote } } + // reject weird pruning schemes + if (hshd.pruning_seed) + { + const uint32_t log_stripes = tools::get_pruning_log_stripes(hshd.pruning_seed); + if (log_stripes != CRYPTONOTE_PRUNING_LOG_STRIPES || tools::get_pruning_stripe(hshd.pruning_seed) > (1u << log_stripes)) + { + MWARNING(context << " peer claim unexpected pruning seed " << epee::string_tools::to_string_hex(hshd.pruning_seed) << ", disconnecting"); + return false; + } + } + context.m_remote_blockchain_height = hshd.current_height; + context.m_pruning_seed = hshd.pruning_seed; +#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED + context.m_pruning_seed = tools::make_pruning_seed(1 + (context.m_remote_address.as<epee::net_utils::ipv4_network_address>().ip()) % (1 << CRYPTONOTE_PRUNING_LOG_STRIPES), CRYPTONOTE_PRUNING_LOG_STRIPES); + LOG_INFO_CC(context, "New connection posing as pruning seed " << epee::string_tools::to_string_hex(context.m_pruning_seed) << ", seed address " << &context.m_pruning_seed); +#endif uint64_t target = m_core.get_target_blockchain_height(); if (target == 0) @@ -298,7 +338,6 @@ namespace cryptonote /* As I don't know if accessing hshd from core could be a good practice, I prefer pushing target height to the core at the same time it is pushed to the user. Nz. */ - m_core.set_target_blockchain_height((hshd.current_height)); int64_t diff = static_cast<int64_t>(hshd.current_height) - static_cast<int64_t>(m_core.get_current_blockchain_height()); uint64_t abs_diff = std::abs(diff); uint64_t max_block_height = std::max(hshd.current_height,m_core.get_current_blockchain_height()); @@ -309,14 +348,31 @@ namespace cryptonote << (0 <= diff ? std::string("behind") : std::string("ahead")) << "] " << ENDL << "SYNCHRONIZATION started"); if (hshd.current_height >= m_core.get_current_blockchain_height() + 5) // don't switch to unsafe mode just for a few blocks + { m_core.safesyncmode(false); + } + if (m_core.get_target_blockchain_height() == 0) // only when sync starts + { + m_sync_timer.resume(); + m_sync_timer.reset(); + m_add_timer.pause(); + m_add_timer.reset(); + m_last_add_end_time = 0; + m_sync_spans_downloaded = 0; + m_sync_old_spans_downloaded = 0; + m_sync_bad_spans_downloaded = 0; + m_sync_download_chain_size = 0; + m_sync_download_objects_size = 0; + } + m_core.set_target_blockchain_height((hshd.current_height)); } - LOG_PRINT_L1("Remote blockchain height: " << hshd.current_height << ", id: " << hshd.top_id); + MINFO(context << "Remote blockchain height: " << hshd.current_height << ", id: " << hshd.top_id); context.m_state = cryptonote_connection_context::state_synchronizing; //let the socket to send response to handshake, but request callback, to let send request data after response LOG_PRINT_CCONTEXT_L2("requesting callback"); ++context.m_callback_request_count; m_p2p->request_callback(context); + MLOG_PEER_STATE("requesting callback"); return true; } //------------------------------------------------------------------------------------------------------------------------ @@ -327,6 +383,7 @@ namespace cryptonote hshd.top_version = m_core.get_ideal_hard_fork_version(hshd.current_height); hshd.cumulative_difficulty = m_core.get_block_cumulative_difficulty(hshd.current_height); hshd.current_height +=1; + hshd.pruning_seed = m_core.get_blockchain_pruning_seed(); return true; } //------------------------------------------------------------------------------------------------------------------------ @@ -389,11 +446,14 @@ namespace cryptonote relay_block(arg, context); }else if(bvc.m_marked_as_orphaned) { + context.m_needed_objects.clear(); context.m_state = cryptonote_connection_context::state_synchronizing; NOTIFY_REQUEST_CHAIN::request r = boost::value_initialized<NOTIFY_REQUEST_CHAIN::request>(); m_core.get_short_chain_history(r.block_ids); - LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); + handler_request_blocks_history( r.block_ids ); // change the limit(?), sleep(?) + MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); post_notify<NOTIFY_REQUEST_CHAIN>(r, context); + MLOG_PEER_STATE("requesting chain"); } return 1; @@ -616,6 +676,7 @@ namespace cryptonote missing_tx_req.missing_tx_indices = std::move(need_tx_indices); m_core.resume_mine(); + MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_FLUFFY_MISSING_TX: missing_tx_indices.size()=" << missing_tx_req.missing_tx_indices.size() ); post_notify<NOTIFY_REQUEST_FLUFFY_MISSING_TX>(missing_tx_req, context); } else // whoo-hoo we've got em all .. @@ -656,11 +717,14 @@ namespace cryptonote } else if( bvc.m_marked_as_orphaned ) { + context.m_needed_objects.clear(); context.m_state = cryptonote_connection_context::state_synchronizing; NOTIFY_REQUEST_CHAIN::request r = boost::value_initialized<NOTIFY_REQUEST_CHAIN::request>(); m_core.get_short_chain_history(r.block_ids); - LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); + handler_request_blocks_history( r.block_ids ); // change the limit(?), sleep(?) + MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); post_notify<NOTIFY_REQUEST_CHAIN>(r, context); + MLOG_PEER_STATE("requesting chain"); } } } @@ -747,7 +811,7 @@ namespace cryptonote fluffy_response.b.txs.push_back(t_serializable_object_to_blob(tx)); } - LOG_PRINT_CCONTEXT_L2 + MLOG_P2P_MESSAGE ( "-->>NOTIFY_RESPONSE_FLUFFY_MISSING_TX: " << ", txs.size()=" << fluffy_response.b.txs.size() @@ -811,7 +875,7 @@ namespace cryptonote drop_connection(context, false, false); return 1; } - LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_RESPONSE_GET_OBJECTS: blocks.size()=" << rsp.blocks.size() << ", txs.size()=" << rsp.txs.size() + MLOG_P2P_MESSAGE("-->>NOTIFY_RESPONSE_GET_OBJECTS: blocks.size()=" << rsp.blocks.size() << ", txs.size()=" << rsp.txs.size() << ", rsp.m_current_blockchain_height=" << rsp.current_blockchain_height << ", missed_ids.size()=" << rsp.missed_ids.size()); post_notify<NOTIFY_RESPONSE_GET_OBJECTS>(rsp, context); //handler_response_blocks_now(sizeof(rsp)); // XXX @@ -834,11 +898,14 @@ namespace cryptonote return avg / m_avg_buffer.size(); } - template<class t_core> int t_cryptonote_protocol_handler<t_core>::handle_response_get_objects(int command, NOTIFY_RESPONSE_GET_OBJECTS::request& arg, cryptonote_connection_context& context) { MLOG_P2P_MESSAGE("Received NOTIFY_RESPONSE_GET_OBJECTS (" << arg.blocks.size() << " blocks, " << arg.txs.size() << " txes)"); + MLOG_PEER_STATE("received objects"); + + boost::posix_time::ptime request_time = context.m_last_request_time; + context.m_last_request_time = boost::date_time::not_a_date_time; // calculate size of request size_t size = 0; @@ -860,6 +927,8 @@ namespace cryptonote CRITICAL_REGION_LOCAL(m_buffer_mutex); m_avg_buffer.push_back(size); } + ++m_sync_spans_downloaded; + m_sync_download_objects_size += size; MDEBUG(context << " downloaded " << size << " bytes worth of blocks"); /*using namespace boost::chrono; @@ -874,6 +943,7 @@ namespace cryptonote LOG_ERROR_CCONTEXT("sent wrong NOTIFY_HAVE_OBJECTS: arg.m_current_blockchain_height=" << arg.current_blockchain_height << " < m_last_response_height=" << context.m_last_response_height << ", dropping connection"); drop_connection(context, false, false); + ++m_sync_bad_spans_downloaded; return 1; } @@ -898,6 +968,7 @@ namespace cryptonote LOG_ERROR_CCONTEXT("sent wrong block: failed to parse and validate block: " << epee::string_tools::buff_to_hex_nodelimer(block_entry.block) << ", dropping connection"); drop_connection(context, false, false); + ++m_sync_bad_spans_downloaded; return 1; } if (b.miner_tx.vin.size() != 1 || b.miner_tx.vin.front().type() != typeid(txin_gen)) @@ -905,6 +976,7 @@ namespace cryptonote LOG_ERROR_CCONTEXT("sent wrong block: block: miner tx does not have exactly one txin_gen input" << epee::string_tools::buff_to_hex_nodelimer(block_entry.block) << ", dropping connection"); drop_connection(context, false, false); + ++m_sync_bad_spans_downloaded; return 1; } if (start_height == std::numeric_limits<uint64_t>::max()) @@ -917,6 +989,7 @@ namespace cryptonote LOG_ERROR_CCONTEXT("sent wrong NOTIFY_RESPONSE_GET_OBJECTS: block with id=" << epee::string_tools::pod_to_hex(get_blob_hash(block_entry.block)) << " wasn't requested, dropping connection"); drop_connection(context, false, false); + ++m_sync_bad_spans_downloaded; return 1; } if(b.tx_hashes.size() != block_entry.txs.size()) @@ -924,6 +997,7 @@ namespace cryptonote LOG_ERROR_CCONTEXT("sent wrong NOTIFY_RESPONSE_GET_OBJECTS: block with id=" << epee::string_tools::pod_to_hex(get_blob_hash(block_entry.block)) << ", tx_hashes.size()=" << b.tx_hashes.size() << " mismatch with block_complete_entry.m_txs.size()=" << block_entry.txs.size() << ", dropping connection"); drop_connection(context, false, false); + ++m_sync_bad_spans_downloaded; return 1; } @@ -931,34 +1005,27 @@ namespace cryptonote block_hashes.push_back(block_hash); } - if(context.m_requested_objects.size()) + if(!context.m_requested_objects.empty()) { - MERROR("returned not all requested objects (context.m_requested_objects.size()=" + MERROR(context << "returned not all requested objects (context.m_requested_objects.size()=" << context.m_requested_objects.size() << "), dropping connection"); drop_connection(context, false, false); + ++m_sync_bad_spans_downloaded; return 1; } - // get the last parsed block, which should be the highest one - const crypto::hash last_block_hash = cryptonote::get_block_hash(b); - if(m_core.have_block(last_block_hash)) - { - const uint64_t subchain_height = start_height + arg.blocks.size(); - LOG_DEBUG_CC(context, "These are old blocks, ignoring: blocks " << start_height << " - " << (subchain_height-1) << ", blockchain height " << m_core.get_current_blockchain_height()); - m_block_queue.remove_spans(context.m_connection_id, start_height); - goto skip; - } - { MLOG_YELLOW(el::Level::Debug, context << " Got NEW BLOCKS inside of " << __FUNCTION__ << ": size: " << arg.blocks.size() - << ", blocks: " << start_height << " - " << (start_height + arg.blocks.size() - 1)); + << ", blocks: " << start_height << " - " << (start_height + arg.blocks.size() - 1) << + " (pruning seed " << epee::string_tools::to_string_hex(context.m_pruning_seed) << ")"); // add that new span to the block queue - const boost::posix_time::time_duration dt = now - context.m_last_request_time; + const boost::posix_time::time_duration dt = now - request_time; const float rate = size * 1e6 / (dt.total_microseconds() + 1); - MDEBUG(context << " adding span: " << arg.blocks.size() << " at height " << start_height << ", " << dt.total_microseconds()/1e6 << " seconds, " << (rate/1e3) << " kB/s, size now " << (m_block_queue.get_data_size() + blocks_size) / 1048576.f << " MB"); + MDEBUG(context << " adding span: " << arg.blocks.size() << " at height " << start_height << ", " << dt.total_microseconds()/1e6 << " seconds, " << (rate/1024) << " kB/s, size now " << (m_block_queue.get_data_size() + blocks_size) / 1048576.f << " MB"); m_block_queue.add_blocks(start_height, arg.blocks, context.m_connection_id, rate, blocks_size); + const crypto::hash last_block_hash = cryptonote::get_block_hash(b); context.m_last_known_hash = last_block_hash; if (!m_core.get_test_drop_download() || !m_core.get_test_drop_download_height()) { // DISCARD BLOCKS for testing @@ -966,7 +1033,6 @@ namespace cryptonote } } -skip: try_add_next_blocks(context); return 1; } @@ -983,15 +1049,22 @@ skip: const boost::unique_lock<boost::mutex> sync{m_sync_lock, boost::try_to_lock}; if (!sync.owns_lock()) { - MINFO("Failed to lock m_sync_lock, going back to download"); + MINFO(context << "Failed to lock m_sync_lock, going back to download"); goto skip; } MDEBUG(context << " lock m_sync_lock, adding blocks to chain..."); + MLOG_PEER_STATE("adding blocks"); { m_core.pause_mine(); - epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler( - boost::bind(&t_core::resume_mine, &m_core)); + m_add_timer.resume(); + bool starting = true; + epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([this, &starting]() { + m_add_timer.pause(); + m_core.resume_mine(); + if (!starting) + m_last_add_end_time = tools::get_tick_count(); + }); while (1) { @@ -1004,22 +1077,38 @@ skip: MDEBUG(context << " no next span found, going back to download"); break; } - MDEBUG(context << " next span in the queue has blocks " << start_height << "-" << (start_height + blocks.size() - 1) - << ", we need " << previous_height); if (blocks.empty()) { - MERROR("Next span has no blocks"); + MERROR(context << "Next span has no blocks"); m_block_queue.remove_spans(span_connection_id, start_height); - break; + continue; } + MDEBUG(context << " next span in the queue has blocks " << start_height << "-" << (start_height + blocks.size() - 1) + << ", we need " << previous_height); + block new_block; + if (!parse_and_validate_block_from_blob(blocks.back().block, new_block)) + { + MERROR(context << "Failed to parse block, but it should already have been parsed"); + m_block_queue.remove_spans(span_connection_id, start_height); + continue; + } + const crypto::hash last_block_hash = cryptonote::get_block_hash(new_block); + if (m_core.have_block(last_block_hash)) + { + const uint64_t subchain_height = start_height + blocks.size(); + LOG_DEBUG_CC(context, "These are old blocks, ignoring: blocks " << start_height << " - " << (subchain_height-1) << ", blockchain height " << m_core.get_current_blockchain_height()); + m_block_queue.remove_spans(span_connection_id, start_height); + ++m_sync_old_spans_downloaded; + continue; + } if (!parse_and_validate_block_from_blob(blocks.front().block, new_block)) { - MERROR("Failed to parse block, but it should already have been parsed"); + MERROR(context << "Failed to parse block, but it should already have been parsed"); m_block_queue.remove_spans(span_connection_id, start_height); - break; + continue; } bool parent_known = m_core.have_block(new_block.prev_id); if (!parent_known) @@ -1032,6 +1121,24 @@ skip: bool parent_requested = m_block_queue.requested(new_block.prev_id); if (!parent_requested) { + // we might be able to ask for that block directly, as we now can request out of order, + // otherwise we continue out of order, unless this block is the one we need, in which + // case we request block hashes, though it might be safer to disconnect ? + if (start_height > previous_height) + { + if (should_drop_connection(context, get_next_needed_pruning_stripe().first)) + { + MDEBUG(context << "Got block with unknown parent which was not requested, but peer does not have that block - dropping connection"); + if (!context.m_is_income) + m_p2p->add_used_stripe_peer(context); + drop_connection(context, false, true); + return 1; + } + MDEBUG(context << "Got block with unknown parent which was not requested, but peer does not have that block - back to download"); + + goto skip; + } + // this can happen if a connection was sicced onto a late span, if it did not have those blocks, // since we don't know that at the sic time LOG_ERROR_CCONTEXT("Got block with unknown parent which was not requested - querying block hashes"); @@ -1047,7 +1154,17 @@ skip: } const boost::posix_time::ptime start = boost::posix_time::microsec_clock::universal_time(); - context.m_last_request_time = start; + + if (starting) + { + starting = false; + if (m_last_add_end_time) + { + const uint64_t tnow = tools::get_tick_count(); + const uint64_t ns = tools::ticks_to_ns(tnow - m_last_add_end_time); + MINFO("Restarting adding block after idle for " << ns/1e9 << " seconds"); + } + } m_core.prepare_handle_incoming_blocks(blocks); @@ -1150,7 +1267,7 @@ skip: } // each download block - MCINFO("sync-info", "Block process time (" << blocks.size() << " blocks, " << num_txs << " txs): " << block_process_time_full + transactions_process_time_full << " (" << transactions_process_time_full << "/" << block_process_time_full << ") ms"); + MDEBUG(context << "Block process time (" << blocks.size() << " blocks, " << num_txs << " txs): " << block_process_time_full + transactions_process_time_full << " (" << transactions_process_time_full << "/" << block_process_time_full << ") ms"); if (!m_core.cleanup_handle_incoming_blocks()) { @@ -1160,40 +1277,54 @@ skip: m_block_queue.remove_spans(span_connection_id, start_height); - if (m_core.get_current_blockchain_height() > previous_height) + const uint64_t current_blockchain_height = m_core.get_current_blockchain_height(); + if (current_blockchain_height > previous_height) { + const uint64_t target_blockchain_height = m_core.get_target_blockchain_height(); const boost::posix_time::time_duration dt = boost::posix_time::microsec_clock::universal_time() - start; + std::string progress_message = ""; + if (current_blockchain_height < target_blockchain_height) + { + uint64_t completion_percent = (current_blockchain_height * 100 / target_blockchain_height); + if (completion_percent == 100) // never show 100% if not actually up to date + completion_percent = 99; + progress_message = " (" + std::to_string(completion_percent) + "%, " + + std::to_string(target_blockchain_height - current_blockchain_height) + " left)"; + } + const uint32_t previous_stripe = tools::get_pruning_stripe(previous_height, target_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES); + const uint32_t current_stripe = tools::get_pruning_stripe(current_blockchain_height, target_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES); std::string timing_message = ""; if (ELPP->vRegistry()->allowed(el::Level::Info, "sync-info")) timing_message = std::string(" (") + std::to_string(dt.total_microseconds()/1e6) + " sec, " - + std::to_string((m_core.get_current_blockchain_height() - previous_height) * 1e6 / dt.total_microseconds()) - + " blocks/sec), " + std::to_string(m_block_queue.get_data_size() / 1048576.f) + " MB queued"; + + std::to_string((current_blockchain_height - previous_height) * 1e6 / dt.total_microseconds()) + + " blocks/sec), " + std::to_string(m_block_queue.get_data_size() / 1048576.f) + " MB queued in " + + std::to_string(m_block_queue.get_num_filled_spans()) + " spans, stripe " + + std::to_string(previous_stripe) + " -> " + std::to_string(current_stripe); if (ELPP->vRegistry()->allowed(el::Level::Debug, "sync-info")) - timing_message += std::string(": ") + m_block_queue.get_overview(); - if(m_core.get_target_blockchain_height() == 0){ - MGINFO_YELLOW(context << " Synced " << m_core.get_current_blockchain_height() << "/" << m_core.get_target_blockchain_height() - << timing_message); - } else { - const int completition_percent = (m_core.get_current_blockchain_height() * 100 / m_core.get_target_blockchain_height()); - if(completition_percent < 99) {//printing completion percent only if % is < of 99 cause for 99 >= this is useless - MGINFO_YELLOW(context << " Synced " << m_core.get_current_blockchain_height() << "/" << m_core.get_target_blockchain_height() - << " (" << completition_percent << "% " << (m_core.get_target_blockchain_height() - m_core.get_current_blockchain_height()) - << " blocks remaining)" << timing_message); - } else { - MGINFO_YELLOW(context << " Synced " << m_core.get_current_blockchain_height() << "/" << m_core.get_target_blockchain_height() - << timing_message); - } - } + timing_message += std::string(": ") + m_block_queue.get_overview(current_blockchain_height); + MGINFO_YELLOW("Synced " << current_blockchain_height << "/" << target_blockchain_height + << progress_message << timing_message); + if (previous_stripe != current_stripe) + notify_new_stripe(context, current_stripe); } } } - if (should_download_next_span(context)) + MLOG_PEER_STATE("stopping adding blocks"); + + if (should_download_next_span(context, false)) { - context.m_needed_objects.clear(); - context.m_last_response_height = 0; force_next_span = true; } + else if (should_drop_connection(context, get_next_needed_pruning_stripe().first)) + { + if (!context.m_is_income) + { + m_p2p->add_used_stripe_peer(context); + drop_connection(context, false, false); + } + return 1; + } } skip: @@ -1207,6 +1338,28 @@ skip: } //------------------------------------------------------------------------------------------------------------------------ template<class t_core> + void t_cryptonote_protocol_handler<t_core>::notify_new_stripe(cryptonote_connection_context& cntxt, uint32_t stripe) + { + m_p2p->for_each_connection([&](cryptonote_connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags)->bool + { + if (cntxt.m_connection_id == context.m_connection_id) + return true; + if (context.m_state == cryptonote_connection_context::state_normal) + { + const uint32_t peer_stripe = tools::get_pruning_stripe(context.m_pruning_seed); + if (stripe && peer_stripe && peer_stripe != stripe) + return true; + context.m_state = cryptonote_connection_context::state_synchronizing; + LOG_PRINT_CCONTEXT_L2("requesting callback"); + ++context.m_callback_request_count; + m_p2p->request_callback(context); + MLOG_PEER_STATE("requesting callback"); + } + return true; + }); + } + //------------------------------------------------------------------------------------------------------------------------ + template<class t_core> bool t_cryptonote_protocol_handler<t_core>::on_idle() { m_idle_peer_kicker.do_call(boost::bind(&t_cryptonote_protocol_handler<t_core>::kick_idle_peers, this)); @@ -1218,30 +1371,24 @@ skip: bool t_cryptonote_protocol_handler<t_core>::kick_idle_peers() { MTRACE("Checking for idle peers..."); - std::vector<boost::uuids::uuid> kick_connections; m_p2p->for_each_connection([&](cryptonote_connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags)->bool { - if (context.m_state == cryptonote_connection_context::state_synchronizing || context.m_state == cryptonote_connection_context::state_before_handshake) + if (context.m_state == cryptonote_connection_context::state_synchronizing && context.m_last_request_time != boost::date_time::not_a_date_time) { - const bool passive = context.m_state == cryptonote_connection_context::state_before_handshake; const boost::posix_time::ptime now = boost::posix_time::microsec_clock::universal_time(); const boost::posix_time::time_duration dt = now - context.m_last_request_time; - const int64_t threshold = passive ? PASSIVE_PEER_KICK_TIME : IDLE_PEER_KICK_TIME; - if (dt.total_microseconds() > threshold) + if (dt.total_microseconds() > IDLE_PEER_KICK_TIME) { - MINFO(context << " kicking " << (passive ? "passive" : "idle") << " peer"); - kick_connections.push_back(context.m_connection_id); + MINFO(context << " kicking idle peer, last update " << (dt.total_microseconds() / 1.e6) << " seconds ago"); + LOG_PRINT_CCONTEXT_L2("requesting callback"); + context.m_last_request_time = boost::date_time::not_a_date_time; + context.m_state = cryptonote_connection_context::state_standby; // we'll go back to adding, then (if we can't), download + ++context.m_callback_request_count; + m_p2p->request_callback(context); } } return true; }); - for (const boost::uuids::uuid &conn_id: kick_connections) - { - m_p2p->for_connection(conn_id, [this](cryptonote_connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags) { - drop_connection(context, false, false); - return true; - }); - } return true; } //------------------------------------------------------------------------------------------------------------------------ @@ -1272,75 +1419,174 @@ skip: drop_connection(context, false, false); return 1; } - LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_RESPONSE_CHAIN_ENTRY: m_start_height=" << r.start_height << ", m_total_height=" << r.total_height << ", m_block_ids.size()=" << r.m_block_ids.size()); + MLOG_P2P_MESSAGE("-->>NOTIFY_RESPONSE_CHAIN_ENTRY: m_start_height=" << r.start_height << ", m_total_height=" << r.total_height << ", m_block_ids.size()=" << r.m_block_ids.size()); post_notify<NOTIFY_RESPONSE_CHAIN_ENTRY>(r, context); return 1; } //------------------------------------------------------------------------------------------------------------------------ template<class t_core> - bool t_cryptonote_protocol_handler<t_core>::should_download_next_span(cryptonote_connection_context& context) const + bool t_cryptonote_protocol_handler<t_core>::should_download_next_span(cryptonote_connection_context& context, bool standby) { std::vector<crypto::hash> hashes; boost::uuids::uuid span_connection_id; boost::posix_time::ptime request_time; + boost::uuids::uuid connection_id; std::pair<uint64_t, uint64_t> span; + bool filled; - span = m_block_queue.get_start_gap_span(); - if (span.second > 0) - { - MDEBUG(context << " we should download it as there is a gap"); - return true; - } - - // if the next span is not scheduled (or there is none) - span = m_block_queue.get_next_span_if_scheduled(hashes, span_connection_id, request_time); - if (span.second == 0) + const uint64_t blockchain_height = m_core.get_current_blockchain_height(); + if (context.m_remote_blockchain_height <= blockchain_height) + return false; + const boost::posix_time::ptime now = boost::posix_time::microsec_clock::universal_time(); + const bool has_next_block = tools::has_unpruned_block(blockchain_height, context.m_remote_blockchain_height, context.m_pruning_seed); + if (has_next_block) { - // we might be in a weird case where there is a filled next span, - // but it starts higher than the current height - uint64_t height; - std::vector<cryptonote::block_complete_entry> bcel; - if (!m_block_queue.get_next_span(height, bcel, span_connection_id, true)) - return false; - if (height > m_core.get_current_blockchain_height()) + if (!m_block_queue.has_next_span(blockchain_height, filled, request_time, connection_id)) { - MDEBUG(context << " we should download it as the next block isn't scheduled"); + MDEBUG(context << " we should download it as no peer reserved it"); return true; } + if (!filled) + { + const long dt = (now - request_time).total_microseconds(); + if (dt >= REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD) + { + MDEBUG(context << " we should download it as it's not been received yet after " << dt/1e6); + return true; + } + + // in standby, be ready to double download early since we're idling anyway + // let the fastest peer trigger first + long threshold; + const double dl_speed = context.m_max_speed_down; + if (standby && dt >= REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD_STANDBY && dl_speed > 0) + { + bool download = false; + if (m_p2p->for_connection(connection_id, [&](cryptonote_connection_context& ctx, nodetool::peerid_type peer_id, uint32_t f)->bool{ + const time_t nowt = time(NULL); + const time_t time_since_last_recv = nowt - ctx.m_last_recv; + const float last_activity = std::min((float)time_since_last_recv, dt/1e6f); + const bool stalled = last_activity > LAST_ACTIVITY_STALL_THRESHOLD; + if (stalled) + { + MDEBUG(context << " we should download it as the downloading peer is stalling for " << nowt - ctx.m_last_recv << " seconds"); + download = true; + return true; + } + + // estimate the standby peer can give us 80% of its max speed + // and let it download if that speed is > N times as fast as the current one + // N starts at 10 after REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD_STANDBY, + // decreases to 1.25 at REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD, + // so that at times goes without the download being done, a retry becomes easier + const float max_multiplier = 10.f; + const float min_multiplier = 1.25f; + float multiplier = max_multiplier; + if (dt/1e6 >= REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD_STANDBY) + { + multiplier = max_multiplier - (dt/1e6-REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD_STANDBY) * (max_multiplier - min_multiplier) / (REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD - REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD_STANDBY); + multiplier = std::min(max_multiplier, std::max(min_multiplier, multiplier)); + } + if (dl_speed * .8f > ctx.m_current_speed_down * multiplier) + { + MDEBUG(context << " we should download it as we are substantially faster (" << dl_speed << " vs " + << ctx.m_current_speed_down << ", multiplier " << multiplier << " after " << dt/1e6 << " seconds)"); + download = true; + return true; + } + return true; + })) + { + if (download) + return true; + } + else + { + MWARNING(context << " we should download it as the downloading peer is unexpectedly not known to us"); + return true; + } + } + } + } + + return false; + } + //------------------------------------------------------------------------------------------------------------------------ + template<class t_core> + bool t_cryptonote_protocol_handler<t_core>::should_drop_connection(cryptonote_connection_context& context, uint32_t next_stripe) + { + if (context.m_anchor) + { + MDEBUG(context << "This is an anchor peer, not dropping"); return false; } - // if it was scheduled by this particular peer - if (span_connection_id == context.m_connection_id) + if (context.m_pruning_seed == 0) + { + MDEBUG(context << "This peer is not striped, not dropping"); return false; + } - float span_speed = m_block_queue.get_speed(span_connection_id); - float speed = m_block_queue.get_speed(context.m_connection_id); - MDEBUG(context << " next span is scheduled for " << span_connection_id << ", speed " << span_speed << ", ours " << speed); - - // we try for that span too if: - // - we're substantially faster, or: - // - we're the fastest and the other one isn't (avoids a peer being waaaay slow but yet unmeasured) - // - the other one asked at least 5 seconds ago - if (span_speed < .25 && speed > .75f) + const uint32_t peer_stripe = tools::get_pruning_stripe(context.m_pruning_seed); + if (next_stripe == peer_stripe) { - MDEBUG(context << " we should download it as we're substantially faster"); - return true; + MDEBUG(context << "This peer has needed stripe " << peer_stripe << ", not dropping"); + return false; } - if (speed == 1.0f && span_speed != 1.0f) + + if (!context.m_needed_objects.empty()) { - MDEBUG(context << " we should download it as we're the fastest peer"); - return true; + const uint64_t next_available_block_height = context.m_last_response_height - context.m_needed_objects.size() + 1; + if (tools::has_unpruned_block(next_available_block_height, context.m_remote_blockchain_height, context.m_pruning_seed)) + { + MDEBUG(context << "This peer has unpruned next block at height " << next_available_block_height << ", not dropping"); + return false; + } } - const boost::posix_time::ptime now = boost::posix_time::microsec_clock::universal_time(); - if ((now - request_time).total_microseconds() > REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD) + + if (next_stripe > 0) { - MDEBUG(context << " we should download it as this span was requested long ago"); - return true; + unsigned int n_out_peers = 0, n_peers_on_next_stripe = 0; + m_p2p->for_each_connection([&](cryptonote_connection_context& ctx, nodetool::peerid_type peer_id, uint32_t support_flags)->bool{ + if (!ctx.m_is_income) + ++n_out_peers; + if (ctx.m_state >= cryptonote_connection_context::state_synchronizing && tools::get_pruning_stripe(ctx.m_pruning_seed) == next_stripe) + ++n_peers_on_next_stripe; + return true; + }); + const uint32_t distance = (peer_stripe + (1<<CRYPTONOTE_PRUNING_LOG_STRIPES) - next_stripe) % (1<<CRYPTONOTE_PRUNING_LOG_STRIPES); + if ((n_out_peers >= m_max_out_peers && n_peers_on_next_stripe == 0) || (distance > 1 && n_peers_on_next_stripe <= 2) || distance > 2) + { + MDEBUG(context << "we want seed " << next_stripe << ", and either " << n_out_peers << " is at max out peers (" + << m_max_out_peers << ") or distance " << distance << " from " << next_stripe << " to " << peer_stripe << + " is too large and we have only " << n_peers_on_next_stripe << " peers on next seed, dropping connection to make space"); + return true; + } } + MDEBUG(context << "End of checks, not dropping"); return false; } //------------------------------------------------------------------------------------------------------------------------ template<class t_core> + void t_cryptonote_protocol_handler<t_core>::skip_unneeded_hashes(cryptonote_connection_context& context, bool check_block_queue) const + { + // take out blocks we already have + size_t skip = 0; + while (skip < context.m_needed_objects.size() && (m_core.have_block(context.m_needed_objects[skip]) || (check_block_queue && m_block_queue.have(context.m_needed_objects[skip])))) + { + // if we're popping the last hash, record it so we can ask again from that hash, + // this prevents never being able to progress on peers we get old hash lists from + if (skip + 1 == context.m_needed_objects.size()) + context.m_last_known_hash = context.m_needed_objects[skip]; + ++skip; + } + if (skip > 0) + { + MDEBUG(context << "skipping " << skip << "/" << context.m_needed_objects.size() << " blocks"); + context.m_needed_objects = std::vector<crypto::hash>(context.m_needed_objects.begin() + skip, context.m_needed_objects.end()); + } + } + //------------------------------------------------------------------------------------------------------------------------ + template<class t_core> bool t_cryptonote_protocol_handler<t_core>::request_missing_objects(cryptonote_connection_context& context, bool check_having_blocks, bool force_next_span) { // flush stale spans @@ -1357,47 +1603,102 @@ skip: { do { - size_t nblocks = m_block_queue.get_num_filled_spans(); + size_t nspans = m_block_queue.get_num_filled_spans(); size_t size = m_block_queue.get_data_size(); - if (nblocks < BLOCK_QUEUE_NBLOCKS_THRESHOLD || size < BLOCK_QUEUE_SIZE_THRESHOLD) + const uint64_t bc_height = m_core.get_current_blockchain_height(); + const auto next_needed_pruning_stripe = get_next_needed_pruning_stripe(); + const uint32_t add_stripe = tools::get_pruning_stripe(bc_height, context.m_remote_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES); + const uint32_t peer_stripe = tools::get_pruning_stripe(context.m_pruning_seed); + const size_t block_queue_size_threshold = m_block_download_max_size ? m_block_download_max_size : BLOCK_QUEUE_SIZE_THRESHOLD; + bool queue_proceed = nspans < BLOCK_QUEUE_NSPANS_THRESHOLD || size < block_queue_size_threshold; + // get rid of blocks we already requested, or already have + skip_unneeded_hashes(context, true); + uint64_t next_needed_height = m_block_queue.get_next_needed_height(bc_height); + uint64_t next_block_height; + if (context.m_needed_objects.empty()) + next_block_height = next_needed_height; + else + next_block_height = context.m_last_response_height - context.m_needed_objects.size() + 1; + bool stripe_proceed_main = (add_stripe == 0 || peer_stripe == 0 || add_stripe == peer_stripe) && (next_block_height < bc_height + BLOCK_QUEUE_FORCE_DOWNLOAD_NEAR_BLOCKS || next_needed_height < bc_height + BLOCK_QUEUE_FORCE_DOWNLOAD_NEAR_BLOCKS); + bool stripe_proceed_secondary = tools::has_unpruned_block(next_block_height, context.m_remote_blockchain_height, context.m_pruning_seed); + bool proceed = stripe_proceed_main || (queue_proceed && stripe_proceed_secondary); + if (!stripe_proceed_main && !stripe_proceed_secondary && should_drop_connection(context, tools::get_pruning_stripe(next_block_height, context.m_remote_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES))) + { + if (!context.m_is_income) + m_p2p->add_used_stripe_peer(context); + return false; // drop outgoing connections + } + + MDEBUG(context << "proceed " << proceed << " (queue " << queue_proceed << ", stripe " << stripe_proceed_main << "/" << + stripe_proceed_secondary << "), " << next_needed_pruning_stripe.first << "-" << next_needed_pruning_stripe.second << + " needed, bc add stripe " << add_stripe << ", we have " << peer_stripe << "), bc_height " << bc_height); + MDEBUG(context << " - next_block_height " << next_block_height << ", seed " << epee::string_tools::to_string_hex(context.m_pruning_seed) << + ", next_needed_height "<< next_needed_height); + MDEBUG(context << " - last_response_height " << context.m_last_response_height << ", m_needed_objects size " << context.m_needed_objects.size()); + + // if we're waiting for next span, try to get it before unblocking threads below, + // or a runaway downloading of future spans might happen + if (stripe_proceed_main && should_download_next_span(context, true)) + { + MDEBUG(context << " we should try for that next span too, we think we could get it faster, resuming"); + force_next_span = true; + MLOG_PEER_STATE("resuming"); + break; + } + + if (proceed) { if (context.m_state != cryptonote_connection_context::state_standby) { - LOG_DEBUG_CC(context, "Block queue is " << nblocks << " and " << size << ", resuming"); + LOG_DEBUG_CC(context, "Block queue is " << nspans << " and " << size << ", resuming"); + MLOG_PEER_STATE("resuming"); } break; } // this one triggers if all threads are in standby, which should not happen, // but happened at least once, so we unblock at least one thread if so - const boost::unique_lock<boost::mutex> sync{m_sync_lock, boost::try_to_lock}; + boost::unique_lock<boost::mutex> sync{m_sync_lock, boost::try_to_lock}; if (sync.owns_lock()) { - LOG_DEBUG_CC(context, "No other thread is adding blocks, resuming"); - break; - } + bool filled = false; + boost::posix_time::ptime time; + boost::uuids::uuid connection_id; + if (m_block_queue.has_next_span(m_core.get_current_blockchain_height(), filled, time, connection_id) && filled) + { + LOG_DEBUG_CC(context, "No other thread is adding blocks, and next span needed is ready, resuming"); + MLOG_PEER_STATE("resuming"); + context.m_state = cryptonote_connection_context::state_standby; + ++context.m_callback_request_count; + m_p2p->request_callback(context); + return true; + } + else + { + sync.unlock(); - if (should_download_next_span(context)) - { - MDEBUG(context << " we should try for that next span too, we think we could get it faster, resuming"); - force_next_span = true; - break; + // if this has gone on for too long, drop incoming connection to guard against some wedge state + if (!context.m_is_income) + { + const uint64_t now = tools::get_tick_count(); + const uint64_t dt = now - m_last_add_end_time; + if (tools::ticks_to_ns(dt) >= DROP_ON_SYNC_WEDGE_THRESHOLD) + { + MDEBUG(context << "Block addition seems to have wedged, dropping connection"); + return false; + } + } + } } if (context.m_state != cryptonote_connection_context::state_standby) { - LOG_DEBUG_CC(context, "Block queue is " << nblocks << " and " << size << ", pausing"); + if (!queue_proceed) + LOG_DEBUG_CC(context, "Block queue is " << nspans << " and " << size << ", pausing"); + else if (!stripe_proceed_main && !stripe_proceed_secondary) + LOG_DEBUG_CC(context, "We do not have the stripe required to download another block, pausing"); context.m_state = cryptonote_connection_context::state_standby; - } - - // this needs doing after we went to standby, so the callback knows what to do - bool filled; - if (m_block_queue.has_next_span(context.m_connection_id, filled) && !filled) - { - MDEBUG(context << " we have the next span, and it is scheduled, resuming"); - ++context.m_callback_request_count; - m_p2p->request_callback(context); - return true; + MLOG_PEER_STATE("pausing"); } return true; @@ -1405,7 +1706,9 @@ skip: context.m_state = cryptonote_connection_context::state_synchronizing; } - MDEBUG(context << " request_missing_objects: check " << check_having_blocks << ", force_next_span " << force_next_span << ", m_needed_objects " << context.m_needed_objects.size() << " lrh " << context.m_last_response_height << ", chain " << m_core.get_current_blockchain_height()); + MDEBUG(context << " request_missing_objects: check " << check_having_blocks << ", force_next_span " << force_next_span + << ", m_needed_objects " << context.m_needed_objects.size() << " lrh " << context.m_last_response_height << ", chain " + << m_core.get_current_blockchain_height() << ", pruning seed " << epee::string_tools::to_string_hex(context.m_pruning_seed)); if(context.m_needed_objects.size() || force_next_span) { //we know objects that we need, request this objects @@ -1414,28 +1717,6 @@ skip: size_t count = 0; const size_t count_limit = m_core.get_block_sync_size(m_core.get_current_blockchain_height()); std::pair<uint64_t, uint64_t> span = std::make_pair(0, 0); - { - MDEBUG(context << " checking for gap"); - span = m_block_queue.get_start_gap_span(); - if (span.second > 0) - { - const uint64_t first_block_height_known = context.m_last_response_height - context.m_needed_objects.size() + 1; - const uint64_t last_block_height_known = context.m_last_response_height; - const uint64_t first_block_height_needed = span.first; - const uint64_t last_block_height_needed = span.first + std::min(span.second, (uint64_t)count_limit) - 1; - MDEBUG(context << " gap found, span: " << span.first << " - " << span.first + span.second - 1 << " (" << last_block_height_needed << ")"); - MDEBUG(context << " current known hashes from " << first_block_height_known << " to " << last_block_height_known); - if (first_block_height_needed < first_block_height_known || last_block_height_needed > last_block_height_known) - { - MDEBUG(context << " we are missing some of the necessary hashes for this gap, requesting chain again"); - context.m_needed_objects.clear(); - context.m_last_response_height = 0; - start_from_current_chain = true; - goto skip; - } - MDEBUG(context << " we have the hashes for this gap"); - } - } if (force_next_span) { if (span.second == 0) @@ -1452,6 +1733,7 @@ skip: req.blocks.push_back(hash); context.m_requested_objects.insert(hash); } + m_block_queue.reset_next_span_time(); } } } @@ -1465,22 +1747,20 @@ skip: context.m_last_response_height = 0; goto skip; } - // take out blocks we already have - size_t skip = 0; - while (skip < context.m_needed_objects.size() && m_core.have_block(context.m_needed_objects[skip])) - { - // if we're popping the last hash, record it so we can ask again from that hash, - // this prevents never being able to progress on peers we get old hash lists from - if (skip + 1 == context.m_needed_objects.size()) - context.m_last_known_hash = context.m_needed_objects[skip]; - ++skip; - } - if (skip > 0) - context.m_needed_objects = std::vector<crypto::hash>(context.m_needed_objects.begin() + skip, context.m_needed_objects.end()); + skip_unneeded_hashes(context, false); const uint64_t first_block_height = context.m_last_response_height - context.m_needed_objects.size() + 1; - span = m_block_queue.reserve_span(first_block_height, context.m_last_response_height, count_limit, context.m_connection_id, context.m_needed_objects); + span = m_block_queue.reserve_span(first_block_height, context.m_last_response_height, count_limit, context.m_connection_id, context.m_pruning_seed, context.m_remote_blockchain_height, context.m_needed_objects); MDEBUG(context << " span from " << first_block_height << ": " << span.first << "/" << span.second); + if (span.second > 0) + { + const uint32_t stripe = tools::get_pruning_stripe(span.first, context.m_remote_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES); + if (context.m_pruning_seed && stripe != tools::get_pruning_stripe(context.m_pruning_seed)) + { + MDEBUG(context << " starting early on next seed (" << span.first << " with stripe " << stripe << + ", context seed " << epee::string_tools::to_string_hex(context.m_pruning_seed) << ")"); + } + } } if (span.second == 0 && !force_next_span) { @@ -1489,6 +1769,8 @@ skip: boost::uuids::uuid span_connection_id; boost::posix_time::ptime time; span = m_block_queue.get_next_span_if_scheduled(hashes, span_connection_id, time); + if (span.second > 0 && !tools::has_unpruned_block(span.first, context.m_remote_blockchain_height, context.m_pruning_seed)) + span = std::make_pair(0, 0); if (span.second > 0) { is_next = true; @@ -1534,17 +1816,67 @@ skip: } context.m_last_request_time = boost::posix_time::microsec_clock::universal_time(); - LOG_PRINT_CCONTEXT_L1("-->>NOTIFY_REQUEST_GET_OBJECTS: blocks.size()=" << req.blocks.size() << ", txs.size()=" << req.txs.size() + MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_GET_OBJECTS: blocks.size()=" << req.blocks.size() << ", txs.size()=" << req.txs.size() << "requested blocks count=" << count << " / " << count_limit << " from " << span.first << ", first hash " << req.blocks.front()); //epee::net_utils::network_throttle_manager::get_global_throttle_inreq().logger_handle_net("log/dr-monero/net/req-all.data", sec, get_avg_block_size()); post_notify<NOTIFY_REQUEST_GET_OBJECTS>(req, context); + MLOG_PEER_STATE("requesting objects"); + return true; + } + + // if we're still around, we might be at a point where the peer is pruned, so we could either + // drop it to make space for other peers, or ask for a span further down the line + const uint32_t next_stripe = get_next_needed_pruning_stripe().first; + const uint32_t peer_stripe = tools::get_pruning_stripe(context.m_pruning_seed); + if (next_stripe && peer_stripe && next_stripe != peer_stripe) + { + // at this point, we have to either close the connection, or start getting blocks past the + // current point, or become dormant + MDEBUG(context << "this peer is pruned at seed " << epee::string_tools::to_string_hex(context.m_pruning_seed) << + ", next stripe needed is " << next_stripe); + if (!context.m_is_income) + { + if (should_drop_connection(context, next_stripe)) + { + m_p2p->add_used_stripe_peer(context); + return false; // drop outgoing connections + } + } + // we'll get back stuck waiting for the go ahead + context.m_state = cryptonote_connection_context::state_normal; + MLOG_PEER_STATE("Nothing to do for now, switching to normal state"); return true; } } skip: context.m_needed_objects.clear(); + + // we might have been called from the "received chain entry" handler, and end up + // here because we can't use any of those blocks (maybe because all of them are + // actually already requested). In this case, if we can add blocks instead, do so + if (m_core.get_current_blockchain_height() < m_core.get_target_blockchain_height()) + { + const boost::unique_lock<boost::mutex> sync{m_sync_lock, boost::try_to_lock}; + if (sync.owns_lock()) + { + uint64_t start_height; + std::vector<cryptonote::block_complete_entry> blocks; + boost::uuids::uuid span_connection_id; + bool filled = false; + if (m_block_queue.get_next_span(start_height, blocks, span_connection_id, filled) && filled) + { + LOG_DEBUG_CC(context, "No other thread is adding blocks, resuming"); + MLOG_PEER_STATE("will try to add blocks next"); + context.m_state = cryptonote_connection_context::state_standby; + ++context.m_callback_request_count; + m_p2p->request_callback(context); + return true; + } + } + } + if(context.m_last_response_height < context.m_remote_blockchain_height-1) {//we have to fetch more objects ids, request blockchain entry @@ -1567,8 +1899,9 @@ skip: //LOG_PRINT_CCONTEXT_L1("r = " << 200); context.m_last_request_time = boost::posix_time::microsec_clock::universal_time(); - LOG_PRINT_CCONTEXT_L1("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() << ", start_from_current_chain " << start_from_current_chain); + MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() << ", start_from_current_chain " << start_from_current_chain); post_notify<NOTIFY_REQUEST_CHAIN>(r, context); + MLOG_PEER_STATE("requesting chain"); }else { CHECK_AND_ASSERT_MES(context.m_last_response_height == context.m_remote_blockchain_height-1 @@ -1608,9 +1941,25 @@ skip: << ENDL << "Use the \"help\" command to see the list of available commands." << ENDL << "**********************************************************************"); + m_sync_timer.pause(); + if (ELPP->vRegistry()->allowed(el::Level::Info, "sync-info")) + { + const uint64_t sync_time = m_sync_timer.value(); + const uint64_t add_time = m_add_timer.value(); + if (sync_time && add_time) + { + MCLOG_YELLOW(el::Level::Info, "sync-info", "Sync time: " << sync_time/1e9/60 << " min, idle time " << + (100.f * (1.0f - add_time / (float)sync_time)) << "%" << ", " << + (10 * m_sync_download_objects_size / 1024 / 1024) / 10.f << " + " << + (10 * m_sync_download_chain_size / 1024 / 1024) / 10.f << " MB downloaded, " << + 100.0f * m_sync_old_spans_downloaded / m_sync_spans_downloaded << "% old spans, " << + 100.0f * m_sync_bad_spans_downloaded / m_sync_spans_downloaded << "% bad spans"); + } + } m_core.on_synchronized(); } m_core.safesyncmode(true); + m_p2p->clear_used_stripe_peers(); return true; } //------------------------------------------------------------------------------------------------------------------------ @@ -1631,6 +1980,11 @@ skip: { MLOG_P2P_MESSAGE("Received NOTIFY_RESPONSE_CHAIN_ENTRY: m_block_ids.size()=" << arg.m_block_ids.size() << ", m_start_height=" << arg.start_height << ", m_total_height=" << arg.total_height); + MLOG_PEER_STATE("received chain"); + + context.m_last_request_time = boost::date_time::not_a_date_time; + + m_sync_download_chain_size += arg.m_block_ids.size() * sizeof(crypto::hash); if(!arg.m_block_ids.size()) { @@ -1644,7 +1998,14 @@ skip: drop_connection(context, true, false); return 1; } + MDEBUG(context << "first block hash " << arg.m_block_ids.front() << ", last " << arg.m_block_ids.back()); + if (arg.total_height >= CRYPTONOTE_MAX_BLOCK_NUMBER || arg.m_block_ids.size() >= CRYPTONOTE_MAX_BLOCK_NUMBER) + { + LOG_ERROR_CCONTEXT("sent wrong NOTIFY_RESPONSE_CHAIN_ENTRY, with total_height=" << arg.total_height << " and block_ids=" << arg.m_block_ids.size()); + drop_connection(context, false, false); + return 1; + } context.m_remote_blockchain_height = arg.total_height; context.m_last_response_height = arg.start_height + arg.m_block_ids.size()-1; if(context.m_last_response_height > context.m_remote_blockchain_height) @@ -1664,6 +2025,7 @@ skip: return 1; } + context.m_needed_objects.clear(); uint64_t added = 0; for(auto& bl_id: arg.m_block_ids) { @@ -1720,13 +2082,13 @@ skip: { std::string fluffyBlob; epee::serialization::store_t_to_binary(fluffy_arg, fluffyBlob); - m_p2p->relay_notify_to_list(NOTIFY_NEW_FLUFFY_BLOCK::ID, fluffyBlob, fluffyConnections); + m_p2p->relay_notify_to_list(NOTIFY_NEW_FLUFFY_BLOCK::ID, epee::strspan<uint8_t>(fluffyBlob), fluffyConnections); } if (!fullConnections.empty()) { std::string fullBlob; epee::serialization::store_t_to_binary(arg, fullBlob); - m_p2p->relay_notify_to_list(NOTIFY_NEW_BLOCK::ID, fullBlob, fullConnections); + m_p2p->relay_notify_to_list(NOTIFY_NEW_BLOCK::ID, epee::strspan<uint8_t>(fullBlob), fullConnections); } return true; @@ -1773,10 +2135,85 @@ skip: } //------------------------------------------------------------------------------------------------------------------------ template<class t_core> + std::string t_cryptonote_protocol_handler<t_core>::get_peers_overview() const + { + std::stringstream ss; + const boost::posix_time::ptime now = boost::posix_time::microsec_clock::universal_time(); + m_p2p->for_each_connection([&](const connection_context &ctx, nodetool::peerid_type peer_id, uint32_t support_flags) { + const uint32_t stripe = tools::get_pruning_stripe(ctx.m_pruning_seed); + char state_char = cryptonote::get_protocol_state_char(ctx.m_state); + ss << stripe + state_char; + if (ctx.m_last_request_time != boost::date_time::not_a_date_time) + ss << (((now - ctx.m_last_request_time).total_microseconds() > IDLE_PEER_KICK_TIME) ? "!" : "?"); + ss << + " "; + return true; + }); + return ss.str(); + } + //------------------------------------------------------------------------------------------------------------------------ + template<class t_core> + std::pair<uint32_t, uint32_t> t_cryptonote_protocol_handler<t_core>::get_next_needed_pruning_stripe() const + { + const uint64_t want_height_from_blockchain = m_core.get_current_blockchain_height(); + const uint64_t want_height_from_block_queue = m_block_queue.get_next_needed_height(want_height_from_blockchain); + const uint64_t want_height = std::max(want_height_from_blockchain, want_height_from_block_queue); + uint64_t blockchain_height = m_core.get_target_blockchain_height(); + // if we don't know the remote chain size yet, assume infinitely large so we get the right stripe if we're not near the tip + if (blockchain_height == 0) + blockchain_height = CRYPTONOTE_MAX_BLOCK_NUMBER; + const uint32_t next_pruning_stripe = tools::get_pruning_stripe(want_height, blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES); + if (next_pruning_stripe == 0) + return std::make_pair(0, 0); + // if we already have a few peers on this stripe, but none on next one, try next one + unsigned int n_next = 0, n_subsequent = 0, n_others = 0; + const uint32_t subsequent_pruning_stripe = 1 + next_pruning_stripe % (1<<CRYPTONOTE_PRUNING_LOG_STRIPES); + m_p2p->for_each_connection([&](const connection_context &context, nodetool::peerid_type peer_id, uint32_t support_flags) { + if (context.m_state >= cryptonote_connection_context::state_synchronizing) + { + if (context.m_pruning_seed == 0 || tools::get_pruning_stripe(context.m_pruning_seed) == next_pruning_stripe) + ++n_next; + else if (tools::get_pruning_stripe(context.m_pruning_seed) == subsequent_pruning_stripe) + ++n_subsequent; + else + ++n_others; + } + return true; + }); + const bool use_next = (n_next > m_max_out_peers / 2 && n_subsequent <= 1) || (n_next > 2 && n_subsequent == 0); + const uint32_t ret_stripe = use_next ? subsequent_pruning_stripe: next_pruning_stripe; + MIDEBUG(const std::string po = get_peers_overview(), "get_next_needed_pruning_stripe: want height " << want_height << " (" << + want_height_from_blockchain << " from blockchain, " << want_height_from_block_queue << " from block queue), stripe " << + next_pruning_stripe << " (" << n_next << "/" << m_max_out_peers << " on it and " << n_subsequent << " on " << + subsequent_pruning_stripe << ", " << n_others << " others) -> " << ret_stripe << " (+" << + (ret_stripe - next_pruning_stripe + (1 << CRYPTONOTE_PRUNING_LOG_STRIPES)) % (1 << CRYPTONOTE_PRUNING_LOG_STRIPES) << + "), current peers " << po); + return std::make_pair(next_pruning_stripe, ret_stripe); + } + //------------------------------------------------------------------------------------------------------------------------ + template<class t_core> + bool t_cryptonote_protocol_handler<t_core>::needs_new_sync_connections() const + { + const uint64_t target = m_core.get_target_blockchain_height(); + const uint64_t height = m_core.get_current_blockchain_height(); + if (target && target <= height) + return false; + size_t n_out_peers = 0; + m_p2p->for_each_connection([&](cryptonote_connection_context& ctx, nodetool::peerid_type peer_id, uint32_t support_flags)->bool{ + if (!ctx.m_is_income) + ++n_out_peers; + return true; + }); + if (n_out_peers >= m_max_out_peers) + return false; + return true; + } + //------------------------------------------------------------------------------------------------------------------------ + template<class t_core> void t_cryptonote_protocol_handler<t_core>::drop_connection(cryptonote_connection_context &context, bool add_fail, bool flush_all_spans) { - LOG_DEBUG_CC(context, "dropping connection id " << context.m_connection_id << - ", add_fail " << add_fail << ", flush_all_spans " << flush_all_spans); + LOG_DEBUG_CC(context, "dropping connection id " << context.m_connection_id << " (pruning seed " << + epee::string_tools::to_string_hex(context.m_pruning_seed) << + "), add_fail " << add_fail << ", flush_all_spans " << flush_all_spans); if (add_fail) m_p2p->add_host_fail(context.m_remote_address); @@ -1803,6 +2240,7 @@ skip: } m_block_queue.flush_spans(context.m_connection_id, false); + MLOG_PEER_STATE("closed"); } //------------------------------------------------------------------------------------------------------------------------ diff --git a/src/daemon/command_parser_executor.cpp b/src/daemon/command_parser_executor.cpp index 853cde9c3..37cb55ec8 100644 --- a/src/daemon/command_parser_executor.cpp +++ b/src/daemon/command_parser_executor.cpp @@ -163,9 +163,21 @@ bool t_command_parser_executor::print_height(const std::vector<std::string>& arg bool t_command_parser_executor::print_block(const std::vector<std::string>& args) { + bool include_hex = false; + + // Assumes that optional flags come after mandatory argument <transaction_hash> + for (unsigned int i = 1; i < args.size(); ++i) { + if (args[i] == "+hex") + include_hex = true; + else + { + std::cout << "unexpected argument: " << args[i] << std::endl; + return true; + } + } if (args.empty()) { - std::cout << "expected: print_block (<block_hash> | <block_height>)" << std::endl; + std::cout << "expected: print_block (<block_hash> | <block_height>) [+hex]" << std::endl; return false; } @@ -173,14 +185,14 @@ bool t_command_parser_executor::print_block(const std::vector<std::string>& args try { uint64_t height = boost::lexical_cast<uint64_t>(arg); - return m_executor.print_block_by_height(height); + return m_executor.print_block_by_height(height, include_hex); } catch (const boost::bad_lexical_cast&) { crypto::hash block_hash; if (parse_hash256(arg, block_hash)) { - return m_executor.print_block_by_hash(block_hash); + return m_executor.print_block_by_hash(block_hash, include_hex); } } @@ -705,4 +717,27 @@ bool t_command_parser_executor::version(const std::vector<std::string>& args) return true; } +bool t_command_parser_executor::prune_blockchain(const std::vector<std::string>& args) +{ + if (args.size() > 1) return false; + + if (args.empty() || args[0] != "confirm") + { + std::cout << "Warning: pruning from within monerod will not shrink the database file size." << std::endl; + std::cout << "Instead, parts of the file will be marked as free, so the file will not grow" << std::endl; + std::cout << "until that newly free space is used up. If you want a smaller file size now," << std::endl; + std::cout << "exit monerod and run monero-blockchain-prune (you will temporarily need more" << std::endl; + std::cout << "disk space for the database conversion though). If you are OK with the database" << std::endl; + std::cout << "file keeping the same size, re-run this command with the \"confirm\" parameter." << std::endl; + return true; + } + + return m_executor.prune_blockchain(); +} + +bool t_command_parser_executor::check_blockchain_pruning(const std::vector<std::string>& args) +{ + return m_executor.check_blockchain_pruning(); +} + } // namespace daemonize diff --git a/src/daemon/command_parser_executor.h b/src/daemon/command_parser_executor.h index e2844e8b7..5b8927908 100644 --- a/src/daemon/command_parser_executor.h +++ b/src/daemon/command_parser_executor.h @@ -142,6 +142,10 @@ public: bool pop_blocks(const std::vector<std::string>& args); bool version(const std::vector<std::string>& args); + + bool prune_blockchain(const std::vector<std::string>& args); + + bool check_blockchain_pruning(const std::vector<std::string>& args); }; } // namespace daemonize diff --git a/src/daemon/command_server.cpp b/src/daemon/command_server.cpp index 527ed2cf1..786d1a601 100644 --- a/src/daemon/command_server.cpp +++ b/src/daemon/command_server.cpp @@ -292,6 +292,16 @@ t_command_server::t_command_server( , std::bind(&t_command_parser_executor::version, &m_parser, p::_1) , "Print version information." ); + m_command_lookup.set_handler( + "prune_blockchain" + , std::bind(&t_command_parser_executor::prune_blockchain, &m_parser, p::_1) + , "Prune the blockchain." + ); + m_command_lookup.set_handler( + "check_blockchain_pruning" + , std::bind(&t_command_parser_executor::check_blockchain_pruning, &m_parser, p::_1) + , "Check the blockchain pruning." + ); } bool t_command_server::process_command_str(const std::string& cmd) diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp index 015e1e1f9..7cd61934f 100644 --- a/src/daemon/rpc_command_executor.cpp +++ b/src/daemon/rpc_command_executor.cpp @@ -31,6 +31,7 @@ #include "string_tools.h" #include "common/password.h" #include "common/scoped_message_writer.h" +#include "common/pruning.h" #include "daemon/rpc_command_executor.h" #include "rpc/core_rpc_server_commands_defs.h" #include "cryptonote_core/cryptonote_core.h" @@ -60,13 +61,14 @@ namespace { peer_id_str >> id_str; epee::string_tools::xtype_to_string(peer.port, port_str); std::string addr_str = ip_str + ":" + port_str; - tools::msg_writer() << boost::format("%-10s %-25s %-25s %s") % prefix % id_str % addr_str % elapsed; + std::string pruning_seed = epee::string_tools::to_string_hex(peer.pruning_seed); + tools::msg_writer() << boost::format("%-10s %-25s %-25s %-4s %s") % prefix % id_str % addr_str % pruning_seed % elapsed; } void print_block_header(cryptonote::block_header_response const & header) { tools::success_msg_writer() - << "timestamp: " << boost::lexical_cast<std::string>(header.timestamp) << std::endl + << "timestamp: " << boost::lexical_cast<std::string>(header.timestamp) << " (" << tools::get_human_readable_timestamp(header.timestamp) << ")" << std::endl << "previous hash: " << header.prev_hash << std::endl << "nonce: " << boost::lexical_cast<std::string>(header.nonce) << std::endl << "is orphan: " << header.orphan_status << std::endl @@ -569,7 +571,7 @@ bool t_rpc_command_executor::print_blockchain_info(uint64_t start_block_index, u if (!first) tools::msg_writer() << "" << std::endl; tools::msg_writer() - << "height: " << header.height << ", timestamp: " << header.timestamp + << "height: " << header.height << ", timestamp: " << header.timestamp << " (" << tools::get_human_readable_timestamp(header.timestamp) << ")" << ", size: " << header.block_size << ", weight: " << header.block_weight << ", transactions: " << header.num_txes << std::endl << "major version: " << (unsigned)header.major_version << ", minor version: " << (unsigned)header.minor_version << std::endl << "block id: " << header.hash << ", previous block id: " << header.prev_hash << std::endl @@ -663,7 +665,7 @@ bool t_rpc_command_executor::print_height() { return true; } -bool t_rpc_command_executor::print_block_by_hash(crypto::hash block_hash) { +bool t_rpc_command_executor::print_block_by_hash(crypto::hash block_hash, bool include_hex) { cryptonote::COMMAND_RPC_GET_BLOCK::request req; cryptonote::COMMAND_RPC_GET_BLOCK::response res; epee::json_rpc::error error_resp; @@ -689,13 +691,15 @@ bool t_rpc_command_executor::print_block_by_hash(crypto::hash block_hash) { } } + if (include_hex) + tools::success_msg_writer() << res.blob << std::endl; print_block_header(res.block_header); tools::success_msg_writer() << res.json << ENDL; return true; } -bool t_rpc_command_executor::print_block_by_height(uint64_t height) { +bool t_rpc_command_executor::print_block_by_height(uint64_t height, bool include_hex) { cryptonote::COMMAND_RPC_GET_BLOCK::request req; cryptonote::COMMAND_RPC_GET_BLOCK::response res; epee::json_rpc::error error_resp; @@ -721,6 +725,8 @@ bool t_rpc_command_executor::print_block_by_height(uint64_t height) { } } + if (include_hex) + tools::success_msg_writer() << res.blob << std::endl; print_block_header(res.block_header); tools::success_msg_writer() << res.json << ENDL; @@ -737,6 +743,7 @@ bool t_rpc_command_executor::print_transaction(crypto::hash transaction_hash, req.txs_hashes.push_back(epee::string_tools::pod_to_hex(transaction_hash)); req.decode_as_json = false; + req.split = true; req.prune = false; if (m_is_rpc) { @@ -762,13 +769,25 @@ bool t_rpc_command_executor::print_transaction(crypto::hash transaction_hash, if (res.txs.front().in_pool) tools::success_msg_writer() << "Found in pool"; else - tools::success_msg_writer() << "Found in blockchain at height " << res.txs.front().block_height; + tools::success_msg_writer() << "Found in blockchain at height " << res.txs.front().block_height << (res.txs.front().prunable_as_hex.empty() ? " (pruned)" : ""); } const std::string &as_hex = (1 == res.txs.size()) ? res.txs.front().as_hex : res.txs_as_hex.front(); + const std::string &pruned_as_hex = (1 == res.txs.size()) ? res.txs.front().pruned_as_hex : ""; + const std::string &prunable_as_hex = (1 == res.txs.size()) ? res.txs.front().prunable_as_hex : ""; // Print raw hex if requested if (include_hex) - tools::success_msg_writer() << as_hex << std::endl; + { + if (!as_hex.empty()) + { + tools::success_msg_writer() << as_hex << std::endl; + } + else + { + std::string output = pruned_as_hex + prunable_as_hex; + tools::success_msg_writer() << output << std::endl; + } + } // Print json if requested if (include_json) @@ -776,17 +795,27 @@ bool t_rpc_command_executor::print_transaction(crypto::hash transaction_hash, crypto::hash tx_hash, tx_prefix_hash; cryptonote::transaction tx; cryptonote::blobdata blob; - if (!string_tools::parse_hexstr_to_binbuff(as_hex, blob)) + std::string source = as_hex.empty() ? pruned_as_hex + prunable_as_hex : as_hex; + bool pruned = !pruned_as_hex.empty() && prunable_as_hex.empty(); + if (!string_tools::parse_hexstr_to_binbuff(source, blob)) { tools::fail_msg_writer() << "Failed to parse tx to get json format"; } - else if (!cryptonote::parse_and_validate_tx_from_blob(blob, tx, tx_hash, tx_prefix_hash)) - { - tools::fail_msg_writer() << "Failed to parse tx blob to get json format"; - } else { - tools::success_msg_writer() << cryptonote::obj_to_json_str(tx) << std::endl; + bool ret; + if (pruned) + ret = cryptonote::parse_and_validate_tx_base_from_blob(blob, tx); + else + ret = cryptonote::parse_and_validate_tx_from_blob(blob, tx); + if (!ret) + { + tools::fail_msg_writer() << "Failed to parse tx blob to get json format"; + } + else + { + tools::success_msg_writer() << cryptonote::obj_to_json_str(tx) << std::endl; + } } } } @@ -1935,6 +1964,8 @@ bool t_rpc_command_executor::sync_info() for (const auto &p: res.peers) current_download += p.info.current_download; tools::success_msg_writer() << "Downloading at " << current_download << " kB/s"; + if (res.next_needed_pruning_seed) + tools::success_msg_writer() << "Next needed pruning seed: " << res.next_needed_pruning_seed; tools::success_msg_writer() << std::to_string(res.peers.size()) << " peers"; for (const auto &p: res.peers) @@ -1942,25 +1973,30 @@ bool t_rpc_command_executor::sync_info() std::string address = epee::string_tools::pad_string(p.info.address, 24); uint64_t nblocks = 0, size = 0; for (const auto &s: res.spans) - if (s.rate > 0.0f && s.connection_id == p.info.connection_id) + if (s.connection_id == p.info.connection_id) nblocks += s.nblocks, size += s.size; - tools::success_msg_writer() << address << " " << epee::string_tools::pad_string(p.info.peer_id, 16, '0', true) << " " << epee::string_tools::pad_string(p.info.state, 16) << " " << p.info.height << " " << p.info.current_download << " kB/s, " << nblocks << " blocks / " << size/1e6 << " MB queued"; + tools::success_msg_writer() << address << " " << epee::string_tools::pad_string(p.info.peer_id, 16, '0', true) << " " << + epee::string_tools::pad_string(p.info.state, 16) << " " << + epee::string_tools::pad_string(epee::string_tools::to_string_hex(p.info.pruning_seed), 8) << " " << p.info.height << " " << + p.info.current_download << " kB/s, " << nblocks << " blocks / " << size/1e6 << " MB queued"; } uint64_t total_size = 0; for (const auto &s: res.spans) total_size += s.size; tools::success_msg_writer() << std::to_string(res.spans.size()) << " spans, " << total_size/1e6 << " MB"; + tools::success_msg_writer() << res.overview; for (const auto &s: res.spans) { std::string address = epee::string_tools::pad_string(s.remote_address, 24); + std::string pruning_seed = epee::string_tools::to_string_hex(tools::get_pruning_seed(s.start_block_height, std::numeric_limits<uint64_t>::max(), CRYPTONOTE_PRUNING_LOG_STRIPES)); if (s.size == 0) { - tools::success_msg_writer() << address << " " << s.nblocks << " (" << s.start_block_height << " - " << (s.start_block_height + s.nblocks - 1) << ") -"; + tools::success_msg_writer() << address << " " << s.nblocks << "/" << pruning_seed << " (" << s.start_block_height << " - " << (s.start_block_height + s.nblocks - 1) << ") -"; } else { - tools::success_msg_writer() << address << " " << s.nblocks << " (" << s.start_block_height << " - " << (s.start_block_height + s.nblocks - 1) << ", " << (uint64_t)(s.size/1e3) << " kB) " << (unsigned)(s.rate/1e3) << " kB/s (" << s.speed/100.0f << ")"; + tools::success_msg_writer() << address << " " << s.nblocks << "/" << pruning_seed << " (" << s.start_block_height << " - " << (s.start_block_height + s.nblocks - 1) << ", " << (uint64_t)(s.size/1e3) << " kB) " << (unsigned)(s.rate/1e3) << " kB/s (" << s.speed/100.0f << ")"; } } @@ -1994,4 +2030,69 @@ bool t_rpc_command_executor::pop_blocks(uint64_t num_blocks) return true; } +bool t_rpc_command_executor::prune_blockchain() +{ + cryptonote::COMMAND_RPC_PRUNE_BLOCKCHAIN::request req; + cryptonote::COMMAND_RPC_PRUNE_BLOCKCHAIN::response res; + std::string fail_message = "Unsuccessful"; + epee::json_rpc::error error_resp; + + req.check = false; + + if (m_is_rpc) + { + if (!m_rpc_client->json_rpc_request(req, res, "prune_blockchain", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_prune_blockchain(req, res, error_resp) || res.status != CORE_RPC_STATUS_OK) + { + tools::fail_msg_writer() << make_error(fail_message, res.status); + return true; + } + } + + tools::success_msg_writer() << "Blockchain pruned: seed " << epee::string_tools::to_string_hex(res.pruning_seed); + return true; +} + +bool t_rpc_command_executor::check_blockchain_pruning() +{ + cryptonote::COMMAND_RPC_PRUNE_BLOCKCHAIN::request req; + cryptonote::COMMAND_RPC_PRUNE_BLOCKCHAIN::response res; + std::string fail_message = "Unsuccessful"; + epee::json_rpc::error error_resp; + + req.check = true; + + if (m_is_rpc) + { + if (!m_rpc_client->json_rpc_request(req, res, "prune_blockchain", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_prune_blockchain(req, res, error_resp) || res.status != CORE_RPC_STATUS_OK) + { + tools::fail_msg_writer() << make_error(fail_message, res.status); + return true; + } + } + + if (res.pruning_seed) + { + tools::success_msg_writer() << "Blockchain pruning checked"; + } + else + { + tools::success_msg_writer() << "Blockchain is not pruned"; + } + return true; +} + }// namespace daemonize diff --git a/src/daemon/rpc_command_executor.h b/src/daemon/rpc_command_executor.h index 592584a5f..de743065b 100644 --- a/src/daemon/rpc_command_executor.h +++ b/src/daemon/rpc_command_executor.h @@ -91,9 +91,9 @@ public: bool print_height(); - bool print_block_by_hash(crypto::hash block_hash); + bool print_block_by_hash(crypto::hash block_hash, bool include_hex); - bool print_block_by_height(uint64_t height); + bool print_block_by_height(uint64_t height, bool include_hex); bool print_transaction(crypto::hash transaction_hash, bool include_hex, bool include_json); @@ -154,6 +154,10 @@ public: bool sync_info(); bool pop_blocks(uint64_t num_blocks); + + bool prune_blockchain(); + + bool check_blockchain_pruning(); }; } // namespace daemonize diff --git a/src/debug_utilities/cn_deserialize.cpp b/src/debug_utilities/cn_deserialize.cpp index 83422083b..f08f2b928 100644 --- a/src/debug_utilities/cn_deserialize.cpp +++ b/src/debug_utilities/cn_deserialize.cpp @@ -180,11 +180,9 @@ int main(int argc, char* argv[]) } else if (cryptonote::parse_and_validate_tx_from_blob(blob, tx) || cryptonote::parse_and_validate_tx_base_from_blob(blob, tx)) { -/* if (tx.pruned) std::cout << "Parsed pruned transaction:" << std::endl; else -*/ std::cout << "Parsed transaction:" << std::endl; std::cout << cryptonote::obj_to_json_str(tx) << std::endl; diff --git a/src/device/device.hpp b/src/device/device.hpp index 399648f01..bdb608907 100644 --- a/src/device/device.hpp +++ b/src/device/device.hpp @@ -208,8 +208,8 @@ namespace hw { return encrypt_payment_id(payment_id, public_key, secret_key); } - virtual bool ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & sharedSec) = 0; - virtual bool ecdhDecode(rct::ecdhTuple & masked, const rct::key & sharedSec) = 0; + virtual bool ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & sharedSec, bool short_amount) = 0; + virtual bool ecdhDecode(rct::ecdhTuple & masked, const rct::key & sharedSec, bool short_amount) = 0; virtual bool add_output_key_mapping(const crypto::public_key &Aout, const crypto::public_key &Bout, const bool is_subaddress, const size_t real_output_index, const rct::key &amount_key, const crypto::public_key &out_eph_public_key) = 0; diff --git a/src/device/device_default.cpp b/src/device/device_default.cpp index 2286998a4..cb2f4e266 100644 --- a/src/device/device_default.cpp +++ b/src/device/device_default.cpp @@ -302,13 +302,13 @@ namespace hw { return true; } - bool device_default::ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & sharedSec) { - rct::ecdhEncode(unmasked, sharedSec); + bool device_default::ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & sharedSec, bool short_amount) { + rct::ecdhEncode(unmasked, sharedSec, short_amount); return true; } - bool device_default::ecdhDecode(rct::ecdhTuple & masked, const rct::key & sharedSec) { - rct::ecdhDecode(masked, sharedSec); + bool device_default::ecdhDecode(rct::ecdhTuple & masked, const rct::key & sharedSec, bool short_amount) { + rct::ecdhDecode(masked, sharedSec, short_amount); return true; } diff --git a/src/device/device_default.hpp b/src/device/device_default.hpp index 5c59a9066..54d159b11 100644 --- a/src/device/device_default.hpp +++ b/src/device/device_default.hpp @@ -111,8 +111,8 @@ namespace hw { bool encrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key) override; - bool ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & sharedSec) override; - bool ecdhDecode(rct::ecdhTuple & masked, const rct::key & sharedSec) override; + bool ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & sharedSec, bool short_amount) override; + bool ecdhDecode(rct::ecdhTuple & masked, const rct::key & sharedSec, bool short_amount) override; bool add_output_key_mapping(const crypto::public_key &Aout, const crypto::public_key &Bout, const bool is_subaddress, const size_t real_output_index, const rct::key &amount_key, const crypto::public_key &out_eph_public_key) override; diff --git a/src/device/device_io.hpp b/src/device/device_io.hpp index 96163a211..1d5e3564c 100644 --- a/src/device/device_io.hpp +++ b/src/device/device_io.hpp @@ -50,7 +50,7 @@ namespace hw { virtual void disconnect() = 0; virtual bool connected() const = 0; - virtual int exchange(unsigned char *command, unsigned int cmd_len, unsigned char *response, unsigned int max_resp_len) = 0; + virtual int exchange(unsigned char *command, unsigned int cmd_len, unsigned char *response, unsigned int max_resp_len, bool user_input) = 0; }; }; }; diff --git a/src/device/device_io_hid.cpp b/src/device/device_io_hid.cpp index 1aadfb9ea..36c7a241b 100644 --- a/src/device/device_io_hid.cpp +++ b/src/device/device_io_hid.cpp @@ -148,7 +148,7 @@ namespace hw { return this->usb_device != NULL; } - int device_io_hid::exchange(unsigned char *command, unsigned int cmd_len, unsigned char *response, unsigned int max_resp_len) { + int device_io_hid::exchange(unsigned char *command, unsigned int cmd_len, unsigned char *response, unsigned int max_resp_len, bool user_input) { unsigned char buffer[400]; unsigned char padding_buffer[MAX_BLOCK+1]; unsigned int result; @@ -177,7 +177,11 @@ namespace hw { //get first response memset(buffer, 0, sizeof(buffer)); - hid_ret = hid_read_timeout(this->usb_device, buffer, MAX_BLOCK, this->timeout); + if (!user_input) { + hid_ret = hid_read_timeout(this->usb_device, buffer, MAX_BLOCK, this->timeout); + } else { + hid_ret = hid_read(this->usb_device, buffer, MAX_BLOCK); + } ASSERT_X(hid_ret>=0, "Unable to read hidapi response. Error "+std::to_string(result)+": "+ safe_hid_error(this->usb_device)); result = (unsigned int)hid_ret; io_hid_log(1, buffer, result); diff --git a/src/device/device_io_hid.hpp b/src/device/device_io_hid.hpp index bb0f0a814..c47eefad2 100644 --- a/src/device/device_io_hid.hpp +++ b/src/device/device_io_hid.hpp @@ -100,7 +100,7 @@ namespace hw { void connect(void *params); void connect(unsigned int vid, unsigned int pid, boost::optional<int> interface_number, boost::optional<unsigned short> usage_page); bool connected() const; - int exchange(unsigned char *command, unsigned int cmd_len, unsigned char *response, unsigned int max_resp_len); + int exchange(unsigned char *command, unsigned int cmd_len, unsigned char *response, unsigned int max_resp_len, bool user_input); void disconnect(); void release(); }; diff --git a/src/device/device_ledger.cpp b/src/device/device_ledger.cpp index bfb41bbe4..9daf62d37 100644 --- a/src/device/device_ledger.cpp +++ b/src/device/device_ledger.cpp @@ -176,7 +176,7 @@ namespace hw { #define INS_GET_RESPONSE 0xc0 - device_ledger::device_ledger(): hw_device(0x0101, 0x05, 64, 120000) { + device_ledger::device_ledger(): hw_device(0x0101, 0x05, 64, 2000) { this->id = device_id++; this->reset_buffer(); this->mode = NONE; @@ -235,6 +235,9 @@ namespace hw { /* IO */ /* ======================================================================= */ + #define IO_SW_DENY 0x6982 + #define IO_SECRET_KEY 0x02 + void device_ledger::logCMD() { if (apdu_verbose) { char strbuffer[1024]; @@ -283,7 +286,12 @@ namespace hw { void device_ledger::send_simple(unsigned char ins, unsigned char p1) { this->length_send = set_command_header_noopt(ins, p1); - this->exchange(); + if (ins == INS_GET_KEY && p1 == IO_SECRET_KEY) { + // export view key user input + this->exchange_wait_on_input(); + } else { + this->exchange(); + } } bool device_ledger::reset() { @@ -294,7 +302,7 @@ namespace hw { unsigned int device_ledger::exchange(unsigned int ok, unsigned int mask) { logCMD(); - this->length_recv = hw_device.exchange(this->buffer_send, this->length_send, this->buffer_recv, BUFFER_SEND_SIZE); + this->length_recv = hw_device.exchange(this->buffer_send, this->length_send, this->buffer_recv, BUFFER_SEND_SIZE, false); ASSERT_X(this->length_recv>=2, "Communication error, less than tow bytes received"); this->length_recv -= 2; @@ -305,6 +313,25 @@ namespace hw { return this->sw; } + unsigned int device_ledger::exchange_wait_on_input(unsigned int ok, unsigned int mask) { + logCMD(); + unsigned int deny = 0; + this->length_recv = hw_device.exchange(this->buffer_send, this->length_send, this->buffer_recv, BUFFER_SEND_SIZE, true); + ASSERT_X(this->length_recv>=2, "Communication error, less than two bytes received"); + + this->length_recv -= 2; + this->sw = (this->buffer_recv[length_recv]<<8) | this->buffer_recv[length_recv+1]; + if (this->sw == IO_SW_DENY) { + // cancel on device + deny = 1; + } else { + ASSERT_SW(this->sw,ok,msk); + } + + logRESP(); + return deny; + } + void device_ledger::reset_buffer() { this->length_send = 0; memset(this->buffer_send, 0, BUFFER_SEND_SIZE); @@ -322,7 +349,7 @@ namespace hw { } const std::string device_ledger::get_name() const { - if (this->full_name.empty() || !this->connected()) { + if (!this->connected()) { return std::string("<disconnected:").append(this->name).append(">"); } return this->name; @@ -1140,13 +1167,13 @@ namespace hw { return true; } - bool device_ledger::ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & AKout) { + bool device_ledger::ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & AKout, bool short_amount) { AUTO_LOCK_CMD(); #ifdef DEBUG_HWDEVICE const rct::key AKout_x = hw::ledger::decrypt(AKout); rct::ecdhTuple unmasked_x = unmasked; - this->controle_device->ecdhEncode(unmasked_x, AKout_x); + this->controle_device->ecdhEncode(unmasked_x, AKout_x, short_amount); #endif int offset = set_command_header_noopt(INS_BLIND); @@ -1177,13 +1204,13 @@ namespace hw { return true; } - bool device_ledger::ecdhDecode(rct::ecdhTuple & masked, const rct::key & AKout) { + bool device_ledger::ecdhDecode(rct::ecdhTuple & masked, const rct::key & AKout, bool short_amount) { AUTO_LOCK_CMD(); #ifdef DEBUG_HWDEVICE const rct::key AKout_x = hw::ledger::decrypt(AKout); rct::ecdhTuple masked_x = masked; - this->controle_device->ecdhDecode(masked_x, AKout_x); + this->controle_device->ecdhDecode(masked_x, AKout_x, short_amount); #endif int offset = set_command_header_noopt(INS_UNBLIND); @@ -1260,7 +1287,8 @@ namespace hw { this->buffer_send[4] = offset-5; this->length_send = offset; - this->exchange(); + // check fee user input + CHECK_AND_ASSERT_THROW_MES(this->exchange_wait_on_input() == 0, "Fee denied on device."); //pseudoOuts if (type == rct::RCTTypeSimple) { @@ -1328,7 +1356,8 @@ namespace hw { this->buffer_send[4] = offset-5; this->length_send = offset; - this->exchange(); + // check transaction user input + CHECK_AND_ASSERT_THROW_MES(this->exchange_wait_on_input() == 0, "Transaction denied on device."); #ifdef DEBUG_HWDEVICE hw::ledger::log_hexbuffer("Prehash AKV input", (char*)&this->buffer_recv[64], 3*32); #endif diff --git a/src/device/device_ledger.hpp b/src/device/device_ledger.hpp index 2f5beb044..584f1e096 100644 --- a/src/device/device_ledger.hpp +++ b/src/device/device_ledger.hpp @@ -85,7 +85,6 @@ namespace hw { //IO hw::io::device_io_hid hw_device; - std::string full_name; unsigned int length_send; unsigned char buffer_send[BUFFER_SEND_SIZE]; unsigned int length_recv; @@ -95,6 +94,7 @@ namespace hw { void logCMD(void); void logRESP(void); unsigned int exchange(unsigned int ok=0x9000, unsigned int mask=0xFFFF); + unsigned int exchange_wait_on_input(unsigned int ok=0x9000, unsigned int mask=0xFFFF); void reset_buffer(void); int set_command_header(unsigned char ins, unsigned char p1 = 0x00, unsigned char p2 = 0x00); int set_command_header_noopt(unsigned char ins, unsigned char p1 = 0x00, unsigned char p2 = 0x00); @@ -191,8 +191,8 @@ namespace hw { bool encrypt_payment_id(crypto::hash8 &payment_id, const crypto::public_key &public_key, const crypto::secret_key &secret_key) override; - bool ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & sharedSec) override; - bool ecdhDecode(rct::ecdhTuple & masked, const rct::key & sharedSec) override; + bool ecdhEncode(rct::ecdhTuple & unmasked, const rct::key & sharedSec, bool short_format) override; + bool ecdhDecode(rct::ecdhTuple & masked, const rct::key & sharedSec, bool short_format) override; bool add_output_key_mapping(const crypto::public_key &Aout, const crypto::public_key &Bout, const bool is_subaddress, const size_t real_output_index, const rct::key &amount_key, const crypto::public_key &out_eph_public_key) override; diff --git a/src/device_trezor/trezor/protocol.cpp b/src/device_trezor/trezor/protocol.cpp index c4a92426c..13506a67f 100644 --- a/src/device_trezor/trezor/protocol.cpp +++ b/src/device_trezor/trezor/protocol.cpp @@ -877,6 +877,9 @@ namespace tx { valueS.SetString(m_ct.enc_salt2.c_str(), m_ct.enc_salt2.size()); json.AddMember("salt2", valueS, json.GetAllocator()); + valueS.SetString(m_ct.tx_prefix_hash.c_str(), m_ct.tx_prefix_hash.size()); + json.AddMember("tx_prefix_hash", valueS, json.GetAllocator()); + valueS.SetString(m_ct.enc_keys.c_str(), m_ct.enc_keys.size()); json.AddMember("enc_keys", valueS, json.GetAllocator()); diff --git a/src/device_trezor/trezor/protocol.hpp b/src/device_trezor/trezor/protocol.hpp index 99211efed..ce0361640 100644 --- a/src/device_trezor/trezor/protocol.hpp +++ b/src/device_trezor/trezor/protocol.hpp @@ -270,7 +270,7 @@ namespace tx { throw std::invalid_argument("RV not initialized"); } auto tp = m_ct.rv->type; - return tp == rct::RCTTypeBulletproof; + return tp == rct::RCTTypeBulletproof || tp == rct::RCTTypeBulletproof2; } bool is_offloading() const { diff --git a/src/mnemonics/electrum-words.cpp b/src/mnemonics/electrum-words.cpp index b1e3bdcd5..68d5b84d3 100644 --- a/src/mnemonics/electrum-words.cpp +++ b/src/mnemonics/electrum-words.cpp @@ -76,8 +76,8 @@ namespace crypto namespace { uint32_t create_checksum_index(const std::vector<epee::wipeable_string> &word_list, - uint32_t unique_prefix_length); - bool checksum_test(std::vector<epee::wipeable_string> seed, uint32_t unique_prefix_length); + const Language::Base *language); + bool checksum_test(std::vector<epee::wipeable_string> seed, const Language::Base *language); /*! * \brief Finds the word list that contains the seed words and puts the indices @@ -116,8 +116,8 @@ namespace for (std::vector<Language::Base*>::iterator it1 = language_instances.begin(); it1 != language_instances.end(); it1++) { - const std::unordered_map<epee::wipeable_string, uint32_t> &word_map = (*it1)->get_word_map(); - const std::unordered_map<epee::wipeable_string, uint32_t> &trimmed_word_map = (*it1)->get_trimmed_word_map(); + const std::unordered_map<epee::wipeable_string, uint32_t, Language::WordHash, Language::WordEqual> &word_map = (*it1)->get_word_map(); + const std::unordered_map<epee::wipeable_string, uint32_t, Language::WordHash, Language::WordEqual> &trimmed_word_map = (*it1)->get_trimmed_word_map(); // To iterate through seed words bool full_match = true; @@ -151,7 +151,7 @@ namespace // if we were using prefix only, and we have a checksum, check it now // to avoid false positives due to prefix set being too common if (has_checksum) - if (!checksum_test(seed, (*it1)->get_unique_prefix_length())) + if (!checksum_test(seed, *it1)) { fallback = *it1; full_match = false; @@ -190,20 +190,20 @@ namespace * \return Checksum index */ uint32_t create_checksum_index(const std::vector<epee::wipeable_string> &word_list, - uint32_t unique_prefix_length) + const Language::Base *language) { - epee::wipeable_string trimmed_words = ""; + epee::wipeable_string trimmed_words = "", word; + const auto &word_map = language->get_word_map(); + const auto &trimmed_word_map = language->get_trimmed_word_map(); + const uint32_t unique_prefix_length = language->get_unique_prefix_length(); for (std::vector<epee::wipeable_string>::const_iterator it = word_list.begin(); it != word_list.end(); it++) { - if (it->length() > unique_prefix_length) - { - trimmed_words += Language::utf8prefix(*it, unique_prefix_length); - } - else - { - trimmed_words += *it; - } + word = Language::utf8prefix(*it, unique_prefix_length); + auto it2 = trimmed_word_map.find(word); + if (it2 == trimmed_word_map.end()) + throw std::runtime_error("Word \"" + std::string(word.data(), word.size()) + "\" not found in trimmed word map in " + language->get_english_language_name()); + trimmed_words += it2->first; } boost::crc_32_type result; result.process_bytes(trimmed_words.data(), trimmed_words.length()); @@ -216,7 +216,7 @@ namespace * \param unique_prefix_length the prefix length of each word to use for checksum * \return True if the test passed false if not. */ - bool checksum_test(std::vector<epee::wipeable_string> seed, uint32_t unique_prefix_length) + bool checksum_test(std::vector<epee::wipeable_string> seed, const Language::Base *language) { if (seed.empty()) return false; @@ -224,13 +224,16 @@ namespace epee::wipeable_string last_word = seed.back(); seed.pop_back(); - epee::wipeable_string checksum = seed[create_checksum_index(seed, unique_prefix_length)]; + const uint32_t unique_prefix_length = language->get_unique_prefix_length(); + + auto idx = create_checksum_index(seed, language); + epee::wipeable_string checksum = seed[idx]; epee::wipeable_string trimmed_checksum = checksum.length() > unique_prefix_length ? Language::utf8prefix(checksum, unique_prefix_length) : checksum; epee::wipeable_string trimmed_last_word = last_word.length() > unique_prefix_length ? Language::utf8prefix(last_word, unique_prefix_length) : last_word; - bool ret = trimmed_checksum == trimmed_last_word; + bool ret = Language::WordEqual()(trimmed_checksum, trimmed_last_word); MINFO("Checksum is " << (ret ? "valid" : "invalid")); return ret; } @@ -301,7 +304,7 @@ namespace crypto if (has_checksum) { - if (!checksum_test(seed, language->get_unique_prefix_length())) + if (!checksum_test(seed, language)) { // Checksum fail MERROR("Invalid seed: invalid checksum"); @@ -424,7 +427,7 @@ namespace crypto memwipe(w, sizeof(w)); } - words += words_store[create_checksum_index(words_store, language->get_unique_prefix_length())]; + words += words_store[create_checksum_index(words_store, language)]; return true; } diff --git a/src/mnemonics/language_base.h b/src/mnemonics/language_base.h index 52e784cef..a6f969604 100644 --- a/src/mnemonics/language_base.h +++ b/src/mnemonics/language_base.h @@ -38,7 +38,9 @@ #include <vector>
#include <unordered_map>
#include <string>
+#include <boost/algorithm/string.hpp>
#include "misc_log_ex.h"
+#include "fnv1.h"
/*!
* \namespace Language
@@ -71,6 +73,92 @@ namespace Language return prefix;
}
+ template<typename T>
+ inline T utf8canonical(const T &s)
+ {
+ T sc = "";
+ size_t avail = s.size();
+ const char *ptr = s.data();
+ wint_t cp = 0;
+ int bytes = 1;
+ char wbuf[8], *wptr;
+ while (avail--)
+ {
+ if ((*ptr & 0x80) == 0)
+ {
+ cp = *ptr++;
+ bytes = 1;
+ }
+ else if ((*ptr & 0xe0) == 0xc0)
+ {
+ if (avail < 1)
+ throw std::runtime_error("Invalid UTF-8");
+ cp = (*ptr++ & 0x1f) << 6;
+ cp |= *ptr++ & 0x3f;
+ --avail;
+ bytes = 2;
+ }
+ else if ((*ptr & 0xf0) == 0xe0)
+ {
+ if (avail < 2)
+ throw std::runtime_error("Invalid UTF-8");
+ cp = (*ptr++ & 0xf) << 12;
+ cp |= (*ptr++ & 0x3f) << 6;
+ cp |= *ptr++ & 0x3f;
+ avail -= 2;
+ bytes = 3;
+ }
+ else if ((*ptr & 0xf8) == 0xf0)
+ {
+ if (avail < 3)
+ throw std::runtime_error("Invalid UTF-8");
+ cp = (*ptr++ & 0x7) << 18;
+ cp |= (*ptr++ & 0x3f) << 12;
+ cp |= (*ptr++ & 0x3f) << 6;
+ cp |= *ptr++ & 0x3f;
+ avail -= 3;
+ bytes = 4;
+ }
+ else
+ throw std::runtime_error("Invalid UTF-8");
+
+ cp = std::towlower(cp);
+ wptr = wbuf;
+ switch (bytes)
+ {
+ case 1: *wptr++ = cp; break;
+ case 2: *wptr++ = 0xc0 | (cp >> 6); *wptr++ = 0x80 | (cp & 0x3f); break;
+ case 3: *wptr++ = 0xe0 | (cp >> 12); *wptr++ = 0x80 | ((cp >> 6) & 0x3f); *wptr++ = 0x80 | (cp & 0x3f); break;
+ case 4: *wptr++ = 0xf0 | (cp >> 18); *wptr += 0x80 | ((cp >> 12) & 0x3f); *wptr++ = 0x80 | ((cp >> 6) & 0x3f); *wptr++ = 0x80 | (cp & 0x3f); break;
+ default: throw std::runtime_error("Invalid UTF-8");
+ }
+ *wptr = 0;
+ sc += T(wbuf, bytes);
+ cp = 0;
+ bytes = 1;
+ }
+ return sc;
+ }
+
+ struct WordHash
+ {
+ std::size_t operator()(const epee::wipeable_string &s) const
+ {
+ const epee::wipeable_string sc = utf8canonical(s);
+ return epee::fnv::FNV1a(sc.data(), sc.size());
+ }
+ };
+
+ struct WordEqual
+ {
+ bool operator()(const epee::wipeable_string &s0, const epee::wipeable_string &s1) const
+ {
+ const epee::wipeable_string s0c = utf8canonical(s0);
+ const epee::wipeable_string s1c = utf8canonical(s1);
+ return s0c == s1c;
+ }
+ };
+
/*!
* \class Base
* \brief A base language class which all languages have to inherit from for
@@ -87,8 +175,8 @@ namespace Language NWORDS = 1626
};
std::vector<std::string> word_list; /*!< A pointer to the array of words */
- std::unordered_map<epee::wipeable_string, uint32_t> word_map; /*!< hash table to find word's index */
- std::unordered_map<epee::wipeable_string, uint32_t> trimmed_word_map; /*!< hash table to find word's trimmed index */
+ std::unordered_map<epee::wipeable_string, uint32_t, WordHash, WordEqual> word_map; /*!< hash table to find word's index */
+ std::unordered_map<epee::wipeable_string, uint32_t, WordHash, WordEqual> trimmed_word_map; /*!< hash table to find word's trimmed index */
std::string language_name; /*!< Name of language */
std::string english_language_name; /*!< Name of language */
uint32_t unique_prefix_length; /*!< Number of unique starting characters to trim the wordlist to when matching */
@@ -159,7 +247,7 @@ namespace Language * \brief Returns a pointer to the word map.
* \return A pointer to the word map.
*/
- const std::unordered_map<epee::wipeable_string, uint32_t>& get_word_map() const
+ const std::unordered_map<epee::wipeable_string, uint32_t, WordHash, WordEqual>& get_word_map() const
{
return word_map;
}
@@ -167,7 +255,7 @@ namespace Language * \brief Returns a pointer to the trimmed word map.
* \return A pointer to the trimmed word map.
*/
- const std::unordered_map<epee::wipeable_string, uint32_t>& get_trimmed_word_map() const
+ const std::unordered_map<epee::wipeable_string, uint32_t, WordHash, WordEqual>& get_trimmed_word_map() const
{
return trimmed_word_map;
}
diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index 4db0a6cb7..09c219a44 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -29,6 +29,7 @@ // Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers #pragma once +#include <array> #include <boost/thread.hpp> #include <boost/program_options/options_description.hpp> #include <boost/program_options/variables_map.hpp> @@ -127,6 +128,11 @@ namespace nodetool virtual bool block_host(const epee::net_utils::network_address &adress, time_t seconds = P2P_IP_BLOCKTIME); virtual bool unblock_host(const epee::net_utils::network_address &address); virtual std::map<std::string, time_t> get_blocked_hosts() { CRITICAL_REGION_LOCAL(m_blocked_hosts_lock); return m_blocked_hosts; } + + virtual void add_used_stripe_peer(const typename t_payload_net_handler::connection_context &context); + virtual void remove_used_stripe_peer(const typename t_payload_net_handler::connection_context &context); + virtual void clear_used_stripe_peers(); + private: const std::vector<std::string> m_seed_nodes_list = { "seeds.moneroseeds.se" @@ -180,10 +186,10 @@ namespace nodetool virtual void on_connection_close(p2p_connection_context& context); virtual void callback(p2p_connection_context& context); //----------------- i_p2p_endpoint ------------------------------------------------------------- - virtual bool relay_notify_to_list(int command, const std::string& data_buff, const std::list<boost::uuids::uuid> &connections); - virtual bool relay_notify_to_all(int command, const std::string& data_buff, const epee::net_utils::connection_context_base& context); - virtual bool invoke_command_to_peer(int command, const std::string& req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context); - virtual bool invoke_notify_to_peer(int command, const std::string& req_buff, const epee::net_utils::connection_context_base& context); + virtual bool relay_notify_to_list(int command, const epee::span<const uint8_t> data_buff, const std::list<boost::uuids::uuid> &connections); + virtual bool relay_notify_to_all(int command, const epee::span<const uint8_t> data_buff, const epee::net_utils::connection_context_base& context); + virtual bool invoke_command_to_peer(int command, const epee::span<const uint8_t> req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context); + virtual bool invoke_notify_to_peer(int command, const epee::span<const uint8_t> req_buff, const epee::net_utils::connection_context_base& context); virtual bool drop_connection(const epee::net_utils::connection_context_base& context); virtual void request_callback(const epee::net_utils::connection_context_base& context); virtual void for_each_connection(std::function<bool(typename t_payload_net_handler::connection_context&, peerid_type, uint32_t)> f); @@ -235,7 +241,13 @@ namespace nodetool bool parse_peers_and_add_to_container(const boost::program_options::variables_map& vm, const command_line::arg_descriptor<std::vector<std::string> > & arg, Container& container); bool set_max_out_peers(const boost::program_options::variables_map& vm, int64_t max); + bool get_max_out_peers() const { return m_config.m_net_config.max_out_connection_count; } + bool get_current_out_peers() const { return m_current_number_of_out_peers; } + bool set_max_in_peers(const boost::program_options::variables_map& vm, int64_t max); + bool get_max_in_peers() const { return m_config.m_net_config.max_in_connection_count; } + bool get_current_in_peers() const { return m_current_number_of_in_peers; } + bool set_tos_flag(const boost::program_options::variables_map& vm, int limit); bool set_rate_up_limit(const boost::program_options::variables_map& vm, int64_t limit); @@ -336,6 +348,9 @@ namespace nodetool epee::critical_section m_host_fails_score_lock; std::map<std::string, uint64_t> m_host_fails_score; + boost::mutex m_used_stripe_peers_mutex; + std::array<std::list<epee::net_utils::network_address>, 1 << CRYPTONOTE_PRUNING_LOG_STRIPES> m_used_stripe_peers; + cryptonote::network_type m_nettype; }; diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index 5b845fe15..c594984d4 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -33,6 +33,7 @@ #include <algorithm> #include <boost/date_time/posix_time/posix_time.hpp> #include <boost/thread/thread.hpp> +#include <boost/uuid/uuid_io.hpp> #include <boost/bind.hpp> #include <atomic> @@ -40,6 +41,7 @@ #include "string_tools.h" #include "common/util.h" #include "common/dns_utils.h" +#include "common/pruning.h" #include "net/net_helper.h" #include "math_helper.h" #include "p2p_protocol_defs.h" @@ -132,6 +134,28 @@ namespace nodetool make_default_config(); } +#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED + std::list<peerlist_entry> plw; + while (m_peerlist.get_white_peers_count()) + { + plw.push_back(peerlist_entry()); + m_peerlist.get_white_peer_by_index(plw.back(), 0); + m_peerlist.remove_from_peer_white(plw.back()); + } + for (auto &e:plw) + m_peerlist.append_with_peer_white(e); + + std::list<peerlist_entry> plg; + while (m_peerlist.get_gray_peers_count()) + { + plg.push_back(peerlist_entry()); + m_peerlist.get_gray_peer_by_index(plg.back(), 0); + m_peerlist.remove_from_peer_gray(plg.back()); + } + for (auto &e:plg) + m_peerlist.append_with_peer_gray(e); +#endif + // always recreate a new peer id make_default_peer_id(); @@ -728,7 +752,7 @@ namespace nodetool std::atomic<bool> hsh_result(false); bool r = epee::net_utils::async_invoke_remote_command2<typename COMMAND_HANDSHAKE::response>(context_.m_connection_id, COMMAND_HANDSHAKE::ID, arg, m_net_server.get_config_object(), - [this, &pi, &ev, &hsh_result, &just_take_peerlist](int code, const typename COMMAND_HANDSHAKE::response& rsp, p2p_connection_context& context) + [this, &pi, &ev, &hsh_result, &just_take_peerlist, &context_](int code, const typename COMMAND_HANDSHAKE::response& rsp, p2p_connection_context& context) { epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){ev.raise();}); @@ -740,7 +764,7 @@ namespace nodetool if(rsp.node_data.network_id != m_network_id) { - LOG_WARNING_CC(context, "COMMAND_HANDSHAKE Failed, wrong network! (" << epee::string_tools::get_str_from_guid_a(rsp.node_data.network_id) << "), closing connection."); + LOG_WARNING_CC(context, "COMMAND_HANDSHAKE Failed, wrong network! (" << rsp.node_data.network_id << "), closing connection."); return; } @@ -761,7 +785,7 @@ namespace nodetool } pi = context.peer_id = rsp.node_data.peer_id; - m_peerlist.set_peer_just_seen(rsp.node_data.peer_id, context.m_remote_address); + m_peerlist.set_peer_just_seen(rsp.node_data.peer_id, context.m_remote_address, context.m_pruning_seed); if(rsp.node_data.peer_id == m_config.m_peer_id) { @@ -769,11 +793,13 @@ namespace nodetool hsh_result = false; return; } + LOG_INFO_CC(context, "New connection handshaked, pruning seed " << epee::string_tools::to_string_hex(context.m_pruning_seed)); LOG_DEBUG_CC(context, " COMMAND_HANDSHAKE INVOKED OK"); }else { LOG_DEBUG_CC(context, " COMMAND_HANDSHAKE(AND CLOSE) INVOKED OK"); } + context_ = context; }, P2P_DEFAULT_HANDSHAKE_INVOKE_TIMEOUT); if(r) @@ -820,7 +846,7 @@ namespace nodetool add_host_fail(context.m_remote_address); } if(!context.m_is_income) - m_peerlist.set_peer_just_seen(context.peer_id, context.m_remote_address); + m_peerlist.set_peer_just_seen(context.peer_id, context.m_remote_address, context.m_pruning_seed); m_payload_handler.process_payload_sync_data(rsp.payload_data, context, false); }); @@ -938,6 +964,7 @@ namespace nodetool const epee::net_utils::ipv4_network_address &ipv4 = na.as<const epee::net_utils::ipv4_network_address>(); typename net_server::t_connection_context con = AUTO_VAL_INIT(con); + con.m_anchor = peer_type == anchor; bool res = m_net_server.connect(epee::string_tools::get_ip_string_from_int32(ipv4.ip()), epee::string_tools::num_to_string_fast(ipv4.port()), m_config.m_net_config.connection_timeout, @@ -977,6 +1004,7 @@ namespace nodetool time_t last_seen; time(&last_seen); pe_local.last_seen = static_cast<int64_t>(last_seen); + pe_local.pruning_seed = con.m_pruning_seed; m_peerlist.append_with_peer_white(pe_local); //update last seen and push it to peerlist manager @@ -1003,6 +1031,7 @@ namespace nodetool const epee::net_utils::ipv4_network_address &ipv4 = na.as<epee::net_utils::ipv4_network_address>(); typename net_server::t_connection_context con = AUTO_VAL_INIT(con); + con.m_anchor = false; bool res = m_net_server.connect(epee::string_tools::get_ip_string_from_int32(ipv4.ip()), epee::string_tools::num_to_string_fast(ipv4.port()), m_config.m_net_config.connection_timeout, @@ -1055,7 +1084,7 @@ namespace nodetool bool node_server<t_payload_net_handler>::make_new_connection_from_anchor_peerlist(const std::vector<anchor_peerlist_entry>& anchor_peerlist) { for (const auto& pe: anchor_peerlist) { - _note("Considering connecting (out) to peer: " << peerid_type(pe.id) << " " << pe.adr.str()); + _note("Considering connecting (out) to anchor peer: " << peerid_type(pe.id) << " " << pe.adr.str()); if(is_peer_used(pe)) { _note("Peer is used"); @@ -1088,11 +1117,7 @@ namespace nodetool template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::make_new_connection_from_peerlist(bool use_white_list) { - size_t local_peers_count = use_white_list ? m_peerlist.get_white_peers_count():m_peerlist.get_gray_peers_count(); - if(!local_peers_count) - return false;//no peers - - size_t max_random_index = std::min<uint64_t>(local_peers_count -1, 20); + size_t max_random_index = 0; std::set<size_t> tried_peers; @@ -1102,21 +1127,54 @@ namespace nodetool { ++rand_count; size_t random_index; + const uint32_t next_needed_pruning_stripe = m_payload_handler.get_next_needed_pruning_stripe().second; - if (use_white_list) { - local_peers_count = m_peerlist.get_white_peers_count(); - if (!local_peers_count) + std::deque<size_t> filtered; + const size_t limit = use_white_list ? 20 : std::numeric_limits<size_t>::max(); + size_t idx = 0; + m_peerlist.foreach (use_white_list, [&filtered, &idx, limit, next_needed_pruning_stripe](const peerlist_entry &pe){ + if (filtered.size() >= limit) return false; - max_random_index = std::min<uint64_t>(local_peers_count -1, 20); - random_index = get_random_index_with_fixed_probability(max_random_index); - } else { - local_peers_count = m_peerlist.get_gray_peers_count(); - if (!local_peers_count) - return false; - random_index = crypto::rand<size_t>() % local_peers_count; + if (next_needed_pruning_stripe == 0 || pe.pruning_seed == 0) + filtered.push_back(idx); + else if (next_needed_pruning_stripe == tools::get_pruning_stripe(pe.pruning_seed)) + filtered.push_front(idx); + ++idx; + return true; + }); + if (filtered.empty()) + { + MDEBUG("No available peer in " << (use_white_list ? "white" : "gray") << " list filtered by " << next_needed_pruning_stripe); + return false; + } + if (use_white_list) + { + // if using the white list, we first pick in the set of peers we've already been using earlier + random_index = get_random_index_with_fixed_probability(std::min<uint64_t>(filtered.size() - 1, 20)); + CRITICAL_REGION_LOCAL(m_used_stripe_peers_mutex); + if (next_needed_pruning_stripe > 0 && next_needed_pruning_stripe <= (1ul << CRYPTONOTE_PRUNING_LOG_STRIPES) && !m_used_stripe_peers[next_needed_pruning_stripe-1].empty()) + { + const epee::net_utils::network_address na = m_used_stripe_peers[next_needed_pruning_stripe-1].front(); + m_used_stripe_peers[next_needed_pruning_stripe-1].pop_front(); + for (size_t i = 0; i < filtered.size(); ++i) + { + peerlist_entry pe; + if (m_peerlist.get_white_peer_by_index(pe, filtered[i]) && pe.adr == na) + { + MDEBUG("Reusing stripe " << next_needed_pruning_stripe << " peer " << pe.adr.str()); + random_index = i; + break; + } + } + } } + else + random_index = crypto::rand<size_t>() % filtered.size(); - CHECK_AND_ASSERT_MES(random_index < local_peers_count, false, "random_starter_index < peers_local.size() failed!!"); + CHECK_AND_ASSERT_MES(random_index < filtered.size(), false, "random_index < filtered.size() failed!!"); + random_index = filtered[random_index]; + CHECK_AND_ASSERT_MES(random_index < (use_white_list ? m_peerlist.get_white_peers_count() : m_peerlist.get_gray_peers_count()), + false, "random_index < peers size failed!!"); if(tried_peers.count(random_index)) continue; @@ -1128,7 +1186,9 @@ namespace nodetool ++try_count; - _note("Considering connecting (out) to peer: " << peerid_to_string(pe.id) << " " << pe.adr.str()); + _note("Considering connecting (out) to " << (use_white_list ? "white" : "gray") << " list peer: " << + peerid_to_string(pe.id) << " " << pe.adr.str() << ", pruning seed " << epee::string_tools::to_string_hex(pe.pruning_seed) << + " (stripe " << next_needed_pruning_stripe << " needed)"); if(is_peer_used(pe)) { _note("Peer is used"); @@ -1142,6 +1202,7 @@ namespace nodetool continue; MDEBUG("Selected peer: " << peerid_to_string(pe.id) << " " << pe.adr.str() + << ", pruning seed " << epee::string_tools::to_string_hex(pe.pruning_seed) << " " << "[peer_list=" << (use_white_list ? white : gray) << "] last_seen: " << (pe.last_seen ? epee::misc_utils::get_time_interval_string(time(NULL) - pe.last_seen) : "never")); @@ -1218,31 +1279,35 @@ namespace nodetool if (!connect_to_peerlist(m_priority_peers)) return false; - size_t expected_white_connections = (m_config.m_net_config.max_out_connection_count*P2P_DEFAULT_WHITELIST_CONNECTIONS_PERCENT)/100; + size_t base_expected_white_connections = (m_config.m_net_config.max_out_connection_count*P2P_DEFAULT_WHITELIST_CONNECTIONS_PERCENT)/100; size_t conn_count = get_outgoing_connections_count(); - if(conn_count < m_config.m_net_config.max_out_connection_count) + while(conn_count < m_config.m_net_config.max_out_connection_count) { + const size_t expected_white_connections = m_payload_handler.get_next_needed_pruning_stripe().second ? m_config.m_net_config.max_out_connection_count : base_expected_white_connections; if(conn_count < expected_white_connections) { //start from anchor list - if(!make_expected_connections_count(anchor, P2P_DEFAULT_ANCHOR_CONNECTIONS_COUNT)) - return false; + while (get_outgoing_connections_count() < P2P_DEFAULT_ANCHOR_CONNECTIONS_COUNT + && make_expected_connections_count(anchor, P2P_DEFAULT_ANCHOR_CONNECTIONS_COUNT)); //then do white list - if(!make_expected_connections_count(white, expected_white_connections)) - return false; + while (get_outgoing_connections_count() < expected_white_connections + && make_expected_connections_count(white, expected_white_connections)); //then do grey list - if(!make_expected_connections_count(gray, m_config.m_net_config.max_out_connection_count)) - return false; + while (get_outgoing_connections_count() < m_config.m_net_config.max_out_connection_count + && make_expected_connections_count(gray, m_config.m_net_config.max_out_connection_count)); }else { //start from grey list - if(!make_expected_connections_count(gray, m_config.m_net_config.max_out_connection_count)) - return false; + while (get_outgoing_connections_count() < m_config.m_net_config.max_out_connection_count + && make_expected_connections_count(gray, m_config.m_net_config.max_out_connection_count)); //and then do white list - if(!make_expected_connections_count(white, m_config.m_net_config.max_out_connection_count)) - return false; + while (get_outgoing_connections_count() < m_config.m_net_config.max_out_connection_count + && make_expected_connections_count(white, m_config.m_net_config.max_out_connection_count)); } + if(m_net_server.is_stop_signal_sent()) + return false; + conn_count = get_outgoing_connections_count(); } if (start_conn_count == get_outgoing_connections_count() && start_conn_count < m_config.m_net_config.max_out_connection_count) @@ -1259,7 +1324,7 @@ namespace nodetool bool node_server<t_payload_net_handler>::make_expected_connections_count(PeerType peer_type, size_t expected_connections) { if (m_offline) - return true; + return false; std::vector<anchor_peerlist_entry> apl; @@ -1269,24 +1334,24 @@ namespace nodetool size_t conn_count = get_outgoing_connections_count(); //add new connections from white peers - while(conn_count < expected_connections) + if(conn_count < expected_connections) { if(m_net_server.is_stop_signal_sent()) return false; + MDEBUG("Making expected connection, type " << peer_type << ", " << conn_count << "/" << expected_connections << " connections"); + if (peer_type == anchor && !make_new_connection_from_anchor_peerlist(apl)) { - break; + return false; } if (peer_type == white && !make_new_connection_from_peerlist(true)) { - break; + return false; } if (peer_type == gray && !make_new_connection_from_peerlist(false)) { - break; + return false; } - - conn_count = get_outgoing_connections_count(); } return true; } @@ -1389,6 +1454,9 @@ namespace nodetool return false; } be.last_seen += delta; +#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED + be.pruning_seed = tools::make_pruning_seed(1 + (be.adr.as<epee::net_utils::ipv4_network_address>().ip()) % (1ul << CRYPTONOTE_PRUNING_LOG_STRIPES), CRYPTONOTE_PRUNING_LOG_STRIPES); +#endif } return true; } @@ -1516,7 +1584,7 @@ namespace nodetool } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::relay_notify_to_list(int command, const std::string& data_buff, const std::list<boost::uuids::uuid> &connections) + bool node_server<t_payload_net_handler>::relay_notify_to_list(int command, const epee::span<const uint8_t> data_buff, const std::list<boost::uuids::uuid> &connections) { for(const auto& c_id: connections) { @@ -1526,7 +1594,7 @@ namespace nodetool } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::relay_notify_to_all(int command, const std::string& data_buff, const epee::net_utils::connection_context_base& context) + bool node_server<t_payload_net_handler>::relay_notify_to_all(int command, const epee::span<const uint8_t> data_buff, const epee::net_utils::connection_context_base& context) { std::list<boost::uuids::uuid> connections; m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) @@ -1545,14 +1613,14 @@ namespace nodetool } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::invoke_notify_to_peer(int command, const std::string& req_buff, const epee::net_utils::connection_context_base& context) + bool node_server<t_payload_net_handler>::invoke_notify_to_peer(int command, const epee::span<const uint8_t> req_buff, const epee::net_utils::connection_context_base& context) { int res = m_net_server.get_config_object().notify(command, req_buff, context.m_connection_id); return res > 0; } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::invoke_command_to_peer(int command, const std::string& req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context) + bool node_server<t_payload_net_handler>::invoke_command_to_peer(int command, const epee::span<const uint8_t> req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context) { int res = m_net_server.get_config_object().invoke(command, req_buff, resp_buff, context.m_connection_id); return res > 0; @@ -1686,7 +1754,7 @@ namespace nodetool if(arg.node_data.network_id != m_network_id) { - LOG_INFO_CC(context, "WRONG NETWORK AGENT CONNECTED! id=" << epee::string_tools::get_str_from_guid_a(arg.node_data.network_id)); + LOG_INFO_CC(context, "WRONG NETWORK AGENT CONNECTED! id=" << arg.node_data.network_id); drop_connection(context); add_host_fail(context.m_remote_address); return 1; @@ -1749,6 +1817,7 @@ namespace nodetool time(&last_seen); pe.last_seen = static_cast<int64_t>(last_seen); pe.id = peer_id_l; + pe.pruning_seed = context.m_pruning_seed; this->m_peerlist.append_with_peer_white(pe); LOG_DEBUG_CC(context, "PING SUCCESS " << context.m_remote_address.host_str() << ":" << port_l); }); @@ -1802,7 +1871,7 @@ namespace nodetool { ss << cntxt.m_remote_address.str() << " \t\tpeer_id " << cntxt.peer_id - << " \t\tconn_id " << epee::string_tools::get_str_from_guid_a(cntxt.m_connection_id) << (cntxt.m_is_income ? " INC":" OUT") + << " \t\tconn_id " << cntxt.m_connection_id << (cntxt.m_is_income ? " INC":" OUT") << std::endl; return true; }); @@ -1884,21 +1953,16 @@ namespace nodetool template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::set_max_out_peers(const boost::program_options::variables_map& vm, int64_t max) { - if(max == -1) { - m_config.m_net_config.max_out_connection_count = P2P_DEFAULT_CONNECTIONS_COUNT; - return true; - } + if(max == -1) + max = P2P_DEFAULT_CONNECTIONS_COUNT; m_config.m_net_config.max_out_connection_count = max; + m_payload_handler.set_max_out_peers(max); return true; } template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::set_max_in_peers(const boost::program_options::variables_map& vm, int64_t max) { - if(max == -1) { - m_config.m_net_config.max_in_connection_count = -1; - return true; - } m_config.m_net_config.max_in_connection_count = max; return true; } @@ -2009,6 +2073,7 @@ namespace nodetool { if (m_offline) return true; if (!m_exclusive_peers.empty()) return true; + if (m_payload_handler.needs_new_sync_connections()) return true; peerlist_entry pe = AUTO_VAL_INIT(pe); @@ -2016,7 +2081,7 @@ namespace nodetool return false; if (!m_peerlist.get_random_gray_peer(pe)) { - return false; + return true; } bool success = check_connection_and_handshake_with_peer(pe.adr, pe.last_seen); @@ -2029,7 +2094,7 @@ namespace nodetool return true; } - m_peerlist.set_peer_just_seen(pe.id, pe.adr); + m_peerlist.set_peer_just_seen(pe.id, pe.adr, pe.pruning_seed); LOG_PRINT_L2("PEER PROMOTED TO WHITE PEER LIST IP address: " << pe.adr.host_str() << " Peer ID: " << peerid_type(pe.id)); @@ -2037,6 +2102,42 @@ namespace nodetool } template<class t_payload_net_handler> + void node_server<t_payload_net_handler>::add_used_stripe_peer(const typename t_payload_net_handler::connection_context &context) + { + const uint32_t stripe = tools::get_pruning_stripe(context.m_pruning_seed); + if (stripe == 0 || stripe > (1ul << CRYPTONOTE_PRUNING_LOG_STRIPES)) + return; + const uint32_t index = stripe - 1; + CRITICAL_REGION_LOCAL(m_used_stripe_peers_mutex); + MINFO("adding stripe " << stripe << " peer: " << context.m_remote_address.str()); + std::remove_if(m_used_stripe_peers[index].begin(), m_used_stripe_peers[index].end(), + [&context](const epee::net_utils::network_address &na){ return context.m_remote_address == na; }); + m_used_stripe_peers[index].push_back(context.m_remote_address); + } + + template<class t_payload_net_handler> + void node_server<t_payload_net_handler>::remove_used_stripe_peer(const typename t_payload_net_handler::connection_context &context) + { + const uint32_t stripe = tools::get_pruning_stripe(context.m_pruning_seed); + if (stripe == 0 || stripe > (1ul << CRYPTONOTE_PRUNING_LOG_STRIPES)) + return; + const uint32_t index = stripe - 1; + CRITICAL_REGION_LOCAL(m_used_stripe_peers_mutex); + MINFO("removing stripe " << stripe << " peer: " << context.m_remote_address.str()); + std::remove_if(m_used_stripe_peers[index].begin(), m_used_stripe_peers[index].end(), + [&context](const epee::net_utils::network_address &na){ return context.m_remote_address == na; }); + } + + template<class t_payload_net_handler> + void node_server<t_payload_net_handler>::clear_used_stripe_peers() + { + CRITICAL_REGION_LOCAL(m_used_stripe_peers_mutex); + MINFO("clearing used stripe peers"); + for (auto &e: m_used_stripe_peers) + e.clear(); + } + + template<class t_payload_net_handler> void node_server<t_payload_net_handler>::add_upnp_port_mapping(uint32_t port) { MDEBUG("Attempting to add IGD port mapping."); diff --git a/src/p2p/net_node_common.h b/src/p2p/net_node_common.h index 218250efa..075c18ef9 100644 --- a/src/p2p/net_node_common.h +++ b/src/p2p/net_node_common.h @@ -43,10 +43,10 @@ namespace nodetool template<class t_connection_context> struct i_p2p_endpoint { - virtual bool relay_notify_to_list(int command, const std::string& data_buff, const std::list<boost::uuids::uuid>& connections)=0; - virtual bool relay_notify_to_all(int command, const std::string& data_buff, const epee::net_utils::connection_context_base& context)=0; - virtual bool invoke_command_to_peer(int command, const std::string& req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context)=0; - virtual bool invoke_notify_to_peer(int command, const std::string& req_buff, const epee::net_utils::connection_context_base& context)=0; + virtual bool relay_notify_to_list(int command, const epee::span<const uint8_t> data_buff, const std::list<boost::uuids::uuid>& connections)=0; + virtual bool relay_notify_to_all(int command, const epee::span<const uint8_t> data_buff, const epee::net_utils::connection_context_base& context)=0; + virtual bool invoke_command_to_peer(int command, const epee::span<const uint8_t> req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context)=0; + virtual bool invoke_notify_to_peer(int command, const epee::span<const uint8_t> req_buff, const epee::net_utils::connection_context_base& context)=0; virtual bool drop_connection(const epee::net_utils::connection_context_base& context)=0; virtual void request_callback(const epee::net_utils::connection_context_base& context)=0; virtual uint64_t get_connections_count()=0; @@ -56,24 +56,27 @@ namespace nodetool virtual bool unblock_host(const epee::net_utils::network_address &address)=0; virtual std::map<std::string, time_t> get_blocked_hosts()=0; virtual bool add_host_fail(const epee::net_utils::network_address &address)=0; + virtual void add_used_stripe_peer(const t_connection_context &context)=0; + virtual void remove_used_stripe_peer(const t_connection_context &context)=0; + virtual void clear_used_stripe_peers()=0; }; template<class t_connection_context> struct p2p_endpoint_stub: public i_p2p_endpoint<t_connection_context> { - virtual bool relay_notify_to_list(int command, const std::string& data_buff, const std::list<boost::uuids::uuid>& connections) + virtual bool relay_notify_to_list(int command, const epee::span<const uint8_t> data_buff, const std::list<boost::uuids::uuid>& connections) { return false; } - virtual bool relay_notify_to_all(int command, const std::string& data_buff, const epee::net_utils::connection_context_base& context) + virtual bool relay_notify_to_all(int command, const epee::span<const uint8_t> data_buff, const epee::net_utils::connection_context_base& context) { return false; } - virtual bool invoke_command_to_peer(int command, const std::string& req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context) + virtual bool invoke_command_to_peer(int command, const epee::span<const uint8_t> req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context) { return false; } - virtual bool invoke_notify_to_peer(int command, const std::string& req_buff, const epee::net_utils::connection_context_base& context) + virtual bool invoke_notify_to_peer(int command, const epee::span<const uint8_t> req_buff, const epee::net_utils::connection_context_base& context) { return true; } @@ -114,5 +117,14 @@ namespace nodetool { return true; } + virtual void add_used_stripe_peer(const t_connection_context &context) + { + } + virtual void remove_used_stripe_peer(const t_connection_context &context) + { + } + virtual void clear_used_stripe_peers() + { + } }; } diff --git a/src/p2p/net_peerlist.h b/src/p2p/net_peerlist.h index e7aad5abe..685fdc193 100644 --- a/src/p2p/net_peerlist.h +++ b/src/p2p/net_peerlist.h @@ -73,16 +73,18 @@ namespace nodetool bool get_peerlist_full(std::vector<peerlist_entry>& pl_gray, std::vector<peerlist_entry>& pl_white); bool get_white_peer_by_index(peerlist_entry& p, size_t i); bool get_gray_peer_by_index(peerlist_entry& p, size_t i); + template<typename F> bool foreach(bool white, const F &f); bool append_with_peer_white(const peerlist_entry& pr); bool append_with_peer_gray(const peerlist_entry& pr); bool append_with_peer_anchor(const anchor_peerlist_entry& ple); - bool set_peer_just_seen(peerid_type peer, const epee::net_utils::network_address& addr); + bool set_peer_just_seen(peerid_type peer, const epee::net_utils::network_address& addr, uint32_t pruning_seed); bool set_peer_unreachable(const peerlist_entry& pr); bool is_host_allowed(const epee::net_utils::network_address &address); bool get_random_gray_peer(peerlist_entry& pe); bool remove_from_peer_gray(const peerlist_entry& pe); bool get_and_empty_anchor_peerlist(std::vector<anchor_peerlist_entry>& apl); bool remove_from_peer_anchor(const epee::net_utils::network_address& addr); + bool remove_from_peer_white(const peerlist_entry& pe); private: struct by_time{}; @@ -356,8 +358,19 @@ namespace nodetool return true; } //-------------------------------------------------------------------------------------------------- + template<typename F> inline + bool peerlist_manager::foreach(bool white, const F &f) + { + CRITICAL_REGION_LOCAL(m_peerlist_lock); + peers_indexed::index<by_time>::type& by_time_index = white ? m_peers_white.get<by_time>() : m_peers_gray.get<by_time>(); + for(const peers_indexed::value_type& vl: boost::adaptors::reverse(by_time_index)) + if (!f(vl)) + return false; + return true; + } + //-------------------------------------------------------------------------------------------------- inline - bool peerlist_manager::set_peer_just_seen(peerid_type peer, const epee::net_utils::network_address& addr) + bool peerlist_manager::set_peer_just_seen(peerid_type peer, const epee::net_utils::network_address& addr, uint32_t pruning_seed) { TRY_ENTRY(); CRITICAL_REGION_LOCAL(m_peerlist_lock); @@ -366,6 +379,7 @@ namespace nodetool ple.adr = addr; ple.id = peer; ple.last_seen = time(NULL); + ple.pruning_seed = pruning_seed; return append_with_peer_white(ple); CATCH_ENTRY_L0("peerlist_manager::set_peer_just_seen()", false); } @@ -469,6 +483,24 @@ namespace nodetool } //-------------------------------------------------------------------------------------------------- inline + bool peerlist_manager::remove_from_peer_white(const peerlist_entry& pe) + { + TRY_ENTRY(); + + CRITICAL_REGION_LOCAL(m_peerlist_lock); + + peers_indexed::index_iterator<by_addr>::type iterator = m_peers_white.get<by_addr>().find(pe.adr); + + if (iterator != m_peers_white.get<by_addr>().end()) { + m_peers_white.erase(iterator); + } + + return true; + + CATCH_ENTRY_L0("peerlist_manager::remove_from_peer_white()", false); + } + //-------------------------------------------------------------------------------------------------- + inline bool peerlist_manager::remove_from_peer_gray(const peerlist_entry& pe) { TRY_ENTRY(); diff --git a/src/p2p/net_peerlist_boost_serialization.h b/src/p2p/net_peerlist_boost_serialization.h index e79207888..2f4a7e661 100644 --- a/src/p2p/net_peerlist_boost_serialization.h +++ b/src/p2p/net_peerlist_boost_serialization.h @@ -33,6 +33,10 @@ #include "net/net_utils_base.h" #include "p2p/p2p_protocol_defs.h" +#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED +#include "common/pruning.h" +#endif + namespace boost { namespace serialization @@ -77,6 +81,19 @@ namespace boost a & pl.adr; a & pl.id; a & pl.last_seen; + if (ver < 1) + { + if (!typename Archive::is_saving()) + pl.pruning_seed = 0; + return; + } + a & pl.pruning_seed; +#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED + if (!typename Archive::is_saving()) + { + pl.pruning_seed = tools::make_pruning_seed(1+pl.adr.as<epee::net_utils::ipv4_network_address>().ip() % (1<<CRYPTONOTE_PRUNING_LOG_STRIPES), CRYPTONOTE_PRUNING_LOG_STRIPES); + } +#endif } template <class Archive, class ver_type> diff --git a/src/p2p/p2p_protocol_defs.h b/src/p2p/p2p_protocol_defs.h index bb9d2635c..939cedf42 100644 --- a/src/p2p/p2p_protocol_defs.h +++ b/src/p2p/p2p_protocol_defs.h @@ -31,6 +31,7 @@ #pragma once #include <boost/uuid/uuid.hpp> +#include <boost/serialization/version.hpp> #include "serialization/keyvalue_serialization.h" #include "net/net_utils_base.h" #include "misc_language.h" @@ -72,11 +73,13 @@ namespace nodetool AddressType adr; peerid_type id; int64_t last_seen; + uint32_t pruning_seed; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(adr) KV_SERIALIZE(id) KV_SERIALIZE(last_seen) + KV_SERIALIZE_OPT(pruning_seed, (uint32_t)0) END_KV_SERIALIZE_MAP() }; typedef peerlist_entry_base<epee::net_utils::network_address> peerlist_entry; @@ -122,7 +125,7 @@ namespace nodetool ss << std::setfill ('0') << std::setw (8) << std::hex << std::noshowbase; for(const peerlist_entry& pe: pl) { - ss << pe.id << "\t" << pe.adr.str() << " \tlast_seen: " << epee::misc_utils::get_time_interval_string(now_time - pe.last_seen) << std::endl; + ss << pe.id << "\t" << pe.adr.str() << " \tpruning seed " << pe.pruning_seed << " \tlast_seen: " << epee::misc_utils::get_time_interval_string(now_time - pe.last_seen) << std::endl; } return ss.str(); } @@ -205,7 +208,7 @@ namespace nodetool { const epee::net_utils::network_address &na = p.adr; const epee::net_utils::ipv4_network_address &ipv4 = na.as<const epee::net_utils::ipv4_network_address>(); - local_peerlist.push_back(peerlist_entry_base<network_address_old>({{ipv4.ip(), ipv4.port()}, p.id, p.last_seen})); + local_peerlist.push_back(peerlist_entry_base<network_address_old>({{ipv4.ip(), ipv4.port()}, p.id, p.last_seen, p.pruning_seed})); } else MDEBUG("Not including in legacy peer list: " << p.adr.str()); @@ -220,7 +223,7 @@ namespace nodetool std::vector<peerlist_entry_base<network_address_old>> local_peerlist; epee::serialization::selector<is_store>::serialize_stl_container_pod_val_as_blob(local_peerlist, stg, hparent_section, "local_peerlist"); for (const auto &p: local_peerlist) - ((response&)this_ref).local_peerlist_new.push_back(peerlist_entry({epee::net_utils::ipv4_network_address(p.adr.ip, p.adr.port), p.id, p.last_seen})); + ((response&)this_ref).local_peerlist_new.push_back(peerlist_entry({epee::net_utils::ipv4_network_address(p.adr.ip, p.adr.port), p.id, p.last_seen, p.pruning_seed})); } } END_KV_SERIALIZE_MAP() @@ -463,5 +466,6 @@ namespace nodetool } +BOOST_CLASS_VERSION(nodetool::peerlist_entry, 1) diff --git a/src/ringct/rctOps.cpp b/src/ringct/rctOps.cpp index 0ec654af6..e39ba16fd 100644 --- a/src/ringct/rctOps.cpp +++ b/src/ringct/rctOps.cpp @@ -670,18 +670,58 @@ namespace rct { //Elliptic Curve Diffie Helman: encodes and decodes the amount b and mask a // where C= aG + bH - void ecdhEncode(ecdhTuple & unmasked, const key & sharedSec) { - key sharedSec1 = hash_to_scalar(sharedSec); - key sharedSec2 = hash_to_scalar(sharedSec1); + static key ecdhHash(const key &k) + { + char data[38]; + rct::key hash; + memcpy(data, "amount", 6); + memcpy(data + 6, &k, sizeof(k)); + cn_fast_hash(hash, data, sizeof(data)); + return hash; + } + static void xor8(key &v, const key &k) + { + for (int i = 0; i < 8; ++i) + v.bytes[i] ^= k.bytes[i]; + } + key genCommitmentMask(const key &sk) + { + char data[15 + sizeof(key)]; + memcpy(data, "commitment_mask", 15); + memcpy(data + 15, &sk, sizeof(sk)); + key scalar; + hash_to_scalar(scalar, data, sizeof(data)); + return scalar; + } + + void ecdhEncode(ecdhTuple & unmasked, const key & sharedSec, bool v2) { //encode - sc_add(unmasked.mask.bytes, unmasked.mask.bytes, sharedSec1.bytes); - sc_add(unmasked.amount.bytes, unmasked.amount.bytes, sharedSec2.bytes); + if (v2) + { + unmasked.mask = zero(); + xor8(unmasked.amount, ecdhHash(sharedSec)); + } + else + { + key sharedSec1 = hash_to_scalar(sharedSec); + key sharedSec2 = hash_to_scalar(sharedSec1); + sc_add(unmasked.mask.bytes, unmasked.mask.bytes, sharedSec1.bytes); + sc_add(unmasked.amount.bytes, unmasked.amount.bytes, sharedSec2.bytes); + } } - void ecdhDecode(ecdhTuple & masked, const key & sharedSec) { - key sharedSec1 = hash_to_scalar(sharedSec); - key sharedSec2 = hash_to_scalar(sharedSec1); + void ecdhDecode(ecdhTuple & masked, const key & sharedSec, bool v2) { //decode - sc_sub(masked.mask.bytes, masked.mask.bytes, sharedSec1.bytes); - sc_sub(masked.amount.bytes, masked.amount.bytes, sharedSec2.bytes); + if (v2) + { + masked.mask = genCommitmentMask(sharedSec); + xor8(masked.amount, ecdhHash(sharedSec)); + } + else + { + key sharedSec1 = hash_to_scalar(sharedSec); + key sharedSec2 = hash_to_scalar(sharedSec1); + sc_sub(masked.mask.bytes, masked.mask.bytes, sharedSec1.bytes); + sc_sub(masked.amount.bytes, masked.amount.bytes, sharedSec2.bytes); + } } } diff --git a/src/ringct/rctOps.h b/src/ringct/rctOps.h index 60e920b3a..dd6d87593 100644 --- a/src/ringct/rctOps.h +++ b/src/ringct/rctOps.h @@ -182,7 +182,8 @@ namespace rct { //Elliptic Curve Diffie Helman: encodes and decodes the amount b and mask a // where C= aG + bH - void ecdhEncode(ecdhTuple & unmasked, const key & sharedSec); - void ecdhDecode(ecdhTuple & masked, const key & sharedSec); + key genCommitmentMask(const key &sk); + void ecdhEncode(ecdhTuple & unmasked, const key & sharedSec, bool v2); + void ecdhDecode(ecdhTuple & masked, const key & sharedSec, bool v2); } #endif /* RCTOPS_H */ diff --git a/src/ringct/rctSigs.cpp b/src/ringct/rctSigs.cpp index baa649f82..81bec487c 100644 --- a/src/ringct/rctSigs.cpp +++ b/src/ringct/rctSigs.cpp @@ -79,9 +79,12 @@ namespace } namespace rct { - Bulletproof proveRangeBulletproof(keyV &C, keyV &masks, const std::vector<uint64_t> &amounts) + Bulletproof proveRangeBulletproof(keyV &C, keyV &masks, const std::vector<uint64_t> &amounts, epee::span<const key> sk) { - masks = rct::skvGen(amounts.size()); + CHECK_AND_ASSERT_THROW_MES(amounts.size() == sk.size(), "Invalid amounts/sk sizes"); + masks.resize(amounts.size()); + for (size_t i = 0; i < masks.size(); ++i) + masks[i] = genCommitmentMask(sk[i]); Bulletproof proof = bulletproof_PROVE(amounts, masks); CHECK_AND_ASSERT_THROW_MES(proof.V.size() == amounts.size(), "V does not have the expected size"); C = proof.V; @@ -416,7 +419,7 @@ namespace rct { hashes.push_back(hash2rct(h)); keyV kv; - if (rv.type == RCTTypeBulletproof) + if (rv.type == RCTTypeBulletproof || rv.type == RCTTypeBulletproof2) { kv.reserve((6*2+9) * rv.p.bulletproofs.size()); for (const auto &p: rv.p.bulletproofs) @@ -605,10 +608,19 @@ namespace rct { keyV tmp(rows + 1); size_t i; keyM M(cols, tmp); + ge_p3 Cp3; + CHECK_AND_ASSERT_MES_L1(ge_frombytes_vartime(&Cp3, C.bytes) == 0, false, "point conv failed"); + ge_cached Ccached; + ge_p3_to_cached(&Ccached, &Cp3); + ge_p1p1 p1; //create the matrix to mg sig for (i = 0; i < cols; i++) { M[i][0] = pubs[i].dest; - subKeys(M[i][1], pubs[i].mask, C); + ge_p3 p3; + CHECK_AND_ASSERT_MES_L1(ge_frombytes_vartime(&p3, pubs[i].mask.bytes) == 0, false, "point conv failed"); + ge_sub(&p1, &p3, &Ccached); + ge_p1p1_to_p3(&p3, &p1); + ge_p3_tobytes(M[i][1].bytes, &p3); } //DP(C); return MLSAG_Ver(message, M, mg, rows); @@ -677,7 +689,7 @@ namespace rct { // must know the destination private key to find the correct amount, else will return a random number // Note: For txn fees, the last index in the amounts vector should contain that // Thus the amounts vector will be "one" longer than the destinations vectort - rctSig genRct(const key &message, const ctkeyV & inSk, const keyV & destinations, const vector<xmr_amount> & amounts, const ctkeyM &mixRing, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, unsigned int index, ctkeyV &outSk, hw::device &hwdev) { + rctSig genRct(const key &message, const ctkeyV & inSk, const keyV & destinations, const vector<xmr_amount> & amounts, const ctkeyM &mixRing, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, unsigned int index, ctkeyV &outSk, const RCTConfig &rct_config, hw::device &hwdev) { CHECK_AND_ASSERT_THROW_MES(amounts.size() == destinations.size() || amounts.size() == destinations.size() + 1, "Different number of amounts/destinations"); CHECK_AND_ASSERT_THROW_MES(amount_keys.size() == destinations.size(), "Different number of amount_keys/destinations"); CHECK_AND_ASSERT_THROW_MES(index < mixRing.size(), "Bad index into mixRing"); @@ -707,7 +719,7 @@ namespace rct { //mask amount and mask rv.ecdhInfo[i].mask = copy(outSk[i].mask); rv.ecdhInfo[i].amount = d2h(amounts[i]); - hwdev.ecdhEncode(rv.ecdhInfo[i], amount_keys[i]); + hwdev.ecdhEncode(rv.ecdhInfo[i], amount_keys[i], rv.type == RCTTypeBulletproof2); } //set txn fee @@ -728,18 +740,18 @@ namespace rct { return rv; } - rctSig genRct(const key &message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const vector<xmr_amount> & amounts, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, const int mixin, hw::device &hwdev) { + rctSig genRct(const key &message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const vector<xmr_amount> & amounts, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, const int mixin, const RCTConfig &rct_config, hw::device &hwdev) { unsigned int index; ctkeyM mixRing; ctkeyV outSk; tie(mixRing, index) = populateFromBlockchain(inPk, mixin); - return genRct(message, inSk, destinations, amounts, mixRing, amount_keys, kLRki, msout, index, outSk, hwdev); + return genRct(message, inSk, destinations, amounts, mixRing, amount_keys, kLRki, msout, index, outSk, rct_config, hwdev); } //RCT simple //for post-rct only - rctSig genRctSimple(const key &message, const ctkeyV & inSk, const keyV & destinations, const vector<xmr_amount> &inamounts, const vector<xmr_amount> &outamounts, xmr_amount txnFee, const ctkeyM & mixRing, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, const std::vector<unsigned int> & index, ctkeyV &outSk, RangeProofType range_proof_type, hw::device &hwdev) { - const bool bulletproof = range_proof_type != RangeProofBorromean; + rctSig genRctSimple(const key &message, const ctkeyV & inSk, const keyV & destinations, const vector<xmr_amount> &inamounts, const vector<xmr_amount> &outamounts, xmr_amount txnFee, const ctkeyM & mixRing, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, const std::vector<unsigned int> & index, ctkeyV &outSk, const RCTConfig &rct_config, hw::device &hwdev) { + const bool bulletproof = rct_config.range_proof_type != RangeProofBorromean; CHECK_AND_ASSERT_THROW_MES(inamounts.size() > 0, "Empty inamounts"); CHECK_AND_ASSERT_THROW_MES(inamounts.size() == inSk.size(), "Different number of inamounts/inSk"); CHECK_AND_ASSERT_THROW_MES(outamounts.size() == destinations.size(), "Different number of amounts/destinations"); @@ -755,7 +767,7 @@ namespace rct { } rctSig rv; - rv.type = bulletproof ? RCTTypeBulletproof : RCTTypeSimple; + rv.type = bulletproof ? (rct_config.bp_version == 0 || rct_config.bp_version >= 2 ? RCTTypeBulletproof2 : RCTTypeBulletproof) : RCTTypeSimple; rv.message = message; rv.outPk.resize(destinations.size()); if (!bulletproof) @@ -784,7 +796,7 @@ namespace rct { std::vector<uint64_t> proof_amounts; size_t n_amounts = outamounts.size(); size_t amounts_proved = 0; - if (range_proof_type == RangeProofPaddedBulletproof) + if (rct_config.range_proof_type == RangeProofPaddedBulletproof) { rct::keyV C, masks; if (hwdev.get_mode() == hw::device::TRANSACTION_CREATE_FAKE) @@ -794,7 +806,8 @@ namespace rct { } else { - rv.p.bulletproofs.push_back(proveRangeBulletproof(C, masks, outamounts)); + const epee::span<const key> keys{&amount_keys[0], amount_keys.size()}; + rv.p.bulletproofs.push_back(proveRangeBulletproof(C, masks, outamounts, keys)); #ifdef DBG CHECK_AND_ASSERT_THROW_MES(verBulletproof(rv.p.bulletproofs.back()), "verBulletproof failed on newly created proof"); #endif @@ -808,7 +821,7 @@ namespace rct { else while (amounts_proved < n_amounts) { size_t batch_size = 1; - if (range_proof_type == RangeProofMultiOutputBulletproof) + if (rct_config.range_proof_type == RangeProofMultiOutputBulletproof) while (batch_size * 2 + amounts_proved <= n_amounts && batch_size * 2 <= BULLETPROOF_MAX_OUTPUTS) batch_size *= 2; rct::keyV C, masks; @@ -822,7 +835,8 @@ namespace rct { } else { - rv.p.bulletproofs.push_back(proveRangeBulletproof(C, masks, batch_amounts)); + const epee::span<const key> keys{&amount_keys[amounts_proved], batch_size}; + rv.p.bulletproofs.push_back(proveRangeBulletproof(C, masks, batch_amounts, keys)); #ifdef DBG CHECK_AND_ASSERT_THROW_MES(verBulletproof(rv.p.bulletproofs.back()), "verBulletproof failed on newly created proof"); #endif @@ -844,7 +858,7 @@ namespace rct { //mask amount and mask rv.ecdhInfo[i].mask = copy(outSk[i].mask); rv.ecdhInfo[i].amount = d2h(outamounts[i]); - hwdev.ecdhEncode(rv.ecdhInfo[i], amount_keys[i]); + hwdev.ecdhEncode(rv.ecdhInfo[i], amount_keys[i], rv.type == RCTTypeBulletproof2); } //set txn fee @@ -862,7 +876,6 @@ namespace rct { sc_add(sumpouts.bytes, a[i].bytes, sumpouts.bytes); genC(pseudoOuts[i], a[i], inamounts[i]); } - rv.mixRing = mixRing; sc_sub(a[i].bytes, sumout.bytes, sumpouts.bytes); genC(pseudoOuts[i], a[i], inamounts[i]); DP(pseudoOuts[i]); @@ -876,7 +889,7 @@ namespace rct { return rv; } - rctSig genRctSimple(const key &message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const vector<xmr_amount> &inamounts, const vector<xmr_amount> &outamounts, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, xmr_amount txnFee, unsigned int mixin, hw::device &hwdev) { + rctSig genRctSimple(const key &message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const vector<xmr_amount> &inamounts, const vector<xmr_amount> &outamounts, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, xmr_amount txnFee, unsigned int mixin, const RCTConfig &rct_config, hw::device &hwdev) { std::vector<unsigned int> index; index.resize(inPk.size()); ctkeyM mixRing; @@ -886,7 +899,7 @@ namespace rct { mixRing[i].resize(mixin+1); index[i] = populateFromBlockchainSimple(mixRing[i], inPk[i], mixin); } - return genRctSimple(message, inSk, destinations, inamounts, outamounts, txnFee, mixRing, amount_keys, kLRki, msout, index, outSk, RangeProofBorromean, hwdev); + return genRctSimple(message, inSk, destinations, inamounts, outamounts, txnFee, mixRing, amount_keys, kLRki, msout, index, outSk, rct_config, hwdev); } //RingCT protocol @@ -976,7 +989,8 @@ namespace rct { { CHECK_AND_ASSERT_MES(rvp, false, "rctSig pointer is NULL"); const rctSig &rv = *rvp; - CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof, false, "verRctSemanticsSimple called on non simple rctSig"); + CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof || rv.type == RCTTypeBulletproof2, + false, "verRctSemanticsSimple called on non simple rctSig"); const bool bulletproof = is_rct_bulletproof(rv.type); if (bulletproof) { @@ -1075,7 +1089,8 @@ namespace rct { { PERF_TIMER(verRctNonSemanticsSimple); - CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof, false, "verRctNonSemanticsSimple called on non simple rctSig"); + CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof || rv.type == RCTTypeBulletproof2, + false, "verRctNonSemanticsSimple called on non simple rctSig"); const bool bulletproof = is_rct_bulletproof(rv.type); // semantics check is early, and mixRing/MGs aren't resolved yet if (bulletproof) @@ -1141,7 +1156,7 @@ namespace rct { //mask amount and mask ecdhTuple ecdh_info = rv.ecdhInfo[i]; - hwdev.ecdhDecode(ecdh_info, sk); + hwdev.ecdhDecode(ecdh_info, sk, rv.type == RCTTypeBulletproof2); mask = ecdh_info.mask; key amount = ecdh_info.amount; key C = rv.outPk[i].mask; @@ -1165,13 +1180,13 @@ namespace rct { } xmr_amount decodeRctSimple(const rctSig & rv, const key & sk, unsigned int i, key &mask, hw::device &hwdev) { - CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof, false, "decodeRct called on non simple rctSig"); + CHECK_AND_ASSERT_MES(rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof || rv.type == RCTTypeBulletproof2, false, "decodeRct called on non simple rctSig"); CHECK_AND_ASSERT_THROW_MES(i < rv.ecdhInfo.size(), "Bad index"); CHECK_AND_ASSERT_THROW_MES(rv.outPk.size() == rv.ecdhInfo.size(), "Mismatched sizes of rv.outPk and rv.ecdhInfo"); //mask amount and mask ecdhTuple ecdh_info = rv.ecdhInfo[i]; - hwdev.ecdhDecode(ecdh_info, sk); + hwdev.ecdhDecode(ecdh_info, sk, rv.type == RCTTypeBulletproof2); mask = ecdh_info.mask; key amount = ecdh_info.amount; key C = rv.outPk[i].mask; @@ -1195,7 +1210,7 @@ namespace rct { } bool signMultisig(rctSig &rv, const std::vector<unsigned int> &indices, const keyV &k, const multisig_out &msout, const key &secret_key) { - CHECK_AND_ASSERT_MES(rv.type == RCTTypeFull || rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof, + CHECK_AND_ASSERT_MES(rv.type == RCTTypeFull || rv.type == RCTTypeSimple || rv.type == RCTTypeBulletproof || rv.type == RCTTypeBulletproof2, false, "unsupported rct type"); CHECK_AND_ASSERT_MES(indices.size() == k.size(), false, "Mismatched k/indices sizes"); CHECK_AND_ASSERT_MES(k.size() == rv.p.MGs.size(), false, "Mismatched k/MGs size"); diff --git a/src/ringct/rctSigs.h b/src/ringct/rctSigs.h index 459edc600..9227eab1e 100644 --- a/src/ringct/rctSigs.h +++ b/src/ringct/rctSigs.h @@ -119,10 +119,10 @@ namespace rct { //decodeRct: (c.f. https://eprint.iacr.org/2015/1098 section 5.1.1) // uses the attached ecdh info to find the amounts represented by each output commitment // must know the destination private key to find the correct amount, else will return a random number - rctSig genRct(const key &message, const ctkeyV & inSk, const keyV & destinations, const std::vector<xmr_amount> & amounts, const ctkeyM &mixRing, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, unsigned int index, ctkeyV &outSk, hw::device &hwdev); - rctSig genRct(const key &message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const std::vector<xmr_amount> & amounts, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, const int mixin, hw::device &hwdev); - rctSig genRctSimple(const key & message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const std::vector<xmr_amount> & inamounts, const std::vector<xmr_amount> & outamounts, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, xmr_amount txnFee, unsigned int mixin, hw::device &hwdev); - rctSig genRctSimple(const key & message, const ctkeyV & inSk, const keyV & destinations, const std::vector<xmr_amount> & inamounts, const std::vector<xmr_amount> & outamounts, xmr_amount txnFee, const ctkeyM & mixRing, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, const std::vector<unsigned int> & index, ctkeyV &outSk, RangeProofType range_proof_type, hw::device &hwdev); + rctSig genRct(const key &message, const ctkeyV & inSk, const keyV & destinations, const std::vector<xmr_amount> & amounts, const ctkeyM &mixRing, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, unsigned int index, ctkeyV &outSk, const RCTConfig &rct_config, hw::device &hwdev); + rctSig genRct(const key &message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const std::vector<xmr_amount> & amounts, const keyV &amount_keys, const multisig_kLRki *kLRki, multisig_out *msout, const int mixin, const RCTConfig &rct_config, hw::device &hwdev); + rctSig genRctSimple(const key & message, const ctkeyV & inSk, const ctkeyV & inPk, const keyV & destinations, const std::vector<xmr_amount> & inamounts, const std::vector<xmr_amount> & outamounts, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, xmr_amount txnFee, unsigned int mixin, const RCTConfig &rct_config, hw::device &hwdev); + rctSig genRctSimple(const key & message, const ctkeyV & inSk, const keyV & destinations, const std::vector<xmr_amount> & inamounts, const std::vector<xmr_amount> & outamounts, xmr_amount txnFee, const ctkeyM & mixRing, const keyV &amount_keys, const std::vector<multisig_kLRki> *kLRki, multisig_out *msout, const std::vector<unsigned int> & index, ctkeyV &outSk, const RCTConfig &rct_config, hw::device &hwdev); bool verRct(const rctSig & rv, bool semantics); static inline bool verRct(const rctSig & rv) { return verRct(rv, true) && verRct(rv, false); } bool verRctSemanticsSimple(const rctSig & rv); diff --git a/src/ringct/rctTypes.cpp b/src/ringct/rctTypes.cpp index 90ed65df0..f01e683cb 100644 --- a/src/ringct/rctTypes.cpp +++ b/src/ringct/rctTypes.cpp @@ -217,6 +217,7 @@ namespace rct { { case RCTTypeSimple: case RCTTypeBulletproof: + case RCTTypeBulletproof2: return true; default: return false; @@ -228,6 +229,7 @@ namespace rct { switch (type) { case RCTTypeBulletproof: + case RCTTypeBulletproof2: return true; default: return false; diff --git a/src/ringct/rctTypes.h b/src/ringct/rctTypes.h index 487ea6f32..50d0f4d91 100644 --- a/src/ringct/rctTypes.h +++ b/src/ringct/rctTypes.h @@ -120,17 +120,14 @@ namespace rct { // If the pedersen commitment to an amount is C = aG + bH, // "mask" contains a 32 byte key a // "amount" contains a hex representation (in 32 bytes) of a 64 bit number - // "senderPk" is not the senders actual public key, but a one-time public key generated for // the purpose of the ECDH exchange struct ecdhTuple { key mask; key amount; - key senderPk; BEGIN_SERIALIZE_OBJECT() - FIELD(mask) + FIELD(mask) // not saved from v2 BPs FIELD(amount) - // FIELD(senderPk) // not serialized, as we do not use it in monero currently END_SERIALIZE() }; @@ -231,8 +228,13 @@ namespace rct { RCTTypeFull = 1, RCTTypeSimple = 2, RCTTypeBulletproof = 3, + RCTTypeBulletproof2 = 4, }; enum RangeProofType { RangeProofBorromean, RangeProofBulletproof, RangeProofMultiOutputBulletproof, RangeProofPaddedBulletproof }; + struct RCTConfig { + RangeProofType range_proof_type; + int bp_version; + }; struct rctSigBase { uint8_t type; key message; @@ -249,7 +251,7 @@ namespace rct { FIELD(type) if (type == RCTTypeNull) return true; - if (type != RCTTypeFull && type != RCTTypeSimple && type != RCTTypeBulletproof) + if (type != RCTTypeFull && type != RCTTypeSimple && type != RCTTypeBulletproof && type != RCTTypeBulletproof2) return false; VARINT_FIELD(txnFee) // inputs/outputs not saved, only here for serialization help @@ -278,7 +280,19 @@ namespace rct { return false; for (size_t i = 0; i < outputs; ++i) { - FIELDS(ecdhInfo[i]) + if (type == RCTTypeBulletproof2) + { + ar.begin_object(); + if (!typename Archive<W>::is_saving()) + memset(ecdhInfo[i].amount.bytes, 0, sizeof(ecdhInfo[i].amount.bytes)); + crypto::hash8 &amount = (crypto::hash8&)ecdhInfo[i].amount; + FIELD(amount); + ar.end_object(); + } + else + { + FIELDS(ecdhInfo[i]) + } if (outputs - i > 1) ar.delimit_array(); } @@ -310,12 +324,15 @@ namespace rct { { if (type == RCTTypeNull) return true; - if (type != RCTTypeFull && type != RCTTypeSimple && type != RCTTypeBulletproof) + if (type != RCTTypeFull && type != RCTTypeSimple && type != RCTTypeBulletproof && type != RCTTypeBulletproof2) return false; - if (type == RCTTypeBulletproof) + if (type == RCTTypeBulletproof || type == RCTTypeBulletproof2) { uint32_t nbp = bulletproofs.size(); - FIELD(nbp) + if (type == RCTTypeBulletproof2) + VARINT_FIELD(nbp) + else + FIELD(nbp) ar.tag("bp"); ar.begin_array(); if (nbp > outputs) @@ -351,7 +368,7 @@ namespace rct { ar.begin_array(); // we keep a byte for size of MGs, because we don't know whether this is // a simple or full rct signature, and it's starting to annoy the hell out of me - size_t mg_elements = (type == RCTTypeSimple || type == RCTTypeBulletproof) ? inputs : 1; + size_t mg_elements = (type == RCTTypeSimple || type == RCTTypeBulletproof || type == RCTTypeBulletproof2) ? inputs : 1; PREPARE_CUSTOM_VECTOR_SERIALIZATION(mg_elements, MGs); if (MGs.size() != mg_elements) return false; @@ -369,7 +386,7 @@ namespace rct { for (size_t j = 0; j < mixin + 1; ++j) { ar.begin_array(); - size_t mg_ss2_elements = ((type == RCTTypeSimple || type == RCTTypeBulletproof) ? 1 : inputs) + 1; + size_t mg_ss2_elements = ((type == RCTTypeSimple || type == RCTTypeBulletproof || type == RCTTypeBulletproof2) ? 1 : inputs) + 1; PREPARE_CUSTOM_VECTOR_SERIALIZATION(mg_ss2_elements, MGs[i].ss[j]); if (MGs[i].ss[j].size() != mg_ss2_elements) return false; @@ -395,7 +412,7 @@ namespace rct { ar.delimit_array(); } ar.end_array(); - if (type == RCTTypeBulletproof) + if (type == RCTTypeBulletproof || type == RCTTypeBulletproof2) { ar.tag("pseudoOuts"); ar.begin_array(); @@ -419,12 +436,12 @@ namespace rct { keyV& get_pseudo_outs() { - return type == RCTTypeBulletproof ? p.pseudoOuts : pseudoOuts; + return type == RCTTypeBulletproof || type == RCTTypeBulletproof2 ? p.pseudoOuts : pseudoOuts; } keyV const& get_pseudo_outs() const { - return type == RCTTypeBulletproof ? p.pseudoOuts : pseudoOuts; + return type == RCTTypeBulletproof || type == RCTTypeBulletproof2 ? p.pseudoOuts : pseudoOuts; } }; diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 0aa25bda7..08674a06e 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -252,19 +252,11 @@ namespace cryptonote pruned_size += bd.first.first.size(); unpruned_size += bd.first.first.size(); res.output_indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices()); - res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices()); - if (!req.no_miner_tx) - { - bool r = m_core.get_tx_outputs_gindexs(bd.first.second, res.output_indices.back().indices.back().indices); - if (!r) - { - res.status = "Failed"; - return false; - } - } ntxes += bd.second.size(); + res.output_indices.back().indices.reserve(1 + bd.second.size()); + if (req.no_miner_tx) + res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices()); res.blocks.back().txs.reserve(bd.second.size()); - res.output_indices.back().indices.reserve(bd.second.size()); for (std::vector<std::pair<crypto::hash, cryptonote::blobdata>>::iterator i = bd.second.begin(); i != bd.second.end(); ++i) { unpruned_size += i->second.size(); @@ -272,14 +264,25 @@ namespace cryptonote i->second.clear(); i->second.shrink_to_fit(); pruned_size += res.blocks.back().txs.back().size(); + } - res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices()); - bool r = m_core.get_tx_outputs_gindexs(i->first, res.output_indices.back().indices.back().indices); + const size_t n_txes_to_lookup = bd.second.size() + (req.no_miner_tx ? 0 : 1); + if (n_txes_to_lookup > 0) + { + std::vector<std::vector<uint64_t>> indices; + bool r = m_core.get_tx_outputs_gindexs(req.no_miner_tx ? bd.second.front().first : bd.first.second, n_txes_to_lookup, indices); if (!r) { res.status = "Failed"; return false; } + if (indices.size() != n_txes_to_lookup || res.output_indices.back().indices.size() != (req.no_miner_tx ? 1 : 0)) + { + res.status = "Failed"; + return false; + } + for (size_t i = 0; i < indices.size(); ++i) + res.output_indices.back().indices.push_back({std::move(indices[i])}); } } @@ -482,8 +485,8 @@ namespace cryptonote vh.push_back(*reinterpret_cast<const crypto::hash*>(b.data())); } std::vector<crypto::hash> missed_txs; - std::vector<transaction> txs; - bool r = m_core.get_transactions(vh, txs, missed_txs); + std::vector<std::tuple<crypto::hash, cryptonote::blobdata, crypto::hash, cryptonote::blobdata>> txs; + bool r = m_core.get_split_transactions_blobs(vh, txs, missed_txs); if(!r) { res.status = "Failed"; @@ -503,7 +506,7 @@ namespace cryptonote if(r) { // sort to match original request - std::vector<transaction> sorted_txs; + std::vector<std::tuple<crypto::hash, cryptonote::blobdata, crypto::hash, cryptonote::blobdata>> sorted_txs; std::vector<tx_info>::const_iterator i; unsigned txs_processed = 0; for (const crypto::hash &h: vh) @@ -516,7 +519,7 @@ namespace cryptonote return true; } // core returns the ones it finds in the right order - if (get_transaction_hash(txs[txs_processed]) != h) + if (std::get<0>(txs[txs_processed]) != h) { res.status = "Failed: tx hash mismatch"; return true; @@ -532,7 +535,16 @@ namespace cryptonote res.status = "Failed to parse and validate tx from blob"; return true; } - sorted_txs.push_back(tx); + std::stringstream ss; + binary_archive<true> ba(ss); + bool r = const_cast<cryptonote::transaction&>(tx).serialize_base(ba); + if (!r) + { + res.status = "Failed to serialize transaction base"; + return true; + } + const cryptonote::blobdata pruned = ss.str(); + sorted_txs.push_back(std::make_tuple(h, pruned, get_transaction_prunable_hash(tx), std::string(i->tx_blob, pruned.size()))); missed_txs.erase(std::find(missed_txs.begin(), missed_txs.end(), h)); pool_tx_hashes.insert(h); const std::string hash_string = epee::string_tools::pod_to_hex(h); @@ -561,11 +573,36 @@ namespace cryptonote crypto::hash tx_hash = *vhi++; e.tx_hash = *txhi++; - pruned_transaction pruned_tx{tx}; - blobdata blob = req.prune ? t_serializable_object_to_blob(pruned_tx) : t_serializable_object_to_blob(tx); - e.as_hex = string_tools::buff_to_hex_nodelimer(blob); - if (req.decode_as_json) - e.as_json = req.prune ? obj_to_json_str(pruned_tx) : obj_to_json_str(tx); + e.prunable_hash = epee::string_tools::pod_to_hex(std::get<2>(tx)); + if (req.split || req.prune || std::get<3>(tx).empty()) + { + e.pruned_as_hex = string_tools::buff_to_hex_nodelimer(std::get<1>(tx)); + if (!req.prune) + e.prunable_as_hex = string_tools::buff_to_hex_nodelimer(std::get<3>(tx)); + } + else + { + cryptonote::blobdata tx_data; + if (req.prune) + tx_data = std::get<1>(tx); + else + tx_data = std::get<1>(tx) + std::get<3>(tx); + e.as_hex = string_tools::buff_to_hex_nodelimer(tx_data); + if (req.decode_as_json && !tx_data.empty()) + { + cryptonote::transaction t; + if (cryptonote::parse_and_validate_tx_from_blob(tx_data, t)) + { + if (req.prune) + { + pruned_transaction pruned_tx{t}; + e.as_json = obj_to_json_str(pruned_tx); + } + else + e.as_json = obj_to_json_str(t); + } + } + } e.in_pool = pool_tx_hashes.find(tx_hash) != pool_tx_hashes.end(); if (e.in_pool) { @@ -703,31 +740,31 @@ namespace cryptonote if(!m_core.handle_incoming_tx(tx_blob, tvc, false, false, req.do_not_relay) || tvc.m_verifivation_failed) { res.status = "Failed"; - res.reason = ""; + std::string reason = ""; if ((res.low_mixin = tvc.m_low_mixin)) - add_reason(res.reason, "bad ring size"); + add_reason(reason, "bad ring size"); if ((res.double_spend = tvc.m_double_spend)) - add_reason(res.reason, "double spend"); + add_reason(reason, "double spend"); if ((res.invalid_input = tvc.m_invalid_input)) - add_reason(res.reason, "invalid input"); + add_reason(reason, "invalid input"); if ((res.invalid_output = tvc.m_invalid_output)) - add_reason(res.reason, "invalid output"); + add_reason(reason, "invalid output"); if ((res.too_big = tvc.m_too_big)) - add_reason(res.reason, "too big"); + add_reason(reason, "too big"); if ((res.overspend = tvc.m_overspend)) - add_reason(res.reason, "overspend"); + add_reason(reason, "overspend"); if ((res.fee_too_low = tvc.m_fee_too_low)) - add_reason(res.reason, "fee too low"); + add_reason(reason, "fee too low"); if ((res.not_rct = tvc.m_not_rct)) - add_reason(res.reason, "tx is not ringct"); - const std::string punctuation = res.reason.empty() ? "" : ": "; + add_reason(reason, "tx is not ringct"); + const std::string punctuation = reason.empty() ? "" : ": "; if (tvc.m_verifivation_failed) { - LOG_PRINT_L0("[on_send_raw_tx]: tx verification failed" << punctuation << res.reason); + LOG_PRINT_L0("[on_send_raw_tx]: tx verification failed" << punctuation << reason); } else { - LOG_PRINT_L0("[on_send_raw_tx]: Failed to process tx" << punctuation << res.reason); + LOG_PRINT_L0("[on_send_raw_tx]: Failed to process tx" << punctuation << reason); } return true; } @@ -866,9 +903,9 @@ namespace cryptonote { if (entry.adr.get_type_id() == epee::net_utils::ipv4_network_address::ID) res.white_list.emplace_back(entry.id, entry.adr.as<epee::net_utils::ipv4_network_address>().ip(), - entry.adr.as<epee::net_utils::ipv4_network_address>().port(), entry.last_seen); + entry.adr.as<epee::net_utils::ipv4_network_address>().port(), entry.last_seen, entry.pruning_seed); else - res.white_list.emplace_back(entry.id, entry.adr.str(), entry.last_seen); + res.white_list.emplace_back(entry.id, entry.adr.str(), entry.last_seen, entry.pruning_seed); } res.gray_list.reserve(gray_list.size()); @@ -876,9 +913,9 @@ namespace cryptonote { if (entry.adr.get_type_id() == epee::net_utils::ipv4_network_address::ID) res.gray_list.emplace_back(entry.id, entry.adr.as<epee::net_utils::ipv4_network_address>().ip(), - entry.adr.as<epee::net_utils::ipv4_network_address>().port(), entry.last_seen); + entry.adr.as<epee::net_utils::ipv4_network_address>().port(), entry.last_seen, entry.pruning_seed); else - res.gray_list.emplace_back(entry.id, entry.adr.str(), entry.last_seen); + res.gray_list.emplace_back(entry.id, entry.adr.str(), entry.last_seen, entry.pruning_seed); } res.status = CORE_RPC_STATUS_OK; @@ -2099,6 +2136,7 @@ namespace cryptonote m_core.get_blockchain_top(res.height, top_hash); ++res.height; // turn top block height into blockchain height res.target_height = m_core.get_target_blockchain_height(); + res.next_needed_pruning_seed = m_p2p.get_payload_object().get_next_needed_pruning_stripe().second; for (const auto &c: m_p2p.get_payload_object().get_connections()) res.peers.push_back({c}); @@ -2113,6 +2151,7 @@ namespace cryptonote res.spans.push_back({span.start_block_height, span.nblocks, span_connection_id, (uint32_t)(span.rate + 0.5f), speed, span.size, address}); return true; }); + res.overview = block_queue.get_overview(res.height); res.status = CORE_RPC_STATUS_OK; return true; @@ -2212,6 +2251,29 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_prune_blockchain(const COMMAND_RPC_PRUNE_BLOCKCHAIN::request& req, COMMAND_RPC_PRUNE_BLOCKCHAIN::response& res, epee::json_rpc::error& error_resp) + { + try + { + if (!(req.check ? m_core.check_blockchain_pruning() : m_core.prune_blockchain())) + { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = req.check ? "Failed to check blockchain pruning" : "Failed to prune blockchain"; + return false; + } + res.pruning_seed = m_core.get_blockchain_pruning_seed(); + } + catch (const std::exception &e) + { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = "Failed to prune blockchain"; + return false; + } + + res.status = CORE_RPC_STATUS_OK; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ const command_line::arg_descriptor<std::string, false, true, 2> core_rpc_server::arg_rpc_bind_port = { diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index 081ccc25d..83a7cfe27 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -153,6 +153,7 @@ namespace cryptonote MAP_JON_RPC_WE_IF("sync_info", on_sync_info, COMMAND_RPC_SYNC_INFO, !m_restricted) MAP_JON_RPC_WE("get_txpool_backlog", on_get_txpool_backlog, COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG) MAP_JON_RPC_WE("get_output_distribution", on_get_output_distribution, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION) + MAP_JON_RPC_WE_IF("prune_blockchain", on_prune_blockchain, COMMAND_RPC_PRUNE_BLOCKCHAIN, !m_restricted) END_JSON_RPC_MAP() END_URI_MAP2() @@ -217,6 +218,7 @@ namespace cryptonote bool on_sync_info(const COMMAND_RPC_SYNC_INFO::request& req, COMMAND_RPC_SYNC_INFO::response& res, epee::json_rpc::error& error_resp); bool on_get_txpool_backlog(const COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::response& res, epee::json_rpc::error& error_resp); bool on_get_output_distribution(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res, epee::json_rpc::error& error_resp); + bool on_prune_blockchain(const COMMAND_RPC_PRUNE_BLOCKCHAIN::request& req, COMMAND_RPC_PRUNE_BLOCKCHAIN::response& res, epee::json_rpc::error& error_resp); //----------------------- private: diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index 0a07930ec..dfad5d6a7 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -84,7 +84,7 @@ namespace cryptonote // advance which version they will stop working with // Don't go over 32767 for any of these #define CORE_RPC_VERSION_MAJOR 2 -#define CORE_RPC_VERSION_MINOR 2 +#define CORE_RPC_VERSION_MINOR 3 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR) @@ -601,11 +601,13 @@ namespace cryptonote std::vector<std::string> txs_hashes; bool decode_as_json; bool prune; + bool split; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(txs_hashes) KV_SERIALIZE(decode_as_json) KV_SERIALIZE_OPT(prune, false) + KV_SERIALIZE_OPT(split, false) END_KV_SERIALIZE_MAP() }; @@ -613,6 +615,9 @@ namespace cryptonote { std::string tx_hash; std::string as_hex; + std::string pruned_as_hex; + std::string prunable_as_hex; + std::string prunable_hash; std::string as_json; bool in_pool; bool double_spend_seen; @@ -623,6 +628,9 @@ namespace cryptonote BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tx_hash) KV_SERIALIZE(as_hex) + KV_SERIALIZE(pruned_as_hex) + KV_SERIALIZE(prunable_as_hex) + KV_SERIALIZE(prunable_hash) KV_SERIALIZE(as_json) KV_SERIALIZE(in_pool) KV_SERIALIZE(double_spend_seen) @@ -1311,14 +1319,15 @@ namespace cryptonote uint32_t ip; uint16_t port; uint64_t last_seen; + uint32_t pruning_seed; peer() = default; - peer(uint64_t id, const std::string &host, uint64_t last_seen) - : id(id), host(host), ip(0), port(0), last_seen(last_seen) + peer(uint64_t id, const std::string &host, uint64_t last_seen, uint32_t pruning_seed) + : id(id), host(host), ip(0), port(0), last_seen(last_seen), pruning_seed(pruning_seed) {} - peer(uint64_t id, uint32_t ip, uint16_t port, uint64_t last_seen) - : id(id), host(std::to_string(ip)), ip(ip), port(port), last_seen(last_seen) + peer(uint64_t id, uint32_t ip, uint16_t port, uint64_t last_seen, uint32_t pruning_seed) + : id(id), host(std::to_string(ip)), ip(ip), port(port), last_seen(last_seen), pruning_seed(pruning_seed) {} BEGIN_KV_SERIALIZE_MAP() @@ -1327,6 +1336,7 @@ namespace cryptonote KV_SERIALIZE(ip) KV_SERIALIZE(port) KV_SERIALIZE(last_seen) + KV_SERIALIZE_OPT(pruning_seed, (uint32_t)0) END_KV_SERIALIZE_MAP() }; @@ -2238,15 +2248,19 @@ namespace cryptonote std::string status; uint64_t height; uint64_t target_height; + uint32_t next_needed_pruning_seed; std::list<peer> peers; std::list<span> spans; + std::string overview; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(status) KV_SERIALIZE(height) KV_SERIALIZE(target_height) + KV_SERIALIZE(next_needed_pruning_seed) KV_SERIALIZE(peers) KV_SERIALIZE(spans) + KV_SERIALIZE(overview) END_KV_SERIALIZE_MAP() }; }; @@ -2351,4 +2365,27 @@ namespace cryptonote }; }; + struct COMMAND_RPC_PRUNE_BLOCKCHAIN + { + struct request + { + bool check; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_OPT(check, false) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + uint32_t pruning_seed; + std::string status; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + KV_SERIALIZE(pruning_seed) + END_KV_SERIALIZE_MAP() + }; + }; + } diff --git a/src/rpc/message_data_structs.h b/src/rpc/message_data_structs.h index e09b6749e..73cf28cec 100644 --- a/src/rpc/message_data_structs.h +++ b/src/rpc/message_data_structs.h @@ -79,6 +79,7 @@ namespace rpc uint32_t ip; uint16_t port; uint64_t last_seen; + uint32_t pruning_seed; }; struct tx_in_pool diff --git a/src/serialization/json_object.cpp b/src/serialization/json_object.cpp index 8b1af9c12..ee4fa4a19 100644 --- a/src/serialization/json_object.cpp +++ b/src/serialization/json_object.cpp @@ -734,6 +734,7 @@ void toJsonValue(rapidjson::Document& doc, const cryptonote::rpc::peer& peer, ra INSERT_INTO_JSON_OBJECT(val, doc, ip, peer.ip); INSERT_INTO_JSON_OBJECT(val, doc, port, peer.port); INSERT_INTO_JSON_OBJECT(val, doc, last_seen, peer.last_seen); + INSERT_INTO_JSON_OBJECT(val, doc, pruning_seed, peer.pruning_seed); } @@ -748,6 +749,7 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::peer& peer) GET_FROM_JSON_OBJECT(val, peer.ip, ip); GET_FROM_JSON_OBJECT(val, peer.port, port); GET_FROM_JSON_OBJECT(val, peer.last_seen, last_seen); + GET_FROM_JSON_OBJECT(val, peer.pruning_seed, pruning_seed); } void toJsonValue(rapidjson::Document& doc, const cryptonote::rpc::tx_in_pool& tx, rapidjson::Value& val) diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index bdb6d2bfe..3cbfb760b 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -113,6 +113,14 @@ typedef cryptonote::simple_wallet sw; #define PRINT_USAGE(usage_help) fail_msg_writer() << boost::format(tr("usage: %s")) % usage_help; +#define LONG_PAYMENT_ID_SUPPORT_CHECK() \ + do { \ + if (!m_long_payment_id_support) { \ + fail_msg_writer() << tr("Long payment IDs are obsolete. Use --long-payment-id-support if you really must use one."); \ + return true; \ + } \ + } while(0) + enum TransferType { Transfer, TransferLocked, @@ -136,10 +144,12 @@ namespace const command_line::arg_descriptor<bool> arg_non_deterministic = {"non-deterministic", sw::tr("Generate non-deterministic view and spend keys"), false}; const command_line::arg_descriptor<bool> arg_allow_mismatched_daemon_version = {"allow-mismatched-daemon-version", sw::tr("Allow communicating with a daemon that uses a different RPC version"), false}; const command_line::arg_descriptor<uint64_t> arg_restore_height = {"restore-height", sw::tr("Restore from specific blockchain height"), 0}; + const command_line::arg_descriptor<std::string> arg_restore_date = {"restore-date", sw::tr("Restore from estimated blockchain height on specified date"), ""}; const command_line::arg_descriptor<bool> arg_do_not_relay = {"do-not-relay", sw::tr("The newly created transaction will not be relayed to the monero network"), false}; const command_line::arg_descriptor<bool> arg_create_address_file = {"create-address-file", sw::tr("Create an address file for new wallets"), false}; const command_line::arg_descriptor<std::string> arg_subaddress_lookahead = {"subaddress-lookahead", tools::wallet2::tr("Set subaddress lookahead sizes to <major>:<minor>"), ""}; const command_line::arg_descriptor<bool> arg_use_english_language_names = {"use-english-language-names", sw::tr("Display English language names"), false}; + const command_line::arg_descriptor<bool> arg_long_payment_id_support = {"long-payment-id-support", sw::tr("Support obsolete long (unencrypted) payment ids"), false}; const command_line::arg_descriptor< std::vector<std::string> > arg_command = {"command", ""}; @@ -150,12 +160,12 @@ namespace const char* USAGE_PAYMENTS("payments <PID_1> [<PID_2> ... <PID_N>]"); const char* USAGE_PAYMENT_ID("payment_id"); const char* USAGE_TRANSFER("transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] (<URI> | <address> <amount>) [<payment_id>]"); - const char* USAGE_LOCKED_TRANSFER("locked_transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] (<URI> | <addr> <amount>) <lockblocks> [<payment_id>]"); - const char* USAGE_LOCKED_SWEEP_ALL("locked_sweep_all [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> <lockblocks> [<payment_id>]"); - const char* USAGE_SWEEP_ALL("sweep_all [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] [outputs=<N>] <address> [<payment_id>]"); - const char* USAGE_SWEEP_BELOW("sweep_below <amount_threshold> [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> [<payment_id>]"); - const char* USAGE_SWEEP_SINGLE("sweep_single [<priority>] [<ring_size>] [outputs=<N>] <key_image> <address> [<payment_id>]"); - const char* USAGE_DONATE("donate [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <amount> [<payment_id>]"); + const char* USAGE_LOCKED_TRANSFER("locked_transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] (<URI> | <addr> <amount>) <lockblocks> [<payment_id (obsolete)>]"); + const char* USAGE_LOCKED_SWEEP_ALL("locked_sweep_all [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> <lockblocks> [<payment_id (obsolete)>]"); + const char* USAGE_SWEEP_ALL("sweep_all [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] [outputs=<N>] <address> [<payment_id (obsolete)>]"); + const char* USAGE_SWEEP_BELOW("sweep_below <amount_threshold> [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> [<payment_id (obsolete)>]"); + const char* USAGE_SWEEP_SINGLE("sweep_single [<priority>] [<ring_size>] [outputs=<N>] <key_image> <address> [<payment_id (obsolete)>]"); + const char* USAGE_DONATE("donate [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <amount> [<payment_id (obsolete)>]"); const char* USAGE_SIGN_TRANSFER("sign_transfer [export_raw]"); const char* USAGE_SET_LOG("set_log <level>|{+,-,}<categories>"); const char* USAGE_ACCOUNT("account\n" @@ -230,12 +240,15 @@ namespace const char* USAGE_VERSION("version"); const char* USAGE_HELP("help [<command>]"); - std::string input_line(const std::string& prompt) + std::string input_line(const std::string& prompt, bool yesno = false) { #ifdef HAVE_READLINE rdln::suspend_readline pause_readline; #endif std::cout << prompt; + if (yesno) + std::cout << " (Y/Yes/N/No)"; + std::cout << ": " << std::flush; std::string buf; #ifdef _WIN32 @@ -247,12 +260,12 @@ namespace return epee::string_tools::trim(buf); } - epee::wipeable_string input_secure_line(const std::string& prompt) + epee::wipeable_string input_secure_line(const char *prompt) { #ifdef HAVE_READLINE rdln::suspend_readline pause_readline; #endif - auto pwd_container = tools::password_container::prompt(false, prompt.c_str(), false); + auto pwd_container = tools::password_container::prompt(false, prompt, false); if (!pwd_container) { MERROR("Failed to read secure line"); @@ -425,10 +438,10 @@ namespace << ", " << dnssec_str << std::endl << sw::tr(" Monero Address = ") << addresses[0] << std::endl - << sw::tr("Is this OK? (Y/n) ") + << sw::tr("Is this OK?") ; // prompt the user for confirmation given the dns query and dnssec status - std::string confirm_dns_ok = input_line(prompt.str()); + std::string confirm_dns_ok = input_line(prompt.str(), true); if (std::cin.eof()) { return {}; @@ -540,7 +553,7 @@ namespace } catch (const tools::error::tx_rejected& e) { - fail_msg_writer() << (boost::format(sw::tr("transaction %s was rejected by daemon with status: ")) % get_transaction_hash(e.tx())) << e.status(); + fail_msg_writer() << (boost::format(sw::tr("transaction %s was rejected by daemon")) % get_transaction_hash(e.tx())); std::string reason = e.reason(); if (!reason.empty()) fail_msg_writer() << sw::tr("Reason: ") << reason; @@ -596,7 +609,7 @@ namespace fail_msg_writer() << boost::format(sw::tr("File %s likely stores wallet private keys! Use a different file name.")) % filename; return false; } - return command_line::is_yes(input_line((boost::format(sw::tr("File %s already exists. Are you sure to overwrite it? (Y/Yes/N/No): ")) % filename).str())); + return command_line::is_yes(input_line((boost::format(sw::tr("File %s already exists. Are you sure to overwrite it?")) % filename).str(), true)); } return true; } @@ -860,6 +873,8 @@ bool simple_wallet::change_password(const std::vector<std::string> &args) bool simple_wallet::payment_id(const std::vector<std::string> &args/* = std::vector<std::string>()*/) { + LONG_PAYMENT_ID_SUPPORT_CHECK(); + crypto::hash payment_id; if (args.size() > 0) { @@ -1165,6 +1180,7 @@ bool simple_wallet::exchange_multisig_keys_main(const std::vector<std::string> & uint32_t threshold, total; m_wallet->multisig(NULL, &threshold, &total); success_msg_writer() << tr("Multisig wallet has been successfully created. Current wallet type: ") << threshold << "/" << total; + success_msg_writer() << tr("Multisig address: ") << m_wallet->get_account().get_public_address_str(m_wallet->nettype()); } } catch (const std::exception &e) @@ -2221,6 +2237,8 @@ bool simple_wallet::set_refresh_type(const std::vector<std::string> &args/* = st bool simple_wallet::set_confirm_missing_payment_id(const std::vector<std::string> &args/* = std::vector<std::string>()*/) { + LONG_PAYMENT_ID_SUPPORT_CHECK(); + const auto pwd_container = get_and_verify_password(); if (pwd_container) { @@ -2781,7 +2799,7 @@ simple_wallet::simple_wallet() m_cmd_binder.set_handler("rescan_bc", boost::bind(&simple_wallet::rescan_blockchain, this, _1), tr(USAGE_RESCAN_BC), - tr("Rescan the blockchain from scratch, losing any information which can not be recovered from the blockchain itself.")); + tr("Rescan the blockchain from scratch. If \"hard\" is specified, you will lose any information which can not be recovered from the blockchain itself.")); m_cmd_binder.set_handler("set_tx_note", boost::bind(&simple_wallet::set_tx_note, this, _1), tr(USAGE_SET_TX_NOTE), @@ -2846,7 +2864,7 @@ simple_wallet::simple_wallet() m_cmd_binder.set_handler("payment_id", boost::bind(&simple_wallet::payment_id, this, _1), tr(USAGE_PAYMENT_ID), - tr("Generate a new random full size payment id. These will be unencrypted on the blockchain, see integrated_address for encrypted short payment ids.")); + tr("Generate a new random full size payment id (obsolete). These will be unencrypted on the blockchain, see integrated_address for encrypted short payment ids.")); m_cmd_binder.set_handler("fee", boost::bind(&simple_wallet::print_fee_info, this, _1), tr("Print the information about the current fee and transaction backlog.")); @@ -3152,9 +3170,9 @@ bool simple_wallet::ask_wallet_create_if_needed() LOG_PRINT_L3("User asked to specify wallet file name."); wallet_path = input_line( tr(m_restoring ? "Specify a new wallet file name for your restored wallet (e.g., MyWallet).\n" - "Wallet file name (or Ctrl-C to quit): " : + "Wallet file name (or Ctrl-C to quit)" : "Specify wallet file name (e.g., MyWallet). If the wallet doesn't exist, it will be created.\n" - "Wallet file name (or Ctrl-C to quit): ") + "Wallet file name (or Ctrl-C to quit)") ); if(std::cin.eof()) { @@ -3201,7 +3219,7 @@ bool simple_wallet::ask_wallet_create_if_needed() if (!m_restoring) { message_writer() << tr("No wallet found with that name. Confirm creation of new wallet named: ") << wallet_path; - confirm_creation = input_line(tr("(Y/Yes/N/No): ")); + confirm_creation = input_line("", true); if(std::cin.eof()) { LOG_ERROR("Unexpected std::cin.eof() - Exited simple_wallet::ask_wallet_create_if_needed()"); @@ -3260,6 +3278,28 @@ static bool might_be_partial_seed(const epee::wipeable_string &words) return seed.size() < 24; } //---------------------------------------------------------------------------------------------------- +static bool datestr_to_int(const std::string &heightstr, uint16_t &year, uint8_t &month, uint8_t &day) +{ + if (heightstr.size() != 10 || heightstr[4] != '-' || heightstr[7] != '-') + { + fail_msg_writer() << tr("date format must be YYYY-MM-DD"); + return false; + } + try + { + year = boost::lexical_cast<uint16_t>(heightstr.substr(0,4)); + // lexical_cast<uint8_t> won't work because uint8_t is treated as character type + month = boost::lexical_cast<uint16_t>(heightstr.substr(5,2)); + day = boost::lexical_cast<uint16_t>(heightstr.substr(8,2)); + } + catch (const boost::bad_lexical_cast &) + { + fail_msg_writer() << tr("bad height parameter: ") << heightstr; + return false; + } + return true; +} +//---------------------------------------------------------------------------------------------------- bool simple_wallet::init(const boost::program_options::variables_map& vm) { epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){ @@ -3388,7 +3428,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) { m_wallet_file = m_generate_from_view_key; // parse address - std::string address_string = input_line("Standard address: "); + std::string address_string = input_line("Standard address"); if (std::cin.eof()) return false; if (address_string.empty()) { @@ -3408,7 +3448,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) } // parse view secret key - epee::wipeable_string viewkey_string = input_secure_line("Secret view key: "); + epee::wipeable_string viewkey_string = input_secure_line("Secret view key"); if (std::cin.eof()) return false; if (viewkey_string.empty()) { @@ -3443,7 +3483,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) { m_wallet_file = m_generate_from_spend_key; // parse spend secret key - epee::wipeable_string spendkey_string = input_secure_line("Secret spend key: "); + epee::wipeable_string spendkey_string = input_secure_line("Secret spend key"); if (std::cin.eof()) return false; if (spendkey_string.empty()) { @@ -3463,7 +3503,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) { m_wallet_file = m_generate_from_keys; // parse address - std::string address_string = input_line("Standard address: "); + std::string address_string = input_line("Standard address"); if (std::cin.eof()) return false; if (address_string.empty()) { @@ -3483,7 +3523,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) } // parse spend secret key - epee::wipeable_string spendkey_string = input_secure_line("Secret spend key: "); + epee::wipeable_string spendkey_string = input_secure_line("Secret spend key"); if (std::cin.eof()) return false; if (spendkey_string.empty()) { @@ -3498,7 +3538,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) } // parse view secret key - epee::wipeable_string viewkey_string = input_secure_line("Secret view key: "); + epee::wipeable_string viewkey_string = input_secure_line("Secret view key"); if (std::cin.eof()) return false; if (viewkey_string.empty()) { @@ -3545,7 +3585,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) unsigned int multisig_n; // parse multisig type - std::string multisig_type_string = input_line("Multisig type (input as M/N with M <= N and M > 1): "); + std::string multisig_type_string = input_line("Multisig type (input as M/N with M <= N and M > 1)"); if (std::cin.eof()) return false; if (multisig_type_string.empty()) @@ -3571,7 +3611,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) message_writer() << boost::format(tr("Generating master wallet from %u of %u multisig wallet keys")) % multisig_m % multisig_n; // parse multisig address - std::string address_string = input_line("Multisig wallet address: "); + std::string address_string = input_line("Multisig wallet address"); if (std::cin.eof()) return false; if (address_string.empty()) { @@ -3586,7 +3626,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) } // parse secret view key - epee::wipeable_string viewkey_string = input_secure_line("Secret view key: "); + epee::wipeable_string viewkey_string = input_secure_line("Secret view key"); if (std::cin.eof()) return false; if (viewkey_string.empty()) @@ -3625,7 +3665,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) // get N secret spend keys from user for(unsigned int i=0; i<multisig_n; ++i) { - spendkey_string = input_secure_line(tr((boost::format(tr("Secret spend key (%u of %u):")) % (i+1) % multisig_m).str().c_str())); + spendkey_string = input_secure_line(tr((boost::format(tr("Secret spend key (%u of %u)")) % (i+1) % multisig_m).str().c_str())); if (std::cin.eof()) return false; if (spendkey_string.empty()) @@ -3698,15 +3738,15 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) if(m_wallet->get_refresh_from_block_height() == 0) { { tools::scoped_message_writer wrt = tools::msg_writer(); - wrt << tr("No restore height is specified."); - wrt << tr("Assumed you are creating a new account, restore will be done from current estimated blockchain height."); - wrt << tr("Use --restore-height if you want to restore an already setup account from a specific height"); + wrt << tr("No restore height is specified.") << " "; + wrt << tr("Assumed you are creating a new account, restore will be done from current estimated blockchain height.") << " "; + wrt << tr("Use --restore-height or --restore-date if you want to restore an already setup account from a specific height."); } - std::string confirm = input_line(tr("Is this okay? (Y/Yes/N/No): ")); + std::string confirm = input_line(tr("Is this okay?"), true); if (std::cin.eof() || !command_line::is_yes(confirm)) CHECK_AND_ASSERT_MES(false, false, tr("account creation aborted")); - m_wallet->set_refresh_from_block_height(m_wallet->estimate_blockchain_height()-1); + m_wallet->set_refresh_from_block_height(m_wallet->estimate_blockchain_height()); m_wallet->explicit_refresh_from_block_height(true); m_restore_height = m_wallet->get_refresh_from_block_height(); } @@ -3729,7 +3769,26 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) if (m_restoring && m_generate_from_json.empty() && m_generate_from_device.empty()) { - m_wallet->explicit_refresh_from_block_height(!command_line::is_arg_defaulted(vm, arg_restore_height)); + m_wallet->explicit_refresh_from_block_height(!(command_line::is_arg_defaulted(vm, arg_restore_height) || + command_line::is_arg_defaulted(vm, arg_restore_date))); + if (command_line::is_arg_defaulted(vm, arg_restore_height) && !command_line::is_arg_defaulted(vm, arg_restore_date)) + { + uint16_t year; + uint8_t month; + uint8_t day; + if (!datestr_to_int(m_restore_date, year, month, day)) + return false; + try + { + m_restore_height = m_wallet->get_blockchain_height_by_date(year, month, day); + success_msg_writer() << tr("Restore height is: ") << m_restore_height; + } + catch (const std::runtime_error& e) + { + fail_msg_writer() << e.what(); + return false; + } + } } if (!m_wallet->explicit_refresh_from_block_height() && m_restoring) { @@ -3739,9 +3798,9 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) { std::string heightstr; if (!connected || version < MAKE_CORE_RPC_VERSION(1, 6)) - heightstr = input_line("Restore from specific blockchain height (optional, default 0): "); + heightstr = input_line("Restore from specific blockchain height (optional, default 0)"); else - heightstr = input_line("Restore from specific blockchain height (optional, default 0),\nor alternatively from specific date (YYYY-MM-DD): "); + heightstr = input_line("Restore from specific blockchain height (optional, default 0),\nor alternatively from specific date (YYYY-MM-DD)"); if (std::cin.eof()) return false; if (heightstr.empty()) @@ -3761,23 +3820,16 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) fail_msg_writer() << tr("bad m_restore_height parameter: ") << heightstr; continue; } - if (heightstr.size() != 10 || heightstr[4] != '-' || heightstr[7] != '-') - { - fail_msg_writer() << tr("date format must be YYYY-MM-DD"); - continue; - } uint16_t year; uint8_t month; // 1, 2, ..., 12 uint8_t day; // 1, 2, ..., 31 try { - year = boost::lexical_cast<uint16_t>(heightstr.substr(0,4)); - // lexical_cast<uint8_t> won't work because uint8_t is treated as character type - month = boost::lexical_cast<uint16_t>(heightstr.substr(5,2)); - day = boost::lexical_cast<uint16_t>(heightstr.substr(8,2)); + if (!datestr_to_int(heightstr, year, month, day)) + return false; m_restore_height = m_wallet->get_blockchain_height_by_date(year, month, day); success_msg_writer() << tr("Restore height is: ") << m_restore_height; - std::string confirm = input_line(tr("Is this okay? (Y/Yes/N/No): ")); + std::string confirm = input_line(tr("Is this okay?"), true); if (std::cin.eof()) return false; if(command_line::is_yes(confirm)) @@ -3800,7 +3852,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) if (m_restore_height >= estimate_height) { success_msg_writer() << tr("Restore height ") << m_restore_height << (" is not yet reached. The current estimated height is ") << estimate_height; - std::string confirm = input_line(tr("Still apply restore height? (Y/Yes/N/No): ")); + std::string confirm = input_line(tr("Still apply restore height?"), true); if (std::cin.eof() || command_line::is_no(confirm)) m_restore_height = 0; } @@ -3861,9 +3913,11 @@ bool simple_wallet::handle_command_line(const boost::program_options::variables_ m_non_deterministic = command_line::get_arg(vm, arg_non_deterministic); m_allow_mismatched_daemon_version = command_line::get_arg(vm, arg_allow_mismatched_daemon_version); m_restore_height = command_line::get_arg(vm, arg_restore_height); + m_restore_date = command_line::get_arg(vm, arg_restore_date); m_do_not_relay = command_line::get_arg(vm, arg_do_not_relay); m_subaddress_lookahead = command_line::get_arg(vm, arg_subaddress_lookahead); m_use_english_language_names = command_line::get_arg(vm, arg_use_english_language_names); + m_long_payment_id_support = command_line::get_arg(vm, arg_long_payment_id_support); m_restoring = !m_generate_from_view_key.empty() || !m_generate_from_spend_key.empty() || !m_generate_from_keys.empty() || @@ -3873,6 +3927,14 @@ bool simple_wallet::handle_command_line(const boost::program_options::variables_ m_restore_deterministic_wallet || m_restore_multisig_wallet; + if (!command_line::is_arg_defaulted(vm, arg_restore_date)) + { + uint16_t year; + uint8_t month, day; + if (!datestr_to_int(m_restore_date, year, month, day)) + return false; + } + return true; } //---------------------------------------------------------------------------------------------------- @@ -3923,7 +3985,7 @@ std::string simple_wallet::get_mnemonic_language() } while (language_number < 0) { - language_choice = input_line(tr("Enter the number corresponding to the language of your choice: ")); + language_choice = input_line(tr("Enter the number corresponding to the language of your choice")); if (std::cin.eof()) return std::string(); try @@ -4562,14 +4624,10 @@ void simple_wallet::on_money_received(uint64_t height, const crypto::hash &txid, tx_extra_nonce extra_nonce; if (find_tx_extra_field_by_type(tx_extra_fields, extra_nonce)) { - crypto::hash8 payment_id8 = crypto::null_hash8; crypto::hash payment_id = crypto::null_hash; - if (get_encrypted_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id8)) - message_writer() << - tr("NOTE: this transaction uses an encrypted payment ID: consider using subaddresses instead"); - else if (get_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id)) + if (get_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id)) message_writer(console_color_red, false) << - tr("WARNING: this transaction uses an unencrypted payment ID: consider using subaddresses instead"); + (m_long_payment_id_support ? tr("WARNING: this transaction uses an unencrypted payment ID: consider using subaddresses instead.") : tr("WARNING: this transaction uses an unencrypted payment ID: these are obsolete. Support will be withdrawn in the future. Use subaddresses instead.")); } } if (m_auto_refresh_refreshing) @@ -5268,6 +5326,8 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri bool r = true; if (tools::wallet2::parse_long_payment_id(payment_id_str, payment_id)) { + LONG_PAYMENT_ID_SUPPORT_CHECK(); + std::string extra_nonce; set_payment_id_to_tx_extra_nonce(extra_nonce, payment_id); r = add_extra_nonce_to_tx_extra(extra, extra_nonce); @@ -5275,19 +5335,6 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri payment_id_seen = true; message_writer() << tr("Unencrypted payment IDs are bad for privacy: ask the recipient to use subaddresses instead"); } - else - { - crypto::hash8 payment_id8; - if (tools::wallet2::parse_short_payment_id(payment_id_str, payment_id8)) - { - std::string extra_nonce; - set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, payment_id8); - r = add_extra_nonce_to_tx_extra(extra, extra_nonce); - local_args.pop_back(); - payment_id_seen = true; - } - } - if(!r) { fail_msg_writer() << tr("payment id failed to encode"); @@ -5343,6 +5390,7 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri info.has_payment_id = true; } de.amount = amount; + de.original = local_args[i]; ++i; } else if (i + 1 < local_args.size()) @@ -5355,6 +5403,7 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri ", " << tr("expected number from 0 to ") << print_money(std::numeric_limits<uint64_t>::max()); return false; } + de.original = local_args[i]; i += 2; } else @@ -5373,6 +5422,7 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri } de.addr = info.address; de.is_subaddress = info.is_subaddress; + de.is_integrated = info.has_payment_id; num_subaddresses += info.is_subaddress; if (info.has_payment_id || !payment_id_uri.empty()) @@ -5391,6 +5441,7 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri } else if (tools::wallet2::parse_payment_id(payment_id_uri, payment_id)) { + LONG_PAYMENT_ID_SUPPORT_CHECK(); set_payment_id_to_tx_extra_nonce(extra_nonce, payment_id); message_writer() << tr("Unencrypted payment IDs are bad for privacy: ask the recipient to use subaddresses instead"); } @@ -5412,9 +5463,9 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri } // prompt is there is no payment id and confirmation is required - if (!payment_id_seen && m_wallet->confirm_missing_payment_id() && dsts.size() > num_subaddresses) + if (m_long_payment_id_support && !payment_id_seen && m_wallet->confirm_missing_payment_id() && dsts.size() > num_subaddresses) { - std::string accepted = input_line(tr("No payment id is included with this transaction. Is this okay? (Y/Yes/N/No): ")); + std::string accepted = input_line(tr("No payment id is included with this transaction. Is this okay?"), true); if (std::cin.eof()) return false; if (!command_line::is_yes(accepted)) @@ -5478,23 +5529,23 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri std::vector<std::pair<uint64_t, uint64_t>> nblocks = m_wallet->estimate_backlog({std::make_pair(worst_fee_per_byte, worst_fee_per_byte)}); if (nblocks.size() != 1) { - prompt << "Internal error checking for backlog. " << tr("Is this okay anyway? (Y/Yes/N/No): "); + prompt << "Internal error checking for backlog. " << tr("Is this okay anyway?"); } else { if (nblocks[0].first > m_wallet->get_confirm_backlog_threshold()) - prompt << (boost::format(tr("There is currently a %u block backlog at that fee level. Is this okay? (Y/Yes/N/No): ")) % nblocks[0].first).str(); + prompt << (boost::format(tr("There is currently a %u block backlog at that fee level. Is this okay?")) % nblocks[0].first).str(); } } catch (const std::exception &e) { - prompt << tr("Failed to check for backlog: ") << e.what() << ENDL << tr("Is this okay anyway? (Y/Yes/N/No): "); + prompt << tr("Failed to check for backlog: ") << e.what() << ENDL << tr("Is this okay anyway?"); } std::string prompt_str = prompt.str(); if (!prompt_str.empty()) { - std::string accepted = input_line(prompt_str); + std::string accepted = input_line(prompt_str, true); if (std::cin.eof()) return false; if (!command_line::is_yes(accepted)) @@ -5580,9 +5631,9 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri { prompt << tr("WARNING: this is a non default ring size, which may harm your privacy. Default is recommended."); } - prompt << ENDL << tr("Is this okay? (Y/Yes/N/No): "); + prompt << ENDL << tr("Is this okay?"); - std::string accepted = input_line(prompt.str()); + std::string accepted = input_line(prompt.str(), true); if (std::cin.eof()) return false; if (!command_line::is_yes(accepted)) @@ -5720,17 +5771,17 @@ bool simple_wallet::sweep_unmixable(const std::vector<std::string> &args_) std::string prompt_str = tr("Sweeping ") + print_money(total_unmixable); if (ptx_vector.size() > 1) { - prompt_str = (boost::format(tr("Sweeping %s in %llu transactions for a total fee of %s. Is this okay? (Y/Yes/N/No): ")) % + prompt_str = (boost::format(tr("Sweeping %s in %llu transactions for a total fee of %s. Is this okay?")) % print_money(total_unmixable) % ((unsigned long long)ptx_vector.size()) % print_money(total_fee)).str(); } else { - prompt_str = (boost::format(tr("Sweeping %s for a total fee of %s. Is this okay? (Y/Yes/N/No): ")) % + prompt_str = (boost::format(tr("Sweeping %s for a total fee of %s. Is this okay?")) % print_money(total_unmixable) % print_money(total_fee)).str(); } - std::string accepted = input_line(prompt_str); + std::string accepted = input_line(prompt_str, true); if (std::cin.eof()) return true; if (!command_line::is_yes(accepted)) @@ -5773,7 +5824,7 @@ bool simple_wallet::sweep_unmixable(const std::vector<std::string> &args_) catch (const tools::error::not_enough_unlocked_money& e) { fail_msg_writer() << tr("Not enough money in unlocked balance"); - std::string accepted = input_line((boost::format(tr("Discarding %s of unmixable outputs that cannot be spent, which can be undone by \"rescan_spent\". Is this okay? (Y/Yes/N/No): ")) % print_money(e.available())).str()); + std::string accepted = input_line((boost::format(tr("Discarding %s of unmixable outputs that cannot be spent, which can be undone by \"rescan_spent\". Is this okay?")) % print_money(e.available())).str(), true); if (std::cin.eof()) return true; if (command_line::is_yes(accepted)) @@ -5935,23 +5986,13 @@ bool simple_wallet::sweep_main(uint64_t below, bool locked, const std::vector<st bool r = tools::wallet2::parse_long_payment_id(payment_id_str, payment_id); if(r) { + LONG_PAYMENT_ID_SUPPORT_CHECK(); + std::string extra_nonce; set_payment_id_to_tx_extra_nonce(extra_nonce, payment_id); r = add_extra_nonce_to_tx_extra(extra, extra_nonce); payment_id_seen = true; } - else - { - crypto::hash8 payment_id8; - r = tools::wallet2::parse_short_payment_id(payment_id_str, payment_id8); - if(r) - { - std::string extra_nonce; - set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, payment_id8); - r = add_extra_nonce_to_tx_extra(extra, extra_nonce); - payment_id_seen = true; - } - } if(!r && local_args.size() == 3) { @@ -5991,9 +6032,9 @@ bool simple_wallet::sweep_main(uint64_t below, bool locked, const std::vector<st } // prompt is there is no payment id and confirmation is required - if (!payment_id_seen && m_wallet->confirm_missing_payment_id() && !info.is_subaddress) + if (m_long_payment_id_support && !payment_id_seen && m_wallet->confirm_missing_payment_id() && !info.is_subaddress) { - std::string accepted = input_line(tr("No payment id is included with this transaction. Is this okay? (Y/Yes/N/No): ")); + std::string accepted = input_line(tr("No payment id is included with this transaction. Is this okay?"), true); if (std::cin.eof()) return true; if (!command_line::is_yes(accepted)) @@ -6041,17 +6082,17 @@ bool simple_wallet::sweep_main(uint64_t below, bool locked, const std::vector<st if (m_wallet->print_ring_members() && !print_ring_members(ptx_vector, prompt)) return true; if (ptx_vector.size() > 1) { - prompt << boost::format(tr("Sweeping %s in %llu transactions for a total fee of %s. Is this okay? (Y/Yes/N/No): ")) % + prompt << boost::format(tr("Sweeping %s in %llu transactions for a total fee of %s. Is this okay?")) % print_money(total_sent) % ((unsigned long long)ptx_vector.size()) % print_money(total_fee); } else { - prompt << boost::format(tr("Sweeping %s for a total fee of %s. Is this okay? (Y/Yes/N/No): ")) % + prompt << boost::format(tr("Sweeping %s for a total fee of %s. Is this okay?")) % print_money(total_sent) % print_money(total_fee); } - std::string accepted = input_line(prompt.str()); + std::string accepted = input_line(prompt.str(), true); if (std::cin.eof()) return true; if (!command_line::is_yes(accepted)) @@ -6202,12 +6243,9 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) std::string extra_nonce; if (tools::wallet2::parse_long_payment_id(local_args.back(), payment_id)) { + LONG_PAYMENT_ID_SUPPORT_CHECK(); set_payment_id_to_tx_extra_nonce(extra_nonce, payment_id); } - else if(tools::wallet2::parse_short_payment_id(local_args.back(), payment_id8)) - { - set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, payment_id8); - } else { fail_msg_writer() << tr("failed to parse Payment ID"); @@ -6263,9 +6301,9 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) } // prompt if there is no payment id and confirmation is required - if (!payment_id_seen && m_wallet->confirm_missing_payment_id() && !info.is_subaddress) + if (m_long_payment_id_support && !payment_id_seen && m_wallet->confirm_missing_payment_id() && !info.is_subaddress) { - std::string accepted = input_line(tr("No payment id is included with this transaction. Is this okay? (Y/Yes/N/No): ")); + std::string accepted = input_line(tr("No payment id is included with this transaction. Is this okay?"), true); if (std::cin.eof()) return true; if (!command_line::is_yes(accepted)) @@ -6307,10 +6345,10 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) std::ostringstream prompt; if (!print_ring_members(ptx_vector, prompt)) return true; - prompt << boost::format(tr("Sweeping %s for a total fee of %s. Is this okay? (Y/Yes/N/No): ")) % + prompt << boost::format(tr("Sweeping %s for a total fee of %s. Is this okay?")) % print_money(total_sent) % print_money(total_fee); - std::string accepted = input_line(prompt.str()); + std::string accepted = input_line(prompt.str(), true); if (std::cin.eof()) return true; if (!command_line::is_yes(accepted)) @@ -6471,14 +6509,29 @@ bool simple_wallet::accept_loaded_tx(const std::function<size_t()> get_num_txes, { if (!payment_id_string.empty()) payment_id_string += ", "; - payment_id_string = std::string("encrypted payment ID ") + epee::string_tools::pod_to_hex(payment_id8); - has_encrypted_payment_id = true; + + // if none of the addresses are integrated addresses, it's a dummy one + bool is_dummy = true; + for (const auto &e: cd.dests) + if (e.is_integrated) + is_dummy = false; + + if (is_dummy) + { + payment_id_string += std::string("dummy encrypted payment ID"); + } + else + { + payment_id_string += std::string("encrypted payment ID ") + epee::string_tools::pod_to_hex(payment_id8); + has_encrypted_payment_id = true; + } } else if (get_payment_id_from_tx_extra_nonce(extra_nonce.nonce, payment_id)) { if (!payment_id_string.empty()) payment_id_string += ", "; - payment_id_string = std::string("unencrypted payment ID ") + epee::string_tools::pod_to_hex(payment_id); + payment_id_string += std::string("unencrypted payment ID ") + epee::string_tools::pod_to_hex(payment_id); + payment_id_string += " (OBSOLETE)"; } } } @@ -6574,8 +6627,8 @@ bool simple_wallet::accept_loaded_tx(const std::function<size_t()> get_num_txes, change_string += tr("no change"); uint64_t fee = amount - amount_to_dests; - std::string prompt_str = (boost::format(tr("Loaded %lu transactions, for %s, fee %s, %s, %s, with min ring size %lu, %s. %sIs this okay? (Y/Yes/N/No): ")) % (unsigned long)get_num_txes() % print_money(amount) % print_money(fee) % dest_string % change_string % (unsigned long)min_ring_size % payment_id_string % extra_message).str(); - return command_line::is_yes(input_line(prompt_str)); + std::string prompt_str = (boost::format(tr("Loaded %lu transactions, for %s, fee %s, %s, %s, with min ring size %lu, %s. %sIs this okay?")) % (unsigned long)get_num_txes() % print_money(amount) % print_money(fee) % dest_string % change_string % (unsigned long)min_ring_size % payment_id_string % extra_message).str(); + return command_line::is_yes(input_line(prompt_str, true)); } //---------------------------------------------------------------------------------------------------- bool simple_wallet::accept_loaded_tx(const tools::wallet2::unsigned_tx_set &txs) @@ -7296,6 +7349,7 @@ bool simple_wallet::get_transfers(std::vector<std::string>& local_args, std::vec const std::string type = pd.m_coinbase ? tr("block") : tr("in"); const bool unlocked = m_wallet->is_transfer_unlocked(pd.m_unlock_time, pd.m_block_height); transfers.push_back({ + type, pd.m_block_height, pd.m_timestamp, type, @@ -7328,6 +7382,7 @@ bool simple_wallet::get_transfers(std::vector<std::string>& local_args, std::vec payment_id = payment_id.substr(0,16); std::string note = m_wallet->get_tx_note(i->first); transfers.push_back({ + "out", pd.m_block_height, pd.m_timestamp, "out", @@ -7365,6 +7420,7 @@ bool simple_wallet::get_transfers(std::vector<std::string>& local_args, std::vec double_spend_note = tr("[Double spend seen on the network: this transaction may or may not end up being mined] "); transfers.push_back({ "pool", + "pool", pd.m_timestamp, "in", false, @@ -7405,6 +7461,7 @@ bool simple_wallet::get_transfers(std::vector<std::string>& local_args, std::vec if ((failed && is_failed) || (!is_failed && pending)) { transfers.push_back({ (is_failed ? "failed" : "pending"), + (is_failed ? "failed" : "pending"), pd.m_timestamp, "out", false, @@ -7452,7 +7509,7 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_) for (const auto& transfer : all_transfers) { - const auto color = transfer.confirmed ? ((transfer.direction == "in" || transfer.direction == "block") ? console_color_green : console_color_magenta) : console_color_white; + const auto color = transfer.type == "failed" ? console_color_red : transfer.confirmed ? ((transfer.direction == "in" || transfer.direction == "block") ? console_color_green : console_color_magenta) : console_color_default; std::string destinations = "-"; if (!transfer.outputs.empty()) @@ -7729,7 +7786,7 @@ bool simple_wallet::rescan_blockchain(const std::vector<std::string> &args_) { message_writer() << tr("Warning: this will lose any information which can not be recovered from the blockchain."); message_writer() << tr("This includes destination addresses, tx secret keys, tx notes, etc"); - std::string confirm = input_line(tr("Rescan anyway ? (Y/Yes/N/No): ")); + std::string confirm = input_line(tr("Rescan anyway?"), true); if(!std::cin.eof()) { if (!command_line::is_yes(confirm)) @@ -7770,8 +7827,9 @@ void simple_wallet::wallet_idle_thread() try { uint64_t fetched_blocks; + bool received_money; if (try_connect_to_daemon(true)) - m_wallet->refresh(m_wallet->is_trusted_daemon(), 0, fetched_blocks); + m_wallet->refresh(m_wallet->is_trusted_daemon(), 0, fetched_blocks, received_money, false); // don't check the pool in background mode } catch(...) {} m_auto_refresh_refreshing = false; @@ -8203,12 +8261,13 @@ bool simple_wallet::address_book(const std::vector<std::string> &args/* = std::v { if (tools::wallet2::parse_long_payment_id(args[3], payment_id)) { + LONG_PAYMENT_ID_SUPPORT_CHECK(); description_start += 2; } else if (tools::wallet2::parse_short_payment_id(args[3], info.payment_id)) { - memcpy(payment_id.data, info.payment_id.data, 8); - description_start += 2; + fail_msg_writer() << tr("Short payment IDs are to be used within an integrated address only"); + return true; } else { @@ -8246,7 +8305,7 @@ bool simple_wallet::address_book(const std::vector<std::string> &args/* = std::v auto& row = address_book[i]; success_msg_writer() << tr("Index: ") << i; success_msg_writer() << tr("Address: ") << get_account_address_as_str(m_wallet->nettype(), row.m_is_subaddress, row.m_address); - success_msg_writer() << tr("Payment ID: ") << row.m_payment_id; + success_msg_writer() << tr("Payment ID: ") << row.m_payment_id << " (OBSOLETE)"; success_msg_writer() << tr("Description: ") << row.m_description << "\n"; } } @@ -8924,10 +8983,12 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_params, arg_electrum_seed ); command_line::add_arg(desc_params, arg_allow_mismatched_daemon_version); command_line::add_arg(desc_params, arg_restore_height); + command_line::add_arg(desc_params, arg_restore_date); command_line::add_arg(desc_params, arg_do_not_relay); command_line::add_arg(desc_params, arg_create_address_file); command_line::add_arg(desc_params, arg_subaddress_lookahead); command_line::add_arg(desc_params, arg_use_english_language_names); + command_line::add_arg(desc_params, arg_long_payment_id_support); po::positional_options_description positional_options; positional_options.add(arg_command.name, -1); diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h index e49da8c18..c3dc16d96 100644 --- a/src/simplewallet/simplewallet.h +++ b/src/simplewallet/simplewallet.h @@ -256,6 +256,7 @@ namespace cryptonote struct transfer_view { + std::string type; boost::variant<uint64_t, std::string> block; uint64_t timestamp; std::string direction; @@ -369,6 +370,7 @@ namespace cryptonote std::string m_mnemonic_language; std::string m_import_path; std::string m_subaddress_lookahead; + std::string m_restore_date; // optional - converted to m_restore_height epee::wipeable_string m_electrum_seed; // electrum-style seed parameter @@ -396,6 +398,8 @@ namespace cryptonote bool m_auto_refresh_refreshing; std::atomic<bool> m_in_manual_refresh; uint32_t m_current_subaddress_account; + + bool m_long_payment_id_support; // MMS mms::message_store& get_message_store() const { return m_wallet->get_message_store(); }; diff --git a/src/wallet/api/wallet.cpp b/src/wallet/api/wallet.cpp index 785d2bf36..935a8d51c 100644 --- a/src/wallet/api/wallet.cpp +++ b/src/wallet/api/wallet.cpp @@ -942,6 +942,12 @@ uint64_t WalletImpl::approximateBlockChainHeight() const { return m_wallet->get_approximate_blockchain_height(); } + +uint64_t WalletImpl::estimateBlockChainHeight() const +{ + return m_wallet->estimate_blockchain_height(); +} + uint64_t WalletImpl::daemonBlockChainHeight() const { if(m_wallet->light_wallet()) { @@ -1401,9 +1407,11 @@ PendingTransaction *WalletImpl::createTransaction(const string &dst_addr, const if (amount) { vector<cryptonote::tx_destination_entry> dsts; cryptonote::tx_destination_entry de; + de.original = dst_addr; de.addr = info.address; de.amount = *amount; de.is_subaddress = info.is_subaddress; + de.is_integrated = info.has_payment_id; dsts.push_back(de); transaction->m_pending_tx = m_wallet->create_transactions_2(dsts, fake_outs_count, 0 /* unlock_time */, adjusted_priority, diff --git a/src/wallet/api/wallet.h b/src/wallet/api/wallet.h index b4637b8e6..55240d64f 100644 --- a/src/wallet/api/wallet.h +++ b/src/wallet/api/wallet.h @@ -109,6 +109,7 @@ public: uint64_t unlockedBalance(uint32_t accountIndex = 0) const override; uint64_t blockChainHeight() const override; uint64_t approximateBlockChainHeight() const override; + uint64_t estimateBlockChainHeight() const override; uint64_t daemonBlockChainHeight() const override; uint64_t daemonBlockChainTargetHeight() const override; bool synchronized() const override; diff --git a/src/wallet/api/wallet2_api.h b/src/wallet/api/wallet2_api.h index 82627de29..5c301974f 100644 --- a/src/wallet/api/wallet2_api.h +++ b/src/wallet/api/wallet2_api.h @@ -575,6 +575,12 @@ struct Wallet virtual uint64_t approximateBlockChainHeight() const = 0; /** + * @brief estimateBlockChainHeight - returns estimate blockchain height. More accurate than approximateBlockChainHeight, + * uses daemon height and falls back to calculation from date/time + * @return + **/ + virtual uint64_t estimateBlockChainHeight() const = 0; + /** * @brief daemonBlockChainHeight - returns daemon blockchain height * @return 0 - in case error communicating with the daemon. * status() will return Status_Error and errorString() will return verbose error description diff --git a/src/wallet/api/wallet_manager.cpp b/src/wallet/api/wallet_manager.cpp index 5b262f1b7..89fe01c0d 100644 --- a/src/wallet/api/wallet_manager.cpp +++ b/src/wallet/api/wallet_manager.cpp @@ -127,6 +127,8 @@ Wallet *WalletManagerImpl::createWalletFromDevice(const std::string &path, WalletImpl * wallet = new WalletImpl(nettype, kdf_rounds); if(restoreHeight > 0){ wallet->setRefreshFromBlockHeight(restoreHeight); + } else { + wallet->setRefreshFromBlockHeight(wallet->estimateBlockChainHeight()); } auto lookahead = tools::parse_subaddress_lookahead(subaddressLookahead); if (lookahead) diff --git a/src/wallet/message_store.h b/src/wallet/message_store.h index 7d26f7889..637bd29a1 100644 --- a/src/wallet/message_store.h +++ b/src/wallet/message_store.h @@ -135,6 +135,7 @@ namespace mms { monero_address_known = false; memset(&monero_address, 0, sizeof(cryptonote::account_public_address)); + me = false; index = 0; auto_config_public_key = crypto::null_pkey; auto_config_secret_key = crypto::null_skey; diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index a0af1e84a..e9237386d 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -192,6 +192,37 @@ namespace return false; } + + void add_reason(std::string &reasons, const char *reason) + { + if (!reasons.empty()) + reasons += ", "; + reasons += reason; + } + + std::string get_text_reason(const cryptonote::COMMAND_RPC_SEND_RAW_TX::response &res) + { + std::string reason; + if (res.low_mixin) + add_reason(reason, "bad ring size"); + if (res.double_spend) + add_reason(reason, "double spend"); + if (res.invalid_input) + add_reason(reason, "invalid input"); + if (res.invalid_output) + add_reason(reason, "invalid output"); + if (res.too_big) + add_reason(reason, "too big"); + if (res.overspend) + add_reason(reason, "overspend"); + if (res.fee_too_low) + add_reason(reason, "fee too low"); + if (res.not_rct) + add_reason(reason, "tx is not ringct"); + if (res.not_relayed) + add_reason(reason, "tx was not relayed"); + return reason; + } } namespace @@ -583,19 +614,6 @@ std::pair<std::unique_ptr<tools::wallet2>, tools::password_container> generate_f return {nullptr, tools::password_container{}}; } -static void throw_on_rpc_response_error(const boost::optional<std::string> &status, const char *method) -{ - // no error - if (!status) - return; - - // empty string -> not connection - THROW_WALLET_EXCEPTION_IF(status->empty(), tools::error::no_connection_to_daemon, method); - - THROW_WALLET_EXCEPTION_IF(*status == CORE_RPC_STATUS_BUSY, tools::error::daemon_busy, method); - THROW_WALLET_EXCEPTION_IF(*status != CORE_RPC_STATUS_OK, tools::error::wallet_generic_rpc_error, method, *status); -} - std::string strjoin(const std::vector<size_t> &V, const char *sep) { std::stringstream ss; @@ -797,6 +815,43 @@ static void setup_shim(hw::wallet_shim * shim, tools::wallet2 * wallet) shim->get_tx_pub_key_from_received_outs = boost::bind(&tools::wallet2::get_tx_pub_key_from_received_outs, wallet, _1); } +bool get_pruned_tx(const cryptonote::COMMAND_RPC_GET_TRANSACTIONS::entry &entry, cryptonote::transaction &tx, crypto::hash &tx_hash) +{ + cryptonote::blobdata bd; + + // easy case if we have the whole tx + if (!entry.as_hex.empty() || (!entry.prunable_as_hex.empty() && !entry.pruned_as_hex.empty())) + { + CHECK_AND_ASSERT_MES(epee::string_tools::parse_hexstr_to_binbuff(entry.as_hex.empty() ? entry.pruned_as_hex + entry.prunable_as_hex : entry.as_hex, bd), false, "Failed to parse tx data"); + CHECK_AND_ASSERT_MES(cryptonote::parse_and_validate_tx_from_blob(bd, tx), false, "Invalid tx data"); + tx_hash = cryptonote::get_transaction_hash(tx); + // if the hash was given, check it matches + CHECK_AND_ASSERT_MES(entry.tx_hash.empty() || epee::string_tools::pod_to_hex(tx_hash) == entry.tx_hash, false, + "Response claims a different hash than the data yields"); + return true; + } + // case of a pruned tx with its prunable data hash + if (!entry.pruned_as_hex.empty() && !entry.prunable_hash.empty()) + { + crypto::hash ph; + CHECK_AND_ASSERT_MES(epee::string_tools::hex_to_pod(entry.prunable_hash, ph), false, "Failed to parse prunable hash"); + CHECK_AND_ASSERT_MES(epee::string_tools::parse_hexstr_to_binbuff(entry.pruned_as_hex, bd), false, "Failed to parse pruned data"); + CHECK_AND_ASSERT_MES(parse_and_validate_tx_base_from_blob(bd, tx), false, "Invalid base tx data"); + // only v2 txes can calculate their txid after pruned + if (bd[0] > 1) + { + tx_hash = cryptonote::get_pruned_transaction_hash(tx, ph); + } + else + { + // for v1, we trust the dameon + CHECK_AND_ASSERT_MES(epee::string_tools::hex_to_pod(entry.tx_hash, tx_hash), false, "Failed to parse tx hash"); + } + return true; + } + return false; +} + //----------------------------------------------------------------- } //namespace @@ -903,6 +958,7 @@ wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended): m_multisig(false), m_multisig_threshold(0), m_node_rpc_proxy(m_http_client, m_daemon_rpc_mutex), + m_account_public_address{crypto::null_pkey, crypto::null_pkey}, m_subaddress_lookahead_major(SUBADDRESS_LOOKAHEAD_MAJOR), m_subaddress_lookahead_minor(SUBADDRESS_LOOKAHEAD_MINOR), m_light_wallet(false), @@ -919,6 +975,7 @@ wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended): m_last_block_reward(0), m_encrypt_keys_after_refresh(boost::none), m_unattended(unattended), + m_devices_registered(false), m_device_last_key_image_sync(0) { } @@ -1353,6 +1410,7 @@ static uint64_t decodeRct(const rct::rctSig & rv, const crypto::key_derivation & { case rct::RCTTypeSimple: case rct::RCTTypeBulletproof: + case rct::RCTTypeBulletproof2: return rct::decodeRctSimple(rv, rct::sk2rct(scalar1), i, mask, hwdev); case rct::RCTTypeFull: return rct::decodeRct(rv, rct::sk2rct(scalar1), i, mask, hwdev); @@ -1368,7 +1426,7 @@ static uint64_t decodeRct(const rct::rctSig & rv, const crypto::key_derivation & } } //---------------------------------------------------------------------------------------------------- -void wallet2::scan_output(const cryptonote::transaction &tx, const crypto::public_key &tx_pub_key, size_t i, tx_scan_info_t &tx_scan_info, int &num_vouts_received, std::unordered_map<cryptonote::subaddress_index, uint64_t> &tx_money_got_in_outs, std::vector<size_t> &outs) +void wallet2::scan_output(const cryptonote::transaction &tx, const crypto::public_key &tx_pub_key, size_t i, tx_scan_info_t &tx_scan_info, int &num_vouts_received, std::unordered_map<cryptonote::subaddress_index, uint64_t> &tx_money_got_in_outs, std::vector<size_t> &outs, bool pool) { THROW_WALLET_EXCEPTION_IF(i >= tx.vout.size(), error::wallet_internal_error, "Invalid vout index"); @@ -1379,7 +1437,7 @@ void wallet2::scan_output(const cryptonote::transaction &tx, const crypto::publi CRITICAL_REGION_LOCAL(password_lock); if (!m_encrypt_keys_after_refresh) { - boost::optional<epee::wipeable_string> pwd = m_callback->on_get_password("output received"); + boost::optional<epee::wipeable_string> pwd = m_callback->on_get_password(pool ? "output found in pool" : "output received"); THROW_WALLET_EXCEPTION_IF(!pwd, error::password_needed, tr("Password is needed to compute key image for incoming monero")); THROW_WALLET_EXCEPTION_IF(!verify_password(*pwd), error::password_needed, tr("Invalid password: password is needed to compute key image for incoming monero")); decrypt_keys(*pwd); @@ -1583,7 +1641,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote if (tx_scan_info[i].received) { hwdev.conceal_derivation(tx_scan_info[i].received->derivation, tx_pub_key, additional_tx_pub_keys.data, derivation, additional_derivations); - scan_output(tx, tx_pub_key, i, tx_scan_info[i], num_vouts_received, tx_money_got_in_outs, outs); + scan_output(tx, tx_pub_key, i, tx_scan_info[i], num_vouts_received, tx_money_got_in_outs, outs, pool); } } } @@ -1606,7 +1664,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote if (tx_scan_info[i].received) { hwdev.conceal_derivation(tx_scan_info[i].received->derivation, tx_pub_key, additional_tx_pub_keys.data, derivation, additional_derivations); - scan_output(tx, tx_pub_key, i, tx_scan_info[i], num_vouts_received, tx_money_got_in_outs, outs); + scan_output(tx, tx_pub_key, i, tx_scan_info[i], num_vouts_received, tx_money_got_in_outs, outs, pool); } } } @@ -1622,7 +1680,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote boost::unique_lock<hw::device> hwdev_lock (hwdev); hwdev.set_mode(hw::device::NONE); hwdev.conceal_derivation(tx_scan_info[i].received->derivation, tx_pub_key, additional_tx_pub_keys.data, derivation, additional_derivations); - scan_output(tx, tx_pub_key, i, tx_scan_info[i], num_vouts_received, tx_money_got_in_outs, outs); + scan_output(tx, tx_pub_key, i, tx_scan_info[i], num_vouts_received, tx_money_got_in_outs, outs, pool); } } } @@ -1662,7 +1720,25 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote td.m_txid = txid; td.m_key_image = tx_scan_info[o].ki; td.m_key_image_known = !m_watch_only && !m_multisig; - td.m_key_image_requested = false; + if (!td.m_key_image_known) + { + // we might have cold signed, and have a mapping to key images + std::unordered_map<crypto::public_key, crypto::key_image>::const_iterator i = m_cold_key_images.find(tx_scan_info[o].in_ephemeral.pub); + if (i != m_cold_key_images.end()) + { + td.m_key_image = i->second; + td.m_key_image_known = true; + } + } + if (m_watch_only) + { + // for view wallets, that flag means "we want to request it" + td.m_key_image_request = true; + } + else + { + td.m_key_image_request = false; + } td.m_key_image_partial = m_multisig; td.m_amount = amount; td.m_pk_index = pk_index - 1; @@ -1684,7 +1760,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote td.m_rct = false; } set_unspent(m_transfers.size()-1); - if (!m_multisig && !m_watch_only) + if (td.m_key_image_known) m_key_images[td.m_key_image] = m_transfers.size()-1; m_pub_keys[tx_scan_info[o].in_ephemeral.pub] = m_transfers.size()-1; if (output_tracker_cache) @@ -2131,7 +2207,7 @@ void wallet2::pull_blocks(uint64_t start_height, uint64_t &blocks_start_height, m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "getblocks.bin"); THROW_WALLET_EXCEPTION_IF(res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "getblocks.bin"); - THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::get_blocks_error, res.status); + THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::get_blocks_error, get_rpc_status(res.status)); THROW_WALLET_EXCEPTION_IF(res.blocks.size() != res.output_indices.size(), error::wallet_internal_error, "mismatched blocks (" + boost::lexical_cast<std::string>(res.blocks.size()) + ") and output_indices (" + boost::lexical_cast<std::string>(res.output_indices.size()) + ") sizes from daemon"); @@ -2153,7 +2229,7 @@ void wallet2::pull_hashes(uint64_t start_height, uint64_t &blocks_start_height, m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "gethashes.bin"); THROW_WALLET_EXCEPTION_IF(res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "gethashes.bin"); - THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::get_hashes_error, res.status); + THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::get_hashes_error, get_rpc_status(res.status)); blocks_start_height = res.start_height; hashes = std::move(res.m_block_ids); @@ -2198,7 +2274,6 @@ void wallet2::process_parsed_blocks(uint64_t start_height, const std::vector<cry const cryptonote::account_keys &keys = m_account.get_keys(); auto gender = [&](wallet2::is_out_data &iod) { - boost::unique_lock<hw::device> hwdev_lock(hwdev); if (!hwdev.generate_key_derivation(iod.pkey, keys.m_view_secret_key, iod.derivation)) { MWARNING("Failed to generate key derivation from tx pubkey, skipping"); @@ -2207,12 +2282,16 @@ void wallet2::process_parsed_blocks(uint64_t start_height, const std::vector<cry } }; - for (auto &slot: tx_cache_data) + for (size_t i = 0; i < tx_cache_data.size(); ++i) { - for (auto &iod: slot.primary) - tpool.submit(&waiter, [&gender, &iod]() { gender(iod); }, true); - for (auto &iod: slot.additional) - tpool.submit(&waiter, [&gender, &iod]() { gender(iod); }, true); + tpool.submit(&waiter, [&hwdev, &gender, &tx_cache_data, i]() { + auto &slot = tx_cache_data[i]; + boost::unique_lock<hw::device> hwdev_lock(hwdev); + for (auto &iod: slot.primary) + gender(iod); + for (auto &iod: slot.additional) + gender(iod); + }, true); } waiter.wait(&tpool); @@ -2312,11 +2391,10 @@ void wallet2::pull_and_parse_next_blocks(uint64_t start_height, uint64_t &blocks THROW_WALLET_EXCEPTION_IF(prev_blocks.size() != prev_parsed_blocks.size(), error::wallet_internal_error, "size mismatch"); // prepend the last 3 blocks, should be enough to guard against a block or two's reorg - std::vector<parsed_block>::const_reverse_iterator i = prev_parsed_blocks.rbegin(); - for (size_t n = 0; n < std::min((size_t)3, prev_parsed_blocks.size()); ++n) + auto s = std::next(prev_parsed_blocks.rbegin(), std::min((size_t)3, prev_parsed_blocks.size())).base(); + for (; s != prev_parsed_blocks.end(); ++s) { - short_chain_history.push_front(i->hash); - ++i; + short_chain_history.push_front(s->hash); } // pull the new blocks @@ -2555,7 +2633,7 @@ void wallet2::update_pool_state(bool refreshed) req.txs_hashes.push_back(epee::string_tools::pod_to_hex(p.first)); MDEBUG("asking for " << txids.size() << " transactions"); req.decode_as_json = false; - req.prune = false; + req.prune = true; m_daemon_rpc_mutex.lock(); bool r = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); @@ -2570,11 +2648,10 @@ void wallet2::update_pool_state(bool refreshed) { cryptonote::transaction tx; cryptonote::blobdata bd; - crypto::hash tx_hash, tx_prefix_hash; - if (epee::string_tools::parse_hexstr_to_binbuff(tx_entry.as_hex, bd)) + crypto::hash tx_hash; + + if (get_pruned_tx(tx_entry, tx, tx_hash)) { - if (cryptonote::parse_and_validate_tx_from_blob(bd, tx, tx_hash, tx_prefix_hash)) - { const std::vector<std::pair<crypto::hash, bool>>::const_iterator i = std::find_if(txids.begin(), txids.end(), [tx_hash](const std::pair<crypto::hash, bool> &e) { return e.first == tx_hash; }); if (i != txids.end()) @@ -2591,11 +2668,6 @@ void wallet2::update_pool_state(bool refreshed) { MERROR("Got txid " << tx_hash << " which we did not ask for"); } - } - else - { - LOG_PRINT_L0("failed to validate transaction from daemon"); - } } else { @@ -2615,7 +2687,7 @@ void wallet2::update_pool_state(bool refreshed) } else { - LOG_PRINT_L0("Error calling gettransactions daemon RPC: r " << r << ", status " << res.status); + LOG_PRINT_L0("Error calling gettransactions daemon RPC: r " << r << ", status " << get_rpc_status(res.status)); } } MTRACE("update_pool_state end"); @@ -2722,7 +2794,7 @@ std::shared_ptr<std::map<std::pair<uint64_t, uint64_t>, size_t>> wallet2::create return cache; } //---------------------------------------------------------------------------------------------------- -void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched, bool& received_money) +void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched, bool& received_money, bool check_pool) { if(m_light_wallet) { @@ -2803,13 +2875,16 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo bool first = true; while(m_run.load(std::memory_order_relaxed)) { + uint64_t next_blocks_start_height; + std::vector<cryptonote::block_complete_entry> next_blocks; + std::vector<parsed_block> next_parsed_blocks; + bool error; try { // pull the next set of blocks while we're processing the current one - uint64_t next_blocks_start_height; - std::vector<cryptonote::block_complete_entry> next_blocks; - std::vector<parsed_block> next_parsed_blocks; - bool error = false; + error = false; + next_blocks.clear(); + next_parsed_blocks.clear(); added_blocks = 0; if (!first && blocks.empty()) { @@ -2847,6 +2922,11 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo start_height = stop_height; throw std::runtime_error(""); // loop again } + catch (const std::exception &e) + { + MERROR("Error parsing blocks: " << e.what()); + error = true; + } blocks_fetched += added_blocks; } waiter.wait(&tpool); @@ -2904,7 +2984,7 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo try { // If stop() is called we don't need to check pending transactions - if(m_run.load(std::memory_order_relaxed)) + if (check_pool && m_run.load(std::memory_order_relaxed)) update_pool_state(refreshed); } catch (...) @@ -4488,6 +4568,23 @@ std::string wallet2::make_multisig(const epee::wipeable_string &password, bool wallet2::finalize_multisig(const epee::wipeable_string &password, const std::unordered_set<crypto::public_key> &pkeys, std::vector<crypto::public_key> signers) { + bool ready; + uint32_t threshold, total; + if (!multisig(&ready, &threshold, &total)) + { + MERROR("This is not a multisig wallet"); + return false; + } + if (ready) + { + MERROR("This multisig wallet is already finalized"); + return false; + } + if (threshold + 1 != total) + { + MERROR("finalize_multisig should only be used for N-1/N wallets, use exchange_multisig_keys instead"); + return false; + } exchange_multisig_keys(password, pkeys, signers); return true; } @@ -5287,7 +5384,7 @@ void wallet2::rescan_spent() m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "is_key_image_spent"); THROW_WALLET_EXCEPTION_IF(daemon_resp.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "is_key_image_spent"); - THROW_WALLET_EXCEPTION_IF(daemon_resp.status != CORE_RPC_STATUS_OK, error::is_key_image_spent_error, daemon_resp.status); + THROW_WALLET_EXCEPTION_IF(daemon_resp.status != CORE_RPC_STATUS_OK, error::is_key_image_spent_error, get_rpc_status(daemon_resp.status)); THROW_WALLET_EXCEPTION_IF(daemon_resp.spent_status.size() != n_outputs, error::wallet_internal_error, "daemon returned wrong response for is_key_image_spent, wrong amounts count = " + std::to_string(daemon_resp.spent_status.size()) + ", expected " + std::to_string(n_outputs)); @@ -5620,7 +5717,7 @@ void wallet2::commit_tx(pending_tx& ptx) m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "submit_raw_tx"); // MyMonero and OpenMonero use different status strings - THROW_WALLET_EXCEPTION_IF(ores.status != "OK" && ores.status != "success" , error::tx_rejected, ptx.tx, ores.status, ores.error); + THROW_WALLET_EXCEPTION_IF(ores.status != "OK" && ores.status != "success" , error::tx_rejected, ptx.tx, get_rpc_status(ores.status), ores.error); } else { @@ -5634,7 +5731,7 @@ void wallet2::commit_tx(pending_tx& ptx) m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "sendrawtransaction"); THROW_WALLET_EXCEPTION_IF(daemon_send_resp.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "sendrawtransaction"); - THROW_WALLET_EXCEPTION_IF(daemon_send_resp.status != CORE_RPC_STATUS_OK, error::tx_rejected, ptx.tx, daemon_send_resp.status, daemon_send_resp.reason); + THROW_WALLET_EXCEPTION_IF(daemon_send_resp.status != CORE_RPC_STATUS_OK, error::tx_rejected, ptx.tx, get_rpc_status(daemon_send_resp.status), get_text_reason(daemon_send_resp)); // sanity checks for (size_t idx: ptx.selected_transfers) { @@ -5831,15 +5928,16 @@ bool wallet2::sign_tx(unsigned_tx_set &exported_txs, std::vector<wallet2::pendin LOG_PRINT_L1(" " << (n+1) << ": " << sd.sources.size() << " inputs, ring size " << sd.sources[0].outputs.size()); signed_txes.ptx.push_back(pending_tx()); tools::wallet2::pending_tx &ptx = signed_txes.ptx.back(); - rct::RangeProofType range_proof_type = rct::RangeProofBorromean; + rct::RCTConfig rct_config = { rct::RangeProofBorromean, 0 }; if (sd.use_bulletproofs) { - range_proof_type = rct::RangeProofPaddedBulletproof; + rct_config.range_proof_type = rct::RangeProofPaddedBulletproof; + rct_config.bp_version = use_fork_rules(HF_VERSION_SMALLER_BP, -10) ? 2 : 1; } crypto::secret_key tx_key; std::vector<crypto::secret_key> additional_tx_keys; rct::multisig_out msout; - bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sd.sources, sd.splitted_dsts, sd.change_dts.addr, sd.extra, ptx.tx, sd.unlock_time, tx_key, additional_tx_keys, sd.use_rct, range_proof_type, m_multisig ? &msout : NULL); + bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sd.sources, sd.splitted_dsts, sd.change_dts.addr, sd.extra, ptx.tx, sd.unlock_time, tx_key, additional_tx_keys, sd.use_rct, rct_config, m_multisig ? &msout : NULL); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sd.sources, sd.splitted_dsts, sd.unlock_time, m_nettype); // we don't test tx size, because we don't know the current limit, due to not having a blockchain, // and it's a bit pointless to fail there anyway, since it'd be a (good) guess only. We sign anyway, @@ -5884,6 +5982,61 @@ bool wallet2::sign_tx(unsigned_tx_set &exported_txs, std::vector<wallet2::pendin txs.back().additional_tx_keys = additional_tx_keys; } + // add key image mapping for these txes + const account_keys &keys = get_account().get_keys(); + hw::device &hwdev = m_account.get_device(); + for (size_t n = 0; n < exported_txs.txes.size(); ++n) + { + const cryptonote::transaction &tx = signed_txes.ptx[n].tx; + + crypto::key_derivation derivation; + std::vector<crypto::key_derivation> additional_derivations; + + // compute public keys from out secret keys + crypto::public_key tx_pub_key; + crypto::secret_key_to_public_key(txs[n].tx_key, tx_pub_key); + std::vector<crypto::public_key> additional_tx_pub_keys; + for (const crypto::secret_key &skey: txs[n].additional_tx_keys) + { + additional_tx_pub_keys.resize(additional_tx_pub_keys.size() + 1); + crypto::secret_key_to_public_key(skey, additional_tx_pub_keys.back()); + } + + // compute derivations + hwdev.set_mode(hw::device::TRANSACTION_PARSE); + if (!hwdev.generate_key_derivation(tx_pub_key, keys.m_view_secret_key, derivation)) + { + MWARNING("Failed to generate key derivation from tx pubkey in " << cryptonote::get_transaction_hash(tx) << ", skipping"); + static_assert(sizeof(derivation) == sizeof(rct::key), "Mismatched sizes of key_derivation and rct::key"); + memcpy(&derivation, rct::identity().bytes, sizeof(derivation)); + } + for (size_t i = 0; i < additional_tx_pub_keys.size(); ++i) + { + additional_derivations.push_back({}); + if (!hwdev.generate_key_derivation(additional_tx_pub_keys[i], keys.m_view_secret_key, additional_derivations.back())) + { + MWARNING("Failed to generate key derivation from additional tx pubkey in " << cryptonote::get_transaction_hash(tx) << ", skipping"); + memcpy(&additional_derivations.back(), rct::identity().bytes, sizeof(crypto::key_derivation)); + } + } + + for (size_t i = 0; i < tx.vout.size(); ++i) + { + if (tx.vout[i].target.type() != typeid(cryptonote::txout_to_key)) + continue; + const cryptonote::txout_to_key &out = boost::get<cryptonote::txout_to_key>(tx.vout[i].target); + // if this output is back to this wallet, we can calculate its key image already + if (!is_out_to_acc_precomp(m_subaddresses, out.key, derivation, additional_derivations, i, hwdev)) + continue; + crypto::key_image ki; + cryptonote::keypair in_ephemeral; + if (generate_key_image_helper(keys, m_subaddresses, out.key, tx_pub_key, additional_tx_pub_keys, i, in_ephemeral, ki, hwdev)) + signed_txes.tx_key_images[out.key] = ki; + else + MERROR("Failed to calculate key image"); + } + } + // add key images signed_txes.key_images.resize(m_transfers.size()); for (size_t i = 0; i < m_transfers.size(); ++i) @@ -6046,6 +6199,10 @@ bool wallet2::parse_tx_from_str(const std::string &signed_tx_st, std::vector<too bool r = import_key_images(signed_txs.key_images); if (!r) return false; + // remember key images for this tx, for when we get those txes from the blockchain + for (const auto &e: signed_txs.tx_key_images) + m_cold_key_images.insert(e); + ptx = signed_txs.ptx; return true; @@ -6242,12 +6399,13 @@ bool wallet2::sign_multisig_tx(multisig_tx_set &exported_txs, std::vector<crypto cryptonote::transaction tx; rct::multisig_out msout = ptx.multisig_sigs.front().msout; auto sources = sd.sources; - rct::RangeProofType range_proof_type = rct::RangeProofBorromean; + rct::RCTConfig rct_config = { rct::RangeProofBorromean, 0 }; if (sd.use_bulletproofs) { - range_proof_type = rct::RangeProofPaddedBulletproof; + rct_config.range_proof_type = rct::RangeProofPaddedBulletproof; + rct_config.bp_version = use_fork_rules(HF_VERSION_SMALLER_BP, -10) ? 2 : 1; } - bool r = cryptonote::construct_tx_with_tx_key(m_account.get_keys(), m_subaddresses, sources, sd.splitted_dsts, ptx.change_dts.addr, sd.extra, tx, sd.unlock_time, ptx.tx_key, ptx.additional_tx_keys, sd.use_rct, range_proof_type, &msout, false); + bool r = cryptonote::construct_tx_with_tx_key(m_account.get_keys(), m_subaddresses, sources, sd.splitted_dsts, ptx.change_dts.addr, sd.extra, tx, sd.unlock_time, ptx.tx_key, ptx.additional_tx_keys, sd.use_rct, rct_config, &msout, false); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sd.sources, sd.splitted_dsts, sd.unlock_time, m_nettype); THROW_WALLET_EXCEPTION_IF(get_transaction_prefix_hash (tx) != get_transaction_prefix_hash(ptx.tx), @@ -6525,7 +6683,7 @@ uint32_t wallet2::adjust_priority(uint32_t priority) m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "getblockheadersrange"); THROW_WALLET_EXCEPTION_IF(getbh_res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "getblockheadersrange"); - THROW_WALLET_EXCEPTION_IF(getbh_res.status != CORE_RPC_STATUS_OK, error::get_blocks_error, getbh_res.status); + THROW_WALLET_EXCEPTION_IF(getbh_res.status != CORE_RPC_STATUS_OK, error::get_blocks_error, get_rpc_status(getbh_res.status)); if (getbh_res.headers.size() != N) { MERROR("Bad blockheaders size"); @@ -6697,11 +6855,12 @@ bool wallet2::find_and_save_rings(bool force) MDEBUG("Found " << std::to_string(txs_hashes.size()) << " transactions"); // get those transactions from the daemon + auto it = txs_hashes.begin(); static const size_t SLICE_SIZE = 200; for (size_t slice = 0; slice < txs_hashes.size(); slice += SLICE_SIZE) { req.decode_as_json = false; - req.prune = false; + req.prune = true; req.txs_hashes.clear(); size_t ntxes = slice + SLICE_SIZE > txs_hashes.size() ? txs_hashes.size() - slice : SLICE_SIZE; for (size_t s = slice; s < slice + ntxes; ++s) @@ -6720,19 +6879,15 @@ bool wallet2::find_and_save_rings(bool force) MDEBUG("Scanning " << res.txs.size() << " transactions"); THROW_WALLET_EXCEPTION_IF(slice + res.txs.size() > txs_hashes.size(), error::wallet_internal_error, "Unexpected tx array size"); - auto it = req.txs_hashes.begin(); for (size_t i = 0; i < res.txs.size(); ++i, ++it) { const auto &tx_info = res.txs[i]; - THROW_WALLET_EXCEPTION_IF(tx_info.tx_hash != epee::string_tools::pod_to_hex(txs_hashes[slice + i]), error::wallet_internal_error, "Wrong txid received"); - THROW_WALLET_EXCEPTION_IF(tx_info.tx_hash != *it, error::wallet_internal_error, "Wrong txid received"); - cryptonote::blobdata bd; - THROW_WALLET_EXCEPTION_IF(!epee::string_tools::parse_hexstr_to_binbuff(tx_info.as_hex, bd), error::wallet_internal_error, "failed to parse tx from hexstr"); - cryptonote::transaction tx; - crypto::hash tx_hash, tx_prefix_hash; - THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(bd, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, "failed to parse tx from blob"); - THROW_WALLET_EXCEPTION_IF(epee::string_tools::pod_to_hex(tx_hash) != tx_info.tx_hash, error::wallet_internal_error, "txid mismatch"); - THROW_WALLET_EXCEPTION_IF(!add_rings(get_ringdb_key(), tx), error::wallet_internal_error, "Failed to save ring"); + cryptonote::transaction tx; + crypto::hash tx_hash; + THROW_WALLET_EXCEPTION_IF(!get_pruned_tx(tx_info, tx, tx_hash), error::wallet_internal_error, + "Failed to get transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(!(tx_hash == *it), error::wallet_internal_error, "Wrong txid received"); + THROW_WALLET_EXCEPTION_IF(!add_rings(get_ringdb_key(), tx), error::wallet_internal_error, "Failed to save ring"); } } @@ -6987,7 +7142,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "transfer_selected"); THROW_WALLET_EXCEPTION_IF(resp_t.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_output_histogram"); - THROW_WALLET_EXCEPTION_IF(resp_t.status != CORE_RPC_STATUS_OK, error::get_histogram_error, resp_t.status); + THROW_WALLET_EXCEPTION_IF(resp_t.status != CORE_RPC_STATUS_OK, error::get_histogram_error, get_rpc_status(resp_t.status)); } // if we want to segregate fake outs pre or post fork, get distribution @@ -7010,7 +7165,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "transfer_selected"); THROW_WALLET_EXCEPTION_IF(resp_t.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_output_distribution"); - THROW_WALLET_EXCEPTION_IF(resp_t.status != CORE_RPC_STATUS_OK, error::get_output_distribution, resp_t.status); + THROW_WALLET_EXCEPTION_IF(resp_t.status != CORE_RPC_STATUS_OK, error::get_output_distribution, get_rpc_status(resp_t.status)); // check we got all data for(size_t idx: selected_transfers) @@ -7409,7 +7564,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "get_outs.bin"); THROW_WALLET_EXCEPTION_IF(daemon_resp.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_outs.bin"); - THROW_WALLET_EXCEPTION_IF(daemon_resp.status != CORE_RPC_STATUS_OK, error::get_outs_error, daemon_resp.status); + THROW_WALLET_EXCEPTION_IF(daemon_resp.status != CORE_RPC_STATUS_OK, error::get_outs_error, get_rpc_status(daemon_resp.status)); THROW_WALLET_EXCEPTION_IF(daemon_resp.outs.size() != req.outputs.size(), error::wallet_internal_error, "daemon returned wrong response for get_outs.bin, wrong amounts count = " + std::to_string(daemon_resp.outs.size()) + ", expected " + std::to_string(req.outputs.size())); @@ -7662,7 +7817,7 @@ void wallet2::transfer_selected(const std::vector<cryptonote::tx_destination_ent std::vector<crypto::secret_key> additional_tx_keys; rct::multisig_out msout; LOG_PRINT_L2("constructing tx"); - bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sources, splitted_dsts, change_dts.addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, rct::RangeProofBulletproof, m_multisig ? &msout : NULL); + bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sources, splitted_dsts, change_dts.addr, extra, tx, unlock_time, tx_key, additional_tx_keys, false, {}, m_multisig ? &msout : NULL); LOG_PRINT_L2("constructed tx, r="<<r); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sources, splitted_dsts, unlock_time, m_nettype); THROW_WALLET_EXCEPTION_IF(upper_transaction_weight_limit <= get_transaction_weight(tx), error::tx_too_big, tx, upper_transaction_weight_limit); @@ -7711,7 +7866,7 @@ void wallet2::transfer_selected(const std::vector<cryptonote::tx_destination_ent void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry> dsts, const std::vector<size_t>& selected_transfers, size_t fake_outputs_count, std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, - uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx &ptx, rct::RangeProofType range_proof_type) + uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx &ptx, const rct::RCTConfig &rct_config) { using namespace cryptonote; // throw if attempting a transaction with no destinations @@ -7893,7 +8048,7 @@ void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry rct::multisig_out msout; LOG_PRINT_L2("constructing tx"); auto sources_copy = sources; - bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sources, splitted_dsts, change_dts.addr, extra, tx, unlock_time, tx_key, additional_tx_keys, true, range_proof_type, m_multisig ? &msout : NULL); + bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sources, splitted_dsts, change_dts.addr, extra, tx, unlock_time, tx_key, additional_tx_keys, true, rct_config, m_multisig ? &msout : NULL); LOG_PRINT_L2("constructed tx, r="<<r); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sources, dsts, unlock_time, m_nettype); THROW_WALLET_EXCEPTION_IF(upper_transaction_weight_limit <= get_transaction_weight(tx), error::tx_too_big, tx, upper_transaction_weight_limit); @@ -7938,7 +8093,7 @@ void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry LOG_PRINT_L2("Creating supplementary multisig transaction"); cryptonote::transaction ms_tx; auto sources_copy_copy = sources_copy; - bool r = cryptonote::construct_tx_with_tx_key(m_account.get_keys(), m_subaddresses, sources_copy_copy, splitted_dsts, change_dts.addr, extra, ms_tx, unlock_time,tx_key, additional_tx_keys, true, range_proof_type, &msout, false); + bool r = cryptonote::construct_tx_with_tx_key(m_account.get_keys(), m_subaddresses, sources_copy_copy, splitted_dsts, change_dts.addr, extra, ms_tx, unlock_time,tx_key, additional_tx_keys, true, rct_config, &msout, false); LOG_PRINT_L2("constructed tx, r="<<r); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sources, splitted_dsts, unlock_time, m_nettype); THROW_WALLET_EXCEPTION_IF(upper_transaction_weight_limit <= get_transaction_weight(tx), error::tx_too_big, tx, upper_transaction_weight_limit); @@ -8244,7 +8399,7 @@ void wallet2::light_wallet_get_unspent_outs() td.m_key_image = unspent_key_image; td.m_key_image_known = !m_watch_only && !m_multisig; - td.m_key_image_requested = false; + td.m_key_image_request = false; td.m_key_image_partial = m_multisig; td.m_amount = o.amount; td.m_pk_index = 0; @@ -8615,15 +8770,16 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp TX() : weight(0), needed_fee(0) {} - void add(const account_public_address &addr, bool is_subaddress, uint64_t amount, unsigned int original_output_index, bool merge_destinations) { + void add(const cryptonote::tx_destination_entry &de, uint64_t amount, unsigned int original_output_index, bool merge_destinations) { if (merge_destinations) { std::vector<cryptonote::tx_destination_entry>::iterator i; - i = std::find_if(dsts.begin(), dsts.end(), [&](const cryptonote::tx_destination_entry &d) { return !memcmp (&d.addr, &addr, sizeof(addr)); }); + i = std::find_if(dsts.begin(), dsts.end(), [&](const cryptonote::tx_destination_entry &d) { return !memcmp (&d.addr, &de.addr, sizeof(de.addr)); }); if (i == dsts.end()) { - dsts.push_back(tx_destination_entry(0,addr,is_subaddress)); + dsts.push_back(de); i = dsts.end() - 1; + i->amount = 0; } i->amount += amount; } @@ -8632,8 +8788,11 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp THROW_WALLET_EXCEPTION_IF(original_output_index > dsts.size(), error::wallet_internal_error, std::string("original_output_index too large: ") + std::to_string(original_output_index) + " > " + std::to_string(dsts.size())); if (original_output_index == dsts.size()) - dsts.push_back(tx_destination_entry(0,addr,is_subaddress)); - THROW_WALLET_EXCEPTION_IF(memcmp(&dsts[original_output_index].addr, &addr, sizeof(addr)), error::wallet_internal_error, "Mismatched destination address"); + { + dsts.push_back(de); + dsts.back().amount = 0; + } + THROW_WALLET_EXCEPTION_IF(memcmp(&dsts[original_output_index].addr, &de.addr, sizeof(de.addr)), error::wallet_internal_error, "Mismatched destination address"); dsts[original_output_index].amount += amount; } } @@ -8645,7 +8804,10 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp const bool use_per_byte_fee = use_fork_rules(HF_VERSION_PER_BYTE_FEE, 0); const bool use_rct = use_fork_rules(4, 0); const bool bulletproof = use_fork_rules(get_bulletproof_fork(), 0); - const rct::RangeProofType range_proof_type = bulletproof ? rct::RangeProofPaddedBulletproof : rct::RangeProofBorromean; + const rct::RCTConfig rct_config { + bulletproof ? rct::RangeProofPaddedBulletproof : rct::RangeProofBorromean, + bulletproof ? (use_fork_rules(HF_VERSION_SMALLER_BP, -10) ? 2 : 1) : 0 + }; const uint64_t base_fee = get_base_fee(); const uint64_t fee_multiplier = get_fee_multiplier(priority, get_fee_algorithm()); @@ -8901,7 +9063,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp // we can fully pay that destination LOG_PRINT_L2("We can fully pay " << get_account_address_as_str(m_nettype, dsts[0].is_subaddress, dsts[0].addr) << " for " << print_money(dsts[0].amount)); - tx.add(dsts[0].addr, dsts[0].is_subaddress, dsts[0].amount, original_output_index, m_merge_destinations); + tx.add(dsts[0], dsts[0].amount, original_output_index, m_merge_destinations); available_amount -= dsts[0].amount; dsts[0].amount = 0; pop_index(dsts, 0); @@ -8912,7 +9074,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp // we can partially fill that destination LOG_PRINT_L2("We can partially pay " << get_account_address_as_str(m_nettype, dsts[0].is_subaddress, dsts[0].addr) << " for " << print_money(available_amount) << "/" << print_money(dsts[0].amount)); - tx.add(dsts[0].addr, dsts[0].is_subaddress, available_amount, original_output_index, m_merge_destinations); + tx.add(dsts[0], available_amount, original_output_index, m_merge_destinations); dsts[0].amount -= available_amount; available_amount = 0; } @@ -8958,7 +9120,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp tx.selected_transfers.size() << " inputs"); if (use_rct) transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, - test_tx, test_ptx, range_proof_type); + test_tx, test_ptx, rct_config); else transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); @@ -9001,7 +9163,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp while (needed_fee > test_ptx.fee) { if (use_rct) transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, - test_tx, test_ptx, range_proof_type); + test_tx, test_ptx, rct_config); else transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); @@ -9074,7 +9236,7 @@ skip_tx: extra, /* const std::vector<uint8_t>& extra, */ test_tx, /* OUT cryptonote::transaction& tx, */ test_ptx, /* OUT cryptonote::transaction& tx, */ - range_proof_type); + rct_config); } else { transfer_selected(tx.dsts, tx.selected_transfers, @@ -9214,7 +9376,10 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton const bool use_per_byte_fee = use_fork_rules(HF_VERSION_PER_BYTE_FEE); const bool use_rct = fake_outs_count > 0 && use_fork_rules(4, 0); const bool bulletproof = use_fork_rules(get_bulletproof_fork(), 0); - const rct::RangeProofType range_proof_type = bulletproof ? rct::RangeProofPaddedBulletproof : rct::RangeProofBorromean; + const rct::RCTConfig rct_config { + bulletproof ? rct::RangeProofPaddedBulletproof : rct::RangeProofBorromean, + bulletproof ? (use_fork_rules(HF_VERSION_SMALLER_BP, -10) ? 2 : 1) : 0, + }; const uint64_t base_fee = get_base_fee(); const uint64_t fee_multiplier = get_fee_multiplier(priority, get_fee_algorithm()); const uint64_t fee_quantization_mask = get_fee_quantization_mask(); @@ -9290,7 +9455,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton tx.selected_transfers.size() << " outputs"); if (use_rct) transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, - test_tx, test_ptx, range_proof_type); + test_tx, test_ptx, rct_config); else transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); @@ -9327,7 +9492,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton } if (use_rct) transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, - test_tx, test_ptx, range_proof_type); + test_tx, test_ptx, rct_config); else transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, outs, unlock_time, needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); @@ -9366,7 +9531,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_from(const crypton pending_tx test_ptx; if (use_rct) { transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, tx.outs, unlock_time, tx.needed_fee, extra, - test_tx, test_ptx, range_proof_type); + test_tx, test_ptx, rct_config); } else { transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, tx.outs, unlock_time, tx.needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx); @@ -9470,7 +9635,7 @@ bool wallet2::use_fork_rules(uint8_t version, int64_t early_blocks) const result = m_node_rpc_proxy.get_earliest_height(version, earliest_height); throw_on_rpc_response_error(result, "get_hard_fork_info"); - bool close_enough = height >= earliest_height - early_blocks; // start using the rules that many blocks beforehand + bool close_enough = height >= earliest_height - early_blocks && earliest_height != std::numeric_limits<uint64_t>::max(); // start using the rules that many blocks beforehand if (close_enough) LOG_PRINT_L2("Using v" << (unsigned)version << " rules"); else @@ -9661,7 +9826,7 @@ void wallet2::set_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_ COMMAND_RPC_GET_TRANSACTIONS::request req = AUTO_VAL_INIT(req); req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); req.decode_as_json = false; - req.prune = false; + req.prune = true; COMMAND_RPC_GET_TRANSACTIONS::response res = AUTO_VAL_INIT(res); bool r; { @@ -9674,11 +9839,10 @@ void wallet2::set_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_ THROW_WALLET_EXCEPTION_IF(res.txs.size() != 1, error::wallet_internal_error, "daemon returned wrong response for gettransactions, wrong txs count = " + std::to_string(res.txs.size()) + ", expected 1"); - cryptonote::blobdata bd; - THROW_WALLET_EXCEPTION_IF(!epee::string_tools::parse_hexstr_to_binbuff(res.txs[0].as_hex, bd), error::wallet_internal_error, "failed to parse tx from hexstr"); cryptonote::transaction tx; - crypto::hash tx_hash, tx_prefix_hash; - THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(bd, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, "failed to parse tx from blob"); + crypto::hash tx_hash; + THROW_WALLET_EXCEPTION_IF(!get_pruned_tx(res.txs[0], tx, tx_hash), error::wallet_internal_error, + "Failed to get transaction from daemon"); THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, "txid mismatch"); std::vector<tx_extra_field> tx_extra_fields; THROW_WALLET_EXCEPTION_IF(!parse_tx_extra(tx.extra, tx_extra_fields), error::wallet_internal_error, "Transaction extra has unsupported format"); @@ -9712,7 +9876,7 @@ std::string wallet2::get_spend_proof(const crypto::hash &txid, const std::string COMMAND_RPC_GET_TRANSACTIONS::request req = AUTO_VAL_INIT(req); req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); req.decode_as_json = false; - req.prune = false; + req.prune = true; COMMAND_RPC_GET_TRANSACTIONS::response res = AUTO_VAL_INIT(res); bool r; { @@ -9725,12 +9889,10 @@ std::string wallet2::get_spend_proof(const crypto::hash &txid, const std::string THROW_WALLET_EXCEPTION_IF(res.txs.size() != 1, error::wallet_internal_error, "daemon returned wrong response for gettransactions, wrong txs count = " + std::to_string(res.txs.size()) + ", expected 1"); - cryptonote::blobdata bd; - THROW_WALLET_EXCEPTION_IF(!epee::string_tools::parse_hexstr_to_binbuff(res.txs[0].as_hex, bd), error::wallet_internal_error, "failed to parse tx from hexstr"); + cryptonote::transaction tx; - crypto::hash tx_hash, tx_prefix_hash; - THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(bd, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, "failed to parse tx from blob"); - THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, "txid mismatch"); + crypto::hash tx_hash; + THROW_WALLET_EXCEPTION_IF(!get_pruned_tx(res.txs[0], tx, tx_hash), error::wallet_internal_error, "Failed to get tx from daemon"); std::vector<std::vector<crypto::signature>> signatures; @@ -9832,7 +9994,7 @@ bool wallet2::check_spend_proof(const crypto::hash &txid, const std::string &mes COMMAND_RPC_GET_TRANSACTIONS::request req = AUTO_VAL_INIT(req); req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); req.decode_as_json = false; - req.prune = false; + req.prune = true; COMMAND_RPC_GET_TRANSACTIONS::response res = AUTO_VAL_INIT(res); bool r; { @@ -9845,12 +10007,10 @@ bool wallet2::check_spend_proof(const crypto::hash &txid, const std::string &mes THROW_WALLET_EXCEPTION_IF(res.txs.size() != 1, error::wallet_internal_error, "daemon returned wrong response for gettransactions, wrong txs count = " + std::to_string(res.txs.size()) + ", expected 1"); - cryptonote::blobdata bd; - THROW_WALLET_EXCEPTION_IF(!epee::string_tools::parse_hexstr_to_binbuff(res.txs[0].as_hex, bd), error::wallet_internal_error, "failed to parse tx from hexstr"); + cryptonote::transaction tx; - crypto::hash tx_hash, tx_prefix_hash; - THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(bd, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, "failed to parse tx from blob"); - THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, "txid mismatch"); + crypto::hash tx_hash; + THROW_WALLET_EXCEPTION_IF(!get_pruned_tx(res.txs[0], tx, tx_hash), error::wallet_internal_error, "failed to get tx from daemon"); // check signature size size_t num_sigs = 0; @@ -9957,24 +10117,30 @@ void wallet2::check_tx_key_helper(const crypto::hash &txid, const crypto::key_de COMMAND_RPC_GET_TRANSACTIONS::response res; req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); req.decode_as_json = false; - req.prune = false; + req.prune = true; m_daemon_rpc_mutex.lock(); bool ok = epee::net_utils::invoke_http_json("/gettransactions", req, res, m_http_client); m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1), error::wallet_internal_error, "Failed to get transaction from daemon"); - cryptonote::blobdata tx_data; + cryptonote::transaction tx; + crypto::hash tx_hash; if (res.txs.size() == 1) - ok = string_tools::parse_hexstr_to_binbuff(res.txs.front().as_hex, tx_data); + { + ok = get_pruned_tx(res.txs.front(), tx, tx_hash); + THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + } else + { + cryptonote::blobdata tx_data; + crypto::hash tx_prefix_hash; ok = string_tools::parse_hexstr_to_binbuff(res.txs_as_hex.front(), tx_data); - THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx, tx_hash, tx_prefix_hash), + error::wallet_internal_error, "Failed to validate transaction from daemon"); + } - crypto::hash tx_hash, tx_prefix_hash; - cryptonote::transaction tx; - THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, - "Failed to validate transaction from daemon"); THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, "Failed to get the right transaction from daemon"); THROW_WALLET_EXCEPTION_IF(!additional_derivations.empty() && additional_derivations.size() != tx.vout.size(), error::wallet_internal_error, @@ -10013,7 +10179,7 @@ void wallet2::check_tx_key_helper(const crypto::hash &txid, const crypto::key_de crypto::secret_key scalar1; hwdev.derivation_to_scalar(found_derivation, n, scalar1); rct::ecdhTuple ecdh_info = tx.rct_signatures.ecdhInfo[n]; - hwdev.ecdhDecode(ecdh_info, rct::sk2rct(scalar1)); + hwdev.ecdhDecode(ecdh_info, rct::sk2rct(scalar1), tx.rct_signatures.type == rct::RCTTypeBulletproof2); const rct::key C = tx.rct_signatures.outPk[n].mask; rct::key Ctmp; THROW_WALLET_EXCEPTION_IF(sc_check(ecdh_info.mask.bytes) != 0, error::wallet_internal_error, "Bad ECDH input mask"); @@ -10097,24 +10263,30 @@ std::string wallet2::get_tx_proof(const crypto::hash &txid, const cryptonote::ac COMMAND_RPC_GET_TRANSACTIONS::response res; req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); req.decode_as_json = false; - req.prune = false; + req.prune = true; m_daemon_rpc_mutex.lock(); bool ok = net_utils::invoke_http_json("/gettransactions", req, res, m_http_client); m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1), error::wallet_internal_error, "Failed to get transaction from daemon"); - cryptonote::blobdata tx_data; + cryptonote::transaction tx; + crypto::hash tx_hash; if (res.txs.size() == 1) - ok = string_tools::parse_hexstr_to_binbuff(res.txs.front().as_hex, tx_data); + { + ok = get_pruned_tx(res.txs.front(), tx, tx_hash); + THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + } else + { + cryptonote::blobdata tx_data; + crypto::hash tx_prefix_hash; ok = string_tools::parse_hexstr_to_binbuff(res.txs_as_hex.front(), tx_data); - THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx, tx_hash, tx_prefix_hash), + error::wallet_internal_error, "Failed to validate transaction from daemon"); + } - crypto::hash tx_hash, tx_prefix_hash; - cryptonote::transaction tx; - THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, - "Failed to validate transaction from daemon"); THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, "Failed to get the right transaction from daemon"); crypto::public_key tx_pub_key = get_tx_pub_key_from_extra(tx); @@ -10209,24 +10381,30 @@ bool wallet2::check_tx_proof(const crypto::hash &txid, const cryptonote::account COMMAND_RPC_GET_TRANSACTIONS::response res; req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); req.decode_as_json = false; - req.prune = false; + req.prune = true; m_daemon_rpc_mutex.lock(); bool ok = net_utils::invoke_http_json("/gettransactions", req, res, m_http_client); m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!ok || (res.txs.size() != 1 && res.txs_as_hex.size() != 1), error::wallet_internal_error, "Failed to get transaction from daemon"); - cryptonote::blobdata tx_data; + cryptonote::transaction tx; + crypto::hash tx_hash; if (res.txs.size() == 1) - ok = string_tools::parse_hexstr_to_binbuff(res.txs.front().as_hex, tx_data); + { + ok = get_pruned_tx(res.txs.front(), tx, tx_hash); + THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + } else + { + cryptonote::blobdata tx_data; + crypto::hash tx_prefix_hash; ok = string_tools::parse_hexstr_to_binbuff(res.txs_as_hex.front(), tx_data); - THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx, tx_hash, tx_prefix_hash), + error::wallet_internal_error, "Failed to validate transaction from daemon"); + } - crypto::hash tx_hash, tx_prefix_hash; - cryptonote::transaction tx; - THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, - "Failed to validate transaction from daemon"); THROW_WALLET_EXCEPTION_IF(tx_hash != txid, error::wallet_internal_error, "Failed to get the right transaction from daemon"); crypto::public_key tx_pub_key = get_tx_pub_key_from_extra(tx); @@ -10445,7 +10623,7 @@ bool wallet2::check_reserve_proof(const cryptonote::account_public_address &addr for (size_t i = 0; i < proofs.size(); ++i) gettx_req.txs_hashes.push_back(epee::string_tools::pod_to_hex(proofs[i].txid)); gettx_req.decode_as_json = false; - gettx_req.prune = false; + gettx_req.prune = true; m_daemon_rpc_mutex.lock(); bool ok = net_utils::invoke_http_json("/gettransactions", gettx_req, gettx_res, m_http_client); m_daemon_rpc_mutex.unlock(); @@ -10469,14 +10647,11 @@ bool wallet2::check_reserve_proof(const cryptonote::account_public_address &addr const reserve_proof_entry& proof = proofs[i]; THROW_WALLET_EXCEPTION_IF(gettx_res.txs[i].in_pool, error::wallet_internal_error, "Tx is unconfirmed"); - cryptonote::blobdata tx_data; - ok = string_tools::parse_hexstr_to_binbuff(gettx_res.txs[i].as_hex, tx_data); + cryptonote::transaction tx; + crypto::hash tx_hash; + ok = get_pruned_tx(gettx_res.txs[i], tx, tx_hash); THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to parse transaction from daemon"); - crypto::hash tx_hash, tx_prefix_hash; - cryptonote::transaction tx; - THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(tx_data, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, - "Failed to validate transaction from daemon"); THROW_WALLET_EXCEPTION_IF(tx_hash != proof.txid, error::wallet_internal_error, "Failed to get the right transaction from daemon"); THROW_WALLET_EXCEPTION_IF(proof.index_in_tx >= tx.vout.size(), error::wallet_internal_error, "index_in_tx is out of bound"); @@ -10518,7 +10693,7 @@ bool wallet2::check_reserve_proof(const cryptonote::account_public_address &addr crypto::secret_key shared_secret; crypto::derivation_to_scalar(derivation, proof.index_in_tx, shared_secret); rct::ecdhTuple ecdh_info = tx.rct_signatures.ecdhInfo[proof.index_in_tx]; - rct::ecdhDecode(ecdh_info, rct::sk2rct(shared_secret)); + rct::ecdhDecode(ecdh_info, rct::sk2rct(shared_secret), tx.rct_signatures.type == rct::RCTTypeBulletproof2); amount = rct::h2d(ecdh_info.amount); } total += amount; @@ -10557,7 +10732,10 @@ uint64_t wallet2::get_daemon_blockchain_height(string &err) const boost::optional<std::string> result = m_node_rpc_proxy.get_height(height); if (result) { - err = *result; + if (m_trusted_daemon) + err = *result; + else + err = "daemon error"; return 0; } @@ -10572,7 +10750,10 @@ uint64_t wallet2::get_daemon_blockchain_target_height(string &err) const auto result = m_node_rpc_proxy.get_target_height(target_height); if (result && *result != CORE_RPC_STATUS_OK) { - err= *result; + if (m_trusted_daemon) + err = *result; + else + err = "daemon error"; return 0; } return target_height; @@ -10840,23 +11021,23 @@ bool wallet2::export_key_images(const std::string &filename) const } //---------------------------------------------------------------------------------------------------- -std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> wallet2::export_key_images() const +std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> wallet2::export_key_images(bool all) const { PERF_TIMER(export_key_images_raw); std::vector<std::pair<crypto::key_image, crypto::signature>> ski; size_t offset = 0; - while (offset < m_transfers.size() && !m_transfers[offset].m_key_image_requested) - ++offset; + if (!all) + { + while (offset < m_transfers.size() && !m_transfers[offset].m_key_image_request) + ++offset; + } ski.reserve(m_transfers.size() - offset); for (size_t n = offset; n < m_transfers.size(); ++n) { const transfer_details &td = m_transfers[n]; - crypto::hash hash; - crypto::cn_fast_hash(&td.m_key_image, sizeof(td.m_key_image), hash); - // get ephemeral public key const cryptonote::tx_out &out = td.m_tx.vout[td.m_internal_output_index]; THROW_WALLET_EXCEPTION_IF(out.target.type() != typeid(txout_to_key), error::wallet_internal_error, @@ -11008,7 +11189,7 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag m_transfers[n + offset].m_key_image = signed_key_images[n].first; m_key_images[m_transfers[n + offset].m_key_image] = n + offset; m_transfers[n + offset].m_key_image_known = true; - m_transfers[n + offset].m_key_image_requested = false; + m_transfers[n + offset].m_key_image_request = false; m_transfers[n + offset].m_key_image_partial = false; } PERF_TIMER_STOP(import_key_images_B); @@ -11080,7 +11261,7 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag COMMAND_RPC_GET_TRANSACTIONS::request gettxs_req; COMMAND_RPC_GET_TRANSACTIONS::response gettxs_res; gettxs_req.decode_as_json = false; - gettxs_req.prune = false; + gettxs_req.prune = true; gettxs_req.txs_hashes.reserve(spent_txids.size()); for (const crypto::hash& spent_txid : spent_txids) gettxs_req.txs_hashes.push_back(epee::string_tools::pod_to_hex(spent_txid)); @@ -11100,17 +11281,16 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag PERF_TIMER_START(import_key_images_F); auto spent_txid = spent_txids.begin(); hw::device &hwdev = m_account.get_device(); + auto it = spent_txids.begin(); for (const COMMAND_RPC_GET_TRANSACTIONS::entry& e : gettxs_res.txs) { THROW_WALLET_EXCEPTION_IF(e.in_pool, error::wallet_internal_error, "spent tx isn't supposed to be in txpool"); - // parse tx - cryptonote::blobdata bd; - THROW_WALLET_EXCEPTION_IF(!epee::string_tools::parse_hexstr_to_binbuff(e.as_hex, bd), error::wallet_internal_error, "parse_hexstr_to_binbuff failed"); cryptonote::transaction spent_tx; - crypto::hash spnet_txid_parsed, spent_txid_prefix; - THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(bd, spent_tx, spnet_txid_parsed, spent_txid_prefix), error::wallet_internal_error, "parse_and_validate_tx_from_blob failed"); - THROW_WALLET_EXCEPTION_IF(*spent_txid != spnet_txid_parsed, error::wallet_internal_error, "parsed txid mismatch"); + crypto::hash spnet_txid_parsed; + THROW_WALLET_EXCEPTION_IF(!get_pruned_tx(e, spent_tx, spnet_txid_parsed), error::wallet_internal_error, "Failed to get tx from daemon"); + THROW_WALLET_EXCEPTION_IF(!(spnet_txid_parsed == *it), error::wallet_internal_error, "parsed txid mismatch"); + ++it; // get received (change) amount uint64_t tx_money_got_in_outs = 0; @@ -11227,7 +11407,7 @@ bool wallet2::import_key_images(std::vector<crypto::key_image> key_images) td.m_key_image = key_images[i]; m_key_images[m_transfers[i].m_key_image] = i; td.m_key_image_known = true; - td.m_key_image_requested = false; + td.m_key_image_request = false; td.m_key_image_partial = false; m_pub_keys[m_transfers[i].get_public_key()] = i; } @@ -11299,7 +11479,7 @@ std::pair<size_t, std::vector<tools::wallet2::transfer_details>> wallet2::export std::vector<tools::wallet2::transfer_details> outs; size_t offset = 0; - while (offset < m_transfers.size() && m_transfers[offset].m_key_image_known) + while (offset < m_transfers.size() && (m_transfers[offset].m_key_image_known && !m_transfers[offset].m_key_image_request)) ++offset; outs.reserve(m_transfers.size() - offset); @@ -11343,7 +11523,7 @@ size_t wallet2::import_outputs(const std::pair<size_t, std::vector<tools::wallet const size_t original_size = m_transfers.size(); m_transfers.resize(offset + outputs.second.size()); for (size_t i = 0; i < offset; ++i) - m_transfers[i].m_key_image_requested = false; + m_transfers[i].m_key_image_request = false; for (size_t i = 0; i < outputs.second.size(); ++i) { transfer_details td = outputs.second[i]; @@ -11384,7 +11564,7 @@ process: THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "Failed to generate key image"); expand_subaddresses(td.m_subaddr_index); td.m_key_image_known = true; - td.m_key_image_requested = true; + td.m_key_image_request = true; td.m_key_image_partial = false; THROW_WALLET_EXCEPTION_IF(in_ephemeral.pub != out_key, error::wallet_internal_error, "key_image generated ephemeral public key not matched with output_key at index " + boost::lexical_cast<std::string>(i + offset)); @@ -11630,7 +11810,7 @@ void wallet2::update_multisig_rescan_info(const std::vector<std::vector<rct::key m_key_images.erase(td.m_key_image); td.m_key_image = get_multisig_composite_key_image(n); td.m_key_image_known = true; - td.m_key_image_requested = false; + td.m_key_image_request = false; td.m_key_image_partial = false; td.m_multisig_k = multisig_k[n]; m_key_images[td.m_key_image] = n; @@ -12013,7 +12193,7 @@ uint64_t wallet2::get_blockchain_height_by_date(uint16_t year, uint8_t month, ui else if (res.status == CORE_RPC_STATUS_BUSY) oss << "daemon is busy"; else - oss << res.status; + oss << get_rpc_status(res.status); throw std::runtime_error(oss.str()); } cryptonote::block blk_min, blk_mid, blk_max; @@ -12232,4 +12412,27 @@ void wallet2::on_passphrase_request(bool on_device, epee::wipeable_string & pass if (0 != m_callback) m_callback->on_passphrase_request(on_device, passphrase); } +//---------------------------------------------------------------------------------------------------- +std::string wallet2::get_rpc_status(const std::string &s) const +{ + if (m_trusted_daemon) + return s; + return "<error>"; +} +//---------------------------------------------------------------------------------------------------- +void wallet2::throw_on_rpc_response_error(const boost::optional<std::string> &status, const char *method) const +{ + // no error + if (!status) + return; + + MERROR("RPC error: " << method << ": status " << *status); + + // empty string -> not connection + THROW_WALLET_EXCEPTION_IF(status->empty(), tools::error::no_connection_to_daemon, method); + + THROW_WALLET_EXCEPTION_IF(*status == CORE_RPC_STATUS_BUSY, tools::error::daemon_busy, method); + THROW_WALLET_EXCEPTION_IF(*status != CORE_RPC_STATUS_OK, tools::error::wallet_generic_rpc_error, method, m_trusted_daemon ? *status : "daemon error"); +} + } diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 5b1988080..3b2dd6076 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -267,7 +267,7 @@ namespace tools uint64_t m_amount; bool m_rct; bool m_key_image_known; - bool m_key_image_requested; + bool m_key_image_request; // view wallets: we want to request it; cold wallets: it was requested size_t m_pk_index; cryptonote::subaddress_index m_subaddr_index; bool m_key_image_partial; @@ -292,7 +292,7 @@ namespace tools FIELD(m_amount) FIELD(m_rct) FIELD(m_key_image_known) - FIELD(m_key_image_requested) + FIELD(m_key_image_request) FIELD(m_pk_index) FIELD(m_subaddr_index) FIELD(m_key_image_partial) @@ -448,6 +448,7 @@ namespace tools { std::vector<pending_tx> ptx; std::vector<crypto::key_image> key_images; + std::unordered_map<crypto::public_key, crypto::key_image> tx_key_images; }; struct multisig_tx_set @@ -732,7 +733,7 @@ namespace tools bool is_deprecated() const; void refresh(bool trusted_daemon); void refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched); - void refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched, bool& received_money); + void refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blocks_fetched, bool& received_money, bool check_pool = true); bool refresh(bool trusted_daemon, uint64_t & blocks_fetched, bool& received_money, bool& ok); void set_refresh_type(RefreshType refresh_type) { m_refresh_type = refresh_type; } @@ -763,7 +764,7 @@ namespace tools uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, T destination_split_strategy, const tx_dust_policy& dust_policy, cryptonote::transaction& tx, pending_tx &ptx); void transfer_selected_rct(std::vector<cryptonote::tx_destination_entry> dsts, const std::vector<size_t>& selected_transfers, size_t fake_outputs_count, std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, - uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx &ptx, rct::RangeProofType range_proof_type); + uint64_t unlock_time, uint64_t fee, const std::vector<uint8_t>& extra, cryptonote::transaction& tx, pending_tx &ptx, const rct::RCTConfig &rct_config); void commit_tx(pending_tx& ptx_vector); void commit_tx(std::vector<pending_tx>& ptx_vector); @@ -927,6 +928,9 @@ namespace tools if(ver < 27) return; a & m_device_last_key_image_sync; + if(ver < 28) + return; + a & m_cold_key_images; } /*! @@ -1113,7 +1117,7 @@ namespace tools std::tuple<size_t, crypto::hash, std::vector<crypto::hash>> export_blockchain() const; void import_blockchain(const std::tuple<size_t, crypto::hash, std::vector<crypto::hash>> &bc); bool export_key_images(const std::string &filename) const; - std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> export_key_images() const; + std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> export_key_images(bool all = false) const; uint64_t import_key_images(const std::vector<std::pair<crypto::key_image, crypto::signature>> &signed_key_images, size_t offset, uint64_t &spent, uint64_t &unspent, bool check_spent = true); uint64_t import_key_images(const std::string &filename, uint64_t &spent, uint64_t &unspent); bool import_key_images(std::vector<crypto::key_image> key_images); @@ -1288,7 +1292,7 @@ namespace tools bool tx_add_fake_output(std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, uint64_t global_index, const crypto::public_key& tx_public_key, const rct::key& mask, uint64_t real_index, bool unlocked) const; bool should_pick_a_second_output(bool use_rct, size_t n_transfers, const std::vector<size_t> &unused_transfers_indices, const std::vector<size_t> &unused_dust_indices) const; std::vector<size_t> get_only_rct(const std::vector<size_t> &unused_dust_indices, const std::vector<size_t> &unused_transfers_indices) const; - void scan_output(const cryptonote::transaction &tx, const crypto::public_key &tx_pub_key, size_t i, tx_scan_info_t &tx_scan_info, int &num_vouts_received, std::unordered_map<cryptonote::subaddress_index, uint64_t> &tx_money_got_in_outs, std::vector<size_t> &outs); + void scan_output(const cryptonote::transaction &tx, const crypto::public_key &tx_pub_key, size_t i, tx_scan_info_t &tx_scan_info, int &num_vouts_received, std::unordered_map<cryptonote::subaddress_index, uint64_t> &tx_money_got_in_outs, std::vector<size_t> &outs, bool pool); void trim_hashchain(); crypto::key_image get_multisig_composite_key_image(size_t n) const; rct::multisig_kLRki get_multisig_composite_kLRki(size_t n, const std::unordered_set<crypto::public_key> &ignore_set, std::unordered_set<rct::key> &used_L, std::unordered_set<rct::key> &new_used_L) const; @@ -1326,6 +1330,9 @@ namespace tools void on_pin_request(epee::wipeable_string & pin); void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase); + std::string get_rpc_status(const std::string &s) const; + void throw_on_rpc_response_error(const boost::optional<std::string> &status, const char *method) const; + cryptonote::account_base m_account; boost::optional<epee::net_utils::http::login> m_daemon_login; std::string m_daemon_address; @@ -1355,6 +1362,7 @@ namespace tools uint64_t m_upper_transaction_weight_limit; //TODO: auto-calc this value or request from daemon, now use some fixed value const std::vector<std::vector<tools::wallet2::multisig_info>> *m_multisig_rescan_info; const std::vector<std::vector<rct::key>> *m_multisig_rescan_k; + std::unordered_map<crypto::public_key, crypto::key_image> m_cold_key_images; std::atomic<bool> m_run; @@ -1449,7 +1457,7 @@ namespace tools std::unique_ptr<wallet_device_callback> m_device_callback; }; } -BOOST_CLASS_VERSION(tools::wallet2, 27) +BOOST_CLASS_VERSION(tools::wallet2, 28) BOOST_CLASS_VERSION(tools::wallet2::transfer_details, 11) BOOST_CLASS_VERSION(tools::wallet2::multisig_info, 1) BOOST_CLASS_VERSION(tools::wallet2::multisig_info::LR, 0) @@ -1461,7 +1469,7 @@ BOOST_CLASS_VERSION(tools::wallet2::confirmed_transfer_details, 6) BOOST_CLASS_VERSION(tools::wallet2::address_book_row, 17) BOOST_CLASS_VERSION(tools::wallet2::reserve_proof_entry, 0) BOOST_CLASS_VERSION(tools::wallet2::unsigned_tx_set, 0) -BOOST_CLASS_VERSION(tools::wallet2::signed_tx_set, 0) +BOOST_CLASS_VERSION(tools::wallet2::signed_tx_set, 1) BOOST_CLASS_VERSION(tools::wallet2::tx_construction_data, 3) BOOST_CLASS_VERSION(tools::wallet2::pending_tx, 3) BOOST_CLASS_VERSION(tools::wallet2::multisig_sig, 0) @@ -1510,7 +1518,7 @@ namespace boost } if (ver < 10) { - x.m_key_image_requested = false; + x.m_key_image_request = false; } } @@ -1598,7 +1606,7 @@ namespace boost initialize_transfer_details(a, x, ver); return; } - a & x.m_key_image_requested; + a & x.m_key_image_request; if (ver < 11) return; a & x.m_uses; @@ -1798,6 +1806,9 @@ namespace boost { a & x.ptx; a & x.key_images; + if (ver < 1) + return; + a & x.tx_key_images; } template <class Archive> diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index d7dc2914e..5dc6ac0e8 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -644,9 +644,11 @@ namespace tools return false; } + de.original = it->address; de.addr = info.address; de.is_subaddress = info.is_subaddress; de.amount = it->amount; + de.is_integrated = info.has_payment_id; dsts.push_back(de); if (info.has_payment_id) @@ -2495,7 +2497,7 @@ namespace tools if (!m_wallet) return not_open(er); try { - std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> ski = m_wallet->export_key_images(); + std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> ski = m_wallet->export_key_images(req.all); res.offset = ski.first; res.signed_key_images.resize(ski.second.size()); for (size_t n = 0; n < ski.second.size(); ++n) diff --git a/src/wallet/wallet_rpc_server_commands_defs.h b/src/wallet/wallet_rpc_server_commands_defs.h index afb8c6e91..f0c1a4e9d 100644 --- a/src/wallet/wallet_rpc_server_commands_defs.h +++ b/src/wallet/wallet_rpc_server_commands_defs.h @@ -47,7 +47,7 @@ // advance which version they will stop working with // Don't go over 32767 for any of these #define WALLET_RPC_VERSION_MAJOR 1 -#define WALLET_RPC_VERSION_MINOR 6 +#define WALLET_RPC_VERSION_MINOR 7 #define MAKE_WALLET_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define WALLET_RPC_VERSION MAKE_WALLET_RPC_VERSION(WALLET_RPC_VERSION_MAJOR, WALLET_RPC_VERSION_MINOR) namespace tools @@ -1565,7 +1565,10 @@ namespace wallet_rpc { struct request { + bool all; + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_OPT(all, false); END_KV_SERIALIZE_MAP() }; |