diff options
Diffstat (limited to '')
30 files changed, 731 insertions, 195 deletions
diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index bb4de3ce6..cb6a8c6f7 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -156,7 +156,8 @@ struct txpool_tx_meta_t uint8_t relayed; uint8_t do_not_relay; uint8_t double_spend_seen: 1; - uint8_t bf_padding: 7; + uint8_t pruned: 1; + uint8_t bf_padding: 6; uint8_t padding[76]; // till 192 bytes }; diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 760e380a9..8e2b5bebf 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -2056,7 +2056,7 @@ bool BlockchainLMDB::prune_worker(int mode, uint32_t pruning_seed) ++n_prunable_records; result = mdb_cursor_get(c_txs_prunable, &k, &v, MDB_SET); if (result == MDB_NOTFOUND) - MWARNING("Already pruned at height " << block_height << "/" << blockchain_height); + MDEBUG("Already pruned at height " << block_height << "/" << blockchain_height); else if (result) throw0(DB_ERROR(lmdb_error("Failed to find transaction prunable data: ", result).c_str())); else @@ -2152,7 +2152,7 @@ bool BlockchainLMDB::prune_worker(int mode, uint32_t pruning_seed) { ++n_prunable_records; if (result == MDB_NOTFOUND) - MWARNING("Already pruned at height " << block_height << "/" << blockchain_height); + MDEBUG("Already pruned at height " << block_height << "/" << blockchain_height); else { MDEBUG("Pruning at height " << block_height << "/" << blockchain_height); @@ -2994,6 +2994,8 @@ bool BlockchainLMDB::get_tx_blob(const crypto::hash& h, cryptonote::blobdata &bd return false; else if (get_result) throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx from hash", get_result).c_str())); + else if (result1.mv_size == 0) + return false; bd.assign(reinterpret_cast<char*>(result0.mv_data), result0.mv_size); bd.append(reinterpret_cast<char*>(result1.mv_data), result1.mv_size); diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp index cb9154f29..bc7489a0d 100644 --- a/src/blockchain_utilities/blockchain_import.cpp +++ b/src/blockchain_utilities/blockchain_import.cpp @@ -191,7 +191,7 @@ int check_flush(cryptonote::core &core, std::vector<block_complete_entry> &block } hashes.push_back(cryptonote::get_block_hash(block)); } - core.prevalidate_block_hashes(core.get_blockchain_storage().get_db().height(), hashes); + core.prevalidate_block_hashes(core.get_blockchain_storage().get_db().height(), hashes, {}); std::vector<block> pblocks; if (!core.prepare_handle_incoming_blocks(blocks, pblocks)) @@ -217,7 +217,7 @@ int check_flush(cryptonote::core &core, std::vector<block_complete_entry> &block if(tvc.m_verifivation_failed) { MERROR("transaction verification failed, tx_id = " - << epee::string_tools::pod_to_hex(get_blob_hash(tx_blob))); + << epee::string_tools::pod_to_hex(get_blob_hash(tx_blob.blob))); core.cleanup_handle_incoming_blocks(); return 1; } @@ -468,13 +468,17 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path { cryptonote::blobdata block; cryptonote::block_to_blob(bp.block, block); - std::vector<cryptonote::blobdata> txs; + std::vector<tx_blob_entry> txs; for (const auto &tx: bp.txs) { - txs.push_back(cryptonote::blobdata()); - cryptonote::tx_to_blob(tx, txs.back()); + txs.push_back({cryptonote::blobdata(), crypto::null_hash}); + cryptonote::tx_to_blob(tx, txs.back().blob); } - blocks.push_back({block, txs}); + block_complete_entry bce; + bce.pruned = false; + bce.block = std::move(block); + bce.txs = std::move(txs); + blocks.push_back(bce); int ret = check_flush(core, blocks, false); if (ret) { diff --git a/src/blockchain_utilities/blocksdat_file.cpp b/src/blockchain_utilities/blocksdat_file.cpp index f56ff5f94..df3c6cafc 100644 --- a/src/blockchain_utilities/blocksdat_file.cpp +++ b/src/blockchain_utilities/blocksdat_file.cpp @@ -99,17 +99,23 @@ bool BlocksdatFile::initialize_file(uint64_t block_stop) return true; } -void BlocksdatFile::write_block(const crypto::hash& block_hash) +void BlocksdatFile::write_block(const crypto::hash& block_hash, uint64_t weight) { m_hashes.push_back(block_hash); + m_weights.push_back(weight); while (m_hashes.size() >= HASH_OF_HASHES_STEP) { crypto::hash hash; crypto::cn_fast_hash(m_hashes.data(), HASH_OF_HASHES_STEP * sizeof(crypto::hash), hash); memmove(m_hashes.data(), m_hashes.data() + HASH_OF_HASHES_STEP, (m_hashes.size() - HASH_OF_HASHES_STEP) * sizeof(crypto::hash)); m_hashes.resize(m_hashes.size() - HASH_OF_HASHES_STEP); - const std::string data(hash.data, sizeof(hash)); - *m_raw_data_file << data; + const std::string data_hashes(hash.data, sizeof(hash)); + *m_raw_data_file << data_hashes; + crypto::cn_fast_hash(m_weights.data(), HASH_OF_HASHES_STEP * sizeof(uint64_t), hash); + memmove(m_weights.data(), m_weights.data() + HASH_OF_HASHES_STEP, (m_weights.size() - HASH_OF_HASHES_STEP) * sizeof(uint64_t)); + m_weights.resize(m_weights.size() - HASH_OF_HASHES_STEP); + const std::string data_weights(hash.data, sizeof(hash)); + *m_raw_data_file << data_weights; } } @@ -154,7 +160,8 @@ bool BlocksdatFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem { // this method's height refers to 0-based height (genesis block = height 0) crypto::hash hash = m_blockchain_storage->get_block_id_by_height(m_cur_height); - write_block(hash); + uint64_t weight = m_blockchain_storage->get_db().get_block_weight(m_cur_height); + write_block(hash, weight); if (m_cur_height % NUM_BLOCKS_PER_CHUNK == 0) { num_blocks_written += NUM_BLOCKS_PER_CHUNK; } diff --git a/src/blockchain_utilities/blocksdat_file.h b/src/blockchain_utilities/blocksdat_file.h index 315713424..72b7afc17 100644 --- a/src/blockchain_utilities/blocksdat_file.h +++ b/src/blockchain_utilities/blocksdat_file.h @@ -72,10 +72,11 @@ protected: bool open_writer(const boost::filesystem::path& file_path, uint64_t block_stop); bool initialize_file(uint64_t block_stop); bool close(); - void write_block(const crypto::hash &block_hash); + void write_block(const crypto::hash &block_hash, uint64_t weight); private: uint64_t m_cur_height; // tracks current height during export std::vector<crypto::hash> m_hashes; + std::vector<uint64_t> m_weights; }; diff --git a/src/blocks/checkpoints.dat b/src/blocks/checkpoints.dat Binary files differindex a7d309753..eb614d58d 100644 --- a/src/blocks/checkpoints.dat +++ b/src/blocks/checkpoints.dat diff --git a/src/cryptonote_basic/connection_context.h b/src/cryptonote_basic/connection_context.h index 51076e8c0..5e8f6685d 100644 --- a/src/cryptonote_basic/connection_context.h +++ b/src/cryptonote_basic/connection_context.h @@ -55,7 +55,7 @@ namespace cryptonote }; state m_state; - std::vector<crypto::hash> m_needed_objects; + std::vector<std::pair<crypto::hash, uint64_t>> m_needed_objects; std::unordered_set<crypto::hash> m_requested_objects; uint64_t m_remote_blockchain_height; uint64_t m_last_response_height; diff --git a/src/cryptonote_basic/cryptonote_basic.h b/src/cryptonote_basic/cryptonote_basic.h index 055c4a22b..e2286ae8c 100644 --- a/src/cryptonote_basic/cryptonote_basic.h +++ b/src/cryptonote_basic/cryptonote_basic.h @@ -195,6 +195,7 @@ namespace cryptonote private: // hash cash mutable std::atomic<bool> hash_valid; + mutable std::atomic<bool> prunable_hash_valid; mutable std::atomic<bool> blob_size_valid; public: @@ -203,6 +204,7 @@ namespace cryptonote // hash cash mutable crypto::hash hash; + mutable crypto::hash prunable_hash; mutable size_t blob_size; bool pruned; @@ -211,22 +213,26 @@ namespace cryptonote std::atomic<unsigned int> prefix_size; transaction(); - transaction(const transaction &t): transaction_prefix(t), hash_valid(false), blob_size_valid(false), signatures(t.signatures), rct_signatures(t.rct_signatures), pruned(t.pruned), unprunable_size(t.unprunable_size.load()), prefix_size(t.prefix_size.load()) { if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } } - transaction &operator=(const transaction &t) { transaction_prefix::operator=(t); set_hash_valid(false); set_blob_size_valid(false); signatures = t.signatures; rct_signatures = t.rct_signatures; if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } pruned = t.pruned; unprunable_size = t.unprunable_size.load(); prefix_size = t.prefix_size.load(); return *this; } + transaction(const transaction &t); + transaction &operator=(const transaction &t); virtual ~transaction(); void set_null(); void invalidate_hashes(); bool is_hash_valid() const { return hash_valid.load(std::memory_order_acquire); } void set_hash_valid(bool v) const { hash_valid.store(v,std::memory_order_release); } + bool is_prunable_hash_valid() const { return prunable_hash_valid.load(std::memory_order_acquire); } + void set_prunable_hash_valid(bool v) const { prunable_hash_valid.store(v,std::memory_order_release); } bool is_blob_size_valid() const { return blob_size_valid.load(std::memory_order_acquire); } void set_blob_size_valid(bool v) const { blob_size_valid.store(v,std::memory_order_release); } - void set_hash(const crypto::hash &h) { hash = h; set_hash_valid(true); } - void set_blob_size(size_t sz) { blob_size = sz; set_blob_size_valid(true); } + void set_hash(const crypto::hash &h) const { hash = h; set_hash_valid(true); } + void set_prunable_hash(const crypto::hash &h) const { prunable_hash = h; set_prunable_hash_valid(true); } + void set_blob_size(size_t sz) const { blob_size = sz; set_blob_size_valid(true); } BEGIN_SERIALIZE_OBJECT() if (!typename Archive<W>::is_saving()) { set_hash_valid(false); + set_prunable_hash_valid(false); set_blob_size_valid(false); } @@ -327,6 +333,63 @@ namespace cryptonote static size_t get_signature_size(const txin_v& tx_in); }; + inline transaction::transaction(const transaction &t): + transaction_prefix(t), + hash_valid(false), + prunable_hash_valid(false), + blob_size_valid(false), + signatures(t.signatures), + rct_signatures(t.rct_signatures), + pruned(t.pruned), + unprunable_size(t.unprunable_size.load()), + prefix_size(t.prefix_size.load()) + { + if (t.is_hash_valid()) + { + hash = t.hash; + set_hash_valid(true); + } + if (t.is_blob_size_valid()) + { + blob_size = t.blob_size; + set_blob_size_valid(true); + } + if (t.is_prunable_hash_valid()) + { + prunable_hash = t.prunable_hash; + set_prunable_hash_valid(true); + } + } + + inline transaction &transaction::operator=(const transaction &t) + { + transaction_prefix::operator=(t); + + set_hash_valid(false); + set_prunable_hash_valid(false); + set_blob_size_valid(false); + signatures = t.signatures; + rct_signatures = t.rct_signatures; + if (t.is_hash_valid()) + { + hash = t.hash; + set_hash_valid(true); + } + if (t.is_prunable_hash_valid()) + { + prunable_hash = t.prunable_hash; + set_prunable_hash_valid(true); + } + if (t.is_blob_size_valid()) + { + blob_size = t.blob_size; + set_blob_size_valid(true); + } + pruned = t.pruned; + unprunable_size = t.unprunable_size.load(); + prefix_size = t.prefix_size.load(); + return *this; + } inline transaction::transaction() @@ -346,6 +409,7 @@ namespace cryptonote signatures.clear(); rct_signatures.type = rct::RCTTypeNull; set_hash_valid(false); + set_prunable_hash_valid(false); set_blob_size_valid(false); pruned = false; unprunable_size = 0; @@ -356,6 +420,7 @@ namespace cryptonote void transaction::invalidate_hashes() { set_hash_valid(false); + set_prunable_hash_valid(false); set_blob_size_valid(false); } @@ -408,6 +473,7 @@ namespace cryptonote void invalidate_hashes() { set_hash_valid(false); } bool is_hash_valid() const { return hash_valid.load(std::memory_order_acquire); } void set_hash_valid(bool v) const { hash_valid.store(v,std::memory_order_release); } + void set_hash(const crypto::hash &h) const { hash = h; set_hash_valid(true); } transaction miner_tx; std::vector<crypto::hash> tx_hashes; diff --git a/src/cryptonote_basic/cryptonote_format_utils.cpp b/src/cryptonote_basic/cryptonote_format_utils.cpp index 7d7de416d..17f861e9a 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.cpp +++ b/src/cryptonote_basic/cryptonote_format_utils.cpp @@ -1011,7 +1011,19 @@ namespace cryptonote crypto::hash get_transaction_prunable_hash(const transaction& t, const cryptonote::blobdata *blobdata) { crypto::hash res; + if (t.is_prunable_hash_valid()) + { +#ifdef ENABLE_HASH_CASH_INTEGRITY_CHECK + CHECK_AND_ASSERT_THROW_MES(!calculate_transaction_prunable_hash(t, blobdata, res) || t.hash == res, "tx hash cash integrity failure"); +#endif + res = t.prunable_hash; + ++tx_hashes_cached_count; + return res; + } + + ++tx_hashes_calculated_count; CHECK_AND_ASSERT_THROW_MES(calculate_transaction_prunable_hash(t, blobdata, res), "Failed to calculate tx prunable hash"); + t.set_prunable_hash(res); return res; } //--------------------------------------------------------------- @@ -1047,11 +1059,14 @@ namespace cryptonote // the tx hash is the hash of the 3 hashes crypto::hash res = cn_fast_hash(hashes, sizeof(hashes)); + t.set_hash(res); return res; } //--------------------------------------------------------------- bool calculate_transaction_hash(const transaction& t, crypto::hash& res, size_t* blob_size) { + CHECK_AND_ASSERT_MES(!t.pruned, false, "Cannot calculate the hash of a pruned transaction"); + // v1 transactions hash the entire blob if (t.version == 1) { @@ -1091,8 +1106,7 @@ namespace cryptonote { if (!t.is_blob_size_valid()) { - t.blob_size = blob.size(); - t.set_blob_size_valid(true); + t.set_blob_size(blob.size()); } *blob_size = t.blob_size; } @@ -1112,8 +1126,7 @@ namespace cryptonote { if (!t.is_blob_size_valid()) { - t.blob_size = get_object_blobsize(t); - t.set_blob_size_valid(true); + t.set_blob_size(get_object_blobsize(t)); } *blob_size = t.blob_size; } @@ -1124,12 +1137,10 @@ namespace cryptonote bool ret = calculate_transaction_hash(t, res, blob_size); if (!ret) return false; - t.hash = res; - t.set_hash_valid(true); + t.set_hash(res); if (blob_size) { - t.blob_size = *blob_size; - t.set_blob_size_valid(true); + t.set_blob_size(*blob_size); } return true; } @@ -1206,8 +1217,7 @@ namespace cryptonote bool ret = calculate_block_hash(b, res); if (!ret) return false; - b.hash = res; - b.set_hash_valid(true); + b.set_hash(res); return true; } //--------------------------------------------------------------- @@ -1273,8 +1283,7 @@ namespace cryptonote { calculate_block_hash(b, *block_hash, &b_blob); ++block_hashes_calculated_count; - b.hash = *block_hash; - b.set_hash_valid(true); + b.set_hash(*block_hash); } return true; } diff --git a/src/cryptonote_config.h b/src/cryptonote_config.h index 173b454f6..23c9dbf41 100644 --- a/src/cryptonote_config.h +++ b/src/cryptonote_config.h @@ -163,7 +163,7 @@ #define PER_KB_FEE_QUANTIZATION_DECIMALS 8 -#define HASH_OF_HASHES_STEP 256 +#define HASH_OF_HASHES_STEP 512 #define DEFAULT_TXPOOL_MAX_WEIGHT 648000000ull // 3 days at 300000, in bytes diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 8ed0e526a..6f57d28a2 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -1255,7 +1255,7 @@ bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_bl } if(base_reward + fee < money_in_use) { - MERROR_VER("coinbase transaction spend too much money (" << print_money(money_in_use) << "). Block reward is " << print_money(base_reward + fee) << "(" << print_money(base_reward) << "+" << print_money(fee) << ")"); + MERROR_VER("coinbase transaction spend too much money (" << print_money(money_in_use) << "). Block reward is " << print_money(base_reward + fee) << "(" << print_money(base_reward) << "+" << print_money(fee) << "), cumulative_block_weight " << cumulative_block_weight); return false; } // From hard fork 2, we allow a miner to claim less block reward than is allowed, in case a miner wants less dust @@ -1907,8 +1907,9 @@ bool Blockchain::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NO std::vector<std::pair<cryptonote::blobdata,block>> blocks; get_blocks(arg.blocks, blocks, rsp.missed_ids); - for (auto& bl: blocks) + for (size_t i = 0; i < blocks.size(); ++i) { + auto& bl = blocks[i]; std::vector<crypto::hash> missed_tx_ids; rsp.blocks.push_back(block_complete_entry()); @@ -1916,8 +1917,8 @@ bool Blockchain::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NO // FIXME: s/rsp.missed_ids/missed_tx_id/ ? Seems like rsp.missed_ids // is for missed blocks, not missed transactions as well. - get_transactions_blobs(bl.second.tx_hashes, e.txs, missed_tx_ids); - + e.pruned = arg.prune; + get_transactions_blobs(bl.second.tx_hashes, e.txs, missed_tx_ids, arg.prune); if (missed_tx_ids.size() != 0) { // do not display an error if the peer asked for an unpruned block which we are not meant to have @@ -1938,6 +1939,9 @@ bool Blockchain::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NO //pack block e.block = std::move(bl.first); + e.block_weight = 0; + if (arg.prune && m_db->block_exists(arg.blocks[i])) + e.block_weight = m_db->get_block_weight(m_db->get_block_height(arg.blocks[i])); } return true; @@ -2216,23 +2220,95 @@ bool Blockchain::get_blocks(const t_ids_container& block_ids, t_blocks_container return true; } //------------------------------------------------------------------ +static bool fill(BlockchainDB *db, const crypto::hash &tx_hash, cryptonote::blobdata &tx, bool pruned) +{ + if (pruned) + { + if (!db->get_pruned_tx_blob(tx_hash, tx)) + { + MDEBUG("Pruned transaction blob not found for " << tx_hash); + return false; + } + } + else + { + if (!db->get_tx_blob(tx_hash, tx)) + { + MDEBUG("Transaction blob not found for " << tx_hash); + return false; + } + } + return true; +} +//------------------------------------------------------------------ +static bool fill(BlockchainDB *db, const crypto::hash &tx_hash, tx_blob_entry &tx, bool pruned) +{ + if (!fill(db, tx_hash, tx.blob, pruned)) + return false; + if (pruned) + { + if (is_v1_tx(tx.blob)) + { + // v1 txes aren't pruned, so fetch the whole thing + cryptonote::blobdata prunable_blob; + if (!db->get_prunable_tx_blob(tx_hash, prunable_blob)) + { + MDEBUG("Prunable transaction blob not found for " << tx_hash); + return false; + } + tx.blob.append(prunable_blob); + tx.prunable_hash = crypto::null_hash; + } + else + { + if (!db->get_prunable_tx_hash(tx_hash, tx.prunable_hash)) + { + MDEBUG("Prunable transaction data hash not found for " << tx_hash); + return false; + } + } + } + return true; +} +//------------------------------------------------------------------ //TODO: return type should be void, throw on exception // alternatively, return true only if no transactions missed -template<class t_ids_container, class t_tx_container, class t_missed_container> -bool Blockchain::get_transactions_blobs(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs, bool pruned) const +bool Blockchain::get_transactions_blobs(const std::vector<crypto::hash>& txs_ids, std::vector<cryptonote::blobdata>& txs, std::vector<crypto::hash>& missed_txs, bool pruned) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); - reserve_container(txs, txs_ids.size()); + txs.reserve(txs_ids.size()); for (const auto& tx_hash : txs_ids) { try { cryptonote::blobdata tx; - if (pruned && m_db->get_pruned_tx_blob(tx_hash, tx)) + if (fill(m_db, tx_hash, tx, pruned)) txs.push_back(std::move(tx)); - else if (!pruned && m_db->get_tx_blob(tx_hash, tx)) + else + missed_txs.push_back(tx_hash); + } + catch (const std::exception& e) + { + return false; + } + } + return true; +} +//------------------------------------------------------------------ +bool Blockchain::get_transactions_blobs(const std::vector<crypto::hash>& txs_ids, std::vector<tx_blob_entry>& txs, std::vector<crypto::hash>& missed_txs, bool pruned) const +{ + LOG_PRINT_L3("Blockchain::" << __func__); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + txs.reserve(txs_ids.size()); + for (const auto& tx_hash : txs_ids) + { + try + { + tx_blob_entry tx; + if (fill(m_db, tx_hash, tx, pruned)) txs.push_back(std::move(tx)); else missed_txs.push_back(tx_hash); @@ -2325,7 +2401,7 @@ bool Blockchain::get_transactions(const t_ids_container& txs_ids, t_tx_container // Find the split point between us and foreign blockchain and return // (by reference) the most recent common block hash along with up to // BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT additional (more recent) hashes. -bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, std::vector<crypto::hash>& hashes, uint64_t& start_height, uint64_t& current_height, bool clip_pruned) const +bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, std::vector<crypto::hash>& hashes, std::vector<uint64_t>* weights, uint64_t& start_height, uint64_t& current_height, bool clip_pruned) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -2342,25 +2418,34 @@ bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qbloc if (clip_pruned) { const uint32_t pruning_seed = get_blockchain_pruning_seed(); - start_height = tools::get_next_unpruned_block_height(start_height, current_height, pruning_seed); + if (start_height < tools::get_next_unpruned_block_height(start_height, current_height, pruning_seed)) + { + MDEBUG("We only have a pruned version of the common ancestor"); + return false; + } stop_height = tools::get_next_pruned_block_height(start_height, current_height, pruning_seed); } size_t count = 0; - hashes.reserve(std::min((size_t)(stop_height - start_height), (size_t)BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT)); + const size_t reserve = std::min((size_t)(stop_height - start_height), (size_t)BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT); + hashes.reserve(reserve); + if (weights) + weights->reserve(reserve); for(size_t i = start_height; i < stop_height && count < BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT; i++, count++) { hashes.push_back(m_db->get_block_hash_from_height(i)); + if (weights) + weights->push_back(m_db->get_block_weight(i)); } return true; } -bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const +bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, bool clip_pruned, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); - bool result = find_blockchain_supplement(qblock_ids, resp.m_block_ids, resp.start_height, resp.total_height, true); + bool result = find_blockchain_supplement(qblock_ids, resp.m_block_ids, &resp.m_block_weights, resp.start_height, resp.total_height, clip_pruned); if (result) { cryptonote::difficulty_type wide_cumulative_difficulty = m_db->get_block_cumulative_difficulty(resp.total_height - 1); @@ -2799,18 +2884,24 @@ bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_pr // II if (rv.type == rct::RCTTypeFull) { - rv.p.MGs.resize(1); - rv.p.MGs[0].II.resize(tx.vin.size()); - for (size_t n = 0; n < tx.vin.size(); ++n) - rv.p.MGs[0].II[n] = rct::ki2rct(boost::get<txin_to_key>(tx.vin[n]).k_image); + if (!tx.pruned) + { + rv.p.MGs.resize(1); + rv.p.MGs[0].II.resize(tx.vin.size()); + for (size_t n = 0; n < tx.vin.size(); ++n) + rv.p.MGs[0].II[n] = rct::ki2rct(boost::get<txin_to_key>(tx.vin[n]).k_image); + } } else if (rv.type == rct::RCTTypeSimple || rv.type == rct::RCTTypeBulletproof || rv.type == rct::RCTTypeBulletproof2) { - CHECK_AND_ASSERT_MES(rv.p.MGs.size() == tx.vin.size(), false, "Bad MGs size"); - for (size_t n = 0; n < tx.vin.size(); ++n) + if (!tx.pruned) { - rv.p.MGs[n].II.resize(1); - rv.p.MGs[n].II[0] = rct::ki2rct(boost::get<txin_to_key>(tx.vin[n]).k_image); + CHECK_AND_ASSERT_MES(rv.p.MGs.size() == tx.vin.size(), false, "Bad MGs size"); + for (size_t n = 0; n < tx.vin.size(); ++n) + { + rv.p.MGs[n].II.resize(1); + rv.p.MGs[n].II[0] = rct::ki2rct(boost::get<txin_to_key>(tx.vin[n]).k_image); + } } } else @@ -2836,6 +2927,10 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, if(pmax_used_block_height) *pmax_used_block_height = 0; + // pruned txes are skipped, as they're only allowed in sync-pruned-blocks mode, which is within the builtin hashes + if (tx.pruned) + return true; + crypto::hash tx_prefix_hash = get_transaction_prefix_hash(tx); const uint8_t hf_version = m_hardfork->get_current_version(); @@ -3539,9 +3634,9 @@ bool Blockchain::flush_txes_from_pool(const std::vector<crypto::hash> &txids) cryptonote::blobdata txblob; size_t tx_weight; uint64_t fee; - bool relayed, do_not_relay, double_spend_seen; + bool relayed, do_not_relay, double_spend_seen, pruned; MINFO("Removing txid " << txid << " from the pool"); - if(m_tx_pool.have_tx(txid) && !m_tx_pool.take_tx(txid, tx, txblob, tx_weight, fee, relayed, do_not_relay, double_spend_seen)) + if(m_tx_pool.have_tx(txid) && !m_tx_pool.take_tx(txid, tx, txblob, tx_weight, fee, relayed, do_not_relay, double_spend_seen, pruned)) { MERROR("Failed to remove txid " << txid << " from the pool"); res = false; @@ -3639,7 +3734,7 @@ leave: #if defined(PER_BLOCK_CHECKPOINT) if (blockchain_height < m_blocks_hash_check.size()) { - const auto &expected_hash = m_blocks_hash_check[blockchain_height]; + const auto &expected_hash = m_blocks_hash_check[blockchain_height].first; if (expected_hash != crypto::null_hash) { if (memcmp(&id, &expected_hash, sizeof(hash)) != 0) @@ -3713,6 +3808,7 @@ leave: uint64_t t_exists = 0; uint64_t t_pool = 0; uint64_t t_dblspnd = 0; + uint64_t n_pruned = 0; TIME_MEASURE_FINISH(t3); // XXX old code adds miner tx here @@ -3728,7 +3824,7 @@ leave: blobdata txblob; size_t tx_weight = 0; uint64_t fee = 0; - bool relayed = false, do_not_relay = false, double_spend_seen = false; + bool relayed = false, do_not_relay = false, double_spend_seen = false, pruned = false; TIME_MEASURE_START(aa); // XXX old code does not check whether tx exists @@ -3745,13 +3841,15 @@ leave: TIME_MEASURE_START(bb); // get transaction with hash <tx_id> from tx_pool - if(!m_tx_pool.take_tx(tx_id, tx_tmp, txblob, tx_weight, fee, relayed, do_not_relay, double_spend_seen)) + if(!m_tx_pool.take_tx(tx_id, tx_tmp, txblob, tx_weight, fee, relayed, do_not_relay, double_spend_seen, pruned)) { MERROR_VER("Block with id: " << id << " has at least one unknown transaction with id: " << tx_id); bvc.m_verifivation_failed = true; return_tx_to_pool(txs); goto leave; } + if (pruned) + ++n_pruned; TIME_MEASURE_FINISH(bb); t_pool += bb; @@ -3822,6 +3920,17 @@ leave: cumulative_block_weight += tx_weight; } + // if we were syncing pruned blocks + if (n_pruned > 0) + { + if (blockchain_height >= m_blocks_hash_check.size() || m_blocks_hash_check[blockchain_height].second == 0) + { + MERROR("Block at " << blockchain_height << " is pruned, but we do not have a weight for it"); + goto leave; + } + cumulative_block_weight = m_blocks_hash_check[blockchain_height].second; + } + m_blocks_txs_check.clear(); TIME_MEASURE_START(vmt); @@ -4262,11 +4371,13 @@ void Blockchain::output_scan_worker(const uint64_t amount, const std::vector<uin } } -uint64_t Blockchain::prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes) +uint64_t Blockchain::prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes, const std::vector<uint64_t> &weights) { // new: . . . . . X X X X X . . . . . . // pre: A A A A B B B B C C C C D D D D + CHECK_AND_ASSERT_MES(weights.empty() || weights.size() == hashes.size(), 0, "Unexpected weights size"); + // easy case: height >= hashes if (height >= m_blocks_hash_of_hashes.size() * HASH_OF_HASHES_STEP) return hashes.size(); @@ -4285,8 +4396,11 @@ uint64_t Blockchain::prevalidate_block_hashes(uint64_t height, const std::vector return hashes.size(); // build hashes vector to hash hashes together - std::vector<crypto::hash> data; - data.reserve(hashes.size() + HASH_OF_HASHES_STEP - 1); // may be a bit too much + std::vector<crypto::hash> data_hashes; + std::vector<uint64_t> data_weights; + data_hashes.reserve(hashes.size() + HASH_OF_HASHES_STEP - 1); // may be a bit too much + if (!weights.empty()) + data_weights.reserve(data_hashes.size()); // we expect height to be either equal or a bit below db height bool disconnected = (height > m_db->height()); @@ -4301,18 +4415,24 @@ uint64_t Blockchain::prevalidate_block_hashes(uint64_t height, const std::vector // we might need some already in the chain for the first part of the first hash for (uint64_t h = first_index * HASH_OF_HASHES_STEP; h < height; ++h) { - data.push_back(m_db->get_block_hash_from_height(h)); + data_hashes.push_back(m_db->get_block_hash_from_height(h)); + if (!weights.empty()) + data_weights.push_back(m_db->get_block_weight(h)); } pop = 0; } // push the data to check - for (const auto &h: hashes) + for (size_t i = 0; i < hashes.size(); ++i) { if (pop) --pop; else - data.push_back(h); + { + data_hashes.push_back(hashes[i]); + if (!weights.empty()) + data_weights.push_back(weights[i]); + } } // hash and check @@ -4322,12 +4442,17 @@ uint64_t Blockchain::prevalidate_block_hashes(uint64_t height, const std::vector if (n < m_blocks_hash_of_hashes.size()) { // if the last index isn't fully filled, we can't tell if valid - if (data.size() < (n - first_index) * HASH_OF_HASHES_STEP + HASH_OF_HASHES_STEP) + if (data_hashes.size() < (n - first_index) * HASH_OF_HASHES_STEP + HASH_OF_HASHES_STEP) break; crypto::hash hash; - cn_fast_hash(data.data() + (n - first_index) * HASH_OF_HASHES_STEP, HASH_OF_HASHES_STEP * sizeof(crypto::hash), hash); - bool valid = hash == m_blocks_hash_of_hashes[n]; + cn_fast_hash(data_hashes.data() + (n - first_index) * HASH_OF_HASHES_STEP, HASH_OF_HASHES_STEP * sizeof(crypto::hash), hash); + bool valid = hash == m_blocks_hash_of_hashes[n].first; + if (valid && !weights.empty()) + { + cn_fast_hash(data_weights.data() + (n - first_index) * HASH_OF_HASHES_STEP, HASH_OF_HASHES_STEP * sizeof(uint64_t), hash); + valid &= hash == m_blocks_hash_of_hashes[n].second; + } // add to the known hashes array if (!valid) @@ -4339,9 +4464,15 @@ uint64_t Blockchain::prevalidate_block_hashes(uint64_t height, const std::vector size_t end = n * HASH_OF_HASHES_STEP + HASH_OF_HASHES_STEP; for (size_t i = n * HASH_OF_HASHES_STEP; i < end; ++i) { - CHECK_AND_ASSERT_MES(m_blocks_hash_check[i] == crypto::null_hash || m_blocks_hash_check[i] == data[i - first_index * HASH_OF_HASHES_STEP], + CHECK_AND_ASSERT_MES(m_blocks_hash_check[i].first == crypto::null_hash || m_blocks_hash_check[i].first == data_hashes[i - first_index * HASH_OF_HASHES_STEP], 0, "Consistency failure in m_blocks_hash_check construction"); - m_blocks_hash_check[i] = data[i - first_index * HASH_OF_HASHES_STEP]; + m_blocks_hash_check[i].first = data_hashes[i - first_index * HASH_OF_HASHES_STEP]; + if (!weights.empty()) + { + CHECK_AND_ASSERT_MES(m_blocks_hash_check[i].second == 0 || m_blocks_hash_check[i].second == data_weights[i - first_index * HASH_OF_HASHES_STEP], + 0, "Consistency failure in m_blocks_hash_check construction"); + m_blocks_hash_check[i].second = data_weights[i - first_index * HASH_OF_HASHES_STEP]; + } } usable += HASH_OF_HASHES_STEP; } @@ -4358,6 +4489,18 @@ uint64_t Blockchain::prevalidate_block_hashes(uint64_t height, const std::vector return usable; } +bool Blockchain::has_block_weights(uint64_t height, uint64_t nblocks) const +{ + CHECK_AND_ASSERT_MES(nblocks > 0, false, "nblocks is 0"); + uint64_t last_block_height = height + nblocks - 1; + if (last_block_height >= m_blocks_hash_check.size()) + return false; + for (uint64_t h = height; h <= last_block_height; ++h) + if (m_blocks_hash_check[h].second == 0) + return false; + return true; +} + //------------------------------------------------------------------ // ND: Speedups: // 1. Thread long_hash computations if possible (m_max_prepare_blocks_threads = nthreads, default = 4) @@ -4399,7 +4542,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete bytes += entry.block.size(); for (const auto &tx_blob : entry.txs) { - bytes += tx_blob.size(); + bytes += tx_blob.blob.size(); } total_txs += entry.txs.size(); } @@ -4555,7 +4698,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete crypto::hash &tx_prefix_hash = txes[tx_index].second; ++tx_index; - if (!parse_and_validate_tx_base_from_blob(tx_blob, tx)) + if (!parse_and_validate_tx_base_from_blob(tx_blob.blob, tx)) SCAN_TABLE_QUIT("Could not parse tx from incoming blocks."); cryptonote::get_transaction_prefix_hash(tx, tx_prefix_hash); @@ -4887,7 +5030,7 @@ void Blockchain::cancel() } #if defined(PER_BLOCK_CHECKPOINT) -static const char expected_block_hashes_hash[] = "7dafb40b414a0e59bfced6682ef519f0b416bc914dd3d622b72e0dd1a47117c2"; +static const char expected_block_hashes_hash[] = "95e60612c1a16f4cd992c335b66daabd98e2d351c2b02b66e43ced0296848d33"; void Blockchain::load_compiled_in_block_hashes(const GetCheckpointsCallback& get_checkpoints) { if (get_checkpoints == nullptr || !m_fast_sync) @@ -4931,19 +5074,21 @@ void Blockchain::load_compiled_in_block_hashes(const GetCheckpointsCallback& get MERROR("Block hash data is too large"); return; } - const size_t size_needed = 4 + nblocks * sizeof(crypto::hash); + const size_t size_needed = 4 + nblocks * (sizeof(crypto::hash) * 2); if(nblocks > 0 && nblocks > (m_db->height() + HASH_OF_HASHES_STEP - 1) / HASH_OF_HASHES_STEP && checkpoints.size() >= size_needed) { p += sizeof(uint32_t); m_blocks_hash_of_hashes.reserve(nblocks); for (uint32_t i = 0; i < nblocks; i++) { - crypto::hash hash; - memcpy(hash.data, p, sizeof(hash.data)); - p += sizeof(hash.data); - m_blocks_hash_of_hashes.push_back(hash); + crypto::hash hash_hashes, hash_weights; + memcpy(hash_hashes.data, p, sizeof(hash_hashes.data)); + p += sizeof(hash_hashes.data); + memcpy(hash_weights.data, p, sizeof(hash_weights.data)); + p += sizeof(hash_weights.data); + m_blocks_hash_of_hashes.push_back(std::make_pair(hash_hashes, hash_weights)); } - m_blocks_hash_check.resize(m_blocks_hash_of_hashes.size() * HASH_OF_HASHES_STEP, crypto::null_hash); + m_blocks_hash_check.resize(m_blocks_hash_of_hashes.size() * HASH_OF_HASHES_STEP, std::make_pair(crypto::null_hash, 0)); MINFO(nblocks << " block hashes loaded"); // FIXME: clear tx_pool because the process might have been @@ -4958,13 +5103,13 @@ void Blockchain::load_compiled_in_block_hashes(const GetCheckpointsCallback& get size_t tx_weight; uint64_t fee; - bool relayed, do_not_relay, double_spend_seen; + bool relayed, do_not_relay, double_spend_seen, pruned; transaction pool_tx; blobdata txblob; for(const transaction &tx : txs) { crypto::hash tx_hash = get_transaction_hash(tx); - m_tx_pool.take_tx(tx_hash, pool_tx, txblob, tx_weight, fee, relayed, do_not_relay, double_spend_seen); + m_tx_pool.take_tx(tx_hash, pool_tx, txblob, tx_weight, fee, relayed, do_not_relay, double_spend_seen, pruned); } } } @@ -5037,6 +5182,5 @@ void Blockchain::cache_block_template(const block &b, const cryptonote::account_ namespace cryptonote { template bool Blockchain::get_transactions(const std::vector<crypto::hash>&, std::vector<transaction>&, std::vector<crypto::hash>&) const; -template bool Blockchain::get_transactions_blobs(const std::vector<crypto::hash>&, std::vector<cryptonote::blobdata>&, std::vector<crypto::hash>&, bool) const; template bool Blockchain::get_split_transactions_blobs(const std::vector<crypto::hash>&, std::vector<std::tuple<crypto::hash, cryptonote::blobdata, crypto::hash, cryptonote::blobdata>>&, std::vector<crypto::hash>&) const; } diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index f58059a6d..fff564d24 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -381,13 +381,14 @@ namespace cryptonote * * @param qblock_ids the foreign chain's "short history" (see get_short_chain_history) * @param hashes the hashes to be returned, return-by-reference + * @param weights the block weights to be returned, return-by-reference * @param start_height the start height, return-by-reference * @param current_height the current blockchain height, return-by-reference * @param clip_pruned whether to constrain results to unpruned data * * @return true if a block found in common, else false */ - bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, std::vector<crypto::hash>& hashes, uint64_t& start_height, uint64_t& current_height, bool clip_pruned) const; + bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, std::vector<crypto::hash>& hashes, std::vector<uint64_t>* weights, uint64_t& start_height, uint64_t& current_height, bool clip_pruned) const; /** * @brief get recent block hashes for a foreign chain @@ -397,11 +398,12 @@ namespace cryptonote * BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT additional (more recent) hashes. * * @param qblock_ids the foreign chain's "short history" (see get_short_chain_history) + * @param clip_pruned clip pruned blocks if true, include them otherwise * @param resp return-by-reference the split height and subsequent blocks' hashes * * @return true if a block found in common, else false */ - bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const; + bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, bool clip_pruned, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const; /** * @brief find the most recent common point between ours and a foreign chain @@ -675,8 +677,8 @@ namespace cryptonote * * @return false if an unexpected exception occurs, else true */ - template<class t_ids_container, class t_tx_container, class t_missed_container> - bool get_transactions_blobs(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs, bool pruned = false) const; + bool get_transactions_blobs(const std::vector<crypto::hash>& txs_ids, std::vector<cryptonote::blobdata>& txs, std::vector<crypto::hash>& missed_txs, bool pruned = false) const; + bool get_transactions_blobs(const std::vector<crypto::hash>& txs_ids, std::vector<tx_blob_entry>& txs, std::vector<crypto::hash>& missed_txs, bool pruned = false) const; template<class t_ids_container, class t_tx_container, class t_missed_container> bool get_split_transactions_blobs(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const; template<class t_ids_container, class t_tx_container, class t_missed_container> @@ -963,9 +965,8 @@ namespace cryptonote cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid) const; bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)>, bool include_blob = false, bool include_unrelayed_txes = true) const; - bool is_within_compiled_block_hash_area(uint64_t height) const; bool is_within_compiled_block_hash_area() const { return is_within_compiled_block_hash_area(m_db->height()); } - uint64_t prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes); + uint64_t prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes, const std::vector<uint64_t> &weights); uint32_t get_blockchain_pruning_seed() const { return m_db->get_blockchain_pruning_seed(); } bool prune_blockchain(uint32_t pruning_seed = 0); bool update_blockchain_pruning(); @@ -995,6 +996,21 @@ namespace cryptonote */ void pop_blocks(uint64_t nblocks); + /** + * @brief checks whether a given block height is included in the precompiled block hash area + * + * @param height the height to check for + */ + bool is_within_compiled_block_hash_area(uint64_t height) const; + + /** + * @brief checks whether we have known weights for the given block heights + * + * @param height the start height to check for + * @param nblocks how many blocks to check from that height + */ + bool has_block_weights(uint64_t height, uint64_t nblocks) const; + #ifndef IN_UNIT_TESTS private: #endif @@ -1022,8 +1038,8 @@ namespace cryptonote std::unordered_map<crypto::hash, crypto::hash> m_blocks_longhash_table; // SHA-3 hashes for each block and for fast pow checking - std::vector<crypto::hash> m_blocks_hash_of_hashes; - std::vector<crypto::hash> m_blocks_hash_check; + std::vector<std::pair<crypto::hash, crypto::hash>> m_blocks_hash_of_hashes; + std::vector<std::pair<crypto::hash, uint64_t>> m_blocks_hash_check; std::vector<crypto::hash> m_blocks_txs_check; blockchain_db_sync_mode m_db_sync_mode; diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index a3a92ab60..785ee7db4 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -113,6 +113,10 @@ namespace cryptonote , "Set maximum size of block download queue in bytes (0 for default)" , 0 }; + const command_line::arg_descriptor<bool> arg_sync_pruned_blocks = { + "sync-pruned-blocks" + , "Allow syncing from nodes with only pruned blocks" + }; static const command_line::arg_descriptor<bool> arg_test_drop_download = { "test-drop-download" @@ -323,6 +327,7 @@ namespace cryptonote command_line::add_arg(desc, arg_offline); command_line::add_arg(desc, arg_disable_dns_checkpoints); command_line::add_arg(desc, arg_block_download_max_size); + command_line::add_arg(desc, arg_sync_pruned_blocks); command_line::add_arg(desc, arg_max_txpool_weight); command_line::add_arg(desc, arg_pad_transactions); command_line::add_arg(desc, arg_block_notify); @@ -744,13 +749,13 @@ namespace cryptonote return false; } //----------------------------------------------------------------------------------------------- - bool core::handle_incoming_tx_pre(const blobdata& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, bool keeped_by_block, bool relayed, bool do_not_relay) + bool core::handle_incoming_tx_pre(const tx_blob_entry& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, bool keeped_by_block, bool relayed, bool do_not_relay) { tvc = boost::value_initialized<tx_verification_context>(); - if(tx_blob.size() > get_max_tx_size()) + if(tx_blob.blob.size() > get_max_tx_size()) { - LOG_PRINT_L1("WRONG TRANSACTION BLOB, too big size " << tx_blob.size() << ", rejected"); + LOG_PRINT_L1("WRONG TRANSACTION BLOB, too big size " << tx_blob.blob.size() << ", rejected"); tvc.m_verifivation_failed = true; tvc.m_too_big = true; return false; @@ -758,7 +763,23 @@ namespace cryptonote tx_hash = crypto::null_hash; - if(!parse_tx_from_blob(tx, tx_hash, tx_blob)) + bool r; + if (tx_blob.prunable_hash == crypto::null_hash) + { + r = parse_tx_from_blob(tx, tx_hash, tx_blob.blob); + } + else + { + r = parse_and_validate_tx_base_from_blob(tx_blob.blob, tx); + if (r) + { + tx.set_prunable_hash(tx_blob.prunable_hash); + tx_hash = cryptonote::get_pruned_transaction_hash(tx, tx_blob.prunable_hash); + tx.set_hash(tx_hash); + } + } + + if (!r) { LOG_PRINT_L1("WRONG TRANSACTION BLOB, Failed to parse, rejected"); tvc.m_verifivation_failed = true; @@ -784,6 +805,7 @@ namespace cryptonote if (tx.version == 0 || tx.version > max_tx_version) { // v2 is the latest one we know + MERROR_VER("Bad tx version (" << tx.version << ", max is " << max_tx_version << ")"); tvc.m_verifivation_failed = true; return false; } @@ -791,7 +813,7 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- - bool core::handle_incoming_tx_post(const blobdata& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, bool keeped_by_block, bool relayed, bool do_not_relay) + bool core::handle_incoming_tx_post(const tx_blob_entry& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, bool keeped_by_block, bool relayed, bool do_not_relay) { if(!check_tx_syntax(tx)) { @@ -920,7 +942,7 @@ namespace cryptonote return ret; } //----------------------------------------------------------------------------------------------- - bool core::handle_incoming_txs(const std::vector<blobdata>& tx_blobs, std::vector<tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) + bool core::handle_incoming_txs(const std::vector<tx_blob_entry>& tx_blobs, std::vector<tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { TRY_ENTRY(); CRITICAL_REGION_LOCAL(m_incoming_tx_lock); @@ -931,7 +953,7 @@ namespace cryptonote tvc.resize(tx_blobs.size()); tools::threadpool& tpool = tools::threadpool::getInstance(); tools::threadpool::waiter waiter; - std::vector<blobdata>::const_iterator it = tx_blobs.begin(); + std::vector<tx_blob_entry>::const_iterator it = tx_blobs.begin(); for (size_t i = 0; i < tx_blobs.size(); i++, ++it) { tpool.submit(&waiter, [&, i, it] { try @@ -1003,8 +1025,12 @@ namespace cryptonote if (already_have[i]) continue; - const size_t weight = get_transaction_weight(results[i].tx, it->size()); - ok &= add_new_tx(results[i].tx, results[i].hash, tx_blobs[i], weight, tvc[i], keeped_by_block, relayed, do_not_relay); + // if it's a pruned tx from an incoming block, we'll get a weight that's technically + // different from the actual transaction weight, but it's OK for our use. Those txes + // will be ignored when mining, and using that "pruned" weight seems appropriate for + // keeping the txpool size constrained + const uint64_t weight = results[i].tx.pruned ? 0 : get_transaction_weight(results[i].tx, it->blob.size()); + ok &= add_new_tx(results[i].tx, results[i].hash, tx_blobs[i].blob, weight, tvc[i], keeped_by_block, relayed, do_not_relay); if(tvc[i].m_verifivation_failed) {MERROR_VER("Transaction verification failed: " << results[i].hash);} else if(tvc[i].m_verifivation_impossible) @@ -1018,9 +1044,9 @@ namespace cryptonote CATCH_ENTRY_L0("core::handle_incoming_txs()", false); } //----------------------------------------------------------------------------------------------- - bool core::handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) + bool core::handle_incoming_tx(const tx_blob_entry& tx_blob, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { - std::vector<cryptonote::blobdata> tx_blobs; + std::vector<tx_blob_entry> tx_blobs; tx_blobs.push_back(tx_blob); std::vector<tx_verification_context> tvcv(1); bool r = handle_incoming_txs(tx_blobs, tvcv, keeped_by_block, relayed, do_not_relay); @@ -1028,6 +1054,11 @@ namespace cryptonote return r; } //----------------------------------------------------------------------------------------------- + bool core::handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) + { + return handle_incoming_tx({tx_blob, crypto::null_hash}, tvc, keeped_by_block, relayed, do_not_relay); + } + //----------------------------------------------------------------------------------------------- bool core::get_stat_info(core_stat_info& st_inf) const { st_inf.mining_speed = m_miner.get_speed(); @@ -1290,9 +1321,9 @@ namespace cryptonote return m_blockchain_storage.create_block_template(b, prev_block, adr, diffic, height, expected_reward, ex_nonce); } //----------------------------------------------------------------------------------------------- - bool core::find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const + bool core::find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, bool clip_pruned, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const { - return m_blockchain_storage.find_blockchain_supplement(qblock_ids, resp); + return m_blockchain_storage.find_blockchain_supplement(qblock_ids, clip_pruned, resp); } //----------------------------------------------------------------------------------------------- bool core::find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_count) const @@ -1338,7 +1369,7 @@ namespace cryptonote { cryptonote::blobdata txblob; CHECK_AND_ASSERT_THROW_MES(pool.get_transaction(tx_hash, txblob), "Transaction not found in pool"); - bce.txs.push_back(txblob); + bce.txs.push_back({txblob, crypto::null_hash}); } return bce; } @@ -1391,7 +1422,7 @@ namespace cryptonote block_to_blob(b, arg.b.block); //pack transactions for(auto& tx: txs) - arg.b.txs.push_back(tx); + arg.b.txs.push_back({tx, crypto::null_hash}); m_pprotocol->relay_block(arg, exclude_context); } @@ -1901,9 +1932,9 @@ namespace cryptonote return m_target_blockchain_height; } //----------------------------------------------------------------------------------------------- - uint64_t core::prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes) + uint64_t core::prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes, const std::vector<uint64_t> &weights) { - return get_blockchain_storage().prevalidate_block_hashes(height, hashes); + return get_blockchain_storage().prevalidate_block_hashes(height, hashes, weights); } //----------------------------------------------------------------------------------------------- uint64_t core::get_free_space() const @@ -1923,6 +1954,16 @@ namespace cryptonote return get_blockchain_storage().prune_blockchain(pruning_seed); } //----------------------------------------------------------------------------------------------- + bool core::is_within_compiled_block_hash_area(uint64_t height) const + { + return get_blockchain_storage().is_within_compiled_block_hash_area(height); + } + //----------------------------------------------------------------------------------------------- + bool core::has_block_weights(uint64_t height, uint64_t nblocks) const + { + return get_blockchain_storage().has_block_weights(height, nblocks); + } + //----------------------------------------------------------------------------------------------- std::time_t core::get_start_time() const { return start_time; diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index badbaf936..0db6350be 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -64,6 +64,7 @@ namespace cryptonote extern const command_line::arg_descriptor<difficulty_type> arg_fixed_difficulty; extern const command_line::arg_descriptor<bool> arg_offline; extern const command_line::arg_descriptor<size_t> arg_block_download_max_size; + extern const command_line::arg_descriptor<bool> arg_sync_pruned_blocks; /************************************************************************/ /* */ @@ -120,6 +121,7 @@ namespace cryptonote * * @return true if the transaction was accepted, false otherwise */ + bool handle_incoming_tx(const tx_blob_entry& tx_blob, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); bool handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); /** @@ -136,7 +138,7 @@ namespace cryptonote * * @return true if the transactions were accepted, false otherwise */ - bool handle_incoming_txs(const std::vector<blobdata>& tx_blobs, std::vector<tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); + bool handle_incoming_txs(const std::vector<tx_blob_entry>& tx_blobs, std::vector<tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); /** * @brief handles an incoming block @@ -522,7 +524,7 @@ namespace cryptonote * * @note see Blockchain::find_blockchain_supplement(const std::list<crypto::hash>&, NOTIFY_RESPONSE_CHAIN_ENTRY::request&) const */ - bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const; + bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, bool clip_pruned, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const; /** * @copydoc Blockchain::find_blockchain_supplement(const uint64_t, const std::list<crypto::hash>&, std::vector<std::pair<cryptonote::blobdata, std::vector<cryptonote::blobdata> > >&, uint64_t&, uint64_t&, size_t) const @@ -779,7 +781,7 @@ namespace cryptonote * * @return number of usable blocks */ - uint64_t prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes); + uint64_t prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes, const std::vector<uint64_t> &weights); /** * @brief get free disk space on the blockchain partition @@ -825,6 +827,18 @@ namespace cryptonote */ bool check_blockchain_pruning(); + /** + * @brief checks whether a given block height is included in the precompiled block hash area + * + * @param height the height to check for + */ + bool is_within_compiled_block_hash_area(uint64_t height) const; + + /** + * @brief checks whether block weights are known for the given range + */ + bool has_block_weights(uint64_t height, uint64_t nblocks) const; + private: /** @@ -910,8 +924,8 @@ namespace cryptonote bool check_tx_semantic(const transaction& tx, bool keeped_by_block) const; void set_semantics_failed(const crypto::hash &tx_hash); - bool handle_incoming_tx_pre(const blobdata& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, bool keeped_by_block, bool relayed, bool do_not_relay); - bool handle_incoming_tx_post(const blobdata& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, bool keeped_by_block, bool relayed, bool do_not_relay); + bool handle_incoming_tx_pre(const tx_blob_entry& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, bool keeped_by_block, bool relayed, bool do_not_relay); + bool handle_incoming_tx_post(const tx_blob_entry &tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, bool keeped_by_block, bool relayed, bool do_not_relay); struct tx_verification_batch_info { const cryptonote::transaction *tx; crypto::hash tx_hash; tx_verification_context &tvc; bool &result; }; bool handle_incoming_tx_accumulated_batch(std::vector<tx_verification_batch_info> &tx_info, bool keeped_by_block); diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index 49d5a8ccc..1455c3ca3 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -247,6 +247,7 @@ namespace cryptonote meta.relayed = relayed; meta.do_not_relay = do_not_relay; meta.double_spend_seen = have_tx_keyimges_as_spent(tx); + meta.pruned = tx.pruned; meta.bf_padding = 0; memset(meta.padding, 0, sizeof(meta.padding)); try @@ -290,6 +291,7 @@ namespace cryptonote meta.relayed = relayed; meta.do_not_relay = do_not_relay; meta.double_spend_seen = false; + meta.pruned = tx.pruned; meta.bf_padding = 0; memset(meta.padding, 0, sizeof(meta.padding)); @@ -460,7 +462,7 @@ namespace cryptonote return true; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::take_tx(const crypto::hash &id, transaction &tx, cryptonote::blobdata &txblob, size_t& tx_weight, uint64_t& fee, bool &relayed, bool &do_not_relay, bool &double_spend_seen) + bool tx_memory_pool::take_tx(const crypto::hash &id, transaction &tx, cryptonote::blobdata &txblob, size_t& tx_weight, uint64_t& fee, bool &relayed, bool &do_not_relay, bool &double_spend_seen, bool &pruned) { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); @@ -482,7 +484,7 @@ namespace cryptonote { tx = ci->second; } - else if (!parse_and_validate_tx_from_blob(txblob, tx)) + else if (!(meta.pruned ? parse_and_validate_tx_base_from_blob(txblob, tx) : parse_and_validate_tx_from_blob(txblob, tx))) { MERROR("Failed to parse tx from txpool"); return false; @@ -496,6 +498,7 @@ namespace cryptonote relayed = meta.relayed; do_not_relay = meta.do_not_relay; double_spend_seen = meta.double_spend_seen; + pruned = meta.pruned; // remove first, in case this throws, so key images aren't removed m_blockchain.remove_txpool_tx(id); @@ -601,7 +604,7 @@ namespace cryptonote txs.reserve(m_blockchain.get_txpool_tx_count()); m_blockchain.for_all_txpool_txes([this, now, &txs](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *){ // 0 fee transactions are never relayed - if(meta.fee > 0 && !meta.do_not_relay && now - meta.last_relayed_time > get_relay_delay(now, meta.receive_time)) + if(!meta.pruned && meta.fee > 0 && !meta.do_not_relay && now - meta.last_relayed_time > get_relay_delay(now, meta.receive_time)) { // if the tx is older than half the max lifetime, we don't re-relay it, to avoid a problem // mentioned by smooth where nodes would flush txes at slightly different times, causing @@ -667,7 +670,7 @@ namespace cryptonote txs.reserve(m_blockchain.get_txpool_tx_count(include_unrelayed_txes)); m_blockchain.for_all_txpool_txes([&txs](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ transaction tx; - if (!parse_and_validate_tx_from_blob(*bd, tx)) + if (!(meta.pruned ? parse_and_validate_tx_base_from_blob(*bd, tx) : parse_and_validate_tx_from_blob(*bd, tx))) { MERROR("Failed to parse tx from txpool"); // continue @@ -791,7 +794,7 @@ namespace cryptonote txi.id_hash = epee::string_tools::pod_to_hex(txid); txi.tx_blob = *bd; transaction tx; - if (!parse_and_validate_tx_from_blob(*bd, tx)) + if (!(meta.pruned ? parse_and_validate_tx_base_from_blob(*bd, tx) : parse_and_validate_tx_from_blob(*bd, tx))) { MERROR("Failed to parse tx from txpool"); // continue @@ -863,7 +866,7 @@ namespace cryptonote m_blockchain.for_all_txpool_txes([&tx_infos, key_image_infos](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ cryptonote::rpc::tx_in_pool txi; txi.tx_hash = txid; - if (!parse_and_validate_tx_from_blob(*bd, txi.tx)) + if (!(meta.pruned ? parse_and_validate_tx_base_from_blob(*bd, txi.tx) : parse_and_validate_tx_from_blob(*bd, txi.tx))) { MERROR("Failed to parse tx from txpool"); // continue @@ -1143,7 +1146,7 @@ namespace cryptonote ss << "id: " << txid << std::endl; if (!short_format) { cryptonote::transaction tx; - if (!parse_and_validate_tx_from_blob(*txblob, tx)) + if (!(meta.pruned ? parse_and_validate_tx_base_from_blob(*txblob, tx) : parse_and_validate_tx_from_blob(*txblob, tx))) { MERROR("Failed to parse tx from txpool"); return true; // continue @@ -1199,6 +1202,12 @@ namespace cryptonote } LOG_PRINT_L2("Considering " << sorted_it->second << ", weight " << meta.weight << ", current block weight " << total_weight << "/" << max_total_weight << ", current coinbase " << print_money(best_coinbase)); + if (meta.pruned) + { + LOG_PRINT_L2(" tx is pruned"); + continue; + } + // Can not exceed maximum block weight if (max_total_weight < total_weight + meta.weight) { @@ -1322,7 +1331,7 @@ namespace cryptonote { cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(txid); cryptonote::transaction tx; - if (!parse_and_validate_tx_from_blob(txblob, tx)) + if (!parse_and_validate_tx_from_blob(txblob, tx)) // remove pruned ones on startup, they're meant to be temporary { MERROR("Failed to parse tx from txpool"); continue; diff --git a/src/cryptonote_core/tx_pool.h b/src/cryptonote_core/tx_pool.h index 877f2b82f..895014d17 100644 --- a/src/cryptonote_core/tx_pool.h +++ b/src/cryptonote_core/tx_pool.h @@ -139,10 +139,11 @@ namespace cryptonote * @param relayed return-by-reference was transaction relayed to us by the network? * @param do_not_relay return-by-reference is transaction not to be relayed to the network? * @param double_spend_seen return-by-reference was a double spend seen for that transaction? + * @param pruned return-by-reference is the tx pruned * * @return true unless the transaction cannot be found in the pool */ - bool take_tx(const crypto::hash &id, transaction &tx, cryptonote::blobdata &txblob, size_t& tx_weight, uint64_t& fee, bool &relayed, bool &do_not_relay, bool &double_spend_seen); + bool take_tx(const crypto::hash &id, transaction &tx, cryptonote::blobdata &txblob, size_t& tx_weight, uint64_t& fee, bool &relayed, bool &do_not_relay, bool &double_spend_seen, bool &pruned); /** * @brief checks if the pool has a transaction with the given hash diff --git a/src/cryptonote_protocol/block_queue.cpp b/src/cryptonote_protocol/block_queue.cpp index b4f9daa74..67f0b3e5d 100644 --- a/src/cryptonote_protocol/block_queue.cpp +++ b/src/cryptonote_protocol/block_queue.cpp @@ -228,13 +228,14 @@ bool block_queue::have(const crypto::hash &hash) const return have_blocks.find(hash) != have_blocks.end(); } -std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, uint32_t pruning_seed, uint64_t blockchain_height, const std::vector<crypto::hash> &block_hashes, boost::posix_time::ptime time) +std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, bool sync_pruned_blocks, uint32_t local_pruning_seed, uint32_t pruning_seed, uint64_t blockchain_height, const std::vector<std::pair<crypto::hash, uint64_t>> &block_hashes, boost::posix_time::ptime time) { boost::unique_lock<boost::recursive_mutex> lock(mutex); MDEBUG("reserve_span: first_block_height " << first_block_height << ", last_block_height " << last_block_height - << ", max " << max_blocks << ", seed " << epee::string_tools::to_string_hex(pruning_seed) << ", blockchain_height " << - blockchain_height << ", block hashes size " << block_hashes.size()); + << ", max " << max_blocks << ", peer seed " << epee::string_tools::to_string_hex(pruning_seed) << ", blockchain_height " << + blockchain_height << ", block hashes size " << block_hashes.size() << ", local seed " << epee::string_tools::to_string_hex(local_pruning_seed) + << ", sync_pruned_blocks " << sync_pruned_blocks); if (last_block_height < first_block_height || max_blocks == 0) { MDEBUG("reserve_span: early out: first_block_height " << first_block_height << ", last_block_height " << last_block_height << ", max_blocks " << max_blocks); @@ -248,22 +249,25 @@ std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_hei // skip everything we've already requested uint64_t span_start_height = last_block_height - block_hashes.size() + 1; - std::vector<crypto::hash>::const_iterator i = block_hashes.begin(); - while (i != block_hashes.end() && requested_internal(*i)) + std::vector<std::pair<crypto::hash, uint64_t>>::const_iterator i = block_hashes.begin(); + while (i != block_hashes.end() && requested_internal((*i).first)) { ++i; ++span_start_height; } - // if the peer's pruned for the starting block and its unpruned stripe comes next, start downloading from there - const uint32_t next_unpruned_height = tools::get_next_unpruned_block_height(span_start_height, blockchain_height, pruning_seed); - MDEBUG("reserve_span: next_unpruned_height " << next_unpruned_height << " from " << span_start_height << " and seed " - << epee::string_tools::to_string_hex(pruning_seed) << ", limit " << span_start_height + CRYPTONOTE_PRUNING_STRIPE_SIZE); - if (next_unpruned_height > span_start_height && next_unpruned_height < span_start_height + CRYPTONOTE_PRUNING_STRIPE_SIZE) + if (!sync_pruned_blocks) { - MDEBUG("We can download from next span: ideal height " << span_start_height << ", next unpruned height " << next_unpruned_height << - "(+" << next_unpruned_height - span_start_height << "), current seed " << pruning_seed); - span_start_height = next_unpruned_height; + // if the peer's pruned for the starting block and its unpruned stripe comes next, start downloading from there + const uint32_t next_unpruned_height = tools::get_next_unpruned_block_height(span_start_height, blockchain_height, pruning_seed); + MDEBUG("reserve_span: next_unpruned_height " << next_unpruned_height << " from " << span_start_height << " and seed " + << epee::string_tools::to_string_hex(pruning_seed) << ", limit " << span_start_height + CRYPTONOTE_PRUNING_STRIPE_SIZE); + if (next_unpruned_height > span_start_height && next_unpruned_height < span_start_height + CRYPTONOTE_PRUNING_STRIPE_SIZE) + { + MDEBUG("We can download from next span: ideal height " << span_start_height << ", next unpruned height " << next_unpruned_height << + "(+" << next_unpruned_height - span_start_height << "), current seed " << pruning_seed); + span_start_height = next_unpruned_height; + } } MDEBUG("span_start_height: " <<span_start_height); const uint64_t block_hashes_start_height = last_block_height - block_hashes.size() + 1; @@ -274,7 +278,7 @@ std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_hei } i = block_hashes.begin() + span_start_height - block_hashes_start_height; - while (i != block_hashes.end() && requested_internal(*i)) + while (i != block_hashes.end() && requested_internal((*i).first)) { ++i; ++span_start_height; @@ -282,9 +286,16 @@ std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_hei uint64_t span_length = 0; std::vector<crypto::hash> hashes; - while (i != block_hashes.end() && span_length < max_blocks && tools::has_unpruned_block(span_start_height + span_length, blockchain_height, pruning_seed)) + bool first_is_pruned = sync_pruned_blocks && !tools::has_unpruned_block(span_start_height + span_length, blockchain_height, local_pruning_seed); + while (i != block_hashes.end() && span_length < max_blocks && (sync_pruned_blocks || tools::has_unpruned_block(span_start_height + span_length, blockchain_height, pruning_seed))) { - hashes.push_back(*i); + // if we want to sync pruned blocks, stop at the first block for which we need full data + if (sync_pruned_blocks && first_is_pruned == tools::has_unpruned_block(span_start_height + span_length, blockchain_height, local_pruning_seed)) + { + MDEBUG("Stopping at " << span_start_height + span_length << " for peer on stripe " << tools::get_pruning_stripe(pruning_seed) << " as we need full data for " << tools::get_pruning_stripe(local_pruning_seed)); + break; + } + hashes.push_back((*i).first); ++i; ++span_length; } diff --git a/src/cryptonote_protocol/block_queue.h b/src/cryptonote_protocol/block_queue.h index 1bef01d67..93c6532e7 100644 --- a/src/cryptonote_protocol/block_queue.h +++ b/src/cryptonote_protocol/block_queue.h @@ -78,7 +78,7 @@ namespace cryptonote void print() const; std::string get_overview(uint64_t blockchain_height) const; bool has_unpruned_height(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed) const; - std::pair<uint64_t, uint64_t> reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, uint32_t pruning_seed, uint64_t blockchain_height, const std::vector<crypto::hash> &block_hashes, boost::posix_time::ptime time = boost::posix_time::microsec_clock::universal_time()); + std::pair<uint64_t, uint64_t> reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, bool sync_pruned_blocks, uint32_t local_pruning_seed, uint32_t pruning_seed, uint64_t blockchain_height, const std::vector<std::pair<crypto::hash, uint64_t>> &block_hashes, boost::posix_time::ptime time = boost::posix_time::microsec_clock::universal_time()); uint64_t get_next_needed_height(uint64_t blockchain_height) const; std::pair<uint64_t, uint64_t> get_next_span_if_scheduled(std::vector<crypto::hash> &hashes, boost::uuids::uuid &connection_id, boost::posix_time::ptime &time) const; void reset_next_span_time(boost::posix_time::ptime t = boost::posix_time::microsec_clock::universal_time()); diff --git a/src/cryptonote_protocol/cryptonote_protocol_defs.h b/src/cryptonote_protocol/cryptonote_protocol_defs.h index b2f8da399..3d594bf83 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_defs.h +++ b/src/cryptonote_protocol/cryptonote_protocol_defs.h @@ -116,14 +116,51 @@ namespace cryptonote /************************************************************************/ /* */ /************************************************************************/ + struct tx_blob_entry + { + blobdata blob; + crypto::hash prunable_hash; + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(blob) + KV_SERIALIZE_VAL_POD_AS_BLOB(prunable_hash) + END_KV_SERIALIZE_MAP() + + tx_blob_entry(const blobdata &bd = {}, const crypto::hash &h = crypto::null_hash): blob(bd), prunable_hash(h) {} + }; struct block_complete_entry { + bool pruned; blobdata block; - std::vector<blobdata> txs; + uint64_t block_weight; + std::vector<tx_blob_entry> txs; BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_OPT(pruned, false) KV_SERIALIZE(block) - KV_SERIALIZE(txs) + KV_SERIALIZE_OPT(block_weight, (uint64_t)0) + if (this_ref.pruned) + { + KV_SERIALIZE(txs) + } + else + { + std::vector<blobdata> txs; + if (is_store) + { + txs.reserve(this_ref.txs.size()); + for (const auto &e: this_ref.txs) txs.push_back(e.blob); + } + epee::serialization::selector<is_store>::serialize(txs, stg, hparent_section, "txs"); + if (!is_store) + { + block_complete_entry &self = const_cast<block_complete_entry&>(this_ref); + self.txs.clear(); + self.txs.reserve(txs.size()); + for (auto &e: txs) self.txs.push_back({std::move(e), crypto::null_hash}); + } + } END_KV_SERIALIZE_MAP() + + block_complete_entry(): pruned(false) {} }; @@ -176,8 +213,11 @@ namespace cryptonote struct request_t { std::vector<crypto::hash> blocks; + bool prune; + BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE_CONTAINER_POD_AS_BLOB(blocks) + KV_SERIALIZE_OPT(prune, false) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<request_t> request; @@ -229,9 +269,11 @@ namespace cryptonote struct request_t { std::list<crypto::hash> block_ids; /*IDs of the first 10 blocks are sequential, next goes with pow(2,n) offset, like 2, 4, 8, 16, 32, 64 and so on, and the last one is always genesis block */ + bool prune; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE_CONTAINER_POD_AS_BLOB(block_ids) + KV_SERIALIZE_OPT(prune, false) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<request_t> request; @@ -248,6 +290,7 @@ namespace cryptonote uint64_t cumulative_difficulty; uint64_t cumulative_difficulty_top64; std::vector<crypto::hash> m_block_ids; + std::vector<uint64_t> m_block_weights; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(start_height) @@ -255,6 +298,7 @@ namespace cryptonote KV_SERIALIZE(cumulative_difficulty) KV_SERIALIZE(cumulative_difficulty_top64) KV_SERIALIZE_CONTAINER_POD_AS_BLOB(m_block_ids) + KV_SERIALIZE_CONTAINER_POD_AS_BLOB(m_block_weights) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<request_t> request; diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.h b/src/cryptonote_protocol/cryptonote_protocol_handler.h index dcc5ec6ed..2666733da 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.h @@ -137,6 +137,7 @@ namespace cryptonote size_t get_synchronizing_connections_count(); bool on_connection_synchronized(); bool should_download_next_span(cryptonote_connection_context& context, bool standby); + bool should_ask_for_pruned_data(cryptonote_connection_context& context, uint64_t first_block_height, uint64_t nblocks, bool check_block_weights) const; void drop_connection(cryptonote_connection_context &context, bool add_fail, bool flush_all_spans); bool kick_idle_peers(); bool check_standby_peers(); @@ -164,6 +165,7 @@ namespace cryptonote uint64_t m_sync_spans_downloaded, m_sync_old_spans_downloaded, m_sync_bad_spans_downloaded; uint64_t m_sync_download_chain_size, m_sync_download_objects_size; size_t m_block_download_max_size; + bool m_sync_pruned_blocks; boost::mutex m_buffer_mutex; double get_avg_block_size(); diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index 65115ee72..c0234bde3 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -106,6 +106,7 @@ namespace cryptonote m_sync_download_objects_size = 0; m_block_download_max_size = command_line::get_arg(vm, cryptonote::arg_block_download_max_size); + m_sync_pruned_blocks = command_line::get_arg(vm, cryptonote::arg_sync_pruned_blocks); return true; } @@ -138,6 +139,7 @@ namespace cryptonote context.m_needed_objects.clear(); m_core.get_short_chain_history(r.block_ids); handler_request_blocks_history( r.block_ids ); // change the limit(?), sleep(?) + r.prune = m_sync_pruned_blocks; MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); post_notify<NOTIFY_REQUEST_CHAIN>(r, context); MLOG_PEER_STATE("requesting chain"); @@ -493,6 +495,7 @@ namespace cryptonote context.m_state = cryptonote_connection_context::state_synchronizing; NOTIFY_REQUEST_CHAIN::request r = boost::value_initialized<NOTIFY_REQUEST_CHAIN::request>(); m_core.get_short_chain_history(r.block_ids); + r.prune = m_sync_pruned_blocks; handler_request_blocks_history( r.block_ids ); // change the limit(?), sleep(?) MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); post_notify<NOTIFY_REQUEST_CHAIN>(r, context); @@ -540,9 +543,9 @@ namespace cryptonote return 1; } } - - std::vector<blobdata> have_tx; - + + std::vector<tx_blob_entry> have_tx; + // Instead of requesting missing transactions by hash like BTC, // we do it by index (thanks to a suggestion from moneromooo) because // we're way cooler .. and also because they're smaller than hashes. @@ -556,7 +559,7 @@ namespace cryptonote for(auto& tx_blob: arg.b.txs) { - if(parse_and_validate_tx_from_blob(tx_blob, tx)) + if(parse_and_validate_tx_from_blob(tx_blob.blob, tx)) { try { @@ -641,7 +644,7 @@ namespace cryptonote LOG_ERROR_CCONTEXT ( "sent wrong tx: failed to parse and validate transaction: " - << epee::string_tools::buff_to_hex_nodelimer(tx_blob) + << epee::string_tools::buff_to_hex_nodelimer(tx_blob.blob) << ", dropping connection" ); @@ -676,7 +679,7 @@ namespace cryptonote cryptonote::blobdata txblob; if(m_core.get_pool_transaction(tx_hash, txblob)) { - have_tx.push_back(txblob); + have_tx.push_back({txblob, crypto::null_hash}); } else { @@ -688,7 +691,7 @@ namespace cryptonote { if (txes.size() == 1) { - have_tx.push_back(tx_to_blob(txes.front())); + have_tx.push_back({tx_to_blob(txes.front()), crypto::null_hash}); } else { @@ -771,6 +774,7 @@ namespace cryptonote NOTIFY_REQUEST_CHAIN::request r = boost::value_initialized<NOTIFY_REQUEST_CHAIN::request>(); m_core.get_short_chain_history(r.block_ids); handler_request_blocks_history( r.block_ids ); // change the limit(?), sleep(?) + r.prune = m_sync_pruned_blocks; MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); post_notify<NOTIFY_REQUEST_CHAIN>(r, context); MLOG_PEER_STATE("requesting chain"); @@ -872,7 +876,7 @@ namespace cryptonote for(auto& tx: txs) { - fluffy_response.b.txs.push_back(t_serializable_object_to_blob(tx)); + fluffy_response.b.txs.push_back({t_serializable_object_to_blob(tx), crypto::null_hash}); } MLOG_P2P_MESSAGE @@ -910,7 +914,7 @@ namespace cryptonote for (size_t i = 0; i < arg.txs.size(); ++i) { cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc); - m_core.handle_incoming_tx(arg.txs[i], tvc, false, true, false); + m_core.handle_incoming_tx({arg.txs[i], crypto::null_hash}, tvc, false, true, false); if(tvc.m_verifivation_failed) { LOG_PRINT_CCONTEXT_L1("Tx verification failed, dropping connection"); @@ -991,7 +995,7 @@ namespace cryptonote for (const auto &element : arg.blocks) { blocks_size += element.block.size(); for (const auto &tx : element.txs) - blocks_size += tx.size(); + blocks_size += tx.blob.size(); } size += blocks_size; @@ -1090,6 +1094,53 @@ namespace cryptonote return 1; } + const bool pruned_ok = should_ask_for_pruned_data(context, start_height, arg.blocks.size(), true); + if (!pruned_ok) + { + // if we don't want pruned data, check we did not get any + for (block_complete_entry& block_entry: arg.blocks) + { + if (block_entry.pruned) + { + MERROR(context << "returned a pruned block, dropping connection"); + drop_connection(context, false, false); + ++m_sync_bad_spans_downloaded; + return 1; + } + if (block_entry.block_weight) + { + MERROR(context << "returned a block weight for a non pruned block, dropping connection"); + drop_connection(context, false, false); + ++m_sync_bad_spans_downloaded; + return 1; + } + for (const tx_blob_entry &tx_entry: block_entry.txs) + { + if (tx_entry.prunable_hash != crypto::null_hash) + { + MERROR(context << "returned at least one pruned object which we did not expect, dropping connection"); + drop_connection(context, false, false); + ++m_sync_bad_spans_downloaded; + return 1; + } + } + } + } + else + { + // we accept pruned data, check that if we got some, then no weights are zero + for (block_complete_entry& block_entry: arg.blocks) + { + if (block_entry.block_weight == 0 && block_entry.pruned) + { + MERROR(context << "returned at least one pruned block with 0 weight, dropping connection"); + drop_connection(context, false, false); + ++m_sync_bad_spans_downloaded; + return 1; + } + } + } + { MLOG_YELLOW(el::Level::Debug, context << " Got NEW BLOCKS inside of " << __FUNCTION__ << ": size: " << arg.blocks.size() << ", blocks: " << start_height << " - " << (start_height + arg.blocks.size() - 1) << @@ -1273,18 +1324,32 @@ namespace cryptonote if (tvc.size() != block_entry.txs.size()) { LOG_ERROR_CCONTEXT("Internal error: tvc.size() != block_entry.txs.size()"); + if (!m_core.cleanup_handle_incoming_blocks()) + { + LOG_PRINT_CCONTEXT_L0("Failure in cleanup_handle_incoming_blocks"); + return 1; + } return 1; } - std::vector<blobdata>::const_iterator it = block_entry.txs.begin(); + std::vector<tx_blob_entry>::const_iterator it = block_entry.txs.begin(); for (size_t i = 0; i < tvc.size(); ++i, ++it) { if(tvc[i].m_verifivation_failed) { if (!m_p2p->for_connection(span_connection_id, [&](cryptonote_connection_context& context, nodetool::peerid_type peer_id, uint32_t f)->bool{ cryptonote::transaction tx; - parse_and_validate_tx_from_blob(*it, tx); // must succeed if we got here + crypto::hash txid; + if (it->prunable_hash == crypto::null_hash) + { + parse_and_validate_tx_from_blob(it->blob, tx, txid); // must succeed if we got here + } + else + { + parse_and_validate_tx_base_from_blob(it->blob, tx); // must succeed if we got here + txid = get_pruned_transaction_hash(tx, it->prunable_hash); + } LOG_ERROR_CCONTEXT("transaction verification failed on NOTIFY_RESPONSE_GET_OBJECTS, tx_id = " - << epee::string_tools::pod_to_hex(cryptonote::get_transaction_hash(tx)) << ", dropping connection"); + << epee::string_tools::pod_to_hex(txid) << ", dropping connection"); drop_connection(context, false, true); return 1; })) @@ -1543,7 +1608,7 @@ skip: { MLOG_P2P_MESSAGE("Received NOTIFY_REQUEST_CHAIN (" << arg.block_ids.size() << " blocks"); NOTIFY_RESPONSE_CHAIN_ENTRY::request r; - if(!m_core.find_blockchain_supplement(arg.block_ids, r)) + if(!m_core.find_blockchain_supplement(arg.block_ids, !arg.prune, r)) { LOG_ERROR_CCONTEXT("Failed to handle NOTIFY_REQUEST_CHAIN."); drop_connection(context, false, false); @@ -1662,6 +1727,12 @@ skip: MDEBUG(context << "This peer has needed stripe " << peer_stripe << ", not dropping"); return false; } + const uint32_t local_stripe = tools::get_pruning_stripe(m_core.get_blockchain_pruning_seed()); + if (m_sync_pruned_blocks && peer_stripe == local_stripe) + { + MDEBUG(context << "We can sync pruned blocks off this peer, not dropping"); + return false; + } if (!context.m_needed_objects.empty()) { @@ -1701,22 +1772,42 @@ skip: { // take out blocks we already have size_t skip = 0; - while (skip < context.m_needed_objects.size() && (m_core.have_block(context.m_needed_objects[skip]) || (check_block_queue && m_block_queue.have(context.m_needed_objects[skip])))) + while (skip < context.m_needed_objects.size() && (m_core.have_block(context.m_needed_objects[skip].first) || (check_block_queue && m_block_queue.have(context.m_needed_objects[skip].first)))) { // if we're popping the last hash, record it so we can ask again from that hash, // this prevents never being able to progress on peers we get old hash lists from if (skip + 1 == context.m_needed_objects.size()) - context.m_last_known_hash = context.m_needed_objects[skip]; + context.m_last_known_hash = context.m_needed_objects[skip].first; ++skip; } if (skip > 0) { MDEBUG(context << "skipping " << skip << "/" << context.m_needed_objects.size() << " blocks"); - context.m_needed_objects = std::vector<crypto::hash>(context.m_needed_objects.begin() + skip, context.m_needed_objects.end()); + context.m_needed_objects = std::vector<std::pair<crypto::hash, uint64_t>>(context.m_needed_objects.begin() + skip, context.m_needed_objects.end()); } } //------------------------------------------------------------------------------------------------------------------------ template<class t_core> + bool t_cryptonote_protocol_handler<t_core>::should_ask_for_pruned_data(cryptonote_connection_context& context, uint64_t first_block_height, uint64_t nblocks, bool check_block_weights) const + { + if (!m_sync_pruned_blocks) + return false; + if (!m_core.is_within_compiled_block_hash_area(first_block_height + nblocks - 1)) + return false; + const uint32_t local_stripe = tools::get_pruning_stripe(m_core.get_blockchain_pruning_seed()); + if (local_stripe == 0) + return false; + // assumes the span size is less or equal to the stripe size + bool full_data_needed = tools::get_pruning_stripe(first_block_height, context.m_remote_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES) == local_stripe + || tools::get_pruning_stripe(first_block_height + nblocks - 1, context.m_remote_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES) == local_stripe; + if (full_data_needed) + return false; + if (check_block_weights && !m_core.has_block_weights(first_block_height, nblocks)) + return false; + return true; + } + //------------------------------------------------------------------------------------------------------------------------ + template<class t_core> bool t_cryptonote_protocol_handler<t_core>::request_missing_objects(cryptonote_connection_context& context, bool check_having_blocks, bool force_next_span) { // flush stale spans @@ -1739,6 +1830,7 @@ skip: const auto next_needed_pruning_stripe = get_next_needed_pruning_stripe(); const uint32_t add_stripe = tools::get_pruning_stripe(bc_height, context.m_remote_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES); const uint32_t peer_stripe = tools::get_pruning_stripe(context.m_pruning_seed); + const uint32_t local_stripe = tools::get_pruning_stripe(m_core.get_blockchain_pruning_seed()); const size_t block_queue_size_threshold = m_block_download_max_size ? m_block_download_max_size : BLOCK_QUEUE_SIZE_THRESHOLD; bool queue_proceed = nspans < BLOCK_QUEUE_NSPANS_THRESHOLD || size < block_queue_size_threshold; // get rid of blocks we already requested, or already have @@ -1749,7 +1841,7 @@ skip: next_block_height = next_needed_height; else next_block_height = context.m_last_response_height - context.m_needed_objects.size() + 1; - bool stripe_proceed_main = (add_stripe == 0 || peer_stripe == 0 || add_stripe == peer_stripe) && (next_block_height < bc_height + BLOCK_QUEUE_FORCE_DOWNLOAD_NEAR_BLOCKS || next_needed_height < bc_height + BLOCK_QUEUE_FORCE_DOWNLOAD_NEAR_BLOCKS); + bool stripe_proceed_main = ((m_sync_pruned_blocks && peer_stripe == local_stripe) || add_stripe == 0 || peer_stripe == 0 || add_stripe == peer_stripe) && (next_block_height < bc_height + BLOCK_QUEUE_FORCE_DOWNLOAD_NEAR_BLOCKS || next_needed_height < bc_height + BLOCK_QUEUE_FORCE_DOWNLOAD_NEAR_BLOCKS); bool stripe_proceed_secondary = tools::has_unpruned_block(next_block_height, context.m_remote_blockchain_height, context.m_pruning_seed); bool proceed = stripe_proceed_main || (queue_proceed && stripe_proceed_secondary); if (!stripe_proceed_main && !stripe_proceed_secondary && should_drop_connection(context, tools::get_pruning_stripe(next_block_height, context.m_remote_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES))) @@ -1812,8 +1904,9 @@ skip: { const uint64_t now = tools::get_tick_count(); const uint64_t dt = now - m_last_add_end_time; - if (tools::ticks_to_ns(dt) >= DROP_ON_SYNC_WEDGE_THRESHOLD) + if (m_last_add_end_time && tools::ticks_to_ns(dt) >= DROP_ON_SYNC_WEDGE_THRESHOLD) { + MDEBUG(context << "ns " << tools::ticks_to_ns(dt) << " from " << m_last_add_end_time << " and " << now); MDEBUG(context << "Block addition seems to have wedged, dropping connection"); return false; } @@ -1880,7 +1973,8 @@ skip: skip_unneeded_hashes(context, false); const uint64_t first_block_height = context.m_last_response_height - context.m_needed_objects.size() + 1; - span = m_block_queue.reserve_span(first_block_height, context.m_last_response_height, count_limit, context.m_connection_id, context.m_pruning_seed, context.m_remote_blockchain_height, context.m_needed_objects); + bool sync_pruned_blocks = m_sync_pruned_blocks && m_core.get_blockchain_pruning_seed(); + span = m_block_queue.reserve_span(first_block_height, context.m_last_response_height, count_limit, context.m_connection_id, sync_pruned_blocks, m_core.get_blockchain_pruning_seed(), context.m_pruning_seed, context.m_remote_blockchain_height, context.m_needed_objects); MDEBUG(context << " span from " << first_block_height << ": " << span.first << "/" << span.second); if (span.second > 0) { @@ -1910,7 +2004,8 @@ skip: ++count; context.m_requested_objects.insert(hash); // that's atrocious O(n) wise, but this is rare - auto i = std::find(context.m_needed_objects.begin(), context.m_needed_objects.end(), hash); + auto i = std::find_if(context.m_needed_objects.begin(), context.m_needed_objects.end(), + [&hash](const std::pair<crypto::hash, uint64_t> &o) { return o.first == hash; }); if (i != context.m_needed_objects.end()) context.m_needed_objects.erase(i); } @@ -1929,7 +2024,7 @@ skip: return false; } if (skip > 0) - context.m_needed_objects = std::vector<crypto::hash>(context.m_needed_objects.begin() + skip, context.m_needed_objects.end()); + context.m_needed_objects = std::vector<std::pair<crypto::hash, uint64_t>>(context.m_needed_objects.begin() + skip, context.m_needed_objects.end()); if (context.m_needed_objects.size() < span.second) { MERROR("ERROR: span " << span.first << "/" << span.second << ", m_needed_objects " << context.m_needed_objects.size()); @@ -1938,18 +2033,37 @@ skip: for (size_t n = 0; n < span.second; ++n) { - req.blocks.push_back(context.m_needed_objects[n]); + req.blocks.push_back(context.m_needed_objects[n].first); ++count; - context.m_requested_objects.insert(context.m_needed_objects[n]); + context.m_requested_objects.insert(context.m_needed_objects[n].first); } - context.m_needed_objects = std::vector<crypto::hash>(context.m_needed_objects.begin() + span.second, context.m_needed_objects.end()); + context.m_needed_objects = std::vector<std::pair<crypto::hash, uint64_t>>(context.m_needed_objects.begin() + span.second, context.m_needed_objects.end()); } + req.prune = should_ask_for_pruned_data(context, span.first, span.second, true); + + // if we need to ask for full data and that peer does not have the right stripe, we can't ask it + if (!req.prune && context.m_pruning_seed) + { + const uint32_t peer_stripe = tools::get_pruning_stripe(context.m_pruning_seed); + const uint32_t first_stripe = tools::get_pruning_stripe(span.first, context.m_remote_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES); + const uint32_t last_stripe = tools::get_pruning_stripe(span.first + span.second - 1, context.m_remote_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES); + if ((first_stripe && peer_stripe != first_stripe) || (last_stripe && peer_stripe != last_stripe)) + { + MDEBUG(context << "We need full data, but the peer does not have it, dropping peer"); + return false; + } + } context.m_last_request_time = boost::posix_time::microsec_clock::universal_time(); MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_GET_OBJECTS: blocks.size()=" << req.blocks.size() << "requested blocks count=" << count << " / " << count_limit << " from " << span.first << ", first hash " << req.blocks.front()); //epee::net_utils::network_throttle_manager::get_global_throttle_inreq().logger_handle_net("log/dr-monero/net/req-all.data", sec, get_avg_block_size()); + MDEBUG("Asking for " << (req.prune ? "pruned" : "full") << " data, start/end " + << tools::get_pruning_stripe(span.first, context.m_remote_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES) + << "/" << tools::get_pruning_stripe(span.first + span.second - 1, context.m_remote_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES) + << ", ours " << tools::get_pruning_stripe(m_core.get_blockchain_pruning_seed()) << ", peer stripe " << tools::get_pruning_stripe(context.m_pruning_seed)); + post_notify<NOTIFY_REQUEST_GET_OBJECTS>(req, context); MLOG_PEER_STATE("requesting objects"); return true; @@ -1959,7 +2073,8 @@ skip: // drop it to make space for other peers, or ask for a span further down the line const uint32_t next_stripe = get_next_needed_pruning_stripe().first; const uint32_t peer_stripe = tools::get_pruning_stripe(context.m_pruning_seed); - if (next_stripe && peer_stripe && next_stripe != peer_stripe) + const uint32_t local_stripe = tools::get_pruning_stripe(m_core.get_blockchain_pruning_seed()); + if (!(m_sync_pruned_blocks && peer_stripe == local_stripe) && next_stripe && peer_stripe && next_stripe != peer_stripe) { // at this point, we have to either close the connection, or start getting blocks past the // current point, or become dormant @@ -2022,6 +2137,7 @@ skip: } handler_request_blocks_history( r.block_ids ); // change the limit(?), sleep(?) + r.prune = m_sync_pruned_blocks; //std::string blob; // for calculate size of request //epee::serialization::store_t_to_binary(r, blob); @@ -2128,6 +2244,12 @@ skip: drop_connection(context, true, false); return 1; } + if (!arg.m_block_weights.empty() && arg.m_block_weights.size() != arg.m_block_ids.size()) + { + LOG_ERROR_CCONTEXT("sent invalid block weight array, dropping connection"); + drop_connection(context, true, false); + return 1; + } MDEBUG(context << "first block hash " << arg.m_block_ids.front() << ", last " << arg.m_block_ids.back()); if (arg.total_height >= CRYPTONOTE_MAX_BLOCK_NUMBER || arg.m_block_ids.size() >= CRYPTONOTE_MAX_BLOCK_NUMBER) @@ -2147,7 +2269,7 @@ skip: return 1; } - uint64_t n_use_blocks = m_core.prevalidate_block_hashes(arg.start_height, arg.m_block_ids); + uint64_t n_use_blocks = m_core.prevalidate_block_hashes(arg.start_height, arg.m_block_ids, arg.m_block_weights); if (n_use_blocks + HASH_OF_HASHES_STEP <= arg.m_block_ids.size()) { LOG_ERROR_CCONTEXT("Most blocks are invalid, dropping connection"); @@ -2157,9 +2279,10 @@ skip: context.m_needed_objects.clear(); uint64_t added = 0; - for(auto& bl_id: arg.m_block_ids) + for (size_t i = 0; i < arg.m_block_ids.size(); ++i) { - context.m_needed_objects.push_back(bl_id); + const uint64_t block_weight = arg.m_block_weights.empty() ? 0 : arg.m_block_weights[i]; + context.m_needed_objects.push_back(std::make_pair(arg.m_block_ids[i], block_weight)); if (++added == n_use_blocks) break; } @@ -2183,7 +2306,7 @@ skip: { NOTIFY_NEW_FLUFFY_BLOCK::request fluffy_arg = AUTO_VAL_INIT(fluffy_arg); fluffy_arg.current_blockchain_height = arg.current_blockchain_height; - std::vector<blobdata> fluffy_txs; + std::vector<tx_blob_entry> fluffy_txs; fluffy_arg.b = arg.b; fluffy_arg.b.txs = fluffy_txs; diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 7706d2cf7..13ec8c920 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -377,6 +377,7 @@ namespace cryptonote for(auto& bd: bs) { res.blocks.resize(res.blocks.size()+1); + res.blocks.back().pruned = req.prune; res.blocks.back().block = bd.first.first; pruned_size += bd.first.first.size(); unpruned_size += bd.first.first.size(); @@ -389,10 +390,10 @@ namespace cryptonote for (std::vector<std::pair<crypto::hash, cryptonote::blobdata>>::iterator i = bd.second.begin(); i != bd.second.end(); ++i) { unpruned_size += i->second.size(); - res.blocks.back().txs.push_back(std::move(i->second)); + res.blocks.back().txs.push_back({std::move(i->second), crypto::null_hash}); i->second.clear(); i->second.shrink_to_fit(); - pruned_size += res.blocks.back().txs.back().size(); + pruned_size += res.blocks.back().txs.back().blob.size(); } const size_t n_txes_to_lookup = bd.second.size() + (req.no_miner_tx ? 0 : 1); @@ -474,7 +475,7 @@ namespace cryptonote res.blocks.resize(res.blocks.size() + 1); res.blocks.back().block = block_to_blob(blk); for (auto& tx : txs) - res.blocks.back().txs.push_back(tx_to_blob(tx)); + res.blocks.back().txs.push_back({tx_to_blob(tx), crypto::null_hash}); } res.status = CORE_RPC_STATUS_OK; return true; @@ -488,7 +489,7 @@ namespace cryptonote return r; res.start_height = req.start_height; - if(!m_core.get_blockchain_storage().find_blockchain_supplement(req.block_ids, res.m_block_ids, res.start_height, res.current_height, false)) + if(!m_core.get_blockchain_storage().find_blockchain_supplement(req.block_ids, res.m_block_ids, NULL, res.start_height, res.current_height, false)) { res.status = "Failed"; add_host_fail(ctx); @@ -912,7 +913,7 @@ namespace cryptonote cryptonote_connection_context fake_context = AUTO_VAL_INIT(fake_context); tx_verification_context tvc = AUTO_VAL_INIT(tvc); - if(!m_core.handle_incoming_tx(tx_blob, tvc, false, false, req.do_not_relay) || tvc.m_verifivation_failed) + if(!m_core.handle_incoming_tx({tx_blob, crypto::null_hash}, tvc, false, false, req.do_not_relay) || tvc.m_verifivation_failed) { res.status = "Failed"; std::string reason = ""; diff --git a/src/rpc/daemon_handler.cpp b/src/rpc/daemon_handler.cpp index 890380dc8..d7e081af3 100644 --- a/src/rpc/daemon_handler.cpp +++ b/src/rpc/daemon_handler.cpp @@ -141,7 +141,7 @@ namespace rpc auto& chain = m_core.get_blockchain_storage(); - if (!chain.find_blockchain_supplement(req.known_hashes, res.hashes, res.start_height, res.current_height, false)) + if (!chain.find_blockchain_supplement(req.known_hashes, res.hashes, NULL, res.start_height, res.current_height, false)) { res.status = Message::STATUS_FAILED; res.error_details = "Blockchain::find_blockchain_supplement() returned false"; @@ -291,7 +291,7 @@ namespace rpc cryptonote_connection_context fake_context = AUTO_VAL_INIT(fake_context); tx_verification_context tvc = AUTO_VAL_INIT(tvc); - if(!m_core.handle_incoming_tx(tx_blob, tvc, false, false, !relay) || tvc.m_verifivation_failed) + if(!m_core.handle_incoming_tx({tx_blob, crypto::null_hash}, tvc, false, false, !relay) || tvc.m_verifivation_failed) { if (tvc.m_verifivation_failed) { diff --git a/src/serialization/json_object.cpp b/src/serialization/json_object.cpp index cc52bde58..42d09aa5f 100644 --- a/src/serialization/json_object.cpp +++ b/src/serialization/json_object.cpp @@ -627,6 +627,25 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::connection_info& inf GET_FROM_JSON_OBJECT(val, info.current_upload, current_upload); } +void toJsonValue(rapidjson::Document& doc, const cryptonote::tx_blob_entry& tx, rapidjson::Value& val) +{ + val.SetObject(); + + INSERT_INTO_JSON_OBJECT(val, doc, blob, tx.blob); + INSERT_INTO_JSON_OBJECT(val, doc, prunable_hash, tx.prunable_hash); +} + +void fromJsonValue(const rapidjson::Value& val, cryptonote::tx_blob_entry& tx) +{ + if (!val.IsObject()) + { + throw WRONG_TYPE("json object"); + } + + GET_FROM_JSON_OBJECT(val, tx.blob, blob); + GET_FROM_JSON_OBJECT(val, tx.prunable_hash, prunable_hash); +} + void toJsonValue(rapidjson::Document& doc, const cryptonote::block_complete_entry& blk, rapidjson::Value& val) { val.SetObject(); diff --git a/src/serialization/json_object.h b/src/serialization/json_object.h index c804d148b..5ef75b863 100644 --- a/src/serialization/json_object.h +++ b/src/serialization/json_object.h @@ -221,6 +221,9 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::tx_out& txout); void toJsonValue(rapidjson::Document& doc, const cryptonote::connection_info& info, rapidjson::Value& val); void fromJsonValue(const rapidjson::Value& val, cryptonote::connection_info& info); +void toJsonValue(rapidjson::Document& doc, const cryptonote::tx_blob_entry& tx, rapidjson::Value& val); +void fromJsonValue(const rapidjson::Value& val, cryptonote::tx_blob_entry& tx); + void toJsonValue(rapidjson::Document& doc, const cryptonote::block_complete_entry& blk, rapidjson::Value& val); void fromJsonValue(const rapidjson::Value& val, cryptonote::block_complete_entry& blk); diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 476c68c74..871eafdeb 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -2735,7 +2735,7 @@ void wallet2::pull_and_parse_next_blocks(uint64_t start_height, uint64_t &blocks for (size_t j = 0; j < blocks[i].txs.size(); ++j) { tpool.submit(&waiter, [&, i, j](){ - if (!parse_and_validate_tx_base_from_blob(blocks[i].txs[j], parsed_blocks[i].txes[j])) + if (!parse_and_validate_tx_base_from_blob(blocks[i].txs[j].blob, parsed_blocks[i].txes[j])) { boost::unique_lock<boost::mutex> lock(error_lock); error = true; diff --git a/tests/core_proxy/core_proxy.cpp b/tests/core_proxy/core_proxy.cpp index 388808269..a49a028f9 100644 --- a/tests/core_proxy/core_proxy.cpp +++ b/tests/core_proxy/core_proxy.cpp @@ -160,7 +160,7 @@ string tx2str(const cryptonote::transaction& tx, const cryptonote::hash256& tx_h return ss.str(); }*/ -bool tests::proxy_core::handle_incoming_tx(const cryptonote::blobdata& tx_blob, cryptonote::tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { +bool tests::proxy_core::handle_incoming_tx(const cryptonote::tx_blob_entry& tx_blob, cryptonote::tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { if (!keeped_by_block) return true; @@ -168,7 +168,13 @@ bool tests::proxy_core::handle_incoming_tx(const cryptonote::blobdata& tx_blob, crypto::hash tx_prefix_hash = null_hash; transaction tx; - if (!parse_and_validate_tx_from_blob(tx_blob, tx, tx_hash, tx_prefix_hash)) { + if (tx_blob.prunable_hash != crypto::null_hash) + { + cerr << "WRONG TRANSACTION, pruned blob rejected" << endl; + return false; + } + + if (!parse_and_validate_tx_from_blob(tx_blob.blob, tx, tx_hash, tx_prefix_hash)) { cerr << "WRONG TRANSACTION BLOB, Failed to parse, rejected" << endl; return false; } @@ -176,7 +182,7 @@ bool tests::proxy_core::handle_incoming_tx(const cryptonote::blobdata& tx_blob, cout << "TX " << endl << endl; cout << tx_hash << endl; cout << tx_prefix_hash << endl; - cout << tx_blob.size() << endl; + cout << tx_blob.blob.size() << endl; //cout << string_tools::buff_to_hex_nodelimer(tx_blob) << endl << endl; cout << obj_to_json_str(tx) << endl; cout << endl << "ENDTX" << endl; @@ -184,7 +190,7 @@ bool tests::proxy_core::handle_incoming_tx(const cryptonote::blobdata& tx_blob, return true; } -bool tests::proxy_core::handle_incoming_txs(const std::vector<blobdata>& tx_blobs, std::vector<tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) +bool tests::proxy_core::handle_incoming_txs(const std::vector<tx_blob_entry>& tx_blobs, std::vector<tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { tvc.resize(tx_blobs.size()); size_t i = 0; diff --git a/tests/core_proxy/core_proxy.h b/tests/core_proxy/core_proxy.h index 6c8b3ccb6..0e41a2be0 100644 --- a/tests/core_proxy/core_proxy.h +++ b/tests/core_proxy/core_proxy.h @@ -75,13 +75,13 @@ namespace tests bool get_stat_info(cryptonote::core_stat_info& st_inf){return true;} bool have_block(const crypto::hash& id); void get_blockchain_top(uint64_t& height, crypto::hash& top_id); - bool handle_incoming_tx(const cryptonote::blobdata& tx_blob, cryptonote::tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); - bool handle_incoming_txs(const std::vector<cryptonote::blobdata>& tx_blobs, std::vector<cryptonote::tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); + bool handle_incoming_tx(const cryptonote::tx_blob_entry& tx_blob, cryptonote::tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); + bool handle_incoming_txs(const std::vector<cryptonote::tx_blob_entry>& tx_blobs, std::vector<cryptonote::tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); bool handle_incoming_block(const cryptonote::blobdata& block_blob, const cryptonote::block *block, cryptonote::block_verification_context& bvc, bool update_miner_blocktemplate = true); void pause_mine(){} void resume_mine(){} bool on_idle(){return true;} - bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, cryptonote::NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp){return true;} + bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, bool clip_pruned, cryptonote::NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp){return true;} bool handle_get_objects(cryptonote::NOTIFY_REQUEST_GET_OBJECTS::request& arg, cryptonote::NOTIFY_RESPONSE_GET_OBJECTS::request& rsp, cryptonote::cryptonote_connection_context& context){return true;} cryptonote::Blockchain &get_blockchain_storage() { throw std::runtime_error("Called invalid member function: please never call get_blockchain_storage on the TESTING class proxy_core."); } bool get_test_drop_download() {return true;} @@ -103,7 +103,9 @@ namespace tests uint64_t get_earliest_ideal_height_for_version(uint8_t version) const { return 0; } cryptonote::difficulty_type get_block_cumulative_difficulty(uint64_t height) const { return 0; } bool fluffy_blocks_enabled() const { return false; } - uint64_t prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes) { return 0; } + uint64_t prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes, const std::vector<uint64_t> &weights) { return 0; } + bool has_block_weights(uint64_t height, uint64_t nblocks) const { return false; } + bool is_within_compiled_block_hash_area(uint64_t height) const { return false; } bool pad_transactions() const { return false; } uint32_t get_blockchain_pruning_seed() const { return 0; } bool prune_blockchain(uint32_t pruning_seed) const { return true; } diff --git a/tests/core_tests/chaingen.h b/tests/core_tests/chaingen.h index c0d31bb8a..b78640dc9 100644 --- a/tests/core_tests/chaingen.h +++ b/tests/core_tests/chaingen.h @@ -544,7 +544,7 @@ public: cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc); size_t pool_size = m_c.get_pool_transactions_count(); - m_c.handle_incoming_tx(t_serializable_object_to_blob(tx), tvc, m_txs_keeped_by_block, false, false); + m_c.handle_incoming_tx({t_serializable_object_to_blob(tx), crypto::null_hash}, tvc, m_txs_keeped_by_block, false, false); bool tx_added = pool_size + 1 == m_c.get_pool_transactions_count(); bool r = m_validator.check_tx_verification_context(tvc, tx_added, m_ev_index, tx); CHECK_AND_NO_ASSERT_MES(r, false, "tx verification context check failed"); @@ -555,12 +555,12 @@ public: { log_event("cryptonote::transaction"); - std::vector<cryptonote::blobdata> tx_blobs; + std::vector<cryptonote::tx_blob_entry> tx_blobs; std::vector<cryptonote::tx_verification_context> tvcs; cryptonote::tx_verification_context tvc0 = AUTO_VAL_INIT(tvc0); for (const auto &tx: txs) { - tx_blobs.push_back(t_serializable_object_to_blob(tx)); + tx_blobs.push_back({t_serializable_object_to_blob(tx)}); tvcs.push_back(tvc0); } size_t pool_size = m_c.get_pool_transactions_count(); @@ -578,7 +578,11 @@ public: cryptonote::block_verification_context bvc = AUTO_VAL_INIT(bvc); cryptonote::blobdata bd = t_serializable_object_to_blob(b); std::vector<cryptonote::block> pblocks; - if (m_c.prepare_handle_incoming_blocks(std::vector<cryptonote::block_complete_entry>(1, {bd, {}}), pblocks)) + cryptonote::block_complete_entry bce; + bce.pruned = false; + bce.block = bd; + bce.txs = {}; + if (m_c.prepare_handle_incoming_blocks(std::vector<cryptonote::block_complete_entry>(1, bce), pblocks)) { m_c.handle_incoming_block(bd, &b, bvc); m_c.cleanup_handle_incoming_blocks(); @@ -608,7 +612,11 @@ public: cryptonote::block_verification_context bvc = AUTO_VAL_INIT(bvc); std::vector<cryptonote::block> pblocks; - if (m_c.prepare_handle_incoming_blocks(std::vector<cryptonote::block_complete_entry>(1, {sr_block.data, {}}), pblocks)) + cryptonote::block_complete_entry bce; + bce.pruned = false; + bce.block = sr_block.data; + bce.txs = {}; + if (m_c.prepare_handle_incoming_blocks(std::vector<cryptonote::block_complete_entry>(1, bce), pblocks)) { m_c.handle_incoming_block(sr_block.data, NULL, bvc); m_c.cleanup_handle_incoming_blocks(); diff --git a/tests/unit_tests/ban.cpp b/tests/unit_tests/ban.cpp index 17fba90c6..32cf2dcc1 100644 --- a/tests/unit_tests/ban.cpp +++ b/tests/unit_tests/ban.cpp @@ -55,13 +55,13 @@ public: bool get_stat_info(cryptonote::core_stat_info& st_inf) const {return true;} bool have_block(const crypto::hash& id) const {return true;} void get_blockchain_top(uint64_t& height, crypto::hash& top_id)const{height=0;top_id=crypto::null_hash;} - bool handle_incoming_tx(const cryptonote::blobdata& tx_blob, cryptonote::tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { return true; } - bool handle_incoming_txs(const std::vector<cryptonote::blobdata>& tx_blob, std::vector<cryptonote::tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { return true; } + bool handle_incoming_tx(const cryptonote::tx_blob_entry& tx_blob, cryptonote::tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { return true; } + bool handle_incoming_txs(const std::vector<cryptonote::tx_blob_entry>& tx_blob, std::vector<cryptonote::tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { return true; } bool handle_incoming_block(const cryptonote::blobdata& block_blob, const cryptonote::block *block, cryptonote::block_verification_context& bvc, bool update_miner_blocktemplate = true) { return true; } void pause_mine(){} void resume_mine(){} bool on_idle(){return true;} - bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, cryptonote::NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp){return true;} + bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, bool clip_pruned, cryptonote::NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp){return true;} bool handle_get_objects(cryptonote::NOTIFY_REQUEST_GET_OBJECTS::request& arg, cryptonote::NOTIFY_RESPONSE_GET_OBJECTS::request& rsp, cryptonote::cryptonote_connection_context& context){return true;} cryptonote::blockchain_storage &get_blockchain_storage() { throw std::runtime_error("Called invalid member function: please never call get_blockchain_storage on the TESTING class test_core."); } bool get_test_drop_download() const {return true;} @@ -83,10 +83,12 @@ public: uint64_t get_earliest_ideal_height_for_version(uint8_t version) const { return 0; } cryptonote::difficulty_type get_block_cumulative_difficulty(uint64_t height) const { return 0; } bool fluffy_blocks_enabled() const { return false; } - uint64_t prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes) { return 0; } + uint64_t prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes, const std::vector<uint64_t> &weights) { return 0; } bool pad_transactions() { return false; } uint32_t get_blockchain_pruning_seed() const { return 0; } bool prune_blockchain(uint32_t pruning_seed = 0) { return true; } + bool is_within_compiled_block_hash_area(uint64_t height) const { return false; } + bool has_block_weights(uint64_t height, uint64_t nblocks) const { return false; } void stop() {} }; |