From 7adceee634354f9291de8dd021669d7ca5cf3f2a Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Wed, 13 Sep 2017 21:03:04 +0100 Subject: precomputed block hashes are now in blocks of N (currently 256) This shaves a lot of space off binaries --- src/blockchain_utilities/blockchain_import.cpp | 20 ++++ src/blockchain_utilities/blocksdat_file.cpp | 14 ++- src/blockchain_utilities/blocksdat_file.h | 1 + src/blocks/checkpoints.dat | Bin 44480036 -> 173732 bytes src/cryptonote_config.h | 3 + src/cryptonote_core/blockchain.cpp | 128 +++++++++++++++++++-- src/cryptonote_core/blockchain.h | 2 + src/cryptonote_core/cryptonote_core.cpp | 5 + src/cryptonote_core/cryptonote_core.h | 7 ++ .../cryptonote_protocol_handler.inl | 12 ++ 10 files changed, 180 insertions(+), 12 deletions(-) (limited to 'src') diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp index 635a70b10..3841911fe 100644 --- a/src/blockchain_utilities/blockchain_import.cpp +++ b/src/blockchain_utilities/blockchain_import.cpp @@ -169,6 +169,26 @@ int check_flush(cryptonote::core &core, std::list &blocks, if (!force && blocks.size() < db_batch_size) return 0; + // wait till we can verify a full HOH without extra, for speed + uint64_t new_height = core.get_blockchain_storage().get_db().height() + blocks.size(); + if (!force && new_height % HASH_OF_HASHES_STEP) + return 0; + + std::list hashes; + for (const auto &b: blocks) + { + cryptonote::block block; + if (!parse_and_validate_block_from_blob(b.block, block)) + { + MERROR("Failed to parse block: " + << epee::string_tools::pod_to_hex(get_blob_hash(b.block))); + core.cleanup_handle_incoming_blocks(); + return 1; + } + hashes.push_back(cryptonote::get_block_hash(block)); + } + core.prevalidate_block_hashes(core.get_blockchain_storage().get_db().height(), hashes); + core.prepare_handle_incoming_blocks(blocks); for(const block_complete_entry& block_entry: blocks) diff --git a/src/blockchain_utilities/blocksdat_file.cpp b/src/blockchain_utilities/blocksdat_file.cpp index 63224225f..32e93345e 100644 --- a/src/blockchain_utilities/blocksdat_file.cpp +++ b/src/blockchain_utilities/blocksdat_file.cpp @@ -82,7 +82,7 @@ bool BlocksdatFile::open_writer(const boost::filesystem::path& file_path, uint64 bool BlocksdatFile::initialize_file(uint64_t block_stop) { - const uint32_t nblocks = block_stop + 1; + const uint32_t nblocks = (block_stop + 1) / HASH_OF_HASHES_STEP; unsigned char nblocksc[4]; nblocksc[0] = nblocks & 0xff; @@ -101,8 +101,16 @@ bool BlocksdatFile::initialize_file(uint64_t block_stop) void BlocksdatFile::write_block(const crypto::hash& block_hash) { - const std::string data(block_hash.data, sizeof(block_hash)); - *m_raw_data_file << data; + m_hashes.push_back(block_hash); + while (m_hashes.size() >= HASH_OF_HASHES_STEP) + { + crypto::hash hash; + crypto::cn_fast_hash(m_hashes.data(), HASH_OF_HASHES_STEP * sizeof(crypto::hash), hash); + memmove(m_hashes.data(), m_hashes.data() + HASH_OF_HASHES_STEP * sizeof(crypto::hash), (m_hashes.size() - HASH_OF_HASHES_STEP) * sizeof(crypto::hash)); + m_hashes.resize(m_hashes.size() - HASH_OF_HASHES_STEP); + const std::string data(hash.data, sizeof(hash)); + *m_raw_data_file << data; + } } bool BlocksdatFile::close() diff --git a/src/blockchain_utilities/blocksdat_file.h b/src/blockchain_utilities/blocksdat_file.h index 0a5913058..d43811772 100644 --- a/src/blockchain_utilities/blocksdat_file.h +++ b/src/blockchain_utilities/blocksdat_file.h @@ -76,4 +76,5 @@ protected: private: uint64_t m_cur_height; // tracks current height during export + std::vector m_hashes; }; diff --git a/src/blocks/checkpoints.dat b/src/blocks/checkpoints.dat index 15fa042cf..c12c3d8a0 100644 Binary files a/src/blocks/checkpoints.dat and b/src/blocks/checkpoints.dat differ diff --git a/src/cryptonote_config.h b/src/cryptonote_config.h index cdb46dda9..f38528d58 100644 --- a/src/cryptonote_config.h +++ b/src/cryptonote_config.h @@ -137,6 +137,9 @@ #define PER_KB_FEE_QUANTIZATION_DECIMALS 8 +#define HASH_OF_HASHES_STEP 256 + + // New constants are intended to go here namespace config { diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 619dbdc07..ece5c46aa 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -3206,13 +3206,21 @@ leave: if (m_db->height() < m_blocks_hash_check.size()) { auto hash = get_block_hash(bl); - if (memcmp(&hash, &m_blocks_hash_check[m_db->height()], sizeof(hash)) != 0) + const auto &expected_hash = m_blocks_hash_check[m_db->height()]; + if (expected_hash != cryptonote::null_hash) { - MERROR_VER("Block with id is INVALID: " << id); - bvc.m_verifivation_failed = true; - goto leave; + if (memcmp(&hash, &expected_hash, sizeof(hash)) != 0) + { + MERROR_VER("Block with id is INVALID: " << id); + bvc.m_verifivation_failed = true; + goto leave; + } + fast_check = true; + } + else + { + MCINFO("verify", "No pre-validated hash at height " << m_db->height() << ", verifying fully"); } - fast_check = true; } else #endif @@ -3657,6 +3665,14 @@ bool Blockchain::cleanup_handle_incoming_blocks(bool force_sync) m_blocks_txs_check.clear(); m_check_txin_table.clear(); + // when we're well clear of the precomputed hashes, free the memory + if (!m_blocks_hash_check.empty() && m_db->height() > m_blocks_hash_check.size() + 4096) + { + MINFO("Dumping block hashes, we're now 4k past " << m_blocks_hash_check.size()); + m_blocks_hash_check.clear(); + m_blocks_hash_check.shrink_to_fit(); + } + CRITICAL_REGION_END(); m_tx_pool.unlock(); @@ -3681,6 +3697,98 @@ void Blockchain::output_scan_worker(const uint64_t amount, const std::vector &hashes) +{ + // new: . . . . . X X X X X . . . . . . + // pre: A A A A B B B B C C C C D D D D + + // easy case: height >= hashes + if (height >= m_blocks_hash_of_hashes.size() * HASH_OF_HASHES_STEP) + return hashes.size(); + + // find hashes encompassing those block + size_t first_index = height / HASH_OF_HASHES_STEP; + size_t last_index = (height + hashes.size() - 1) / HASH_OF_HASHES_STEP; + MDEBUG("Blocks " << height << " - " << (height + hashes.size() - 1) << " start at " << first_index << " and end at " << last_index); + + // case of not enough to calculate even a single hash + if (first_index == last_index && hashes.size() < HASH_OF_HASHES_STEP && (height + hashes.size()) % HASH_OF_HASHES_STEP) + return hashes.size(); + + // build hashes vector to hash hashes together + std::vector data; + data.reserve(hashes.size() + HASH_OF_HASHES_STEP - 1); // may be a bit too much + + // we expect height to be either equal or a bit below db height + bool disconnected = (height > m_db->height()); + size_t pop; + if (disconnected && height % HASH_OF_HASHES_STEP) + { + ++first_index; + pop = HASH_OF_HASHES_STEP - height % HASH_OF_HASHES_STEP; + } + else + { + // we might need some already in the chain for the first part of the first hash + for (uint64_t h = first_index * HASH_OF_HASHES_STEP; h < height; ++h) + { + data.push_back(m_db->get_block_hash_from_height(h)); + } + pop = 0; + } + + // push the data to check + for (const auto &h: hashes) + { + if (pop) + --pop; + else + data.push_back(h); + } + + // hash and check + uint64_t usable = first_index * HASH_OF_HASHES_STEP - height; // may start negative, but unsigned under/overflow is not UB + for (size_t n = first_index; n <= last_index; ++n) + { + if (n < m_blocks_hash_of_hashes.size()) + { + // if the last index isn't fully filled, we can't tell if valid + if (data.size() < (n - first_index) * HASH_OF_HASHES_STEP + HASH_OF_HASHES_STEP) + break; + + crypto::hash hash; + cn_fast_hash(data.data() + (n - first_index) * HASH_OF_HASHES_STEP, HASH_OF_HASHES_STEP * sizeof(crypto::hash), hash); + bool valid = hash == m_blocks_hash_of_hashes[n]; + + // add to the known hashes array + if (!valid) + { + MWARNING("invalid hash for blocks " << n * HASH_OF_HASHES_STEP << " - " << (n * HASH_OF_HASHES_STEP + HASH_OF_HASHES_STEP - 1)); + break; + } + + size_t end = n * HASH_OF_HASHES_STEP + HASH_OF_HASHES_STEP; + for (size_t i = n * HASH_OF_HASHES_STEP; i < end; ++i) + { + CHECK_AND_ASSERT_MES(m_blocks_hash_check[i] == cryptonote::null_hash || m_blocks_hash_check[i] == data[i - first_index * HASH_OF_HASHES_STEP], + 0, "Consistency failure in m_blocks_hash_check construction"); + m_blocks_hash_check[i] = data[i - first_index * HASH_OF_HASHES_STEP]; + } + usable += HASH_OF_HASHES_STEP; + } + else + { + // if after the end of the precomputed blocks, accept anything + usable += HASH_OF_HASHES_STEP; + if (usable > hashes.size()) + usable = hashes.size(); + } + } + MDEBUG("usable: " << usable << " / " << hashes.size()); + CHECK_AND_ASSERT_MES(usable < std::numeric_limits::max() / 2, 0, "usable is negative"); + return usable; +} + //------------------------------------------------------------------ // ND: Speedups: // 1. Thread long_hash computations if possible (m_max_prepare_blocks_threads = nthreads, default = 4) @@ -4144,7 +4252,7 @@ void Blockchain::cancel() } #if defined(PER_BLOCK_CHECKPOINT) -static const char expected_block_hashes_hash[] = "d3ca80d50661684cde0e715d46d7c19704d2e216b21ed088af9fd4ef37ed4d65"; +static const char expected_block_hashes_hash[] = "4b553162ee4e7af3c53666506591489c68560b9175e6e941dc96c89f96f0e35c"; void Blockchain::load_compiled_in_block_hashes() { if (m_fast_sync && get_blocks_dat_start(m_testnet) != nullptr && get_blocks_dat_size(m_testnet) > 0) @@ -4180,16 +4288,18 @@ void Blockchain::load_compiled_in_block_hashes() const unsigned char *p = get_blocks_dat_start(m_testnet); const uint32_t nblocks = *p | ((*(p+1))<<8) | ((*(p+2))<<16) | ((*(p+3))<<24); const size_t size_needed = 4 + nblocks * sizeof(crypto::hash); - if(nblocks > 0 && nblocks > m_db->height() && get_blocks_dat_size(m_testnet) >= size_needed) + if(nblocks > 0 && nblocks * HASH_OF_HASHES_STEP > m_db->height() && get_blocks_dat_size(m_testnet) >= size_needed) { p += sizeof(uint32_t); + m_blocks_hash_of_hashes.reserve(nblocks); for (uint32_t i = 0; i < nblocks; i++) { crypto::hash hash; memcpy(hash.data, p, sizeof(hash.data)); p += sizeof(hash.data); - m_blocks_hash_check.push_back(hash); + m_blocks_hash_of_hashes.push_back(hash); } + m_blocks_hash_check.resize(m_blocks_hash_of_hashes.size() * HASH_OF_HASHES_STEP, cryptonote::null_hash); MINFO(nblocks << " block hashes loaded"); // FIXME: clear tx_pool because the process might have been @@ -4220,7 +4330,7 @@ void Blockchain::load_compiled_in_block_hashes() bool Blockchain::is_within_compiled_block_hash_area(uint64_t height) const { #if defined(PER_BLOCK_CHECKPOINT) - return height < m_blocks_hash_check.size(); + return height < m_blocks_hash_of_hashes.size() * HASH_OF_HASHES_STEP; #else return false; #endif diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index e2da535cd..4f3262878 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -955,6 +955,7 @@ namespace cryptonote bool is_within_compiled_block_hash_area(uint64_t height) const; bool is_within_compiled_block_hash_area() const { return is_within_compiled_block_hash_area(m_db->height()); } + uint64_t prevalidate_block_hashes(uint64_t height, const std::list &hashes); void lock(); void unlock(); @@ -995,6 +996,7 @@ namespace cryptonote std::unordered_map> m_check_txin_table; // SHA-3 hashes for each block and for fast pow checking + std::vector m_blocks_hash_of_hashes; std::vector m_blocks_hash_check; std::vector m_blocks_txs_check; diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 56485aedf..293fb42e9 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -1400,6 +1400,11 @@ namespace cryptonote return m_target_blockchain_height; } //----------------------------------------------------------------------------------------------- + uint64_t core::prevalidate_block_hashes(uint64_t height, const std::list &hashes) + { + return get_blockchain_storage().prevalidate_block_hashes(height, hashes); + } + //----------------------------------------------------------------------------------------------- std::time_t core::get_start_time() const { return start_time; diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 4ea33dbe1..f0ba16bea 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -752,6 +752,13 @@ namespace cryptonote */ bool fluffy_blocks_enabled() const { return m_fluffy_blocks_enabled; } + /** + * @brief check a set of hashes against the precompiled hash set + * + * @return number of usable blocks + */ + uint64_t prevalidate_block_hashes(uint64_t height, const std::list &hashes); + private: /** diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index 803d948cc..f599e0c8d 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -1583,10 +1583,22 @@ skip: return 1; } + uint64_t n_use_blocks = m_core.prevalidate_block_hashes(arg.start_height, arg.m_block_ids); + if (n_use_blocks == 0) + { + LOG_ERROR_CCONTEXT("Peer yielded no usable blocks, dropping connection"); + drop_connection(context, false, false); + return 1; + } + + uint64_t added = 0; for(auto& bl_id: arg.m_block_ids) { context.m_needed_objects.push_back(bl_id); + if (++added == n_use_blocks) + break; } + context.m_last_response_height -= arg.m_block_ids.size() - n_use_blocks; if (!request_missing_objects(context, false)) { -- cgit v1.2.3