From aba548cbf755277e99e3e3350f8232b103fb1ab5 Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Mon, 6 Oct 2014 19:46:25 -0400 Subject: import of BlockchainDB files tried rebasing, tree-filter, and many other things. at this point, the history of these files previous to this can live on in my bc2 branch, as I'm importing them as-is to here. --- src/cryptonote_core/blockchain.cpp | 2137 +++++++++++++++++++++++++++++++++ src/cryptonote_core/blockchain.h | 221 ++++ src/cryptonote_core/blockchain_db.cpp | 135 +++ src/cryptonote_core/blockchain_db.h | 510 ++++++++ 4 files changed, 3003 insertions(+) create mode 100644 src/cryptonote_core/blockchain.cpp create mode 100644 src/cryptonote_core/blockchain.h create mode 100644 src/cryptonote_core/blockchain_db.cpp create mode 100644 src/cryptonote_core/blockchain_db.h (limited to 'src') diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp new file mode 100644 index 000000000..22ae7a39a --- /dev/null +++ b/src/cryptonote_core/blockchain.cpp @@ -0,0 +1,2137 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +#include +#include +#include +#include + +#include "include_base_utils.h" +#include "cryptonote_basic_impl.h" +#include "blockchain.h" +#include "cryptonote_format_utils.h" +#include "cryptonote_boost_serialization.h" +#include "cryptonote_config.h" +#include "miner.h" +#include "misc_language.h" +#include "profile_tools.h" +#include "file_io_utils.h" +#include "common/boost_serialization_helper.h" +#include "warnings.h" +#include "crypto/hash.h" +//#include "serialization/json_archive.h" + +/* TODO: + * Clean up code: + * Possibly change how outputs are referred to/indexed in blockchain and wallets + * + */ + +using namespace cryptonote; + +DISABLE_VS_WARNINGS(4267) + +//------------------------------------------------------------------ +// TODO: initialize m_db with a concrete implementation of BlockchainDB +Blockchain::Blockchain(tx_memory_pool& tx_pool):m_db(), m_tx_pool(tx_pool), m_current_block_cumul_sz_limit(0), m_is_in_checkpoint_zone(false), m_is_blockchain_storing(false) +{ + if (m_db == NULL) + { + throw new DB_ERROR("database pointer null in blockchain init"); + } +} +//------------------------------------------------------------------ +//TODO: is this still needed? I don't think so - tewinget +template +void Blockchain::serialize(archive_t & ar, const unsigned int version) +{ + if(version < 11) + return; + CRITICAL_REGION_LOCAL(m_blockchain_lock); + ar & m_blocks; + ar & m_blocks_index; + ar & m_transactions; + ar & m_spent_keys; + ar & m_alternative_chains; + ar & m_outputs; + ar & m_invalid_blocks; + ar & m_current_block_cumul_sz_limit; + /*serialization bug workaround*/ + if(version > 11) + { + uint64_t total_check_count = m_db->height() + m_blocks_index.size() + m_transactions.size() + m_spent_keys.size() + m_alternative_chains.size() + m_outputs.size() + m_invalid_blocks.size() + m_current_block_cumul_sz_limit; + if(archive_t::is_saving::value) + { + ar & total_check_count; + }else + { + uint64_t total_check_count_loaded = 0; + ar & total_check_count_loaded; + if(total_check_count != total_check_count_loaded) + { + LOG_ERROR("Blockchain storage data corruption detected. total_count loaded from file = " << total_check_count_loaded << ", expected = " << total_check_count); + + LOG_PRINT_L0("Blockchain storage:" << std::endl << + "m_blocks: " << m_db->height() << std::endl << + "m_blocks_index: " << m_blocks_index.size() << std::endl << + "m_transactions: " << m_transactions.size() << std::endl << + "m_spent_keys: " << m_spent_keys.size() << std::endl << + "m_alternative_chains: " << m_alternative_chains.size() << std::endl << + "m_outputs: " << m_outputs.size() << std::endl << + "m_invalid_blocks: " << m_invalid_blocks.size() << std::endl << + "m_current_block_cumul_sz_limit: " << m_current_block_cumul_sz_limit); + + throw std::runtime_error("Blockchain data corruption"); + } + } + } + + + LOG_PRINT_L2("Blockchain storage:" << std::endl << + "m_blocks: " << m_db->height() << std::endl << + "m_blocks_index: " << m_blocks_index.size() << std::endl << + "m_transactions: " << m_transactions.size() << std::endl << + "m_spent_keys: " << m_spent_keys.size() << std::endl << + "m_alternative_chains: " << m_alternative_chains.size() << std::endl << + "m_outputs: " << m_outputs.size() << std::endl << + "m_invalid_blocks: " << m_invalid_blocks.size() << std::endl << + "m_current_block_cumul_sz_limit: " << m_current_block_cumul_sz_limit); +} +//------------------------------------------------------------------ +bool Blockchain::have_tx(const crypto::hash &id) +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + return m_db->tx_exists(id); +} +//------------------------------------------------------------------ +bool Blockchain::have_tx_keyimg_as_spent(const crypto::key_image &key_im) +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + return m_db->has_key_image(key_im); +} +//------------------------------------------------------------------ +// This function makes sure that each "input" in an input (mixins) exists +// and collects the public key for each from the transaction it was included in +// via the visitor passed to it. +template +bool Blockchain::scan_outputkeys_for_indexes(const txin_to_key& tx_in_to_key, visitor_t& vis, uint64_t* pmax_related_block_height) +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + // verify that the input has key offsets (that it exists properly, really) + if(!tx_in_to_key.key_offsets.size()) + return false; + + // cryptonote_format_utils uses relative offsets for indexing to the global + // outputs list. that is to say that absolute offset #2 is absolute offset + // #1 plus relative offset #2. + // TODO: Investigate if this is necessary / why this is done. + std::vector absolute_offsets = relative_output_offsets_to_absolute(tx_in_to_key.key_offsets); + + + //std::vector >& amount_outs_vec = it->second; + size_t count = 0; + for (const uint64_t& i : absolute_offsets) + { + try + { + // get tx hash and output index for output + auto output_index = m_db->get_output_tx_and_index(tx_in_to_key.amount, i); + + // get tx that output is from + auto tx = m_db->get_tx(output_index.first); + + // make sure output index is within range for the given transaction + if (output_index.second >= tx.vout.size()) + { + LOG_PRINT_L0("Output does not exist. tx = " << output_index.first << ", index = " << output_index.second); + return false; + } + + // call to the passed boost visitor to grab the public key for the output + if(!vis.handle_output(tx, tx.vout[output_index.second])) + { + LOG_PRINT_L0("Failed to handle_output for output no = " << count << ", with absolute offset " << i); + return false; + } + + // if on last output and pmax_related_block_height not null pointer + if(++count == absolute_offsets.size() && pmax_related_block_height) + { + // set *pmax_related_block_height to tx block height for this output + auto h = m_db->get_tx_block_height(output_index.first); + if(*pmax_related_block_height < h) + { + *pmax_related_block_height = h; + } + } + + } + catch (const OUTPUT_DNE& e) + { + LOG_PRINT_L0("Output does not exist: " << e.what()); + return false; + } + catch (const TX_DNE& e) + { + LOG_PRINT_L0("Transaction does not exist: " << e.what()); + return false; + } + + } + + return true; +} +//------------------------------------------------------------------ +uint64_t Blockchain::get_current_blockchain_height() +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + return m_db->height(); +} +//------------------------------------------------------------------ +//FIXME: possibly move this into the constructor, to avoid accidentally +// dereferencing a null BlockchainDB pointer +bool Blockchain::init(const std::string& config_folder) +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + m_config_folder = config_folder; + LOG_PRINT_L0("Loading blockchain..."); + + //FIXME: update filename for BlockchainDB + const std::string filename = m_config_folder + "/" CRYPTONOTE_BLOCKCHAINDATA_FILENAME; + try + { + m_db->open(filename); + } + catch (const DB_OPEN_FAILURE& e) + { + LOG_PRINT_L0("No blockchain file found, attempting to create one."); + try + { + m_db->create(filename); + } + catch (const DB_CREATE_FAILURE& db_create_error) + { + LOG_PRINT_L0("Unable to create BlockchainDB! This is not good..."); + //TODO: make sure whatever calls this handles the return value properly + return false; + } + } + + // if the blockchain is new, add the genesis block + // this feels kinda kludgy to do it this way, but can be looked at later. + if(!m_db->height()) + { + LOG_PRINT_L0("Blockchain not loaded, generating genesis block."); + block bl = boost::value_initialized(); + block_verification_context bvc = boost::value_initialized(); + generate_genesis_block(bl); + add_new_block(bl, bvc); + CHECK_AND_ASSERT_MES(!bvc.m_verification_failed, false, "Failed to add genesis block to blockchain"); + } + + // check how far behind we are + uint64_t top_block_timestamp = m_db->get_top_block_timestamp(); + uint64_t timestamp_diff = time(NULL) - top_block_timestamp; + + // genesis block has no timestamp, could probably change it to have timestamp of 1341378000... + if(!top_block_timestamp) + timestamp_diff = time(NULL) - 1341378000; + LOG_PRINT_GREEN("Blockchain initialized. last block: " << m_db->height() - 1 << ", " << epee::misc_utils::get_time_interval_string(timestamp_diff) << " time ago, current difficulty: " << get_difficulty_for_next_block(), LOG_LEVEL_0); + + return true; +} +//------------------------------------------------------------------ +bool Blockchain::store_blockchain() +{ + // TODO: make sure if this throws that it is not simply ignored higher + // up the call stack + try + { + m_db->sync(); + } + catch (const std::exception& e) + { + LOG_PRINT_L0(std::string("Error syncing blockchain db: ") + e.what() + "-- shutting down now to prevent issues!"); + throw; + } + catch (...) + { + LOG_PRINT_L0("There was an issue storing the blockchain, shutting down now to prevent issues!"); + throw; + } + LOG_PRINT_L0("Blockchain stored OK."); + return true; +} +//------------------------------------------------------------------ +bool Blockchain::deinit() +{ + // as this should be called if handling a SIGSEGV, need to check + // if m_db is a NULL pointer (and thus may have caused the illegal + // memory operation), otherwise we may cause a loop. + if (m_db == NULL) + { + throw new DB_ERROR("The db pointer is null in Blockchain, the blockchain may be corrupt!"); + } + + try + { + m_db->close(); + } + catch (const std::exception& e) + { + LOG_PRINT_L0(std::string("Error closing blockchain db: ") + e.what()); + } + catch (...) + { + LOG_PRINT_L0("There was an issue closing/storing the blockchain, shutting down now to prevent issues!"); + } + return true; +} +//------------------------------------------------------------------ +// This function tells BlockchainDB to remove the top block from the +// blockchain and then returns all transactions (except the miner tx, of course) +// from it to the tx_pool +block Blockchain::pop_block_from_blockchain() +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + block popped_block; + std::vector popped_txs; + + try + { + m_db->pop_block(popped_block, popped_txs); + } + // anything that could cause this to throw is likely catastrophic, + // so we re-throw + catch (const std::exception& e) + { + LOG_ERROR("Error popping block from blockchain: " << e.what()); + throw; + } + catch (...) + { + LOG_ERROR("Error popping block from blockchain, throwing!"); + throw; + } + + // return transactions from popped block to the tx_pool + for (transaction& tx : popped_txs) + { + if (!is_coinbase(tx)) + { + cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc); + bool r = m_tx_pool.add_tx(tx, tvc, true); + if (!r) + { + LOG_ERROR("Error returning transaction to tx_pool"); + } + } + } + + return popped_block; +} +//------------------------------------------------------------------ +bool Blockchain::reset_and_set_genesis_block(const block& b) +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + m_transactions.clear(); + m_spent_keys.clear(); + m_blocks.clear(); + m_blocks_index.clear(); + m_alternative_chains.clear(); + m_outputs.clear(); + m_db->reset(); + + block_verification_context bvc = boost::value_initialized(); + add_new_block(b, bvc); + return bvc.m_added_to_main_chain && !bvc.m_verification_failed; +} +//------------------------------------------------------------------ +//TODO: move to BlockchainDB subclass +bool Blockchain::purge_transaction_keyimages_from_blockchain(const transaction& tx, bool strict_check) +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + struct purge_transaction_visitor: public boost::static_visitor + { + key_images_container& m_spent_keys; + bool m_strict_check; + purge_transaction_visitor(key_images_container& spent_keys, bool strict_check):m_spent_keys(spent_keys), m_strict_check(strict_check){} + + bool operator()(const txin_to_key& inp) const + { + //const crypto::key_image& ki = inp.k_image; + auto r = m_spent_keys.find(inp.k_image); + if(r != m_spent_keys.end()) + { + m_spent_keys.erase(r); + }else + { + CHECK_AND_ASSERT_MES(!m_strict_check, false, "purge_block_data_from_blockchain: key image in transaction not found"); + } + return true; + } + bool operator()(const txin_gen& inp) const + { + return true; + } + bool operator()(const txin_to_script& tx) const + { + return false; + } + + bool operator()(const txin_to_scripthash& tx) const + { + return false; + } + }; + + BOOST_FOREACH(const txin_v& in, tx.vin) + { + bool r = boost::apply_visitor(purge_transaction_visitor(m_spent_keys, strict_check), in); + CHECK_AND_ASSERT_MES(!strict_check || r, false, "failed to process purge_transaction_visitor"); + } + return true; +} +//------------------------------------------------------------------ +crypto::hash Blockchain::get_tail_id(uint64_t& height) +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + height = m_db->height(); + return get_tail_id(); +} +//------------------------------------------------------------------ +crypto::hash Blockchain::get_tail_id() +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + return m_db->top_block_hash(); +} +//------------------------------------------------------------------ +/*TODO: this function was...poorly written. As such, I'm not entirely + * certain on what it was supposed to be doing. Need to look into this, + * but it doesn't seem terribly important just yet. + * + * puts into list a list of hashes representing certain blocks + * from the blockchain in reverse chronological order + * + * the blocks chosen, at the time of this writing, are: + * the most recent 11 + * powers of 2 less recent from there, so 13, 17, 25, etc... + * + */ +bool Blockchain::get_short_chain_history(std::list& ids) +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + uint64_t i = 0; + uint64_t current_multiplier = 1; + uint64_t sz = m_db->height(); + + if(!sz) + return true; + + uint64_t current_back_offset = 0; + while(current_back_offset < sz) + { + ids.push_back(m_db->get_block_hash_from_height(sz-current_back_offset)); + if(i < 10) + { + ++current_back_offset; + } + else + { + current_multiplier *= 2; + current_back_offset += current_multiplier; + } + ++i; + } + + return true; +} +//------------------------------------------------------------------ +crypto::hash Blockchain::get_block_id_by_height(uint64_t height) +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + try + { + return m_db->get_block_hash_from_height(height); + } + catch (const BLOCK_DNE& e) + { + } + catch (const std::exception& e) + { + LOG_PRINT_L0(std::string("Something went wrong fetching block hash by height: ") + e.what()); + throw; + } + catch (...) + { + LOG_PRINT_L0(std::string("Something went wrong fetching block hash by height")); + throw; + } + return null_hash; +} +//------------------------------------------------------------------ +bool Blockchain::get_block_by_hash(const crypto::hash &h, block &blk) +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + // try to find block in main chain + try + { + blk = m_db->get_block(h); + return true; + } + // try to find block in alternative chain + catch (const BLOCK_DNE& e) + { + blocks_ext_by_hash::const_iterator it_alt = m_alternative_chains.find(h); + if (m_alternative_chains.end() != it_alt) { + blk = it_alt->second.bl; + return true; + } + } + catch (const std::exception& e) + { + LOG_PRINT_L0(std::string("Something went wrong fetching block by hash: ") + e.what()); + throw; + } + catch (...) + { + LOG_PRINT_L0(std::string("Something went wrong fetching block hash by hash")); + throw; + } + + return false; +} +//------------------------------------------------------------------ +void Blockchain::get_all_known_block_ids(std::list &main, std::list &alt, std::list &invalid) +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + main = m_db->get_hashes_range(0, m_db->height()); + + BOOST_FOREACH(blocks_ext_by_hash::value_type &v, m_alternative_chains) + alt.push_back(v.first); + + BOOST_FOREACH(blocks_ext_by_hash::value_type &v, m_invalid_blocks) + invalid.push_back(v.first); +} +//------------------------------------------------------------------ +// This function aggregates the cumulative difficulties and timestamps of the +// last DIFFICULTY_BLOCKS_COUNT blocks and passes them to next_difficulty, +// returning the result of that call. Ignores the genesis block, and can use +// less blocks than desired if there aren't enough. +difficulty_type Blockchain::get_difficulty_for_next_block() +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + std::vector timestamps; + std::vector cumulative_difficulties; + auto h = m_db->height(); + + size_t offset = h - std::min(h, static_cast(DIFFICULTY_BLOCKS_COUNT)); + + // because BlockchainDB::height() returns the index of the top block, the + // first index we need to get needs to be one + // higher than height() - DIFFICULTY_BLOCKS_COUNT. This also conveniently + // makes sure we don't use the genesis block. + ++offset; + + for(; offset <= h; offset++) + { + timestamps.push_back(m_db->get_block_timestamp(offset)); + cumulative_difficulties.push_back(m_db->get_block_cumulative_difficulty(offset)); + } + return next_difficulty(timestamps, cumulative_difficulties); +} +//------------------------------------------------------------------ +// This function removes blocks from the blockchain until it gets to the +// position where the blockchain switch started and then re-adds the blocks +// that had been removed. +bool Blockchain::rollback_blockchain_switching(std::list& original_chain) +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + // remove blocks from blockchain until we get back to where we were. + // Conveniently, that's until our top block's hash == the parent block of + // the first block we need to add back. + while (m_db->top_block_hash() != original_chain.front().prev_id) + { + pop_block_from_blockchain(); + } + + //return back original chain + for (auto& bl : original_chain) + { + block_verification_context bvc = boost::value_initialized(); + bool r = handle_block_to_main_chain(bl, bvc); + CHECK_AND_ASSERT_MES(r && bvc.m_added_to_main_chain, false, "PANIC!!! failed to add (again) block while chain switching during the rollback!"); + } + + return true; +} +//------------------------------------------------------------------ +// This function attempts to switch to an alternate chain, returning +// boolean based on success therein. +bool Blockchain::switch_to_alternative_blockchain(std::list& alt_chain, bool discard_disconnected_chain) +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + // if empty alt chain passed (not sure how that could happen), return false + CHECK_AND_ASSERT_MES(alt_chain.size(), false, "switch_to_alternative_blockchain: empty chain passed"); + + // verify that main chain has front of alt chain's parent block + if (!m_db->block_exists(alt_chain.front()->second.bl.prev_id)) + { + LOG_ERROR("Attempting to move to an alternate chain, but it doesn't appear to connect to the main chain!"); + return false; + } + + // pop blocks from the blockchain until the top block is the parent + // of the front block of the alt chain. + std::list disconnected_chain; + while (m_db->top_block_hash() != alt_chain.front()->second.bl.prev_id) + { + block b = pop_block_from_blockchain(); + disconnected_chain.push_front(b); + } + + auto split_height = m_db->height(); + + //connecting new alternative chain + for(auto alt_ch_iter = alt_chain.begin(); alt_ch_iter != alt_chain.end(); alt_ch_iter++) + { + auto ch_ent = *alt_ch_iter; + block_verification_context bvc = boost::value_initialized(); + + // add block to main chain + bool r = handle_block_to_main_chain(ch_ent->second.bl, bvc); + + // if adding block to main chain failed, rollback to previous state and + // return false + if(!r || !bvc.m_added_to_main_chain) + { + LOG_PRINT_L0("Failed to switch to alternative blockchain"); + rollback_blockchain_switching(disconnected_chain); + + // FIXME: Why do we keep invalid blocks around? Possibly in case we hear + // about them again so we can immediately dismiss them, but needs some + // looking into. + add_block_as_invalid(ch_ent->second, get_block_hash(ch_ent->second.bl)); + LOG_PRINT_L0("The block was inserted as invalid while connecting new alternative chain, block_id: " << get_block_hash(ch_ent->second.bl)); + m_alternative_chains.erase(ch_ent); + + for(auto alt_ch_to_orph_iter = ++alt_ch_iter; alt_ch_to_orph_iter != alt_chain.end(); alt_ch_to_orph_iter++) + { + add_block_as_invalid((*alt_ch_iter)->second, (*alt_ch_iter)->first); + m_alternative_chains.erase(*alt_ch_to_orph_iter); + } + return false; + } + } + + // if we're to keep the disconnected blocks, add them as alternates + if(!discard_disconnected_chain) + { + //pushing old chain as alternative chain + for (auto& old_ch_ent : disconnected_chain) + { + block_verification_context bvc = boost::value_initialized(); + bool r = handle_alternative_block(old_ch_ent, get_block_hash(old_ch_ent), bvc); + if(!r) + { + LOG_ERROR("Failed to push ex-main chain blocks to alternative chain "); + // previously this would fail the blockchain switching, but I don't + // think this is bad enough to warrant that. + } + } + } + + //removing alt_chain entries from alternative chain + BOOST_FOREACH(auto ch_ent, alt_chain) + { + m_alternative_chains.erase(ch_ent); + } + + LOG_PRINT_GREEN("REORGANIZE SUCCESS! on height: " << split_height << ", new blockchain size: " << m_db->height(), LOG_LEVEL_0); + return true; +} +//------------------------------------------------------------------ +// This function calculates the difficulty target for the block being added to +// an alternate chain. +difficulty_type Blockchain::get_next_difficulty_for_alternative_chain(const std::list& alt_chain, block_extended_info& bei) +{ + std::vector timestamps; + std::vector cumulative_difficulties; + + // if the alt chain isn't long enough to calculate the difficulty target + // based on its blocks alone, need to get more blocks from the main chain + if(alt_chain.size()< DIFFICULTY_BLOCKS_COUNT) + { + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + // Figure out start and stop offsets for main chain blocks + size_t main_chain_stop_offset = alt_chain.size() ? alt_chain.front()->second.height : bei.height; + size_t main_chain_count = DIFFICULTY_BLOCKS_COUNT - std::min(static_cast(DIFFICULTY_BLOCKS_COUNT), alt_chain.size()); + main_chain_count = std::min(main_chain_count, main_chain_stop_offset); + size_t main_chain_start_offset = main_chain_stop_offset - main_chain_count; + + if(!main_chain_start_offset) + ++main_chain_start_offset; //skip genesis block + + // get difficulties and timestamps from relevant main chain blocks + for(; main_chain_start_offset < main_chain_stop_offset; ++main_chain_start_offset) + { + timestamps.push_back(m_db->get_block_timestamp(main_chain_start_offset)); + cumulative_difficulties.push_back(m_db->get_block_cumulative_difficulty(main_chain_start_offset)); + } + + // make sure we haven't accidentally grabbed too many blocks...maybe don't need this check? + CHECK_AND_ASSERT_MES((alt_chain.size() + timestamps.size()) <= DIFFICULTY_BLOCKS_COUNT, false, + "Internal error, alt_chain.size()[" << alt_chain.size() + << "] + vtimestampsec.size()[" << timestamps.size() + << "] NOT <= DIFFICULTY_WINDOW[]" << DIFFICULTY_BLOCKS_COUNT + ); + + for (auto it : alt_chain) + { + timestamps.push_back(it->second.bl.timestamp); + cumulative_difficulties.push_back(it->second.cumulative_difficulty); + } + } + // if the alt chain is long enough for the difficulty calc, grab difficulties + // and timestamps from it alone + else + { + timestamps.resize(static_cast(DIFFICULTY_BLOCKS_COUNT)); + cumulative_difficulties.resize(static_cast(DIFFICULTY_BLOCKS_COUNT)); + size_t count = 0; + size_t max_i = timestamps.size()-1; + // get difficulties and timestamps from most recent blocks in alt chain + BOOST_REVERSE_FOREACH(auto it, alt_chain) + { + timestamps[max_i - count] = it->second.bl.timestamp; + cumulative_difficulties[max_i - count] = it->second.cumulative_difficulty; + count++; + if(count >= DIFFICULTY_BLOCKS_COUNT) + break; + } + } + + // calculate the difficulty target for the block and return it + return next_difficulty(timestamps, cumulative_difficulties); +} +//------------------------------------------------------------------ +// This function does a sanity check on basic things that all miner +// transactions have in common, such as: +// one input, of type txin_gen, with height set to the block's height +// correct miner tx unlock time +// a non-overflowing tx amount (dubious necessity on this check) +bool Blockchain::prevalidate_miner_transaction(const block& b, uint64_t height) +{ + CHECK_AND_ASSERT_MES(b.miner_tx.vin.size() == 1, false, "coinbase transaction in the block has no inputs"); + CHECK_AND_ASSERT_MES(b.miner_tx.vin[0].type() == typeid(txin_gen), false, "coinbase transaction in the block has the wrong type"); + if(boost::get(b.miner_tx.vin[0]).height != height) + { + LOG_PRINT_RED_L0("The miner transaction in block has invalid height: " << boost::get(b.miner_tx.vin[0]).height << ", expected: " << height); + return false; + } + CHECK_AND_ASSERT_MES(b.miner_tx.unlock_time == height + CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW, + false, + "coinbase transaction transaction have wrong unlock time=" << b.miner_tx.unlock_time << ", expected " << height + CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW); + + //check outs overflow + //NOTE: not entirely sure this is necessary, given that this function is + // designed simply to make sure the total amount for a transaction + // does not overflow a uint64_t, and this transaction *is* a uint64_t... + if(!check_outs_overflow(b.miner_tx)) + { + LOG_PRINT_RED_L0("miner transaction have money overflow in block " << get_block_hash(b)); + return false; + } + + return true; +} +//------------------------------------------------------------------ +// This function validates the miner transaction reward +bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins) +{ + //validate reward + uint64_t money_in_use = 0; + BOOST_FOREACH(auto& o, b.miner_tx.vout) + money_in_use += o.amount; + + std::vector last_blocks_sizes; + get_last_n_blocks_sizes(last_blocks_sizes, CRYPTONOTE_REWARD_BLOCKS_WINDOW); + if (!get_block_reward(epee::misc_utils::median(last_blocks_sizes), cumulative_block_size, already_generated_coins, base_reward)) { + LOG_PRINT_L0("block size " << cumulative_block_size << " is bigger than allowed for this blockchain"); + return false; + } + if(base_reward + fee < money_in_use) + { + LOG_ERROR("coinbase transaction spend too much money (" << print_money(money_in_use) << "). Block reward is " << print_money(base_reward + fee) << "(" << print_money(base_reward) << "+" << print_money(fee) << ")"); + return false; + } + if(base_reward + fee != money_in_use) + { + LOG_ERROR("coinbase transaction doesn't use full amount of block reward: spent: " + << print_money(money_in_use) << ", block reward " << print_money(base_reward + fee) << "(" << print_money(base_reward) << "+" << print_money(fee) << ")"); + return false; + } + return true; +} +//------------------------------------------------------------------ +// get the block sizes of the last blocks, starting at +// and return by reference . +void Blockchain::get_last_n_blocks_sizes(std::vector& sz, size_t count) +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + auto h = m_db->height(); + + // this function is meaningless for an empty blockchain...granted it should never be empty + if(h == 0) + return; + + // add size of last blocks to vector (or less, if blockchain size < count) + size_t start_offset = (h+1) - std::min((h+1), count); + for(size_t i = start_offset; i <= h; i++) + { + sz.push_back(m_db->get_block_size(i)); + } +} +//------------------------------------------------------------------ +uint64_t Blockchain::get_current_cumulative_blocksize_limit() +{ + return m_current_block_cumul_sz_limit; +} +//------------------------------------------------------------------ +//TODO: This function only needed minor modification to work with BlockchainDB, +// and *works*. As such, to reduce the number of things that might break +// in moving to BlockchainDB, this function will remain otherwise +// unchanged for the time being. +// +// This function makes a new block for a miner to mine the hash for +// +// FIXME: this codebase references #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) +// in a lot of places. That flag is not referenced in any of the code +// nor any of the makefiles, howeve. Need to look into whether or not it's +// necessary at all. +bool Blockchain::create_block_template(block& b, const account_public_address& miner_address, difficulty_type& diffic, uint64_t& height, const blobdata& ex_nonce) +{ + size_t median_size; + uint64_t already_generated_coins; + + CRITICAL_REGION_BEGIN(m_blockchain_lock); + b.major_version = CURRENT_BLOCK_MAJOR_VERSION; + b.minor_version = CURRENT_BLOCK_MINOR_VERSION; + b.prev_id = get_tail_id(); + b.timestamp = time(NULL); + + auto old_height = m_db->height(); + height = old_height + 1; + diffic = get_difficulty_for_next_block(); + CHECK_AND_ASSERT_MES(diffic, false, "difficulty owverhead."); + + median_size = m_current_block_cumul_sz_limit / 2; + already_generated_coins = m_db->get_block_already_generated_coins(old_height); + + CRITICAL_REGION_END(); + + size_t txs_size; + uint64_t fee; + if (!m_tx_pool.fill_block_template(b, median_size, already_generated_coins, txs_size, fee)) { + return false; + } +#if defined(DEBUG_CREATE_BLOCK_TEMPLATE) + size_t real_txs_size = 0; + uint64_t real_fee = 0; + CRITICAL_REGION_BEGIN(m_tx_pool.m_transactions_lock); + BOOST_FOREACH(crypto::hash &cur_hash, b.tx_hashes) { + auto cur_res = m_tx_pool.m_transactions.find(cur_hash); + if (cur_res == m_tx_pool.m_transactions.end()) { + LOG_ERROR("Creating block template: error: transaction not found"); + continue; + } + tx_memory_pool::tx_details &cur_tx = cur_res->second; + real_txs_size += cur_tx.blob_size; + real_fee += cur_tx.fee; + if (cur_tx.blob_size != get_object_blobsize(cur_tx.tx)) { + LOG_ERROR("Creating block template: error: invalid transaction size"); + } + uint64_t inputs_amount; + if (!get_inputs_money_amount(cur_tx.tx, inputs_amount)) { + LOG_ERROR("Creating block template: error: cannot get inputs amount"); + } else if (cur_tx.fee != inputs_amount - get_outs_money_amount(cur_tx.tx)) { + LOG_ERROR("Creating block template: error: invalid fee"); + } + } + if (txs_size != real_txs_size) { + LOG_ERROR("Creating block template: error: wrongly calculated transaction size"); + } + if (fee != real_fee) { + LOG_ERROR("Creating block template: error: wrongly calculated fee"); + } + CRITICAL_REGION_END(); + LOG_PRINT_L1("Creating block template: height " << height << + ", median size " << median_size << + ", already generated coins " << already_generated_coins << + ", transaction size " << txs_size << + ", fee " << fee); +#endif + + /* + two-phase miner transaction generation: we don't know exact block size until we prepare block, but we don't know reward until we know + block size, so first miner transaction generated with fake amount of money, and with phase we know think we know expected block size + */ + //make blocks coin-base tx looks close to real coinbase tx to get truthful blob size + bool r = construct_miner_tx(height, median_size, already_generated_coins, txs_size, fee, miner_address, b.miner_tx, ex_nonce, 11); + CHECK_AND_ASSERT_MES(r, false, "Failed to construc miner tx, first chance"); + size_t cumulative_size = txs_size + get_object_blobsize(b.miner_tx); +#if defined(DEBUG_CREATE_BLOCK_TEMPLATE) + LOG_PRINT_L1("Creating block template: miner tx size " << get_object_blobsize(b.miner_tx) << + ", cumulative size " << cumulative_size); +#endif + for (size_t try_count = 0; try_count != 10; ++try_count) { + r = construct_miner_tx(height, median_size, already_generated_coins, cumulative_size, fee, miner_address, b.miner_tx, ex_nonce, 11); + + CHECK_AND_ASSERT_MES(r, false, "Failed to construc miner tx, second chance"); + size_t coinbase_blob_size = get_object_blobsize(b.miner_tx); + if (coinbase_blob_size > cumulative_size - txs_size) { + cumulative_size = txs_size + coinbase_blob_size; +#if defined(DEBUG_CREATE_BLOCK_TEMPLATE) + LOG_PRINT_L1("Creating block template: miner tx size " << coinbase_blob_size << + ", cumulative size " << cumulative_size << " is greater then before"); +#endif + continue; + } + + if (coinbase_blob_size < cumulative_size - txs_size) { + size_t delta = cumulative_size - txs_size - coinbase_blob_size; +#if defined(DEBUG_CREATE_BLOCK_TEMPLATE) + LOG_PRINT_L1("Creating block template: miner tx size " << coinbase_blob_size << + ", cumulative size " << txs_size + coinbase_blob_size << + " is less then before, adding " << delta << " zero bytes"); +#endif + b.miner_tx.extra.insert(b.miner_tx.extra.end(), delta, 0); + //here could be 1 byte difference, because of extra field counter is varint, and it can become from 1-byte len to 2-bytes len. + if (cumulative_size != txs_size + get_object_blobsize(b.miner_tx)) { + CHECK_AND_ASSERT_MES(cumulative_size + 1 == txs_size + get_object_blobsize(b.miner_tx), false, "unexpected case: cumulative_size=" << cumulative_size << " + 1 is not equal txs_cumulative_size=" << txs_size << " + get_object_blobsize(b.miner_tx)=" << get_object_blobsize(b.miner_tx)); + b.miner_tx.extra.resize(b.miner_tx.extra.size() - 1); + if (cumulative_size != txs_size + get_object_blobsize(b.miner_tx)) { + //fuck, not lucky, -1 makes varint-counter size smaller, in that case we continue to grow with cumulative_size + LOG_PRINT_RED("Miner tx creation have no luck with delta_extra size = " << delta << " and " << delta - 1 , LOG_LEVEL_2); + cumulative_size += delta - 1; + continue; + } + LOG_PRINT_GREEN("Setting extra for block: " << b.miner_tx.extra.size() << ", try_count=" << try_count, LOG_LEVEL_1); + } + } + CHECK_AND_ASSERT_MES(cumulative_size == txs_size + get_object_blobsize(b.miner_tx), false, "unexpected case: cumulative_size=" << cumulative_size << " is not equal txs_cumulative_size=" << txs_size << " + get_object_blobsize(b.miner_tx)=" << get_object_blobsize(b.miner_tx)); +#if defined(DEBUG_CREATE_BLOCK_TEMPLATE) + LOG_PRINT_L1("Creating block template: miner tx size " << coinbase_blob_size << + ", cumulative size " << cumulative_size << " is now good"); +#endif + return true; + } + LOG_ERROR("Failed to create_block_template with " << 10 << " tries"); + return false; +} +//------------------------------------------------------------------ +// for an alternate chain, get the timestamps from the main chain to complete +// the needed number of timestamps for the BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW. +bool Blockchain::complete_timestamps_vector(uint64_t start_top_height, std::vector& timestamps) +{ + + if(timestamps.size() >= BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW) + return true; + + CRITICAL_REGION_LOCAL(m_blockchain_lock); + size_t need_elements = BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW - timestamps.size(); + CHECK_AND_ASSERT_MES(start_top_height <= m_db->height(), false, "internal error: passed start_height > " << " m_db->height() -- " << start_top_height << " > " << m_db->height()); + size_t stop_offset = start_top_height > need_elements ? start_top_height - need_elements : 0; + while (start_top_height != stop_offset); + { + timestamps.push_back(m_db->get_block_timestamp(start_top_height)); + --start_top_height; + } + return true; +} +//------------------------------------------------------------------ +// If a block is to be added and its parent block is not the current +// main chain top block, then we need to see if we know about its parent block. +// If its parent block is part of a known forked chain, then we need to see +// if that chain is long enough to become the main chain and re-org accordingly +// if so. If not, we need to hang on to the block in case it becomes part of +// a long forked chain eventually. +bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id, block_verification_context& bvc) +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + uint64_t block_height = get_block_height(b); + if(0 == block_height) + { + LOG_ERROR("Block with id: " << epee::string_tools::pod_to_hex(id) << " (as alternative) have wrong miner transaction"); + bvc.m_verification_failed = true; + return false; + } + // TODO: this basically says if the blockchain is smaller than the first + // checkpoint then alternate blocks are allowed...this seems backwards, but + // I'm not sure. Needs further investigating. + if (!m_checkpoints.is_alternative_block_allowed(get_current_blockchain_height(), block_height)) + { + LOG_PRINT_RED_L0("Block with id: " << id + << std::endl << " can't be accepted for alternative chain, block height: " << block_height + << std::endl << " blockchain height: " << get_current_blockchain_height()); + bvc.m_verification_failed = true; + return false; + } + + //block is not related with head of main chain + //first of all - look in alternative chains container + auto it_prev = m_alternative_chains.find(b.prev_id); + bool parent_in_main = m_db->block_exists(b.prev_id); + if(it_prev != m_alternative_chains.end() || parent_in_main) + { + //we have new block in alternative chain + + //build alternative subchain, front -> mainchain, back -> alternative head + blocks_ext_by_hash::iterator alt_it = it_prev; //m_alternative_chains.find() + std::list alt_chain; + std::vector timestamps; + while(alt_it != m_alternative_chains.end()) + { + alt_chain.push_front(alt_it); + timestamps.push_back(alt_it->second.bl.timestamp); + alt_it = m_alternative_chains.find(alt_it->second.bl.prev_id); + } + + // if block to be added connects to known blocks that aren't part of the + // main chain -- that is, if we're adding on to an alternate chain + if(alt_chain.size()) + { + // make sure alt chain doesn't somehow start past the end of the main chain + CHECK_AND_ASSERT_MES(m_db->height() > alt_chain.front()->second.height, false, "main blockchain wrong height"); + + // make sure that the blockchain contains the block that should connect + // this alternate chain with it. + if (!m_db->block_exists(alt_chain.front()->second.bl.prev_id)) + { + LOG_PRINT_L1("alternate chain does not appear to connect to main chain..."); + return false; + } + + // make sure block connects correctly to the main chain + auto h = m_db->get_block_hash_from_height(alt_chain.front()->second.height - 1); + CHECK_AND_ASSERT_MES(h == alt_chain.front()->second.bl.prev_id, false, "alternative chain have wrong connection to main chain"); + complete_timestamps_vector(m_db->get_block_height(alt_chain.front()->second.bl.prev_id), timestamps); + } + // if block not associated with known alternate chain + else + { + // if block parent is not part of main chain or an alternate chain, + // we ignore it + CHECK_AND_ASSERT_MES(parent_in_main, false, "internal error: broken imperative condition it_main_prev != m_blocks_index.end()"); + + complete_timestamps_vector(m_db->get_block_height(b.prev_id), timestamps); + } + + // verify that the block's timestamp is within the acceptable range + // (not earlier than the median of the last X blocks) + if(!check_block_timestamp(timestamps, b)) + { + LOG_PRINT_RED_L0("Block with id: " << id + << std::endl << " for alternative chain, have invalid timestamp: " << b.timestamp); + bvc.m_verification_failed = true; + return false; + } + + // FIXME: consider moving away from block_extended_info at some point + block_extended_info bei = boost::value_initialized(); + bei.bl = b; + bei.height = alt_chain.size() ? it_prev->second.height + 1 : m_db->get_block_height(b.prev_id) + 1; + + bool is_a_checkpoint; + if(!m_checkpoints.check_block(bei.height, id, is_a_checkpoint)) + { + LOG_ERROR("CHECKPOINT VALIDATION FAILED"); + bvc.m_verification_failed = true; + return false; + } + + // Check the block's hash against the difficulty target for its alt chain + m_is_in_checkpoint_zone = false; + difficulty_type current_diff = get_next_difficulty_for_alternative_chain(alt_chain, bei); + CHECK_AND_ASSERT_MES(current_diff, false, "!!!!!!! DIFFICULTY OVERHEAD !!!!!!!"); + crypto::hash proof_of_work = null_hash; + get_block_longhash(bei.bl, proof_of_work, bei.height); + if(!check_hash(proof_of_work, current_diff)) + { + LOG_PRINT_RED_L0("Block with id: " << id + << std::endl << " for alternative chain, have not enough proof of work: " << proof_of_work + << std::endl << " expected difficulty: " << current_diff); + bvc.m_verification_failed = true; + return false; + } + + if(!prevalidate_miner_transaction(b, bei.height)) + { + LOG_PRINT_RED_L0("Block with id: " << epee::string_tools::pod_to_hex(id) + << " (as alternative) have wrong miner transaction."); + bvc.m_verification_failed = true; + return false; + + } + + // FIXME: + // this brings up an interesting point: consider allowing to get block + // difficulty both by height OR by hash, not just height. + bei.cumulative_difficulty = alt_chain.size() ? it_prev->second.cumulative_difficulty : m_db->get_block_cumulative_difficulty(m_db->get_block_height(b.prev_id)); + bei.cumulative_difficulty += current_diff; + + // add block to alternate blocks storage, + // as well as the current "alt chain" container + auto i_res = m_alternative_chains.insert(blocks_ext_by_hash::value_type(id, bei)); + CHECK_AND_ASSERT_MES(i_res.second, false, "insertion of new alternative block returned as it already exist"); + alt_chain.push_back(i_res.first); + + // FIXME: is it even possible for a checkpoint to show up not on the main chain? + if(is_a_checkpoint) + { + //do reorganize! + LOG_PRINT_GREEN("###### REORGANIZE on height: " << alt_chain.front()->second.height << " of " << m_db->height() - 1 << + ", checkpoint is found in alternative chain on height " << bei.height, LOG_LEVEL_0); + + bool r = switch_to_alternative_blockchain(alt_chain, true); + + bvc.m_added_to_main_chain = r; + bvc.m_verification_failed = !r; + + return r; + } + else if(m_blocks.back().cumulative_difficulty < bei.cumulative_difficulty) //check if difficulty bigger then in main chain + { + //do reorganize! + LOG_PRINT_GREEN("###### REORGANIZE on height: " + << alt_chain.front()->second.height << " of " << m_db->height() + << " with cum_difficulty " << m_db->get_block_cumulative_difficulty(m_db->height()) + << std::endl << " alternative blockchain size: " << alt_chain.size() + << " with cum_difficulty " << bei.cumulative_difficulty, LOG_LEVEL_0 + ); + + bool r = switch_to_alternative_blockchain(alt_chain, false); + if(r) bvc.m_added_to_main_chain = true; + else bvc.m_verification_failed = true; + return r; + } + else + { + LOG_PRINT_BLUE("----- BLOCK ADDED AS ALTERNATIVE ON HEIGHT " << bei.height + << std::endl << "id:\t" << id + << std::endl << "PoW:\t" << proof_of_work + << std::endl << "difficulty:\t" << current_diff, LOG_LEVEL_0); + return true; + } + } + else + { + //block orphaned + bvc.m_marked_as_orphaned = true; + LOG_PRINT_RED_L0("Block recognized as orphaned and rejected, id = " << id); + } + + return true; +} +//------------------------------------------------------------------ +bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::list& blocks, std::list& txs) +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + if(start_offset > m_db->height()) + return false; + + if (!get_blocks(start_offset, count, blocks)) + { + return false; + } + + for(const block& blk : blocks) + { + std::list missed_ids; + get_transactions(blk.tx_hashes, txs, missed_ids); + CHECK_AND_ASSERT_MES(!missed_ids.size(), false, "have missed transactions in own block in main blockchain"); + } + + return true; +} +//------------------------------------------------------------------ +bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::list& blocks) +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + if(start_offset > m_db->height()) + return false; + + for(size_t i = start_offset; i < start_offset + count && i <= m_db->height();i++) + { + blocks.push_back(m_db->get_block_from_height(i)); + } + return true; +} +//------------------------------------------------------------------ +//TODO: This function *looks* like it won't need to be rewritten +// to use BlockchainDB, as it calls other functions that were, +// but it warrants some looking into later. +bool Blockchain::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NOTIFY_RESPONSE_GET_OBJECTS::request& rsp) +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + rsp.current_blockchain_height = get_current_blockchain_height(); + std::list blocks; + get_blocks(arg.blocks, blocks, rsp.missed_ids); + + BOOST_FOREACH(const auto& bl, blocks) + { + std::list missed_tx_id; + std::list txs; + get_transactions(bl.tx_hashes, txs, rsp.missed_ids); + CHECK_AND_ASSERT_MES(!missed_tx_id.size(), false, "Internal error: have missed missed_tx_id.size()=" << missed_tx_id.size() + << std::endl << "for block id = " << get_block_hash(bl)); + rsp.blocks.push_back(block_complete_entry()); + block_complete_entry& e = rsp.blocks.back(); + //pack block + e.block = t_serializable_object_to_blob(bl); + //pack transactions + BOOST_FOREACH(transaction& tx, txs) + e.txs.push_back(t_serializable_object_to_blob(tx)); + + } + //get another transactions, if need + std::list txs; + get_transactions(arg.txs, txs, rsp.missed_ids); + //pack aside transactions + BOOST_FOREACH(const auto& tx, txs) + rsp.txs.push_back(t_serializable_object_to_blob(tx)); + + return true; +} +//------------------------------------------------------------------ +bool Blockchain::get_alternative_blocks(std::list& blocks) +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + BOOST_FOREACH(const auto& alt_bl, m_alternative_chains) + { + blocks.push_back(alt_bl.second.bl); + } + return true; +} +//------------------------------------------------------------------ +size_t Blockchain::get_alternative_blocks_count() +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + return m_alternative_chains.size(); +} +//------------------------------------------------------------------ +// This function adds the output specified by to the result_outs container +// unlocked and other such checks should be done by here. +void Blockchain::add_out_to_get_random_outs(COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i) +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry& oen = *result_outs.outs.insert(result_outs.outs.end(), COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry()); + oen.global_amount_index = i; + oen.out_key = m_db->get_output_key(amount, i); +} +//------------------------------------------------------------------ +// This function takes an RPC request for mixins and creates an RPC response +// with the requested mixins. +// TODO: figure out why this returns boolean / if we should be returning false +// in some cases +bool Blockchain::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) +{ + srand(static_cast(time(NULL))); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + // for each amount that we need to get mixins for, get random outputs + // from BlockchainDB where is req.outs_count (number of mixins). + for (uint64_t amount : req.amounts) + { + // create outs_for_amount struct and populate amount field + COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs = *res.outs.insert(res.outs.end(), COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount()); + result_outs.amount = amount; + + std::unordered_set seen_indices; + + // if there aren't enough outputs to mix with (or just enough), + // use all of them. Eventually this should become impossible. + if (m_db->get_num_outputs(amount) <= req.outs_count) + { + for (uint64_t i = 0; i < m_db->get_num_outputs(amount); i++) + { + // get tx_hash, tx_out_index from DB + tx_out_index toi = m_db->get_output_tx_and_index(amount, i); + + // if tx is unlocked, add output to result_outs + if (is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first))) + { + add_out_to_get_random_outs(result_outs, amount, i); + } + + } + } + else + { + // while we still need more mixins + auto num_outs = m_db->get_num_outputs(amount); + while (result_outs.outs.size() < req.outs_count) + { + // if we've gone through every possible output, we've gotten all we can + if (seen_indices.size() == num_outs) + { + break; + } + + // get a random output index from the DB. If we've already seen it, + // return to the top of the loop and try again, otherwise add it to the + // list of output indices we've seen. + uint64_t i = m_db->get_random_output(amount); + if (seen_indices.count(i)) + { + continue; + } + seen_indices.emplace(i); + + // get tx_hash, tx_out_index from DB + tx_out_index toi = m_db->get_output_tx_and_index(amount, i); + + // if the output's transaction is unlocked, add the output's index to + // our list. + if (is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first))) + { + add_out_to_get_random_outs(result_outs, amount, i); + } + } + } + } + return true; +} +//------------------------------------------------------------------ +// This function takes a list of block hashes from another node +// on the network to find where the split point is between us and them. +// This is used to see what to send another node that needs to sync. +bool Blockchain::find_blockchain_supplement(const std::list& qblock_ids, uint64_t& starter_offset) +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + // make sure the request includes at least the genesis block, otherwise + // how can we expect to sync from the client that the block list came from? + if(!qblock_ids.size() /*|| !req.m_total_height*/) + { + LOG_ERROR("Client sent wrong NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << qblock_ids.size() << /*", m_height=" << req.m_total_height <<*/ ", dropping connection"); + return false; + } + + // make sure that the last block in the request's block list matches + // the genesis block + auto gen_hash = m_db->get_block_hash_from_height(0); + if(qblock_ids.back() != gen_hash) + { + LOG_ERROR("Client sent wrong NOTIFY_REQUEST_CHAIN: genesis block missmatch: " << std::endl << "id: " + << qblock_ids.back() << ", " << std::endl << "expected: " << gen_hash + << "," << std::endl << " dropping connection"); + return false; + } + + // Find the first block the foreign chain has that we also have. + // Assume qblock_ids is in reverse-chronological order. + auto bl_it = qblock_ids.begin(); + uint64_t split_height = 0; + for(; bl_it != qblock_ids.end(); bl_it++) + { + try + { + split_height = m_db->get_block_height(*bl_it); + break; + } + catch (const BLOCK_DNE& e) + { + continue; + } + catch (const std::exception& e) + { + LOG_PRINT_L1("Non-critical error trying to find block by hash in BlockchainDB, hash: " << *bl_it); + return false; + } + } + + // this should be impossible, as we checked that we share the genesis block, + // but just in case... + if(bl_it == qblock_ids.end()) + { + LOG_ERROR("Internal error handling connection, can't find split point"); + return false; + } + + // if split_height remains 0, we didn't have any but the genesis block in common + if(split_height == 0) + { + LOG_ERROR("Ours and foreign blockchain have only genesis block in common... o.O"); + return false; + } + + //we start to put block ids INCLUDING last known id, just to make other side be sure + starter_offset = split_height; + return true; +} +//------------------------------------------------------------------ +uint64_t Blockchain::block_difficulty(uint64_t i) +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + try + { + return m_db->get_block_difficulty(i); + } + catch (const BLOCK_DNE& e) + { + LOG_PRINT_L0("Attempted to get block difficulty for height above blockchain height"); + } + return 0; +} +//------------------------------------------------------------------ +template +void Blockchain::get_blocks(const t_ids_container& block_ids, t_blocks_container& blocks, t_missed_container& missed_bs) +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + for (const auto& block_hash : block_ids) + { + try + { + blocks.push_back(m_db->get_block(block_hash)); + } + catch (const BLOCK_DNE& e) + { + missed_bs.push_back(block_hash); + } + } +} +//------------------------------------------------------------------ +template +void Blockchain::get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + for (const auto& tx_hash : txs_ids) + { + try + { + txs.push_back(m_db->get_tx(tx_hash)); + } + catch (const TX_DNE& e) + { + missed_txs.push_back(tx_hash); + } + } +} +//------------------------------------------------------------------ +void Blockchain::print_blockchain(uint64_t start_index, uint64_t end_index) +{ + std::stringstream ss; + CRITICAL_REGION_LOCAL(m_blockchain_lock); + auto h = m_db->height(); + if(start_index > h) + { + LOG_PRINT_L0("Wrong starter index set: " << start_index << ", expected max index " << h); + return; + } + + for(size_t i = start_index; i <= h && i != end_index; i++) + { + ss << "height " << i + << ", timestamp " << m_db->get_block_timestamp(i) + << ", cumul_dif " << m_db->get_block_cumulative_difficulty(i) + << ", size " << m_db->get_block_size(i) + << "\nid\t\t" << m_db->get_block_hash_from_height(i) + << "\ndifficulty\t\t" << m_db->get_block_difficulty(i) + << ", nonce " << m_db->get_block_from_height(i).nonce + << ", tx_count " << m_db->get_block_from_height(i).tx_hashes.size() + << std::endl; + } + LOG_PRINT_L1("Current blockchain:" << std::endl << ss.str()); + LOG_PRINT_L0("Blockchain printed with log level 1"); +} +//------------------------------------------------------------------ +void Blockchain::print_blockchain_index() +{ + std::stringstream ss; + CRITICAL_REGION_LOCAL(m_blockchain_lock); + for(uint64_t i = 0; i <= m_db->height(); i++) + { + ss << "height: " << i << ", hash: " << m_db->get_block_hash_from_height(i); + } + + LOG_PRINT_L0("Current blockchain index:" << std::endl + << ss.str() + ); +} +//------------------------------------------------------------------ +//TODO: remove this function and references to it +void Blockchain::print_blockchain_outs(const std::string& file) +{ + return; +} +//------------------------------------------------------------------ +// Find the split point between us and foreign blockchain and return +// (by reference) the most recent common block hash along with up to +// BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT additional (more recent) hashes. +bool Blockchain::find_blockchain_supplement(const std::list& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + // if we can't find the split point, return false + if(!find_blockchain_supplement(qblock_ids, resp.start_height)) + { + return false; + } + + resp.total_height = get_current_blockchain_height(); + size_t count = 0; + for(size_t i = resp.start_height; i <= m_db->height() && count < BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT; i++, count++) + { + resp.m_block_ids.push_back(m_db->get_block_hash_from_height(i)); + } + return true; +} +//------------------------------------------------------------------ +//FIXME: change argument to std::vector, low priority +// find split point between ours and foreign blockchain (or start at +// blockchain height ), and return up to max_count FULL +// blocks by reference. +bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count) +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + // if a specific start height has been requested + if(req_start_block > 0) + { + // if requested height is higher than our chain, return false -- we can't help + if (req_start_block > m_db->height()) + { + return false; + } + start_height = req_start_block; + } + else + { + if(!find_blockchain_supplement(qblock_ids, start_height)) + { + return false; + } + } + + total_height = get_current_blockchain_height(); + size_t count = 0; + for(size_t i = start_height; i <= m_db->height() && count < max_count; i++, count++) + { + blocks.resize(blocks.size()+1); + blocks.back().first = m_db->get_block_from_height(i); + std::list mis; + get_transactions(m_blocks[i].bl.tx_hashes, blocks.back().second, mis); + CHECK_AND_ASSERT_MES(!mis.size(), false, "internal error, transaction from block not found"); + } + return true; +} +//------------------------------------------------------------------ +bool Blockchain::add_block_as_invalid(const block& bl, const crypto::hash& h) +{ + block_extended_info bei = AUTO_VAL_INIT(bei); + bei.bl = bl; + return add_block_as_invalid(bei, h); +} +//------------------------------------------------------------------ +bool Blockchain::add_block_as_invalid(const block_extended_info& bei, const crypto::hash& h) +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + auto i_res = m_invalid_blocks.insert(std::map::value_type(h, bei)); + CHECK_AND_ASSERT_MES(i_res.second, false, "at insertion invalid by tx returned status existed"); + LOG_PRINT_L0("BLOCK ADDED AS INVALID: " << h << std::endl << ", prev_id=" << bei.bl.prev_id << ", m_invalid_blocks count=" << m_invalid_blocks.size()); + return true; +} +//------------------------------------------------------------------ +bool Blockchain::have_block(const crypto::hash& id) +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + if(m_db->block_exists(id)) + return true; + + if(m_alternative_chains.count(id)) + return true; + + if(m_invalid_blocks.count(id)) + return true; + + return false; +} +//------------------------------------------------------------------ +bool Blockchain::handle_block_to_main_chain(const block& bl, block_verification_context& bvc) +{ + crypto::hash id = get_block_hash(bl); + return handle_block_to_main_chain(bl, id, bvc); +} +//------------------------------------------------------------------ +size_t Blockchain::get_total_transactions() +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + return m_db->get_tx_count(); +} +//------------------------------------------------------------------ +// This function checks each input in the transaction to make sure it +// has not been used already, and adds its key to the container . +// +// This container should be managed by the code that validates blocks so we don't +// have to store the used keys in a given block in the permanent storage only to +// remove them later if the block fails validation. +bool Blockchain::check_for_double_spend(const transaction& tx, key_images_container& keys_this_block) +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + struct add_transaction_input_visitor: public boost::static_visitor + { + key_images_container& m_spent_keys; + BlockchainDB* m_db; + add_transaction_input_visitor(key_images_container& spent_keys, BlockchainDB* db):m_spent_keys(spent_keys), m_db(db) + {} + bool operator()(const txin_to_key& in) const + { + const crypto::key_image& ki = in.k_image; + + // attempt to insert the newly-spent key into the container of + // keys spent this block. If this fails, the key was spent already + // in this block, return false to flag that a double spend was detected. + // + // if the insert into the block-wide spent keys container succeeds, + // check the blockchain-wide spent keys container and make sure the + // key wasn't used in another block already. + auto r = m_spent_keys.insert(ki); + if(!r.second || m_db->has_key_image(ki)) + { + //double spend detected + return false; + } + + // if no double-spend detected, return true + return true; + } + + bool operator()(const txin_gen& tx) const{return true;} + bool operator()(const txin_to_script& tx) const{return false;} + bool operator()(const txin_to_scripthash& tx) const{return false;} + }; + + for (const txin_v& in : tx.vin) + { + if(!boost::apply_visitor(add_transaction_input_visitor(keys_this_block, m_db), in)) + { + LOG_ERROR("Double spend detected!"); + return false; + } + } + + return true; +} +//------------------------------------------------------------------ +bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector& indexs) +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + if (!m_db->tx_exists(tx_id)) + { + LOG_PRINT_L0("warning: get_tx_outputs_gindexs failed to find transaction with id = " << tx_id); + return false; + } + + indexs = m_db->get_tx_output_indices(tx_id); + return true; +} +//------------------------------------------------------------------ +// This function overloads its sister function with +// an extra value (hash of highest block that holds an output used as input) +// as a return-by-reference. +bool Blockchain::check_tx_inputs(const transaction& tx, uint64_t& max_used_block_height, crypto::hash& max_used_block_id) +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + bool res = check_tx_inputs(tx, &max_used_block_height); + if(!res) return false; + CHECK_AND_ASSERT_MES(max_used_block_height < m_db->height(), false, "internal error: max used block index=" << max_used_block_height << " is not less then blockchain size = " << m_db->height()); + max_used_block_id = m_db->get_block_hash_from_height(max_used_block_height); + return true; +} +//------------------------------------------------------------------ +bool Blockchain::have_tx_keyimges_as_spent(const transaction &tx) +{ + BOOST_FOREACH(const txin_v& in, tx.vin) + { + CHECKED_GET_SPECIFIC_VARIANT(in, const txin_to_key, in_to_key, true); + if(have_tx_keyimg_as_spent(in_to_key.k_image)) + return true; + } + return false; +} +//------------------------------------------------------------------ +// This function validates transaction inputs and their keys. Previously +// it also performed double spend checking, but that has been moved to its +// own function. +bool Blockchain::check_tx_inputs(const transaction& tx, uint64_t* pmax_used_block_height) +{ + size_t sig_index = 0; + if(pmax_used_block_height) + *pmax_used_block_height = 0; + + crypto::hash tx_prefix_hash = get_transaction_prefix_hash(tx); + + for (const auto& txin : tx.vin) + { + // make sure output being spent is of type txin_to_key, rather than + // e.g. txin_gen, which is only used for miner transactions + CHECK_AND_ASSERT_MES(txin.type() == typeid(txin_to_key), false, "wrong type id in tx input at Blockchain::check_tx_inputs"); + const txin_to_key& in_to_key = boost::get(txin); + + // make sure tx output has key offset(s) (is signed to be used) + CHECK_AND_ASSERT_MES(in_to_key.key_offsets.size(), false, "empty in_to_key.key_offsets in transaction with id " << get_transaction_hash(tx)); + + // basically, make sure number of inputs == number of signatures + CHECK_AND_ASSERT_MES(sig_index < tx.signatures.size(), false, "wrong transaction: not signature entry for input with index= " << sig_index); + + // make sure that output being spent matches up correctly with the + // signature spending it. + if(!check_tx_input(in_to_key, tx_prefix_hash, tx.signatures[sig_index], pmax_used_block_height)) + { + LOG_PRINT_L0("Failed to check ring signature for tx " << get_transaction_hash(tx)); + return false; + } + + sig_index++; + } + + return true; +} +//------------------------------------------------------------------ +// This function checks to see if a tx is unlocked. unlock_time is either +// a block index or a unix time. +bool Blockchain::is_tx_spendtime_unlocked(uint64_t unlock_time) +{ + if(unlock_time < CRYPTONOTE_MAX_BLOCK_NUMBER) + { + //interpret as block index + if(get_current_blockchain_height() + CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_BLOCKS >= unlock_time) + return true; + else + return false; + }else + { + //interpret as time + uint64_t current_time = static_cast(time(NULL)); + if(current_time + CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_SECONDS >= unlock_time) + return true; + else + return false; + } + return false; +} +//------------------------------------------------------------------ +// This function locates all outputs associated with a given input (mixins) +// and validates that they exist and are usable. It also checks the ring +// signature for each input. +bool Blockchain::check_tx_input(const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector& sig, uint64_t* pmax_related_block_height) +{ + CRITICAL_REGION_LOCAL(m_blockchain_lock); + + struct outputs_visitor + { + std::vector& m_results_collector; + Blockchain& m_bch; + outputs_visitor(std::vector& results_collector, Blockchain& bch):m_results_collector(results_collector), m_bch(bch) + {} + bool handle_output(const transaction& tx, const tx_out& out) + { + //check tx unlock time + if(!m_bch.is_tx_spendtime_unlocked(tx.unlock_time)) + { + LOG_PRINT_L0("One of outputs for one of inputs have wrong tx.unlock_time = " << tx.unlock_time); + return false; + } + + if(out.target.type() != typeid(txout_to_key)) + { + LOG_PRINT_L0("Output have wrong type id, which=" << out.target.which()); + return false; + } + + m_results_collector.push_back(&boost::get(out.target).key); + return true; + } + }; + + //check ring signature + std::vector output_keys; + outputs_visitor vi(output_keys, *this); + if(!scan_outputkeys_for_indexes(txin, vi, pmax_related_block_height)) + { + LOG_PRINT_L0("Failed to get output keys for tx with amount = " << print_money(txin.amount) << " and count indexes " << txin.key_offsets.size()); + return false; + } + + if(txin.key_offsets.size() != output_keys.size()) + { + LOG_PRINT_L0("Output keys for tx with amount = " << txin.amount << " and count indexes " << txin.key_offsets.size() << " returned wrong keys count " << output_keys.size()); + return false; + } + CHECK_AND_ASSERT_MES(sig.size() == output_keys.size(), false, "internal error: tx signatures count=" << sig.size() << " mismatch with outputs keys count for inputs=" << output_keys.size()); + if(m_is_in_checkpoint_zone) + return true; + return crypto::check_ring_signature(tx_prefix_hash, txin.k_image, output_keys, sig.data()); +} +//------------------------------------------------------------------ +//TODO: Is this intended to do something else? Need to look into the todo there. +uint64_t Blockchain::get_adjusted_time() +{ + //TODO: add collecting median time + return time(NULL); +} +//------------------------------------------------------------------ +bool Blockchain::check_block_timestamp(const std::vector& timestamps, const block& b) +{ + uint64_t median_ts = epee::misc_utils::median(timestamps); + + if(b.timestamp < median_ts) + { + LOG_PRINT_L0("Timestamp of block with id: " << get_block_hash(b) << ", " << b.timestamp << ", less than median of last " << BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW << " blocks, " << median_ts); + return false; + } + + return true; +} +//------------------------------------------------------------------ +// This function grabs the timestamps from the most recent blocks, +// where n = BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW. If there are not those many +// blocks in the blockchain, the timestap is assumed to be valid. If there +// are, this function returns: +// true if the block's timestamp is not less than the timestamp of the +// median of the selected blocks +// false otherwise +bool Blockchain::check_block_timestamp(const block& b) +{ + if(b.timestamp > get_adjusted_time() + CRYPTONOTE_BLOCK_FUTURE_TIME_LIMIT) + { + LOG_PRINT_L0("Timestamp of block with id: " << get_block_hash(b) << ", " << b.timestamp << ", bigger than adjusted time + 2 hours"); + return false; + } + + // if not enough blocks, no proper median yet, return true + if(m_db->height() < BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW + 1) + { + return true; + } + + std::vector timestamps; + auto h = m_db->height(); + + // need most recent 60 blocks, get index of first of those + // using +1 because BlockchainDB::height() returns the index of the top block, + // not the size of the blockchain (0-indexed) + size_t offset = h - BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW + 1; + for(;offset <= h; ++offset) + { + timestamps.push_back(m_db->get_block_timestamp(offset)); + } + + return check_block_timestamp(timestamps, b); +} +//------------------------------------------------------------------ +// Needs to validate the block and acquire each transaction from the +// transaction mem_pool, then pass the block and transactions to +// m_db->add_block() +bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& id, block_verification_context& bvc) +{ + // if we already have the block, return false + if (have_block(id)) + { + LOG_PRINT_L0("Attempting to add block to main chain, but it's already either there or in an alternate chain. hash: " << id); + bvc.m_verification_failed = true; + return false; + } + + TIME_MEASURE_START(block_processing_time); + CRITICAL_REGION_LOCAL(m_blockchain_lock); + if(bl.prev_id != get_tail_id()) + { + LOG_PRINT_L0("Block with id: " << id << std::endl + << "have wrong prev_id: " << bl.prev_id << std::endl + << "expected: " << get_tail_id()); + return false; + } + + // make sure block timestamp is not less than the median timestamp + // of a set number of the most recent blocks. + if(!check_block_timestamp(bl)) + { + LOG_PRINT_L0("Block with id: " << id << std::endl + << "have invalid timestamp: " << bl.timestamp); + bvc.m_verification_failed = true; + return false; + } + + //check proof of work + TIME_MEASURE_START(target_calculating_time); + + // get the target difficulty for the block. + // the calculation can overflow, among other failure cases, + // so we need to check the return type. + // FIXME: get_difficulty_for_next_block can also assert, look into + // changing this to throwing exceptions instead so we can clean up. + difficulty_type current_diffic = get_difficulty_for_next_block(); + CHECK_AND_ASSERT_MES(current_diffic, false, "!!!!!!!!! difficulty overhead !!!!!!!!!"); + + TIME_MEASURE_FINISH(target_calculating_time); + + TIME_MEASURE_START(longhash_calculating_time); + + crypto::hash proof_of_work = null_hash; + + // Formerly the code below contained an if loop with the following condition + // !m_checkpoints.is_in_checkpoint_zone(get_current_blockchain_height()) + // however, this caused the daemon to not bother checking PoW for blocks + // before checkpoints, which is very dangerous behaviour. We moved the PoW + // validation out of the next chunk of code to make sure that we correctly + // check PoW now. + // FIXME: height parameter is not used...should it be used or should it not + // be a parameter? + proof_of_work = get_block_longhash(bl, m_db->height()); + + // validate proof_of_work versus difficulty target + if(!check_hash(proof_of_work, current_diffic)) + { + LOG_PRINT_L0("Block with id: " << id << std::endl + << "have not enough proof of work: " << proof_of_work << std::endl + << "nexpected difficulty: " << current_diffic ); + bvc.m_verification_failed = true; + return false; + } + + // If we're at a checkpoint, ensure that our hardcoded checkpoint hash + // is correct. + if(m_checkpoints.is_in_checkpoint_zone(get_current_blockchain_height())) + { + if(!m_checkpoints.check_block(get_current_blockchain_height(), id)) + { + LOG_ERROR("CHECKPOINT VALIDATION FAILED"); + bvc.m_verification_failed = true; + return false; + } + } + + TIME_MEASURE_FINISH(longhash_calculating_time); + + // sanity check basic miner tx properties + if(!prevalidate_miner_transaction(bl, m_db->height())) + { + LOG_PRINT_L0("Block with id: " << id + << " failed to pass prevalidation"); + bvc.m_verification_failed = true; + return false; + } + + size_t coinbase_blob_size = get_object_blobsize(bl.miner_tx); + size_t cumulative_block_size = coinbase_blob_size; + + std::vector txs; + key_images_container keys; + + // add miner transaction to list of block's transactions. + txs.push_back(bl.miner_tx); + + uint64_t fee_summary = 0; + + // Iterate over the block's transaction hashes, grabbing each + // from the tx_pool and validating them. Each is then added + // to txs. Keys spent in each are added to by the double spend check. + for (const crypto::hash& tx_id : bl.tx_hashes) + { + transaction tx; + size_t blob_size = 0; + uint64_t fee = 0; + + if (m_db->tx_exists(tx_id)) + { + LOG_PRINT_L0("Block with id: " << id << " attempting to add transaction already in blockchain with id: " << tx_id); + bvc.m_verification_failed = true; + break; + } + + // get transaction with hash from tx_pool + if(!m_tx_pool.take_tx(tx_id, tx, blob_size, fee)) + { + LOG_PRINT_L0("Block with id: " << id << "have at least one unknown transaction with id: " << tx_id); + bvc.m_verification_failed = true; + break; + } + + // add the transaction to the temp list of transactions, so we can either + // store the list of transactions all at once or return the ones we've + // taken from the tx_pool back to it if the block fails verification. + txs.push_back(tx); + + // validate that transaction inputs and the keys spending them are correct. + if(!check_tx_inputs(tx)) + { + LOG_PRINT_L0("Block with id: " << id << "have at least one transaction (id: " << tx_id << ") with wrong inputs."); + + //TODO: why is this done? make sure that keeping invalid blocks makes sense. + add_block_as_invalid(bl, id); + LOG_PRINT_L0("Block with id " << id << " added as invalid becouse of wrong inputs in transactions"); + bvc.m_verification_failed = true; + break; + } + + if (!check_for_double_spend(tx, keys)) + { + LOG_PRINT_L0("Double spend detected in transaction (id: " << tx_id); + bvc.m_verification_failed = true; + break; + } + + fee_summary += fee; + cumulative_block_size += blob_size; + } + + uint64_t base_reward = 0; + uint64_t already_generated_coins = m_db->height() ? m_db->get_block_already_generated_coins(m_db->height()) : 0; + if(!validate_miner_transaction(bl, cumulative_block_size, fee_summary, base_reward, already_generated_coins)) + { + LOG_PRINT_L0("Block with id: " << id + << " have wrong miner transaction"); + bvc.m_verification_failed = true; + } + + + block_extended_info bei = boost::value_initialized(); + size_t block_size; + difficulty_type cumulative_difficulty; + + // populate various metadata about the block to be stored alongside it. + block_size = cumulative_block_size; + cumulative_difficulty = current_diffic; + already_generated_coins = already_generated_coins + base_reward; + if(m_db->height()) + cumulative_difficulty += m_db->get_block_cumulative_difficulty(m_db->height()); + + update_next_cumulative_size_limit(); + + TIME_MEASURE_FINISH(block_processing_time); + + uint64_t new_height = 0; + bool add_success = true; + try + { + new_height = m_db->add_block(bl, block_size, cumulative_difficulty, already_generated_coins, txs); + } + catch (const std::exception& e) + { + //TODO: figure out the best way to deal with this failure + LOG_ERROR("Error adding block with hash: " << id << " to blockchain, what = " << e.what()); + add_success = false; + } + + // if we failed for any reason to verify the block, return taken + // transactions to the tx_pool. + if (bvc.m_verification_failed || !add_success) + { + // return taken transactions to transaction pool + for (auto& tx : txs) + { + cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc); + if (!m_tx_pool.add_tx(tx, tvc, true)) + { + LOG_PRINT_L0("Failed to return taken transaction with hash: " << get_transaction_hash(tx) << " to tx_pool"); + } + } + return false; + } + + LOG_PRINT_L1("+++++ BLOCK SUCCESSFULLY ADDED" << std::endl << "id:\t" << id + << std::endl << "PoW:\t" << proof_of_work + << std::endl << "HEIGHT " << new_height << ", difficulty:\t" << current_diffic + << std::endl << "block reward: " << print_money(fee_summary + base_reward) << "(" << print_money(base_reward) << " + " << print_money(fee_summary) + << "), coinbase_blob_size: " << coinbase_blob_size << ", cumulative size: " << cumulative_block_size + << ", " << block_processing_time << "("<< target_calculating_time << "/" << longhash_calculating_time << ")ms"); + + bvc.m_added_to_main_chain = true; + + // appears to be a NOP *and* is called elsewhere. wat? + m_tx_pool.on_blockchain_inc(new_height, id); + + return true; +} +//------------------------------------------------------------------ +bool Blockchain::update_next_cumulative_size_limit() +{ + std::vector sz; + get_last_n_blocks_sizes(sz, CRYPTONOTE_REWARD_BLOCKS_WINDOW); + + uint64_t median = epee::misc_utils::median(sz); + if(median <= CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE) + median = CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE; + + m_current_block_cumul_sz_limit = median*2; + return true; +} +//------------------------------------------------------------------ +bool Blockchain::add_new_block(const block& bl_, block_verification_context& bvc) +{ + //copy block here to let modify block.target + block bl = bl_; + crypto::hash id = get_block_hash(bl); + CRITICAL_REGION_LOCAL(m_tx_pool);//to avoid deadlock lets lock tx_pool for whole add/reorganize process + CRITICAL_REGION_LOCAL1(m_blockchain_lock); + if(have_block(id)) + { + LOG_PRINT_L3("block with id = " << id << " already exists"); + bvc.m_already_exists = true; + return false; + } + + //check that block refers to chain tail + if(!(bl.prev_id == get_tail_id())) + { + //chain switching or wrong block + bvc.m_added_to_main_chain = false; + return handle_alternative_block(bl, id, bvc); + //never relay alternative blocks + } + + return handle_block_to_main_chain(bl, id, bvc); +} diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h new file mode 100644 index 000000000..27e5cde8c --- /dev/null +++ b/src/cryptonote_core/blockchain.h @@ -0,0 +1,221 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "syncobj.h" +#include "string_tools.h" +#include "tx_pool.h" +#include "cryptonote_basic.h" +#include "common/util.h" +#include "cryptonote_protocol/cryptonote_protocol_defs.h" +#include "rpc/core_rpc_server_commands_defs.h" +#include "difficulty.h" +#include "cryptonote_core/cryptonote_format_utils.h" +#include "verification_context.h" +#include "crypto/hash.h" +#include "checkpoints.h" +#include "blockchain_db.h" + +namespace cryptonote +{ + + /************************************************************************/ + /* */ + /************************************************************************/ + class Blockchain + { + public: + struct transaction_chain_entry + { + transaction tx; + uint64_t m_keeper_block_height; + size_t m_blob_size; + std::vector m_global_output_indexes; + }; + + struct block_extended_info + { + block bl; + uint64_t height; + size_t block_cumulative_size; + difficulty_type cumulative_difficulty; + uint64_t already_generated_coins; + }; + + Blockchain(tx_memory_pool& tx_pool); + + bool init() { return init(tools::get_default_data_dir()); } + bool init(const std::string& config_folder); + bool deinit(); + + void set_checkpoints(checkpoints&& chk_pts) { m_checkpoints = chk_pts; } + + //bool push_new_block(); + bool get_blocks(uint64_t start_offset, size_t count, std::list& blocks, std::list& txs); + bool get_blocks(uint64_t start_offset, size_t count, std::list& blocks); + bool get_alternative_blocks(std::list& blocks); + size_t get_alternative_blocks_count(); + crypto::hash get_block_id_by_height(uint64_t height); + bool get_block_by_hash(const crypto::hash &h, block &blk); + void get_all_known_block_ids(std::list &main, std::list &alt, std::list &invalid); + + template + void serialize(archive_t & ar, const unsigned int version); + + bool have_tx(const crypto::hash &id); + bool have_tx_keyimges_as_spent(const transaction &tx); + bool have_tx_keyimg_as_spent(const crypto::key_image &key_im); + + template + bool scan_outputkeys_for_indexes(const txin_to_key& tx_in_to_key, visitor_t& vis, uint64_t* pmax_related_block_height = NULL); + + uint64_t get_current_blockchain_height(); + crypto::hash get_tail_id(); + crypto::hash get_tail_id(uint64_t& height); + difficulty_type get_difficulty_for_next_block(); + bool add_new_block(const block& bl_, block_verification_context& bvc); + bool reset_and_set_genesis_block(const block& b); + bool create_block_template(block& b, const account_public_address& miner_address, difficulty_type& di, uint64_t& height, const blobdata& ex_nonce); + bool have_block(const crypto::hash& id); + size_t get_total_transactions(); + bool get_short_chain_history(std::list& ids); + bool find_blockchain_supplement(const std::list& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp); + bool find_blockchain_supplement(const std::list& qblock_ids, uint64_t& starter_offset); + bool find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count); + bool handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NOTIFY_RESPONSE_GET_OBJECTS::request& rsp); + bool handle_get_objects(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res); + bool get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res); + bool get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector& indexs); + bool store_blockchain(); + bool check_tx_input(const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector& sig, uint64_t* pmax_related_block_height = NULL); + bool check_tx_inputs(const transaction& tx, uint64_t* pmax_used_block_height = NULL); + bool check_tx_inputs(const transaction& tx, uint64_t& pmax_used_block_height, crypto::hash& max_used_block_id); + uint64_t get_current_cumulative_blocksize_limit(); + bool is_storing_blockchain(){return m_is_blockchain_storing;} + uint64_t block_difficulty(uint64_t i); + + template + void get_blocks(const t_ids_container& block_ids, t_blocks_container& blocks, t_missed_container& missed_bs); + + template + void get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs); + + //debug functions + void print_blockchain(uint64_t start_index, uint64_t end_index); + void print_blockchain_index(); + void print_blockchain_outs(const std::string& file); + + private: + typedef std::unordered_map blocks_by_id_index; + typedef std::unordered_map transactions_container; + typedef std::unordered_set key_images_container; + typedef std::vector blocks_container; + typedef std::unordered_map blocks_ext_by_hash; + typedef std::unordered_map blocks_by_hash; + typedef std::map>> outputs_container; //crypto::hash - tx hash, size_t - index of out in transaction + + BlockchainDB* m_db; + + tx_memory_pool& m_tx_pool; + epee::critical_section m_blockchain_lock; // TODO: add here reader/writer lock + + // main chain + blocks_container m_blocks; // height -> block_extended_info + blocks_by_id_index m_blocks_index; // crypto::hash -> height + transactions_container m_transactions; + key_images_container m_spent_keys; + size_t m_current_block_cumul_sz_limit; + + + // all alternative chains + blocks_ext_by_hash m_alternative_chains; // crypto::hash -> block_extended_info + + // some invalid blocks + blocks_ext_by_hash m_invalid_blocks; // crypto::hash -> block_extended_info + outputs_container m_outputs; + + + std::string m_config_folder; + checkpoints m_checkpoints; + std::atomic m_is_in_checkpoint_zone; + std::atomic m_is_blockchain_storing; + + bool switch_to_alternative_blockchain(std::list& alt_chain, bool discard_disconnected_chain); + block pop_block_from_blockchain(); + bool purge_transaction_from_blockchain(const crypto::hash& tx_id); + bool purge_transaction_keyimages_from_blockchain(const transaction& tx, bool strict_check); + + bool handle_block_to_main_chain(const block& bl, block_verification_context& bvc); + bool handle_block_to_main_chain(const block& bl, const crypto::hash& id, block_verification_context& bvc); + bool handle_alternative_block(const block& b, const crypto::hash& id, block_verification_context& bvc); + difficulty_type get_next_difficulty_for_alternative_chain(const std::list& alt_chain, block_extended_info& bei); + bool prevalidate_miner_transaction(const block& b, uint64_t height); + bool validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins); + bool validate_transaction(const block& b, uint64_t height, const transaction& tx); + bool rollback_blockchain_switching(std::list& original_chain); + bool add_transaction_from_block(const transaction& tx, const crypto::hash& tx_id, const crypto::hash& bl_id, uint64_t bl_height); + bool push_transaction_to_global_outs_index(const transaction& tx, const crypto::hash& tx_id, std::vector& global_indexes); + bool pop_transaction_from_global_index(const transaction& tx, const crypto::hash& tx_id); + void get_last_n_blocks_sizes(std::vector& sz, size_t count); + void add_out_to_get_random_outs(COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i); + bool is_tx_spendtime_unlocked(uint64_t unlock_time); + bool add_block_as_invalid(const block& bl, const crypto::hash& h); + bool add_block_as_invalid(const block_extended_info& bei, const crypto::hash& h); + bool check_block_timestamp(const block& b); + bool check_block_timestamp(const std::vector& timestamps, const block& b); + uint64_t get_adjusted_time(); + bool complete_timestamps_vector(uint64_t start_height, std::vector& timestamps); + bool update_next_cumulative_size_limit(); + + bool check_for_double_spend(const transaction& tx, key_images_container& keys_this_block); + }; + + + /************************************************************************/ + /* */ + /************************************************************************/ + + #define CURRENT_BLOCKCHAIN_STORAGE_ARCHIVE_VER 12 + + //------------------------------------------------------------------ + +} // namespace cryptonote + +BOOST_CLASS_VERSION(cryptonote::Blockchain, CURRENT_BLOCKCHAIN_STORAGE_ARCHIVE_VER) diff --git a/src/cryptonote_core/blockchain_db.cpp b/src/cryptonote_core/blockchain_db.cpp new file mode 100644 index 000000000..606b34310 --- /dev/null +++ b/src/cryptonote_core/blockchain_db.cpp @@ -0,0 +1,135 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "cryptonote_core/blockchain_db.h" +#include "cryptonote_format_utils.h" + +namespace cryptonote +{ + +void BlockchainDB::pop_block() +{ + block blk; + std::vector txs; + pop_block(blk, txs); +} + +void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transaction& tx) +{ + crypto::hash tx_hash = get_transaction_hash(tx); + + add_transaction_data(blk_hash, tx); + + // iterate tx.vout using indices instead of C++11 foreach syntax because + // we need the index + for (uint64_t i = 0; i < tx.vout.size(); ++i) + { + add_output(tx_hash, tx.vout[i], i); + } + + for (const txin_v& tx_input : tx.vin) + { + if (tx_input.type() == typeid(txin_to_key)) + { + add_spent_key(boost::get(tx_input).k_image); + } + } +} + +uint64_t BlockchainDB::add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const std::vector& txs + ) +{ + try + { + // call out to subclass implementation to add the block & metadata + add_block(blk, block_size, cumulative_difficulty, coins_generated); + + crypto::hash blk_hash = get_block_hash(blk); + // call out to add the transactions + for (const transaction& tx : txs) + { + add_transaction(blk_hash, tx); + } + } + // in case any of the add_block process goes awry, undo + catch (const std::exception& e) + { + LOG_ERROR("Error adding block to db: " << e.what()); + try + { + pop_block(); + } + // if undoing goes wrong as well, we need to throw, as blockchain + // will be in a bad state + catch (const std::exception& e) + { + LOG_ERROR("Error undoing partially added block: " << e.what()); + throw; + } + throw; + } + + return height(); +} + +void BlockchainDB::pop_block(block& blk, std::vector& txs) +{ + blk = get_top_block(); + + for (const auto& h : blk.tx_hashes) + { + txs.push_back(get_tx(h)); + remove_transaction(h); + } +} + +void BlockchainDB::remove_transaction(const crypto::hash& tx_hash) +{ + transaction tx = get_tx(tx_hash); + + for (const tx_out& tx_output : tx.vout) + { + remove_output(tx_output); + } + + for (const txin_v& tx_input : tx.vin) + { + if (tx_input.type() == typeid(txin_to_key)) + { + remove_spent_key(boost::get(tx_input).k_image); + } + } + + remove_transaction_data(tx_hash); +} + +} // namespace cryptonote diff --git a/src/cryptonote_core/blockchain_db.h b/src/cryptonote_core/blockchain_db.h new file mode 100644 index 000000000..5728f7065 --- /dev/null +++ b/src/cryptonote_core/blockchain_db.h @@ -0,0 +1,510 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include +#include +#include "crypto/hash.h" +#include "cryptonote_core/cryptonote_basic.h" +#include "cryptonote_core/difficulty.h" + +/* DB Driver Interface + * + * The DB interface is a store for the canonical block chain. + * It serves as a persistent storage for the blockchain. + * + * For the sake of efficiency, the reference implementation will also + * store some blockchain data outside of the blocks, such as spent + * transfer key images, unspent transaction outputs, etc. + * + * Transactions are duplicated so that we don't have to fetch a whole block + * in order to fetch a transaction from that block. If this is deemed + * unnecessary later, this can change. + * + * Spent key images are duplicated outside of the blocks so it is quick + * to verify an output hasn't already been spent + * + * Unspent transaction outputs are duplicated to quickly gather random + * outputs to use for mixins + * + * IMPORTANT: + * A concrete implementation of this interface should populate these + * duplicated members! It is possible to have a partial implementation + * of this interface call to private members of the interface to be added + * later that will then populate as needed. + * + * General: + * open() + * close() + * sync() + * reset() + * + * Lock and unlock provided for reorg externally, and for block + * additions internally, this way threaded reads are completely fine + * unless the blockchain is changing. + * bool lock() + * unlock() + * + * Blocks: + * bool block_exists(hash) + * height add_block(block, block_size, cumulative_difficulty, coins_generated, transactions) + * block get_block(hash) + * height get_block_height(hash) + * header get_block_header(hash) + * block get_block_from_height(height) + * size_t get_block_size(height) + * difficulty get_block_cumulative_difficulty(height) + * uint64_t get_block_already_generated_coins(height) + * uint64_t get_block_timestamp(height) + * uint64_t get_top_block_timestamp() + * hash get_block_hash_from_height(height) + * blocks get_blocks_range(height1, height2) + * hashes get_hashes_range(height1, height2) + * hash top_block_hash() + * block get_top_block() + * height height() + * void pop_block(block&, tx_list&) + * + * Transactions: + * bool tx_exists(hash) + * uint64_t get_tx_unlock_time(hash) + * tx get_tx(hash) + * uint64_t get_tx_count() + * tx_list get_tx_list(hash_list) + * height get_tx_block_height(hash) + * + * Outputs: + * index get_random_output(amount) + * uint64_t get_num_outputs(amount) + * pub_key get_output_key(amount, index) + * tx_out get_output(tx_hash, index) + * hash,index get_output_tx_and_index(amount, index) + * vec get_tx_output_indices(tx_hash) + * + * + * Spent Output Key Images: + * bool has_key_image(key_image) + * + * Exceptions: + * DB_ERROR -- generic + * DB_OPEN_FAILURE + * DB_CREATE_FAILURE + * DB_SYNC_FAILURE + * BLOCK_DNE + * BLOCK_PARENT_DNE + * BLOCK_EXISTS + * BLOCK_INVALID -- considering making this multiple errors + * TX_DNE + * TX_EXISTS + * OUTPUT_DNE + */ + +namespace cryptonote +{ + +/*********************************** + * Exception Definitions + ***********************************/ +class DB_ERROR : public std::exception +{ + private: + std::string m; + public: + DB_ERROR() : m("Generic DB Error") { } + DB_ERROR(const char* s) : m(s) { } + + virtual ~DB_ERROR() { } + + const char* what() const throw() + { + return m.c_str(); + } +}; + +class DB_OPEN_FAILURE : public std::exception +{ + private: + std::string m; + public: + DB_OPEN_FAILURE() : m("Failed to open the db") { } + DB_OPEN_FAILURE(const char* s) : m(s) { } + + virtual ~DB_OPEN_FAILURE() { } + + const char* what() const throw() + { + return m.c_str(); + } +}; + +class DB_CREATE_FAILURE : public std::exception +{ + private: + std::string m; + public: + DB_CREATE_FAILURE() : m("Failed to create the db") { } + DB_CREATE_FAILURE(const char* s) : m(s) { } + + virtual ~DB_CREATE_FAILURE() { } + + const char* what() const throw() + { + return m.c_str(); + } +}; + +class DB_SYNC_FAILURE : public std::exception +{ + private: + std::string m; + public: + DB_SYNC_FAILURE() : m("Failed to sync the db") { } + DB_SYNC_FAILURE(const char* s) : m(s) { } + + virtual ~DB_SYNC_FAILURE() { } + + const char* what() const throw() + { + return m.c_str(); + } +}; + +class BLOCK_DNE : public std::exception +{ + private: + std::string m; + public: + BLOCK_DNE() : m("The block requested does not exist") { } + BLOCK_DNE(const char* s) : m(s) { } + + virtual ~BLOCK_DNE() { } + + const char* what() const throw() + { + return m.c_str(); + } +}; + +class BLOCK_PARENT_DNE : public std::exception +{ + private: + std::string m; + public: + BLOCK_PARENT_DNE() : m("The parent of the block does not exist") { } + BLOCK_PARENT_DNE(const char* s) : m(s) { } + + virtual ~BLOCK_PARENT_DNE() { } + + const char* what() const throw() + { + return m.c_str(); + } +}; + +class BLOCK_EXISTS : public std::exception +{ + private: + std::string m; + public: + BLOCK_EXISTS() : m("The block to be added already exists!") { } + BLOCK_EXISTS(const char* s) : m(s) { } + + virtual ~BLOCK_EXISTS() { } + + const char* what() const throw() + { + return m.c_str(); + } +}; + +class BLOCK_INVALID : public std::exception +{ + private: + std::string m; + public: + BLOCK_INVALID() : m("The block to be added did not pass validation!") { } + BLOCK_INVALID(const char* s) : m(s) { } + + virtual ~BLOCK_INVALID() { } + + const char* what() const throw() + { + return m.c_str(); + } +}; + +class TX_DNE : public std::exception +{ + private: + std::string m; + public: + TX_DNE() : m("The transaction requested does not exist") { } + TX_DNE(const char* s) : m(s) { } + + virtual ~TX_DNE() { } + + const char* what() const throw() + { + return m.c_str(); + } +}; + +class TX_EXISTS : public std::exception +{ + private: + std::string m; + public: + TX_EXISTS() : m("The transaction to be added already exists!") { } + TX_EXISTS(const char* s) : m(s) { } + + virtual ~TX_EXISTS() { } + + const char* what() const throw() + { + return m.c_str(); + } +}; + +class OUTPUT_DNE : public std::exception +{ + private: + std::string m; + public: + OUTPUT_DNE() : m("The transaction requested does not exist") { } + OUTPUT_DNE(const char* s) : m(s) { } + + virtual ~OUTPUT_DNE() { } + + const char* what() const throw() + { + return m.c_str(); + } +}; + +/*********************************** + * End of Exception Definitions + ***********************************/ + + +class BlockchainDB +{ +private: + /********************************************************************* + * private virtual members + *********************************************************************/ + + // tells the subclass to add the block and metadata to storage + virtual void add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + ) = 0; + + // tells the subclass to remove data about a block + virtual void remove_block(const crypto::hash& blk_hash) = 0; + + // tells the subclass to store the transaction and its metadata + virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx) = 0; + + // tells the subclass to remove data about a transaction + virtual void remove_transaction_data(const crypto::hash& tx_hash) = 0; + + // tells the subclass to store an output + virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index) = 0; + + // tells the subclass to remove an output + virtual void remove_output(const tx_out& tx_output) = 0; + + // tells the subclass to store a spent key + virtual void add_spent_key(const crypto::key_image& k_image) = 0; + + // tells the subclass to remove a spent key + virtual void remove_spent_key(const crypto::key_image& k_image) = 0; + + + /********************************************************************* + * private concrete members + *********************************************************************/ + // private version of pop_block, for undoing if an add_block goes tits up + void pop_block(); + + // helper function for add_transactions, to add each individual tx + void add_transaction(const crypto::hash& blk_hash, const transaction& tx); + + // helper function to remove transaction from blockchain + void remove_transaction(const crypto::hash& tx_hash); + + +public: + // open the db at location , or create it if there isn't one. + virtual void open(const std::string& filename) = 0; + + // make sure implementation has a create function as well + virtual void create(const std::string& filename) = 0; + + // close and sync the db + virtual void close() = 0; + + // sync the db + virtual void sync() = 0; + + // reset the db -- USE WITH CARE + virtual void reset() = 0; + + + // FIXME: these are just for functionality mocking, need to implement + // RAII-friendly and multi-read one-write friendly locking mechanism + // + // acquire db lock + virtual bool lock() = 0; + + // release db lock + virtual void unlock() = 0; + + + // adds a block with the given metadata to the top of the blockchain, returns the new height + uint64_t add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const std::vector& txs + ); + + // return true if a block with hash exists in the blockchain + virtual bool block_exists(const crypto::hash& h) = 0; + + // return block with hash + virtual block get_block(const crypto::hash& h) = 0; + + // return the height of the block with hash on the blockchain, + // throw if it doesn't exist + virtual uint64_t get_block_height(const crypto::hash& h) = 0; + + // return header for block with hash + virtual block_header get_block_header(const crypto::hash& h) = 0; + + // return block at height + virtual block get_block_from_height(const uint64_t& height) = 0; + + // return timestamp of block at height + virtual uint64_t get_block_timestamp(const uint64_t& height) = 0; + + // return timestamp of most recent block + virtual uint64_t get_top_block_timestamp() = 0; + + // return block size of block at height + virtual size_t get_block_size(const uint64_t& height) = 0; + + // return cumulative difficulty up to and including block at height + virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height) = 0; + + // return difficulty of block at height + virtual difficulty_type get_block_difficulty(const uint64_t& height) = 0; + + // return number of coins generated up to and including block at height + virtual uint64_t get_block_already_generated_coins(const uint64_t& height) = 0; + + // return hash of block at height + virtual crypto::hash get_block_hash_from_height(const uint64_t& height) = 0; + + // return list of blocks in range of height. + virtual std::list get_blocks_range(const uint64_t& h1, const uint64_t& h2) = 0; + + // return list of block hashes in range of height + virtual std::list get_hashes_range(const uint64_t& h1, const uint64_t& h2) = 0; + + // return the hash of the top block on the chain + virtual crypto::hash top_block_hash() = 0; + + // return the block at the top of the blockchain + virtual block get_top_block() = 0; + + // return the index of the top block on the chain + // NOTE: for convenience using heights as indices, this is not the total + // size of the blockchain, but rather the index of the top block. As the + // chain is 0-indexed, the total size will be height() + 1. + virtual uint64_t height() = 0; + + // pops the top block off the blockchain. + // Returns by reference the popped block and its associated transactions + // + // IMPORTANT: + // When a block is popped, the transactions associated with it need to be + // removed, as well as outputs and spent key images associated with + // those transactions. + void pop_block(block& blk, std::vector& txs); + + + // return true if a transaction with hash exists + virtual bool tx_exists(const crypto::hash& h) = 0; + + // return unlock time of tx with hash + virtual uint64_t get_tx_unlock_time(const crypto::hash& h) = 0; + + // return tx with hash + // throw if no such tx exists + virtual transaction get_tx(const crypto::hash& h) = 0; + + // returns the total number of transactions in all blocks + virtual uint64_t get_tx_count() = 0; + + // return list of tx with hashes . + // TODO: decide if a missing hash means return empty list + // or just skip that hash + virtual std::list get_tx_list(const std::vector& hlist) = 0; + + // returns height of block that contains transaction with hash + virtual uint64_t get_tx_block_height(const crypto::hash& h) = 0; + + // return global output index of a random output of amount + virtual uint64_t get_random_output(const uint64_t& amount) = 0; + + // returns the total number of outputs of amount + virtual uint64_t get_num_outputs(const uint64_t& amount) = 0; + + // return public key for output with global output amount and index + virtual crypto::public_key get_output_key(const uint64_t& amount, const uint64_t& index) = 0; + + // returns the output indexed by in the transaction with hash + virtual tx_out get_output(const crypto::hash& h, const uint64_t& index) = 0; + + // returns the transaction-local reference for the output with at + // return type is pair of tx hash and index + virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) = 0; + + // return a vector of indices corresponding to the global output index for + // each output in the transaction with hash + virtual std::vector get_tx_output_indices(const crypto::hash& h) = 0; + + // returns true if key image is present in spent key images storage + virtual bool has_key_image(const crypto::key_image& img) = 0; + +}; // class BlockchainDB + + +} // namespace cryptonote -- cgit v1.2.3 From 67515b8b19b7a9ea4f1666519a624f5efa44cdf3 Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Mon, 6 Oct 2014 19:54:46 -0400 Subject: missing typedef --- src/cryptonote_core/blockchain_db.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src') diff --git a/src/cryptonote_core/blockchain_db.h b/src/cryptonote_core/blockchain_db.h index 5728f7065..5359909f7 100644 --- a/src/cryptonote_core/blockchain_db.h +++ b/src/cryptonote_core/blockchain_db.h @@ -127,6 +127,9 @@ namespace cryptonote { +// typedef for convenience +typedef std::pair tx_out_index; + /*********************************** * Exception Definitions ***********************************/ -- cgit v1.2.3 From 1ffbeb2d2ef07eeba5a0380f73ca87de3ee0c0c9 Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Mon, 6 Oct 2014 19:56:31 -0400 Subject: stupid past me, fixing typos and shit... --- src/cryptonote_core/blockchain.cpp | 42 +++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 21 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 22ae7a39a..579574cd0 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -255,7 +255,7 @@ bool Blockchain::init(const std::string& config_folder) block_verification_context bvc = boost::value_initialized(); generate_genesis_block(bl); add_new_block(bl, bvc); - CHECK_AND_ASSERT_MES(!bvc.m_verification_failed, false, "Failed to add genesis block to blockchain"); + CHECK_AND_ASSERT_MES(!bvc.m_verifivation_failed, false, "Failed to add genesis block to blockchain"); } // check how far behind we are @@ -374,7 +374,7 @@ bool Blockchain::reset_and_set_genesis_block(const block& b) block_verification_context bvc = boost::value_initialized(); add_new_block(b, bvc); - return bvc.m_added_to_main_chain && !bvc.m_verification_failed; + return bvc.m_added_to_main_chain && !bvc.m_verifivation_failed; } //------------------------------------------------------------------ //TODO: move to BlockchainDB subclass @@ -999,7 +999,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id if(0 == block_height) { LOG_ERROR("Block with id: " << epee::string_tools::pod_to_hex(id) << " (as alternative) have wrong miner transaction"); - bvc.m_verification_failed = true; + bvc.m_verifivation_failed = true; return false; } // TODO: this basically says if the blockchain is smaller than the first @@ -1010,7 +1010,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id LOG_PRINT_RED_L0("Block with id: " << id << std::endl << " can't be accepted for alternative chain, block height: " << block_height << std::endl << " blockchain height: " << get_current_blockchain_height()); - bvc.m_verification_failed = true; + bvc.m_verifivation_failed = true; return false; } @@ -1069,7 +1069,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id { LOG_PRINT_RED_L0("Block with id: " << id << std::endl << " for alternative chain, have invalid timestamp: " << b.timestamp); - bvc.m_verification_failed = true; + bvc.m_verifivation_failed = true; return false; } @@ -1082,7 +1082,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id if(!m_checkpoints.check_block(bei.height, id, is_a_checkpoint)) { LOG_ERROR("CHECKPOINT VALIDATION FAILED"); - bvc.m_verification_failed = true; + bvc.m_verifivation_failed = true; return false; } @@ -1097,7 +1097,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id LOG_PRINT_RED_L0("Block with id: " << id << std::endl << " for alternative chain, have not enough proof of work: " << proof_of_work << std::endl << " expected difficulty: " << current_diff); - bvc.m_verification_failed = true; + bvc.m_verifivation_failed = true; return false; } @@ -1105,7 +1105,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id { LOG_PRINT_RED_L0("Block with id: " << epee::string_tools::pod_to_hex(id) << " (as alternative) have wrong miner transaction."); - bvc.m_verification_failed = true; + bvc.m_verifivation_failed = true; return false; } @@ -1132,7 +1132,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id bool r = switch_to_alternative_blockchain(alt_chain, true); bvc.m_added_to_main_chain = r; - bvc.m_verification_failed = !r; + bvc.m_verifivation_failed = !r; return r; } @@ -1148,7 +1148,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id bool r = switch_to_alternative_blockchain(alt_chain, false); if(r) bvc.m_added_to_main_chain = true; - else bvc.m_verification_failed = true; + else bvc.m_verifivation_failed = true; return r; } else @@ -1881,7 +1881,7 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& if (have_block(id)) { LOG_PRINT_L0("Attempting to add block to main chain, but it's already either there or in an alternate chain. hash: " << id); - bvc.m_verification_failed = true; + bvc.m_verifivation_failed = true; return false; } @@ -1901,7 +1901,7 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& { LOG_PRINT_L0("Block with id: " << id << std::endl << "have invalid timestamp: " << bl.timestamp); - bvc.m_verification_failed = true; + bvc.m_verifivation_failed = true; return false; } @@ -1938,7 +1938,7 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& LOG_PRINT_L0("Block with id: " << id << std::endl << "have not enough proof of work: " << proof_of_work << std::endl << "nexpected difficulty: " << current_diffic ); - bvc.m_verification_failed = true; + bvc.m_verifivation_failed = true; return false; } @@ -1949,7 +1949,7 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& if(!m_checkpoints.check_block(get_current_blockchain_height(), id)) { LOG_ERROR("CHECKPOINT VALIDATION FAILED"); - bvc.m_verification_failed = true; + bvc.m_verifivation_failed = true; return false; } } @@ -1961,7 +1961,7 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& { LOG_PRINT_L0("Block with id: " << id << " failed to pass prevalidation"); - bvc.m_verification_failed = true; + bvc.m_verifivation_failed = true; return false; } @@ -1988,7 +1988,7 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& if (m_db->tx_exists(tx_id)) { LOG_PRINT_L0("Block with id: " << id << " attempting to add transaction already in blockchain with id: " << tx_id); - bvc.m_verification_failed = true; + bvc.m_verifivation_failed = true; break; } @@ -1996,7 +1996,7 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& if(!m_tx_pool.take_tx(tx_id, tx, blob_size, fee)) { LOG_PRINT_L0("Block with id: " << id << "have at least one unknown transaction with id: " << tx_id); - bvc.m_verification_failed = true; + bvc.m_verifivation_failed = true; break; } @@ -2013,14 +2013,14 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& //TODO: why is this done? make sure that keeping invalid blocks makes sense. add_block_as_invalid(bl, id); LOG_PRINT_L0("Block with id " << id << " added as invalid becouse of wrong inputs in transactions"); - bvc.m_verification_failed = true; + bvc.m_verifivation_failed = true; break; } if (!check_for_double_spend(tx, keys)) { LOG_PRINT_L0("Double spend detected in transaction (id: " << tx_id); - bvc.m_verification_failed = true; + bvc.m_verifivation_failed = true; break; } @@ -2034,7 +2034,7 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& { LOG_PRINT_L0("Block with id: " << id << " have wrong miner transaction"); - bvc.m_verification_failed = true; + bvc.m_verifivation_failed = true; } @@ -2068,7 +2068,7 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& // if we failed for any reason to verify the block, return taken // transactions to the tx_pool. - if (bvc.m_verification_failed || !add_success) + if (bvc.m_verifivation_failed || !add_success) { // return taken transactions to transaction pool for (auto& tx : txs) -- cgit v1.2.3 From 07733f98c02490bf972ae5f5f7ba25a49a846dec Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Mon, 13 Oct 2014 00:31:21 -0400 Subject: update new blockchain to build with new changes Still need to add in the new checkpointing functionality, as well as touch up a few things, but is okay for now. --- src/cryptonote_core/blockchain.cpp | 40 +++++++++++++++++++++++++++++--------- src/cryptonote_core/blockchain.h | 7 +++---- 2 files changed, 34 insertions(+), 13 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 579574cd0..828ad96e4 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -218,7 +218,7 @@ uint64_t Blockchain::get_current_blockchain_height() //------------------------------------------------------------------ //FIXME: possibly move this into the constructor, to avoid accidentally // dereferencing a null BlockchainDB pointer -bool Blockchain::init(const std::string& config_folder) +bool Blockchain::init(const std::string& config_folder, bool testnet) { CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -248,15 +248,29 @@ bool Blockchain::init(const std::string& config_folder) // if the blockchain is new, add the genesis block // this feels kinda kludgy to do it this way, but can be looked at later. + // TODO: add function to create and store genesis block, + // taking testnet into account if(!m_db->height()) { LOG_PRINT_L0("Blockchain not loaded, generating genesis block."); block bl = boost::value_initialized(); block_verification_context bvc = boost::value_initialized(); - generate_genesis_block(bl); + if (testnet) + { + generate_genesis_block(bl, config::testnet::GENESIS_TX, config::testnet::GENESIS_NONCE); + } + else + { + generate_genesis_block(bl, config::GENESIS_TX, config::GENESIS_NONCE); + } add_new_block(bl, bvc); CHECK_AND_ASSERT_MES(!bvc.m_verifivation_failed, false, "Failed to add genesis block to blockchain"); } + // TODO: if blockchain load successful, verify blockchain against both + // hard-coded and runtime-loaded (and enforced) checkpoints. + else + { + } // check how far behind we are uint64_t top_block_timestamp = m_db->get_top_block_timestamp(); @@ -576,14 +590,12 @@ difficulty_type Blockchain::get_difficulty_for_next_block() // This function removes blocks from the blockchain until it gets to the // position where the blockchain switch started and then re-adds the blocks // that had been removed. -bool Blockchain::rollback_blockchain_switching(std::list& original_chain) +bool Blockchain::rollback_blockchain_switching(std::list& original_chain, uint64_t rollback_height) { CRITICAL_REGION_LOCAL(m_blockchain_lock); - // remove blocks from blockchain until we get back to where we were. - // Conveniently, that's until our top block's hash == the parent block of - // the first block we need to add back. - while (m_db->top_block_hash() != original_chain.front().prev_id) + // remove blocks from blockchain until we get back to where we should be. + while (m_db->height() != rollback_height) { pop_block_from_blockchain(); } @@ -596,6 +608,11 @@ bool Blockchain::rollback_blockchain_switching(std::list& original_chain) CHECK_AND_ASSERT_MES(r && bvc.m_added_to_main_chain, false, "PANIC!!! failed to add (again) block while chain switching during the rollback!"); } + LOG_PRINT_L1("Rollback to height " << rollback_height << " was successful."); + if (original_chain.size()) + { + LOG_PRINT_L1("Restoration to previous blockchain successful as well."); + } return true; } //------------------------------------------------------------------ @@ -640,7 +657,11 @@ bool Blockchain::switch_to_alternative_blockchain(std::listheight()); // FIXME: Why do we keep invalid blocks around? Possibly in case we hear // about them again so we can immediately dismiss them, but needs some @@ -1823,7 +1844,8 @@ uint64_t Blockchain::get_adjusted_time() return time(NULL); } //------------------------------------------------------------------ -bool Blockchain::check_block_timestamp(const std::vector& timestamps, const block& b) +//TODO: revisit, has changed a bit on upstream +bool Blockchain::check_block_timestamp(std::vector& timestamps, const block& b) { uint64_t median_ts = epee::misc_utils::median(timestamps); diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index 27e5cde8c..884de122d 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -81,8 +81,7 @@ namespace cryptonote Blockchain(tx_memory_pool& tx_pool); - bool init() { return init(tools::get_default_data_dir()); } - bool init(const std::string& config_folder); + bool init(const std::string& config_folder, bool testnet = false); bool deinit(); void set_checkpoints(checkpoints&& chk_pts) { m_checkpoints = chk_pts; } @@ -189,7 +188,7 @@ namespace cryptonote bool prevalidate_miner_transaction(const block& b, uint64_t height); bool validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins); bool validate_transaction(const block& b, uint64_t height, const transaction& tx); - bool rollback_blockchain_switching(std::list& original_chain); + bool rollback_blockchain_switching(std::list& original_chain, uint64_t rollback_height); bool add_transaction_from_block(const transaction& tx, const crypto::hash& tx_id, const crypto::hash& bl_id, uint64_t bl_height); bool push_transaction_to_global_outs_index(const transaction& tx, const crypto::hash& tx_id, std::vector& global_indexes); bool pop_transaction_from_global_index(const transaction& tx, const crypto::hash& tx_id); @@ -199,7 +198,7 @@ namespace cryptonote bool add_block_as_invalid(const block& bl, const crypto::hash& h); bool add_block_as_invalid(const block_extended_info& bei, const crypto::hash& h); bool check_block_timestamp(const block& b); - bool check_block_timestamp(const std::vector& timestamps, const block& b); + bool check_block_timestamp(std::vector& timestamps, const block& b); uint64_t get_adjusted_time(); bool complete_timestamps_vector(uint64_t start_height, std::vector& timestamps); bool update_next_cumulative_size_limit(); -- cgit v1.2.3 From bc44bc19f4fa9e7eabb1a1b82127398c1a201048 Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Wed, 15 Oct 2014 18:33:53 -0400 Subject: Initial commit of BlockchainDB tests, other misc miscellaneous changes to BlockchainDB/blockchain as well, namely replacing instances of std::list with std::vector --- src/cryptonote_core/blockchain.cpp | 7 ++++++- src/cryptonote_core/blockchain_db.h | 24 +++++++++++++++++++----- 2 files changed, 25 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 828ad96e4..a35ce846c 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -547,11 +547,16 @@ bool Blockchain::get_block_by_hash(const crypto::hash &h, block &blk) return false; } //------------------------------------------------------------------ +//FIXME: this function does not seem to be called from anywhere, but +// if it ever is, should probably change std::list for std::vector void Blockchain::get_all_known_block_ids(std::list &main, std::list &alt, std::list &invalid) { CRITICAL_REGION_LOCAL(m_blockchain_lock); - main = m_db->get_hashes_range(0, m_db->height()); + for (auto& a : m_db->get_hashes_range(0, m_db->height())) + { + main.push_back(a); + } BOOST_FOREACH(blocks_ext_by_hash::value_type &v, m_alternative_chains) alt.push_back(v.first); diff --git a/src/cryptonote_core/blockchain_db.h b/src/cryptonote_core/blockchain_db.h index 5359909f7..251f9272b 100644 --- a/src/cryptonote_core/blockchain_db.h +++ b/src/cryptonote_core/blockchain_db.h @@ -25,6 +25,8 @@ // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#ifndef BLOCKCHAIN_DB_H +#define BLOCKCHAIN_DB_H #include #include @@ -70,6 +72,8 @@ * bool lock() * unlock() * + * vector get_filenames() + * * Blocks: * bool block_exists(hash) * height add_block(block, block_size, cumulative_difficulty, coins_generated, transactions) @@ -364,6 +368,11 @@ private: public: + + // virtual dtor + virtual ~BlockchainDB() { }; + + // open the db at location , or create it if there isn't one. virtual void open(const std::string& filename) = 0; @@ -379,6 +388,9 @@ public: // reset the db -- USE WITH CARE virtual void reset() = 0; + // get all files used by this db (if any) + virtual std::vector get_filenames() = 0; + // FIXME: these are just for functionality mocking, need to implement // RAII-friendly and multi-read one-write friendly locking mechanism @@ -435,11 +447,11 @@ public: // return hash of block at height virtual crypto::hash get_block_hash_from_height(const uint64_t& height) = 0; - // return list of blocks in range of height. - virtual std::list get_blocks_range(const uint64_t& h1, const uint64_t& h2) = 0; + // return vector of blocks in range of height (inclusively) + virtual std::vector get_blocks_range(const uint64_t& h1, const uint64_t& h2) = 0; - // return list of block hashes in range of height - virtual std::list get_hashes_range(const uint64_t& h1, const uint64_t& h2) = 0; + // return vector of block hashes in range of height (inclusively) + virtual std::vector get_hashes_range(const uint64_t& h1, const uint64_t& h2) = 0; // return the hash of the top block on the chain virtual crypto::hash top_block_hash() = 0; @@ -479,7 +491,7 @@ public: // return list of tx with hashes . // TODO: decide if a missing hash means return empty list // or just skip that hash - virtual std::list get_tx_list(const std::vector& hlist) = 0; + virtual std::vector get_tx_list(const std::vector& hlist) = 0; // returns height of block that contains transaction with hash virtual uint64_t get_tx_block_height(const crypto::hash& h) = 0; @@ -511,3 +523,5 @@ public: } // namespace cryptonote + +#endif // BLOCKCHAIN_DB_H -- cgit v1.2.3 From b98b96489f0b84daea4bdb5291dfa1e5e54ae612 Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Tue, 21 Oct 2014 16:33:43 -0400 Subject: Initial commit of lmdb BlockchainDB impl --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 319 ++++++++++++++++++++++ src/cryptonote_core/BlockchainDB_impl/db_lmdb.h | 153 +++++++++++ 2 files changed, 472 insertions(+) create mode 100644 src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp create mode 100644 src/cryptonote_core/BlockchainDB_impl/db_lmdb.h (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp new file mode 100644 index 000000000..7deae302c --- /dev/null +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -0,0 +1,319 @@ +// Copyright (c) 2014, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "db_lmdb.h" + +namespace cryptonote +{ + +#define CN_LMDB_DBI_OPEN( txn, flags, dbi ) \ + if (mdb_dbi_open(txn, NULL, flags, &dbi)) \ + { \ + throw DB_OPEN_FAILURE( "Failed to open db handle for ##dbi" ); \ + } + + +void BlockchainLMDB::add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + ) +{ +} + +void BlockchainLMDB::remove_block(const crypto::hash& blk_hash) +{ +} + +void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx) +{ +} + +void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash) +{ +} + +void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index) +{ +} + +void BlockchainLMDB::remove_output(const tx_out& tx_output) +{ +} + +void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image) +{ +} + +void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image) +{ +} + +BlockchainLMDB::~BlockchainLMDB() +{ +} + +BlockchainLMDB::BlockchainLMDB() +{ +} + +void BlockchainLMDB::open(const std::string& filename) +{ + if (mdb_env_create(&m_env)) + { + throw DB_ERROR("Failed to create lmdb environment"); + } + if (mdb_env_open(m_env, filename.c_str(), 0, 0664)) + { + throw DB_ERROR("Failed to open lmdb environment"); + } + if (mdb_env_set_maxdbs(m_env, 15)) + { + throw DB_ERROR("Failed to set max number of dbs"); + } + MDB_txn* txn; + if (mdb_txn_begin(m_env, NULL, 0, &txn)) + { + throw DB_ERROR("Failed to create a transaction for the db"); + } + + CN_LMDB_DBI_OPEN(txn, MDB_CREATE | MDB_DUPSORT, m_blocks) + CN_LMDB_DBI_OPEN(txn, MDB_CREATE | MDB_DUPSORT, m_block_txs) + CN_LMDB_DBI_OPEN(txn, MDB_CREATE | MDB_DUPSORT, m_block_hashes) + CN_LMDB_DBI_OPEN(txn, MDB_CREATE | MDB_DUPSORT, m_block_sizes) + CN_LMDB_DBI_OPEN(txn, MDB_CREATE | MDB_DUPSORT, m_block_diffs) + CN_LMDB_DBI_OPEN(txn, MDB_CREATE | MDB_DUPSORT, m_block_coins) + CN_LMDB_DBI_OPEN(txn, MDB_CREATE | MDB_DUPSORT, m_txs) + + if (mdb_txn_commit(txn)) + { + throw DB_OPEN_FAILURE("Failed to commit db open transaction"); + } + + // from here, init should be finished +} + +// unused for now, create will happen on open if doesn't exist +void BlockchainLMDB::create(const std::string& filename) +{ +} + +void BlockchainLMDB::close() +{ +} + +void BlockchainLMDB::sync() +{ +} + +void BlockchainLMDB::reset() +{ +} + +std::vector BlockchainLMDB::get_filenames() +{ + std::vector retval; + return retval; +} + + +bool BlockchainLMDB::lock() +{ + return false; +} + +void BlockchainLMDB::unlock() +{ +} + + +bool BlockchainLMDB::block_exists(const crypto::hash& h) +{ + return false; +} + +block BlockchainLMDB::get_block(const crypto::hash& h) +{ + block b; + return b; +} + +uint64_t BlockchainLMDB::get_block_height(const crypto::hash& h) +{ + return 0; +} + +block_header BlockchainLMDB::get_block_header(const crypto::hash& h) +{ + block_header bh; + return bh; +} + +block BlockchainLMDB::get_block_from_height(const uint64_t& height) +{ + block b; + return b; +} + +uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) +{ + return 0; +} + +uint64_t BlockchainLMDB::get_top_block_timestamp() +{ + return 0; +} + +size_t BlockchainLMDB::get_block_size(const uint64_t& height) +{ + return 0; +} + +difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& height) +{ + return 0; +} + +difficulty_type BlockchainLMDB::get_block_difficulty(const uint64_t& height) +{ + return 0; +} + +uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& height) +{ + return 0; +} + +crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height) +{ + crypto::hash h; + return h; +} + +std::vector BlockchainLMDB::get_blocks_range(const uint64_t& h1, const uint64_t& h2) +{ + std::vector v; + return v; +} + +std::vector BlockchainLMDB::get_hashes_range(const uint64_t& h1, const uint64_t& h2) +{ + std::vector v; + return v; +} + +crypto::hash BlockchainLMDB::top_block_hash() +{ + crypto::hash h; + return h; +} + +block BlockchainLMDB::get_top_block() +{ + block b; + return b; +} + +uint64_t BlockchainLMDB::height() +{ + return 0; +} + + +bool BlockchainLMDB::tx_exists(const crypto::hash& h) +{ + return false; +} + +uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) +{ + return 0; +} + +transaction BlockchainLMDB::get_tx(const crypto::hash& h) +{ + transaction t; + return t; +} + +uint64_t BlockchainLMDB::get_tx_count() +{ + return 0; +} + +std::vector BlockchainLMDB::get_tx_list(const std::vector& hlist) +{ + std::vector v; + return v; +} + +uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) +{ + return 0; +} + +uint64_t BlockchainLMDB::get_random_output(const uint64_t& amount) +{ + return 0; +} + +uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) +{ + return 0; +} + +crypto::public_key BlockchainLMDB::get_output_key(const uint64_t& amount, const uint64_t& index) +{ + crypto::public_key pk; + return pk; +} + +tx_out BlockchainLMDB::get_output(const crypto::hash& h, const uint64_t& index) +{ + tx_out o; + return o; +} + +tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) +{ + tx_out_index i; + return i; +} + +std::vector BlockchainLMDB::get_tx_output_indices(const crypto::hash& h) +{ + std::vector v; + return v; +} + +bool BlockchainLMDB::has_key_image(const crypto::key_image& img) +{ + return false; +} + +} // namespace cryptonote diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h new file mode 100644 index 000000000..50e8d3375 --- /dev/null +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h @@ -0,0 +1,153 @@ +// Copyright (c) 2014, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "cryptonote_core/blockchain_db.h" + +#include + +namespace cryptonote +{ + +class BlockchainLMDB : public BlockchainDB +{ +public: + BlockchainLMDB(); + ~BlockchainLMDB(); + + virtual void open(const std::string& filename); + + virtual void create(const std::string& filename); + + virtual void close(); + + virtual void sync(); + + virtual void reset(); + + virtual std::vector get_filenames(); + + virtual bool lock(); + + virtual void unlock(); + + virtual bool block_exists(const crypto::hash& h); + + virtual block get_block(const crypto::hash& h); + + virtual uint64_t get_block_height(const crypto::hash& h); + + virtual block_header get_block_header(const crypto::hash& h); + + virtual block get_block_from_height(const uint64_t& height); + + virtual uint64_t get_block_timestamp(const uint64_t& height) ; + + virtual uint64_t get_top_block_timestamp(); + + virtual size_t get_block_size(const uint64_t& height); + + virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height); + + virtual difficulty_type get_block_difficulty(const uint64_t& height); + + virtual uint64_t get_block_already_generated_coins(const uint64_t& height); + + virtual crypto::hash get_block_hash_from_height(const uint64_t& height); + + virtual std::vector get_blocks_range(const uint64_t& h1, const uint64_t& h2); + + virtual std::vector get_hashes_range(const uint64_t& h1, const uint64_t& h2); + + virtual crypto::hash top_block_hash(); + + virtual block get_top_block(); + + virtual uint64_t height(); + + virtual bool tx_exists(const crypto::hash& h); + + virtual uint64_t get_tx_unlock_time(const crypto::hash& h); + + virtual transaction get_tx(const crypto::hash& h); + + virtual uint64_t get_tx_count(); + + virtual std::vector get_tx_list(const std::vector& hlist); + + virtual uint64_t get_tx_block_height(const crypto::hash& h); + + virtual uint64_t get_random_output(const uint64_t& amount); + + virtual uint64_t get_num_outputs(const uint64_t& amount); + + virtual crypto::public_key get_output_key(const uint64_t& amount, const uint64_t& index); + + virtual tx_out get_output(const crypto::hash& h, const uint64_t& index); + + virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index); + + virtual std::vector get_tx_output_indices(const crypto::hash& h); + + virtual bool has_key_image(const crypto::key_image& img); + +private: + virtual void add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + ); + + virtual void remove_block(const crypto::hash& blk_hash); + + virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx); + + virtual void remove_transaction_data(const crypto::hash& tx_hash); + + virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index); + + virtual void remove_output(const tx_out& tx_output); + + virtual void add_spent_key(const crypto::key_image& k_image); + + virtual void remove_spent_key(const crypto::key_image& k_image); + + MDB_env* m_env; + + MDB_dbi m_blocks; + MDB_dbi m_block_txs; + MDB_dbi m_block_hashes; + MDB_dbi m_block_sizes; + MDB_dbi m_block_diffs; + MDB_dbi m_block_coins; + + MDB_dbi m_txs; + + MDB_dbi m_spent; + MDB_dbi m_utxo; +}; + +} // namespace cryptonote -- cgit v1.2.3 From db00ce0173f49ef9e1d07019b803155c9d4fe945 Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Thu, 23 Oct 2014 15:37:10 -0400 Subject: Parts of LMDB impl of BlockchainDB done and working The rest should just be tedious copypasta and modification. --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 377 ++++++++++++++++++++-- src/cryptonote_core/BlockchainDB_impl/db_lmdb.h | 58 +++- src/cryptonote_core/blockchain_db.cpp | 47 +-- src/cryptonote_core/blockchain_db.h | 18 +- 4 files changed, 431 insertions(+), 69 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index 7deae302c..7330ad867 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -27,15 +27,21 @@ #include "db_lmdb.h" +#include +#include // std::unique_ptr +#include // memcpy + +#include "cryptonote_core/cryptonote_format_utils.h" + namespace cryptonote { -#define CN_LMDB_DBI_OPEN( txn, flags, dbi ) \ - if (mdb_dbi_open(txn, NULL, flags, &dbi)) \ - { \ - throw DB_OPEN_FAILURE( "Failed to open db handle for ##dbi" ); \ - } - +const char* LMDB_BLOCKS = "blocks"; +const char* LMDB_BLOCK_HASHES = "block_hashes"; +const char* LMDB_BLOCK_SIZES = "block_sizes"; +const char* LMDB_BLOCK_DIFFS = "block_diffs"; +const char* LMDB_BLOCK_COINS = "block_coins"; +const char* LMDB_TXS = "txs"; void BlockchainLMDB::add_block( const block& blk , const size_t& block_size @@ -43,34 +49,121 @@ void BlockchainLMDB::add_block( const block& blk , const uint64_t& coins_generated ) { + check_open(); + + crypto::hash h = get_block_hash(blk); + MDB_val val_h; + val_h.mv_size = sizeof(crypto::hash); + val_h.mv_data = &h; + + MDB_val unused; + if (mdb_get(*m_write_txn, m_block_hashes, &val_h, &unused) == 0) + { + LOG_PRINT_L1("Attempting to add block that's already in the db"); + throw BLOCK_EXISTS("Attempting to add block that's already in the db"); + } + + if (m_height > 0) + { + MDB_val parent_key; + crypto::hash parent = blk.prev_id; + parent_key.mv_size = sizeof(crypto::hash); + parent_key.mv_data = &parent; + + MDB_val parent_h; + if (mdb_get(*m_write_txn, m_block_hashes, &parent_key, &parent_h)) + { + LOG_PRINT_L0("Failed to get top block hash to check for new block's parent"); + throw DB_ERROR("Failed to get top block hash to check for new block's parent"); + } + + uint64_t parent_height = *(uint64_t *)parent_h.mv_data; + if (parent_height != m_height - 1) + { + LOG_PRINT_L0("Top block is not new block's parent"); + throw BLOCK_PARENT_DNE("Top block is not new block's parent"); + } + } + + MDB_val key; + key.mv_size = sizeof(uint64_t); + key.mv_data = &m_height; + + auto bd = block_to_blob(blk); + + // const-correctness be trolling, yo + std::unique_ptr bd_cpy(new char[bd.size()]); + memcpy(bd_cpy.get(), bd.data(), bd.size()); + + MDB_val blob; + blob.mv_size = bd.size(); + blob.mv_data = bd_cpy.get(); + if (mdb_put(*m_write_txn, m_blocks, &key, &blob, 0)) + { + LOG_PRINT_L0("Failed to add block blob to db transaction"); + throw DB_ERROR("Failed to add block blob to db transaction"); + } + + size_t size_cpy = block_size; + MDB_val sz; + sz.mv_size = sizeof(block_size); + sz.mv_data = &size_cpy; + if (mdb_put(*m_write_txn, m_block_sizes, &key, &sz, 0)) + { + LOG_PRINT_L0("Failed to add block size to db transaction"); + throw DB_ERROR("Failed to add block size to db transaction"); + } + + if (mdb_put(*m_write_txn, m_block_hashes, &val_h, &key, 0)) + { + LOG_PRINT_L0("Failed to add block size to db transaction"); + throw DB_ERROR("Failed to add block size to db transaction"); + } + } -void BlockchainLMDB::remove_block(const crypto::hash& blk_hash) +void BlockchainLMDB::remove_block() { + check_open(); } void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx) { + check_open(); } void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash) { + check_open(); } void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index) { + check_open(); } void BlockchainLMDB::remove_output(const tx_out& tx_output) { + check_open(); } void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image) { + check_open(); } void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image) { + check_open(); +} + +void BlockchainLMDB::check_open() +{ + if (!m_open) + { + LOG_PRINT_L0("DB operation attempted on a not-open DB instance"); + throw DB_ERROR("DB operation attempted on a not-open DB instance"); + } } BlockchainLMDB::~BlockchainLMDB() @@ -79,41 +172,113 @@ BlockchainLMDB::~BlockchainLMDB() BlockchainLMDB::BlockchainLMDB() { + // initialize folder to something "safe" just in case + // someone accidentally misuses this class... + m_folder = "thishsouldnotexistbecauseitisgibberish"; + m_open = false; } void BlockchainLMDB::open(const std::string& filename) { - if (mdb_env_create(&m_env)) + + if (m_open) { - throw DB_ERROR("Failed to create lmdb environment"); + LOG_PRINT_L0("Attempted to open db, but it's already open"); + throw DB_OPEN_FAILURE("Attempted to open db, but it's already open"); } - if (mdb_env_open(m_env, filename.c_str(), 0, 0664)) + + boost::filesystem::path direc(filename); + if (boost::filesystem::exists(direc)) + { + if (!boost::filesystem::is_directory(direc)) + { + LOG_PRINT_L0("LMDB needs a directory path, but a file was passed"); + throw DB_OPEN_FAILURE("LMDB needs a directory path, but a file was passed"); + } + } + else { - throw DB_ERROR("Failed to open lmdb environment"); + if (!boost::filesystem::create_directory(direc)) + { + LOG_PRINT_L0("Failed to create directory " << filename); + throw DB_OPEN_FAILURE(std::string("Failed to create directory ").append(filename).c_str()); + } + } + + m_folder = filename; + + // set up lmdb environment + if (mdb_env_create(&m_env)) + { + LOG_PRINT_L0("Failed to create lmdb environment"); + throw DB_OPEN_FAILURE("Failed to create lmdb environment"); } if (mdb_env_set_maxdbs(m_env, 15)) { - throw DB_ERROR("Failed to set max number of dbs"); + LOG_PRINT_L0("Failed to set max number of dbs"); + throw DB_OPEN_FAILURE("Failed to set max number of dbs"); + } + if (mdb_env_open(m_env, filename.c_str(), 0, 0664)) + { + LOG_PRINT_L0("Failed to open lmdb environment"); + throw DB_OPEN_FAILURE("Failed to open lmdb environment"); } - MDB_txn* txn; - if (mdb_txn_begin(m_env, NULL, 0, &txn)) + + // get a read/write MDB_txn + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, 0, txn)) { + LOG_PRINT_L0("Failed to create a transaction for the db"); throw DB_ERROR("Failed to create a transaction for the db"); } - CN_LMDB_DBI_OPEN(txn, MDB_CREATE | MDB_DUPSORT, m_blocks) - CN_LMDB_DBI_OPEN(txn, MDB_CREATE | MDB_DUPSORT, m_block_txs) - CN_LMDB_DBI_OPEN(txn, MDB_CREATE | MDB_DUPSORT, m_block_hashes) - CN_LMDB_DBI_OPEN(txn, MDB_CREATE | MDB_DUPSORT, m_block_sizes) - CN_LMDB_DBI_OPEN(txn, MDB_CREATE | MDB_DUPSORT, m_block_diffs) - CN_LMDB_DBI_OPEN(txn, MDB_CREATE | MDB_DUPSORT, m_block_coins) - CN_LMDB_DBI_OPEN(txn, MDB_CREATE | MDB_DUPSORT, m_txs) + // open necessary databases, and set properties as needed + // uses macros to avoid having to change things too many places + if (mdb_dbi_open(txn, LMDB_BLOCKS, MDB_INTEGERKEY | MDB_CREATE, &m_blocks)) + { + LOG_PRINT_L0("Failed to open db handle for m_blocks"); + throw DB_OPEN_FAILURE( "Failed to open db handle for m_blocks" ); + } + + if (mdb_dbi_open(txn, LMDB_BLOCK_HASHES, MDB_INTEGERKEY | MDB_CREATE, &m_block_hashes)) + { + LOG_PRINT_L0("Failed to open db handle for m_block_hashes"); + throw DB_OPEN_FAILURE( "Failed to open db handle for m_block_hashes" ); + } + if (mdb_dbi_open(txn, LMDB_BLOCK_SIZES, MDB_INTEGERKEY | MDB_CREATE, &m_block_sizes)) + { + LOG_PRINT_L0("Failed to open db handle for m_block_sizes"); + throw DB_OPEN_FAILURE( "Failed to open db handle for m_block_sizes" ); + } + if (mdb_dbi_open(txn, LMDB_BLOCK_DIFFS, MDB_INTEGERKEY | MDB_CREATE, &m_block_diffs)) + { + LOG_PRINT_L0("Failed to open db handle for m_block_diffs"); + throw DB_OPEN_FAILURE( "Failed to open db handle for m_block_diffs" ); + } + if (mdb_dbi_open(txn, LMDB_BLOCK_COINS, MDB_INTEGERKEY | MDB_CREATE, &m_block_coins)) + { + LOG_PRINT_L0("Failed to open db handle for m_block_coins"); + throw DB_OPEN_FAILURE( "Failed to open db handle for m_block_coins" ); + } + if (mdb_dbi_open(txn, LMDB_TXS, MDB_CREATE, &m_txs)) + { + LOG_PRINT_L0("Failed to open db handle for m_txs"); + throw DB_OPEN_FAILURE( "Failed to open db handle for m_txs" ); + } - if (mdb_txn_commit(txn)) + // get and keep current height + MDB_stat db_stats; + if (mdb_stat(txn, m_blocks, &db_stats)) { - throw DB_OPEN_FAILURE("Failed to commit db open transaction"); + LOG_PRINT_L0("Failed to query m_blocks"); + throw DB_ERROR("Failed to query m_blocks"); } + m_height = db_stats.ms_entries; + + // commit the transaction + txn.commit(); + m_open = true; // from here, init should be finished } @@ -136,184 +301,340 @@ void BlockchainLMDB::reset() std::vector BlockchainLMDB::get_filenames() { - std::vector retval; - return retval; + std::vector filenames; + + boost::filesystem::path datafile(m_folder); + datafile /= "data.mdb"; + boost::filesystem::path lockfile(m_folder); + lockfile /= "lock.mdb"; + + filenames.push_back(datafile.string()); + filenames.push_back(lockfile.string()); + + return filenames; } bool BlockchainLMDB::lock() { + check_open(); return false; } void BlockchainLMDB::unlock() { + check_open(); } bool BlockchainLMDB::block_exists(const crypto::hash& h) { - return false; + check_open(); + + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + { + LOG_PRINT_L0("Failed to create a transaction for the db"); + throw DB_ERROR("Failed to create a transaction for the db"); + } + + crypto::hash key_cpy = h; + MDB_val key; + key.mv_size = sizeof(crypto::hash); + key.mv_data = &key_cpy; + + MDB_val result; + auto get_result = mdb_get(txn, m_block_hashes, &key, &result); + if (get_result == MDB_NOTFOUND) + { + LOG_PRINT_L1("Block with hash " << epee::string_tools::pod_to_hex(h) << "not found in db"); + return false; + } + else if (get_result) + { + LOG_PRINT_L0("DB error attempting to fetch block index from hash"); + throw DB_ERROR("DB error attempting to fetch block index from hash"); + } + + return true; } block BlockchainLMDB::get_block(const crypto::hash& h) { - block b; - return b; + check_open(); + + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + { + LOG_PRINT_L0("Failed to create a transaction for the db"); + throw DB_ERROR("Failed to create a transaction for the db"); + } + + crypto::hash key_cpy = h; + MDB_val key; + key.mv_size = sizeof(crypto::hash); + key.mv_data = &key_cpy; + + MDB_val result; + auto get_result = mdb_get(txn, m_block_hashes, &key, &result); + if (get_result == MDB_NOTFOUND) + { + LOG_PRINT_L1("Attempted to retrieve non-existent block"); + throw BLOCK_DNE("Attempted to retrieve non-existent block"); + } + else if (get_result) + { + LOG_PRINT_L0("Error attempting to retrieve a block index from the db"); + throw DB_ERROR("Error attempting to retrieve a block index from the db"); + } + + txn.commit(); + + uint64_t index = *(uint64_t*)result.mv_data; + + return get_block_from_height(index); } uint64_t BlockchainLMDB::get_block_height(const crypto::hash& h) { + check_open(); return 0; } block_header BlockchainLMDB::get_block_header(const crypto::hash& h) { + check_open(); block_header bh; return bh; } block BlockchainLMDB::get_block_from_height(const uint64_t& height) { + check_open(); + + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + { + LOG_PRINT_L0("Failed to create a transaction for the db"); + throw DB_ERROR("Failed to create a transaction for the db"); + } + + uint64_t height_cpy = height; + MDB_val key; + key.mv_size = sizeof(uint64_t); + key.mv_data = &height_cpy; + MDB_val result; + auto get_result = mdb_get(txn, m_blocks, &key, &result); + if (get_result == MDB_NOTFOUND) + { + LOG_PRINT_L0("Attempted to get block from height " << height << ", but no such block exists"); + throw DB_ERROR("Attempt to get block from height failed -- block not in db"); + } + else if (get_result) + { + LOG_PRINT_L0("Error attempting to retrieve a block from the db"); + throw DB_ERROR("Error attempting to retrieve a block from the db"); + } + + txn.commit(); + + blobdata bd; + bd.assign(reinterpret_cast(result.mv_data), result.mv_size); + block b; + if (!parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Failed to parse block from blob retrieved from the db"); + throw DB_ERROR("Failed to parse block from blob retrieved from the db"); + } + return b; } uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) { + check_open(); return 0; } uint64_t BlockchainLMDB::get_top_block_timestamp() { + check_open(); return 0; } size_t BlockchainLMDB::get_block_size(const uint64_t& height) { + check_open(); return 0; } difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& height) { + check_open(); return 0; } difficulty_type BlockchainLMDB::get_block_difficulty(const uint64_t& height) { + check_open(); return 0; } uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& height) { + check_open(); return 0; } crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height) { + check_open(); crypto::hash h; return h; } std::vector BlockchainLMDB::get_blocks_range(const uint64_t& h1, const uint64_t& h2) { + check_open(); std::vector v; return v; } std::vector BlockchainLMDB::get_hashes_range(const uint64_t& h1, const uint64_t& h2) { + check_open(); std::vector v; return v; } crypto::hash BlockchainLMDB::top_block_hash() { + check_open(); crypto::hash h; return h; } block BlockchainLMDB::get_top_block() { + check_open(); block b; return b; } uint64_t BlockchainLMDB::height() { + check_open(); return 0; } bool BlockchainLMDB::tx_exists(const crypto::hash& h) { + check_open(); return false; } uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) { + check_open(); return 0; } transaction BlockchainLMDB::get_tx(const crypto::hash& h) { + check_open(); transaction t; return t; } uint64_t BlockchainLMDB::get_tx_count() { + check_open(); return 0; } std::vector BlockchainLMDB::get_tx_list(const std::vector& hlist) { + check_open(); std::vector v; return v; } uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) { + check_open(); return 0; } uint64_t BlockchainLMDB::get_random_output(const uint64_t& amount) { + check_open(); return 0; } uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) { + check_open(); return 0; } crypto::public_key BlockchainLMDB::get_output_key(const uint64_t& amount, const uint64_t& index) { + check_open(); crypto::public_key pk; return pk; } tx_out BlockchainLMDB::get_output(const crypto::hash& h, const uint64_t& index) { + check_open(); tx_out o; return o; } tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) { + check_open(); tx_out_index i; return i; } std::vector BlockchainLMDB::get_tx_output_indices(const crypto::hash& h) { + check_open(); std::vector v; return v; } bool BlockchainLMDB::has_key_image(const crypto::key_image& img) { + check_open(); return false; } +uint64_t BlockchainLMDB::add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const std::vector& txs + ) +{ + check_open(); + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, 0, txn)) + { + LOG_PRINT_L0("Failed to create a transaction for the db"); + throw DB_ERROR("Failed to create a transaction for the db"); + } + m_write_txn = &txn; + + BlockchainDB::add_block(blk, block_size, cumulative_difficulty, coins_generated, txs); + + txn.commit(); + + m_height++; + return m_height - 1; +} + } // namespace cryptonote diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h index 50e8d3375..b2cf5f30a 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h @@ -32,6 +32,47 @@ namespace cryptonote { +struct txn_safe +{ + txn_safe() : m_txn(NULL) { } + ~txn_safe() + { + if(m_txn != NULL) + { + mdb_txn_abort(m_txn); + } + } + + void commit(std::string message = "") + { + if (message.size() == 0) + { + message = "Failed to commit a transaction to the db"; + } + + if (mdb_txn_commit(m_txn)) + { + m_txn = NULL; + LOG_PRINT_L0(message); + throw DB_ERROR(message.c_str()); + } + m_txn = NULL; + } + + operator MDB_txn*() + { + return m_txn; + } + + operator MDB_txn**() + { + return &m_txn; + } + + MDB_txn* m_txn; +}; + + class BlockchainLMDB : public BlockchainDB { public: @@ -114,6 +155,13 @@ public: virtual bool has_key_image(const crypto::key_image& img); + virtual uint64_t add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const std::vector& txs + ); + private: virtual void add_block( const block& blk , const size_t& block_size @@ -121,7 +169,7 @@ private: , const uint64_t& coins_generated ); - virtual void remove_block(const crypto::hash& blk_hash); + virtual void remove_block(); virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx); @@ -135,10 +183,11 @@ private: virtual void remove_spent_key(const crypto::key_image& k_image); + void check_open(); + MDB_env* m_env; MDB_dbi m_blocks; - MDB_dbi m_block_txs; MDB_dbi m_block_hashes; MDB_dbi m_block_sizes; MDB_dbi m_block_diffs; @@ -148,6 +197,11 @@ private: MDB_dbi m_spent; MDB_dbi m_utxo; + + bool m_open; + uint64_t m_height; + std::string m_folder; + txn_safe* m_write_txn; }; } // namespace cryptonote diff --git a/src/cryptonote_core/blockchain_db.cpp b/src/cryptonote_core/blockchain_db.cpp index 606b34310..02912be10 100644 --- a/src/cryptonote_core/blockchain_db.cpp +++ b/src/cryptonote_core/blockchain_db.cpp @@ -62,40 +62,22 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti } uint64_t BlockchainDB::add_block( const block& blk - , const size_t& block_size - , const difficulty_type& cumulative_difficulty - , const uint64_t& coins_generated - , const std::vector& txs - ) + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const std::vector& txs + ) { - try - { - // call out to subclass implementation to add the block & metadata - add_block(blk, block_size, cumulative_difficulty, coins_generated); + // call out to subclass implementation to add the block & metadata + add_block(blk, block_size, cumulative_difficulty, coins_generated); - crypto::hash blk_hash = get_block_hash(blk); - // call out to add the transactions - for (const transaction& tx : txs) - { - add_transaction(blk_hash, tx); - } - } - // in case any of the add_block process goes awry, undo - catch (const std::exception& e) + crypto::hash blk_hash = get_block_hash(blk); + // call out to add the transactions + + add_transaction(blk_hash, blk.miner_tx); + for (const transaction& tx : txs) { - LOG_ERROR("Error adding block to db: " << e.what()); - try - { - pop_block(); - } - // if undoing goes wrong as well, we need to throw, as blockchain - // will be in a bad state - catch (const std::exception& e) - { - LOG_ERROR("Error undoing partially added block: " << e.what()); - throw; - } - throw; + add_transaction(blk_hash, tx); } return height(); @@ -104,7 +86,10 @@ uint64_t BlockchainDB::add_block( const block& blk void BlockchainDB::pop_block(block& blk, std::vector& txs) { blk = get_top_block(); + + remove_block(); + remove_transaction(get_transaction_hash(blk.miner_tx)); for (const auto& h : blk.tx_hashes) { txs.push_back(get_tx(h)); diff --git a/src/cryptonote_core/blockchain_db.h b/src/cryptonote_core/blockchain_db.h index 251f9272b..b3c73a0d8 100644 --- a/src/cryptonote_core/blockchain_db.h +++ b/src/cryptonote_core/blockchain_db.h @@ -332,8 +332,8 @@ private: , const uint64_t& coins_generated ) = 0; - // tells the subclass to remove data about a block - virtual void remove_block(const crypto::hash& blk_hash) = 0; + // tells the subclass to remove data about the top block + virtual void remove_block() = 0; // tells the subclass to store the transaction and its metadata virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx) = 0; @@ -403,12 +403,14 @@ public: // adds a block with the given metadata to the top of the blockchain, returns the new height - uint64_t add_block( const block& blk - , const size_t& block_size - , const difficulty_type& cumulative_difficulty - , const uint64_t& coins_generated - , const std::vector& txs - ); + // NOTE: subclass implementations of this (or the functions it calls) need + // to handle undoing any partially-added blocks in the event of a failure. + virtual uint64_t add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const std::vector& txs + ); // return true if a block with hash exists in the blockchain virtual bool block_exists(const crypto::hash& h) = 0; -- cgit v1.2.3 From a0af217d9ad42ab21d8e79987d8a07eeb457045d Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Thu, 23 Oct 2014 19:47:36 -0400 Subject: Adding block data to LMDB BlockchainDB coded Still needs testing (and need to write a few more unit tests), but everything should be there. Lots of unfortunate duplication, but...well, I can't see a way around it using LMDB. A couple of other minor changes in this commit, only slightly relevant. --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 218 +++++++++++++++++++++- src/cryptonote_core/BlockchainDB_impl/db_lmdb.h | 12 +- src/cryptonote_core/blockchain_db.h | 36 +++- 3 files changed, 258 insertions(+), 8 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index 7330ad867..23c5fd4fc 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -42,6 +42,14 @@ const char* LMDB_BLOCK_SIZES = "block_sizes"; const char* LMDB_BLOCK_DIFFS = "block_diffs"; const char* LMDB_BLOCK_COINS = "block_coins"; const char* LMDB_TXS = "txs"; +const char* LMDB_TX_HEIGHTS = "tx_heights"; +const char* LMDB_TX_OUTPUTS = "tx_outputs"; +const char* LMDB_OUTPUT_TXS = "output_txs"; +const char* LMDB_OUTPUT_INDICES = "output_indices"; +const char* LMDB_OUTPUT_AMOUNTS = "output_amounts"; +const char* LMDB_OUTPUTS = "outputs"; +const char* LMDB_OUTPUT_GINDICES = "output_gindices"; +const char* LMDB_SPENT_KEYS = "spent_keys"; void BlockchainLMDB::add_block( const block& blk , const size_t& block_size @@ -130,6 +138,42 @@ void BlockchainLMDB::remove_block() void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx) { check_open(); + + crypto::hash h = get_transaction_hash(tx); + MDB_val val_h; + val_h.mv_size = sizeof(crypto::hash); + val_h.mv_data = &h; + + MDB_val unused; + if (mdb_get(*m_write_txn, m_txs, &val_h, &unused) == 0) + { + LOG_PRINT_L1("Attempting to add transaction that's already in the db"); + throw TX_EXISTS("Attempting to add transaction that's already in the db"); + } + + auto bd = tx_to_blob(tx); + + // const-correctness be trolling, yo + std::unique_ptr bd_cpy(new char[bd.size()]); + memcpy(bd_cpy.get(), bd.data(), bd.size()); + + MDB_val blob; + blob.mv_size = bd.size(); + blob.mv_data = bd_cpy.get(); + if (mdb_put(*m_write_txn, m_txs, &val_h, &blob, 0)) + { + LOG_PRINT_L0("Failed to add tx blob to db transaction"); + throw DB_ERROR("Failed to add tx blob to db transaction"); + } + + MDB_val height; + height.mv_size = sizeof(uint64_t); + height.mv_data = &m_height; + if (mdb_put(*m_write_txn, m_tx_heights, &val_h, &height, 0)) + { + LOG_PRINT_L0("Failed to add tx block height to db transaction"); + throw DB_ERROR("Failed to add tx block height to db transaction"); + } } void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash) @@ -140,6 +184,66 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash) void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index) { check_open(); + + MDB_val k; + MDB_val v; + + k.mv_size = sizeof(uint64_t); + k.mv_data = &m_num_outputs; + crypto::hash h_cpy = tx_hash; + v.mv_size = sizeof(crypto::hash); + v.mv_data = &h_cpy; + if (mdb_put(*m_write_txn, m_output_txs, &k, &v, 0)) + { + LOG_PRINT_L0("Failed to add output tx hash to db transaction"); + throw DB_ERROR("Failed to add output tx hash to db transaction"); + } + if (mdb_put(*m_write_txn, m_tx_outputs, &v, &k, 0)) + { + LOG_PRINT_L0("Failed to add tx output index to db transaction"); + throw DB_ERROR("Failed to add tx output index to db transaction"); + } + + uint64_t index_cpy = local_index; + v.mv_size = sizeof(uint64_t); + v.mv_data = &index_cpy; + if (mdb_put(*m_write_txn, m_output_indices, &k, &v, 0)) + { + LOG_PRINT_L0("Failed to add tx output index to db transaction"); + throw DB_ERROR("Failed to add tx output index to db transaction"); + } + + uint64_t amount = tx_output.amount; + v.mv_size = sizeof(uint64_t); + v.mv_data = &amount; + if (mdb_put(*m_write_txn, m_output_amounts, &v, &k, 0)) + { + LOG_PRINT_L0("Failed to add output amount to db transaction"); + throw DB_ERROR("Failed to add output amount to db transaction"); + } + + blobdata b; + t_serializable_object_to_blob(tx_output, b); + /* + * use this later to deserialize + std::stringstream ss; + ss << tx_blob; + binary_archive ba(ss); + bool r = ::serialization::serialize(ba, tx); + */ + v.mv_size = b.size(); + v.mv_data = &b; + if (mdb_put(*m_write_txn, m_outputs, &k, &v, 0)) + { + LOG_PRINT_L0("Failed to add output to db transaction"); + throw DB_ERROR("Failed to add output to db transaction"); + } + if (mdb_put(*m_write_txn, m_output_gindices, &v, &k, 0)) + { + LOG_PRINT_L0("Failed to add output global index to db transaction"); + throw DB_ERROR("Failed to add output global index to db transaction"); + } + } void BlockchainLMDB::remove_output(const tx_out& tx_output) @@ -150,6 +254,27 @@ void BlockchainLMDB::remove_output(const tx_out& tx_output) void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image) { check_open(); + + crypto::key_image key = k_image; + MDB_val val_key; + val_key.mv_size = sizeof(crypto::key_image); + val_key.mv_data = &key; + + MDB_val unused; + if (mdb_get(*m_write_txn, m_spent_keys, &val_key, &unused) == 0) + { + LOG_PRINT_L1("Attempting to add spent key image that's already in the db"); + throw KEY_IMAGE_EXISTS("Attempting to add spent key image that's already in the db"); + } + + char anything = '\0'; + unused.mv_size = sizeof(char); + unused.mv_data = &anything; + if (mdb_put(*m_write_txn, m_spent_keys, &val_key, &unused, 0)) + { + LOG_PRINT_L1("Error adding spent key image to db transaction"); + throw DB_ERROR("Error adding spent key image to db transaction"); + } } void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image) @@ -260,11 +385,54 @@ void BlockchainLMDB::open(const std::string& filename) LOG_PRINT_L0("Failed to open db handle for m_block_coins"); throw DB_OPEN_FAILURE( "Failed to open db handle for m_block_coins" ); } + if (mdb_dbi_open(txn, LMDB_TXS, MDB_CREATE, &m_txs)) { LOG_PRINT_L0("Failed to open db handle for m_txs"); throw DB_OPEN_FAILURE( "Failed to open db handle for m_txs" ); } + if (mdb_dbi_open(txn, LMDB_TX_HEIGHTS, MDB_CREATE, &m_tx_heights)) + { + LOG_PRINT_L0("Failed to open db handle for m_tx_heights"); + throw DB_OPEN_FAILURE( "Failed to open db handle for m_tx_heights" ); + } + if (mdb_dbi_open(txn, LMDB_TX_OUTPUTS, MDB_DUPSORT | MDB_CREATE, &m_tx_outputs)) + { + LOG_PRINT_L0("Failed to open db handle for m_tx_outputs"); + throw DB_OPEN_FAILURE( "Failed to open db handle for m_tx_outputs" ); + } + + if (mdb_dbi_open(txn, LMDB_OUTPUT_TXS, MDB_INTEGERKEY | MDB_CREATE, &m_output_txs)) + { + LOG_PRINT_L0("Failed to open db handle for m_output_txs"); + throw DB_OPEN_FAILURE( "Failed to open db handle for m_output_txs" ); + } + if (mdb_dbi_open(txn, LMDB_OUTPUT_INDICES, MDB_INTEGERKEY | MDB_CREATE, &m_output_indices)) + { + LOG_PRINT_L0("Failed to open db handle for m_output_indices"); + throw DB_OPEN_FAILURE( "Failed to open db handle for m_output_indices" ); + } + if (mdb_dbi_open(txn, LMDB_OUTPUT_GINDICES, MDB_CREATE, &m_output_gindices)) + { + LOG_PRINT_L0("Failed to open db handle for m_output_gindices"); + throw DB_OPEN_FAILURE( "Failed to open db handle for m_output_gindices" ); + } + if (mdb_dbi_open(txn, LMDB_OUTPUT_AMOUNTS, MDB_INTEGERKEY | MDB_DUPSORT | MDB_CREATE, &m_output_amounts)) + { + LOG_PRINT_L0("Failed to open db handle for m_output_amounts"); + throw DB_OPEN_FAILURE( "Failed to open db handle for m_output_amounts" ); + } + if (mdb_dbi_open(txn, LMDB_OUTPUTS, MDB_INTEGERKEY | MDB_CREATE, &m_outputs)) + { + LOG_PRINT_L0("Failed to open db handle for m_outputs"); + throw DB_OPEN_FAILURE( "Failed to open db handle for m_outputs" ); + } + + if (mdb_dbi_open(txn, LMDB_SPENT_KEYS, MDB_CREATE, &m_spent_keys)) + { + LOG_PRINT_L0("Failed to open db handle for m_outputs"); + throw DB_OPEN_FAILURE( "Failed to open db handle for m_outputs" ); + } // get and keep current height MDB_stat db_stats; @@ -275,6 +443,14 @@ void BlockchainLMDB::open(const std::string& filename) } m_height = db_stats.ms_entries; + // get and keep current number of outputs + if (mdb_stat(txn, m_outputs, &db_stats)) + { + LOG_PRINT_L0("Failed to query m_outputs"); + throw DB_ERROR("Failed to query m_outputs"); + } + m_num_outputs = db_stats.ms_entries; + // commit the transaction txn.commit(); @@ -347,6 +523,7 @@ bool BlockchainLMDB::block_exists(const crypto::hash& h) auto get_result = mdb_get(txn, m_block_hashes, &key, &result); if (get_result == MDB_NOTFOUND) { + txn.commit(); LOG_PRINT_L1("Block with hash " << epee::string_tools::pod_to_hex(h) << "not found in db"); return false; } @@ -356,6 +533,7 @@ bool BlockchainLMDB::block_exists(const crypto::hash& h) throw DB_ERROR("DB error attempting to fetch block index from hash"); } + txn.commit(); return true; } @@ -525,7 +703,8 @@ block BlockchainLMDB::get_top_block() uint64_t BlockchainLMDB::height() { check_open(); - return 0; + + return m_height; } @@ -610,6 +789,27 @@ std::vector BlockchainLMDB::get_tx_output_indices(const crypto::hash& bool BlockchainLMDB::has_key_image(const crypto::key_image& img) { check_open(); + + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + { + LOG_PRINT_L0("Failed to create a transaction for the db"); + throw DB_ERROR("Failed to create a transaction for the db"); + } + + crypto::key_image key = img; + MDB_val val_key; + val_key.mv_size = sizeof(crypto::key_image); + val_key.mv_data = &key; + + MDB_val unused; + if (mdb_get(txn, m_spent_keys, &val_key, &unused) == 0) + { + txn.commit(); + return true; + } + + txn.commit(); return false; } @@ -629,12 +829,20 @@ uint64_t BlockchainLMDB::add_block( const block& blk } m_write_txn = &txn; - BlockchainDB::add_block(blk, block_size, cumulative_difficulty, coins_generated, txs); + uint64_t num_outputs = m_num_outputs; + try + { + BlockchainDB::add_block(blk, block_size, cumulative_difficulty, coins_generated, txs); - txn.commit(); + txn.commit(); + } + catch (...) + { + m_num_outputs = num_outputs; + throw; + } - m_height++; - return m_height - 1; + return ++m_height; } } // namespace cryptonote diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h index b2cf5f30a..7b012e4f6 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h @@ -194,12 +194,20 @@ private: MDB_dbi m_block_coins; MDB_dbi m_txs; + MDB_dbi m_tx_heights; + MDB_dbi m_tx_outputs; - MDB_dbi m_spent; - MDB_dbi m_utxo; + MDB_dbi m_output_txs; + MDB_dbi m_output_indices; + MDB_dbi m_output_gindices; + MDB_dbi m_output_amounts; + MDB_dbi m_outputs; + + MDB_dbi m_spent_keys; bool m_open; uint64_t m_height; + uint64_t m_num_outputs; std::string m_folder; txn_safe* m_write_txn; }; diff --git a/src/cryptonote_core/blockchain_db.h b/src/cryptonote_core/blockchain_db.h index b3c73a0d8..f59678869 100644 --- a/src/cryptonote_core/blockchain_db.h +++ b/src/cryptonote_core/blockchain_db.h @@ -126,6 +126,8 @@ * TX_DNE * TX_EXISTS * OUTPUT_DNE + * OUTPUT_EXISTS + * KEY_IMAGE_EXISTS */ namespace cryptonote @@ -302,7 +304,7 @@ class OUTPUT_DNE : public std::exception private: std::string m; public: - OUTPUT_DNE() : m("The transaction requested does not exist") { } + OUTPUT_DNE() : m("The output requested does not exist!") { } OUTPUT_DNE(const char* s) : m(s) { } virtual ~OUTPUT_DNE() { } @@ -313,6 +315,38 @@ class OUTPUT_DNE : public std::exception } }; +class OUTPUT_EXISTS : public std::exception +{ + private: + std::string m; + public: + OUTPUT_EXISTS() : m("The output to be added already exists!") { } + OUTPUT_EXISTS(const char* s) : m(s) { } + + virtual ~OUTPUT_EXISTS() { } + + const char* what() const throw() + { + return m.c_str(); + } +}; + +class KEY_IMAGE_EXISTS : public std::exception +{ + private: + std::string m; + public: + KEY_IMAGE_EXISTS() : m("The spent key image to be added already exists!") { } + KEY_IMAGE_EXISTS(const char* s) : m(s) { } + + virtual ~KEY_IMAGE_EXISTS() { } + + const char* what() const throw() + { + return m.c_str(); + } +}; + /*********************************** * End of Exception Definitions ***********************************/ -- cgit v1.2.3 From e47e343a1c579f1d92b473d1517c7de58ffb71cc Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Thu, 23 Oct 2014 20:32:31 -0400 Subject: LMDB blockchain: remove outputs and spent keys --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 63 +++++++++++++++++++++++ 1 file changed, 63 insertions(+) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index 23c5fd4fc..5fd9f3d3d 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -249,6 +249,58 @@ void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_ou void BlockchainLMDB::remove_output(const tx_out& tx_output) { check_open(); + + MDB_val k; + MDB_val v; + + blobdata b; + t_serializable_object_to_blob(tx_output, b); + k.mv_size = b.size(); + k.mv_data = &b; + + if (mdb_get(*m_write_txn, m_output_gindices, &k, &v)) + { + LOG_PRINT_L1("Attempting to remove output that does not exist"); + throw OUTPUT_DNE("Attempting to remove output that does not exist"); + } + + uint64_t gindex = *(uint64_t*)v.mv_data; + + auto result = mdb_del(*m_write_txn, m_output_gindices, &k, NULL); + if (result != 0 && result != MDB_NOTFOUND) + { + LOG_PRINT_L1("Error adding removal of output global index to db transaction"); + throw DB_ERROR("Error adding removal of output global index to db transaction"); + } + + result = mdb_del(*m_write_txn, m_output_indices, &v, NULL); + if (result != 0 && result != MDB_NOTFOUND) + { + LOG_PRINT_L1("Error adding removal of output tx index to db transaction"); + throw DB_ERROR("Error adding removal of output tx index to db transaction"); + } + + result = mdb_del(*m_write_txn, m_output_txs, &v, NULL); + if (result != 0 && result != MDB_NOTFOUND) + { + LOG_PRINT_L1("Error adding removal of output tx hash to db transaction"); + throw DB_ERROR("Error adding removal of output tx hash to db transaction"); + } + + result = mdb_del(*m_write_txn, m_output_amounts, &v, NULL); + if (result != 0 && result != MDB_NOTFOUND) + { + LOG_PRINT_L1("Error adding removal of output amount to db transaction"); + throw DB_ERROR("Error adding removal of output amount to db transaction"); + } + + result = mdb_del(*m_write_txn, m_outputs, &v, NULL); + if (result != 0 && result != MDB_NOTFOUND) + { + LOG_PRINT_L1("Error adding removal of output to db transaction"); + throw DB_ERROR("Error adding removal of output to db transaction"); + } + } void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image) @@ -280,6 +332,17 @@ void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image) void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image) { check_open(); + + crypto::key_image key_cpy = k_image; + MDB_val k; + k.mv_size = sizeof(crypto::key_image); + k.mv_data = &key_cpy; + auto result = mdb_del(*m_write_txn, m_spent_keys, &val_key, NULL); + if (result != 0 && result != MDB_NOTFOUND) + { + LOG_PRINT_L1("Error adding removal of key image to db transaction"); + throw DB_ERROR("Error adding removal of key image to db transaction"); + } } void BlockchainLMDB::check_open() -- cgit v1.2.3 From d8c570b5888bed6c8d4bc7fa21f7cf6372eb6e6b Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Mon, 27 Oct 2014 20:45:33 -0400 Subject: All LMDB BlockchainDB implemented, not tested All of the functionality for the LMDB implementation of BlockchainDB is implemented, but only what is in tests/unit_tests/BlockchainDB.cpp has been tested. This is basically add a block, see if you can get the block and a tx from the block. More tests should be added at some point. --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 998 +++++++++++++++++++--- src/cryptonote_core/BlockchainDB_impl/db_lmdb.h | 43 + 2 files changed, 903 insertions(+), 138 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index 5fd9f3d3d..218bd4b63 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -32,18 +32,56 @@ #include // memcpy #include "cryptonote_core/cryptonote_format_utils.h" +#include "crypto/crypto.h" -namespace cryptonote +namespace +{ + +// cursor needs to be closed when it goes out of scope, +// this helps if the function using it throws +struct lmdb_cur { + lmdb_cur(MDB_txn* txn, MDB_dbi dbi) + { + if (mdb_cursor_open(txn, dbi, &m_cur)) + { + LOG_PRINT_L0("Error opening db cursor"); + throw cryptonote::DB_ERROR("Error opening db cursor"); + } + done = false; + } + + ~lmdb_cur() { close(); } + + operator MDB_cursor*() { return m_cur; } + operator MDB_cursor**() { return &m_cur; } + + void close() + { + if (!done) + { + mdb_cursor_close(m_cur); + done = true; + } + } + + MDB_cursor* m_cur; + bool done; +}; const char* LMDB_BLOCKS = "blocks"; +const char* LMDB_BLOCK_TIMESTAMPS = "block_timestamps"; +const char* LMDB_BLOCK_HEIGHTS = "block_heights"; const char* LMDB_BLOCK_HASHES = "block_hashes"; const char* LMDB_BLOCK_SIZES = "block_sizes"; const char* LMDB_BLOCK_DIFFS = "block_diffs"; const char* LMDB_BLOCK_COINS = "block_coins"; + const char* LMDB_TXS = "txs"; +const char* LMDB_TX_UNLOCKS = "tx_unlocks"; const char* LMDB_TX_HEIGHTS = "tx_heights"; const char* LMDB_TX_OUTPUTS = "tx_outputs"; + const char* LMDB_OUTPUT_TXS = "output_txs"; const char* LMDB_OUTPUT_INDICES = "output_indices"; const char* LMDB_OUTPUT_AMOUNTS = "output_amounts"; @@ -51,6 +89,20 @@ const char* LMDB_OUTPUTS = "outputs"; const char* LMDB_OUTPUT_GINDICES = "output_gindices"; const char* LMDB_SPENT_KEYS = "spent_keys"; +inline void lmdb_db_open(MDB_txn* txn, const char* name, int flags, MDB_dbi& dbi, const std::string& error_string) +{ + if (mdb_dbi_open(txn, name, flags, &dbi)) + { + LOG_PRINT_L0(error_string); + throw cryptonote::DB_OPEN_FAILURE(error_string.c_str()); + } +} + +} // anonymous namespace + +namespace cryptonote +{ + void BlockchainLMDB::add_block( const block& blk , const size_t& block_size , const difficulty_type& cumulative_difficulty @@ -65,7 +117,7 @@ void BlockchainLMDB::add_block( const block& blk val_h.mv_data = &h; MDB_val unused; - if (mdb_get(*m_write_txn, m_block_hashes, &val_h, &unused) == 0) + if (mdb_get(*m_write_txn, m_block_heights, &val_h, &unused) == 0) { LOG_PRINT_L1("Attempting to add block that's already in the db"); throw BLOCK_EXISTS("Attempting to add block that's already in the db"); @@ -79,7 +131,7 @@ void BlockchainLMDB::add_block( const block& blk parent_key.mv_data = &parent; MDB_val parent_h; - if (mdb_get(*m_write_txn, m_block_hashes, &parent_key, &parent_h)) + if (mdb_get(*m_write_txn, m_block_heights, &parent_key, &parent_h)) { LOG_PRINT_L0("Failed to get top block hash to check for new block's parent"); throw DB_ERROR("Failed to get top block hash to check for new block's parent"); @@ -122,10 +174,43 @@ void BlockchainLMDB::add_block( const block& blk throw DB_ERROR("Failed to add block size to db transaction"); } - if (mdb_put(*m_write_txn, m_block_hashes, &val_h, &key, 0)) + uint64_t time_cpy = blk.timestamp; + sz.mv_size = sizeof(time_cpy); + sz.mv_data = &time_cpy; + if (mdb_put(*m_write_txn, m_block_timestamps, &key, &sz, 0)) { - LOG_PRINT_L0("Failed to add block size to db transaction"); - throw DB_ERROR("Failed to add block size to db transaction"); + LOG_PRINT_L0("Failed to add block timestamp to db transaction"); + throw DB_ERROR("Failed to add block timestamp to db transaction"); + } + + difficulty_type diff_cpy = cumulative_difficulty; + sz.mv_size = sizeof(cumulative_difficulty); + sz.mv_data = &diff_cpy; + if (mdb_put(*m_write_txn, m_block_diffs, &key, &sz, 0)) + { + LOG_PRINT_L0("Failed to add block cumulative difficulty to db transaction"); + throw DB_ERROR("Failed to add block cumulative difficulty to db transaction"); + } + + uint64_t coins_cpy = coins_generated; + sz.mv_size = sizeof(coins_generated); + sz.mv_data = &coins_cpy; + if (mdb_put(*m_write_txn, m_block_coins, &key, &sz, 0)) + { + LOG_PRINT_L0("Failed to add block total generated coins to db transaction"); + throw DB_ERROR("Failed to add block total generated coins to db transaction"); + } + + if (mdb_put(*m_write_txn, m_block_heights, &val_h, &key, 0)) + { + LOG_PRINT_L0("Failed to add block height by hash to db transaction"); + throw DB_ERROR("Failed to add block height by hash to db transaction"); + } + + if (mdb_put(*m_write_txn, m_block_hashes, &key, &val_h, 0)) + { + LOG_PRINT_L0("Failed to add block hash to db transaction"); + throw DB_ERROR("Failed to add block hash to db transaction"); } } @@ -133,6 +218,54 @@ void BlockchainLMDB::add_block( const block& blk void BlockchainLMDB::remove_block() { check_open(); + + MDB_val k; + uint64_t height = m_height - 1; + k.mv_size = sizeof(uint64_t); + k.mv_data = &height; + + MDB_val h; + if (mdb_get(*m_write_txn, m_block_hashes, &k, &h)) + { + LOG_PRINT_L1("Attempting to remove block that's not in the db"); + throw BLOCK_DNE("Attempting to remove block that's not in the db"); + } + + if (mdb_del(*m_write_txn, m_blocks, &k, NULL)) + { + LOG_PRINT_L1("Failed to add removal of block to db transaction"); + throw DB_ERROR("Failed to add removal of block to db transaction"); + } + + if (mdb_del(*m_write_txn, m_block_sizes, &k, NULL)) + { + LOG_PRINT_L1("Failed to add removal of block size to db transaction"); + throw DB_ERROR("Failed to add removal of block size to db transaction"); + } + + if (mdb_del(*m_write_txn, m_block_diffs, &k, NULL)) + { + LOG_PRINT_L1("Failed to add removal of block cumulative difficulty to db transaction"); + throw DB_ERROR("Failed to add removal of block cumulative difficulty to db transaction"); + } + + if (mdb_del(*m_write_txn, m_block_coins, &k, NULL)) + { + LOG_PRINT_L1("Failed to add removal of block total generated coins to db transaction"); + throw DB_ERROR("Failed to add removal of block total generated coins to db transaction"); + } + + if (mdb_del(*m_write_txn, m_block_heights, &h, NULL)) + { + LOG_PRINT_L1("Failed to add removal of block height by hash to db transaction"); + throw DB_ERROR("Failed to add removal of block height by hash to db transaction"); + } + + if (mdb_del(*m_write_txn, m_block_hashes, &k, NULL)) + { + LOG_PRINT_L1("Failed to add removal of block hash to db transaction"); + throw DB_ERROR("Failed to add removal of block hash to db transaction"); + } } void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx) @@ -174,11 +307,54 @@ void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const tr LOG_PRINT_L0("Failed to add tx block height to db transaction"); throw DB_ERROR("Failed to add tx block height to db transaction"); } + + uint64_t unlock_cpy = tx.unlock_time; + height.mv_size = sizeof(unlock_cpy); + height.mv_data = &unlock_cpy; + if (mdb_put(*m_write_txn, m_tx_unlocks, &val_h, &height, 0)) + { + LOG_PRINT_L0("Failed to add tx unlock time to db transaction"); + throw DB_ERROR("Failed to add tx unlock time to db transaction"); + } } void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash) { check_open(); + + crypto::hash h = tx_hash; + MDB_val val_h; + val_h.mv_size = sizeof(crypto::hash); + val_h.mv_data = &h; + + MDB_val unused; + if (mdb_get(*m_write_txn, m_txs, &val_h, &unused)) + { + LOG_PRINT_L1("Attempting to remove transaction that isn't in the db"); + throw TX_DNE("Attempting to remove transaction that isn't in the db"); + } + + if (mdb_del(*m_write_txn, m_txs, &val_h, NULL)) + { + LOG_PRINT_L1("Failed to add removal of tx to db transaction"); + throw DB_ERROR("Failed to add removal of tx to db transaction"); + } + if (mdb_del(*m_write_txn, m_tx_unlocks, &val_h, NULL)) + { + LOG_PRINT_L1("Failed to add removal of tx unlock time to db transaction"); + throw DB_ERROR("Failed to add removal of tx unlock time to db transaction"); + } + if (mdb_del(*m_write_txn, m_tx_heights, &val_h, NULL)) + { + LOG_PRINT_L1("Failed to add removal of tx block height to db transaction"); + throw DB_ERROR("Failed to add removal of tx block height to db transaction"); + } + if (mdb_del(*m_write_txn, m_tx_outputs, &val_h, NULL)) + { + LOG_PRINT_L1("Failed to add removal of tx outputs to db transaction"); + throw DB_ERROR("Failed to add removal of tx outputs to db transaction"); + } + } void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index) @@ -222,15 +398,8 @@ void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_ou throw DB_ERROR("Failed to add output amount to db transaction"); } - blobdata b; - t_serializable_object_to_blob(tx_output, b); - /* - * use this later to deserialize - std::stringstream ss; - ss << tx_blob; - binary_archive ba(ss); - bool r = ::serialization::serialize(ba, tx); - */ + blobdata b = output_to_blob(tx_output); + v.mv_size = b.size(); v.mv_data = &b; if (mdb_put(*m_write_txn, m_outputs, &k, &v, 0)) @@ -337,7 +506,7 @@ void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image) MDB_val k; k.mv_size = sizeof(crypto::key_image); k.mv_data = &key_cpy; - auto result = mdb_del(*m_write_txn, m_spent_keys, &val_key, NULL); + auto result = mdb_del(*m_write_txn, m_spent_keys, &k, NULL); if (result != 0 && result != MDB_NOTFOUND) { LOG_PRINT_L1("Error adding removal of key image to db transaction"); @@ -345,6 +514,38 @@ void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image) } } +blobdata BlockchainLMDB::output_to_blob(const tx_out& output) +{ + blobdata b; + if (!t_serializable_object_to_blob(output, b)) + { + LOG_PRINT_L1("Error serializing output to blob"); + throw DB_ERROR("Error serializing output to blob"); + } + return b; +} + +tx_out BlockchainLMDB::output_from_blob(const blobdata& blob) +{ + std::stringstream ss; + ss << blob; + binary_archive ba(ss); + tx_out o; + + if (!(::serialization::serialize(ba, o))) + { + LOG_PRINT_L1("Error deserializing tx output blob"); + throw DB_ERROR("Error deserializing tx output blob"); + } + + return o; +} + +uint64_t BlockchainLMDB::get_output_global_index(const uint64_t& amount, const uint64_t& index) +{ + return 0; +} + void BlockchainLMDB::check_open() { if (!m_open) @@ -399,17 +600,17 @@ void BlockchainLMDB::open(const std::string& filename) if (mdb_env_create(&m_env)) { LOG_PRINT_L0("Failed to create lmdb environment"); - throw DB_OPEN_FAILURE("Failed to create lmdb environment"); + throw DB_ERROR("Failed to create lmdb environment"); } - if (mdb_env_set_maxdbs(m_env, 15)) + if (mdb_env_set_maxdbs(m_env, 20)) { LOG_PRINT_L0("Failed to set max number of dbs"); - throw DB_OPEN_FAILURE("Failed to set max number of dbs"); + throw DB_ERROR("Failed to set max number of dbs"); } if (mdb_env_open(m_env, filename.c_str(), 0, 0664)) { LOG_PRINT_L0("Failed to open lmdb environment"); - throw DB_OPEN_FAILURE("Failed to open lmdb environment"); + throw DB_ERROR("Failed to open lmdb environment"); } // get a read/write MDB_txn @@ -422,80 +623,27 @@ void BlockchainLMDB::open(const std::string& filename) // open necessary databases, and set properties as needed // uses macros to avoid having to change things too many places - if (mdb_dbi_open(txn, LMDB_BLOCKS, MDB_INTEGERKEY | MDB_CREATE, &m_blocks)) - { - LOG_PRINT_L0("Failed to open db handle for m_blocks"); - throw DB_OPEN_FAILURE( "Failed to open db handle for m_blocks" ); - } + lmdb_db_open(txn, LMDB_BLOCKS, MDB_INTEGERKEY | MDB_CREATE, m_blocks, "Failed to open db handle for m_blocks"); - if (mdb_dbi_open(txn, LMDB_BLOCK_HASHES, MDB_INTEGERKEY | MDB_CREATE, &m_block_hashes)) - { - LOG_PRINT_L0("Failed to open db handle for m_block_hashes"); - throw DB_OPEN_FAILURE( "Failed to open db handle for m_block_hashes" ); - } - if (mdb_dbi_open(txn, LMDB_BLOCK_SIZES, MDB_INTEGERKEY | MDB_CREATE, &m_block_sizes)) - { - LOG_PRINT_L0("Failed to open db handle for m_block_sizes"); - throw DB_OPEN_FAILURE( "Failed to open db handle for m_block_sizes" ); - } - if (mdb_dbi_open(txn, LMDB_BLOCK_DIFFS, MDB_INTEGERKEY | MDB_CREATE, &m_block_diffs)) - { - LOG_PRINT_L0("Failed to open db handle for m_block_diffs"); - throw DB_OPEN_FAILURE( "Failed to open db handle for m_block_diffs" ); - } - if (mdb_dbi_open(txn, LMDB_BLOCK_COINS, MDB_INTEGERKEY | MDB_CREATE, &m_block_coins)) - { - LOG_PRINT_L0("Failed to open db handle for m_block_coins"); - throw DB_OPEN_FAILURE( "Failed to open db handle for m_block_coins" ); - } + lmdb_db_open(txn, LMDB_BLOCK_TIMESTAMPS, MDB_INTEGERKEY | MDB_CREATE, m_block_timestamps, "Failed to open db handle for m_block_timestamps"); + lmdb_db_open(txn, LMDB_BLOCK_HEIGHTS, MDB_CREATE, m_block_heights, "Failed to open db handle for m_block_heights"); + lmdb_db_open(txn, LMDB_BLOCK_HASHES, MDB_INTEGERKEY | MDB_CREATE, m_block_hashes, "Failed to open db handle for m_block_hashes"); + lmdb_db_open(txn, LMDB_BLOCK_SIZES, MDB_INTEGERKEY | MDB_CREATE, m_block_sizes, "Failed to open db handle for m_block_sizes"); + lmdb_db_open(txn, LMDB_BLOCK_DIFFS, MDB_INTEGERKEY | MDB_CREATE, m_block_diffs, "Failed to open db handle for m_block_diffs"); + lmdb_db_open(txn, LMDB_BLOCK_COINS, MDB_INTEGERKEY | MDB_CREATE, m_block_coins, "Failed to open db handle for m_block_coins"); - if (mdb_dbi_open(txn, LMDB_TXS, MDB_CREATE, &m_txs)) - { - LOG_PRINT_L0("Failed to open db handle for m_txs"); - throw DB_OPEN_FAILURE( "Failed to open db handle for m_txs" ); - } - if (mdb_dbi_open(txn, LMDB_TX_HEIGHTS, MDB_CREATE, &m_tx_heights)) - { - LOG_PRINT_L0("Failed to open db handle for m_tx_heights"); - throw DB_OPEN_FAILURE( "Failed to open db handle for m_tx_heights" ); - } - if (mdb_dbi_open(txn, LMDB_TX_OUTPUTS, MDB_DUPSORT | MDB_CREATE, &m_tx_outputs)) - { - LOG_PRINT_L0("Failed to open db handle for m_tx_outputs"); - throw DB_OPEN_FAILURE( "Failed to open db handle for m_tx_outputs" ); - } + lmdb_db_open(txn, LMDB_TXS, MDB_CREATE, m_txs, "Failed to open db handle for m_txs"); + lmdb_db_open(txn, LMDB_TX_UNLOCKS, MDB_CREATE, m_tx_unlocks, "Failed to open db handle for m_tx_unlocks"); + lmdb_db_open(txn, LMDB_TX_HEIGHTS, MDB_CREATE, m_tx_heights, "Failed to open db handle for m_tx_heights"); + lmdb_db_open(txn, LMDB_TX_OUTPUTS, MDB_DUPSORT | MDB_CREATE, m_tx_outputs, "Failed to open db handle for m_tx_outputs"); - if (mdb_dbi_open(txn, LMDB_OUTPUT_TXS, MDB_INTEGERKEY | MDB_CREATE, &m_output_txs)) - { - LOG_PRINT_L0("Failed to open db handle for m_output_txs"); - throw DB_OPEN_FAILURE( "Failed to open db handle for m_output_txs" ); - } - if (mdb_dbi_open(txn, LMDB_OUTPUT_INDICES, MDB_INTEGERKEY | MDB_CREATE, &m_output_indices)) - { - LOG_PRINT_L0("Failed to open db handle for m_output_indices"); - throw DB_OPEN_FAILURE( "Failed to open db handle for m_output_indices" ); - } - if (mdb_dbi_open(txn, LMDB_OUTPUT_GINDICES, MDB_CREATE, &m_output_gindices)) - { - LOG_PRINT_L0("Failed to open db handle for m_output_gindices"); - throw DB_OPEN_FAILURE( "Failed to open db handle for m_output_gindices" ); - } - if (mdb_dbi_open(txn, LMDB_OUTPUT_AMOUNTS, MDB_INTEGERKEY | MDB_DUPSORT | MDB_CREATE, &m_output_amounts)) - { - LOG_PRINT_L0("Failed to open db handle for m_output_amounts"); - throw DB_OPEN_FAILURE( "Failed to open db handle for m_output_amounts" ); - } - if (mdb_dbi_open(txn, LMDB_OUTPUTS, MDB_INTEGERKEY | MDB_CREATE, &m_outputs)) - { - LOG_PRINT_L0("Failed to open db handle for m_outputs"); - throw DB_OPEN_FAILURE( "Failed to open db handle for m_outputs" ); - } + lmdb_db_open(txn, LMDB_OUTPUT_TXS, MDB_INTEGERKEY | MDB_CREATE, m_output_txs, "Failed to open db handle for m_output_txs"); + lmdb_db_open(txn, LMDB_OUTPUT_INDICES, MDB_INTEGERKEY | MDB_CREATE, m_output_indices, "Failed to open db handle for m_output_indices"); + lmdb_db_open(txn, LMDB_OUTPUT_GINDICES, MDB_CREATE, m_output_gindices, "Failed to open db handle for m_output_gindices"); + lmdb_db_open(txn, LMDB_OUTPUT_AMOUNTS, MDB_INTEGERKEY | MDB_DUPSORT | MDB_CREATE, m_output_amounts, "Failed to open db handle for m_output_amounts"); + lmdb_db_open(txn, LMDB_OUTPUTS, MDB_INTEGERKEY | MDB_CREATE, m_outputs, "Failed to open db handle for m_outputs"); - if (mdb_dbi_open(txn, LMDB_SPENT_KEYS, MDB_CREATE, &m_spent_keys)) - { - LOG_PRINT_L0("Failed to open db handle for m_outputs"); - throw DB_OPEN_FAILURE( "Failed to open db handle for m_outputs" ); - } + lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_CREATE, m_spent_keys, "Failed to open db handle for m_outputs"); // get and keep current height MDB_stat db_stats; @@ -528,14 +676,18 @@ void BlockchainLMDB::create(const std::string& filename) void BlockchainLMDB::close() { + // FIXME: not yet thread safe!!! Use with care. + mdb_env_close(m_env); } void BlockchainLMDB::sync() { + // LMDB documentation leads me to believe this is unnecessary } void BlockchainLMDB::reset() { + // TODO: this } std::vector BlockchainLMDB::get_filenames() @@ -553,13 +705,14 @@ std::vector BlockchainLMDB::get_filenames() return filenames; } - +// TODO: this? bool BlockchainLMDB::lock() { check_open(); return false; } +// TODO: this? void BlockchainLMDB::unlock() { check_open(); @@ -583,7 +736,7 @@ bool BlockchainLMDB::block_exists(const crypto::hash& h) key.mv_data = &key_cpy; MDB_val result; - auto get_result = mdb_get(txn, m_block_hashes, &key, &result); + auto get_result = mdb_get(txn, m_block_heights, &key, &result); if (get_result == MDB_NOTFOUND) { txn.commit(); @@ -604,6 +757,13 @@ block BlockchainLMDB::get_block(const crypto::hash& h) { check_open(); + return get_block_from_height(get_block_height(h)); +} + +uint64_t BlockchainLMDB::get_block_height(const crypto::hash& h) +{ + check_open(); + txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) { @@ -617,36 +777,28 @@ block BlockchainLMDB::get_block(const crypto::hash& h) key.mv_data = &key_cpy; MDB_val result; - auto get_result = mdb_get(txn, m_block_hashes, &key, &result); + auto get_result = mdb_get(txn, m_block_heights, &key, &result); if (get_result == MDB_NOTFOUND) { - LOG_PRINT_L1("Attempted to retrieve non-existent block"); - throw BLOCK_DNE("Attempted to retrieve non-existent block"); + LOG_PRINT_L1("Attempted to retrieve non-existent block height"); + throw BLOCK_DNE("Attempted to retrieve non-existent block height"); } else if (get_result) { - LOG_PRINT_L0("Error attempting to retrieve a block index from the db"); - throw DB_ERROR("Error attempting to retrieve a block index from the db"); + LOG_PRINT_L0("Error attempting to retrieve a block height from the db"); + throw DB_ERROR("Error attempting to retrieve a block height from the db"); } txn.commit(); - - uint64_t index = *(uint64_t*)result.mv_data; - - return get_block_from_height(index); -} - -uint64_t BlockchainLMDB::get_block_height(const crypto::hash& h) -{ - check_open(); - return 0; + return *(uint64_t*)result.mv_data; } block_header BlockchainLMDB::get_block_header(const crypto::hash& h) { check_open(); - block_header bh; - return bh; + + // block_header object is automatically cast from block object + return get_block(h); } block BlockchainLMDB::get_block_from_height(const uint64_t& height) @@ -695,50 +847,202 @@ block BlockchainLMDB::get_block_from_height(const uint64_t& height) uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) { check_open(); - return 0; + + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + { + LOG_PRINT_L0("Failed to create a transaction for the db"); + throw DB_ERROR("Failed to create a transaction for the db"); + } + + uint64_t height_cpy = height; + MDB_val key; + key.mv_size = sizeof(uint64_t); + key.mv_data = &height_cpy; + MDB_val result; + auto get_result = mdb_get(txn, m_block_timestamps, &key, &result); + if (get_result == MDB_NOTFOUND) + { + LOG_PRINT_L0("Attempted to get timestamp from height " << height << ", but no such timestamp exists"); + throw DB_ERROR("Attempt to get timestamp from height failed -- timestamp not in db"); + } + else if (get_result) + { + LOG_PRINT_L0("Error attempting to retrieve a timestamp from the db"); + throw DB_ERROR("Error attempting to retrieve a timestamp from the db"); + } + + txn.commit(); + return *(uint64_t*)result.mv_data; } uint64_t BlockchainLMDB::get_top_block_timestamp() { check_open(); - return 0; + + // if no blocks, return 0 + if (m_height == 0) + { + return 0; + } + + return get_block_timestamp(m_height - 1); } size_t BlockchainLMDB::get_block_size(const uint64_t& height) { check_open(); - return 0; + + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + { + LOG_PRINT_L0("Failed to create a transaction for the db"); + throw DB_ERROR("Failed to create a transaction for the db"); + } + + uint64_t height_cpy = height; + MDB_val key; + key.mv_size = sizeof(uint64_t); + key.mv_data = &height_cpy; + MDB_val result; + auto get_result = mdb_get(txn, m_block_sizes, &key, &result); + if (get_result == MDB_NOTFOUND) + { + LOG_PRINT_L0("Attempted to get block size from height " << height << ", but no such block size exists"); + throw DB_ERROR("Attempt to get block size from height failed -- block size not in db"); + } + else if (get_result) + { + LOG_PRINT_L0("Error attempting to retrieve a block size from the db"); + throw DB_ERROR("Error attempting to retrieve a block size from the db"); + } + + txn.commit(); + return *(size_t*)result.mv_data; } difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& height) { check_open(); - return 0; + + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + { + LOG_PRINT_L0("Failed to create a transaction for the db"); + throw DB_ERROR("Failed to create a transaction for the db"); + } + + uint64_t height_cpy = height; + MDB_val key; + key.mv_size = sizeof(uint64_t); + key.mv_data = &height_cpy; + MDB_val result; + auto get_result = mdb_get(txn, m_block_diffs, &key, &result); + if (get_result == MDB_NOTFOUND) + { + LOG_PRINT_L0("Attempted to get cumulative difficulty from height " << height << ", but no such cumulative difficulty exists"); + throw DB_ERROR("Attempt to get cumulative difficulty from height failed -- block size not in db"); + } + else if (get_result) + { + LOG_PRINT_L0("Error attempting to retrieve a cumulative difficulty from the db"); + throw DB_ERROR("Error attempting to retrieve a cumulative difficulty from the db"); + } + + txn.commit(); + return *(difficulty_type*)result.mv_data; } difficulty_type BlockchainLMDB::get_block_difficulty(const uint64_t& height) { check_open(); - return 0; + + difficulty_type diff1 = 0; + difficulty_type diff2 = 0; + + diff1 = get_block_cumulative_difficulty(height); + if (height != 0) + { + diff2 = get_block_cumulative_difficulty(height - 1); + } + + return diff1 - diff2; } uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& height) { check_open(); - return 0; + + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + { + LOG_PRINT_L0("Failed to create a transaction for the db"); + throw DB_ERROR("Failed to create a transaction for the db"); + } + + uint64_t height_cpy = height; + MDB_val key; + key.mv_size = sizeof(uint64_t); + key.mv_data = &height_cpy; + MDB_val result; + auto get_result = mdb_get(txn, m_block_coins, &key, &result); + if (get_result == MDB_NOTFOUND) + { + LOG_PRINT_L0("Attempted to get total generated coins from height " << height << ", but no such total generated coins exists"); + throw DB_ERROR("Attempt to get total generated coins from height failed -- block size not in db"); + } + else if (get_result) + { + LOG_PRINT_L0("Error attempting to retrieve a total generated coins from the db"); + throw DB_ERROR("Error attempting to retrieve a total generated coins from the db"); + } + + txn.commit(); + return *(uint64_t*)result.mv_data; } crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height) { check_open(); - crypto::hash h; - return h; -} -std::vector BlockchainLMDB::get_blocks_range(const uint64_t& h1, const uint64_t& h2) -{ + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + { + LOG_PRINT_L0("Failed to create a transaction for the db"); + throw DB_ERROR("Failed to create a transaction for the db"); + } + + uint64_t height_cpy = height; + MDB_val key; + key.mv_size = sizeof(uint64_t); + key.mv_data = &height_cpy; + MDB_val result; + auto get_result = mdb_get(txn, m_block_hashes, &key, &result); + if (get_result == MDB_NOTFOUND) + { + LOG_PRINT_L0("Attempted to get hash from height " << height << ", but no such hash exists"); + throw DB_ERROR("Attempt to get hash from height failed -- block size not in db"); + } + else if (get_result) + { + LOG_PRINT_L0("Error attempting to retrieve a block hash from the db"); + throw DB_ERROR("Error attempting to retrieve a block hash from the db"); + } + + txn.commit(); + return *(crypto::hash*)result.mv_data; +} + +std::vector BlockchainLMDB::get_blocks_range(const uint64_t& h1, const uint64_t& h2) +{ check_open(); std::vector v; + + for (uint64_t height = h1; height <= h2; ++height) + { + v.push_back(get_block_from_height(height)); + } + return v; } @@ -746,19 +1050,35 @@ std::vector BlockchainLMDB::get_hashes_range(const uint64_t& h1, c { check_open(); std::vector v; + + for (uint64_t height = h1; height <= h2; ++height) + { + v.push_back(get_block_hash_from_height(height)); + } + return v; } crypto::hash BlockchainLMDB::top_block_hash() { check_open(); - crypto::hash h; - return h; + if (m_height != 0) + { + return get_block_hash_from_height(m_height - 1); + } + + return null_hash; } block BlockchainLMDB::get_top_block() { check_open(); + + if (m_height != 0) + { + return get_block_from_height(m_height - 1); + } + block b; return b; } @@ -774,79 +1094,481 @@ uint64_t BlockchainLMDB::height() bool BlockchainLMDB::tx_exists(const crypto::hash& h) { check_open(); - return false; + + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + { + LOG_PRINT_L0("Failed to create a transaction for the db"); + throw DB_ERROR("Failed to create a transaction for the db"); + } + + crypto::hash key_cpy = h; + MDB_val key; + key.mv_size = sizeof(crypto::hash); + key.mv_data = &key_cpy; + + MDB_val result; + auto get_result = mdb_get(txn, m_txs, &key, &result); + if (get_result == MDB_NOTFOUND) + { + txn.commit(); + LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << "not found in db"); + return false; + } + else if (get_result) + { + LOG_PRINT_L0("DB error attempting to fetch transaction from hash"); + throw DB_ERROR("DB error attempting to fetch transaction from hash"); + } + + return true; } uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) { check_open(); - return 0; + + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + { + LOG_PRINT_L0("Failed to create a transaction for the db"); + throw DB_ERROR("Failed to create a transaction for the db"); + } + + crypto::hash key_cpy = h; + MDB_val key; + key.mv_size = sizeof(crypto::hash); + key.mv_data = &key_cpy; + + MDB_val result; + auto get_result = mdb_get(txn, m_tx_unlocks, &key, &result); + if (get_result == MDB_NOTFOUND) + { + LOG_PRINT_L1("tx unlock time with hash " << epee::string_tools::pod_to_hex(h) << "not found in db"); + throw TX_DNE("Attempting to get unlock time for tx, but tx not in db"); + } + else if (get_result) + { + LOG_PRINT_L0("DB error attempting to fetch tx unlock time from hash"); + throw DB_ERROR("DB error attempting to fetch tx unlock time from hash"); + } + + return *(uint64_t*)result.mv_data; } transaction BlockchainLMDB::get_tx(const crypto::hash& h) { check_open(); - transaction t; - return t; + + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + { + LOG_PRINT_L0("Failed to create a transaction for the db"); + throw DB_ERROR("Failed to create a transaction for the db"); + } + + crypto::hash key_cpy = h; + MDB_val key; + key.mv_size = sizeof(crypto::hash); + key.mv_data = &key_cpy; + + MDB_val result; + auto get_result = mdb_get(txn, m_txs, &key, &result); + if (get_result == MDB_NOTFOUND) + { + LOG_PRINT_L1("tx with hash " << epee::string_tools::pod_to_hex(h) << "not found in db"); + throw TX_DNE("Attempting to get tx, but tx not in db"); + } + else if (get_result) + { + LOG_PRINT_L0("DB error attempting to fetch tx from hash"); + throw DB_ERROR("DB error attempting to fetch tx from hash"); + } + + blobdata bd; + bd.assign(reinterpret_cast(result.mv_data), result.mv_size); + + transaction tx; + if (!parse_and_validate_tx_from_blob(bd, tx)) + { + LOG_PRINT_L0("Failed to parse tx from blob retrieved from the db"); + throw DB_ERROR("Failed to parse tx from blob retrieved from the db"); + } + + return tx; } uint64_t BlockchainLMDB::get_tx_count() { check_open(); - return 0; + + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + { + LOG_PRINT_L0("Failed to create a transaction for the db"); + throw DB_ERROR("Failed to create a transaction for the db"); + } + + MDB_stat db_stats; + if (mdb_stat(txn, m_txs, &db_stats)) + { + LOG_PRINT_L0("Failed to query m_txs"); + throw DB_ERROR("Failed to query m_txs"); + } + + txn.commit(); + + return db_stats.ms_entries; } std::vector BlockchainLMDB::get_tx_list(const std::vector& hlist) { check_open(); std::vector v; + + for (auto& h : hlist) + { + v.push_back(get_tx(h)); + } + return v; } uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) { check_open(); - return 0; + + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + { + LOG_PRINT_L0("Failed to create a transaction for the db"); + throw DB_ERROR("Failed to create a transaction for the db"); + } + + crypto::hash key_cpy = h; + MDB_val key; + key.mv_size = sizeof(crypto::hash); + key.mv_data = &key_cpy; + + MDB_val result; + auto get_result = mdb_get(txn, m_tx_heights, &key, &result); + if (get_result == MDB_NOTFOUND) + { + LOG_PRINT_L1("tx height with hash " << epee::string_tools::pod_to_hex(h) << "not found in db"); + throw TX_DNE("Attempting to get height for tx, but tx not in db"); + } + else if (get_result) + { + LOG_PRINT_L0("DB error attempting to fetch tx height from hash"); + throw DB_ERROR("DB error attempting to fetch tx height from hash"); + } + + return *(uint64_t*)result.mv_data; } +//FIXME: make sure the random method used here is appropriate uint64_t BlockchainLMDB::get_random_output(const uint64_t& amount) { check_open(); - return 0; + + uint64_t num_outputs = get_num_outputs(amount); + if (num_outputs == 0) + { + LOG_PRINT_L1("Attempting to get a random output for an amount, but none exist"); + throw OUTPUT_DNE("Attempting to get a random output for an amount, but none exist"); + } + + return crypto::rand() % num_outputs; } uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) { check_open(); - return 0; + + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + { + LOG_PRINT_L0("Failed to create a transaction for the db"); + throw DB_ERROR("Failed to create a transaction for the db"); + } + + lmdb_cur cur(txn, m_output_amounts); + + uint64_t amount_cpy = amount; + MDB_val k; + k.mv_size = sizeof(amount_cpy); + k.mv_data = &amount_cpy; + + MDB_val v; + auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + { + return 0; + } + else if (result) + { + LOG_PRINT_L0("DB error attempting to get number of outputs of an amount"); + throw DB_ERROR("DB error attempting to get number of outputs of an amount"); + } + + size_t num_elems = 0; + mdb_cursor_count(cur, &num_elems); + + txn.commit(); + + return num_elems; } crypto::public_key BlockchainLMDB::get_output_key(const uint64_t& amount, const uint64_t& index) { check_open(); - crypto::public_key pk; - return pk; + + uint64_t global = get_output_global_index(amount, index); + + tx_out output = get_output(global); + + if (output.target.type() != typeid(txout_to_key)) + { + LOG_PRINT_L1("Attempting to get public key for wrong type of output"); + throw DB_ERROR("Attempting to get public key for wrong type of output"); + } + + return boost::get(output.target).key; } tx_out BlockchainLMDB::get_output(const crypto::hash& h, const uint64_t& index) { check_open(); - tx_out o; - return o; + + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + { + LOG_PRINT_L0("Failed to create a transaction for the db"); + throw DB_ERROR("Failed to create a transaction for the db"); + } + + MDB_val k; + crypto::hash h_cpy = h; + k.mv_size = sizeof(h_cpy); + k.mv_data = &h_cpy; + + lmdb_cur cur(txn, m_tx_outputs); + + MDB_val v; + + auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + { + LOG_PRINT_L1("Attempting to get an output by tx hash and tx index, but output not found"); + throw OUTPUT_DNE("Attempting to get an output by tx hash and tx index, but output not found"); + } + else if (result) + { + LOG_PRINT_L0("DB error attempting to get an output"); + throw DB_ERROR("DB error attempting to get an output"); + } + + size_t num_elems = 0; + mdb_cursor_count(cur, &num_elems); + if (num_elems <= index) + { + LOG_PRINT_L1("Attempting to get an output by tx hash and tx index, but output not found"); + throw OUTPUT_DNE("Attempting to get an output by tx hash and tx index, but output not found"); + } + + mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); + + if (index != 0) + { + for (uint64_t i = 0; i < index; ++i) + { + mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); + } + } + + mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); + + blobdata b; + b = *(blobdata*)v.mv_data; + + cur.close(); + txn.commit(); + + return output_from_blob(b); +} + +tx_out BlockchainLMDB::get_output(const uint64_t& index) +{ + check_open(); + + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + { + LOG_PRINT_L0("Failed to create a transaction for the db"); + throw DB_ERROR("Failed to create a transaction for the db"); + } + + uint64_t index_cpy = index; + MDB_val k; + k.mv_size = sizeof(index_cpy); + k.mv_data = &index_cpy; + + MDB_val v; + auto get_result = mdb_get(txn, m_outputs, &k, &v); + if (get_result == MDB_NOTFOUND) + { + LOG_PRINT_L0("Attempting to get output by global index, but output does not exist"); + throw OUTPUT_DNE(); + } + else if (get_result) + { + LOG_PRINT_L0("Error attempting to retrieve an output from the db"); + throw DB_ERROR("Error attempting to retrieve an output from the db"); + } + + blobdata b = *(blobdata*)v.mv_data; + + return output_from_blob(b); } tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) { check_open(); - tx_out_index i; - return i; + + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + { + LOG_PRINT_L0("Failed to create a transaction for the db"); + throw DB_ERROR("Failed to create a transaction for the db"); + } + + lmdb_cur cur(txn, m_output_amounts); + + uint64_t amount_cpy = amount; + MDB_val k; + k.mv_size = sizeof(amount_cpy); + k.mv_data = &amount_cpy; + + MDB_val v; + + auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + { + LOG_PRINT_L1("Attempting to get an output index by amount and amount index, but output not found"); + throw OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found"); + } + else if (result) + { + LOG_PRINT_L0("DB error attempting to get an output"); + throw DB_ERROR("DB error attempting to get an output"); + } + + size_t num_elems = 0; + mdb_cursor_count(cur, &num_elems); + if (num_elems <= index) + { + LOG_PRINT_L1("Attempting to get an output index by amount and amount index, but output not found"); + throw OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found"); + } + + mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); + + if (index != 0) + { + for (uint64_t i = 0; i < index; ++i) + { + mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); + } + } + + mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); + + uint64_t glob_index = *(uint64_t*)v.mv_data; + + cur.close(); + + k.mv_size = sizeof(glob_index); + k.mv_data = &glob_index; + auto get_result = mdb_get(txn, m_output_txs, &k, &v); + if (get_result == MDB_NOTFOUND) + { + LOG_PRINT_L1("output with given index not in db"); + throw OUTPUT_DNE("output with given index not in db"); + } + else if (get_result) + { + LOG_PRINT_L0("DB error attempting to fetch output tx hash"); + throw DB_ERROR("DB error attempting to fetch output tx hash"); + } + + crypto::hash tx_hash = *(crypto::hash*)v.mv_data; + + get_result = mdb_get(txn, m_output_indices, &k, &v); + if (get_result == MDB_NOTFOUND) + { + LOG_PRINT_L1("output with given index not in db"); + throw OUTPUT_DNE("output with given index not in db"); + } + else if (get_result) + { + LOG_PRINT_L0("DB error attempting to fetch output tx index"); + throw DB_ERROR("DB error attempting to fetch output tx index"); + } + + txn.commit(); + + return tx_out_index(tx_hash, *(uint64_t *)v.mv_data); } std::vector BlockchainLMDB::get_tx_output_indices(const crypto::hash& h) { check_open(); - std::vector v; - return v; + std::vector index_vec; + + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + { + LOG_PRINT_L0("Failed to create a transaction for the db"); + throw DB_ERROR("Failed to create a transaction for the db"); + } + + MDB_val k; + crypto::hash h_cpy = h; + k.mv_size = sizeof(h_cpy); + k.mv_data = &h_cpy; + + lmdb_cur cur(txn, m_tx_outputs); + + MDB_val v; + + auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + { + LOG_PRINT_L1("Attempting to get an output by tx hash and tx index, but output not found"); + throw OUTPUT_DNE("Attempting to get an output by tx hash and tx index, but output not found"); + } + else if (result) + { + LOG_PRINT_L0("DB error attempting to get an output"); + throw DB_ERROR("DB error attempting to get an output"); + } + + size_t num_elems = 0; + mdb_cursor_count(cur, &num_elems); + + mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); + + for (uint64_t i = 0; i < num_elems; ++i) + { + mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); + mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); + index_vec.push_back(*(uint64_t *)v.mv_data); + } + + cur.close(); + txn.commit(); + + return index_vec; } bool BlockchainLMDB::has_key_image(const crypto::key_image& img) diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h index 7b012e4f6..a0e9c0a8c 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h @@ -26,6 +26,7 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "cryptonote_core/blockchain_db.h" +#include "cryptonote_protocol/blobdatatype.h" // for type blobdata #include @@ -149,6 +150,17 @@ public: virtual tx_out get_output(const crypto::hash& h, const uint64_t& index); + /** + * @brief get an output from its global index + * + * @param index global index of the output desired + * + * @return the output associated with the index. + * Will throw OUTPUT_DNE if not output has that global index. + * Will throw DB_ERROR if there is a non-specific LMDB error in fetching + */ + tx_out get_output(const uint64_t& index); + virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index); virtual std::vector get_tx_output_indices(const crypto::hash& h); @@ -183,17 +195,48 @@ private: virtual void remove_spent_key(const crypto::key_image& k_image); + /** + * @brief convert a tx output to a blob for storage + * + * @param output the output to convert + * + * @return the resultant blob + */ + blobdata output_to_blob(const tx_out& output); + + /** + * @brief convert a tx output blob to a tx output + * + * @param blob the blob to convert + * + * @return the resultant tx output + */ + tx_out output_from_blob(const blobdata& blob); + + /** + * @brief get the global index of the index-th output of the given amount + * + * @param amount the output amount + * @param index the index into the set of outputs of that amount + * + * @return the global index of the desired output + */ + uint64_t get_output_global_index(const uint64_t& amount, const uint64_t& index); + void check_open(); MDB_env* m_env; MDB_dbi m_blocks; + MDB_dbi m_block_heights; MDB_dbi m_block_hashes; + MDB_dbi m_block_timestamps; MDB_dbi m_block_sizes; MDB_dbi m_block_diffs; MDB_dbi m_block_coins; MDB_dbi m_txs; + MDB_dbi m_tx_unlocks; MDB_dbi m_tx_heights; MDB_dbi m_tx_outputs; -- cgit v1.2.3 From 74a1a89e27c719811a1fcc0578e4795802555225 Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Mon, 27 Oct 2014 23:44:45 -0400 Subject: Integrate BlockchainDB into cryptonote_core Probably needs more looking at -- lot of things were done...in a rushed sort of way. That said, it all builds and *should* be at least testable. update for rebase (warptangent 2015-01-04) fix conflicts with upstream CMakeLists.txt files src/CMakeLists.txt (remove edits from original commit) tests/CMakeLists.txt (remove edits from original commit) src/cryptonote_core/CMakeLists.txt (edit) - use blockchain db .cpp and .h files - add LMDB_LIBRARIES --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 1 + src/cryptonote_core/CMakeLists.txt | 7 ++++ src/cryptonote_core/blockchain.cpp | 39 +++++++++++++++++++---- src/cryptonote_core/blockchain.h | 10 ++++-- src/cryptonote_core/cryptonote_core.cpp | 6 ++-- src/cryptonote_core/cryptonote_core.h | 6 ++-- src/cryptonote_core/tx_pool.cpp | 4 +-- src/cryptonote_core/tx_pool.h | 9 ++---- 8 files changed, 58 insertions(+), 24 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index 218bd4b63..6275ae388 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -672,6 +672,7 @@ void BlockchainLMDB::open(const std::string& filename) // unused for now, create will happen on open if doesn't exist void BlockchainLMDB::create(const std::string& filename) { + throw DB_CREATE_FAILURE("create() is not implemented for this BlockchainDB, open() will create files if needed."); } void BlockchainLMDB::close() diff --git a/src/cryptonote_core/CMakeLists.txt b/src/cryptonote_core/CMakeLists.txt index 3abf93f3c..93c3cb51e 100644 --- a/src/cryptonote_core/CMakeLists.txt +++ b/src/cryptonote_core/CMakeLists.txt @@ -29,6 +29,9 @@ set(cryptonote_core_sources account.cpp blockchain_storage.cpp + blockchain.cpp + blockchain_db.cpp + BlockchainDB_impl/db_lmdb.cpp checkpoints.cpp checkpoints_create.cpp cryptonote_basic_impl.cpp @@ -45,6 +48,9 @@ set(cryptonote_core_private_headers account_boost_serialization.h blockchain_storage.h blockchain_storage_boost_serialization.h + blockchain.h + blockchain_db.h + BlockchainDB_impl/db_lmdb.h checkpoints.h checkpoints_create.h connection_context.h @@ -77,4 +83,5 @@ target_link_libraries(cryptonote_core ${Boost_FILESYSTEM_LIBRARY} ${Boost_SYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY} + ${LMDB_LIBRARIES} ${EXTRA_LIBRARIES}) diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index a35ce846c..0f5765281 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -32,10 +32,14 @@ #include #include #include +#include #include "include_base_utils.h" #include "cryptonote_basic_impl.h" +#include "tx_pool.h" #include "blockchain.h" +#include "cryptonote_core/blockchain_db.h" +#include "cryptonote_core/BlockchainDB_impl/db_lmdb.h" #include "cryptonote_format_utils.h" #include "cryptonote_boost_serialization.h" #include "cryptonote_config.h" @@ -62,10 +66,6 @@ DISABLE_VS_WARNINGS(4267) // TODO: initialize m_db with a concrete implementation of BlockchainDB Blockchain::Blockchain(tx_memory_pool& tx_pool):m_db(), m_tx_pool(tx_pool), m_current_block_cumul_sz_limit(0), m_is_in_checkpoint_zone(false), m_is_blockchain_storing(false) { - if (m_db == NULL) - { - throw new DB_ERROR("database pointer null in blockchain init"); - } } //------------------------------------------------------------------ //TODO: is this still needed? I don't think so - tewinget @@ -222,11 +222,23 @@ bool Blockchain::init(const std::string& config_folder, bool testnet) { CRITICAL_REGION_LOCAL(m_blockchain_lock); + m_db = new BlockchainLMDB(); + m_config_folder = config_folder; + m_testnet = testnet; + + boost::filesystem::path folder(m_config_folder); + + // append "testnet" directory as needed + if (testnet) + { + folder /= "testnet"; + } + LOG_PRINT_L0("Loading blockchain..."); //FIXME: update filename for BlockchainDB - const std::string filename = m_config_folder + "/" CRYPTONOTE_BLOCKCHAINDATA_FILENAME; + const std::string filename = folder.string(); try { m_db->open(filename); @@ -328,6 +340,8 @@ bool Blockchain::deinit() { LOG_PRINT_L0("There was an issue closing/storing the blockchain, shutting down now to prevent issues!"); } + + delete m_db; return true; } //------------------------------------------------------------------ @@ -1450,7 +1464,7 @@ uint64_t Blockchain::block_difficulty(uint64_t i) } //------------------------------------------------------------------ template -void Blockchain::get_blocks(const t_ids_container& block_ids, t_blocks_container& blocks, t_missed_container& missed_bs) +bool Blockchain::get_blocks(const t_ids_container& block_ids, t_blocks_container& blocks, t_missed_container& missed_bs) { CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -1464,11 +1478,16 @@ void Blockchain::get_blocks(const t_ids_container& block_ids, t_blocks_container { missed_bs.push_back(block_hash); } + catch (const std::exception& e) + { + return false; + } } + return true; } //------------------------------------------------------------------ template -void Blockchain::get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) +bool Blockchain::get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) { CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -1482,7 +1501,13 @@ void Blockchain::get_transactions(const t_ids_container& txs_ids, t_tx_container { missed_txs.push_back(tx_hash); } + //FIXME: is this the correct way to handle this? + catch (const std::exception& e) + { + return false; + } } + return true; } //------------------------------------------------------------------ void Blockchain::print_blockchain(uint64_t start_index, uint64_t end_index) diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index 884de122d..4024fa741 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -41,7 +41,6 @@ #include "syncobj.h" #include "string_tools.h" -#include "tx_pool.h" #include "cryptonote_basic.h" #include "common/util.h" #include "cryptonote_protocol/cryptonote_protocol_defs.h" @@ -55,6 +54,7 @@ namespace cryptonote { + class tx_memory_pool; /************************************************************************/ /* */ @@ -131,16 +131,19 @@ namespace cryptonote uint64_t block_difficulty(uint64_t i); template - void get_blocks(const t_ids_container& block_ids, t_blocks_container& blocks, t_missed_container& missed_bs); + bool get_blocks(const t_ids_container& block_ids, t_blocks_container& blocks, t_missed_container& missed_bs); template - void get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs); + bool get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs); //debug functions void print_blockchain(uint64_t start_index, uint64_t end_index); void print_blockchain_index(); void print_blockchain_outs(const std::string& file); + void set_enforce_dns_checkpoints(bool enforce) { } + bool update_checkpoints(const std::string& path, bool dns) { return true; } + private: typedef std::unordered_map blocks_by_id_index; typedef std::unordered_map transactions_container; @@ -175,6 +178,7 @@ namespace cryptonote checkpoints m_checkpoints; std::atomic m_is_in_checkpoint_zone; std::atomic m_is_blockchain_storing; + bool m_testnet; bool switch_to_alternative_blockchain(std::list& alt_chain, bool discard_disconnected_chain); block pop_block_from_blockchain(); diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index b8b5dc008..e2c533fe5 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -290,9 +290,9 @@ namespace cryptonote return false; } - if(!keeped_by_block && get_object_blobsize(tx) >= m_blockchain_storage.get_current_comulative_blocksize_limit() - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE) + if(!keeped_by_block && get_object_blobsize(tx) >= m_blockchain_storage.get_current_cumulative_blocksize_limit() - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE) { - LOG_PRINT_RED_L1("tx is too large " << get_object_blobsize(tx) << ", expected not bigger than " << m_blockchain_storage.get_current_comulative_blocksize_limit() - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE); + LOG_PRINT_RED_L1("tx is too large " << get_object_blobsize(tx) << ", expected not bigger than " << m_blockchain_storage.get_current_cumulative_blocksize_limit() - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE); return false; } @@ -577,7 +577,7 @@ namespace cryptonote m_starter_message_showed = true; } - m_store_blockchain_interval.do_call(boost::bind(&blockchain_storage::store_blockchain, &m_blockchain_storage)); + m_store_blockchain_interval.do_call(boost::bind(&Blockchain::store_blockchain, &m_blockchain_storage)); m_miner.on_idle(); m_mempool.on_idle(); return true; diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 748f2b665..8ee0d8a8d 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -39,7 +39,7 @@ #include "cryptonote_protocol/cryptonote_protocol_handler_common.h" #include "storages/portable_storage_template_helper.h" #include "tx_pool.h" -#include "blockchain_storage.h" +#include "blockchain.h" #include "miner.h" #include "connection_context.h" #include "cryptonote_core/cryptonote_stat_info.h" @@ -112,7 +112,7 @@ namespace cryptonote bool get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res); void pause_mine(); void resume_mine(); - blockchain_storage& get_blockchain_storage(){return m_blockchain_storage;} + Blockchain& get_blockchain_storage(){return m_blockchain_storage;} //debug functions void print_blockchain(uint64_t start_index, uint64_t end_index); void print_blockchain_index(); @@ -149,7 +149,7 @@ namespace cryptonote tx_memory_pool m_mempool; - blockchain_storage m_blockchain_storage; + Blockchain m_blockchain_storage; i_cryptonote_protocol* m_pprotocol; epee::critical_section m_incoming_tx_lock; //m_miner and m_miner_addres are probably temporary here diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index 691d16492..e6c20d814 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -37,7 +37,7 @@ #include "cryptonote_format_utils.h" #include "cryptonote_boost_serialization.h" #include "cryptonote_config.h" -#include "blockchain_storage.h" +#include "blockchain.h" #include "common/boost_serialization_helper.h" #include "common/int-util.h" #include "misc_language.h" @@ -54,7 +54,7 @@ namespace cryptonote } //--------------------------------------------------------------------------------- - tx_memory_pool::tx_memory_pool(blockchain_storage& bchs): m_blockchain(bchs) + tx_memory_pool::tx_memory_pool(Blockchain& bchs): m_blockchain(bchs) { } diff --git a/src/cryptonote_core/tx_pool.h b/src/cryptonote_core/tx_pool.h index 9c1c2b1aa..7ff8c5e1c 100644 --- a/src/cryptonote_core/tx_pool.h +++ b/src/cryptonote_core/tx_pool.h @@ -47,7 +47,7 @@ namespace cryptonote { - class blockchain_storage; + class Blockchain; /************************************************************************/ /* */ /************************************************************************/ @@ -55,7 +55,7 @@ namespace cryptonote class tx_memory_pool: boost::noncopyable { public: - tx_memory_pool(blockchain_storage& bchs); + tx_memory_pool(Blockchain& bchs); bool add_tx(const transaction &tx, const crypto::hash &id, size_t blob_size, tx_verification_context& tvc, bool keeped_by_block); bool add_tx(const transaction &tx, tx_verification_context& tvc, bool keeped_by_block); //gets tx and remove it from pool @@ -127,7 +127,7 @@ namespace cryptonote //transactions_container m_alternative_transactions; std::string m_config_folder; - blockchain_storage& m_blockchain; + Blockchain& m_blockchain; /************************************************************************/ /* */ /************************************************************************/ @@ -170,9 +170,6 @@ namespace cryptonote uint64_t operator()(const txin_to_scripthash& tx) const {return 0;} }; -#if defined(DEBUG_CREATE_BLOCK_TEMPLATE) - friend class blockchain_storage; -#endif }; } -- cgit v1.2.3 From 90f402e258d6ec8b0bb27e78be013af5ba463953 Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Tue, 28 Oct 2014 13:43:50 -0400 Subject: minor fixes to Blockchain.cpp --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 2 +- src/cryptonote_core/blockchain.cpp | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index 6275ae388..3c7b1a442 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -1022,7 +1022,7 @@ crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height) if (get_result == MDB_NOTFOUND) { LOG_PRINT_L0("Attempted to get hash from height " << height << ", but no such hash exists"); - throw DB_ERROR("Attempt to get hash from height failed -- block size not in db"); + throw BLOCK_DNE("Attempt to get hash from height failed -- hash not in db"); } else if (get_result) { diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 0f5765281..25d1ae33c 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -1541,9 +1541,13 @@ void Blockchain::print_blockchain_index() { std::stringstream ss; CRITICAL_REGION_LOCAL(m_blockchain_lock); - for(uint64_t i = 0; i <= m_db->height(); i++) + auto height = m_db->height(); + if (height != 0) { - ss << "height: " << i << ", hash: " << m_db->get_block_hash_from_height(i); + for(uint64_t i = 0; i <= height; i++) + { + ss << "height: " << i << ", hash: " << m_db->get_block_hash_from_height(i); + } } LOG_PRINT_L0("Current blockchain index:" << std::endl -- cgit v1.2.3 From 006afe2172ea0a480e61805ed2232f625b9aff7c Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Tue, 28 Oct 2014 15:30:16 -0400 Subject: Minor bug fixes and debug prints Blockchain and BlockchainLMDB classes now have a debug print at the beginning of each function at log level 2. These can be removed at any time, but for now are quite useful. Blockchain runs, and adds the genesis block just fine, but for some reason isn't getting new blocks. --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 56 ++++++++++++++++++ src/cryptonote_core/blockchain.cpp | 72 +++++++++++++++++++++-- 2 files changed, 122 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index 3c7b1a442..731964dc7 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -109,6 +109,7 @@ void BlockchainLMDB::add_block( const block& blk , const uint64_t& coins_generated ) { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); check_open(); crypto::hash h = get_block_hash(blk); @@ -217,6 +218,7 @@ void BlockchainLMDB::add_block( const block& blk void BlockchainLMDB::remove_block() { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); check_open(); MDB_val k; @@ -270,6 +272,7 @@ void BlockchainLMDB::remove_block() void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx) { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); check_open(); crypto::hash h = get_transaction_hash(tx); @@ -320,6 +323,7 @@ void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const tr void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash) { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); check_open(); crypto::hash h = tx_hash; @@ -359,6 +363,7 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash) void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index) { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); check_open(); MDB_val k; @@ -417,6 +422,7 @@ void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_ou void BlockchainLMDB::remove_output(const tx_out& tx_output) { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); check_open(); MDB_val k; @@ -474,6 +480,7 @@ void BlockchainLMDB::remove_output(const tx_out& tx_output) void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image) { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); check_open(); crypto::key_image key = k_image; @@ -500,6 +507,7 @@ void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image) void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image) { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); check_open(); crypto::key_image key_cpy = k_image; @@ -516,6 +524,7 @@ void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image) blobdata BlockchainLMDB::output_to_blob(const tx_out& output) { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); blobdata b; if (!t_serializable_object_to_blob(output, b)) { @@ -527,6 +536,7 @@ blobdata BlockchainLMDB::output_to_blob(const tx_out& output) tx_out BlockchainLMDB::output_from_blob(const blobdata& blob) { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); std::stringstream ss; ss << blob; binary_archive ba(ss); @@ -543,11 +553,13 @@ tx_out BlockchainLMDB::output_from_blob(const blobdata& blob) uint64_t BlockchainLMDB::get_output_global_index(const uint64_t& amount, const uint64_t& index) { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); return 0; } void BlockchainLMDB::check_open() { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); if (!m_open) { LOG_PRINT_L0("DB operation attempted on a not-open DB instance"); @@ -557,18 +569,22 @@ void BlockchainLMDB::check_open() BlockchainLMDB::~BlockchainLMDB() { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); } BlockchainLMDB::BlockchainLMDB() { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); // initialize folder to something "safe" just in case // someone accidentally misuses this class... m_folder = "thishsouldnotexistbecauseitisgibberish"; m_open = false; + m_height = 0; } void BlockchainLMDB::open(const std::string& filename) { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); if (m_open) { @@ -652,6 +668,7 @@ void BlockchainLMDB::open(const std::string& filename) LOG_PRINT_L0("Failed to query m_blocks"); throw DB_ERROR("Failed to query m_blocks"); } + LOG_PRINT_L2("Setting m_height to: " << db_stats.ms_entries); m_height = db_stats.ms_entries; // get and keep current number of outputs @@ -672,27 +689,32 @@ void BlockchainLMDB::open(const std::string& filename) // unused for now, create will happen on open if doesn't exist void BlockchainLMDB::create(const std::string& filename) { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); throw DB_CREATE_FAILURE("create() is not implemented for this BlockchainDB, open() will create files if needed."); } void BlockchainLMDB::close() { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); // FIXME: not yet thread safe!!! Use with care. mdb_env_close(m_env); } void BlockchainLMDB::sync() { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); // LMDB documentation leads me to believe this is unnecessary } void BlockchainLMDB::reset() { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); // TODO: this } std::vector BlockchainLMDB::get_filenames() { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); std::vector filenames; boost::filesystem::path datafile(m_folder); @@ -709,6 +731,7 @@ std::vector BlockchainLMDB::get_filenames() // TODO: this? bool BlockchainLMDB::lock() { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); check_open(); return false; } @@ -716,12 +739,14 @@ bool BlockchainLMDB::lock() // TODO: this? void BlockchainLMDB::unlock() { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); check_open(); } bool BlockchainLMDB::block_exists(const crypto::hash& h) { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); check_open(); txn_safe txn; @@ -756,6 +781,7 @@ bool BlockchainLMDB::block_exists(const crypto::hash& h) block BlockchainLMDB::get_block(const crypto::hash& h) { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); check_open(); return get_block_from_height(get_block_height(h)); @@ -763,6 +789,7 @@ block BlockchainLMDB::get_block(const crypto::hash& h) uint64_t BlockchainLMDB::get_block_height(const crypto::hash& h) { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); check_open(); txn_safe txn; @@ -796,6 +823,7 @@ uint64_t BlockchainLMDB::get_block_height(const crypto::hash& h) block_header BlockchainLMDB::get_block_header(const crypto::hash& h) { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); check_open(); // block_header object is automatically cast from block object @@ -804,6 +832,7 @@ block_header BlockchainLMDB::get_block_header(const crypto::hash& h) block BlockchainLMDB::get_block_from_height(const uint64_t& height) { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); check_open(); txn_safe txn; @@ -847,6 +876,7 @@ block BlockchainLMDB::get_block_from_height(const uint64_t& height) uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); check_open(); txn_safe txn; @@ -879,6 +909,7 @@ uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) uint64_t BlockchainLMDB::get_top_block_timestamp() { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); check_open(); // if no blocks, return 0 @@ -892,6 +923,7 @@ uint64_t BlockchainLMDB::get_top_block_timestamp() size_t BlockchainLMDB::get_block_size(const uint64_t& height) { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); check_open(); txn_safe txn; @@ -924,6 +956,7 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& height) { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); check_open(); txn_safe txn; @@ -956,6 +989,7 @@ difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& difficulty_type BlockchainLMDB::get_block_difficulty(const uint64_t& height) { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); check_open(); difficulty_type diff1 = 0; @@ -972,6 +1006,7 @@ difficulty_type BlockchainLMDB::get_block_difficulty(const uint64_t& height) uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& height) { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); check_open(); txn_safe txn; @@ -1004,6 +1039,7 @@ uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& heigh crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height) { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); check_open(); txn_safe txn; @@ -1036,6 +1072,7 @@ crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height) std::vector BlockchainLMDB::get_blocks_range(const uint64_t& h1, const uint64_t& h2) { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); check_open(); std::vector v; @@ -1049,6 +1086,7 @@ std::vector BlockchainLMDB::get_blocks_range(const uint64_t& h1, const ui std::vector BlockchainLMDB::get_hashes_range(const uint64_t& h1, const uint64_t& h2) { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); check_open(); std::vector v; @@ -1062,6 +1100,7 @@ std::vector BlockchainLMDB::get_hashes_range(const uint64_t& h1, c crypto::hash BlockchainLMDB::top_block_hash() { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); check_open(); if (m_height != 0) { @@ -1073,6 +1112,7 @@ crypto::hash BlockchainLMDB::top_block_hash() block BlockchainLMDB::get_top_block() { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); check_open(); if (m_height != 0) @@ -1086,6 +1126,7 @@ block BlockchainLMDB::get_top_block() uint64_t BlockchainLMDB::height() { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); check_open(); return m_height; @@ -1094,6 +1135,7 @@ uint64_t BlockchainLMDB::height() bool BlockchainLMDB::tx_exists(const crypto::hash& h) { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); check_open(); txn_safe txn; @@ -1127,6 +1169,7 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); check_open(); txn_safe txn; @@ -1159,6 +1202,7 @@ uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) transaction BlockchainLMDB::get_tx(const crypto::hash& h) { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); check_open(); txn_safe txn; @@ -1201,6 +1245,7 @@ transaction BlockchainLMDB::get_tx(const crypto::hash& h) uint64_t BlockchainLMDB::get_tx_count() { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); check_open(); txn_safe txn; @@ -1224,6 +1269,7 @@ uint64_t BlockchainLMDB::get_tx_count() std::vector BlockchainLMDB::get_tx_list(const std::vector& hlist) { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); check_open(); std::vector v; @@ -1237,6 +1283,7 @@ std::vector BlockchainLMDB::get_tx_list(const std::vector BlockchainLMDB::get_tx_output_indices(const crypto::hash& h) { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); check_open(); std::vector index_vec; @@ -1574,6 +1628,7 @@ std::vector BlockchainLMDB::get_tx_output_indices(const crypto::hash& bool BlockchainLMDB::has_key_image(const crypto::key_image& img) { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); check_open(); txn_safe txn; @@ -1606,6 +1661,7 @@ uint64_t BlockchainLMDB::add_block( const block& blk , const std::vector& txs ) { + LOG_PRINT_L2("BlockchainLMDB::" << __func__); check_open(); txn_safe txn; if (mdb_txn_begin(m_env, NULL, 0, txn)) diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 25d1ae33c..5e1e53272 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -66,12 +66,14 @@ DISABLE_VS_WARNINGS(4267) // TODO: initialize m_db with a concrete implementation of BlockchainDB Blockchain::Blockchain(tx_memory_pool& tx_pool):m_db(), m_tx_pool(tx_pool), m_current_block_cumul_sz_limit(0), m_is_in_checkpoint_zone(false), m_is_blockchain_storing(false) { + LOG_PRINT_L2("Blockchain::" << __func__); } //------------------------------------------------------------------ //TODO: is this still needed? I don't think so - tewinget template void Blockchain::serialize(archive_t & ar, const unsigned int version) { + LOG_PRINT_L2("Blockchain::" << __func__); if(version < 11) return; CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -127,12 +129,14 @@ void Blockchain::serialize(archive_t & ar, const unsigned int version) //------------------------------------------------------------------ bool Blockchain::have_tx(const crypto::hash &id) { + LOG_PRINT_L2("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); return m_db->tx_exists(id); } //------------------------------------------------------------------ bool Blockchain::have_tx_keyimg_as_spent(const crypto::key_image &key_im) { + LOG_PRINT_L2("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); return m_db->has_key_image(key_im); } @@ -143,6 +147,7 @@ bool Blockchain::have_tx_keyimg_as_spent(const crypto::key_image &key_im) template bool Blockchain::scan_outputkeys_for_indexes(const txin_to_key& tx_in_to_key, visitor_t& vis, uint64_t* pmax_related_block_height) { + LOG_PRINT_L2("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); // verify that the input has key offsets (that it exists properly, really) @@ -212,6 +217,7 @@ bool Blockchain::scan_outputkeys_for_indexes(const txin_to_key& tx_in_to_key, vi //------------------------------------------------------------------ uint64_t Blockchain::get_current_blockchain_height() { + LOG_PRINT_L2("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); return m_db->height(); } @@ -220,6 +226,7 @@ uint64_t Blockchain::get_current_blockchain_height() // dereferencing a null BlockchainDB pointer bool Blockchain::init(const std::string& config_folder, bool testnet) { + LOG_PRINT_L2("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); m_db = new BlockchainLMDB(); @@ -298,6 +305,7 @@ bool Blockchain::init(const std::string& config_folder, bool testnet) //------------------------------------------------------------------ bool Blockchain::store_blockchain() { + LOG_PRINT_L2("Blockchain::" << __func__); // TODO: make sure if this throws that it is not simply ignored higher // up the call stack try @@ -320,6 +328,7 @@ bool Blockchain::store_blockchain() //------------------------------------------------------------------ bool Blockchain::deinit() { + LOG_PRINT_L2("Blockchain::" << __func__); // as this should be called if handling a SIGSEGV, need to check // if m_db is a NULL pointer (and thus may have caused the illegal // memory operation), otherwise we may cause a loop. @@ -350,6 +359,7 @@ bool Blockchain::deinit() // from it to the tx_pool block Blockchain::pop_block_from_blockchain() { + LOG_PRINT_L2("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); block popped_block; @@ -391,6 +401,7 @@ block Blockchain::pop_block_from_blockchain() //------------------------------------------------------------------ bool Blockchain::reset_and_set_genesis_block(const block& b) { + LOG_PRINT_L2("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); m_transactions.clear(); m_spent_keys.clear(); @@ -408,6 +419,7 @@ bool Blockchain::reset_and_set_genesis_block(const block& b) //TODO: move to BlockchainDB subclass bool Blockchain::purge_transaction_keyimages_from_blockchain(const transaction& tx, bool strict_check) { + LOG_PRINT_L2("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); struct purge_transaction_visitor: public boost::static_visitor { @@ -453,6 +465,7 @@ bool Blockchain::purge_transaction_keyimages_from_blockchain(const transaction& //------------------------------------------------------------------ crypto::hash Blockchain::get_tail_id(uint64_t& height) { + LOG_PRINT_L2("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); height = m_db->height(); return get_tail_id(); @@ -460,6 +473,7 @@ crypto::hash Blockchain::get_tail_id(uint64_t& height) //------------------------------------------------------------------ crypto::hash Blockchain::get_tail_id() { + LOG_PRINT_L2("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); return m_db->top_block_hash(); } @@ -478,16 +492,17 @@ crypto::hash Blockchain::get_tail_id() */ bool Blockchain::get_short_chain_history(std::list& ids) { + LOG_PRINT_L2("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); uint64_t i = 0; uint64_t current_multiplier = 1; - uint64_t sz = m_db->height(); + uint64_t sz = m_db->height() - 1; if(!sz) return true; uint64_t current_back_offset = 0; - while(current_back_offset < sz) + while(current_back_offset <= sz) { ids.push_back(m_db->get_block_hash_from_height(sz-current_back_offset)); if(i < 10) @@ -507,6 +522,7 @@ bool Blockchain::get_short_chain_history(std::list& ids) //------------------------------------------------------------------ crypto::hash Blockchain::get_block_id_by_height(uint64_t height) { + LOG_PRINT_L2("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); try { @@ -530,6 +546,7 @@ crypto::hash Blockchain::get_block_id_by_height(uint64_t height) //------------------------------------------------------------------ bool Blockchain::get_block_by_hash(const crypto::hash &h, block &blk) { + LOG_PRINT_L2("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); // try to find block in main chain @@ -565,6 +582,7 @@ bool Blockchain::get_block_by_hash(const crypto::hash &h, block &blk) // if it ever is, should probably change std::list for std::vector void Blockchain::get_all_known_block_ids(std::list &main, std::list &alt, std::list &invalid) { + LOG_PRINT_L2("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); for (auto& a : m_db->get_hashes_range(0, m_db->height())) @@ -585,6 +603,7 @@ void Blockchain::get_all_known_block_ids(std::list &main, std::lis // less blocks than desired if there aren't enough. difficulty_type Blockchain::get_difficulty_for_next_block() { + LOG_PRINT_L2("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); std::vector timestamps; std::vector cumulative_difficulties; @@ -598,7 +617,7 @@ difficulty_type Blockchain::get_difficulty_for_next_block() // makes sure we don't use the genesis block. ++offset; - for(; offset <= h; offset++) + for(; offset < h; offset++) { timestamps.push_back(m_db->get_block_timestamp(offset)); cumulative_difficulties.push_back(m_db->get_block_cumulative_difficulty(offset)); @@ -611,6 +630,7 @@ difficulty_type Blockchain::get_difficulty_for_next_block() // that had been removed. bool Blockchain::rollback_blockchain_switching(std::list& original_chain, uint64_t rollback_height) { + LOG_PRINT_L2("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); // remove blocks from blockchain until we get back to where we should be. @@ -639,6 +659,7 @@ bool Blockchain::rollback_blockchain_switching(std::list& original_chain, // boolean based on success therein. bool Blockchain::switch_to_alternative_blockchain(std::list& alt_chain, bool discard_disconnected_chain) { + LOG_PRINT_L2("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); // if empty alt chain passed (not sure how that could happen), return false @@ -729,6 +750,7 @@ bool Blockchain::switch_to_alternative_blockchain(std::list& alt_chain, block_extended_info& bei) { + LOG_PRINT_L2("Blockchain::" << __func__); std::vector timestamps; std::vector cumulative_difficulties; @@ -797,6 +819,7 @@ difficulty_type Blockchain::get_next_difficulty_for_alternative_chain(const std: // a non-overflowing tx amount (dubious necessity on this check) bool Blockchain::prevalidate_miner_transaction(const block& b, uint64_t height) { + LOG_PRINT_L2("Blockchain::" << __func__); CHECK_AND_ASSERT_MES(b.miner_tx.vin.size() == 1, false, "coinbase transaction in the block has no inputs"); CHECK_AND_ASSERT_MES(b.miner_tx.vin[0].type() == typeid(txin_gen), false, "coinbase transaction in the block has the wrong type"); if(boost::get(b.miner_tx.vin[0]).height != height) @@ -824,6 +847,7 @@ bool Blockchain::prevalidate_miner_transaction(const block& b, uint64_t height) // This function validates the miner transaction reward bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins) { + LOG_PRINT_L2("Blockchain::" << __func__); //validate reward uint64_t money_in_use = 0; BOOST_FOREACH(auto& o, b.miner_tx.vout) @@ -853,6 +877,7 @@ bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_bl // and return by reference . void Blockchain::get_last_n_blocks_sizes(std::vector& sz, size_t count) { + LOG_PRINT_L2("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); auto h = m_db->height(); @@ -870,6 +895,7 @@ void Blockchain::get_last_n_blocks_sizes(std::vector& sz, size_t count) //------------------------------------------------------------------ uint64_t Blockchain::get_current_cumulative_blocksize_limit() { + LOG_PRINT_L2("Blockchain::" << __func__); return m_current_block_cumul_sz_limit; } //------------------------------------------------------------------ @@ -886,6 +912,7 @@ uint64_t Blockchain::get_current_cumulative_blocksize_limit() // necessary at all. bool Blockchain::create_block_template(block& b, const account_public_address& miner_address, difficulty_type& diffic, uint64_t& height, const blobdata& ex_nonce) { + LOG_PRINT_L2("Blockchain::" << __func__); size_t median_size; uint64_t already_generated_coins; @@ -1009,6 +1036,7 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m // the needed number of timestamps for the BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW. bool Blockchain::complete_timestamps_vector(uint64_t start_top_height, std::vector& timestamps) { + LOG_PRINT_L2("Blockchain::" << __func__); if(timestamps.size() >= BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW) return true; @@ -1033,6 +1061,7 @@ bool Blockchain::complete_timestamps_vector(uint64_t start_top_height, std::vect // a long forked chain eventually. bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id, block_verification_context& bvc) { + LOG_PRINT_L2("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); uint64_t block_height = get_block_height(b); @@ -1212,6 +1241,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id //------------------------------------------------------------------ bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::list& blocks, std::list& txs) { + LOG_PRINT_L2("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); if(start_offset > m_db->height()) return false; @@ -1233,6 +1263,7 @@ bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::list& blocks) { + LOG_PRINT_L2("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); if(start_offset > m_db->height()) return false; @@ -1249,6 +1280,7 @@ bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::list blocks; @@ -1282,6 +1314,7 @@ bool Blockchain::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NO //------------------------------------------------------------------ bool Blockchain::get_alternative_blocks(std::list& blocks) { + LOG_PRINT_L2("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); BOOST_FOREACH(const auto& alt_bl, m_alternative_chains) @@ -1293,6 +1326,7 @@ bool Blockchain::get_alternative_blocks(std::list& blocks) //------------------------------------------------------------------ size_t Blockchain::get_alternative_blocks_count() { + LOG_PRINT_L2("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); return m_alternative_chains.size(); } @@ -1301,6 +1335,7 @@ size_t Blockchain::get_alternative_blocks_count() // unlocked and other such checks should be done by here. void Blockchain::add_out_to_get_random_outs(COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i) { + LOG_PRINT_L2("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry& oen = *result_outs.outs.insert(result_outs.outs.end(), COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry()); @@ -1314,6 +1349,7 @@ void Blockchain::add_out_to_get_random_outs(COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_A // in some cases bool Blockchain::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) { + LOG_PRINT_L2("Blockchain::" << __func__); srand(static_cast(time(NULL))); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -1386,6 +1422,7 @@ bool Blockchain::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUT // This is used to see what to send another node that needs to sync. bool Blockchain::find_blockchain_supplement(const std::list& qblock_ids, uint64_t& starter_offset) { + LOG_PRINT_L2("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); // make sure the request includes at least the genesis block, otherwise @@ -1451,6 +1488,7 @@ bool Blockchain::find_blockchain_supplement(const std::list& qbloc //------------------------------------------------------------------ uint64_t Blockchain::block_difficulty(uint64_t i) { + LOG_PRINT_L2("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); try { @@ -1466,6 +1504,7 @@ uint64_t Blockchain::block_difficulty(uint64_t i) template bool Blockchain::get_blocks(const t_ids_container& block_ids, t_blocks_container& blocks, t_missed_container& missed_bs) { + LOG_PRINT_L2("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); for (const auto& block_hash : block_ids) @@ -1489,6 +1528,7 @@ bool Blockchain::get_blocks(const t_ids_container& block_ids, t_blocks_container template bool Blockchain::get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) { + LOG_PRINT_L2("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); for (const auto& tx_hash : txs_ids) @@ -1512,6 +1552,7 @@ bool Blockchain::get_transactions(const t_ids_container& txs_ids, t_tx_container //------------------------------------------------------------------ void Blockchain::print_blockchain(uint64_t start_index, uint64_t end_index) { + LOG_PRINT_L2("Blockchain::" << __func__); std::stringstream ss; CRITICAL_REGION_LOCAL(m_blockchain_lock); auto h = m_db->height(); @@ -1539,6 +1580,7 @@ void Blockchain::print_blockchain(uint64_t start_index, uint64_t end_index) //------------------------------------------------------------------ void Blockchain::print_blockchain_index() { + LOG_PRINT_L2("Blockchain::" << __func__); std::stringstream ss; CRITICAL_REGION_LOCAL(m_blockchain_lock); auto height = m_db->height(); @@ -1558,6 +1600,7 @@ void Blockchain::print_blockchain_index() //TODO: remove this function and references to it void Blockchain::print_blockchain_outs(const std::string& file) { + LOG_PRINT_L2("Blockchain::" << __func__); return; } //------------------------------------------------------------------ @@ -1566,6 +1609,7 @@ void Blockchain::print_blockchain_outs(const std::string& file) // BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT additional (more recent) hashes. bool Blockchain::find_blockchain_supplement(const std::list& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) { + LOG_PRINT_L2("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); // if we can't find the split point, return false @@ -1589,6 +1633,7 @@ bool Blockchain::find_blockchain_supplement(const std::list& qbloc // blocks by reference. bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count) { + LOG_PRINT_L2("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); // if a specific start height has been requested @@ -1624,6 +1669,7 @@ bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, cons //------------------------------------------------------------------ bool Blockchain::add_block_as_invalid(const block& bl, const crypto::hash& h) { + LOG_PRINT_L2("Blockchain::" << __func__); block_extended_info bei = AUTO_VAL_INIT(bei); bei.bl = bl; return add_block_as_invalid(bei, h); @@ -1631,6 +1677,7 @@ bool Blockchain::add_block_as_invalid(const block& bl, const crypto::hash& h) //------------------------------------------------------------------ bool Blockchain::add_block_as_invalid(const block_extended_info& bei, const crypto::hash& h) { + LOG_PRINT_L2("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); auto i_res = m_invalid_blocks.insert(std::map::value_type(h, bei)); CHECK_AND_ASSERT_MES(i_res.second, false, "at insertion invalid by tx returned status existed"); @@ -1640,6 +1687,7 @@ bool Blockchain::add_block_as_invalid(const block_extended_info& bei, const cryp //------------------------------------------------------------------ bool Blockchain::have_block(const crypto::hash& id) { + LOG_PRINT_L2("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); if(m_db->block_exists(id)) @@ -1656,12 +1704,14 @@ bool Blockchain::have_block(const crypto::hash& id) //------------------------------------------------------------------ bool Blockchain::handle_block_to_main_chain(const block& bl, block_verification_context& bvc) { + LOG_PRINT_L2("Blockchain::" << __func__); crypto::hash id = get_block_hash(bl); return handle_block_to_main_chain(bl, id, bvc); } //------------------------------------------------------------------ size_t Blockchain::get_total_transactions() { + LOG_PRINT_L2("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); return m_db->get_tx_count(); } @@ -1674,6 +1724,7 @@ size_t Blockchain::get_total_transactions() // remove them later if the block fails validation. bool Blockchain::check_for_double_spend(const transaction& tx, key_images_container& keys_this_block) { + LOG_PRINT_L2("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); struct add_transaction_input_visitor: public boost::static_visitor { @@ -1722,6 +1773,7 @@ bool Blockchain::check_for_double_spend(const transaction& tx, key_images_contai //------------------------------------------------------------------ bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector& indexs) { + LOG_PRINT_L2("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); if (!m_db->tx_exists(tx_id)) { @@ -1738,6 +1790,7 @@ bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector& sig, uint64_t* pmax_related_block_height) { + LOG_PRINT_L2("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); struct outputs_visitor @@ -1874,6 +1931,7 @@ bool Blockchain::check_tx_input(const txin_to_key& txin, const crypto::hash& tx_ //TODO: Is this intended to do something else? Need to look into the todo there. uint64_t Blockchain::get_adjusted_time() { + LOG_PRINT_L2("Blockchain::" << __func__); //TODO: add collecting median time return time(NULL); } @@ -1881,6 +1939,7 @@ uint64_t Blockchain::get_adjusted_time() //TODO: revisit, has changed a bit on upstream bool Blockchain::check_block_timestamp(std::vector& timestamps, const block& b) { + LOG_PRINT_L2("Blockchain::" << __func__); uint64_t median_ts = epee::misc_utils::median(timestamps); if(b.timestamp < median_ts) @@ -1901,6 +1960,7 @@ bool Blockchain::check_block_timestamp(std::vector& timestamps, const // false otherwise bool Blockchain::check_block_timestamp(const block& b) { + LOG_PRINT_L2("Blockchain::" << __func__); if(b.timestamp > get_adjusted_time() + CRYPTONOTE_BLOCK_FUTURE_TIME_LIMIT) { LOG_PRINT_L0("Timestamp of block with id: " << get_block_hash(b) << ", " << b.timestamp << ", bigger than adjusted time + 2 hours"); @@ -1933,6 +1993,7 @@ bool Blockchain::check_block_timestamp(const block& b) // m_db->add_block() bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& id, block_verification_context& bvc) { + LOG_PRINT_L2("Blockchain::" << __func__); // if we already have the block, return false if (have_block(id)) { @@ -2027,9 +2088,6 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& std::vector txs; key_images_container keys; - // add miner transaction to list of block's transactions. - txs.push_back(bl.miner_tx); - uint64_t fee_summary = 0; // Iterate over the block's transaction hashes, grabbing each @@ -2155,6 +2213,7 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& //------------------------------------------------------------------ bool Blockchain::update_next_cumulative_size_limit() { + LOG_PRINT_L2("Blockchain::" << __func__); std::vector sz; get_last_n_blocks_sizes(sz, CRYPTONOTE_REWARD_BLOCKS_WINDOW); @@ -2168,6 +2227,7 @@ bool Blockchain::update_next_cumulative_size_limit() //------------------------------------------------------------------ bool Blockchain::add_new_block(const block& bl_, block_verification_context& bvc) { + LOG_PRINT_L2("Blockchain::" << __func__); //copy block here to let modify block.target block bl = bl_; crypto::hash id = get_block_hash(bl); -- cgit v1.2.3 From 1a546e32227492cec4238ff0634e7c59b23ce9bb Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Tue, 28 Oct 2014 22:25:03 -0400 Subject: some bug fixes, but still needs work There are quite a few debug prints in this commit that will need removed later, but for posterity (in case someone wants to debug this while I'm away), I left them in. Currently errors when syncing on the first block that has a "real" transaction. Seems to not be able to validate the ring signature, but I can't for the life of me figure out what's going wrong. --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 23 +++++++++- src/cryptonote_core/blockchain.cpp | 54 ++++++++++++----------- src/cryptonote_core/blockchain_db.cpp | 31 +++++++++---- 3 files changed, 73 insertions(+), 35 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index 731964dc7..e26300a38 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -34,6 +34,8 @@ #include "cryptonote_core/cryptonote_format_utils.h" #include "crypto/crypto.h" +using epee::string_tools::pod_to_hex; + namespace { @@ -69,6 +71,14 @@ struct lmdb_cur bool done; }; +auto compare_uint64 = [](const MDB_val *a, const MDB_val *b) { + uint64_t va = *(uint64_t*)a->mv_data; + uint64_t vb = *(uint64_t*)b->mv_data; + if (va < vb) return -1; + else if (va == vb) return 0; + else return 1; +}; + const char* LMDB_BLOCKS = "blocks"; const char* LMDB_BLOCK_TIMESTAMPS = "block_timestamps"; const char* LMDB_BLOCK_HEIGHTS = "block_heights"; @@ -418,6 +428,9 @@ void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_ou throw DB_ERROR("Failed to add output global index to db transaction"); } + LOG_PRINT_L0(__func__ << ": amount == " << amount << ", tx_index == " << local_index << "amount_index == " << get_num_outputs(amount) << ", tx_hash == " << pod_to_hex(tx_hash)); + + m_num_outputs++; } void BlockchainLMDB::remove_output(const tx_out& tx_output) @@ -661,6 +674,9 @@ void BlockchainLMDB::open(const std::string& filename) lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_CREATE, m_spent_keys, "Failed to open db handle for m_outputs"); + mdb_set_dupsort(txn, m_output_amounts, compare_uint64); + mdb_set_dupsort(txn, m_tx_outputs, compare_uint64); + // get and keep current height MDB_stat db_stats; if (mdb_stat(txn, m_blocks, &db_stats)) @@ -1508,8 +1524,8 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); if (result == MDB_NOTFOUND) { - LOG_PRINT_L1("Attempting to get an output index by amount and amount index, but output not found"); - throw OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found"); + LOG_PRINT_L1("Attempting to get an output index by amount and amount index, but amount not found"); + throw OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found"); } else if (result) { @@ -1519,6 +1535,7 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con size_t num_elems = 0; mdb_cursor_count(cur, &num_elems); + LOG_PRINT_L0(__func__ << ": amount == " << amount << ", index == " << index << ", num_elem for amount == " << num_elems); if (num_elems <= index) { LOG_PRINT_L1("Attempting to get an output index by amount and amount index, but output not found"); @@ -1571,6 +1588,8 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con txn.commit(); + LOG_PRINT_L0(__func__ << ": tx_hash == " << pod_to_hex(tx_hash) << " tx_index == " << *(uint64_t*)v.mv_data); + return tx_out_index(tx_hash, *(uint64_t *)v.mv_data); } diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 5e1e53272..382be9e12 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -180,6 +180,8 @@ bool Blockchain::scan_outputkeys_for_indexes(const txin_to_key& tx_in_to_key, vi return false; } +LOG_PRINT_L0(__func__ << ": amount == " << tx.vout[output_index.second].amount); + // call to the passed boost visitor to grab the public key for the output if(!vis.handle_output(tx, tx.vout[output_index.second])) { @@ -496,15 +498,15 @@ bool Blockchain::get_short_chain_history(std::list& ids) CRITICAL_REGION_LOCAL(m_blockchain_lock); uint64_t i = 0; uint64_t current_multiplier = 1; - uint64_t sz = m_db->height() - 1; + uint64_t sz = m_db->height(); if(!sz) return true; uint64_t current_back_offset = 0; - while(current_back_offset <= sz) + while(current_back_offset < sz) { - ids.push_back(m_db->get_block_hash_from_height(sz-current_back_offset)); + ids.push_back(m_db->get_block_hash_from_height(sz - current_back_offset - 1)); if(i < 10) { ++current_back_offset; @@ -585,7 +587,7 @@ void Blockchain::get_all_known_block_ids(std::list &main, std::lis LOG_PRINT_L2("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); - for (auto& a : m_db->get_hashes_range(0, m_db->height())) + for (auto& a : m_db->get_hashes_range(0, m_db->height() - 1)) { main.push_back(a); } @@ -861,13 +863,13 @@ bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_bl } if(base_reward + fee < money_in_use) { - LOG_ERROR("coinbase transaction spend too much money (" << print_money(money_in_use) << "). Block reward is " << print_money(base_reward + fee) << "(" << print_money(base_reward) << "+" << print_money(fee) << ")"); + LOG_ERROR("coinbase transaction spend too much money (" << money_in_use << "). Block reward is " << base_reward + fee << "(" << base_reward << "+" << fee << ")"); return false; } if(base_reward + fee != money_in_use) { LOG_ERROR("coinbase transaction doesn't use full amount of block reward: spent: " - << print_money(money_in_use) << ", block reward " << print_money(base_reward + fee) << "(" << print_money(base_reward) << "+" << print_money(fee) << ")"); + << money_in_use << ", block reward " << base_reward + fee << "(" << base_reward << "+" << fee << ")"); return false; } return true; @@ -886,8 +888,8 @@ void Blockchain::get_last_n_blocks_sizes(std::vector& sz, size_t count) return; // add size of last blocks to vector (or less, if blockchain size < count) - size_t start_offset = (h+1) - std::min((h+1), count); - for(size_t i = start_offset; i <= h; i++) + size_t start_offset = h - std::min(h, count); + for(size_t i = start_offset; i < h; i++) { sz.push_back(m_db->get_block_size(i)); } @@ -922,13 +924,12 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m b.prev_id = get_tail_id(); b.timestamp = time(NULL); - auto old_height = m_db->height(); - height = old_height + 1; + height = m_db->height(); diffic = get_difficulty_for_next_block(); CHECK_AND_ASSERT_MES(diffic, false, "difficulty owverhead."); median_size = m_current_block_cumul_sz_limit / 2; - already_generated_coins = m_db->get_block_already_generated_coins(old_height); + already_generated_coins = m_db->get_block_already_generated_coins(height - 1); CRITICAL_REGION_END(); @@ -1043,7 +1044,7 @@ bool Blockchain::complete_timestamps_vector(uint64_t start_top_height, std::vect CRITICAL_REGION_LOCAL(m_blockchain_lock); size_t need_elements = BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW - timestamps.size(); - CHECK_AND_ASSERT_MES(start_top_height <= m_db->height(), false, "internal error: passed start_height > " << " m_db->height() -- " << start_top_height << " > " << m_db->height()); + CHECK_AND_ASSERT_MES(start_top_height < m_db->height(), false, "internal error: passed start_height not < " << " m_db->height() -- " << start_top_height << " >= " << m_db->height()); size_t stop_offset = start_top_height > need_elements ? start_top_height - need_elements : 0; while (start_top_height != stop_offset); { @@ -1107,7 +1108,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id if(alt_chain.size()) { // make sure alt chain doesn't somehow start past the end of the main chain - CHECK_AND_ASSERT_MES(m_db->height() > alt_chain.front()->second.height, false, "main blockchain wrong height"); + CHECK_AND_ASSERT_MES(m_db->height() - 1 > alt_chain.front()->second.height, false, "main blockchain wrong height"); // make sure that the blockchain contains the block that should connect // this alternate chain with it. @@ -1979,8 +1980,8 @@ bool Blockchain::check_block_timestamp(const block& b) // need most recent 60 blocks, get index of first of those // using +1 because BlockchainDB::height() returns the index of the top block, // not the size of the blockchain (0-indexed) - size_t offset = h - BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW + 1; - for(;offset <= h; ++offset) + size_t offset = h - BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW - 1; + for(;offset < h; ++offset) { timestamps.push_back(m_db->get_block_timestamp(offset)); } @@ -2143,7 +2144,7 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& } uint64_t base_reward = 0; - uint64_t already_generated_coins = m_db->height() ? m_db->get_block_already_generated_coins(m_db->height()) : 0; + uint64_t already_generated_coins = m_db->height() ? m_db->get_block_already_generated_coins(m_db->height() - 1) : 0; if(!validate_miner_transaction(bl, cumulative_block_size, fee_summary, base_reward, already_generated_coins)) { LOG_PRINT_L0("Block with id: " << id @@ -2161,7 +2162,7 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& cumulative_difficulty = current_diffic; already_generated_coins = already_generated_coins + base_reward; if(m_db->height()) - cumulative_difficulty += m_db->get_block_cumulative_difficulty(m_db->height()); + cumulative_difficulty += m_db->get_block_cumulative_difficulty(m_db->height() - 1); update_next_cumulative_size_limit(); @@ -2169,15 +2170,18 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& uint64_t new_height = 0; bool add_success = true; - try - { - new_height = m_db->add_block(bl, block_size, cumulative_difficulty, already_generated_coins, txs); - } - catch (const std::exception& e) + if (!bvc.m_verifivation_failed) { - //TODO: figure out the best way to deal with this failure - LOG_ERROR("Error adding block with hash: " << id << " to blockchain, what = " << e.what()); - add_success = false; + try + { + new_height = m_db->add_block(bl, block_size, cumulative_difficulty, already_generated_coins, txs); + } + catch (const std::exception& e) + { + //TODO: figure out the best way to deal with this failure + LOG_ERROR("Error adding block with hash: " << id << " to blockchain, what = " << e.what()); + add_success = false; + } } // if we failed for any reason to verify the block, return taken diff --git a/src/cryptonote_core/blockchain_db.cpp b/src/cryptonote_core/blockchain_db.cpp index 02912be10..c7109dd20 100644 --- a/src/cryptonote_core/blockchain_db.cpp +++ b/src/cryptonote_core/blockchain_db.cpp @@ -29,6 +29,8 @@ #include "cryptonote_core/blockchain_db.h" #include "cryptonote_format_utils.h" +using epee::string_tools::pod_to_hex; + namespace cryptonote { @@ -43,20 +45,31 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti { crypto::hash tx_hash = get_transaction_hash(tx); + LOG_PRINT_L0("Adding tx with hash " << pod_to_hex(tx_hash) << " to BlockchainDB instance"); + LOG_PRINT_L0("Included in block with hash " << pod_to_hex(blk_hash)); + LOG_PRINT_L0("Unlock time == " << tx.unlock_time); + add_transaction_data(blk_hash, tx); // iterate tx.vout using indices instead of C++11 foreach syntax because // we need the index - for (uint64_t i = 0; i < tx.vout.size(); ++i) + if (tx.vout.size() != 0) // it may be technically possible for a tx to have no outputs { - add_output(tx_hash, tx.vout[i], i); - } + for (uint64_t i = 0; i < tx.vout.size(); ++i) + { + add_output(tx_hash, tx.vout[i], i); + } - for (const txin_v& tx_input : tx.vin) - { - if (tx_input.type() == typeid(txin_to_key)) + for (const txin_v& tx_input : tx.vin) { - add_spent_key(boost::get(tx_input).k_image); + if (tx_input.type() == typeid(txin_to_key)) + { + add_spent_key(boost::get(tx_input).k_image); + } + else + { + LOG_PRINT_L0("Is miner tx"); + } } } } @@ -68,10 +81,12 @@ uint64_t BlockchainDB::add_block( const block& blk , const std::vector& txs ) { + crypto::hash blk_hash = get_block_hash(blk); + LOG_PRINT_L0("Adding block with hash " << pod_to_hex(blk_hash) << " to BlockchainDB instance."); + // call out to subclass implementation to add the block & metadata add_block(blk, block_size, cumulative_difficulty, coins_generated); - crypto::hash blk_hash = get_block_hash(blk); // call out to add the transactions add_transaction(blk_hash, blk.miner_tx); -- cgit v1.2.3 From 09159131111b2c2c6ff6c620fa3b73624d38a2be Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Thu, 30 Oct 2014 00:58:14 -0400 Subject: BlockchainLMDB seems to be working*! * - Well, mostly. Haven't let it sync too far just yet. Currently trying to figure out the best way to deal with LMDB/mmap virtual memory pages. --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 122 ++++++++-------- src/cryptonote_core/blockchain.cpp | 165 ++++++++++++---------- src/cryptonote_core/blockchain_db.cpp | 9 -- 3 files changed, 155 insertions(+), 141 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index e26300a38..58f0d920f 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -119,7 +119,7 @@ void BlockchainLMDB::add_block( const block& blk , const uint64_t& coins_generated ) { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); crypto::hash h = get_block_hash(blk); @@ -228,7 +228,7 @@ void BlockchainLMDB::add_block( const block& blk void BlockchainLMDB::remove_block() { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); MDB_val k; @@ -282,7 +282,7 @@ void BlockchainLMDB::remove_block() void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx) { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); crypto::hash h = get_transaction_hash(tx); @@ -333,7 +333,7 @@ void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const tr void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash) { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); crypto::hash h = tx_hash; @@ -373,7 +373,7 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash) void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index) { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); MDB_val k; @@ -407,9 +407,10 @@ void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_ou uint64_t amount = tx_output.amount; v.mv_size = sizeof(uint64_t); v.mv_data = &amount; - if (mdb_put(*m_write_txn, m_output_amounts, &v, &k, 0)) + if (auto result = mdb_put(*m_write_txn, m_output_amounts, &v, &k, 0)) { LOG_PRINT_L0("Failed to add output amount to db transaction"); + LOG_PRINT_L0("E: " << mdb_strerror(result)); throw DB_ERROR("Failed to add output amount to db transaction"); } @@ -428,14 +429,12 @@ void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_ou throw DB_ERROR("Failed to add output global index to db transaction"); } - LOG_PRINT_L0(__func__ << ": amount == " << amount << ", tx_index == " << local_index << "amount_index == " << get_num_outputs(amount) << ", tx_hash == " << pod_to_hex(tx_hash)); - m_num_outputs++; } void BlockchainLMDB::remove_output(const tx_out& tx_output) { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); MDB_val k; @@ -493,7 +492,7 @@ void BlockchainLMDB::remove_output(const tx_out& tx_output) void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image) { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); crypto::key_image key = k_image; @@ -520,7 +519,7 @@ void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image) void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image) { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); crypto::key_image key_cpy = k_image; @@ -537,7 +536,7 @@ void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image) blobdata BlockchainLMDB::output_to_blob(const tx_out& output) { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); blobdata b; if (!t_serializable_object_to_blob(output, b)) { @@ -549,7 +548,7 @@ blobdata BlockchainLMDB::output_to_blob(const tx_out& output) tx_out BlockchainLMDB::output_from_blob(const blobdata& blob) { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); std::stringstream ss; ss << blob; binary_archive ba(ss); @@ -566,13 +565,13 @@ tx_out BlockchainLMDB::output_from_blob(const blobdata& blob) uint64_t BlockchainLMDB::get_output_global_index(const uint64_t& amount, const uint64_t& index) { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); return 0; } void BlockchainLMDB::check_open() { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); if (!m_open) { LOG_PRINT_L0("DB operation attempted on a not-open DB instance"); @@ -582,12 +581,12 @@ void BlockchainLMDB::check_open() BlockchainLMDB::~BlockchainLMDB() { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); } BlockchainLMDB::BlockchainLMDB() { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); // initialize folder to something "safe" just in case // someone accidentally misuses this class... m_folder = "thishsouldnotexistbecauseitisgibberish"; @@ -597,7 +596,7 @@ BlockchainLMDB::BlockchainLMDB() void BlockchainLMDB::open(const std::string& filename) { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); if (m_open) { @@ -636,9 +635,16 @@ void BlockchainLMDB::open(const std::string& filename) LOG_PRINT_L0("Failed to set max number of dbs"); throw DB_ERROR("Failed to set max number of dbs"); } - if (mdb_env_open(m_env, filename.c_str(), 0, 0664)) + if (auto result = mdb_env_set_mapsize(m_env, 1 << 29)) + { + LOG_PRINT_L0("Failed to set max memory map size"); + LOG_PRINT_L0("E: " << mdb_strerror(result)); + throw DB_ERROR("Failed to set max memory map size"); + } + if (auto result = mdb_env_open(m_env, filename.c_str(), 0, 0664)) { LOG_PRINT_L0("Failed to open lmdb environment"); + LOG_PRINT_L0("E: " << mdb_strerror(result)); throw DB_ERROR("Failed to open lmdb environment"); } @@ -705,32 +711,32 @@ void BlockchainLMDB::open(const std::string& filename) // unused for now, create will happen on open if doesn't exist void BlockchainLMDB::create(const std::string& filename) { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); throw DB_CREATE_FAILURE("create() is not implemented for this BlockchainDB, open() will create files if needed."); } void BlockchainLMDB::close() { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); // FIXME: not yet thread safe!!! Use with care. mdb_env_close(m_env); } void BlockchainLMDB::sync() { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); // LMDB documentation leads me to believe this is unnecessary } void BlockchainLMDB::reset() { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); // TODO: this } std::vector BlockchainLMDB::get_filenames() { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); std::vector filenames; boost::filesystem::path datafile(m_folder); @@ -747,7 +753,7 @@ std::vector BlockchainLMDB::get_filenames() // TODO: this? bool BlockchainLMDB::lock() { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); return false; } @@ -755,14 +761,14 @@ bool BlockchainLMDB::lock() // TODO: this? void BlockchainLMDB::unlock() { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); } bool BlockchainLMDB::block_exists(const crypto::hash& h) { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); txn_safe txn; @@ -797,7 +803,7 @@ bool BlockchainLMDB::block_exists(const crypto::hash& h) block BlockchainLMDB::get_block(const crypto::hash& h) { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); return get_block_from_height(get_block_height(h)); @@ -805,7 +811,7 @@ block BlockchainLMDB::get_block(const crypto::hash& h) uint64_t BlockchainLMDB::get_block_height(const crypto::hash& h) { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); txn_safe txn; @@ -839,7 +845,7 @@ uint64_t BlockchainLMDB::get_block_height(const crypto::hash& h) block_header BlockchainLMDB::get_block_header(const crypto::hash& h) { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); // block_header object is automatically cast from block object @@ -848,7 +854,7 @@ block_header BlockchainLMDB::get_block_header(const crypto::hash& h) block BlockchainLMDB::get_block_from_height(const uint64_t& height) { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); txn_safe txn; @@ -892,7 +898,7 @@ block BlockchainLMDB::get_block_from_height(const uint64_t& height) uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); txn_safe txn; @@ -925,7 +931,7 @@ uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) uint64_t BlockchainLMDB::get_top_block_timestamp() { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); // if no blocks, return 0 @@ -939,7 +945,7 @@ uint64_t BlockchainLMDB::get_top_block_timestamp() size_t BlockchainLMDB::get_block_size(const uint64_t& height) { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); txn_safe txn; @@ -972,7 +978,7 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& height) { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); txn_safe txn; @@ -1005,7 +1011,7 @@ difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& difficulty_type BlockchainLMDB::get_block_difficulty(const uint64_t& height) { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); difficulty_type diff1 = 0; @@ -1022,7 +1028,7 @@ difficulty_type BlockchainLMDB::get_block_difficulty(const uint64_t& height) uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& height) { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); txn_safe txn; @@ -1055,7 +1061,7 @@ uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& heigh crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height) { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); txn_safe txn; @@ -1088,7 +1094,7 @@ crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height) std::vector BlockchainLMDB::get_blocks_range(const uint64_t& h1, const uint64_t& h2) { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); std::vector v; @@ -1102,7 +1108,7 @@ std::vector BlockchainLMDB::get_blocks_range(const uint64_t& h1, const ui std::vector BlockchainLMDB::get_hashes_range(const uint64_t& h1, const uint64_t& h2) { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); std::vector v; @@ -1116,7 +1122,7 @@ std::vector BlockchainLMDB::get_hashes_range(const uint64_t& h1, c crypto::hash BlockchainLMDB::top_block_hash() { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); if (m_height != 0) { @@ -1128,7 +1134,7 @@ crypto::hash BlockchainLMDB::top_block_hash() block BlockchainLMDB::get_top_block() { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); if (m_height != 0) @@ -1142,7 +1148,7 @@ block BlockchainLMDB::get_top_block() uint64_t BlockchainLMDB::height() { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); return m_height; @@ -1151,7 +1157,7 @@ uint64_t BlockchainLMDB::height() bool BlockchainLMDB::tx_exists(const crypto::hash& h) { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); txn_safe txn; @@ -1185,7 +1191,7 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); txn_safe txn; @@ -1218,7 +1224,7 @@ uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) transaction BlockchainLMDB::get_tx(const crypto::hash& h) { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); txn_safe txn; @@ -1261,7 +1267,7 @@ transaction BlockchainLMDB::get_tx(const crypto::hash& h) uint64_t BlockchainLMDB::get_tx_count() { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); txn_safe txn; @@ -1285,7 +1291,7 @@ uint64_t BlockchainLMDB::get_tx_count() std::vector BlockchainLMDB::get_tx_list(const std::vector& hlist) { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); std::vector v; @@ -1299,7 +1305,7 @@ std::vector BlockchainLMDB::get_tx_list(const std::vector BlockchainLMDB::get_tx_output_indices(const crypto::hash& h) { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); std::vector index_vec; @@ -1647,7 +1653,7 @@ std::vector BlockchainLMDB::get_tx_output_indices(const crypto::hash& bool BlockchainLMDB::has_key_image(const crypto::key_image& img) { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); txn_safe txn; @@ -1680,7 +1686,7 @@ uint64_t BlockchainLMDB::add_block( const block& blk , const std::vector& txs ) { - LOG_PRINT_L2("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); txn_safe txn; if (mdb_txn_begin(m_env, NULL, 0, txn)) diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 382be9e12..39bf39df3 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -59,6 +59,7 @@ */ using namespace cryptonote; +using epee::string_tools::pod_to_hex; DISABLE_VS_WARNINGS(4267) @@ -66,14 +67,14 @@ DISABLE_VS_WARNINGS(4267) // TODO: initialize m_db with a concrete implementation of BlockchainDB Blockchain::Blockchain(tx_memory_pool& tx_pool):m_db(), m_tx_pool(tx_pool), m_current_block_cumul_sz_limit(0), m_is_in_checkpoint_zone(false), m_is_blockchain_storing(false) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); } //------------------------------------------------------------------ //TODO: is this still needed? I don't think so - tewinget template void Blockchain::serialize(archive_t & ar, const unsigned int version) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); if(version < 11) return; CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -116,7 +117,7 @@ void Blockchain::serialize(archive_t & ar, const unsigned int version) } - LOG_PRINT_L2("Blockchain storage:" << std::endl << + LOG_PRINT_L3("Blockchain storage:" << std::endl << "m_blocks: " << m_db->height() << std::endl << "m_blocks_index: " << m_blocks_index.size() << std::endl << "m_transactions: " << m_transactions.size() << std::endl << @@ -129,14 +130,14 @@ void Blockchain::serialize(archive_t & ar, const unsigned int version) //------------------------------------------------------------------ bool Blockchain::have_tx(const crypto::hash &id) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); return m_db->tx_exists(id); } //------------------------------------------------------------------ bool Blockchain::have_tx_keyimg_as_spent(const crypto::key_image &key_im) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); return m_db->has_key_image(key_im); } @@ -147,7 +148,7 @@ bool Blockchain::have_tx_keyimg_as_spent(const crypto::key_image &key_im) template bool Blockchain::scan_outputkeys_for_indexes(const txin_to_key& tx_in_to_key, visitor_t& vis, uint64_t* pmax_related_block_height) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); // verify that the input has key offsets (that it exists properly, really) @@ -180,8 +181,6 @@ bool Blockchain::scan_outputkeys_for_indexes(const txin_to_key& tx_in_to_key, vi return false; } -LOG_PRINT_L0(__func__ << ": amount == " << tx.vout[output_index.second].amount); - // call to the passed boost visitor to grab the public key for the output if(!vis.handle_output(tx, tx.vout[output_index.second])) { @@ -219,7 +218,7 @@ LOG_PRINT_L0(__func__ << ": amount == " << tx.vout[output_index.second].amount); //------------------------------------------------------------------ uint64_t Blockchain::get_current_blockchain_height() { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); return m_db->height(); } @@ -228,7 +227,7 @@ uint64_t Blockchain::get_current_blockchain_height() // dereferencing a null BlockchainDB pointer bool Blockchain::init(const std::string& config_folder, bool testnet) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); m_db = new BlockchainLMDB(); @@ -307,7 +306,7 @@ bool Blockchain::init(const std::string& config_folder, bool testnet) //------------------------------------------------------------------ bool Blockchain::store_blockchain() { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); // TODO: make sure if this throws that it is not simply ignored higher // up the call stack try @@ -330,7 +329,7 @@ bool Blockchain::store_blockchain() //------------------------------------------------------------------ bool Blockchain::deinit() { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); // as this should be called if handling a SIGSEGV, need to check // if m_db is a NULL pointer (and thus may have caused the illegal // memory operation), otherwise we may cause a loop. @@ -361,7 +360,7 @@ bool Blockchain::deinit() // from it to the tx_pool block Blockchain::pop_block_from_blockchain() { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); block popped_block; @@ -403,7 +402,7 @@ block Blockchain::pop_block_from_blockchain() //------------------------------------------------------------------ bool Blockchain::reset_and_set_genesis_block(const block& b) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); m_transactions.clear(); m_spent_keys.clear(); @@ -421,7 +420,7 @@ bool Blockchain::reset_and_set_genesis_block(const block& b) //TODO: move to BlockchainDB subclass bool Blockchain::purge_transaction_keyimages_from_blockchain(const transaction& tx, bool strict_check) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); struct purge_transaction_visitor: public boost::static_visitor { @@ -467,7 +466,7 @@ bool Blockchain::purge_transaction_keyimages_from_blockchain(const transaction& //------------------------------------------------------------------ crypto::hash Blockchain::get_tail_id(uint64_t& height) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); height = m_db->height(); return get_tail_id(); @@ -475,7 +474,7 @@ crypto::hash Blockchain::get_tail_id(uint64_t& height) //------------------------------------------------------------------ crypto::hash Blockchain::get_tail_id() { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); return m_db->top_block_hash(); } @@ -494,7 +493,7 @@ crypto::hash Blockchain::get_tail_id() */ bool Blockchain::get_short_chain_history(std::list& ids) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); uint64_t i = 0; uint64_t current_multiplier = 1; @@ -503,10 +502,16 @@ bool Blockchain::get_short_chain_history(std::list& ids) if(!sz) return true; - uint64_t current_back_offset = 0; + bool genesis_included = false; + uint64_t current_back_offset = 1; while(current_back_offset < sz) { - ids.push_back(m_db->get_block_hash_from_height(sz - current_back_offset - 1)); + ids.push_back(m_db->get_block_hash_from_height(sz - current_back_offset)); + + if(sz-current_back_offset == 0) + { + genesis_included = true; + } if(i < 10) { ++current_back_offset; @@ -519,12 +524,17 @@ bool Blockchain::get_short_chain_history(std::list& ids) ++i; } + if (!genesis_included) + { + ids.push_back(m_db->get_block_hash_from_height(0)); + } + return true; } //------------------------------------------------------------------ crypto::hash Blockchain::get_block_id_by_height(uint64_t height) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); try { @@ -548,7 +558,7 @@ crypto::hash Blockchain::get_block_id_by_height(uint64_t height) //------------------------------------------------------------------ bool Blockchain::get_block_by_hash(const crypto::hash &h, block &blk) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); // try to find block in main chain @@ -584,7 +594,7 @@ bool Blockchain::get_block_by_hash(const crypto::hash &h, block &blk) // if it ever is, should probably change std::list for std::vector void Blockchain::get_all_known_block_ids(std::list &main, std::list &alt, std::list &invalid) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); for (auto& a : m_db->get_hashes_range(0, m_db->height() - 1)) @@ -605,7 +615,7 @@ void Blockchain::get_all_known_block_ids(std::list &main, std::lis // less blocks than desired if there aren't enough. difficulty_type Blockchain::get_difficulty_for_next_block() { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); std::vector timestamps; std::vector cumulative_difficulties; @@ -632,7 +642,7 @@ difficulty_type Blockchain::get_difficulty_for_next_block() // that had been removed. bool Blockchain::rollback_blockchain_switching(std::list& original_chain, uint64_t rollback_height) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); // remove blocks from blockchain until we get back to where we should be. @@ -661,7 +671,7 @@ bool Blockchain::rollback_blockchain_switching(std::list& original_chain, // boolean based on success therein. bool Blockchain::switch_to_alternative_blockchain(std::list& alt_chain, bool discard_disconnected_chain) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); // if empty alt chain passed (not sure how that could happen), return false @@ -752,7 +762,7 @@ bool Blockchain::switch_to_alternative_blockchain(std::list& alt_chain, block_extended_info& bei) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); std::vector timestamps; std::vector cumulative_difficulties; @@ -821,7 +831,7 @@ difficulty_type Blockchain::get_next_difficulty_for_alternative_chain(const std: // a non-overflowing tx amount (dubious necessity on this check) bool Blockchain::prevalidate_miner_transaction(const block& b, uint64_t height) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); CHECK_AND_ASSERT_MES(b.miner_tx.vin.size() == 1, false, "coinbase transaction in the block has no inputs"); CHECK_AND_ASSERT_MES(b.miner_tx.vin[0].type() == typeid(txin_gen), false, "coinbase transaction in the block has the wrong type"); if(boost::get(b.miner_tx.vin[0]).height != height) @@ -849,7 +859,7 @@ bool Blockchain::prevalidate_miner_transaction(const block& b, uint64_t height) // This function validates the miner transaction reward bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); //validate reward uint64_t money_in_use = 0; BOOST_FOREACH(auto& o, b.miner_tx.vout) @@ -879,7 +889,7 @@ bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_bl // and return by reference . void Blockchain::get_last_n_blocks_sizes(std::vector& sz, size_t count) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); auto h = m_db->height(); @@ -897,7 +907,7 @@ void Blockchain::get_last_n_blocks_sizes(std::vector& sz, size_t count) //------------------------------------------------------------------ uint64_t Blockchain::get_current_cumulative_blocksize_limit() { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); return m_current_block_cumul_sz_limit; } //------------------------------------------------------------------ @@ -914,7 +924,7 @@ uint64_t Blockchain::get_current_cumulative_blocksize_limit() // necessary at all. bool Blockchain::create_block_template(block& b, const account_public_address& miner_address, difficulty_type& diffic, uint64_t& height, const blobdata& ex_nonce) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); size_t median_size; uint64_t already_generated_coins; @@ -1037,7 +1047,7 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m // the needed number of timestamps for the BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW. bool Blockchain::complete_timestamps_vector(uint64_t start_top_height, std::vector& timestamps) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); if(timestamps.size() >= BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW) return true; @@ -1062,7 +1072,7 @@ bool Blockchain::complete_timestamps_vector(uint64_t start_top_height, std::vect // a long forked chain eventually. bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id, block_verification_context& bvc) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); uint64_t block_height = get_block_height(b); @@ -1242,7 +1252,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id //------------------------------------------------------------------ bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::list& blocks, std::list& txs) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); if(start_offset > m_db->height()) return false; @@ -1264,7 +1274,7 @@ bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::list& blocks) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); if(start_offset > m_db->height()) return false; @@ -1281,7 +1291,7 @@ bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::list blocks; @@ -1315,7 +1325,7 @@ bool Blockchain::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NO //------------------------------------------------------------------ bool Blockchain::get_alternative_blocks(std::list& blocks) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); BOOST_FOREACH(const auto& alt_bl, m_alternative_chains) @@ -1327,7 +1337,7 @@ bool Blockchain::get_alternative_blocks(std::list& blocks) //------------------------------------------------------------------ size_t Blockchain::get_alternative_blocks_count() { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); return m_alternative_chains.size(); } @@ -1336,7 +1346,7 @@ size_t Blockchain::get_alternative_blocks_count() // unlocked and other such checks should be done by here. void Blockchain::add_out_to_get_random_outs(COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry& oen = *result_outs.outs.insert(result_outs.outs.end(), COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::out_entry()); @@ -1350,7 +1360,7 @@ void Blockchain::add_out_to_get_random_outs(COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_A // in some cases bool Blockchain::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); srand(static_cast(time(NULL))); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -1423,7 +1433,7 @@ bool Blockchain::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUT // This is used to see what to send another node that needs to sync. bool Blockchain::find_blockchain_supplement(const std::list& qblock_ids, uint64_t& starter_offset) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); // make sure the request includes at least the genesis block, otherwise @@ -1489,7 +1499,7 @@ bool Blockchain::find_blockchain_supplement(const std::list& qbloc //------------------------------------------------------------------ uint64_t Blockchain::block_difficulty(uint64_t i) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); try { @@ -1505,7 +1515,7 @@ uint64_t Blockchain::block_difficulty(uint64_t i) template bool Blockchain::get_blocks(const t_ids_container& block_ids, t_blocks_container& blocks, t_missed_container& missed_bs) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); for (const auto& block_hash : block_ids) @@ -1529,7 +1539,7 @@ bool Blockchain::get_blocks(const t_ids_container& block_ids, t_blocks_container template bool Blockchain::get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); for (const auto& tx_hash : txs_ids) @@ -1553,7 +1563,7 @@ bool Blockchain::get_transactions(const t_ids_container& txs_ids, t_tx_container //------------------------------------------------------------------ void Blockchain::print_blockchain(uint64_t start_index, uint64_t end_index) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); std::stringstream ss; CRITICAL_REGION_LOCAL(m_blockchain_lock); auto h = m_db->height(); @@ -1581,7 +1591,7 @@ void Blockchain::print_blockchain(uint64_t start_index, uint64_t end_index) //------------------------------------------------------------------ void Blockchain::print_blockchain_index() { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); std::stringstream ss; CRITICAL_REGION_LOCAL(m_blockchain_lock); auto height = m_db->height(); @@ -1601,7 +1611,7 @@ void Blockchain::print_blockchain_index() //TODO: remove this function and references to it void Blockchain::print_blockchain_outs(const std::string& file) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); return; } //------------------------------------------------------------------ @@ -1610,7 +1620,7 @@ void Blockchain::print_blockchain_outs(const std::string& file) // BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT additional (more recent) hashes. bool Blockchain::find_blockchain_supplement(const std::list& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); // if we can't find the split point, return false @@ -1634,7 +1644,7 @@ bool Blockchain::find_blockchain_supplement(const std::list& qbloc // blocks by reference. bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); // if a specific start height has been requested @@ -1670,7 +1680,7 @@ bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, cons //------------------------------------------------------------------ bool Blockchain::add_block_as_invalid(const block& bl, const crypto::hash& h) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); block_extended_info bei = AUTO_VAL_INIT(bei); bei.bl = bl; return add_block_as_invalid(bei, h); @@ -1678,7 +1688,7 @@ bool Blockchain::add_block_as_invalid(const block& bl, const crypto::hash& h) //------------------------------------------------------------------ bool Blockchain::add_block_as_invalid(const block_extended_info& bei, const crypto::hash& h) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); auto i_res = m_invalid_blocks.insert(std::map::value_type(h, bei)); CHECK_AND_ASSERT_MES(i_res.second, false, "at insertion invalid by tx returned status existed"); @@ -1688,7 +1698,7 @@ bool Blockchain::add_block_as_invalid(const block_extended_info& bei, const cryp //------------------------------------------------------------------ bool Blockchain::have_block(const crypto::hash& id) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); if(m_db->block_exists(id)) @@ -1705,14 +1715,14 @@ bool Blockchain::have_block(const crypto::hash& id) //------------------------------------------------------------------ bool Blockchain::handle_block_to_main_chain(const block& bl, block_verification_context& bvc) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); crypto::hash id = get_block_hash(bl); return handle_block_to_main_chain(bl, id, bvc); } //------------------------------------------------------------------ size_t Blockchain::get_total_transactions() { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); return m_db->get_tx_count(); } @@ -1725,7 +1735,7 @@ size_t Blockchain::get_total_transactions() // remove them later if the block fails validation. bool Blockchain::check_for_double_spend(const transaction& tx, key_images_container& keys_this_block) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); struct add_transaction_input_visitor: public boost::static_visitor { @@ -1774,7 +1784,7 @@ bool Blockchain::check_for_double_spend(const transaction& tx, key_images_contai //------------------------------------------------------------------ bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector& indexs) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); if (!m_db->tx_exists(tx_id)) { @@ -1791,7 +1801,7 @@ bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector& sig, uint64_t* pmax_related_block_height) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); struct outputs_visitor { - std::vector& m_results_collector; + std::vector& m_p_output_keys; + std::vector& m_output_keys; Blockchain& m_bch; - outputs_visitor(std::vector& results_collector, Blockchain& bch):m_results_collector(results_collector), m_bch(bch) + outputs_visitor(std::vector& output_keys, std::vector& p_output_keys, Blockchain& bch) : m_output_keys(output_keys), m_p_output_keys(p_output_keys), m_bch(bch) {} bool handle_output(const transaction& tx, const tx_out& out) { @@ -1904,20 +1915,26 @@ bool Blockchain::check_tx_input(const txin_to_key& txin, const crypto::hash& tx_ return false; } - m_results_collector.push_back(&boost::get(out.target).key); + m_output_keys.push_back(boost::get(out.target).key); return true; } }; //check ring signature - std::vector output_keys; - outputs_visitor vi(output_keys, *this); + std::vector output_keys; + std::vector p_output_keys; + outputs_visitor vi(output_keys, p_output_keys, *this); if(!scan_outputkeys_for_indexes(txin, vi, pmax_related_block_height)) { LOG_PRINT_L0("Failed to get output keys for tx with amount = " << print_money(txin.amount) << " and count indexes " << txin.key_offsets.size()); return false; } + for (auto& k : output_keys) + { + p_output_keys.push_back(&k); + } + if(txin.key_offsets.size() != output_keys.size()) { LOG_PRINT_L0("Output keys for tx with amount = " << txin.amount << " and count indexes " << txin.key_offsets.size() << " returned wrong keys count " << output_keys.size()); @@ -1926,13 +1943,13 @@ bool Blockchain::check_tx_input(const txin_to_key& txin, const crypto::hash& tx_ CHECK_AND_ASSERT_MES(sig.size() == output_keys.size(), false, "internal error: tx signatures count=" << sig.size() << " mismatch with outputs keys count for inputs=" << output_keys.size()); if(m_is_in_checkpoint_zone) return true; - return crypto::check_ring_signature(tx_prefix_hash, txin.k_image, output_keys, sig.data()); + return crypto::check_ring_signature(tx_prefix_hash, txin.k_image, p_output_keys, sig.data()); } //------------------------------------------------------------------ //TODO: Is this intended to do something else? Need to look into the todo there. uint64_t Blockchain::get_adjusted_time() { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); //TODO: add collecting median time return time(NULL); } @@ -1940,7 +1957,7 @@ uint64_t Blockchain::get_adjusted_time() //TODO: revisit, has changed a bit on upstream bool Blockchain::check_block_timestamp(std::vector& timestamps, const block& b) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); uint64_t median_ts = epee::misc_utils::median(timestamps); if(b.timestamp < median_ts) @@ -1961,7 +1978,7 @@ bool Blockchain::check_block_timestamp(std::vector& timestamps, const // false otherwise bool Blockchain::check_block_timestamp(const block& b) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); if(b.timestamp > get_adjusted_time() + CRYPTONOTE_BLOCK_FUTURE_TIME_LIMIT) { LOG_PRINT_L0("Timestamp of block with id: " << get_block_hash(b) << ", " << b.timestamp << ", bigger than adjusted time + 2 hours"); @@ -1994,7 +2011,7 @@ bool Blockchain::check_block_timestamp(const block& b) // m_db->add_block() bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& id, block_verification_context& bvc) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); // if we already have the block, return false if (have_block(id)) { @@ -2217,7 +2234,7 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& //------------------------------------------------------------------ bool Blockchain::update_next_cumulative_size_limit() { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); std::vector sz; get_last_n_blocks_sizes(sz, CRYPTONOTE_REWARD_BLOCKS_WINDOW); @@ -2231,7 +2248,7 @@ bool Blockchain::update_next_cumulative_size_limit() //------------------------------------------------------------------ bool Blockchain::add_new_block(const block& bl_, block_verification_context& bvc) { - LOG_PRINT_L2("Blockchain::" << __func__); + LOG_PRINT_L3("Blockchain::" << __func__); //copy block here to let modify block.target block bl = bl_; crypto::hash id = get_block_hash(bl); diff --git a/src/cryptonote_core/blockchain_db.cpp b/src/cryptonote_core/blockchain_db.cpp index c7109dd20..439cf5ded 100644 --- a/src/cryptonote_core/blockchain_db.cpp +++ b/src/cryptonote_core/blockchain_db.cpp @@ -45,10 +45,6 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti { crypto::hash tx_hash = get_transaction_hash(tx); - LOG_PRINT_L0("Adding tx with hash " << pod_to_hex(tx_hash) << " to BlockchainDB instance"); - LOG_PRINT_L0("Included in block with hash " << pod_to_hex(blk_hash)); - LOG_PRINT_L0("Unlock time == " << tx.unlock_time); - add_transaction_data(blk_hash, tx); // iterate tx.vout using indices instead of C++11 foreach syntax because @@ -66,10 +62,6 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti { add_spent_key(boost::get(tx_input).k_image); } - else - { - LOG_PRINT_L0("Is miner tx"); - } } } } @@ -82,7 +74,6 @@ uint64_t BlockchainDB::add_block( const block& blk ) { crypto::hash blk_hash = get_block_hash(blk); - LOG_PRINT_L0("Adding block with hash " << pod_to_hex(blk_hash) << " to BlockchainDB instance."); // call out to subclass implementation to add the block & metadata add_block(blk, block_size, cumulative_difficulty, coins_generated); -- cgit v1.2.3 From 71b18d716637066d758243cea191dedcc237b3e5 Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Thu, 30 Oct 2014 14:57:39 -0400 Subject: moar bug fixes, removed debug prints --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 3 --- src/cryptonote_core/blockchain.cpp | 9 ++++----- 2 files changed, 4 insertions(+), 8 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index 58f0d920f..033e177dc 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -1541,7 +1541,6 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con size_t num_elems = 0; mdb_cursor_count(cur, &num_elems); - LOG_PRINT_L0(__func__ << ": amount == " << amount << ", index == " << index << ", num_elem for amount == " << num_elems); if (num_elems <= index) { LOG_PRINT_L1("Attempting to get an output index by amount and amount index, but output not found"); @@ -1594,8 +1593,6 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con txn.commit(); - LOG_PRINT_L0(__func__ << ": tx_hash == " << pod_to_hex(tx_hash) << " tx_index == " << *(uint64_t*)v.mv_data); - return tx_out_index(tx_hash, *(uint64_t *)v.mv_data); } diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 39bf39df3..37ff4248e 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -623,11 +623,10 @@ difficulty_type Blockchain::get_difficulty_for_next_block() size_t offset = h - std::min(h, static_cast(DIFFICULTY_BLOCKS_COUNT)); - // because BlockchainDB::height() returns the index of the top block, the - // first index we need to get needs to be one - // higher than height() - DIFFICULTY_BLOCKS_COUNT. This also conveniently - // makes sure we don't use the genesis block. - ++offset; + if (offset == 0) + { + ++offset; + } for(; offset < h; offset++) { -- cgit v1.2.3 From ab7951d99a6aad3f4de7503e67f89cc664863b87 Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Thu, 30 Oct 2014 18:33:35 -0400 Subject: Minor bugfixes, redundancy removal Minor bugfixes in block removal Storing outputs outside their transactions is largely unnecessary, and thus has been removed. --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 106 ++++++++++++++++++++-- src/cryptonote_core/BlockchainDB_impl/db_lmdb.h | 6 ++ src/cryptonote_core/blockchain_db.h | 2 +- 3 files changed, 104 insertions(+), 10 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index 033e177dc..34451facd 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -363,6 +363,9 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash) LOG_PRINT_L1("Failed to add removal of tx block height to db transaction"); throw DB_ERROR("Failed to add removal of tx block height to db transaction"); } + + remove_tx_outputs(tx_hash); + if (mdb_del(*m_write_txn, m_tx_outputs, &val_h, NULL)) { LOG_PRINT_L1("Failed to add removal of tx outputs to db transaction"); @@ -414,6 +417,8 @@ void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_ou throw DB_ERROR("Failed to add output amount to db transaction"); } +/****** Uncomment if ever outputs actually need to be stored in this manner + * blobdata b = output_to_blob(tx_output); v.mv_size = b.size(); @@ -428,11 +433,59 @@ void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_ou LOG_PRINT_L0("Failed to add output global index to db transaction"); throw DB_ERROR("Failed to add output global index to db transaction"); } +************************************************************************/ m_num_outputs++; } +void BlockchainLMDB::remove_tx_outputs(const crypto::hash& tx_hash) +{ + + lmdb_cur cur(*m_write_txn, m_tx_outputs); + + crypto::hash h_cpy = tx_hash; + MDB_val k; + k.mv_size = sizeof(h_cpy); + k.mv_data = &h_cpy; + + MDB_val v; + + auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + { + LOG_ERROR("Attempting to remove a tx's outputs, but none found. Continuing, but...be wary, because that's weird."); + } + else if (result) + { + LOG_PRINT_L0("DB error attempting to get an output"); + throw DB_ERROR("DB error attempting to get an output"); + } + else + { + size_t num_elems = 0; + mdb_cursor_count(cur, &num_elems); + + mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); + + for (uint64_t i = 0; i < num_elems; ++i) + { + remove_output(*(uint64_t*)v.mv_data); + if (i < num_elems - 1) + { + mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); + } + } + } + + cur.close(); +} + void BlockchainLMDB::remove_output(const tx_out& tx_output) +{ + return; +} + +void BlockchainLMDB::remove_output(const uint64_t& out_index) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -440,6 +493,7 @@ void BlockchainLMDB::remove_output(const tx_out& tx_output) MDB_val k; MDB_val v; +/****** Uncomment if ever outputs actually need to be stored in this manner blobdata b; t_serializable_object_to_blob(tx_output, b); k.mv_size = b.size(); @@ -460,7 +514,15 @@ void BlockchainLMDB::remove_output(const tx_out& tx_output) throw DB_ERROR("Error adding removal of output global index to db transaction"); } - result = mdb_del(*m_write_txn, m_output_indices, &v, NULL); + result = mdb_del(*m_write_txn, m_outputs, &v, NULL); + if (result != 0 && result != MDB_NOTFOUND) + { + LOG_PRINT_L1("Error adding removal of output to db transaction"); + throw DB_ERROR("Error adding removal of output to db transaction"); + } +*********************************************************************/ + + auto result = mdb_del(*m_write_txn, m_output_indices, &v, NULL); if (result != 0 && result != MDB_NOTFOUND) { LOG_PRINT_L1("Error adding removal of output tx index to db transaction"); @@ -481,13 +543,7 @@ void BlockchainLMDB::remove_output(const tx_out& tx_output) throw DB_ERROR("Error adding removal of output amount to db transaction"); } - result = mdb_del(*m_write_txn, m_outputs, &v, NULL); - if (result != 0 && result != MDB_NOTFOUND) - { - LOG_PRINT_L1("Error adding removal of output to db transaction"); - throw DB_ERROR("Error adding removal of output to db transaction"); - } - + m_num_outputs--; } void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image) @@ -674,9 +730,12 @@ void BlockchainLMDB::open(const std::string& filename) lmdb_db_open(txn, LMDB_OUTPUT_TXS, MDB_INTEGERKEY | MDB_CREATE, m_output_txs, "Failed to open db handle for m_output_txs"); lmdb_db_open(txn, LMDB_OUTPUT_INDICES, MDB_INTEGERKEY | MDB_CREATE, m_output_indices, "Failed to open db handle for m_output_indices"); - lmdb_db_open(txn, LMDB_OUTPUT_GINDICES, MDB_CREATE, m_output_gindices, "Failed to open db handle for m_output_gindices"); lmdb_db_open(txn, LMDB_OUTPUT_AMOUNTS, MDB_INTEGERKEY | MDB_DUPSORT | MDB_CREATE, m_output_amounts, "Failed to open db handle for m_output_amounts"); + +/*************** not used, but kept for posterity lmdb_db_open(txn, LMDB_OUTPUTS, MDB_INTEGERKEY | MDB_CREATE, m_outputs, "Failed to open db handle for m_outputs"); + lmdb_db_open(txn, LMDB_OUTPUT_GINDICES, MDB_CREATE, m_output_gindices, "Failed to open db handle for m_output_gindices"); +*************************************************/ lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_CREATE, m_spent_keys, "Failed to open db handle for m_outputs"); @@ -1471,8 +1530,11 @@ tx_out BlockchainLMDB::get_output(const crypto::hash& h, const uint64_t& index) return output_from_blob(b); } +// As this is not used, its return is now a blank output. +// This will save on space in the db. tx_out BlockchainLMDB::get_output(const uint64_t& index) { + return tx_out(); LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -1709,4 +1771,30 @@ uint64_t BlockchainLMDB::add_block( const block& blk return ++m_height; } +void BlockchainLMDB::pop_block(block& blk, std::vector& txs) +{ + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, 0, txn)) + { + LOG_PRINT_L0("Failed to create a transaction for the db"); + throw DB_ERROR("Failed to create a transaction for the db"); + } + m_write_txn = &txn; + + uint64_t num_outputs = m_num_outputs; + try + { + BlockchainDB::pop_block(blk, txs); + + txn.commit(); + } + catch (...) + { + m_num_outputs = num_outputs; + throw; + } + + --m_height; +} + } // namespace cryptonote diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h index a0e9c0a8c..5bc1650c8 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h @@ -174,6 +174,8 @@ public: , const std::vector& txs ); + virtual void pop_block(block& blk, std::vector& txs); + private: virtual void add_block( const block& blk , const size_t& block_size @@ -191,6 +193,10 @@ private: virtual void remove_output(const tx_out& tx_output); + void remove_tx_outputs(const crypto::hash& tx_hash); + + void remove_output(const uint64_t& out_index); + virtual void add_spent_key(const crypto::key_image& k_image); virtual void remove_spent_key(const crypto::key_image& k_image); diff --git a/src/cryptonote_core/blockchain_db.h b/src/cryptonote_core/blockchain_db.h index f59678869..3418b946b 100644 --- a/src/cryptonote_core/blockchain_db.h +++ b/src/cryptonote_core/blockchain_db.h @@ -508,7 +508,7 @@ public: // When a block is popped, the transactions associated with it need to be // removed, as well as outputs and spent key images associated with // those transactions. - void pop_block(block& blk, std::vector& txs); + virtual void pop_block(block& blk, std::vector& txs); // return true if a transaction with hash exists -- cgit v1.2.3 From 006e106ae9235198e286d6503e53fa41865e1601 Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Fri, 31 Oct 2014 17:34:15 -0400 Subject: Store output pubkeys separately, bug fixes --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 62 +++++++++++++++++++---- src/cryptonote_core/BlockchainDB_impl/db_lmdb.h | 1 + 2 files changed, 54 insertions(+), 9 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index 34451facd..bd775ac88 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -95,6 +95,7 @@ const char* LMDB_TX_OUTPUTS = "tx_outputs"; const char* LMDB_OUTPUT_TXS = "output_txs"; const char* LMDB_OUTPUT_INDICES = "output_indices"; const char* LMDB_OUTPUT_AMOUNTS = "output_amounts"; +const char* LMDB_OUTPUT_KEYS = "output_keys"; const char* LMDB_OUTPUTS = "outputs"; const char* LMDB_OUTPUT_GINDICES = "output_gindices"; const char* LMDB_SPENT_KEYS = "spent_keys"; @@ -417,6 +418,20 @@ void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_ou throw DB_ERROR("Failed to add output amount to db transaction"); } + if (tx_output.target.type() == typeid(txout_to_key)) + { + crypto::public_key pubkey = boost::get(tx_output.target).key; + + v.mv_size = sizeof(pubkey); + v.mv_data = &pubkey; + if (mdb_put(*m_write_txn, m_output_keys, &k, &v, 0)) + { + LOG_PRINT_L0("Failed to add output pubkey to db transaction"); + throw DB_ERROR("Failed to add output pubkey to db transaction"); + } + } + + /****** Uncomment if ever outputs actually need to be stored in this manner * blobdata b = output_to_blob(tx_output); @@ -543,6 +558,17 @@ void BlockchainLMDB::remove_output(const uint64_t& out_index) throw DB_ERROR("Error adding removal of output amount to db transaction"); } + result = mdb_del(*m_write_txn, m_output_keys, &v, NULL); + if (result == MDB_NOTFOUND) + { + LOG_PRINT_L2("Removing output, no public key found."); + } + else if (result) + { + LOG_PRINT_L1("Error adding removal of output pubkey to db transaction"); + throw DB_ERROR("Error adding removal of output pubkey to db transaction"); + } + m_num_outputs--; } @@ -691,7 +717,8 @@ void BlockchainLMDB::open(const std::string& filename) LOG_PRINT_L0("Failed to set max number of dbs"); throw DB_ERROR("Failed to set max number of dbs"); } - if (auto result = mdb_env_set_mapsize(m_env, 1 << 29)) + size_t mapsize = 1LL << 32; + if (auto result = mdb_env_set_mapsize(m_env, mapsize)) { LOG_PRINT_L0("Failed to set max memory map size"); LOG_PRINT_L0("E: " << mdb_strerror(result)); @@ -731,6 +758,7 @@ void BlockchainLMDB::open(const std::string& filename) lmdb_db_open(txn, LMDB_OUTPUT_TXS, MDB_INTEGERKEY | MDB_CREATE, m_output_txs, "Failed to open db handle for m_output_txs"); lmdb_db_open(txn, LMDB_OUTPUT_INDICES, MDB_INTEGERKEY | MDB_CREATE, m_output_indices, "Failed to open db handle for m_output_indices"); lmdb_db_open(txn, LMDB_OUTPUT_AMOUNTS, MDB_INTEGERKEY | MDB_DUPSORT | MDB_CREATE, m_output_amounts, "Failed to open db handle for m_output_amounts"); + lmdb_db_open(txn, LMDB_OUTPUT_KEYS, MDB_INTEGERKEY | MDB_CREATE, m_output_keys, "Failed to open db handle for m_output_keys"); /*************** not used, but kept for posterity lmdb_db_open(txn, LMDB_OUTPUTS, MDB_INTEGERKEY | MDB_CREATE, m_outputs, "Failed to open db handle for m_outputs"); @@ -753,10 +781,10 @@ void BlockchainLMDB::open(const std::string& filename) m_height = db_stats.ms_entries; // get and keep current number of outputs - if (mdb_stat(txn, m_outputs, &db_stats)) + if (mdb_stat(txn, m_output_indices, &db_stats)) { - LOG_PRINT_L0("Failed to query m_outputs"); - throw DB_ERROR("Failed to query m_outputs"); + LOG_PRINT_L0("Failed to query m_output_indices"); + throw DB_ERROR("Failed to query m_output_indices"); } m_num_outputs = db_stats.ms_entries; @@ -1457,15 +1485,31 @@ crypto::public_key BlockchainLMDB::get_output_key(const uint64_t& amount, const uint64_t global = get_output_global_index(amount, index); - tx_out output = get_output(global); + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + { + LOG_PRINT_L0("Failed to create a transaction for the db"); + throw DB_ERROR("Failed to create a transaction for the db"); + } - if (output.target.type() != typeid(txout_to_key)) + MDB_val k; + k.mv_size = sizeof(global); + k.mv_data = &global; + + MDB_val v; + auto get_result = mdb_get(txn, m_output_keys, &k, &v); + if (get_result == MDB_NOTFOUND) + { + LOG_PRINT_L0("Attempting to get output pubkey by global index, but key does not exist"); + throw DB_ERROR("Attempting to get output pubkey by global index, but key does not exist"); + } + else if (get_result) { - LOG_PRINT_L1("Attempting to get public key for wrong type of output"); - throw DB_ERROR("Attempting to get public key for wrong type of output"); + LOG_PRINT_L0("Error attempting to retrieve an output pubkey from the db"); + throw DB_ERROR("Error attempting to retrieve an output pubkey from the db"); } - return boost::get(output.target).key; + return *(crypto::public_key*)v.mv_data; } tx_out BlockchainLMDB::get_output(const crypto::hash& h, const uint64_t& index) diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h index 5bc1650c8..e74ca4b73 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h @@ -250,6 +250,7 @@ private: MDB_dbi m_output_indices; MDB_dbi m_output_gindices; MDB_dbi m_output_amounts; + MDB_dbi m_output_keys; MDB_dbi m_outputs; MDB_dbi m_spent_keys; -- cgit v1.2.3 From 26a7db38eb59110e71a8a6a6d05124d2c5417d2b Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Sat, 1 Nov 2014 19:03:37 -0400 Subject: add new checkpointing behavior to Blockchain class --- src/cryptonote_core/blockchain.cpp | 75 +++++++++++++++++++++++++++++++++++++- src/cryptonote_core/blockchain.h | 6 ++- 2 files changed, 78 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 37ff4248e..03eac11d4 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -50,6 +50,7 @@ #include "common/boost_serialization_helper.h" #include "warnings.h" #include "crypto/hash.h" +#include "cryptonote_core/checkpoints_create.h" //#include "serialization/json_archive.h" /* TODO: @@ -65,7 +66,7 @@ DISABLE_VS_WARNINGS(4267) //------------------------------------------------------------------ // TODO: initialize m_db with a concrete implementation of BlockchainDB -Blockchain::Blockchain(tx_memory_pool& tx_pool):m_db(), m_tx_pool(tx_pool), m_current_block_cumul_sz_limit(0), m_is_in_checkpoint_zone(false), m_is_blockchain_storing(false) +Blockchain::Blockchain(tx_memory_pool& tx_pool):m_db(), m_tx_pool(tx_pool), m_current_block_cumul_sz_limit(0), m_is_in_checkpoint_zone(false), m_is_blockchain_storing(false), m_testnet(false), m_enforce_dns_checkpoints(false) { LOG_PRINT_L3("Blockchain::" << __func__); } @@ -2271,3 +2272,75 @@ bool Blockchain::add_new_block(const block& bl_, block_verification_context& bvc return handle_block_to_main_chain(bl, id, bvc); } +//------------------------------------------------------------------ +void Blockchain::check_against_checkpoints(checkpoints& points, bool enforce) +{ + const auto& pts = points.get_points(); + + for (const auto& pt : pts) + { + // if the checkpoint is for a block we don't have yet, move on + if (pt.first >= m_db->height()) + { + continue; + } + + if (!points.check_block(pt.first, m_db->get_block_hash_from_height(pt.first))) + { + // if asked to enforce checkpoints, roll back to a couple of blocks before the checkpoint + if (enforce) + { + LOG_ERROR("Local blockchain failed to pass a checkpoint, rolling back!"); + std::list empty; + rollback_blockchain_switching(empty, pt.first - 2); + } + else + { + LOG_ERROR("WARNING: local blockchain failed to pass a MoneroPulse checkpoint, and you could be on a fork. You should either sync up from scratch, OR download a fresh blockchain bootstrap, OR enable checkpoint enforcing with the --enforce-dns-checkpointing command-line option"); + } + } + } +} +//------------------------------------------------------------------ +// returns false if any of the checkpoints loading returns false. +// That should happen only if a checkpoint is added that conflicts +// with an existing checkpoint. +bool Blockchain::update_checkpoints(const std::string& file_path, bool check_dns) +{ + if (!cryptonote::load_checkpoints_from_json(m_checkpoints, file_path)) + { + return false; + } + + // if we're checking both dns and json, load checkpoints from dns. + // if we're not hard-enforcing dns checkpoints, handle accordingly + if (m_enforce_dns_checkpoints && check_dns) + { + if (!cryptonote::load_checkpoints_from_dns(m_checkpoints)) + { + return false; + } + } + else if (check_dns) + { + checkpoints dns_points; + cryptonote::load_checkpoints_from_dns(dns_points); + if (m_checkpoints.check_for_conflicts(dns_points)) + { + check_against_checkpoints(dns_points, false); + } + else + { + LOG_PRINT_L0("One or more checkpoints fetched from DNS conflicted with existing checkpoints!"); + } + } + + check_against_checkpoints(m_checkpoints, true); + + return true; +} +//------------------------------------------------------------------ +void Blockchain::set_enforce_dns_checkpoints(bool enforce_checkpoints) +{ + m_enforce_dns_checkpoints = enforce_checkpoints; +} diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index 4024fa741..75a729a09 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -141,8 +141,9 @@ namespace cryptonote void print_blockchain_index(); void print_blockchain_outs(const std::string& file); - void set_enforce_dns_checkpoints(bool enforce) { } - bool update_checkpoints(const std::string& path, bool dns) { return true; } + void check_against_checkpoints(checkpoints& points, bool enforce); + void set_enforce_dns_checkpoints(bool enforce); + bool update_checkpoints(const std::string& file_path, bool check_dns); private: typedef std::unordered_map blocks_by_id_index; @@ -179,6 +180,7 @@ namespace cryptonote std::atomic m_is_in_checkpoint_zone; std::atomic m_is_blockchain_storing; bool m_testnet; + bool m_enforce_dns_checkpoints; bool switch_to_alternative_blockchain(std::list& alt_chain, bool discard_disconnected_chain); block pop_block_from_blockchain(); -- cgit v1.2.3 From 4af0918501c40c20f9b42d1d2d7da154d365a58b Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Sat, 1 Nov 2014 20:55:22 -0400 Subject: very, VERY primitive blockchain converter hard-coded config folder, hard-coded BlockchainDB subclass. Needs finessing, but should be testable this way. update for rebase (warptangent 2015-01-04) fix conflicts with upstream CMakeLists.txt files src/CMakeLists.txt (edit original commit) src/blockchain_converter/CMakeLists.txt (add) --- src/CMakeLists.txt | 2 + src/blockchain_converter/CMakeLists.txt | 51 ++++++++++++ src/blockchain_converter/blockchain_converter.cpp | 99 +++++++++++++++++++++++ src/cryptonote_core/blockchain_storage.cpp | 22 ++--- src/cryptonote_core/blockchain_storage.h | 11 ++- 5 files changed, 171 insertions(+), 14 deletions(-) create mode 100644 src/blockchain_converter/CMakeLists.txt create mode 100644 src/blockchain_converter/blockchain_converter.cpp (limited to 'src') diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a80dfe378..7c4492b01 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -97,3 +97,5 @@ add_subdirectory(connectivity_tool) add_subdirectory(miner) add_subdirectory(simplewallet) add_subdirectory(daemon) + +add_subdirectory(blockchain_converter) diff --git a/src/blockchain_converter/CMakeLists.txt b/src/blockchain_converter/CMakeLists.txt new file mode 100644 index 000000000..713ba18ef --- /dev/null +++ b/src/blockchain_converter/CMakeLists.txt @@ -0,0 +1,51 @@ +# Copyright (c) 2014-2015, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set(blockchain_converter_sources + blockchain_converter.cpp + ) + +set(blockchain_converter_private_headers) + +bitmonero_private_headers(blockchain_converter + ${blockchain_converter_private_headers}) + +bitmonero_add_executable(blockchain_converter + ${blockchain_converter_sources} + ${blockchain_converter_private_headers}) + +target_link_libraries(blockchain_converter + LINK_PRIVATE + cryptonote_core + ${CMAKE_THREAD_LIBS_INIT}) + +add_dependencies(blockchain_converter + version) +set_property(TARGET blockchain_converter + PROPERTY + OUTPUT_NAME "blockchain_converter") diff --git a/src/blockchain_converter/blockchain_converter.cpp b/src/blockchain_converter/blockchain_converter.cpp new file mode 100644 index 000000000..5dc937b3b --- /dev/null +++ b/src/blockchain_converter/blockchain_converter.cpp @@ -0,0 +1,99 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "include_base_utils.h" +#include "common/util.h" +#include "warnings.h" +#include "crypto/crypto.h" +#include "cryptonote_config.h" +#include "cryptonote_core/cryptonote_format_utils.h" +#include "misc_language.h" +#include "cryptonote_core/blockchain_storage.h" +#include "cryptonote_core/blockchain_db.h" +#include "cryptonote_core/blockchain.h" +#include "cryptonote_core/BlockchainDB_impl/db_lmdb.h" +#include "cryptonote_core/tx_pool.h" + +using namespace cryptonote; + +struct fake_core +{ + tx_memory_pool m_pool; + Blockchain dummy; + + blockchain_storage m_storage; + + + fake_core() : m_pool(dummy), dummy(m_pool), m_storage(&m_pool) + { + m_pool.init("~/.bitmonero"); + m_storage.init("~/.bitmonero", false); + } +}; + +int main(int argc, char* argv[]) +{ + fake_core c; + + BlockchainDB *blockchain; + + blockchain = new BlockchainLMDB(); + + blockchain->open("~/.bitmonero"); + + for (uint64_t i = 0; i < c.m_storage.get_current_blockchain_height(); ++i) + { + block b = c.m_storage.get_block(i); + size_t bsize = c.m_storage.get_block_size(i); + difficulty_type bdiff = c.m_storage.get_block_cumulative_difficulty(i); + uint64_t bcoins = c.m_storage.get_block_coins_generated(i); + std::vector txs; + std::vector missed; + + c.m_storage.get_transactions(b.tx_hashes, txs, missed); + if (missed.size()) + { + std::cerr << "Missed transaction(s) for block at height " << i << ", exiting" << std::endl; + delete blockchain; + return 1; + } + + try + { + blockchain->add_block(b, bsize, bdiff, bcoins, txs); + } + catch (const std::exception& e) + { + std::cerr << "Error adding block to new blockchain: " << e.what() << std::endl; + delete blockchain; + return 2; + } + } + + return 0; +} diff --git a/src/cryptonote_core/blockchain_storage.cpp b/src/cryptonote_core/blockchain_storage.cpp index e2b6f2326..e40eb6a0f 100644 --- a/src/cryptonote_core/blockchain_storage.cpp +++ b/src/cryptonote_core/blockchain_storage.cpp @@ -231,7 +231,7 @@ bool blockchain_storage::pop_block_from_blockchain() m_blocks_index.erase(bl_ind); //pop block from core m_blocks.pop_back(); - m_tx_pool.on_blockchain_dec(m_blocks.size()-1, get_tail_id()); + m_tx_pool->on_blockchain_dec(m_blocks.size()-1, get_tail_id()); return true; } //------------------------------------------------------------------ @@ -307,7 +307,7 @@ bool blockchain_storage::purge_transaction_from_blockchain(const crypto::hash& t if(!is_coinbase(tx)) { cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc); - bool r = m_tx_pool.add_tx(tx, tvc, true); + bool r = m_tx_pool->add_tx(tx, tvc, true); CHECK_AND_ASSERT_MES(r, false, "purge_block_data_from_blockchain: failed to add transaction to transaction pool"); } @@ -676,16 +676,16 @@ bool blockchain_storage::create_block_template(block& b, const account_public_ad size_t txs_size; uint64_t fee; - if (!m_tx_pool.fill_block_template(b, median_size, already_generated_coins, txs_size, fee)) { + if (!m_tx_pool->fill_block_template(b, median_size, already_generated_coins, txs_size, fee)) { return false; } #if defined(DEBUG_CREATE_BLOCK_TEMPLATE) size_t real_txs_size = 0; uint64_t real_fee = 0; - CRITICAL_REGION_BEGIN(m_tx_pool.m_transactions_lock); + CRITICAL_REGION_BEGIN(m_tx_pool->m_transactions_lock); BOOST_FOREACH(crypto::hash &cur_hash, b.tx_hashes) { - auto cur_res = m_tx_pool.m_transactions.find(cur_hash); - if (cur_res == m_tx_pool.m_transactions.end()) { + auto cur_res = m_tx_pool->m_transactions.find(cur_hash); + if (cur_res == m_tx_pool->m_transactions.end()) { LOG_ERROR("Creating block template: error: transaction not found"); continue; } @@ -1658,7 +1658,7 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt transaction tx; size_t blob_size = 0; uint64_t fee = 0; - if(!m_tx_pool.take_tx(tx_id, tx, blob_size, fee)) + if(!m_tx_pool->take_tx(tx_id, tx, blob_size, fee)) { LOG_PRINT_L1("Block with id: " << id << "has at least one unknown transaction with id: " << tx_id); purge_block_data_from_blockchain(bl, tx_processed_count); @@ -1670,7 +1670,7 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt { LOG_PRINT_L1("Block with id: " << id << "has at least one transaction (id: " << tx_id << ") with wrong inputs."); cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc); - bool add_res = m_tx_pool.add_tx(tx, tvc, true); + bool add_res = m_tx_pool->add_tx(tx, tvc, true); CHECK_AND_ASSERT_MES2(add_res, "handle_block_to_main_chain: failed to add transaction back to transaction pool"); purge_block_data_from_blockchain(bl, tx_processed_count); add_block_as_invalid(bl, id); @@ -1683,7 +1683,7 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt { LOG_PRINT_L1("Block with id: " << id << " failed to add transaction to blockchain storage"); cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc); - bool add_res = m_tx_pool.add_tx(tx, tvc, true); + bool add_res = m_tx_pool->add_tx(tx, tvc, true); CHECK_AND_ASSERT_MES2(add_res, "handle_block_to_main_chain: failed to add transaction back to transaction pool"); purge_block_data_from_blockchain(bl, tx_processed_count); bvc.m_verifivation_failed = true; @@ -1738,7 +1738,7 @@ bool blockchain_storage::handle_block_to_main_chain(const block& bl, const crypt /*if(!m_orphanes_reorganize_in_work) review_orphaned_blocks_with_new_block_id(id, true);*/ - m_tx_pool.on_blockchain_inc(bei.height, id); + m_tx_pool->on_blockchain_inc(bei.height, id); //LOG_PRINT_L0("BLOCK: " << ENDL << "" << dump_obj_as_json(bei.bl)); return true; } @@ -1761,7 +1761,7 @@ bool blockchain_storage::add_new_block(const block& bl_, block_verification_cont //copy block here to let modify block.target block bl = bl_; crypto::hash id = get_block_hash(bl); - CRITICAL_REGION_LOCAL(m_tx_pool);//to avoid deadlock lets lock tx_pool for whole add/reorganize process + CRITICAL_REGION_LOCAL(*m_tx_pool);//to avoid deadlock lets lock tx_pool for whole add/reorganize process CRITICAL_REGION_LOCAL1(m_blockchain_lock); if(have_block(id)) { diff --git a/src/cryptonote_core/blockchain_storage.h b/src/cryptonote_core/blockchain_storage.h index 1bfdf7bd0..671da40a8 100644 --- a/src/cryptonote_core/blockchain_storage.h +++ b/src/cryptonote_core/blockchain_storage.h @@ -78,7 +78,7 @@ namespace cryptonote uint64_t already_generated_coins; }; - blockchain_storage(tx_memory_pool& tx_pool):m_tx_pool(tx_pool), m_current_block_cumul_sz_limit(0), m_is_in_checkpoint_zone(false), m_is_blockchain_storing(false), m_enforce_dns_checkpoints(false) + blockchain_storage(tx_memory_pool* tx_pool):m_tx_pool(tx_pool), m_current_block_cumul_sz_limit(0), m_is_in_checkpoint_zone(false), m_is_blockchain_storing(false), m_enforce_dns_checkpoints(false) {}; bool init() { return init(tools::get_default_data_dir(), true); } @@ -166,7 +166,7 @@ namespace cryptonote if(it == m_transactions.end()) { transaction tx; - if(!m_tx_pool.get_transaction(tx_id, tx)) + if(!m_tx_pool->get_transaction(tx_id, tx)) missed_txs.push_back(tx_id); else txs.push_back(tx); @@ -184,6 +184,11 @@ namespace cryptonote bool update_checkpoints(const std::string& file_path, bool check_dns); void set_enforce_dns_checkpoints(bool enforce_checkpoints); + block get_block(uint64_t height) { return m_blocks[height].bl; } + size_t get_block_size(uint64_t height) { return m_blocks[height].block_cumulative_size; } + difficulty_type get_block_cumulative_difficulty(uint64_t height) { return m_blocks[height].cumulative_difficulty; } + uint64_t get_block_coins_generated(uint64_t height) { return m_blocks[height].already_generated_coins; } + private: typedef std::unordered_map blocks_by_id_index; typedef std::unordered_map transactions_container; @@ -193,7 +198,7 @@ namespace cryptonote typedef std::unordered_map blocks_by_hash; typedef std::map>> outputs_container; //crypto::hash - tx hash, size_t - index of out in transaction - tx_memory_pool& m_tx_pool; + tx_memory_pool* m_tx_pool; epee::critical_section m_blockchain_lock; // TODO: add here reader/writer lock // main chain -- cgit v1.2.3 From 9455e0cd58871812294ceeeef44b053aea8e0671 Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Sat, 1 Nov 2014 21:13:15 -0400 Subject: ~ didn't work, need hard path. debug print. --- src/blockchain_converter/blockchain_converter.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/blockchain_converter/blockchain_converter.cpp b/src/blockchain_converter/blockchain_converter.cpp index 5dc937b3b..b18fd3525 100644 --- a/src/blockchain_converter/blockchain_converter.cpp +++ b/src/blockchain_converter/blockchain_converter.cpp @@ -38,6 +38,7 @@ #include "cryptonote_core/blockchain.h" #include "cryptonote_core/BlockchainDB_impl/db_lmdb.h" #include "cryptonote_core/tx_pool.h" +#include using namespace cryptonote; @@ -51,8 +52,8 @@ struct fake_core fake_core() : m_pool(dummy), dummy(m_pool), m_storage(&m_pool) { - m_pool.init("~/.bitmonero"); - m_storage.init("~/.bitmonero", false); + m_pool.init("/home/user/.bitmonero"); + m_storage.init("/home/user/.bitmonero", false); } }; @@ -64,10 +65,11 @@ int main(int argc, char* argv[]) blockchain = new BlockchainLMDB(); - blockchain->open("~/.bitmonero"); + blockchain->open("/home/user/.bitmonero"); for (uint64_t i = 0; i < c.m_storage.get_current_blockchain_height(); ++i) { + if (i % 10 == 0) std::cout << "block " << i << std::endl; block b = c.m_storage.get_block(i); size_t bsize = c.m_storage.get_block_size(i); difficulty_type bdiff = c.m_storage.get_block_cumulative_difficulty(i); -- cgit v1.2.3 From 6c8b8acfe45f8b2f20df48fed88ea074a26d653f Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Wed, 5 Nov 2014 21:35:51 -0500 Subject: more blockchain height-related fixes, syncing other nodes code this time --- src/cryptonote_core/blockchain.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 03eac11d4..21ec14d3e 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -1631,7 +1631,7 @@ bool Blockchain::find_blockchain_supplement(const std::list& qbloc resp.total_height = get_current_blockchain_height(); size_t count = 0; - for(size_t i = resp.start_height; i <= m_db->height() && count < BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT; i++, count++) + for(size_t i = resp.start_height; i < resp.total_height && count < BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT; i++, count++) { resp.m_block_ids.push_back(m_db->get_block_hash_from_height(i)); } @@ -1651,7 +1651,7 @@ bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, cons if(req_start_block > 0) { // if requested height is higher than our chain, return false -- we can't help - if (req_start_block > m_db->height()) + if (req_start_block >= m_db->height()) { return false; } @@ -1667,12 +1667,12 @@ bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, cons total_height = get_current_blockchain_height(); size_t count = 0; - for(size_t i = start_height; i <= m_db->height() && count < max_count; i++, count++) + for(size_t i = start_height; i < total_height && count < max_count; i++, count++) { blocks.resize(blocks.size()+1); blocks.back().first = m_db->get_block_from_height(i); std::list mis; - get_transactions(m_blocks[i].bl.tx_hashes, blocks.back().second, mis); + get_transactions(blocks.back().first.tx_hashes, blocks.back().second, mis); CHECK_AND_ASSERT_MES(!mis.size(), false, "internal error, transaction from block not found"); } return true; -- cgit v1.2.3 From 8e1b7e2ad44d31b7d3e91dadbadd0eef99aa1e54 Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Wed, 12 Nov 2014 18:27:25 -0500 Subject: raised maximum mapsize for lmdb to ~16GB --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index bd775ac88..a123a3643 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -717,7 +717,7 @@ void BlockchainLMDB::open(const std::string& filename) LOG_PRINT_L0("Failed to set max number of dbs"); throw DB_ERROR("Failed to set max number of dbs"); } - size_t mapsize = 1LL << 32; + size_t mapsize = 1LL << 34; if (auto result = mdb_env_set_mapsize(m_env, mapsize)) { LOG_PRINT_L0("Failed to set max memory map size"); -- cgit v1.2.3 From 215e63b79fe32b6755882c9626a946be7bc48395 Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Sun, 16 Nov 2014 15:19:48 -0500 Subject: extraneous semicolon in Blockchain::complete_timestamps_vector credit here: https://bitcointalk.org/index.php?topic=583449.msg9562845#msg9562845 --- src/cryptonote_core/blockchain.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 21ec14d3e..647b9405d 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -1056,7 +1056,7 @@ bool Blockchain::complete_timestamps_vector(uint64_t start_top_height, std::vect size_t need_elements = BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW - timestamps.size(); CHECK_AND_ASSERT_MES(start_top_height < m_db->height(), false, "internal error: passed start_height not < " << " m_db->height() -- " << start_top_height << " >= " << m_db->height()); size_t stop_offset = start_top_height > need_elements ? start_top_height - need_elements : 0; - while (start_top_height != stop_offset); + while (start_top_height != stop_offset) { timestamps.push_back(m_db->get_block_timestamp(start_top_height)); --start_top_height; -- cgit v1.2.3 From 0886183568a41b41c83e9cd7f624de295d8367c3 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sat, 6 Dec 2014 15:47:49 +0000 Subject: build: add liblmdb to the cmake autodetection system update for rebase (warptangent 2015-01-04) src/cryptonote_core/CMakeLists.txt (edit) - replace LMDB_LIBRARIES with LMDB_LIBRARY set from autodetection --- src/cryptonote_core/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/cryptonote_core/CMakeLists.txt b/src/cryptonote_core/CMakeLists.txt index 93c3cb51e..fc9cc629b 100644 --- a/src/cryptonote_core/CMakeLists.txt +++ b/src/cryptonote_core/CMakeLists.txt @@ -83,5 +83,5 @@ target_link_libraries(cryptonote_core ${Boost_FILESYSTEM_LIBRARY} ${Boost_SYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY} - ${LMDB_LIBRARIES} + ${LMDB_LIBRARY} ${EXTRA_LIBRARIES}) -- cgit v1.2.3 From 11129b9ee4e1f022ca1ae1c26d8f50e8aa8d3e03 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sat, 6 Dec 2014 16:12:25 +0000 Subject: blockchain_converter: use the actual blockchain location --- src/blockchain_converter/blockchain_converter.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/blockchain_converter/blockchain_converter.cpp b/src/blockchain_converter/blockchain_converter.cpp index b18fd3525..a2b3a375f 100644 --- a/src/blockchain_converter/blockchain_converter.cpp +++ b/src/blockchain_converter/blockchain_converter.cpp @@ -52,20 +52,22 @@ struct fake_core fake_core() : m_pool(dummy), dummy(m_pool), m_storage(&m_pool) { - m_pool.init("/home/user/.bitmonero"); - m_storage.init("/home/user/.bitmonero", false); + boost::filesystem::path default_data_path {tools::get_default_data_dir()}; + m_pool.init(default_data_path.string()); + m_storage.init(default_data_path.string(), false); } }; int main(int argc, char* argv[]) { fake_core c; + boost::filesystem::path default_data_path {tools::get_default_data_dir()}; BlockchainDB *blockchain; blockchain = new BlockchainLMDB(); - blockchain->open("/home/user/.bitmonero"); + blockchain->open(default_data_path.string()); for (uint64_t i = 0; i < c.m_storage.get_current_blockchain_height(); ++i) { -- cgit v1.2.3 From 10fd6cab6ce65e114e9148d905cd0fa6a9b86e07 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sat, 6 Dec 2014 21:21:17 +0000 Subject: blockchain_db: factor some exception code Ideally, the log would go in the exception's ctor, but two log levels are used, so I'd need to specify the level in the ctor, which isn't great as it's not really related to the exception. --- src/cryptonote_core/blockchain_db.h | 196 +++++++++--------------------------- 1 file changed, 48 insertions(+), 148 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/blockchain_db.h b/src/cryptonote_core/blockchain_db.h index 3418b946b..95a01b9e3 100644 --- a/src/cryptonote_core/blockchain_db.h +++ b/src/cryptonote_core/blockchain_db.h @@ -139,15 +139,16 @@ typedef std::pair tx_out_index; /*********************************** * Exception Definitions ***********************************/ -class DB_ERROR : public std::exception +class DB_EXCEPTION : public std::exception { private: std::string m; - public: - DB_ERROR() : m("Generic DB Error") { } - DB_ERROR(const char* s) : m(s) { } - virtual ~DB_ERROR() { } + protected: + DB_EXCEPTION(const char *s) : m(s) { } + + public: + virtual ~DB_EXCEPTION() { } const char* what() const throw() { @@ -155,196 +156,95 @@ class DB_ERROR : public std::exception } }; -class DB_OPEN_FAILURE : public std::exception +class DB_ERROR : public DB_EXCEPTION { - private: - std::string m; public: - DB_OPEN_FAILURE() : m("Failed to open the db") { } - DB_OPEN_FAILURE(const char* s) : m(s) { } - - virtual ~DB_OPEN_FAILURE() { } - - const char* what() const throw() - { - return m.c_str(); - } + DB_ERROR() : DB_EXCEPTION("Generic DB Error") { } + DB_ERROR(const char* s) : DB_EXCEPTION(s) { } }; -class DB_CREATE_FAILURE : public std::exception +class DB_OPEN_FAILURE : public DB_EXCEPTION { - private: - std::string m; public: - DB_CREATE_FAILURE() : m("Failed to create the db") { } - DB_CREATE_FAILURE(const char* s) : m(s) { } - - virtual ~DB_CREATE_FAILURE() { } - - const char* what() const throw() - { - return m.c_str(); - } + DB_OPEN_FAILURE() : DB_EXCEPTION("Failed to open the db") { } + DB_OPEN_FAILURE(const char* s) : DB_EXCEPTION(s) { } }; -class DB_SYNC_FAILURE : public std::exception +class DB_CREATE_FAILURE : public DB_EXCEPTION { - private: - std::string m; public: - DB_SYNC_FAILURE() : m("Failed to sync the db") { } - DB_SYNC_FAILURE(const char* s) : m(s) { } - - virtual ~DB_SYNC_FAILURE() { } - - const char* what() const throw() - { - return m.c_str(); - } + DB_CREATE_FAILURE() : DB_EXCEPTION("Failed to create the db") { } + DB_CREATE_FAILURE(const char* s) : DB_EXCEPTION(s) { } }; -class BLOCK_DNE : public std::exception +class DB_SYNC_FAILURE : public DB_EXCEPTION { - private: - std::string m; public: - BLOCK_DNE() : m("The block requested does not exist") { } - BLOCK_DNE(const char* s) : m(s) { } - - virtual ~BLOCK_DNE() { } - - const char* what() const throw() - { - return m.c_str(); - } + DB_SYNC_FAILURE() : DB_EXCEPTION("Failed to sync the db") { } + DB_SYNC_FAILURE(const char* s) : DB_EXCEPTION(s) { } }; -class BLOCK_PARENT_DNE : public std::exception +class BLOCK_DNE : public DB_EXCEPTION { - private: - std::string m; public: - BLOCK_PARENT_DNE() : m("The parent of the block does not exist") { } - BLOCK_PARENT_DNE(const char* s) : m(s) { } - - virtual ~BLOCK_PARENT_DNE() { } - - const char* what() const throw() - { - return m.c_str(); - } + BLOCK_DNE() : DB_EXCEPTION("The block requested does not exist") { } + BLOCK_DNE(const char* s) : DB_EXCEPTION(s) { } }; -class BLOCK_EXISTS : public std::exception +class BLOCK_PARENT_DNE : public DB_EXCEPTION { - private: - std::string m; public: - BLOCK_EXISTS() : m("The block to be added already exists!") { } - BLOCK_EXISTS(const char* s) : m(s) { } - - virtual ~BLOCK_EXISTS() { } - - const char* what() const throw() - { - return m.c_str(); - } + BLOCK_PARENT_DNE() : DB_EXCEPTION("The parent of the block does not exist") { } + BLOCK_PARENT_DNE(const char* s) : DB_EXCEPTION(s) { } }; -class BLOCK_INVALID : public std::exception +class BLOCK_EXISTS : public DB_EXCEPTION { - private: - std::string m; public: - BLOCK_INVALID() : m("The block to be added did not pass validation!") { } - BLOCK_INVALID(const char* s) : m(s) { } - - virtual ~BLOCK_INVALID() { } - - const char* what() const throw() - { - return m.c_str(); - } + BLOCK_EXISTS() : DB_EXCEPTION("The block to be added already exists!") { } + BLOCK_EXISTS(const char* s) : DB_EXCEPTION(s) { } }; -class TX_DNE : public std::exception +class BLOCK_INVALID : public DB_EXCEPTION { - private: - std::string m; public: - TX_DNE() : m("The transaction requested does not exist") { } - TX_DNE(const char* s) : m(s) { } - - virtual ~TX_DNE() { } - - const char* what() const throw() - { - return m.c_str(); - } + BLOCK_INVALID() : DB_EXCEPTION("The block to be added did not pass validation!") { } + BLOCK_INVALID(const char* s) : DB_EXCEPTION(s) { } }; -class TX_EXISTS : public std::exception +class TX_DNE : public DB_EXCEPTION { - private: - std::string m; public: - TX_EXISTS() : m("The transaction to be added already exists!") { } - TX_EXISTS(const char* s) : m(s) { } - - virtual ~TX_EXISTS() { } - - const char* what() const throw() - { - return m.c_str(); - } + TX_DNE() : DB_EXCEPTION("The transaction requested does not exist") { } + TX_DNE(const char* s) : DB_EXCEPTION(s) { } }; -class OUTPUT_DNE : public std::exception +class TX_EXISTS : public DB_EXCEPTION { - private: - std::string m; public: - OUTPUT_DNE() : m("The output requested does not exist!") { } - OUTPUT_DNE(const char* s) : m(s) { } - - virtual ~OUTPUT_DNE() { } - - const char* what() const throw() - { - return m.c_str(); - } + TX_EXISTS() : DB_EXCEPTION("The transaction to be added already exists!") { } + TX_EXISTS(const char* s) : DB_EXCEPTION(s) { } }; -class OUTPUT_EXISTS : public std::exception +class OUTPUT_DNE : public DB_EXCEPTION { - private: - std::string m; public: - OUTPUT_EXISTS() : m("The output to be added already exists!") { } - OUTPUT_EXISTS(const char* s) : m(s) { } - - virtual ~OUTPUT_EXISTS() { } - - const char* what() const throw() - { - return m.c_str(); - } + OUTPUT_DNE() : DB_EXCEPTION("The output requested does not exist!") { } + OUTPUT_DNE(const char* s) : DB_EXCEPTION(s) { } }; -class KEY_IMAGE_EXISTS : public std::exception +class OUTPUT_EXISTS : public DB_EXCEPTION { - private: - std::string m; public: - KEY_IMAGE_EXISTS() : m("The spent key image to be added already exists!") { } - KEY_IMAGE_EXISTS(const char* s) : m(s) { } - - virtual ~KEY_IMAGE_EXISTS() { } + OUTPUT_EXISTS() : DB_EXCEPTION("The output to be added already exists!") { } + OUTPUT_EXISTS(const char* s) : DB_EXCEPTION(s) { } +}; - const char* what() const throw() - { - return m.c_str(); - } +class KEY_IMAGE_EXISTS : public DB_EXCEPTION +{ + public: + KEY_IMAGE_EXISTS() : DB_EXCEPTION("The spent key image to be added already exists!") { } + KEY_IMAGE_EXISTS(const char* s) : DB_EXCEPTION(s) { } }; /*********************************** -- cgit v1.2.3 From 23f3cb4c0e3e290a6de65ebce890dfd5a7b0d45b Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sat, 6 Dec 2014 21:37:22 +0000 Subject: blockchain_db: add consts where appropriate --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 110 +++++++++++----------- src/cryptonote_core/BlockchainDB_impl/db_lmdb.h | 70 +++++++------- src/cryptonote_core/blockchain_db.h | 62 ++++++------ 3 files changed, 121 insertions(+), 121 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index a123a3643..2901f9ba7 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -79,26 +79,26 @@ auto compare_uint64 = [](const MDB_val *a, const MDB_val *b) { else return 1; }; -const char* LMDB_BLOCKS = "blocks"; -const char* LMDB_BLOCK_TIMESTAMPS = "block_timestamps"; -const char* LMDB_BLOCK_HEIGHTS = "block_heights"; -const char* LMDB_BLOCK_HASHES = "block_hashes"; -const char* LMDB_BLOCK_SIZES = "block_sizes"; -const char* LMDB_BLOCK_DIFFS = "block_diffs"; -const char* LMDB_BLOCK_COINS = "block_coins"; - -const char* LMDB_TXS = "txs"; -const char* LMDB_TX_UNLOCKS = "tx_unlocks"; -const char* LMDB_TX_HEIGHTS = "tx_heights"; -const char* LMDB_TX_OUTPUTS = "tx_outputs"; - -const char* LMDB_OUTPUT_TXS = "output_txs"; -const char* LMDB_OUTPUT_INDICES = "output_indices"; -const char* LMDB_OUTPUT_AMOUNTS = "output_amounts"; -const char* LMDB_OUTPUT_KEYS = "output_keys"; -const char* LMDB_OUTPUTS = "outputs"; -const char* LMDB_OUTPUT_GINDICES = "output_gindices"; -const char* LMDB_SPENT_KEYS = "spent_keys"; +const char* const LMDB_BLOCKS = "blocks"; +const char* const LMDB_BLOCK_TIMESTAMPS = "block_timestamps"; +const char* const LMDB_BLOCK_HEIGHTS = "block_heights"; +const char* const LMDB_BLOCK_HASHES = "block_hashes"; +const char* const LMDB_BLOCK_SIZES = "block_sizes"; +const char* const LMDB_BLOCK_DIFFS = "block_diffs"; +const char* const LMDB_BLOCK_COINS = "block_coins"; + +const char* const LMDB_TXS = "txs"; +const char* const LMDB_TX_UNLOCKS = "tx_unlocks"; +const char* const LMDB_TX_HEIGHTS = "tx_heights"; +const char* const LMDB_TX_OUTPUTS = "tx_outputs"; + +const char* const LMDB_OUTPUT_TXS = "output_txs"; +const char* const LMDB_OUTPUT_INDICES = "output_indices"; +const char* const LMDB_OUTPUT_AMOUNTS = "output_amounts"; +const char* const LMDB_OUTPUT_KEYS = "output_keys"; +const char* const LMDB_OUTPUTS = "outputs"; +const char* const LMDB_OUTPUT_GINDICES = "output_gindices"; +const char* const LMDB_SPENT_KEYS = "spent_keys"; inline void lmdb_db_open(MDB_txn* txn, const char* name, int flags, MDB_dbi& dbi, const std::string& error_string) { @@ -628,7 +628,7 @@ blobdata BlockchainLMDB::output_to_blob(const tx_out& output) return b; } -tx_out BlockchainLMDB::output_from_blob(const blobdata& blob) +tx_out BlockchainLMDB::output_from_blob(const blobdata& blob) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); std::stringstream ss; @@ -645,13 +645,13 @@ tx_out BlockchainLMDB::output_from_blob(const blobdata& blob) return o; } -uint64_t BlockchainLMDB::get_output_global_index(const uint64_t& amount, const uint64_t& index) +uint64_t BlockchainLMDB::get_output_global_index(const uint64_t& amount, const uint64_t& index) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); return 0; } -void BlockchainLMDB::check_open() +void BlockchainLMDB::check_open() const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); if (!m_open) @@ -821,7 +821,7 @@ void BlockchainLMDB::reset() // TODO: this } -std::vector BlockchainLMDB::get_filenames() +std::vector BlockchainLMDB::get_filenames() const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); std::vector filenames; @@ -853,7 +853,7 @@ void BlockchainLMDB::unlock() } -bool BlockchainLMDB::block_exists(const crypto::hash& h) +bool BlockchainLMDB::block_exists(const crypto::hash& h) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -888,7 +888,7 @@ bool BlockchainLMDB::block_exists(const crypto::hash& h) return true; } -block BlockchainLMDB::get_block(const crypto::hash& h) +block BlockchainLMDB::get_block(const crypto::hash& h) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -896,7 +896,7 @@ block BlockchainLMDB::get_block(const crypto::hash& h) return get_block_from_height(get_block_height(h)); } -uint64_t BlockchainLMDB::get_block_height(const crypto::hash& h) +uint64_t BlockchainLMDB::get_block_height(const crypto::hash& h) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -930,7 +930,7 @@ uint64_t BlockchainLMDB::get_block_height(const crypto::hash& h) return *(uint64_t*)result.mv_data; } -block_header BlockchainLMDB::get_block_header(const crypto::hash& h) +block_header BlockchainLMDB::get_block_header(const crypto::hash& h) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -939,7 +939,7 @@ block_header BlockchainLMDB::get_block_header(const crypto::hash& h) return get_block(h); } -block BlockchainLMDB::get_block_from_height(const uint64_t& height) +block BlockchainLMDB::get_block_from_height(const uint64_t& height) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -983,7 +983,7 @@ block BlockchainLMDB::get_block_from_height(const uint64_t& height) return b; } -uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) +uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -1016,7 +1016,7 @@ uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) return *(uint64_t*)result.mv_data; } -uint64_t BlockchainLMDB::get_top_block_timestamp() +uint64_t BlockchainLMDB::get_top_block_timestamp() const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -1030,7 +1030,7 @@ uint64_t BlockchainLMDB::get_top_block_timestamp() return get_block_timestamp(m_height - 1); } -size_t BlockchainLMDB::get_block_size(const uint64_t& height) +size_t BlockchainLMDB::get_block_size(const uint64_t& height) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -1063,7 +1063,7 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) return *(size_t*)result.mv_data; } -difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& height) +difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& height) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -1096,7 +1096,7 @@ difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& return *(difficulty_type*)result.mv_data; } -difficulty_type BlockchainLMDB::get_block_difficulty(const uint64_t& height) +difficulty_type BlockchainLMDB::get_block_difficulty(const uint64_t& height) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -1113,7 +1113,7 @@ difficulty_type BlockchainLMDB::get_block_difficulty(const uint64_t& height) return diff1 - diff2; } -uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& height) +uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& height) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -1146,7 +1146,7 @@ uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& heigh return *(uint64_t*)result.mv_data; } -crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height) +crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -1179,7 +1179,7 @@ crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height) return *(crypto::hash*)result.mv_data; } -std::vector BlockchainLMDB::get_blocks_range(const uint64_t& h1, const uint64_t& h2) +std::vector BlockchainLMDB::get_blocks_range(const uint64_t& h1, const uint64_t& h2) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -1193,7 +1193,7 @@ std::vector BlockchainLMDB::get_blocks_range(const uint64_t& h1, const ui return v; } -std::vector BlockchainLMDB::get_hashes_range(const uint64_t& h1, const uint64_t& h2) +std::vector BlockchainLMDB::get_hashes_range(const uint64_t& h1, const uint64_t& h2) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -1207,7 +1207,7 @@ std::vector BlockchainLMDB::get_hashes_range(const uint64_t& h1, c return v; } -crypto::hash BlockchainLMDB::top_block_hash() +crypto::hash BlockchainLMDB::top_block_hash() const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -1219,7 +1219,7 @@ crypto::hash BlockchainLMDB::top_block_hash() return null_hash; } -block BlockchainLMDB::get_top_block() +block BlockchainLMDB::get_top_block() const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -1233,7 +1233,7 @@ block BlockchainLMDB::get_top_block() return b; } -uint64_t BlockchainLMDB::height() +uint64_t BlockchainLMDB::height() const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -1242,7 +1242,7 @@ uint64_t BlockchainLMDB::height() } -bool BlockchainLMDB::tx_exists(const crypto::hash& h) +bool BlockchainLMDB::tx_exists(const crypto::hash& h) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -1276,7 +1276,7 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) return true; } -uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) +uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -1309,7 +1309,7 @@ uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) return *(uint64_t*)result.mv_data; } -transaction BlockchainLMDB::get_tx(const crypto::hash& h) +transaction BlockchainLMDB::get_tx(const crypto::hash& h) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -1352,7 +1352,7 @@ transaction BlockchainLMDB::get_tx(const crypto::hash& h) return tx; } -uint64_t BlockchainLMDB::get_tx_count() +uint64_t BlockchainLMDB::get_tx_count() const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -1376,7 +1376,7 @@ uint64_t BlockchainLMDB::get_tx_count() return db_stats.ms_entries; } -std::vector BlockchainLMDB::get_tx_list(const std::vector& hlist) +std::vector BlockchainLMDB::get_tx_list(const std::vector& hlist) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -1390,7 +1390,7 @@ std::vector BlockchainLMDB::get_tx_list(const std::vector() % num_outputs; } -uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) +uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -1478,7 +1478,7 @@ uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) return num_elems; } -crypto::public_key BlockchainLMDB::get_output_key(const uint64_t& amount, const uint64_t& index) +crypto::public_key BlockchainLMDB::get_output_key(const uint64_t& amount, const uint64_t& index) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -1512,7 +1512,7 @@ crypto::public_key BlockchainLMDB::get_output_key(const uint64_t& amount, const return *(crypto::public_key*)v.mv_data; } -tx_out BlockchainLMDB::get_output(const crypto::hash& h, const uint64_t& index) +tx_out BlockchainLMDB::get_output(const crypto::hash& h, const uint64_t& index) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -1576,7 +1576,7 @@ tx_out BlockchainLMDB::get_output(const crypto::hash& h, const uint64_t& index) // As this is not used, its return is now a blank output. // This will save on space in the db. -tx_out BlockchainLMDB::get_output(const uint64_t& index) +tx_out BlockchainLMDB::get_output(const uint64_t& index) const { return tx_out(); LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -1612,7 +1612,7 @@ tx_out BlockchainLMDB::get_output(const uint64_t& index) return output_from_blob(b); } -tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) +tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -1702,7 +1702,7 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con return tx_out_index(tx_hash, *(uint64_t *)v.mv_data); } -std::vector BlockchainLMDB::get_tx_output_indices(const crypto::hash& h) +std::vector BlockchainLMDB::get_tx_output_indices(const crypto::hash& h) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -1754,7 +1754,7 @@ std::vector BlockchainLMDB::get_tx_output_indices(const crypto::hash& return index_vec; } -bool BlockchainLMDB::has_key_image(const crypto::key_image& img) +bool BlockchainLMDB::has_key_image(const crypto::key_image& img) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h index e74ca4b73..3bac5d740 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h @@ -90,65 +90,65 @@ public: virtual void reset(); - virtual std::vector get_filenames(); + virtual std::vector get_filenames() const; virtual bool lock(); virtual void unlock(); - virtual bool block_exists(const crypto::hash& h); + virtual bool block_exists(const crypto::hash& h) const; - virtual block get_block(const crypto::hash& h); + virtual block get_block(const crypto::hash& h) const; - virtual uint64_t get_block_height(const crypto::hash& h); + virtual uint64_t get_block_height(const crypto::hash& h) const; - virtual block_header get_block_header(const crypto::hash& h); + virtual block_header get_block_header(const crypto::hash& h) const; - virtual block get_block_from_height(const uint64_t& height); + virtual block get_block_from_height(const uint64_t& height) const; - virtual uint64_t get_block_timestamp(const uint64_t& height) ; + virtual uint64_t get_block_timestamp(const uint64_t& height) const; - virtual uint64_t get_top_block_timestamp(); + virtual uint64_t get_top_block_timestamp() const; - virtual size_t get_block_size(const uint64_t& height); + virtual size_t get_block_size(const uint64_t& height) const; - virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height); + virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const; - virtual difficulty_type get_block_difficulty(const uint64_t& height); + virtual difficulty_type get_block_difficulty(const uint64_t& height) const; - virtual uint64_t get_block_already_generated_coins(const uint64_t& height); + virtual uint64_t get_block_already_generated_coins(const uint64_t& height) const; - virtual crypto::hash get_block_hash_from_height(const uint64_t& height); + virtual crypto::hash get_block_hash_from_height(const uint64_t& height) const; - virtual std::vector get_blocks_range(const uint64_t& h1, const uint64_t& h2); + virtual std::vector get_blocks_range(const uint64_t& h1, const uint64_t& h2) const; - virtual std::vector get_hashes_range(const uint64_t& h1, const uint64_t& h2); + virtual std::vector get_hashes_range(const uint64_t& h1, const uint64_t& h2) const; - virtual crypto::hash top_block_hash(); + virtual crypto::hash top_block_hash() const; - virtual block get_top_block(); + virtual block get_top_block() const; - virtual uint64_t height(); + virtual uint64_t height() const; - virtual bool tx_exists(const crypto::hash& h); + virtual bool tx_exists(const crypto::hash& h) const; - virtual uint64_t get_tx_unlock_time(const crypto::hash& h); + virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const; - virtual transaction get_tx(const crypto::hash& h); + virtual transaction get_tx(const crypto::hash& h) const; - virtual uint64_t get_tx_count(); + virtual uint64_t get_tx_count() const; - virtual std::vector get_tx_list(const std::vector& hlist); + virtual std::vector get_tx_list(const std::vector& hlist) const; - virtual uint64_t get_tx_block_height(const crypto::hash& h); + virtual uint64_t get_tx_block_height(const crypto::hash& h) const; - virtual uint64_t get_random_output(const uint64_t& amount); + virtual uint64_t get_random_output(const uint64_t& amount) const; - virtual uint64_t get_num_outputs(const uint64_t& amount); + virtual uint64_t get_num_outputs(const uint64_t& amount) const; - virtual crypto::public_key get_output_key(const uint64_t& amount, const uint64_t& index); + virtual crypto::public_key get_output_key(const uint64_t& amount, const uint64_t& index) const; - virtual tx_out get_output(const crypto::hash& h, const uint64_t& index); + virtual tx_out get_output(const crypto::hash& h, const uint64_t& index) const; /** * @brief get an output from its global index @@ -159,13 +159,13 @@ public: * Will throw OUTPUT_DNE if not output has that global index. * Will throw DB_ERROR if there is a non-specific LMDB error in fetching */ - tx_out get_output(const uint64_t& index); + tx_out get_output(const uint64_t& index) const; - virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index); + virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const; - virtual std::vector get_tx_output_indices(const crypto::hash& h); + virtual std::vector get_tx_output_indices(const crypto::hash& h) const; - virtual bool has_key_image(const crypto::key_image& img); + virtual bool has_key_image(const crypto::key_image& img) const; virtual uint64_t add_block( const block& blk , const size_t& block_size @@ -217,7 +217,7 @@ private: * * @return the resultant tx output */ - tx_out output_from_blob(const blobdata& blob); + tx_out output_from_blob(const blobdata& blob) const; /** * @brief get the global index of the index-th output of the given amount @@ -227,9 +227,9 @@ private: * * @return the global index of the desired output */ - uint64_t get_output_global_index(const uint64_t& amount, const uint64_t& index); + uint64_t get_output_global_index(const uint64_t& amount, const uint64_t& index) const; - void check_open(); + void check_open() const; MDB_env* m_env; diff --git a/src/cryptonote_core/blockchain_db.h b/src/cryptonote_core/blockchain_db.h index 95a01b9e3..b9ea79f01 100644 --- a/src/cryptonote_core/blockchain_db.h +++ b/src/cryptonote_core/blockchain_db.h @@ -323,7 +323,7 @@ public: virtual void reset() = 0; // get all files used by this db (if any) - virtual std::vector get_filenames() = 0; + virtual std::vector get_filenames() const = 0; // FIXME: these are just for functionality mocking, need to implement @@ -347,59 +347,59 @@ public: ); // return true if a block with hash exists in the blockchain - virtual bool block_exists(const crypto::hash& h) = 0; + virtual bool block_exists(const crypto::hash& h) const = 0; // return block with hash - virtual block get_block(const crypto::hash& h) = 0; + virtual block get_block(const crypto::hash& h) const = 0; // return the height of the block with hash on the blockchain, // throw if it doesn't exist - virtual uint64_t get_block_height(const crypto::hash& h) = 0; + virtual uint64_t get_block_height(const crypto::hash& h) const = 0; // return header for block with hash - virtual block_header get_block_header(const crypto::hash& h) = 0; + virtual block_header get_block_header(const crypto::hash& h) const = 0; // return block at height - virtual block get_block_from_height(const uint64_t& height) = 0; + virtual block get_block_from_height(const uint64_t& height) const = 0; // return timestamp of block at height - virtual uint64_t get_block_timestamp(const uint64_t& height) = 0; + virtual uint64_t get_block_timestamp(const uint64_t& height) const = 0; // return timestamp of most recent block - virtual uint64_t get_top_block_timestamp() = 0; + virtual uint64_t get_top_block_timestamp() const = 0; // return block size of block at height - virtual size_t get_block_size(const uint64_t& height) = 0; + virtual size_t get_block_size(const uint64_t& height) const = 0; // return cumulative difficulty up to and including block at height - virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height) = 0; + virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const = 0; // return difficulty of block at height - virtual difficulty_type get_block_difficulty(const uint64_t& height) = 0; + virtual difficulty_type get_block_difficulty(const uint64_t& height) const = 0; // return number of coins generated up to and including block at height - virtual uint64_t get_block_already_generated_coins(const uint64_t& height) = 0; + virtual uint64_t get_block_already_generated_coins(const uint64_t& height) const = 0; // return hash of block at height - virtual crypto::hash get_block_hash_from_height(const uint64_t& height) = 0; + virtual crypto::hash get_block_hash_from_height(const uint64_t& height) const = 0; // return vector of blocks in range of height (inclusively) - virtual std::vector get_blocks_range(const uint64_t& h1, const uint64_t& h2) = 0; + virtual std::vector get_blocks_range(const uint64_t& h1, const uint64_t& h2) const = 0; // return vector of block hashes in range of height (inclusively) - virtual std::vector get_hashes_range(const uint64_t& h1, const uint64_t& h2) = 0; + virtual std::vector get_hashes_range(const uint64_t& h1, const uint64_t& h2) const = 0; // return the hash of the top block on the chain - virtual crypto::hash top_block_hash() = 0; + virtual crypto::hash top_block_hash() const = 0; // return the block at the top of the blockchain - virtual block get_top_block() = 0; + virtual block get_top_block() const = 0; // return the index of the top block on the chain // NOTE: for convenience using heights as indices, this is not the total // size of the blockchain, but rather the index of the top block. As the // chain is 0-indexed, the total size will be height() + 1. - virtual uint64_t height() = 0; + virtual uint64_t height() const = 0; // pops the top block off the blockchain. // Returns by reference the popped block and its associated transactions @@ -412,48 +412,48 @@ public: // return true if a transaction with hash exists - virtual bool tx_exists(const crypto::hash& h) = 0; + virtual bool tx_exists(const crypto::hash& h) const = 0; // return unlock time of tx with hash - virtual uint64_t get_tx_unlock_time(const crypto::hash& h) = 0; + virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const = 0; // return tx with hash // throw if no such tx exists - virtual transaction get_tx(const crypto::hash& h) = 0; + virtual transaction get_tx(const crypto::hash& h) const = 0; // returns the total number of transactions in all blocks - virtual uint64_t get_tx_count() = 0; + virtual uint64_t get_tx_count() const = 0; // return list of tx with hashes . // TODO: decide if a missing hash means return empty list // or just skip that hash - virtual std::vector get_tx_list(const std::vector& hlist) = 0; + virtual std::vector get_tx_list(const std::vector& hlist) const = 0; // returns height of block that contains transaction with hash - virtual uint64_t get_tx_block_height(const crypto::hash& h) = 0; + virtual uint64_t get_tx_block_height(const crypto::hash& h) const = 0; // return global output index of a random output of amount - virtual uint64_t get_random_output(const uint64_t& amount) = 0; + virtual uint64_t get_random_output(const uint64_t& amount) const = 0; // returns the total number of outputs of amount - virtual uint64_t get_num_outputs(const uint64_t& amount) = 0; + virtual uint64_t get_num_outputs(const uint64_t& amount) const = 0; // return public key for output with global output amount and index - virtual crypto::public_key get_output_key(const uint64_t& amount, const uint64_t& index) = 0; + virtual crypto::public_key get_output_key(const uint64_t& amount, const uint64_t& index) const = 0; // returns the output indexed by in the transaction with hash - virtual tx_out get_output(const crypto::hash& h, const uint64_t& index) = 0; + virtual tx_out get_output(const crypto::hash& h, const uint64_t& index) const = 0; // returns the transaction-local reference for the output with at // return type is pair of tx hash and index - virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) = 0; + virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const = 0; // return a vector of indices corresponding to the global output index for // each output in the transaction with hash - virtual std::vector get_tx_output_indices(const crypto::hash& h) = 0; + virtual std::vector get_tx_output_indices(const crypto::hash& h) const = 0; // returns true if key image is present in spent key images storage - virtual bool has_key_image(const crypto::key_image& img) = 0; + virtual bool has_key_image(const crypto::key_image& img) const = 0; }; // class BlockchainDB -- cgit v1.2.3 From b7270ab60e74700da4cfb126b2509de4e947eb24 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sat, 6 Dec 2014 22:40:33 +0000 Subject: blockchain: add consts where appropriate --- src/cryptonote_core/blockchain.cpp | 86 +++++++++++++++++++------------------- src/cryptonote_core/blockchain.h | 82 ++++++++++++++++++------------------ 2 files changed, 84 insertions(+), 84 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 647b9405d..75296cf46 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -129,14 +129,14 @@ void Blockchain::serialize(archive_t & ar, const unsigned int version) "m_current_block_cumul_sz_limit: " << m_current_block_cumul_sz_limit); } //------------------------------------------------------------------ -bool Blockchain::have_tx(const crypto::hash &id) +bool Blockchain::have_tx(const crypto::hash &id) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); return m_db->tx_exists(id); } //------------------------------------------------------------------ -bool Blockchain::have_tx_keyimg_as_spent(const crypto::key_image &key_im) +bool Blockchain::have_tx_keyimg_as_spent(const crypto::key_image &key_im) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -147,7 +147,7 @@ bool Blockchain::have_tx_keyimg_as_spent(const crypto::key_image &key_im) // and collects the public key for each from the transaction it was included in // via the visitor passed to it. template -bool Blockchain::scan_outputkeys_for_indexes(const txin_to_key& tx_in_to_key, visitor_t& vis, uint64_t* pmax_related_block_height) +bool Blockchain::scan_outputkeys_for_indexes(const txin_to_key& tx_in_to_key, visitor_t& vis, uint64_t* pmax_related_block_height) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -217,7 +217,7 @@ bool Blockchain::scan_outputkeys_for_indexes(const txin_to_key& tx_in_to_key, vi return true; } //------------------------------------------------------------------ -uint64_t Blockchain::get_current_blockchain_height() +uint64_t Blockchain::get_current_blockchain_height() const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -465,7 +465,7 @@ bool Blockchain::purge_transaction_keyimages_from_blockchain(const transaction& return true; } //------------------------------------------------------------------ -crypto::hash Blockchain::get_tail_id(uint64_t& height) +crypto::hash Blockchain::get_tail_id(uint64_t& height) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -473,7 +473,7 @@ crypto::hash Blockchain::get_tail_id(uint64_t& height) return get_tail_id(); } //------------------------------------------------------------------ -crypto::hash Blockchain::get_tail_id() +crypto::hash Blockchain::get_tail_id() const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -492,7 +492,7 @@ crypto::hash Blockchain::get_tail_id() * powers of 2 less recent from there, so 13, 17, 25, etc... * */ -bool Blockchain::get_short_chain_history(std::list& ids) +bool Blockchain::get_short_chain_history(std::list& ids) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -533,7 +533,7 @@ bool Blockchain::get_short_chain_history(std::list& ids) return true; } //------------------------------------------------------------------ -crypto::hash Blockchain::get_block_id_by_height(uint64_t height) +crypto::hash Blockchain::get_block_id_by_height(uint64_t height) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -557,7 +557,7 @@ crypto::hash Blockchain::get_block_id_by_height(uint64_t height) return null_hash; } //------------------------------------------------------------------ -bool Blockchain::get_block_by_hash(const crypto::hash &h, block &blk) +bool Blockchain::get_block_by_hash(const crypto::hash &h, block &blk) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -593,7 +593,7 @@ bool Blockchain::get_block_by_hash(const crypto::hash &h, block &blk) //------------------------------------------------------------------ //FIXME: this function does not seem to be called from anywhere, but // if it ever is, should probably change std::list for std::vector -void Blockchain::get_all_known_block_ids(std::list &main, std::list &alt, std::list &invalid) +void Blockchain::get_all_known_block_ids(std::list &main, std::list &alt, std::list &invalid) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -603,10 +603,10 @@ void Blockchain::get_all_known_block_ids(std::list &main, std::lis main.push_back(a); } - BOOST_FOREACH(blocks_ext_by_hash::value_type &v, m_alternative_chains) + BOOST_FOREACH(const blocks_ext_by_hash::value_type &v, m_alternative_chains) alt.push_back(v.first); - BOOST_FOREACH(blocks_ext_by_hash::value_type &v, m_invalid_blocks) + BOOST_FOREACH(const blocks_ext_by_hash::value_type &v, m_invalid_blocks) invalid.push_back(v.first); } //------------------------------------------------------------------ @@ -614,7 +614,7 @@ void Blockchain::get_all_known_block_ids(std::list &main, std::lis // last DIFFICULTY_BLOCKS_COUNT blocks and passes them to next_difficulty, // returning the result of that call. Ignores the genesis block, and can use // less blocks than desired if there aren't enough. -difficulty_type Blockchain::get_difficulty_for_next_block() +difficulty_type Blockchain::get_difficulty_for_next_block() const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -760,7 +760,7 @@ bool Blockchain::switch_to_alternative_blockchain(std::list& alt_chain, block_extended_info& bei) +difficulty_type Blockchain::get_next_difficulty_for_alternative_chain(const std::list& alt_chain, block_extended_info& bei) const { LOG_PRINT_L3("Blockchain::" << __func__); std::vector timestamps; @@ -887,7 +887,7 @@ bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_bl //------------------------------------------------------------------ // get the block sizes of the last blocks, starting at // and return by reference . -void Blockchain::get_last_n_blocks_sizes(std::vector& sz, size_t count) +void Blockchain::get_last_n_blocks_sizes(std::vector& sz, size_t count) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -905,7 +905,7 @@ void Blockchain::get_last_n_blocks_sizes(std::vector& sz, size_t count) } } //------------------------------------------------------------------ -uint64_t Blockchain::get_current_cumulative_blocksize_limit() +uint64_t Blockchain::get_current_cumulative_blocksize_limit() const { LOG_PRINT_L3("Blockchain::" << __func__); return m_current_block_cumul_sz_limit; @@ -922,7 +922,7 @@ uint64_t Blockchain::get_current_cumulative_blocksize_limit() // in a lot of places. That flag is not referenced in any of the code // nor any of the makefiles, howeve. Need to look into whether or not it's // necessary at all. -bool Blockchain::create_block_template(block& b, const account_public_address& miner_address, difficulty_type& diffic, uint64_t& height, const blobdata& ex_nonce) +bool Blockchain::create_block_template(block& b, const account_public_address& miner_address, difficulty_type& diffic, uint64_t& height, const blobdata& ex_nonce) const { LOG_PRINT_L3("Blockchain::" << __func__); size_t median_size; @@ -1250,7 +1250,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id return true; } //------------------------------------------------------------------ -bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::list& blocks, std::list& txs) +bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::list& blocks, std::list& txs) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -1272,7 +1272,7 @@ bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::list& blocks) +bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::list& blocks) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -1323,7 +1323,7 @@ bool Blockchain::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NO return true; } //------------------------------------------------------------------ -bool Blockchain::get_alternative_blocks(std::list& blocks) +bool Blockchain::get_alternative_blocks(std::list& blocks) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -1335,7 +1335,7 @@ bool Blockchain::get_alternative_blocks(std::list& blocks) return true; } //------------------------------------------------------------------ -size_t Blockchain::get_alternative_blocks_count() +size_t Blockchain::get_alternative_blocks_count() const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -1344,7 +1344,7 @@ size_t Blockchain::get_alternative_blocks_count() //------------------------------------------------------------------ // This function adds the output specified by to the result_outs container // unlocked and other such checks should be done by here. -void Blockchain::add_out_to_get_random_outs(COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i) +void Blockchain::add_out_to_get_random_outs(COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -1358,7 +1358,7 @@ void Blockchain::add_out_to_get_random_outs(COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_A // with the requested mixins. // TODO: figure out why this returns boolean / if we should be returning false // in some cases -bool Blockchain::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) +bool Blockchain::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) const { LOG_PRINT_L3("Blockchain::" << __func__); srand(static_cast(time(NULL))); @@ -1431,7 +1431,7 @@ bool Blockchain::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUT // This function takes a list of block hashes from another node // on the network to find where the split point is between us and them. // This is used to see what to send another node that needs to sync. -bool Blockchain::find_blockchain_supplement(const std::list& qblock_ids, uint64_t& starter_offset) +bool Blockchain::find_blockchain_supplement(const std::list& qblock_ids, uint64_t& starter_offset) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -1497,7 +1497,7 @@ bool Blockchain::find_blockchain_supplement(const std::list& qbloc return true; } //------------------------------------------------------------------ -uint64_t Blockchain::block_difficulty(uint64_t i) +uint64_t Blockchain::block_difficulty(uint64_t i) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -1513,7 +1513,7 @@ uint64_t Blockchain::block_difficulty(uint64_t i) } //------------------------------------------------------------------ template -bool Blockchain::get_blocks(const t_ids_container& block_ids, t_blocks_container& blocks, t_missed_container& missed_bs) +bool Blockchain::get_blocks(const t_ids_container& block_ids, t_blocks_container& blocks, t_missed_container& missed_bs) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -1537,7 +1537,7 @@ bool Blockchain::get_blocks(const t_ids_container& block_ids, t_blocks_container } //------------------------------------------------------------------ template -bool Blockchain::get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) +bool Blockchain::get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -1618,7 +1618,7 @@ void Blockchain::print_blockchain_outs(const std::string& file) // Find the split point between us and foreign blockchain and return // (by reference) the most recent common block hash along with up to // BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT additional (more recent) hashes. -bool Blockchain::find_blockchain_supplement(const std::list& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) +bool Blockchain::find_blockchain_supplement(const std::list& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -1642,7 +1642,7 @@ bool Blockchain::find_blockchain_supplement(const std::list& qbloc // find split point between ours and foreign blockchain (or start at // blockchain height ), and return up to max_count FULL // blocks by reference. -bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count) +bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -1696,7 +1696,7 @@ bool Blockchain::add_block_as_invalid(const block_extended_info& bei, const cryp return true; } //------------------------------------------------------------------ -bool Blockchain::have_block(const crypto::hash& id) +bool Blockchain::have_block(const crypto::hash& id) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -1720,7 +1720,7 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, block_verification_ return handle_block_to_main_chain(bl, id, bvc); } //------------------------------------------------------------------ -size_t Blockchain::get_total_transactions() +size_t Blockchain::get_total_transactions() const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -1733,7 +1733,7 @@ size_t Blockchain::get_total_transactions() // This container should be managed by the code that validates blocks so we don't // have to store the used keys in a given block in the permanent storage only to // remove them later if the block fails validation. -bool Blockchain::check_for_double_spend(const transaction& tx, key_images_container& keys_this_block) +bool Blockchain::check_for_double_spend(const transaction& tx, key_images_container& keys_this_block) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -1782,7 +1782,7 @@ bool Blockchain::check_for_double_spend(const transaction& tx, key_images_contai return true; } //------------------------------------------------------------------ -bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector& indexs) +bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector& indexs) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -1799,7 +1799,7 @@ bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector& sig, uint64_t* pmax_related_block_height) +bool Blockchain::check_tx_input(const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector& sig, uint64_t* pmax_related_block_height) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -1897,8 +1897,8 @@ bool Blockchain::check_tx_input(const txin_to_key& txin, const crypto::hash& tx_ { std::vector& m_p_output_keys; std::vector& m_output_keys; - Blockchain& m_bch; - outputs_visitor(std::vector& output_keys, std::vector& p_output_keys, Blockchain& bch) : m_output_keys(output_keys), m_p_output_keys(p_output_keys), m_bch(bch) + const Blockchain& m_bch; + outputs_visitor(std::vector& output_keys, std::vector& p_output_keys, const Blockchain& bch) : m_output_keys(output_keys), m_p_output_keys(p_output_keys), m_bch(bch) {} bool handle_output(const transaction& tx, const tx_out& out) { @@ -1947,7 +1947,7 @@ bool Blockchain::check_tx_input(const txin_to_key& txin, const crypto::hash& tx_ } //------------------------------------------------------------------ //TODO: Is this intended to do something else? Need to look into the todo there. -uint64_t Blockchain::get_adjusted_time() +uint64_t Blockchain::get_adjusted_time() const { LOG_PRINT_L3("Blockchain::" << __func__); //TODO: add collecting median time @@ -1955,7 +1955,7 @@ uint64_t Blockchain::get_adjusted_time() } //------------------------------------------------------------------ //TODO: revisit, has changed a bit on upstream -bool Blockchain::check_block_timestamp(std::vector& timestamps, const block& b) +bool Blockchain::check_block_timestamp(std::vector& timestamps, const block& b) const { LOG_PRINT_L3("Blockchain::" << __func__); uint64_t median_ts = epee::misc_utils::median(timestamps); @@ -1976,7 +1976,7 @@ bool Blockchain::check_block_timestamp(std::vector& timestamps, const // true if the block's timestamp is not less than the timestamp of the // median of the selected blocks // false otherwise -bool Blockchain::check_block_timestamp(const block& b) +bool Blockchain::check_block_timestamp(const block& b) const { LOG_PRINT_L3("Blockchain::" << __func__); if(b.timestamp > get_adjusted_time() + CRYPTONOTE_BLOCK_FUTURE_TIME_LIMIT) diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index 75a729a09..6ab963ac7 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -87,54 +87,54 @@ namespace cryptonote void set_checkpoints(checkpoints&& chk_pts) { m_checkpoints = chk_pts; } //bool push_new_block(); - bool get_blocks(uint64_t start_offset, size_t count, std::list& blocks, std::list& txs); - bool get_blocks(uint64_t start_offset, size_t count, std::list& blocks); - bool get_alternative_blocks(std::list& blocks); - size_t get_alternative_blocks_count(); - crypto::hash get_block_id_by_height(uint64_t height); - bool get_block_by_hash(const crypto::hash &h, block &blk); - void get_all_known_block_ids(std::list &main, std::list &alt, std::list &invalid); + bool get_blocks(uint64_t start_offset, size_t count, std::list& blocks, std::list& txs) const; + bool get_blocks(uint64_t start_offset, size_t count, std::list& blocks) const; + bool get_alternative_blocks(std::list& blocks) const; + size_t get_alternative_blocks_count() const; + crypto::hash get_block_id_by_height(uint64_t height) const; + bool get_block_by_hash(const crypto::hash &h, block &blk) const; + void get_all_known_block_ids(std::list &main, std::list &alt, std::list &invalid) const; template void serialize(archive_t & ar, const unsigned int version); - bool have_tx(const crypto::hash &id); - bool have_tx_keyimges_as_spent(const transaction &tx); - bool have_tx_keyimg_as_spent(const crypto::key_image &key_im); + bool have_tx(const crypto::hash &id) const; + bool have_tx_keyimges_as_spent(const transaction &tx) const; + bool have_tx_keyimg_as_spent(const crypto::key_image &key_im) const; template - bool scan_outputkeys_for_indexes(const txin_to_key& tx_in_to_key, visitor_t& vis, uint64_t* pmax_related_block_height = NULL); + bool scan_outputkeys_for_indexes(const txin_to_key& tx_in_to_key, visitor_t& vis, uint64_t* pmax_related_block_height = NULL) const; - uint64_t get_current_blockchain_height(); - crypto::hash get_tail_id(); - crypto::hash get_tail_id(uint64_t& height); - difficulty_type get_difficulty_for_next_block(); + uint64_t get_current_blockchain_height() const; + crypto::hash get_tail_id() const; + crypto::hash get_tail_id(uint64_t& height) const; + difficulty_type get_difficulty_for_next_block() const; bool add_new_block(const block& bl_, block_verification_context& bvc); bool reset_and_set_genesis_block(const block& b); - bool create_block_template(block& b, const account_public_address& miner_address, difficulty_type& di, uint64_t& height, const blobdata& ex_nonce); - bool have_block(const crypto::hash& id); - size_t get_total_transactions(); - bool get_short_chain_history(std::list& ids); - bool find_blockchain_supplement(const std::list& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp); - bool find_blockchain_supplement(const std::list& qblock_ids, uint64_t& starter_offset); - bool find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count); + bool create_block_template(block& b, const account_public_address& miner_address, difficulty_type& di, uint64_t& height, const blobdata& ex_nonce) const; + bool have_block(const crypto::hash& id) const; + size_t get_total_transactions() const; + bool get_short_chain_history(std::list& ids) const; + bool find_blockchain_supplement(const std::list& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const; + bool find_blockchain_supplement(const std::list& qblock_ids, uint64_t& starter_offset) const; + bool find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count) const; bool handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NOTIFY_RESPONSE_GET_OBJECTS::request& rsp); bool handle_get_objects(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res); - bool get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res); - bool get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector& indexs); + bool get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) const; + bool get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector& indexs) const; bool store_blockchain(); - bool check_tx_input(const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector& sig, uint64_t* pmax_related_block_height = NULL); - bool check_tx_inputs(const transaction& tx, uint64_t* pmax_used_block_height = NULL); - bool check_tx_inputs(const transaction& tx, uint64_t& pmax_used_block_height, crypto::hash& max_used_block_id); - uint64_t get_current_cumulative_blocksize_limit(); - bool is_storing_blockchain(){return m_is_blockchain_storing;} - uint64_t block_difficulty(uint64_t i); + bool check_tx_input(const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector& sig, uint64_t* pmax_related_block_height = NULL) const; + bool check_tx_inputs(const transaction& tx, uint64_t* pmax_used_block_height = NULL) const; + bool check_tx_inputs(const transaction& tx, uint64_t& pmax_used_block_height, crypto::hash& max_used_block_id) const; + uint64_t get_current_cumulative_blocksize_limit() const; + bool is_storing_blockchain()const{return m_is_blockchain_storing;} + uint64_t block_difficulty(uint64_t i) const; template - bool get_blocks(const t_ids_container& block_ids, t_blocks_container& blocks, t_missed_container& missed_bs); + bool get_blocks(const t_ids_container& block_ids, t_blocks_container& blocks, t_missed_container& missed_bs) const; template - bool get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs); + bool get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const; //debug functions void print_blockchain(uint64_t start_index, uint64_t end_index); @@ -157,7 +157,7 @@ namespace cryptonote BlockchainDB* m_db; tx_memory_pool& m_tx_pool; - epee::critical_section m_blockchain_lock; // TODO: add here reader/writer lock + mutable epee::critical_section m_blockchain_lock; // TODO: add here reader/writer lock // main chain blocks_container m_blocks; // height -> block_extended_info @@ -190,7 +190,7 @@ namespace cryptonote bool handle_block_to_main_chain(const block& bl, block_verification_context& bvc); bool handle_block_to_main_chain(const block& bl, const crypto::hash& id, block_verification_context& bvc); bool handle_alternative_block(const block& b, const crypto::hash& id, block_verification_context& bvc); - difficulty_type get_next_difficulty_for_alternative_chain(const std::list& alt_chain, block_extended_info& bei); + difficulty_type get_next_difficulty_for_alternative_chain(const std::list& alt_chain, block_extended_info& bei) const; bool prevalidate_miner_transaction(const block& b, uint64_t height); bool validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins); bool validate_transaction(const block& b, uint64_t height, const transaction& tx); @@ -198,18 +198,18 @@ namespace cryptonote bool add_transaction_from_block(const transaction& tx, const crypto::hash& tx_id, const crypto::hash& bl_id, uint64_t bl_height); bool push_transaction_to_global_outs_index(const transaction& tx, const crypto::hash& tx_id, std::vector& global_indexes); bool pop_transaction_from_global_index(const transaction& tx, const crypto::hash& tx_id); - void get_last_n_blocks_sizes(std::vector& sz, size_t count); - void add_out_to_get_random_outs(COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i); - bool is_tx_spendtime_unlocked(uint64_t unlock_time); + void get_last_n_blocks_sizes(std::vector& sz, size_t count) const; + void add_out_to_get_random_outs(COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i) const; + bool is_tx_spendtime_unlocked(uint64_t unlock_time) const; bool add_block_as_invalid(const block& bl, const crypto::hash& h); bool add_block_as_invalid(const block_extended_info& bei, const crypto::hash& h); - bool check_block_timestamp(const block& b); - bool check_block_timestamp(std::vector& timestamps, const block& b); - uint64_t get_adjusted_time(); + bool check_block_timestamp(const block& b) const; + bool check_block_timestamp(std::vector& timestamps, const block& b) const; + uint64_t get_adjusted_time() const; bool complete_timestamps_vector(uint64_t start_height, std::vector& timestamps); bool update_next_cumulative_size_limit(); - bool check_for_double_spend(const transaction& tx, key_images_container& keys_this_block); + bool check_for_double_spend(const transaction& tx, key_images_container& keys_this_block) const; }; -- cgit v1.2.3 From 256162fcd5cffaec5d1670e9cc32635ea540ac2d Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sat, 6 Dec 2014 22:46:51 +0000 Subject: checkpoints: add consts where appropriate --- src/cryptonote_core/checkpoints.cpp | 8 ++++---- src/cryptonote_core/checkpoints.h | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/checkpoints.cpp b/src/cryptonote_core/checkpoints.cpp index 73668ab36..58edda7c9 100644 --- a/src/cryptonote_core/checkpoints.cpp +++ b/src/cryptonote_core/checkpoints.cpp @@ -99,7 +99,7 @@ namespace cryptonote return checkpoint_height < block_height; } //--------------------------------------------------------------------------- - uint64_t checkpoints::get_max_height() + uint64_t checkpoints::get_max_height() const { std::map< uint64_t, crypto::hash >::const_iterator highest = std::max_element( m_points.begin(), m_points.end(), @@ -108,18 +108,18 @@ namespace cryptonote return highest->first; } //--------------------------------------------------------------------------- - const std::map& checkpoints::get_points() + const std::map& checkpoints::get_points() const { return m_points; } - bool checkpoints::check_for_conflicts(checkpoints& other) + bool checkpoints::check_for_conflicts(const checkpoints& other) const { for (auto& pt : other.get_points()) { if (m_points.count(pt.first)) { - CHECK_AND_ASSERT_MES(pt.second == m_points[pt.first], false, "Checkpoint at given height already exists, and hash for new checkpoint was different!"); + CHECK_AND_ASSERT_MES(pt.second == m_points.at(pt.first), false, "Checkpoint at given height already exists, and hash for new checkpoint was different!"); } } return true; diff --git a/src/cryptonote_core/checkpoints.h b/src/cryptonote_core/checkpoints.h index a39ce1964..55d765b71 100644 --- a/src/cryptonote_core/checkpoints.h +++ b/src/cryptonote_core/checkpoints.h @@ -45,9 +45,9 @@ namespace cryptonote bool check_block(uint64_t height, const crypto::hash& h) const; bool check_block(uint64_t height, const crypto::hash& h, bool& is_a_checkpoint) const; bool is_alternative_block_allowed(uint64_t blockchain_height, uint64_t block_height) const; - uint64_t get_max_height(); - const std::map& get_points(); - bool check_for_conflicts(checkpoints& other); + uint64_t get_max_height() const; + const std::map& get_points() const; + bool check_for_conflicts(const checkpoints& other) const; private: std::map m_points; }; -- cgit v1.2.3 From 98bdadcad7a276f2196cfebfe2148d36f65872d1 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sat, 6 Dec 2014 22:53:27 +0000 Subject: blockchain_converter: delete blockchain on succesful exit While the dtor implementation does not actually do anything, other paths do delete it, and the dtor might do someting later. --- src/blockchain_converter/blockchain_converter.cpp | 1 + 1 file changed, 1 insertion(+) (limited to 'src') diff --git a/src/blockchain_converter/blockchain_converter.cpp b/src/blockchain_converter/blockchain_converter.cpp index a2b3a375f..4653cf66c 100644 --- a/src/blockchain_converter/blockchain_converter.cpp +++ b/src/blockchain_converter/blockchain_converter.cpp @@ -99,5 +99,6 @@ int main(int argc, char* argv[]) } } + delete blockchain; return 0; } -- cgit v1.2.3 From 8e41b1e7353215a83835818e1f8ab86ba90169aa Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sat, 6 Dec 2014 23:35:26 +0000 Subject: blockchain_storage: add consts where appropriate --- src/cryptonote_core/blockchain_storage.cpp | 106 +++++++++++++-------------- src/cryptonote_core/blockchain_storage.h | 114 ++++++++++++++--------------- 2 files changed, 110 insertions(+), 110 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/blockchain_storage.cpp b/src/cryptonote_core/blockchain_storage.cpp index e40eb6a0f..b3458ba47 100644 --- a/src/cryptonote_core/blockchain_storage.cpp +++ b/src/cryptonote_core/blockchain_storage.cpp @@ -55,19 +55,19 @@ using namespace cryptonote; DISABLE_VS_WARNINGS(4267) //------------------------------------------------------------------ -bool blockchain_storage::have_tx(const crypto::hash &id) +bool blockchain_storage::have_tx(const crypto::hash &id) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); return m_transactions.find(id) != m_transactions.end(); } //------------------------------------------------------------------ -bool blockchain_storage::have_tx_keyimg_as_spent(const crypto::key_image &key_im) +bool blockchain_storage::have_tx_keyimg_as_spent(const crypto::key_image &key_im) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); return m_spent_keys.find(key_im) != m_spent_keys.end(); } //------------------------------------------------------------------ -transaction *blockchain_storage::get_tx(const crypto::hash &id) +const transaction *blockchain_storage::get_tx(const crypto::hash &id) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); auto it = m_transactions.find(id); @@ -77,7 +77,7 @@ transaction *blockchain_storage::get_tx(const crypto::hash &id) return &it->second.tx; } //------------------------------------------------------------------ -uint64_t blockchain_storage::get_current_blockchain_height() +uint64_t blockchain_storage::get_current_blockchain_height() const { CRITICAL_REGION_LOCAL(m_blockchain_lock); return m_blocks.size(); @@ -333,14 +333,14 @@ bool blockchain_storage::purge_block_data_from_blockchain(const block& bl, size_ return res; } //------------------------------------------------------------------ -crypto::hash blockchain_storage::get_tail_id(uint64_t& height) +crypto::hash blockchain_storage::get_tail_id(uint64_t& height) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); height = get_current_blockchain_height()-1; return get_tail_id(); } //------------------------------------------------------------------ -crypto::hash blockchain_storage::get_tail_id() +crypto::hash blockchain_storage::get_tail_id() const { CRITICAL_REGION_LOCAL(m_blockchain_lock); crypto::hash id = null_hash; @@ -351,7 +351,7 @@ crypto::hash blockchain_storage::get_tail_id() return id; } //------------------------------------------------------------------ -bool blockchain_storage::get_short_chain_history(std::list& ids) +bool blockchain_storage::get_short_chain_history(std::list& ids) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); size_t i = 0; @@ -381,7 +381,7 @@ bool blockchain_storage::get_short_chain_history(std::list& ids) return true; } //------------------------------------------------------------------ -crypto::hash blockchain_storage::get_block_id_by_height(uint64_t height) +crypto::hash blockchain_storage::get_block_id_by_height(uint64_t height) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); if(height >= m_blocks.size()) @@ -390,7 +390,7 @@ crypto::hash blockchain_storage::get_block_id_by_height(uint64_t height) return get_block_hash(m_blocks[height].bl); } //------------------------------------------------------------------ -bool blockchain_storage::get_block_by_hash(const crypto::hash &h, block &blk) { +bool blockchain_storage::get_block_by_hash(const crypto::hash &h, block &blk) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); // try to find block in main chain @@ -410,20 +410,20 @@ bool blockchain_storage::get_block_by_hash(const crypto::hash &h, block &blk) { return false; } //------------------------------------------------------------------ -void blockchain_storage::get_all_known_block_ids(std::list &main, std::list &alt, std::list &invalid) { +void blockchain_storage::get_all_known_block_ids(std::list &main, std::list &alt, std::list &invalid) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); - BOOST_FOREACH(blocks_by_id_index::value_type &v, m_blocks_index) + BOOST_FOREACH(const blocks_by_id_index::value_type &v, m_blocks_index) main.push_back(v.first); - BOOST_FOREACH(blocks_ext_by_hash::value_type &v, m_alternative_chains) + BOOST_FOREACH(const blocks_ext_by_hash::value_type &v, m_alternative_chains) alt.push_back(v.first); - BOOST_FOREACH(blocks_ext_by_hash::value_type &v, m_invalid_blocks) + BOOST_FOREACH(const blocks_ext_by_hash::value_type &v, m_invalid_blocks) invalid.push_back(v.first); } //------------------------------------------------------------------ -difficulty_type blockchain_storage::get_difficulty_for_next_block() +difficulty_type blockchain_storage::get_difficulty_for_next_block() const { CRITICAL_REGION_LOCAL(m_blockchain_lock); std::vector timestamps; @@ -535,7 +535,7 @@ bool blockchain_storage::switch_to_alternative_blockchain(std::list& alt_chain, block_extended_info& bei) +difficulty_type blockchain_storage::get_next_difficulty_for_alternative_chain(const std::list& alt_chain, block_extended_info& bei) const { std::vector timestamps; std::vector commulative_difficulties; @@ -580,7 +580,7 @@ difficulty_type blockchain_storage::get_next_difficulty_for_alternative_chain(co return next_difficulty(timestamps, commulative_difficulties); } //------------------------------------------------------------------ -bool blockchain_storage::prevalidate_miner_transaction(const block& b, uint64_t height) +bool blockchain_storage::prevalidate_miner_transaction(const block& b, uint64_t height) const { CHECK_AND_ASSERT_MES(b.miner_tx.vin.size() == 1, false, "coinbase transaction in the block has no inputs"); CHECK_AND_ASSERT_MES(b.miner_tx.vin[0].type() == typeid(txin_gen), false, "coinbase transaction in the block has the wrong type"); @@ -603,7 +603,7 @@ bool blockchain_storage::prevalidate_miner_transaction(const block& b, uint64_t return true; } //------------------------------------------------------------------ -bool blockchain_storage::validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins) +bool blockchain_storage::validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins) const { //validate reward uint64_t money_in_use = 0; @@ -630,7 +630,7 @@ bool blockchain_storage::validate_miner_transaction(const block& b, size_t cumul return true; } //------------------------------------------------------------------ -bool blockchain_storage::get_backward_blocks_sizes(size_t from_height, std::vector& sz, size_t count) +bool blockchain_storage::get_backward_blocks_sizes(size_t from_height, std::vector& sz, size_t count) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); CHECK_AND_ASSERT_MES(from_height < m_blocks.size(), false, "Internal error: get_backward_blocks_sizes called with from_height=" << from_height << ", blockchain height = " << m_blocks.size()); @@ -642,7 +642,7 @@ bool blockchain_storage::get_backward_blocks_sizes(size_t from_height, std::vect return true; } //------------------------------------------------------------------ -bool blockchain_storage::get_last_n_blocks_sizes(std::vector& sz, size_t count) +bool blockchain_storage::get_last_n_blocks_sizes(std::vector& sz, size_t count) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); if(!m_blocks.size()) @@ -650,12 +650,12 @@ bool blockchain_storage::get_last_n_blocks_sizes(std::vector& sz, size_t return get_backward_blocks_sizes(m_blocks.size() -1, sz, count); } //------------------------------------------------------------------ -uint64_t blockchain_storage::get_current_comulative_blocksize_limit() +uint64_t blockchain_storage::get_current_comulative_blocksize_limit() const { return m_current_block_cumul_sz_limit; } //------------------------------------------------------------------ -bool blockchain_storage::create_block_template(block& b, const account_public_address& miner_address, difficulty_type& diffic, uint64_t& height, const blobdata& ex_nonce) +bool blockchain_storage::create_block_template(block& b, const account_public_address& miner_address, difficulty_type& diffic, uint64_t& height, const blobdata& ex_nonce) const { size_t median_size; uint64_t already_generated_coins; @@ -774,7 +774,7 @@ bool blockchain_storage::create_block_template(block& b, const account_public_ad return false; } //------------------------------------------------------------------ -bool blockchain_storage::complete_timestamps_vector(uint64_t start_top_height, std::vector& timestamps) +bool blockchain_storage::complete_timestamps_vector(uint64_t start_top_height, std::vector& timestamps) const { if(timestamps.size() >= BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW) @@ -939,7 +939,7 @@ bool blockchain_storage::handle_alternative_block(const block& b, const crypto:: return true; } //------------------------------------------------------------------ -bool blockchain_storage::get_blocks(uint64_t start_offset, size_t count, std::list& blocks, std::list& txs) +bool blockchain_storage::get_blocks(uint64_t start_offset, size_t count, std::list& blocks, std::list& txs) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); if(start_offset >= m_blocks.size()) @@ -955,7 +955,7 @@ bool blockchain_storage::get_blocks(uint64_t start_offset, size_t count, std::li return true; } //------------------------------------------------------------------ -bool blockchain_storage::get_blocks(uint64_t start_offset, size_t count, std::list& blocks) +bool blockchain_storage::get_blocks(uint64_t start_offset, size_t count, std::list& blocks) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); if(start_offset >= m_blocks.size()) @@ -999,7 +999,7 @@ bool blockchain_storage::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& return true; } //------------------------------------------------------------------ -bool blockchain_storage::get_alternative_blocks(std::list& blocks) +bool blockchain_storage::get_alternative_blocks(std::list& blocks) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -1010,21 +1010,21 @@ bool blockchain_storage::get_alternative_blocks(std::list& blocks) return true; } //------------------------------------------------------------------ -size_t blockchain_storage::get_alternative_blocks_count() +size_t blockchain_storage::get_alternative_blocks_count() const { CRITICAL_REGION_LOCAL(m_blockchain_lock); return m_alternative_chains.size(); } //------------------------------------------------------------------ -bool blockchain_storage::add_out_to_get_random_outs(std::vector >& amount_outs, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i) +bool blockchain_storage::add_out_to_get_random_outs(const std::vector >& amount_outs, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); - transactions_container::iterator tx_it = m_transactions.find(amount_outs[i].first); + transactions_container::const_iterator tx_it = m_transactions.find(amount_outs[i].first); CHECK_AND_ASSERT_MES(tx_it != m_transactions.end(), false, "internal error: transaction with id " << amount_outs[i].first << ENDL << ", used in mounts global index for amount=" << amount << ": i=" << i << "not found in transactions index"); CHECK_AND_ASSERT_MES(tx_it->second.tx.vout.size() > amount_outs[i].second, false, "internal error: in global outs index, transaction out index=" << amount_outs[i].second << " more than transaction outputs = " << tx_it->second.tx.vout.size() << ", for tx id = " << amount_outs[i].first); - transaction& tx = tx_it->second.tx; + const transaction& tx = tx_it->second.tx; CHECK_AND_ASSERT_MES(tx.vout[amount_outs[i].second].target.type() == typeid(txout_to_key), false, "unknown tx out type"); //check if transaction is unlocked @@ -1037,7 +1037,7 @@ bool blockchain_storage::add_out_to_get_random_outs(std::vector >& amount_outs) +size_t blockchain_storage::find_end_of_allowed_index(const std::vector >& amount_outs) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); if(!amount_outs.size()) @@ -1046,7 +1046,7 @@ size_t blockchain_storage::find_end_of_allowed_index(const std::vector >& amount_outs = it->second; + const std::vector >& amount_outs = it->second; //it is not good idea to use top fresh outs, because it increases possibility of transaction canceling on split //lets find upper bound of not fresh outs size_t up_index_limit = find_end_of_allowed_index(amount_outs); @@ -1096,7 +1096,7 @@ bool blockchain_storage::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDO return true; } //------------------------------------------------------------------ -bool blockchain_storage::find_blockchain_supplement(const std::list& qblock_ids, uint64_t& starter_offset) +bool blockchain_storage::find_blockchain_supplement(const std::list& qblock_ids, uint64_t& starter_offset) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -1143,7 +1143,7 @@ bool blockchain_storage::find_blockchain_supplement(const std::list& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) +bool blockchain_storage::find_blockchain_supplement(const std::list& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); if(!find_blockchain_supplement(qblock_ids, resp.start_height)) @@ -1219,7 +1219,7 @@ bool blockchain_storage::find_blockchain_supplement(const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count) +bool blockchain_storage::find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); if(req_start_block > 0) { @@ -1258,7 +1258,7 @@ bool blockchain_storage::add_block_as_invalid(const block_extended_info& bei, co return true; } //------------------------------------------------------------------ -bool blockchain_storage::have_block(const crypto::hash& id) +bool blockchain_storage::have_block(const crypto::hash& id) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); if(m_blocks_index.count(id)) @@ -1296,13 +1296,13 @@ bool blockchain_storage::push_transaction_to_global_outs_index(const transaction return true; } //------------------------------------------------------------------ -size_t blockchain_storage::get_total_transactions() +size_t blockchain_storage::get_total_transactions() const { CRITICAL_REGION_LOCAL(m_blockchain_lock); return m_transactions.size(); } //------------------------------------------------------------------ -bool blockchain_storage::get_outs(uint64_t amount, std::list& pkeys) +bool blockchain_storage::get_outs(uint64_t amount, std::list& pkeys) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); auto it = m_outputs.find(amount); @@ -1392,7 +1392,7 @@ bool blockchain_storage::add_transaction_from_block(const transaction& tx, const return true; } //------------------------------------------------------------------ -bool blockchain_storage::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector& indexs) +bool blockchain_storage::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector& indexs) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); auto it = m_transactions.find(tx_id); @@ -1407,7 +1407,7 @@ bool blockchain_storage::get_tx_outputs_gindexs(const crypto::hash& tx_id, std:: return true; } //------------------------------------------------------------------ -bool blockchain_storage::check_tx_inputs(const transaction& tx, uint64_t& max_used_block_height, crypto::hash& max_used_block_id) +bool blockchain_storage::check_tx_inputs(const transaction& tx, uint64_t& max_used_block_height, crypto::hash& max_used_block_id) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); bool res = check_tx_inputs(tx, &max_used_block_height); @@ -1417,7 +1417,7 @@ bool blockchain_storage::check_tx_inputs(const transaction& tx, uint64_t& max_us return true; } //------------------------------------------------------------------ -bool blockchain_storage::have_tx_keyimges_as_spent(const transaction &tx) +bool blockchain_storage::have_tx_keyimges_as_spent(const transaction &tx) const { BOOST_FOREACH(const txin_v& in, tx.vin) { @@ -1428,13 +1428,13 @@ bool blockchain_storage::have_tx_keyimges_as_spent(const transaction &tx) return false; } //------------------------------------------------------------------ -bool blockchain_storage::check_tx_inputs(const transaction& tx, uint64_t* pmax_used_block_height) +bool blockchain_storage::check_tx_inputs(const transaction& tx, uint64_t* pmax_used_block_height) const { crypto::hash tx_prefix_hash = get_transaction_prefix_hash(tx); return check_tx_inputs(tx, tx_prefix_hash, pmax_used_block_height); } //------------------------------------------------------------------ -bool blockchain_storage::check_tx_inputs(const transaction& tx, const crypto::hash& tx_prefix_hash, uint64_t* pmax_used_block_height) +bool blockchain_storage::check_tx_inputs(const transaction& tx, const crypto::hash& tx_prefix_hash, uint64_t* pmax_used_block_height) const { size_t sig_index = 0; if(pmax_used_block_height) @@ -1466,7 +1466,7 @@ bool blockchain_storage::check_tx_inputs(const transaction& tx, const crypto::ha return true; } //------------------------------------------------------------------ -bool blockchain_storage::is_tx_spendtime_unlocked(uint64_t unlock_time) +bool blockchain_storage::is_tx_spendtime_unlocked(uint64_t unlock_time) const { if(unlock_time < CRYPTONOTE_MAX_BLOCK_NUMBER) { @@ -1487,15 +1487,15 @@ bool blockchain_storage::is_tx_spendtime_unlocked(uint64_t unlock_time) return false; } //------------------------------------------------------------------ -bool blockchain_storage::check_tx_input(const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector& sig, uint64_t* pmax_related_block_height) +bool blockchain_storage::check_tx_input(const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector& sig, uint64_t* pmax_related_block_height) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); struct outputs_visitor { std::vector& m_results_collector; - blockchain_storage& m_bch; - outputs_visitor(std::vector& results_collector, blockchain_storage& bch):m_results_collector(results_collector), m_bch(bch) + const blockchain_storage& m_bch; + outputs_visitor(std::vector& results_collector, const blockchain_storage& bch):m_results_collector(results_collector), m_bch(bch) {} bool handle_output(const transaction& tx, const tx_out& out) { @@ -1537,13 +1537,13 @@ bool blockchain_storage::check_tx_input(const txin_to_key& txin, const crypto::h return crypto::check_ring_signature(tx_prefix_hash, txin.k_image, output_keys, sig.data()); } //------------------------------------------------------------------ -uint64_t blockchain_storage::get_adjusted_time() +uint64_t blockchain_storage::get_adjusted_time() const { //TODO: add collecting median time return time(NULL); } //------------------------------------------------------------------ -bool blockchain_storage::check_block_timestamp_main(const block& b) +bool blockchain_storage::check_block_timestamp_main(const block& b) const { if(b.timestamp > get_adjusted_time() + CRYPTONOTE_BLOCK_FUTURE_TIME_LIMIT) { @@ -1559,7 +1559,7 @@ bool blockchain_storage::check_block_timestamp_main(const block& b) return check_block_timestamp(std::move(timestamps), b); } //------------------------------------------------------------------ -bool blockchain_storage::check_block_timestamp(std::vector timestamps, const block& b) +bool blockchain_storage::check_block_timestamp(std::vector timestamps, const block& b) const { if(timestamps.size() < BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW) return true; @@ -1782,7 +1782,7 @@ bool blockchain_storage::add_new_block(const block& bl_, block_verification_cont return handle_block_to_main_chain(bl, id, bvc); } //------------------------------------------------------------------ -void blockchain_storage::check_against_checkpoints(checkpoints& points, bool enforce) +void blockchain_storage::check_against_checkpoints(const checkpoints& points, bool enforce) { const auto& pts = points.get_points(); diff --git a/src/cryptonote_core/blockchain_storage.h b/src/cryptonote_core/blockchain_storage.h index 671da40a8..178ef38fd 100644 --- a/src/cryptonote_core/blockchain_storage.h +++ b/src/cryptonote_core/blockchain_storage.h @@ -88,55 +88,55 @@ namespace cryptonote void set_checkpoints(checkpoints&& chk_pts) { m_checkpoints = chk_pts; } //bool push_new_block(); - bool get_blocks(uint64_t start_offset, size_t count, std::list& blocks, std::list& txs); - bool get_blocks(uint64_t start_offset, size_t count, std::list& blocks); - bool get_alternative_blocks(std::list& blocks); - size_t get_alternative_blocks_count(); - crypto::hash get_block_id_by_height(uint64_t height); - bool get_block_by_hash(const crypto::hash &h, block &blk); - void get_all_known_block_ids(std::list &main, std::list &alt, std::list &invalid); + bool get_blocks(uint64_t start_offset, size_t count, std::list& blocks, std::list& txs) const; + bool get_blocks(uint64_t start_offset, size_t count, std::list& blocks) const; + bool get_alternative_blocks(std::list& blocks) const; + size_t get_alternative_blocks_count() const; + crypto::hash get_block_id_by_height(uint64_t height) const; + bool get_block_by_hash(const crypto::hash &h, block &blk) const; + void get_all_known_block_ids(std::list &main, std::list &alt, std::list &invalid) const; template void serialize(archive_t & ar, const unsigned int version); - bool have_tx(const crypto::hash &id); - bool have_tx_keyimges_as_spent(const transaction &tx); - bool have_tx_keyimg_as_spent(const crypto::key_image &key_im); - transaction *get_tx(const crypto::hash &id); + bool have_tx(const crypto::hash &id) const; + bool have_tx_keyimges_as_spent(const transaction &tx) const; + bool have_tx_keyimg_as_spent(const crypto::key_image &key_im) const; + const transaction *get_tx(const crypto::hash &id) const; template - bool scan_outputkeys_for_indexes(const txin_to_key& tx_in_to_key, visitor_t& vis, uint64_t* pmax_related_block_height = NULL); + bool scan_outputkeys_for_indexes(const txin_to_key& tx_in_to_key, visitor_t& vis, uint64_t* pmax_related_block_height = NULL) const; - uint64_t get_current_blockchain_height(); - crypto::hash get_tail_id(); - crypto::hash get_tail_id(uint64_t& height); - difficulty_type get_difficulty_for_next_block(); + uint64_t get_current_blockchain_height() const; + crypto::hash get_tail_id() const; + crypto::hash get_tail_id(uint64_t& height) const; + difficulty_type get_difficulty_for_next_block() const; bool add_new_block(const block& bl_, block_verification_context& bvc); bool reset_and_set_genesis_block(const block& b); - bool create_block_template(block& b, const account_public_address& miner_address, difficulty_type& di, uint64_t& height, const blobdata& ex_nonce); - bool have_block(const crypto::hash& id); - size_t get_total_transactions(); - bool get_outs(uint64_t amount, std::list& pkeys); - bool get_short_chain_history(std::list& ids); - bool find_blockchain_supplement(const std::list& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp); - bool find_blockchain_supplement(const std::list& qblock_ids, uint64_t& starter_offset); - bool find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count); + bool create_block_template(block& b, const account_public_address& miner_address, difficulty_type& di, uint64_t& height, const blobdata& ex_nonce) const; + bool have_block(const crypto::hash& id) const; + size_t get_total_transactions() const; + bool get_outs(uint64_t amount, std::list& pkeys) const; + bool get_short_chain_history(std::list& ids) const; + bool find_blockchain_supplement(const std::list& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const; + bool find_blockchain_supplement(const std::list& qblock_ids, uint64_t& starter_offset) const; + bool find_blockchain_supplement(const uint64_t req_start_block, const std::list& qblock_ids, std::list > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count) const; bool handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NOTIFY_RESPONSE_GET_OBJECTS::request& rsp); bool handle_get_objects(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res); - bool get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res); - bool get_backward_blocks_sizes(size_t from_height, std::vector& sz, size_t count); - bool get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector& indexs); + bool get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) const; + bool get_backward_blocks_sizes(size_t from_height, std::vector& sz, size_t count) const; + bool get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vector& indexs) const; bool store_blockchain(); - bool check_tx_input(const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector& sig, uint64_t* pmax_related_block_height = NULL); - bool check_tx_inputs(const transaction& tx, const crypto::hash& tx_prefix_hash, uint64_t* pmax_used_block_height = NULL); - bool check_tx_inputs(const transaction& tx, uint64_t* pmax_used_block_height = NULL); - bool check_tx_inputs(const transaction& tx, uint64_t& pmax_used_block_height, crypto::hash& max_used_block_id); - uint64_t get_current_comulative_blocksize_limit(); - bool is_storing_blockchain(){return m_is_blockchain_storing;} - uint64_t block_difficulty(size_t i); + bool check_tx_input(const txin_to_key& txin, const crypto::hash& tx_prefix_hash, const std::vector& sig, uint64_t* pmax_related_block_height = NULL) const; + bool check_tx_inputs(const transaction& tx, const crypto::hash& tx_prefix_hash, uint64_t* pmax_used_block_height = NULL) const; + bool check_tx_inputs(const transaction& tx, uint64_t* pmax_used_block_height = NULL) const; + bool check_tx_inputs(const transaction& tx, uint64_t& pmax_used_block_height, crypto::hash& max_used_block_id) const; + uint64_t get_current_comulative_blocksize_limit() const; + bool is_storing_blockchain()const{return m_is_blockchain_storing;} + uint64_t block_difficulty(size_t i) const; template - bool get_blocks(const t_ids_container& block_ids, t_blocks_container& blocks, t_missed_container& missed_bs) + bool get_blocks(const t_ids_container& block_ids, t_blocks_container& blocks, t_missed_container& missed_bs) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -156,7 +156,7 @@ namespace cryptonote } template - bool get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) + bool get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -180,14 +180,14 @@ namespace cryptonote void print_blockchain(uint64_t start_index, uint64_t end_index); void print_blockchain_index(); void print_blockchain_outs(const std::string& file); - void check_against_checkpoints(checkpoints& points, bool enforce); + void check_against_checkpoints(const checkpoints& points, bool enforce); bool update_checkpoints(const std::string& file_path, bool check_dns); void set_enforce_dns_checkpoints(bool enforce_checkpoints); - block get_block(uint64_t height) { return m_blocks[height].bl; } - size_t get_block_size(uint64_t height) { return m_blocks[height].block_cumulative_size; } - difficulty_type get_block_cumulative_difficulty(uint64_t height) { return m_blocks[height].cumulative_difficulty; } - uint64_t get_block_coins_generated(uint64_t height) { return m_blocks[height].already_generated_coins; } + block get_block(uint64_t height) const { return m_blocks[height].bl; } + size_t get_block_size(uint64_t height) const { return m_blocks[height].block_cumulative_size; } + difficulty_type get_block_cumulative_difficulty(uint64_t height) const { return m_blocks[height].cumulative_difficulty; } + uint64_t get_block_coins_generated(uint64_t height) const { return m_blocks[height].already_generated_coins; } private: typedef std::unordered_map blocks_by_id_index; @@ -199,7 +199,7 @@ namespace cryptonote typedef std::map>> outputs_container; //crypto::hash - tx hash, size_t - index of out in transaction tx_memory_pool* m_tx_pool; - epee::critical_section m_blockchain_lock; // TODO: add here reader/writer lock + mutable epee::critical_section m_blockchain_lock; // TODO: add here reader/writer lock // main chain blocks_container m_blocks; // height -> block_extended_info @@ -233,24 +233,24 @@ namespace cryptonote bool handle_block_to_main_chain(const block& bl, block_verification_context& bvc); bool handle_block_to_main_chain(const block& bl, const crypto::hash& id, block_verification_context& bvc); bool handle_alternative_block(const block& b, const crypto::hash& id, block_verification_context& bvc); - difficulty_type get_next_difficulty_for_alternative_chain(const std::list& alt_chain, block_extended_info& bei); - bool prevalidate_miner_transaction(const block& b, uint64_t height); - bool validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins); - bool validate_transaction(const block& b, uint64_t height, const transaction& tx); + difficulty_type get_next_difficulty_for_alternative_chain(const std::list& alt_chain, block_extended_info& bei) const; + bool prevalidate_miner_transaction(const block& b, uint64_t height) const; + bool validate_miner_transaction(const block& b, size_t cumulative_block_size, uint64_t fee, uint64_t& base_reward, uint64_t already_generated_coins) const; + bool validate_transaction(const block& b, uint64_t height, const transaction& tx) const; bool rollback_blockchain_switching(std::list& original_chain, size_t rollback_height); bool add_transaction_from_block(const transaction& tx, const crypto::hash& tx_id, const crypto::hash& bl_id, uint64_t bl_height); bool push_transaction_to_global_outs_index(const transaction& tx, const crypto::hash& tx_id, std::vector& global_indexes); bool pop_transaction_from_global_index(const transaction& tx, const crypto::hash& tx_id); - bool get_last_n_blocks_sizes(std::vector& sz, size_t count); - bool add_out_to_get_random_outs(std::vector >& amount_outs, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i); - bool is_tx_spendtime_unlocked(uint64_t unlock_time); + bool get_last_n_blocks_sizes(std::vector& sz, size_t count) const; + bool add_out_to_get_random_outs(const std::vector >& amount_outs, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::outs_for_amount& result_outs, uint64_t amount, size_t i) const; + bool is_tx_spendtime_unlocked(uint64_t unlock_time) const; bool add_block_as_invalid(const block& bl, const crypto::hash& h); bool add_block_as_invalid(const block_extended_info& bei, const crypto::hash& h); - size_t find_end_of_allowed_index(const std::vector >& amount_outs); - bool check_block_timestamp_main(const block& b); - bool check_block_timestamp(std::vector timestamps, const block& b); - uint64_t get_adjusted_time(); - bool complete_timestamps_vector(uint64_t start_height, std::vector& timestamps); + size_t find_end_of_allowed_index(const std::vector >& amount_outs) const; + bool check_block_timestamp_main(const block& b) const; + bool check_block_timestamp(std::vector timestamps, const block& b) const; + uint64_t get_adjusted_time() const; + bool complete_timestamps_vector(uint64_t start_height, std::vector& timestamps) const; bool update_next_comulative_size_limit(); bool store_genesis_block(bool testnet); }; @@ -320,7 +320,7 @@ namespace cryptonote //------------------------------------------------------------------ template - bool blockchain_storage::scan_outputkeys_for_indexes(const txin_to_key& tx_in_to_key, visitor_t& vis, uint64_t* pmax_related_block_height) + bool blockchain_storage::scan_outputkeys_for_indexes(const txin_to_key& tx_in_to_key, visitor_t& vis, uint64_t* pmax_related_block_height) const { CRITICAL_REGION_LOCAL(m_blockchain_lock); auto it = m_outputs.find(tx_in_to_key.amount); @@ -330,7 +330,7 @@ namespace cryptonote std::vector absolute_offsets = relative_output_offsets_to_absolute(tx_in_to_key.key_offsets); - std::vector >& amount_outs_vec = it->second; + const std::vector >& amount_outs_vec = it->second; size_t count = 0; BOOST_FOREACH(uint64_t i, absolute_offsets) { @@ -339,7 +339,7 @@ namespace cryptonote LOG_PRINT_L0("Wrong index in transaction inputs: " << i << ", expected maximum " << amount_outs_vec.size() - 1); return false; } - transactions_container::iterator tx_it = m_transactions.find(amount_outs_vec[i].first); + transactions_container::const_iterator tx_it = m_transactions.find(amount_outs_vec[i].first); CHECK_AND_ASSERT_MES(tx_it != m_transactions.end(), false, "Wrong transaction id in output indexes: " << epee::string_tools::pod_to_hex(amount_outs_vec[i].first)); CHECK_AND_ASSERT_MES(amount_outs_vec[i].second < tx_it->second.tx.vout.size(), false, "Wrong index in transaction outputs: " << amount_outs_vec[i].second << ", expected less then " << tx_it->second.tx.vout.size()); -- cgit v1.2.3 From a3157d7b697cc4313460c37a3e8740afc149b2d3 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 7 Dec 2014 11:10:24 +0000 Subject: blockchain_storage: refactor genesis block creation The existing assert is kept as it is stricter than the function's internal assert. --- src/cryptonote_core/blockchain_storage.cpp | 20 +++++--------------- src/cryptonote_core/blockchain_storage.h | 2 +- 2 files changed, 6 insertions(+), 16 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/blockchain_storage.cpp b/src/cryptonote_core/blockchain_storage.cpp index b3458ba47..11bd1f2ac 100644 --- a/src/cryptonote_core/blockchain_storage.cpp +++ b/src/cryptonote_core/blockchain_storage.cpp @@ -113,24 +113,14 @@ bool blockchain_storage::init(const std::string& config_folder, bool testnet) else { LOG_PRINT_L0("Can't load blockchain storage from file, generating genesis block."); - block bl = boost::value_initialized(); - block_verification_context bvc = boost::value_initialized(); - if (testnet) - { - generate_genesis_block(bl, config::testnet::GENESIS_TX, config::testnet::GENESIS_NONCE); - } - else - { - generate_genesis_block(bl, config::GENESIS_TX, config::GENESIS_NONCE); - } - add_new_block(bl, bvc); - CHECK_AND_ASSERT_MES(!bvc.m_verifivation_failed && bvc.m_added_to_main_chain, false, "Failed to add genesis block to blockchain"); + if (!store_genesis_block(testnet, true)) + return false; } if(!m_blocks.size()) { LOG_PRINT_L0("Blockchain not loaded, generating genesis block."); - if (!store_genesis_block(testnet)) { + if (!store_genesis_block(testnet, false)) { return false; } } else { @@ -159,7 +149,7 @@ bool blockchain_storage::init(const std::string& config_folder, bool testnet) return true; } //------------------------------------------------------------------ -bool blockchain_storage::store_genesis_block(bool testnet) { +bool blockchain_storage::store_genesis_block(bool testnet, bool check_added) { block bl = ::boost::value_initialized(); block_verification_context bvc = boost::value_initialized(); @@ -173,7 +163,7 @@ bool blockchain_storage::store_genesis_block(bool testnet) { } add_new_block(bl, bvc); - CHECK_AND_ASSERT_MES(!bvc.m_verifivation_failed, false, "Failed to add genesis block to blockchain"); + CHECK_AND_ASSERT_MES(!bvc.m_verifivation_failed && (bvc.m_added_to_main_chain || !check_added), false, "Failed to add genesis block to blockchain"); return true; } //------------------------------------------------------------------ diff --git a/src/cryptonote_core/blockchain_storage.h b/src/cryptonote_core/blockchain_storage.h index 178ef38fd..e26d55b64 100644 --- a/src/cryptonote_core/blockchain_storage.h +++ b/src/cryptonote_core/blockchain_storage.h @@ -252,7 +252,7 @@ namespace cryptonote uint64_t get_adjusted_time() const; bool complete_timestamps_vector(uint64_t start_height, std::vector& timestamps) const; bool update_next_comulative_size_limit(); - bool store_genesis_block(bool testnet); + bool store_genesis_block(bool testnet, bool check_added = false); }; -- cgit v1.2.3 From 29b5876ad1a6b6eeeae941fe63d37d79cfa3fe24 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 7 Dec 2014 11:23:40 +0000 Subject: db_lmdb: make cursor internal members private --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 1 + 1 file changed, 1 insertion(+) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index 2901f9ba7..632c2cc60 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -67,6 +67,7 @@ struct lmdb_cur } } +private: MDB_cursor* m_cur; bool done; }; -- cgit v1.2.3 From 1d23db220ad584b3af1cc243cd3a1193bb1deeed Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 7 Dec 2014 11:36:44 +0000 Subject: db_lmdb: do not keep a dangling pointer to stack objects --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index 632c2cc60..278bec14f 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -1804,12 +1804,14 @@ uint64_t BlockchainLMDB::add_block( const block& blk try { BlockchainDB::add_block(blk, block_size, cumulative_difficulty, coins_generated, txs); + m_write_txn = NULL; txn.commit(); } catch (...) { m_num_outputs = num_outputs; + m_write_txn = NULL; throw; } @@ -1830,12 +1832,14 @@ void BlockchainLMDB::pop_block(block& blk, std::vector& txs) try { BlockchainDB::pop_block(blk, txs); + m_write_txn = NULL; txn.commit(); } catch (...) { m_num_outputs = num_outputs; + m_write_txn = NULL; throw; } -- cgit v1.2.3 From 198368b2e1674307539d6646dac8f4ad57170469 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 7 Dec 2014 13:17:25 +0000 Subject: blockchain: fix wallet syncing from scratch When the wallet syncs from the first block, it is fine to start at the genesis block. --- src/cryptonote_core/blockchain.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 75296cf46..b5c4f9ca4 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -1486,7 +1486,8 @@ bool Blockchain::find_blockchain_supplement(const std::list& qbloc } // if split_height remains 0, we didn't have any but the genesis block in common - if(split_height == 0) + // which is only fine if the blocks just have the genesis block + if(split_height == 0 && qblock_ids.size() > 1) { LOG_ERROR("Ours and foreign blockchain have only genesis block in common... o.O"); return false; -- cgit v1.2.3 From c93a18663774f2961aac94bfebf4954af0dc7196 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Thu, 11 Dec 2014 19:30:55 +0000 Subject: db_lmdb: do not cast const away --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 26 +++++++++++------------ 1 file changed, 13 insertions(+), 13 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index 278bec14f..701b80a2c 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -73,8 +73,8 @@ private: }; auto compare_uint64 = [](const MDB_val *a, const MDB_val *b) { - uint64_t va = *(uint64_t*)a->mv_data; - uint64_t vb = *(uint64_t*)b->mv_data; + const uint64_t va = *(const uint64_t*)a->mv_data; + const uint64_t vb = *(const uint64_t*)b->mv_data; if (va < vb) return -1; else if (va == vb) return 0; else return 1; @@ -150,7 +150,7 @@ void BlockchainLMDB::add_block( const block& blk throw DB_ERROR("Failed to get top block hash to check for new block's parent"); } - uint64_t parent_height = *(uint64_t *)parent_h.mv_data; + uint64_t parent_height = *(const uint64_t *)parent_h.mv_data; if (parent_height != m_height - 1) { LOG_PRINT_L0("Top block is not new block's parent"); @@ -485,7 +485,7 @@ void BlockchainLMDB::remove_tx_outputs(const crypto::hash& tx_hash) for (uint64_t i = 0; i < num_elems; ++i) { - remove_output(*(uint64_t*)v.mv_data); + remove_output(*(const uint64_t*)v.mv_data); if (i < num_elems - 1) { mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); @@ -928,7 +928,7 @@ uint64_t BlockchainLMDB::get_block_height(const crypto::hash& h) const } txn.commit(); - return *(uint64_t*)result.mv_data; + return *(const uint64_t*)result.mv_data; } block_header BlockchainLMDB::get_block_header(const crypto::hash& h) const @@ -1014,7 +1014,7 @@ uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const } txn.commit(); - return *(uint64_t*)result.mv_data; + return *(const uint64_t*)result.mv_data; } uint64_t BlockchainLMDB::get_top_block_timestamp() const @@ -1061,7 +1061,7 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) const } txn.commit(); - return *(size_t*)result.mv_data; + return *(const size_t*)result.mv_data; } difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& height) const @@ -1144,7 +1144,7 @@ uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& heigh } txn.commit(); - return *(uint64_t*)result.mv_data; + return *(const uint64_t*)result.mv_data; } crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height) const @@ -1307,7 +1307,7 @@ uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) const throw DB_ERROR("DB error attempting to fetch tx unlock time from hash"); } - return *(uint64_t*)result.mv_data; + return *(const uint64_t*)result.mv_data; } transaction BlockchainLMDB::get_tx(const crypto::hash& h) const @@ -1421,7 +1421,7 @@ uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const throw DB_ERROR("DB error attempting to fetch tx height from hash"); } - return *(uint64_t*)result.mv_data; + return *(const uint64_t*)result.mv_data; } //FIXME: make sure the random method used here is appropriate @@ -1666,7 +1666,7 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); - uint64_t glob_index = *(uint64_t*)v.mv_data; + uint64_t glob_index = *(const uint64_t*)v.mv_data; cur.close(); @@ -1700,7 +1700,7 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con txn.commit(); - return tx_out_index(tx_hash, *(uint64_t *)v.mv_data); + return tx_out_index(tx_hash, *(const uint64_t *)v.mv_data); } std::vector BlockchainLMDB::get_tx_output_indices(const crypto::hash& h) const @@ -1746,7 +1746,7 @@ std::vector BlockchainLMDB::get_tx_output_indices(const crypto::hash& { mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); - index_vec.push_back(*(uint64_t *)v.mv_data); + index_vec.push_back(*(const uint64_t *)v.mv_data); } cur.close(); -- cgit v1.2.3 From 1c578ad3f8ad57b291a4f651ef897120f7735b20 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Thu, 11 Dec 2014 19:36:27 +0000 Subject: db_lmdb: remove block timestamp too when removing a block --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index 701b80a2c..3b4de0a62 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -269,6 +269,12 @@ void BlockchainLMDB::remove_block() throw DB_ERROR("Failed to add removal of block total generated coins to db transaction"); } + if (mdb_del(*m_write_txn, m_block_timestamps, &k, NULL)) + { + LOG_PRINT_L1("Failed to add removal of block timestamp to db transaction"); + throw DB_ERROR("Failed to add removal of block timestamp to db transaction"); + } + if (mdb_del(*m_write_txn, m_block_heights, &h, NULL)) { LOG_PRINT_L1("Failed to add removal of block height by hash to db transaction"); -- cgit v1.2.3 From 1860658eeca1a311e95ff2b0cf3a3dabe135484a Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Thu, 11 Dec 2014 19:37:07 +0000 Subject: blockchain: do not append "testnet" to the data directory It is already there (unless overridden via command line). --- src/cryptonote_core/blockchain.cpp | 6 ------ 1 file changed, 6 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index b5c4f9ca4..87be9d566 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -238,12 +238,6 @@ bool Blockchain::init(const std::string& config_folder, bool testnet) boost::filesystem::path folder(m_config_folder); - // append "testnet" directory as needed - if (testnet) - { - folder /= "testnet"; - } - LOG_PRINT_L0("Loading blockchain..."); //FIXME: update filename for BlockchainDB -- cgit v1.2.3 From 2b9f73787259dfbf164516925beb5befd8f1df6b Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Fri, 12 Dec 2014 10:28:46 +0000 Subject: blockchain_converter: only call data path function once --- src/blockchain_converter/blockchain_converter.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/blockchain_converter/blockchain_converter.cpp b/src/blockchain_converter/blockchain_converter.cpp index 4653cf66c..1ae1efc72 100644 --- a/src/blockchain_converter/blockchain_converter.cpp +++ b/src/blockchain_converter/blockchain_converter.cpp @@ -50,18 +50,17 @@ struct fake_core blockchain_storage m_storage; - fake_core() : m_pool(dummy), dummy(m_pool), m_storage(&m_pool) + fake_core(const boost::filesystem::path &path) : m_pool(dummy), dummy(m_pool), m_storage(&m_pool) { - boost::filesystem::path default_data_path {tools::get_default_data_dir()}; - m_pool.init(default_data_path.string()); - m_storage.init(default_data_path.string(), false); + m_pool.init(path.string()); + m_storage.init(path.string(), false); } }; int main(int argc, char* argv[]) { - fake_core c; boost::filesystem::path default_data_path {tools::get_default_data_dir()}; + fake_core c(default_data_path); BlockchainDB *blockchain; -- cgit v1.2.3 From 609cf7fc92987da0585651a6e95225575ecacd09 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Fri, 12 Dec 2014 11:01:17 +0000 Subject: blockchain_converter: a bit more user friendly output --- src/blockchain_converter/blockchain_converter.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/blockchain_converter/blockchain_converter.cpp b/src/blockchain_converter/blockchain_converter.cpp index 1ae1efc72..f2dd10b18 100644 --- a/src/blockchain_converter/blockchain_converter.cpp +++ b/src/blockchain_converter/blockchain_converter.cpp @@ -68,9 +68,13 @@ int main(int argc, char* argv[]) blockchain->open(default_data_path.string()); - for (uint64_t i = 0; i < c.m_storage.get_current_blockchain_height(); ++i) + for (uint64_t height, i = 0; i < (height = c.m_storage.get_current_blockchain_height()); ++i) { - if (i % 10 == 0) std::cout << "block " << i << std::endl; + if (i % 10 == 0) + { + std::cout << "\r \r" << "block " << i << "/" << height + << " (" << (i+1)*100/height<< "%)" << std::flush; + } block b = c.m_storage.get_block(i); size_t bsize = c.m_storage.get_block_size(i); difficulty_type bdiff = c.m_storage.get_block_cumulative_difficulty(i); @@ -81,6 +85,7 @@ int main(int argc, char* argv[]) c.m_storage.get_transactions(b.tx_hashes, txs, missed); if (missed.size()) { + std::cout << std::endl; std::cerr << "Missed transaction(s) for block at height " << i << ", exiting" << std::endl; delete blockchain; return 1; @@ -92,12 +97,14 @@ int main(int argc, char* argv[]) } catch (const std::exception& e) { + std::cout << std::endl; std::cerr << "Error adding block to new blockchain: " << e.what() << std::endl; delete blockchain; return 2; } } + std::cout << std::endl; delete blockchain; return 0; } -- cgit v1.2.3 From 3fcb8daf6e337e9eeb41b00d0583758a09eed823 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Fri, 12 Dec 2014 13:27:05 +0000 Subject: db_lmdb: factor the MDB_val setup code It makes the code simpler, avoids possible copy/paste errors (wrong sizeof, etc), and generally unclutters the calling code. --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 271 ++++++---------------- 1 file changed, 74 insertions(+), 197 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index 3b4de0a62..a0f42bebc 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -72,6 +72,31 @@ private: bool done; }; +template +struct MDB_val_copy: public MDB_val +{ + MDB_val_copy(const T &t): t_copy(t) + { + mv_size = sizeof (T); + mv_data = &t_copy; + } +private: + T t_copy; +}; + +template<> +struct MDB_val_copy: public MDB_val +{ + MDB_val_copy(const cryptonote::blobdata &bd): data(new char[bd.size()]) + { + memcpy(data.get(), bd.data(), bd.size()); + mv_size = bd.size(); + mv_data = data.get(); + } +private: + std::unique_ptr data; +}; + auto compare_uint64 = [](const MDB_val *a, const MDB_val *b) { const uint64_t va = *(const uint64_t*)a->mv_data; const uint64_t vb = *(const uint64_t*)b->mv_data; @@ -124,11 +149,7 @@ void BlockchainLMDB::add_block( const block& blk LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - crypto::hash h = get_block_hash(blk); - MDB_val val_h; - val_h.mv_size = sizeof(crypto::hash); - val_h.mv_data = &h; - + MDB_val_copy val_h(get_block_hash(blk)); MDB_val unused; if (mdb_get(*m_write_txn, m_block_heights, &val_h, &unused) == 0) { @@ -138,11 +159,7 @@ void BlockchainLMDB::add_block( const block& blk if (m_height > 0) { - MDB_val parent_key; - crypto::hash parent = blk.prev_id; - parent_key.mv_size = sizeof(crypto::hash); - parent_key.mv_data = &parent; - + MDB_val_copy parent_key(blk.prev_id); MDB_val parent_h; if (mdb_get(*m_write_txn, m_block_heights, &parent_key, &parent_h)) { @@ -158,57 +175,38 @@ void BlockchainLMDB::add_block( const block& blk } } - MDB_val key; - key.mv_size = sizeof(uint64_t); - key.mv_data = &m_height; - - auto bd = block_to_blob(blk); + MDB_val_copy key(m_height); - // const-correctness be trolling, yo - std::unique_ptr bd_cpy(new char[bd.size()]); - memcpy(bd_cpy.get(), bd.data(), bd.size()); - - MDB_val blob; - blob.mv_size = bd.size(); - blob.mv_data = bd_cpy.get(); + MDB_val_copy blob(block_to_blob(blk)); if (mdb_put(*m_write_txn, m_blocks, &key, &blob, 0)) { LOG_PRINT_L0("Failed to add block blob to db transaction"); throw DB_ERROR("Failed to add block blob to db transaction"); } - size_t size_cpy = block_size; - MDB_val sz; - sz.mv_size = sizeof(block_size); - sz.mv_data = &size_cpy; + MDB_val_copy sz(block_size); if (mdb_put(*m_write_txn, m_block_sizes, &key, &sz, 0)) { LOG_PRINT_L0("Failed to add block size to db transaction"); throw DB_ERROR("Failed to add block size to db transaction"); } - uint64_t time_cpy = blk.timestamp; - sz.mv_size = sizeof(time_cpy); - sz.mv_data = &time_cpy; - if (mdb_put(*m_write_txn, m_block_timestamps, &key, &sz, 0)) + MDB_val_copy ts(blk.timestamp); + if (mdb_put(*m_write_txn, m_block_timestamps, &key, &ts, 0)) { LOG_PRINT_L0("Failed to add block timestamp to db transaction"); throw DB_ERROR("Failed to add block timestamp to db transaction"); } - difficulty_type diff_cpy = cumulative_difficulty; - sz.mv_size = sizeof(cumulative_difficulty); - sz.mv_data = &diff_cpy; - if (mdb_put(*m_write_txn, m_block_diffs, &key, &sz, 0)) + MDB_val_copy diff(cumulative_difficulty); + if (mdb_put(*m_write_txn, m_block_diffs, &key, &diff, 0)) { LOG_PRINT_L0("Failed to add block cumulative difficulty to db transaction"); throw DB_ERROR("Failed to add block cumulative difficulty to db transaction"); } - uint64_t coins_cpy = coins_generated; - sz.mv_size = sizeof(coins_generated); - sz.mv_data = &coins_cpy; - if (mdb_put(*m_write_txn, m_block_coins, &key, &sz, 0)) + MDB_val_copy coinsgen(coins_generated); + if (mdb_put(*m_write_txn, m_block_coins, &key, &coinsgen, 0)) { LOG_PRINT_L0("Failed to add block total generated coins to db transaction"); throw DB_ERROR("Failed to add block total generated coins to db transaction"); @@ -233,11 +231,7 @@ void BlockchainLMDB::remove_block() LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - MDB_val k; - uint64_t height = m_height - 1; - k.mv_size = sizeof(uint64_t); - k.mv_data = &height; - + MDB_val_copy k(m_height - 1); MDB_val h; if (mdb_get(*m_write_txn, m_block_hashes, &k, &h)) { @@ -293,11 +287,7 @@ void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const tr LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - crypto::hash h = get_transaction_hash(tx); - MDB_val val_h; - val_h.mv_size = sizeof(crypto::hash); - val_h.mv_data = &h; - + MDB_val_copy val_h(get_transaction_hash(tx)); MDB_val unused; if (mdb_get(*m_write_txn, m_txs, &val_h, &unused) == 0) { @@ -305,34 +295,22 @@ void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const tr throw TX_EXISTS("Attempting to add transaction that's already in the db"); } - auto bd = tx_to_blob(tx); - - // const-correctness be trolling, yo - std::unique_ptr bd_cpy(new char[bd.size()]); - memcpy(bd_cpy.get(), bd.data(), bd.size()); - - MDB_val blob; - blob.mv_size = bd.size(); - blob.mv_data = bd_cpy.get(); + MDB_val_copy blob(tx_to_blob(tx)); if (mdb_put(*m_write_txn, m_txs, &val_h, &blob, 0)) { LOG_PRINT_L0("Failed to add tx blob to db transaction"); throw DB_ERROR("Failed to add tx blob to db transaction"); } - MDB_val height; - height.mv_size = sizeof(uint64_t); - height.mv_data = &m_height; + MDB_val_copy height(m_height); if (mdb_put(*m_write_txn, m_tx_heights, &val_h, &height, 0)) { LOG_PRINT_L0("Failed to add tx block height to db transaction"); throw DB_ERROR("Failed to add tx block height to db transaction"); } - uint64_t unlock_cpy = tx.unlock_time; - height.mv_size = sizeof(unlock_cpy); - height.mv_data = &unlock_cpy; - if (mdb_put(*m_write_txn, m_tx_unlocks, &val_h, &height, 0)) + MDB_val_copy unlock_time(tx.unlock_time); + if (mdb_put(*m_write_txn, m_tx_unlocks, &val_h, &unlock_time, 0)) { LOG_PRINT_L0("Failed to add tx unlock time to db transaction"); throw DB_ERROR("Failed to add tx unlock time to db transaction"); @@ -344,11 +322,7 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash) LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - crypto::hash h = tx_hash; - MDB_val val_h; - val_h.mv_size = sizeof(crypto::hash); - val_h.mv_data = &h; - + MDB_val_copy val_h(tx_hash); MDB_val unused; if (mdb_get(*m_write_txn, m_txs, &val_h, &unused)) { @@ -387,14 +361,9 @@ void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_ou LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - MDB_val k; - MDB_val v; + MDB_val_copy k(m_num_outputs); + MDB_val_copy v(tx_hash); - k.mv_size = sizeof(uint64_t); - k.mv_data = &m_num_outputs; - crypto::hash h_cpy = tx_hash; - v.mv_size = sizeof(crypto::hash); - v.mv_data = &h_cpy; if (mdb_put(*m_write_txn, m_output_txs, &k, &v, 0)) { LOG_PRINT_L0("Failed to add output tx hash to db transaction"); @@ -406,19 +375,15 @@ void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_ou throw DB_ERROR("Failed to add tx output index to db transaction"); } - uint64_t index_cpy = local_index; - v.mv_size = sizeof(uint64_t); - v.mv_data = &index_cpy; - if (mdb_put(*m_write_txn, m_output_indices, &k, &v, 0)) + MDB_val_copy val_local_index(local_index); + if (mdb_put(*m_write_txn, m_output_indices, &k, &val_local_index, 0)) { LOG_PRINT_L0("Failed to add tx output index to db transaction"); throw DB_ERROR("Failed to add tx output index to db transaction"); } - uint64_t amount = tx_output.amount; - v.mv_size = sizeof(uint64_t); - v.mv_data = &amount; - if (auto result = mdb_put(*m_write_txn, m_output_amounts, &v, &k, 0)) + MDB_val_copy val_amount(tx_output.amount); + if (auto result = mdb_put(*m_write_txn, m_output_amounts, &val_amount, &k, 0)) { LOG_PRINT_L0("Failed to add output amount to db transaction"); LOG_PRINT_L0("E: " << mdb_strerror(result)); @@ -427,11 +392,8 @@ void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_ou if (tx_output.target.type() == typeid(txout_to_key)) { - crypto::public_key pubkey = boost::get(tx_output.target).key; - - v.mv_size = sizeof(pubkey); - v.mv_data = &pubkey; - if (mdb_put(*m_write_txn, m_output_keys, &k, &v, 0)) + MDB_val_copy val_pubkey(boost::get(tx_output.target).key); + if (mdb_put(*m_write_txn, m_output_keys, &k, &val_pubkey, 0)) { LOG_PRINT_L0("Failed to add output pubkey to db transaction"); throw DB_ERROR("Failed to add output pubkey to db transaction"); @@ -465,11 +427,7 @@ void BlockchainLMDB::remove_tx_outputs(const crypto::hash& tx_hash) lmdb_cur cur(*m_write_txn, m_tx_outputs); - crypto::hash h_cpy = tx_hash; - MDB_val k; - k.mv_size = sizeof(h_cpy); - k.mv_data = &h_cpy; - + MDB_val_copy k(tx_hash); MDB_val v; auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); @@ -584,11 +542,7 @@ void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image) LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - crypto::key_image key = k_image; - MDB_val val_key; - val_key.mv_size = sizeof(crypto::key_image); - val_key.mv_data = &key; - + MDB_val_copy val_key(k_image); MDB_val unused; if (mdb_get(*m_write_txn, m_spent_keys, &val_key, &unused) == 0) { @@ -611,10 +565,7 @@ void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image) LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - crypto::key_image key_cpy = k_image; - MDB_val k; - k.mv_size = sizeof(crypto::key_image); - k.mv_data = &key_cpy; + MDB_val_copy k(k_image); auto result = mdb_del(*m_write_txn, m_spent_keys, &k, NULL); if (result != 0 && result != MDB_NOTFOUND) { @@ -872,11 +823,7 @@ bool BlockchainLMDB::block_exists(const crypto::hash& h) const throw DB_ERROR("Failed to create a transaction for the db"); } - crypto::hash key_cpy = h; - MDB_val key; - key.mv_size = sizeof(crypto::hash); - key.mv_data = &key_cpy; - + MDB_val_copy key(h); MDB_val result; auto get_result = mdb_get(txn, m_block_heights, &key, &result); if (get_result == MDB_NOTFOUND) @@ -915,11 +862,7 @@ uint64_t BlockchainLMDB::get_block_height(const crypto::hash& h) const throw DB_ERROR("Failed to create a transaction for the db"); } - crypto::hash key_cpy = h; - MDB_val key; - key.mv_size = sizeof(crypto::hash); - key.mv_data = &key_cpy; - + MDB_val_copy key(h); MDB_val result; auto get_result = mdb_get(txn, m_block_heights, &key, &result); if (get_result == MDB_NOTFOUND) @@ -958,10 +901,7 @@ block BlockchainLMDB::get_block_from_height(const uint64_t& height) const throw DB_ERROR("Failed to create a transaction for the db"); } - uint64_t height_cpy = height; - MDB_val key; - key.mv_size = sizeof(uint64_t); - key.mv_data = &height_cpy; + MDB_val_copy key(height); MDB_val result; auto get_result = mdb_get(txn, m_blocks, &key, &result); if (get_result == MDB_NOTFOUND) @@ -1002,10 +942,7 @@ uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const throw DB_ERROR("Failed to create a transaction for the db"); } - uint64_t height_cpy = height; - MDB_val key; - key.mv_size = sizeof(uint64_t); - key.mv_data = &height_cpy; + MDB_val_copy key(height); MDB_val result; auto get_result = mdb_get(txn, m_block_timestamps, &key, &result); if (get_result == MDB_NOTFOUND) @@ -1049,10 +986,7 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) const throw DB_ERROR("Failed to create a transaction for the db"); } - uint64_t height_cpy = height; - MDB_val key; - key.mv_size = sizeof(uint64_t); - key.mv_data = &height_cpy; + MDB_val_copy key(height); MDB_val result; auto get_result = mdb_get(txn, m_block_sizes, &key, &result); if (get_result == MDB_NOTFOUND) @@ -1082,10 +1016,7 @@ difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& throw DB_ERROR("Failed to create a transaction for the db"); } - uint64_t height_cpy = height; - MDB_val key; - key.mv_size = sizeof(uint64_t); - key.mv_data = &height_cpy; + MDB_val_copy key(height); MDB_val result; auto get_result = mdb_get(txn, m_block_diffs, &key, &result); if (get_result == MDB_NOTFOUND) @@ -1132,10 +1063,7 @@ uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& heigh throw DB_ERROR("Failed to create a transaction for the db"); } - uint64_t height_cpy = height; - MDB_val key; - key.mv_size = sizeof(uint64_t); - key.mv_data = &height_cpy; + MDB_val_copy key(height); MDB_val result; auto get_result = mdb_get(txn, m_block_coins, &key, &result); if (get_result == MDB_NOTFOUND) @@ -1165,10 +1093,7 @@ crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height) throw DB_ERROR("Failed to create a transaction for the db"); } - uint64_t height_cpy = height; - MDB_val key; - key.mv_size = sizeof(uint64_t); - key.mv_data = &height_cpy; + MDB_val_copy key(height); MDB_val result; auto get_result = mdb_get(txn, m_block_hashes, &key, &result); if (get_result == MDB_NOTFOUND) @@ -1261,11 +1186,7 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) const throw DB_ERROR("Failed to create a transaction for the db"); } - crypto::hash key_cpy = h; - MDB_val key; - key.mv_size = sizeof(crypto::hash); - key.mv_data = &key_cpy; - + MDB_val_copy key(h); MDB_val result; auto get_result = mdb_get(txn, m_txs, &key, &result); if (get_result == MDB_NOTFOUND) @@ -1295,11 +1216,7 @@ uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) const throw DB_ERROR("Failed to create a transaction for the db"); } - crypto::hash key_cpy = h; - MDB_val key; - key.mv_size = sizeof(crypto::hash); - key.mv_data = &key_cpy; - + MDB_val_copy key(h); MDB_val result; auto get_result = mdb_get(txn, m_tx_unlocks, &key, &result); if (get_result == MDB_NOTFOUND) @@ -1328,11 +1245,7 @@ transaction BlockchainLMDB::get_tx(const crypto::hash& h) const throw DB_ERROR("Failed to create a transaction for the db"); } - crypto::hash key_cpy = h; - MDB_val key; - key.mv_size = sizeof(crypto::hash); - key.mv_data = &key_cpy; - + MDB_val_copy key(h); MDB_val result; auto get_result = mdb_get(txn, m_txs, &key, &result); if (get_result == MDB_NOTFOUND) @@ -1409,11 +1322,7 @@ uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const throw DB_ERROR("Failed to create a transaction for the db"); } - crypto::hash key_cpy = h; - MDB_val key; - key.mv_size = sizeof(crypto::hash); - key.mv_data = &key_cpy; - + MDB_val_copy key(h); MDB_val result; auto get_result = mdb_get(txn, m_tx_heights, &key, &result); if (get_result == MDB_NOTFOUND) @@ -1460,11 +1369,7 @@ uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) const lmdb_cur cur(txn, m_output_amounts); - uint64_t amount_cpy = amount; - MDB_val k; - k.mv_size = sizeof(amount_cpy); - k.mv_data = &amount_cpy; - + MDB_val_copy k(amount); MDB_val v; auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); if (result == MDB_NOTFOUND) @@ -1490,8 +1395,6 @@ crypto::public_key BlockchainLMDB::get_output_key(const uint64_t& amount, const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - uint64_t global = get_output_global_index(amount, index); - txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) { @@ -1499,10 +1402,7 @@ crypto::public_key BlockchainLMDB::get_output_key(const uint64_t& amount, const throw DB_ERROR("Failed to create a transaction for the db"); } - MDB_val k; - k.mv_size = sizeof(global); - k.mv_data = &global; - + MDB_val_copy k(get_output_global_index(amount, index)); MDB_val v; auto get_result = mdb_get(txn, m_output_keys, &k, &v); if (get_result == MDB_NOTFOUND) @@ -1531,15 +1431,10 @@ tx_out BlockchainLMDB::get_output(const crypto::hash& h, const uint64_t& index) throw DB_ERROR("Failed to create a transaction for the db"); } - MDB_val k; - crypto::hash h_cpy = h; - k.mv_size = sizeof(h_cpy); - k.mv_data = &h_cpy; - lmdb_cur cur(txn, m_tx_outputs); + MDB_val_copy k(h); MDB_val v; - auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); if (result == MDB_NOTFOUND) { @@ -1596,11 +1491,7 @@ tx_out BlockchainLMDB::get_output(const uint64_t& index) const throw DB_ERROR("Failed to create a transaction for the db"); } - uint64_t index_cpy = index; - MDB_val k; - k.mv_size = sizeof(index_cpy); - k.mv_data = &index_cpy; - + MDB_val_copy k(index); MDB_val v; auto get_result = mdb_get(txn, m_outputs, &k, &v); if (get_result == MDB_NOTFOUND) @@ -1633,11 +1524,7 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con lmdb_cur cur(txn, m_output_amounts); - uint64_t amount_cpy = amount; - MDB_val k; - k.mv_size = sizeof(amount_cpy); - k.mv_data = &amount_cpy; - + MDB_val_copy k(amount); MDB_val v; auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); @@ -1676,8 +1563,7 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con cur.close(); - k.mv_size = sizeof(glob_index); - k.mv_data = &glob_index; + k = MDB_val_copy(glob_index); auto get_result = mdb_get(txn, m_output_txs, &k, &v); if (get_result == MDB_NOTFOUND) { @@ -1722,15 +1608,10 @@ std::vector BlockchainLMDB::get_tx_output_indices(const crypto::hash& throw DB_ERROR("Failed to create a transaction for the db"); } - MDB_val k; - crypto::hash h_cpy = h; - k.mv_size = sizeof(h_cpy); - k.mv_data = &h_cpy; - lmdb_cur cur(txn, m_tx_outputs); + MDB_val_copy k(h); MDB_val v; - auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); if (result == MDB_NOTFOUND) { @@ -1773,11 +1654,7 @@ bool BlockchainLMDB::has_key_image(const crypto::key_image& img) const throw DB_ERROR("Failed to create a transaction for the db"); } - crypto::key_image key = img; - MDB_val val_key; - val_key.mv_size = sizeof(crypto::key_image); - val_key.mv_data = &key; - + MDB_val_copy val_key(img); MDB_val unused; if (mdb_get(txn, m_spent_keys, &val_key, &unused) == 0) { -- cgit v1.2.3 From 3a3459d59ba655ea5a19cbb3ebb9de2890e87ea3 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Fri, 12 Dec 2014 21:34:45 +0000 Subject: db_lmdb: factor all the log+throw code paths --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 623 +++++----------------- 1 file changed, 138 insertions(+), 485 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index a0f42bebc..f53e5bac2 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -39,6 +39,18 @@ using epee::string_tools::pod_to_hex; namespace { +inline void throw0(const std::exception &e) +{ + LOG_PRINT_L0(e.what()); + throw e; +} + +inline void throw1(const std::exception &e) +{ + LOG_PRINT_L1(e.what()); + throw e; +} + // cursor needs to be closed when it goes out of scope, // this helps if the function using it throws struct lmdb_cur @@ -46,10 +58,7 @@ struct lmdb_cur lmdb_cur(MDB_txn* txn, MDB_dbi dbi) { if (mdb_cursor_open(txn, dbi, &m_cur)) - { - LOG_PRINT_L0("Error opening db cursor"); - throw cryptonote::DB_ERROR("Error opening db cursor"); - } + throw0(cryptonote::DB_ERROR("Error opening db cursor")); done = false; } @@ -129,10 +138,7 @@ const char* const LMDB_SPENT_KEYS = "spent_keys"; inline void lmdb_db_open(MDB_txn* txn, const char* name, int flags, MDB_dbi& dbi, const std::string& error_string) { if (mdb_dbi_open(txn, name, flags, &dbi)) - { - LOG_PRINT_L0(error_string); - throw cryptonote::DB_OPEN_FAILURE(error_string.c_str()); - } + throw0(cryptonote::DB_OPEN_FAILURE(error_string.c_str())); } } // anonymous namespace @@ -152,77 +158,47 @@ void BlockchainLMDB::add_block( const block& blk MDB_val_copy val_h(get_block_hash(blk)); MDB_val unused; if (mdb_get(*m_write_txn, m_block_heights, &val_h, &unused) == 0) - { - LOG_PRINT_L1("Attempting to add block that's already in the db"); - throw BLOCK_EXISTS("Attempting to add block that's already in the db"); - } + throw1(BLOCK_EXISTS("Attempting to add block that's already in the db")); if (m_height > 0) { MDB_val_copy parent_key(blk.prev_id); MDB_val parent_h; if (mdb_get(*m_write_txn, m_block_heights, &parent_key, &parent_h)) - { - LOG_PRINT_L0("Failed to get top block hash to check for new block's parent"); - throw DB_ERROR("Failed to get top block hash to check for new block's parent"); - } + throw0(DB_ERROR("Failed to get top block hash to check for new block's parent")); uint64_t parent_height = *(const uint64_t *)parent_h.mv_data; if (parent_height != m_height - 1) - { - LOG_PRINT_L0("Top block is not new block's parent"); - throw BLOCK_PARENT_DNE("Top block is not new block's parent"); - } + throw0(BLOCK_PARENT_DNE("Top block is not new block's parent")); } MDB_val_copy key(m_height); MDB_val_copy blob(block_to_blob(blk)); if (mdb_put(*m_write_txn, m_blocks, &key, &blob, 0)) - { - LOG_PRINT_L0("Failed to add block blob to db transaction"); - throw DB_ERROR("Failed to add block blob to db transaction"); - } + throw0(DB_ERROR("Failed to add block blob to db transaction")); MDB_val_copy sz(block_size); if (mdb_put(*m_write_txn, m_block_sizes, &key, &sz, 0)) - { - LOG_PRINT_L0("Failed to add block size to db transaction"); - throw DB_ERROR("Failed to add block size to db transaction"); - } + throw0(DB_ERROR("Failed to add block size to db transaction")); MDB_val_copy ts(blk.timestamp); if (mdb_put(*m_write_txn, m_block_timestamps, &key, &ts, 0)) - { - LOG_PRINT_L0("Failed to add block timestamp to db transaction"); - throw DB_ERROR("Failed to add block timestamp to db transaction"); - } + throw0(DB_ERROR("Failed to add block timestamp to db transaction")); MDB_val_copy diff(cumulative_difficulty); if (mdb_put(*m_write_txn, m_block_diffs, &key, &diff, 0)) - { - LOG_PRINT_L0("Failed to add block cumulative difficulty to db transaction"); - throw DB_ERROR("Failed to add block cumulative difficulty to db transaction"); - } + throw0(DB_ERROR("Failed to add block cumulative difficulty to db transaction")); MDB_val_copy coinsgen(coins_generated); if (mdb_put(*m_write_txn, m_block_coins, &key, &coinsgen, 0)) - { - LOG_PRINT_L0("Failed to add block total generated coins to db transaction"); - throw DB_ERROR("Failed to add block total generated coins to db transaction"); - } + throw0(DB_ERROR("Failed to add block total generated coins to db transaction")); if (mdb_put(*m_write_txn, m_block_heights, &val_h, &key, 0)) - { - LOG_PRINT_L0("Failed to add block height by hash to db transaction"); - throw DB_ERROR("Failed to add block height by hash to db transaction"); - } + throw0(DB_ERROR("Failed to add block height by hash to db transaction")); if (mdb_put(*m_write_txn, m_block_hashes, &key, &val_h, 0)) - { - LOG_PRINT_L0("Failed to add block hash to db transaction"); - throw DB_ERROR("Failed to add block hash to db transaction"); - } + throw0(DB_ERROR("Failed to add block hash to db transaction")); } @@ -234,52 +210,28 @@ void BlockchainLMDB::remove_block() MDB_val_copy k(m_height - 1); MDB_val h; if (mdb_get(*m_write_txn, m_block_hashes, &k, &h)) - { - LOG_PRINT_L1("Attempting to remove block that's not in the db"); - throw BLOCK_DNE("Attempting to remove block that's not in the db"); - } + throw1(BLOCK_DNE("Attempting to remove block that's not in the db")); if (mdb_del(*m_write_txn, m_blocks, &k, NULL)) - { - LOG_PRINT_L1("Failed to add removal of block to db transaction"); - throw DB_ERROR("Failed to add removal of block to db transaction"); - } + throw1(DB_ERROR("Failed to add removal of block to db transaction")); if (mdb_del(*m_write_txn, m_block_sizes, &k, NULL)) - { - LOG_PRINT_L1("Failed to add removal of block size to db transaction"); - throw DB_ERROR("Failed to add removal of block size to db transaction"); - } + throw1(DB_ERROR("Failed to add removal of block size to db transaction")); if (mdb_del(*m_write_txn, m_block_diffs, &k, NULL)) - { - LOG_PRINT_L1("Failed to add removal of block cumulative difficulty to db transaction"); - throw DB_ERROR("Failed to add removal of block cumulative difficulty to db transaction"); - } + throw1(DB_ERROR("Failed to add removal of block cumulative difficulty to db transaction")); if (mdb_del(*m_write_txn, m_block_coins, &k, NULL)) - { - LOG_PRINT_L1("Failed to add removal of block total generated coins to db transaction"); - throw DB_ERROR("Failed to add removal of block total generated coins to db transaction"); - } + throw1(DB_ERROR("Failed to add removal of block total generated coins to db transaction")); if (mdb_del(*m_write_txn, m_block_timestamps, &k, NULL)) - { - LOG_PRINT_L1("Failed to add removal of block timestamp to db transaction"); - throw DB_ERROR("Failed to add removal of block timestamp to db transaction"); - } + throw1(DB_ERROR("Failed to add removal of block timestamp to db transaction")); if (mdb_del(*m_write_txn, m_block_heights, &h, NULL)) - { - LOG_PRINT_L1("Failed to add removal of block height by hash to db transaction"); - throw DB_ERROR("Failed to add removal of block height by hash to db transaction"); - } + throw1(DB_ERROR("Failed to add removal of block height by hash to db transaction")); if (mdb_del(*m_write_txn, m_block_hashes, &k, NULL)) - { - LOG_PRINT_L1("Failed to add removal of block hash to db transaction"); - throw DB_ERROR("Failed to add removal of block hash to db transaction"); - } + throw1(DB_ERROR("Failed to add removal of block hash to db transaction")); } void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx) @@ -290,31 +242,19 @@ void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const tr MDB_val_copy val_h(get_transaction_hash(tx)); MDB_val unused; if (mdb_get(*m_write_txn, m_txs, &val_h, &unused) == 0) - { - LOG_PRINT_L1("Attempting to add transaction that's already in the db"); - throw TX_EXISTS("Attempting to add transaction that's already in the db"); - } + throw1(TX_EXISTS("Attempting to add transaction that's already in the db")); MDB_val_copy blob(tx_to_blob(tx)); if (mdb_put(*m_write_txn, m_txs, &val_h, &blob, 0)) - { - LOG_PRINT_L0("Failed to add tx blob to db transaction"); - throw DB_ERROR("Failed to add tx blob to db transaction"); - } + throw0(DB_ERROR("Failed to add tx blob to db transaction")); MDB_val_copy height(m_height); if (mdb_put(*m_write_txn, m_tx_heights, &val_h, &height, 0)) - { - LOG_PRINT_L0("Failed to add tx block height to db transaction"); - throw DB_ERROR("Failed to add tx block height to db transaction"); - } + throw0(DB_ERROR("Failed to add tx block height to db transaction")); MDB_val_copy unlock_time(tx.unlock_time); if (mdb_put(*m_write_txn, m_tx_unlocks, &val_h, &unlock_time, 0)) - { - LOG_PRINT_L0("Failed to add tx unlock time to db transaction"); - throw DB_ERROR("Failed to add tx unlock time to db transaction"); - } + throw0(DB_ERROR("Failed to add tx unlock time to db transaction")); } void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash) @@ -325,34 +265,19 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash) MDB_val_copy val_h(tx_hash); MDB_val unused; if (mdb_get(*m_write_txn, m_txs, &val_h, &unused)) - { - LOG_PRINT_L1("Attempting to remove transaction that isn't in the db"); - throw TX_DNE("Attempting to remove transaction that isn't in the db"); - } + throw1(TX_DNE("Attempting to remove transaction that isn't in the db")); if (mdb_del(*m_write_txn, m_txs, &val_h, NULL)) - { - LOG_PRINT_L1("Failed to add removal of tx to db transaction"); - throw DB_ERROR("Failed to add removal of tx to db transaction"); - } + throw1(DB_ERROR("Failed to add removal of tx to db transaction")); if (mdb_del(*m_write_txn, m_tx_unlocks, &val_h, NULL)) - { - LOG_PRINT_L1("Failed to add removal of tx unlock time to db transaction"); - throw DB_ERROR("Failed to add removal of tx unlock time to db transaction"); - } + throw1(DB_ERROR("Failed to add removal of tx unlock time to db transaction")); if (mdb_del(*m_write_txn, m_tx_heights, &val_h, NULL)) - { - LOG_PRINT_L1("Failed to add removal of tx block height to db transaction"); - throw DB_ERROR("Failed to add removal of tx block height to db transaction"); - } + throw1(DB_ERROR("Failed to add removal of tx block height to db transaction")); remove_tx_outputs(tx_hash); if (mdb_del(*m_write_txn, m_tx_outputs, &val_h, NULL)) - { - LOG_PRINT_L1("Failed to add removal of tx outputs to db transaction"); - throw DB_ERROR("Failed to add removal of tx outputs to db transaction"); - } + throw1(DB_ERROR("Failed to add removal of tx outputs to db transaction")); } @@ -365,39 +290,23 @@ void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_ou MDB_val_copy v(tx_hash); if (mdb_put(*m_write_txn, m_output_txs, &k, &v, 0)) - { - LOG_PRINT_L0("Failed to add output tx hash to db transaction"); - throw DB_ERROR("Failed to add output tx hash to db transaction"); - } + throw0(DB_ERROR("Failed to add output tx hash to db transaction")); if (mdb_put(*m_write_txn, m_tx_outputs, &v, &k, 0)) - { - LOG_PRINT_L0("Failed to add tx output index to db transaction"); - throw DB_ERROR("Failed to add tx output index to db transaction"); - } + throw0(DB_ERROR("Failed to add tx output index to db transaction")); MDB_val_copy val_local_index(local_index); if (mdb_put(*m_write_txn, m_output_indices, &k, &val_local_index, 0)) - { - LOG_PRINT_L0("Failed to add tx output index to db transaction"); - throw DB_ERROR("Failed to add tx output index to db transaction"); - } + throw0(DB_ERROR("Failed to add tx output index to db transaction")); MDB_val_copy val_amount(tx_output.amount); if (auto result = mdb_put(*m_write_txn, m_output_amounts, &val_amount, &k, 0)) - { - LOG_PRINT_L0("Failed to add output amount to db transaction"); - LOG_PRINT_L0("E: " << mdb_strerror(result)); - throw DB_ERROR("Failed to add output amount to db transaction"); - } + throw0(DB_ERROR(std::string("Failed to add output amount to db transaction").append(mdb_strerror(result)).c_str())); if (tx_output.target.type() == typeid(txout_to_key)) { MDB_val_copy val_pubkey(boost::get(tx_output.target).key); if (mdb_put(*m_write_txn, m_output_keys, &k, &val_pubkey, 0)) - { - LOG_PRINT_L0("Failed to add output pubkey to db transaction"); - throw DB_ERROR("Failed to add output pubkey to db transaction"); - } + throw0(DB_ERROR("Failed to add output pubkey to db transaction")); } @@ -408,15 +317,9 @@ void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_ou v.mv_size = b.size(); v.mv_data = &b; if (mdb_put(*m_write_txn, m_outputs, &k, &v, 0)) - { - LOG_PRINT_L0("Failed to add output to db transaction"); - throw DB_ERROR("Failed to add output to db transaction"); - } + throw0(DB_ERROR("Failed to add output to db transaction")); if (mdb_put(*m_write_txn, m_output_gindices, &v, &k, 0)) - { - LOG_PRINT_L0("Failed to add output global index to db transaction"); - throw DB_ERROR("Failed to add output global index to db transaction"); - } + throw0(DB_ERROR("Failed to add output global index to db transaction")); ************************************************************************/ m_num_outputs++; @@ -437,8 +340,7 @@ void BlockchainLMDB::remove_tx_outputs(const crypto::hash& tx_hash) } else if (result) { - LOG_PRINT_L0("DB error attempting to get an output"); - throw DB_ERROR("DB error attempting to get an output"); + throw0(DB_ERROR("DB error attempting to get an output")); } else { @@ -480,48 +382,30 @@ void BlockchainLMDB::remove_output(const uint64_t& out_index) k.mv_data = &b; if (mdb_get(*m_write_txn, m_output_gindices, &k, &v)) - { - LOG_PRINT_L1("Attempting to remove output that does not exist"); - throw OUTPUT_DNE("Attempting to remove output that does not exist"); - } + throw1(OUTPUT_DNE("Attempting to remove output that does not exist")); uint64_t gindex = *(uint64_t*)v.mv_data; auto result = mdb_del(*m_write_txn, m_output_gindices, &k, NULL); if (result != 0 && result != MDB_NOTFOUND) - { - LOG_PRINT_L1("Error adding removal of output global index to db transaction"); - throw DB_ERROR("Error adding removal of output global index to db transaction"); - } + throw1(DB_ERROR("Error adding removal of output global index to db transaction")); result = mdb_del(*m_write_txn, m_outputs, &v, NULL); if (result != 0 && result != MDB_NOTFOUND) - { - LOG_PRINT_L1("Error adding removal of output to db transaction"); - throw DB_ERROR("Error adding removal of output to db transaction"); - } + throw1(DB_ERROR("Error adding removal of output to db transaction")); *********************************************************************/ auto result = mdb_del(*m_write_txn, m_output_indices, &v, NULL); if (result != 0 && result != MDB_NOTFOUND) - { - LOG_PRINT_L1("Error adding removal of output tx index to db transaction"); - throw DB_ERROR("Error adding removal of output tx index to db transaction"); - } + throw1(DB_ERROR("Error adding removal of output tx index to db transaction")); result = mdb_del(*m_write_txn, m_output_txs, &v, NULL); if (result != 0 && result != MDB_NOTFOUND) - { - LOG_PRINT_L1("Error adding removal of output tx hash to db transaction"); - throw DB_ERROR("Error adding removal of output tx hash to db transaction"); - } + throw1(DB_ERROR("Error adding removal of output tx hash to db transaction")); result = mdb_del(*m_write_txn, m_output_amounts, &v, NULL); if (result != 0 && result != MDB_NOTFOUND) - { - LOG_PRINT_L1("Error adding removal of output amount to db transaction"); - throw DB_ERROR("Error adding removal of output amount to db transaction"); - } + throw1(DB_ERROR("Error adding removal of output amount to db transaction")); result = mdb_del(*m_write_txn, m_output_keys, &v, NULL); if (result == MDB_NOTFOUND) @@ -529,10 +413,7 @@ void BlockchainLMDB::remove_output(const uint64_t& out_index) LOG_PRINT_L2("Removing output, no public key found."); } else if (result) - { - LOG_PRINT_L1("Error adding removal of output pubkey to db transaction"); - throw DB_ERROR("Error adding removal of output pubkey to db transaction"); - } + throw1(DB_ERROR("Error adding removal of output pubkey to db transaction")); m_num_outputs--; } @@ -545,19 +426,13 @@ void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image) MDB_val_copy val_key(k_image); MDB_val unused; if (mdb_get(*m_write_txn, m_spent_keys, &val_key, &unused) == 0) - { - LOG_PRINT_L1("Attempting to add spent key image that's already in the db"); - throw KEY_IMAGE_EXISTS("Attempting to add spent key image that's already in the db"); - } + throw1(KEY_IMAGE_EXISTS("Attempting to add spent key image that's already in the db")); char anything = '\0'; unused.mv_size = sizeof(char); unused.mv_data = &anything; if (mdb_put(*m_write_txn, m_spent_keys, &val_key, &unused, 0)) - { - LOG_PRINT_L1("Error adding spent key image to db transaction"); - throw DB_ERROR("Error adding spent key image to db transaction"); - } + throw1(DB_ERROR("Error adding spent key image to db transaction")); } void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image) @@ -568,10 +443,7 @@ void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image) MDB_val_copy k(k_image); auto result = mdb_del(*m_write_txn, m_spent_keys, &k, NULL); if (result != 0 && result != MDB_NOTFOUND) - { - LOG_PRINT_L1("Error adding removal of key image to db transaction"); - throw DB_ERROR("Error adding removal of key image to db transaction"); - } + throw1(DB_ERROR("Error adding removal of key image to db transaction")); } blobdata BlockchainLMDB::output_to_blob(const tx_out& output) @@ -579,10 +451,7 @@ blobdata BlockchainLMDB::output_to_blob(const tx_out& output) LOG_PRINT_L3("BlockchainLMDB::" << __func__); blobdata b; if (!t_serializable_object_to_blob(output, b)) - { - LOG_PRINT_L1("Error serializing output to blob"); - throw DB_ERROR("Error serializing output to blob"); - } + throw1(DB_ERROR("Error serializing output to blob")); return b; } @@ -595,10 +464,7 @@ tx_out BlockchainLMDB::output_from_blob(const blobdata& blob) const tx_out o; if (!(::serialization::serialize(ba, o))) - { - LOG_PRINT_L1("Error deserializing tx output blob"); - throw DB_ERROR("Error deserializing tx output blob"); - } + throw1(DB_ERROR("Error deserializing tx output blob")); return o; } @@ -613,10 +479,7 @@ void BlockchainLMDB::check_open() const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); if (!m_open) - { - LOG_PRINT_L0("DB operation attempted on a not-open DB instance"); - throw DB_ERROR("DB operation attempted on a not-open DB instance"); - } + throw0(DB_ERROR("DB operation attempted on a not-open DB instance")); } BlockchainLMDB::~BlockchainLMDB() @@ -639,63 +502,38 @@ void BlockchainLMDB::open(const std::string& filename) LOG_PRINT_L3("BlockchainLMDB::" << __func__); if (m_open) - { - LOG_PRINT_L0("Attempted to open db, but it's already open"); - throw DB_OPEN_FAILURE("Attempted to open db, but it's already open"); - } + throw0(DB_OPEN_FAILURE("Attempted to open db, but it's already open")); boost::filesystem::path direc(filename); if (boost::filesystem::exists(direc)) { if (!boost::filesystem::is_directory(direc)) - { - LOG_PRINT_L0("LMDB needs a directory path, but a file was passed"); - throw DB_OPEN_FAILURE("LMDB needs a directory path, but a file was passed"); - } + throw0(DB_OPEN_FAILURE("LMDB needs a directory path, but a file was passed")); } else { if (!boost::filesystem::create_directory(direc)) - { - LOG_PRINT_L0("Failed to create directory " << filename); - throw DB_OPEN_FAILURE(std::string("Failed to create directory ").append(filename).c_str()); - } + throw0(DB_OPEN_FAILURE(std::string("Failed to create directory ").append(filename).c_str())); } m_folder = filename; // set up lmdb environment if (mdb_env_create(&m_env)) - { - LOG_PRINT_L0("Failed to create lmdb environment"); - throw DB_ERROR("Failed to create lmdb environment"); - } + throw0(DB_ERROR("Failed to create lmdb environment")); if (mdb_env_set_maxdbs(m_env, 20)) - { - LOG_PRINT_L0("Failed to set max number of dbs"); - throw DB_ERROR("Failed to set max number of dbs"); - } + throw0(DB_ERROR("Failed to set max number of dbs")); + size_t mapsize = 1LL << 34; if (auto result = mdb_env_set_mapsize(m_env, mapsize)) - { - LOG_PRINT_L0("Failed to set max memory map size"); - LOG_PRINT_L0("E: " << mdb_strerror(result)); - throw DB_ERROR("Failed to set max memory map size"); - } + throw0(DB_ERROR(std::string("Failed to set max memory map size: ").append(mdb_strerror(result)).c_str())); if (auto result = mdb_env_open(m_env, filename.c_str(), 0, 0664)) - { - LOG_PRINT_L0("Failed to open lmdb environment"); - LOG_PRINT_L0("E: " << mdb_strerror(result)); - throw DB_ERROR("Failed to open lmdb environment"); - } + throw0(DB_ERROR(std::string("Failed to open lmdb environment: ").append(mdb_strerror(result)).c_str())); // get a read/write MDB_txn txn_safe txn; if (mdb_txn_begin(m_env, NULL, 0, txn)) - { - LOG_PRINT_L0("Failed to create a transaction for the db"); - throw DB_ERROR("Failed to create a transaction for the db"); - } + throw0(DB_ERROR("Failed to create a transaction for the db")); // open necessary databases, and set properties as needed // uses macros to avoid having to change things too many places @@ -731,19 +569,13 @@ void BlockchainLMDB::open(const std::string& filename) // get and keep current height MDB_stat db_stats; if (mdb_stat(txn, m_blocks, &db_stats)) - { - LOG_PRINT_L0("Failed to query m_blocks"); - throw DB_ERROR("Failed to query m_blocks"); - } + throw0(DB_ERROR("Failed to query m_blocks")); LOG_PRINT_L2("Setting m_height to: " << db_stats.ms_entries); m_height = db_stats.ms_entries; // get and keep current number of outputs if (mdb_stat(txn, m_output_indices, &db_stats)) - { - LOG_PRINT_L0("Failed to query m_output_indices"); - throw DB_ERROR("Failed to query m_output_indices"); - } + throw0(DB_ERROR("Failed to query m_output_indices")); m_num_outputs = db_stats.ms_entries; // commit the transaction @@ -818,10 +650,7 @@ bool BlockchainLMDB::block_exists(const crypto::hash& h) const txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - { - LOG_PRINT_L0("Failed to create a transaction for the db"); - throw DB_ERROR("Failed to create a transaction for the db"); - } + throw0(DB_ERROR("Failed to create a transaction for the db")); MDB_val_copy key(h); MDB_val result; @@ -833,10 +662,7 @@ bool BlockchainLMDB::block_exists(const crypto::hash& h) const return false; } else if (get_result) - { - LOG_PRINT_L0("DB error attempting to fetch block index from hash"); - throw DB_ERROR("DB error attempting to fetch block index from hash"); - } + throw0(DB_ERROR("DB error attempting to fetch block index from hash")); txn.commit(); return true; @@ -857,24 +683,15 @@ uint64_t BlockchainLMDB::get_block_height(const crypto::hash& h) const txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - { - LOG_PRINT_L0("Failed to create a transaction for the db"); - throw DB_ERROR("Failed to create a transaction for the db"); - } + throw0(DB_ERROR("Failed to create a transaction for the db")); MDB_val_copy key(h); MDB_val result; auto get_result = mdb_get(txn, m_block_heights, &key, &result); if (get_result == MDB_NOTFOUND) - { - LOG_PRINT_L1("Attempted to retrieve non-existent block height"); - throw BLOCK_DNE("Attempted to retrieve non-existent block height"); - } + throw1(BLOCK_DNE("Attempted to retrieve non-existent block height")); else if (get_result) - { - LOG_PRINT_L0("Error attempting to retrieve a block height from the db"); - throw DB_ERROR("Error attempting to retrieve a block height from the db"); - } + throw0(DB_ERROR("Error attempting to retrieve a block height from the db")); txn.commit(); return *(const uint64_t*)result.mv_data; @@ -896,24 +713,17 @@ block BlockchainLMDB::get_block_from_height(const uint64_t& height) const txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - { - LOG_PRINT_L0("Failed to create a transaction for the db"); - throw DB_ERROR("Failed to create a transaction for the db"); - } + throw0(DB_ERROR("Failed to create a transaction for the db")); MDB_val_copy key(height); MDB_val result; auto get_result = mdb_get(txn, m_blocks, &key, &result); if (get_result == MDB_NOTFOUND) { - LOG_PRINT_L0("Attempted to get block from height " << height << ", but no such block exists"); - throw DB_ERROR("Attempt to get block from height failed -- block not in db"); + throw0(DB_ERROR(std::string("Attempt to get block from height ").append(boost::lexical_cast(height)).append(" failed -- block not in db").c_str())); } else if (get_result) - { - LOG_PRINT_L0("Error attempting to retrieve a block from the db"); - throw DB_ERROR("Error attempting to retrieve a block from the db"); - } + throw0(DB_ERROR("Error attempting to retrieve a block from the db")); txn.commit(); @@ -922,10 +732,7 @@ block BlockchainLMDB::get_block_from_height(const uint64_t& height) const block b; if (!parse_and_validate_block_from_blob(bd, b)) - { - LOG_PRINT_L0("Failed to parse block from blob retrieved from the db"); - throw DB_ERROR("Failed to parse block from blob retrieved from the db"); - } + throw0(DB_ERROR("Failed to parse block from blob retrieved from the db")); return b; } @@ -937,24 +744,17 @@ uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - { - LOG_PRINT_L0("Failed to create a transaction for the db"); - throw DB_ERROR("Failed to create a transaction for the db"); - } + throw0(DB_ERROR("Failed to create a transaction for the db")); MDB_val_copy key(height); MDB_val result; auto get_result = mdb_get(txn, m_block_timestamps, &key, &result); if (get_result == MDB_NOTFOUND) { - LOG_PRINT_L0("Attempted to get timestamp from height " << height << ", but no such timestamp exists"); - throw DB_ERROR("Attempt to get timestamp from height failed -- timestamp not in db"); + throw0(DB_ERROR(std::string("Attempt to get timestamp from height ").append(boost::lexical_cast(height)).append(" failed -- timestamp not in db").c_str())); } else if (get_result) - { - LOG_PRINT_L0("Error attempting to retrieve a timestamp from the db"); - throw DB_ERROR("Error attempting to retrieve a timestamp from the db"); - } + throw0(DB_ERROR("Error attempting to retrieve a timestamp from the db")); txn.commit(); return *(const uint64_t*)result.mv_data; @@ -981,24 +781,17 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) const txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - { - LOG_PRINT_L0("Failed to create a transaction for the db"); - throw DB_ERROR("Failed to create a transaction for the db"); - } + throw0(DB_ERROR("Failed to create a transaction for the db")); MDB_val_copy key(height); MDB_val result; auto get_result = mdb_get(txn, m_block_sizes, &key, &result); if (get_result == MDB_NOTFOUND) { - LOG_PRINT_L0("Attempted to get block size from height " << height << ", but no such block size exists"); - throw DB_ERROR("Attempt to get block size from height failed -- block size not in db"); + throw0(DB_ERROR(std::string("Attempt to get block size from height ").append(boost::lexical_cast(height)).append(" failed -- block size not in db").c_str())); } else if (get_result) - { - LOG_PRINT_L0("Error attempting to retrieve a block size from the db"); - throw DB_ERROR("Error attempting to retrieve a block size from the db"); - } + throw0(DB_ERROR("Error attempting to retrieve a block size from the db")); txn.commit(); return *(const size_t*)result.mv_data; @@ -1011,24 +804,17 @@ difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - { - LOG_PRINT_L0("Failed to create a transaction for the db"); - throw DB_ERROR("Failed to create a transaction for the db"); - } + throw0(DB_ERROR("Failed to create a transaction for the db")); MDB_val_copy key(height); MDB_val result; auto get_result = mdb_get(txn, m_block_diffs, &key, &result); if (get_result == MDB_NOTFOUND) { - LOG_PRINT_L0("Attempted to get cumulative difficulty from height " << height << ", but no such cumulative difficulty exists"); - throw DB_ERROR("Attempt to get cumulative difficulty from height failed -- block size not in db"); + throw0(DB_ERROR(std::string("Attempt to get cumulative difficulty from height ").append(boost::lexical_cast(height)).append(" failed -- block size not in db").c_str())); } else if (get_result) - { - LOG_PRINT_L0("Error attempting to retrieve a cumulative difficulty from the db"); - throw DB_ERROR("Error attempting to retrieve a cumulative difficulty from the db"); - } + throw0(DB_ERROR("Error attempting to retrieve a cumulative difficulty from the db")); txn.commit(); return *(difficulty_type*)result.mv_data; @@ -1058,24 +844,17 @@ uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& heigh txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - { - LOG_PRINT_L0("Failed to create a transaction for the db"); - throw DB_ERROR("Failed to create a transaction for the db"); - } + throw0(DB_ERROR("Failed to create a transaction for the db")); MDB_val_copy key(height); MDB_val result; auto get_result = mdb_get(txn, m_block_coins, &key, &result); if (get_result == MDB_NOTFOUND) { - LOG_PRINT_L0("Attempted to get total generated coins from height " << height << ", but no such total generated coins exists"); - throw DB_ERROR("Attempt to get total generated coins from height failed -- block size not in db"); + throw0(DB_ERROR(std::string("Attempt to get generated coins from height ").append(boost::lexical_cast(height)).append(" failed -- block size not in db").c_str())); } else if (get_result) - { - LOG_PRINT_L0("Error attempting to retrieve a total generated coins from the db"); - throw DB_ERROR("Error attempting to retrieve a total generated coins from the db"); - } + throw0(DB_ERROR("Error attempting to retrieve a total generated coins from the db")); txn.commit(); return *(const uint64_t*)result.mv_data; @@ -1088,24 +867,17 @@ crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height) txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - { - LOG_PRINT_L0("Failed to create a transaction for the db"); - throw DB_ERROR("Failed to create a transaction for the db"); - } + throw0(DB_ERROR("Failed to create a transaction for the db")); MDB_val_copy key(height); MDB_val result; auto get_result = mdb_get(txn, m_block_hashes, &key, &result); if (get_result == MDB_NOTFOUND) { - LOG_PRINT_L0("Attempted to get hash from height " << height << ", but no such hash exists"); - throw BLOCK_DNE("Attempt to get hash from height failed -- hash not in db"); + throw0(BLOCK_DNE(std::string("Attempt to get hash from height ").append(boost::lexical_cast(height)).append(" failed -- hash not in db").c_str())); } else if (get_result) - { - LOG_PRINT_L0("Error attempting to retrieve a block hash from the db"); - throw DB_ERROR("Error attempting to retrieve a block hash from the db"); - } + throw0(DB_ERROR("Error attempting to retrieve a block hash from the db")); txn.commit(); return *(crypto::hash*)result.mv_data; @@ -1181,10 +953,7 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) const txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - { - LOG_PRINT_L0("Failed to create a transaction for the db"); - throw DB_ERROR("Failed to create a transaction for the db"); - } + throw0(DB_ERROR("Failed to create a transaction for the db")); MDB_val_copy key(h); MDB_val result; @@ -1196,10 +965,7 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) const return false; } else if (get_result) - { - LOG_PRINT_L0("DB error attempting to fetch transaction from hash"); - throw DB_ERROR("DB error attempting to fetch transaction from hash"); - } + throw0(DB_ERROR("DB error attempting to fetch transaction from hash")); return true; } @@ -1211,24 +977,15 @@ uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) const txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - { - LOG_PRINT_L0("Failed to create a transaction for the db"); - throw DB_ERROR("Failed to create a transaction for the db"); - } + throw0(DB_ERROR("Failed to create a transaction for the db")); MDB_val_copy key(h); MDB_val result; auto get_result = mdb_get(txn, m_tx_unlocks, &key, &result); if (get_result == MDB_NOTFOUND) - { - LOG_PRINT_L1("tx unlock time with hash " << epee::string_tools::pod_to_hex(h) << "not found in db"); - throw TX_DNE("Attempting to get unlock time for tx, but tx not in db"); - } + throw1(TX_DNE(std::string("tx unlock time with hash ").append(epee::string_tools::pod_to_hex(h)).append("not found in db").c_str())); else if (get_result) - { - LOG_PRINT_L0("DB error attempting to fetch tx unlock time from hash"); - throw DB_ERROR("DB error attempting to fetch tx unlock time from hash"); - } + throw0(DB_ERROR("DB error attempting to fetch tx unlock time from hash")); return *(const uint64_t*)result.mv_data; } @@ -1240,34 +997,22 @@ transaction BlockchainLMDB::get_tx(const crypto::hash& h) const txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - { - LOG_PRINT_L0("Failed to create a transaction for the db"); - throw DB_ERROR("Failed to create a transaction for the db"); - } + throw0(DB_ERROR("Failed to create a transaction for the db")); MDB_val_copy key(h); MDB_val result; auto get_result = mdb_get(txn, m_txs, &key, &result); if (get_result == MDB_NOTFOUND) - { - LOG_PRINT_L1("tx with hash " << epee::string_tools::pod_to_hex(h) << "not found in db"); - throw TX_DNE("Attempting to get tx, but tx not in db"); - } + throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(h)).append("not found in db").c_str())); else if (get_result) - { - LOG_PRINT_L0("DB error attempting to fetch tx from hash"); - throw DB_ERROR("DB error attempting to fetch tx from hash"); - } + throw0(DB_ERROR("DB error attempting to fetch tx from hash")); blobdata bd; bd.assign(reinterpret_cast(result.mv_data), result.mv_size); transaction tx; if (!parse_and_validate_tx_from_blob(bd, tx)) - { - LOG_PRINT_L0("Failed to parse tx from blob retrieved from the db"); - throw DB_ERROR("Failed to parse tx from blob retrieved from the db"); - } + throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); return tx; } @@ -1279,17 +1024,11 @@ uint64_t BlockchainLMDB::get_tx_count() const txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - { - LOG_PRINT_L0("Failed to create a transaction for the db"); - throw DB_ERROR("Failed to create a transaction for the db"); - } + throw0(DB_ERROR("Failed to create a transaction for the db")); MDB_stat db_stats; if (mdb_stat(txn, m_txs, &db_stats)) - { - LOG_PRINT_L0("Failed to query m_txs"); - throw DB_ERROR("Failed to query m_txs"); - } + throw0(DB_ERROR("Failed to query m_txs")); txn.commit(); @@ -1317,24 +1056,17 @@ uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - { - LOG_PRINT_L0("Failed to create a transaction for the db"); - throw DB_ERROR("Failed to create a transaction for the db"); - } + throw0(DB_ERROR("Failed to create a transaction for the db")); MDB_val_copy key(h); MDB_val result; auto get_result = mdb_get(txn, m_tx_heights, &key, &result); if (get_result == MDB_NOTFOUND) { - LOG_PRINT_L1("tx height with hash " << epee::string_tools::pod_to_hex(h) << "not found in db"); - throw TX_DNE("Attempting to get height for tx, but tx not in db"); + throw1(TX_DNE(std::string("tx height with hash ").append(epee::string_tools::pod_to_hex(h)).append("not found in db").c_str())); } else if (get_result) - { - LOG_PRINT_L0("DB error attempting to fetch tx height from hash"); - throw DB_ERROR("DB error attempting to fetch tx height from hash"); - } + throw0(DB_ERROR("DB error attempting to fetch tx height from hash")); return *(const uint64_t*)result.mv_data; } @@ -1347,10 +1079,7 @@ uint64_t BlockchainLMDB::get_random_output(const uint64_t& amount) const uint64_t num_outputs = get_num_outputs(amount); if (num_outputs == 0) - { - LOG_PRINT_L1("Attempting to get a random output for an amount, but none exist"); - throw OUTPUT_DNE("Attempting to get a random output for an amount, but none exist"); - } + throw1(OUTPUT_DNE("Attempting to get a random output for an amount, but none exist")); return crypto::rand() % num_outputs; } @@ -1362,10 +1091,7 @@ uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) const txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - { - LOG_PRINT_L0("Failed to create a transaction for the db"); - throw DB_ERROR("Failed to create a transaction for the db"); - } + throw0(DB_ERROR("Failed to create a transaction for the db")); lmdb_cur cur(txn, m_output_amounts); @@ -1377,10 +1103,7 @@ uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) const return 0; } else if (result) - { - LOG_PRINT_L0("DB error attempting to get number of outputs of an amount"); - throw DB_ERROR("DB error attempting to get number of outputs of an amount"); - } + throw0(DB_ERROR("DB error attempting to get number of outputs of an amount")); size_t num_elems = 0; mdb_cursor_count(cur, &num_elems); @@ -1397,24 +1120,15 @@ crypto::public_key BlockchainLMDB::get_output_key(const uint64_t& amount, const txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - { - LOG_PRINT_L0("Failed to create a transaction for the db"); - throw DB_ERROR("Failed to create a transaction for the db"); - } + throw0(DB_ERROR("Failed to create a transaction for the db")); MDB_val_copy k(get_output_global_index(amount, index)); MDB_val v; auto get_result = mdb_get(txn, m_output_keys, &k, &v); if (get_result == MDB_NOTFOUND) - { - LOG_PRINT_L0("Attempting to get output pubkey by global index, but key does not exist"); - throw DB_ERROR("Attempting to get output pubkey by global index, but key does not exist"); - } + throw0(DB_ERROR("Attempting to get output pubkey by global index, but key does not exist")); else if (get_result) - { - LOG_PRINT_L0("Error attempting to retrieve an output pubkey from the db"); - throw DB_ERROR("Error attempting to retrieve an output pubkey from the db"); - } + throw0(DB_ERROR("Error attempting to retrieve an output pubkey from the db")); return *(crypto::public_key*)v.mv_data; } @@ -1426,10 +1140,7 @@ tx_out BlockchainLMDB::get_output(const crypto::hash& h, const uint64_t& index) txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - { - LOG_PRINT_L0("Failed to create a transaction for the db"); - throw DB_ERROR("Failed to create a transaction for the db"); - } + throw0(DB_ERROR("Failed to create a transaction for the db")); lmdb_cur cur(txn, m_tx_outputs); @@ -1437,23 +1148,14 @@ tx_out BlockchainLMDB::get_output(const crypto::hash& h, const uint64_t& index) MDB_val v; auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); if (result == MDB_NOTFOUND) - { - LOG_PRINT_L1("Attempting to get an output by tx hash and tx index, but output not found"); - throw OUTPUT_DNE("Attempting to get an output by tx hash and tx index, but output not found"); - } + throw1(OUTPUT_DNE("Attempting to get an output by tx hash and tx index, but output not found")); else if (result) - { - LOG_PRINT_L0("DB error attempting to get an output"); - throw DB_ERROR("DB error attempting to get an output"); - } + throw0(DB_ERROR("DB error attempting to get an output")); size_t num_elems = 0; mdb_cursor_count(cur, &num_elems); if (num_elems <= index) - { - LOG_PRINT_L1("Attempting to get an output by tx hash and tx index, but output not found"); - throw OUTPUT_DNE("Attempting to get an output by tx hash and tx index, but output not found"); - } + throw1(OUTPUT_DNE("Attempting to get an output by tx hash and tx index, but output not found")); mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); @@ -1486,24 +1188,17 @@ tx_out BlockchainLMDB::get_output(const uint64_t& index) const txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - { - LOG_PRINT_L0("Failed to create a transaction for the db"); - throw DB_ERROR("Failed to create a transaction for the db"); - } + throw0(DB_ERROR("Failed to create a transaction for the db")); MDB_val_copy k(index); MDB_val v; auto get_result = mdb_get(txn, m_outputs, &k, &v); if (get_result == MDB_NOTFOUND) { - LOG_PRINT_L0("Attempting to get output by global index, but output does not exist"); - throw OUTPUT_DNE(); + throw OUTPUT_DNE("Attempting to get output by global index, but output does not exist"); } else if (get_result) - { - LOG_PRINT_L0("Error attempting to retrieve an output from the db"); - throw DB_ERROR("Error attempting to retrieve an output from the db"); - } + throw0(DB_ERROR("Error attempting to retrieve an output from the db")); blobdata b = *(blobdata*)v.mv_data; @@ -1517,10 +1212,7 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - { - LOG_PRINT_L0("Failed to create a transaction for the db"); - throw DB_ERROR("Failed to create a transaction for the db"); - } + throw0(DB_ERROR("Failed to create a transaction for the db")); lmdb_cur cur(txn, m_output_amounts); @@ -1529,23 +1221,14 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); if (result == MDB_NOTFOUND) - { - LOG_PRINT_L1("Attempting to get an output index by amount and amount index, but amount not found"); - throw OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found"); - } + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); else if (result) - { - LOG_PRINT_L0("DB error attempting to get an output"); - throw DB_ERROR("DB error attempting to get an output"); - } + throw0(DB_ERROR("DB error attempting to get an output")); size_t num_elems = 0; mdb_cursor_count(cur, &num_elems); if (num_elems <= index) - { - LOG_PRINT_L1("Attempting to get an output index by amount and amount index, but output not found"); - throw OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found"); - } + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found")); mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); @@ -1566,29 +1249,17 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con k = MDB_val_copy(glob_index); auto get_result = mdb_get(txn, m_output_txs, &k, &v); if (get_result == MDB_NOTFOUND) - { - LOG_PRINT_L1("output with given index not in db"); - throw OUTPUT_DNE("output with given index not in db"); - } + throw1(OUTPUT_DNE("output with given index not in db")); else if (get_result) - { - LOG_PRINT_L0("DB error attempting to fetch output tx hash"); - throw DB_ERROR("DB error attempting to fetch output tx hash"); - } + throw0(DB_ERROR("DB error attempting to fetch output tx hash")); crypto::hash tx_hash = *(crypto::hash*)v.mv_data; get_result = mdb_get(txn, m_output_indices, &k, &v); if (get_result == MDB_NOTFOUND) - { - LOG_PRINT_L1("output with given index not in db"); - throw OUTPUT_DNE("output with given index not in db"); - } + throw1(OUTPUT_DNE("output with given index not in db")); else if (get_result) - { - LOG_PRINT_L0("DB error attempting to fetch output tx index"); - throw DB_ERROR("DB error attempting to fetch output tx index"); - } + throw0(DB_ERROR("DB error attempting to fetch output tx index")); txn.commit(); @@ -1603,10 +1274,7 @@ std::vector BlockchainLMDB::get_tx_output_indices(const crypto::hash& txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - { - LOG_PRINT_L0("Failed to create a transaction for the db"); - throw DB_ERROR("Failed to create a transaction for the db"); - } + throw0(DB_ERROR("Failed to create a transaction for the db")); lmdb_cur cur(txn, m_tx_outputs); @@ -1614,15 +1282,9 @@ std::vector BlockchainLMDB::get_tx_output_indices(const crypto::hash& MDB_val v; auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); if (result == MDB_NOTFOUND) - { - LOG_PRINT_L1("Attempting to get an output by tx hash and tx index, but output not found"); - throw OUTPUT_DNE("Attempting to get an output by tx hash and tx index, but output not found"); - } + throw1(OUTPUT_DNE("Attempting to get an output by tx hash and tx index, but output not found")); else if (result) - { - LOG_PRINT_L0("DB error attempting to get an output"); - throw DB_ERROR("DB error attempting to get an output"); - } + throw0(DB_ERROR("DB error attempting to get an output")); size_t num_elems = 0; mdb_cursor_count(cur, &num_elems); @@ -1649,10 +1311,7 @@ bool BlockchainLMDB::has_key_image(const crypto::key_image& img) const txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - { - LOG_PRINT_L0("Failed to create a transaction for the db"); - throw DB_ERROR("Failed to create a transaction for the db"); - } + throw0(DB_ERROR("Failed to create a transaction for the db")); MDB_val_copy val_key(img); MDB_val unused; @@ -1677,10 +1336,7 @@ uint64_t BlockchainLMDB::add_block( const block& blk check_open(); txn_safe txn; if (mdb_txn_begin(m_env, NULL, 0, txn)) - { - LOG_PRINT_L0("Failed to create a transaction for the db"); - throw DB_ERROR("Failed to create a transaction for the db"); - } + throw0(DB_ERROR("Failed to create a transaction for the db")); m_write_txn = &txn; uint64_t num_outputs = m_num_outputs; @@ -1705,10 +1361,7 @@ void BlockchainLMDB::pop_block(block& blk, std::vector& txs) { txn_safe txn; if (mdb_txn_begin(m_env, NULL, 0, txn)) - { - LOG_PRINT_L0("Failed to create a transaction for the db"); - throw DB_ERROR("Failed to create a transaction for the db"); - } + throw0(DB_ERROR("Failed to create a transaction for the db")); m_write_txn = &txn; uint64_t num_outputs = m_num_outputs; -- cgit v1.2.3 From 4c2a45288a6bc89ce2e48c2854943904dcd01bab Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Fri, 12 Dec 2014 23:20:41 +0000 Subject: db_lmdb: catch attempt to remove block from an empty blockchain It would probably have thrown when not finding a block at height 2^64-1, but better make things clear. --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index f53e5bac2..a4dbb1d88 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -207,6 +207,9 @@ void BlockchainLMDB::remove_block() LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); + if (m_height == 0) + throw0(BLOCK_DNE ("Attempting to remove block from an empty blockchain")); + MDB_val_copy k(m_height - 1); MDB_val h; if (mdb_get(*m_write_txn, m_block_hashes, &k, &h)) -- cgit v1.2.3 From 59d2b0ed1cb9f25c1cf25d75cda864a44a094bde Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Fri, 12 Dec 2014 23:24:10 +0000 Subject: db_lmdb: do not give the group database write permissions --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index a4dbb1d88..9d0728f6c 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -530,7 +530,7 @@ void BlockchainLMDB::open(const std::string& filename) size_t mapsize = 1LL << 34; if (auto result = mdb_env_set_mapsize(m_env, mapsize)) throw0(DB_ERROR(std::string("Failed to set max memory map size: ").append(mdb_strerror(result)).c_str())); - if (auto result = mdb_env_open(m_env, filename.c_str(), 0, 0664)) + if (auto result = mdb_env_open(m_env, filename.c_str(), 0, 0644)) throw0(DB_ERROR(std::string("Failed to open lmdb environment: ").append(mdb_strerror(result)).c_str())); // get a read/write MDB_txn -- cgit v1.2.3 From 1362846dd7c7e50b8804e24ece1a750472cc4485 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Fri, 19 Dec 2014 18:18:21 +0000 Subject: blockchain_converter: add --testnet for converting testnet blockchain --- src/blockchain_converter/blockchain_converter.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/blockchain_converter/blockchain_converter.cpp b/src/blockchain_converter/blockchain_converter.cpp index f2dd10b18..57822700f 100644 --- a/src/blockchain_converter/blockchain_converter.cpp +++ b/src/blockchain_converter/blockchain_converter.cpp @@ -59,7 +59,12 @@ struct fake_core int main(int argc, char* argv[]) { - boost::filesystem::path default_data_path {tools::get_default_data_dir()}; + std::string dir = tools::get_default_data_dir(); + boost::filesystem::path default_data_path {dir}; + if (argc >= 2 && !strcmp(argv[1], "--testnet")) { + default_data_path /= "testnet"; + } + fake_core c(default_data_path); BlockchainDB *blockchain; -- cgit v1.2.3 From c3fa07b44b95996ceb9a15964211593ceed517c2 Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Sun, 14 Dec 2014 14:30:38 -0500 Subject: update comments to reflect changed code --- src/cryptonote_core/blockchain_db.h | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/blockchain_db.h b/src/cryptonote_core/blockchain_db.h index b9ea79f01..aa82672f2 100644 --- a/src/cryptonote_core/blockchain_db.h +++ b/src/cryptonote_core/blockchain_db.h @@ -395,10 +395,7 @@ public: // return the block at the top of the blockchain virtual block get_top_block() const = 0; - // return the index of the top block on the chain - // NOTE: for convenience using heights as indices, this is not the total - // size of the blockchain, but rather the index of the top block. As the - // chain is 0-indexed, the total size will be height() + 1. + // return the height of the chain virtual uint64_t height() const = 0; // pops the top block off the blockchain. -- cgit v1.2.3 From 57b80c541e01df8ad83aa6f8611c2ebc5669c716 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sat, 20 Dec 2014 18:35:54 +0000 Subject: db_lmdb: remove redundant checks --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index e96b9f90f..ee5c0fa9b 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -1162,12 +1162,9 @@ tx_out BlockchainLMDB::get_output(const crypto::hash& h, const uint64_t& index) mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); - if (index != 0) + for (uint64_t i = 0; i < index; ++i) { - for (uint64_t i = 0; i < index; ++i) - { - mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); - } + mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); } mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); @@ -1264,12 +1261,9 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); - if (index != 0) + for (uint64_t i = 0; i < index; ++i) { - for (uint64_t i = 0; i < index; ++i) - { - mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); - } + mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); } mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); -- cgit v1.2.3 From c50cd956743838dc256a1fe81722c80898cd2ec9 Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Sun, 14 Dec 2014 15:20:41 -0500 Subject: Fixes a bug with getting output metadata from BlockchainDB Thanks to moneromooo-monero for spotting the bug. --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 46 +++++++++++++++-------- src/cryptonote_core/BlockchainDB_impl/db_lmdb.h | 2 + src/cryptonote_core/blockchain.cpp | 2 +- src/cryptonote_core/blockchain_db.h | 4 ++ 4 files changed, 37 insertions(+), 17 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index 9d0728f6c..e96b9f90f 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -1208,6 +1208,35 @@ tx_out BlockchainLMDB::get_output(const uint64_t& index) const return output_from_blob(b); } +tx_out_index BlockchainLMDB::get_output_tx_and_index_from_global(const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + MDB_val_copy k(index); + MDB_val v; + + auto get_result = mdb_get(txn, m_output_txs, &k, &v); + if (get_result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("output with given index not in db")); + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch output tx hash")); + + crypto::hash tx_hash = *(crypto::hash*)v.mv_data; + + get_result = mdb_get(txn, m_output_indices, &k, &v); + if (get_result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("output with given index not in db")); + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch output tx index")); + + return tx_out_index(tx_hash, *(const uint64_t *)v.mv_data); +} + tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -1249,24 +1278,9 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con cur.close(); - k = MDB_val_copy(glob_index); - auto get_result = mdb_get(txn, m_output_txs, &k, &v); - if (get_result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("output with given index not in db")); - else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch output tx hash")); - - crypto::hash tx_hash = *(crypto::hash*)v.mv_data; - - get_result = mdb_get(txn, m_output_indices, &k, &v); - if (get_result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("output with given index not in db")); - else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch output tx index")); - txn.commit(); - return tx_out_index(tx_hash, *(const uint64_t *)v.mv_data); + return get_output_tx_and_index_from_global(glob_index); } std::vector BlockchainLMDB::get_tx_output_indices(const crypto::hash& h) const diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h index 3bac5d740..6696ef492 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h @@ -161,6 +161,8 @@ public: */ tx_out get_output(const uint64_t& index) const; + virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const; + virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const; virtual std::vector get_tx_output_indices(const crypto::hash& h) const; diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 87be9d566..c4767765d 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -170,7 +170,7 @@ bool Blockchain::scan_outputkeys_for_indexes(const txin_to_key& tx_in_to_key, vi try { // get tx hash and output index for output - auto output_index = m_db->get_output_tx_and_index(tx_in_to_key.amount, i); + auto output_index = m_db->get_output_tx_and_index_from_global(i); // get tx that output is from auto tx = m_db->get_tx(output_index.first); diff --git a/src/cryptonote_core/blockchain_db.h b/src/cryptonote_core/blockchain_db.h index aa82672f2..a3c7bc26b 100644 --- a/src/cryptonote_core/blockchain_db.h +++ b/src/cryptonote_core/blockchain_db.h @@ -107,6 +107,7 @@ * uint64_t get_num_outputs(amount) * pub_key get_output_key(amount, index) * tx_out get_output(tx_hash, index) + * hash,index get_output_tx_and_index_from_global(index) * hash,index get_output_tx_and_index(amount, index) * vec get_tx_output_indices(tx_hash) * @@ -441,6 +442,9 @@ public: // returns the output indexed by in the transaction with hash virtual tx_out get_output(const crypto::hash& h, const uint64_t& index) const = 0; + // returns the tx hash associated with an output, referenced by global output index + virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const = 0; + // returns the transaction-local reference for the output with at // return type is pair of tx hash and index virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const = 0; -- cgit v1.2.3 From ad8200a5731e462cf191e49570359283d760b372 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sat, 20 Dec 2014 18:36:22 +0000 Subject: db_lmdb: fix global index calculation off by 1 This finally fixes raw tx being accepted by the daemon. --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index ee5c0fa9b..5b71dcf08 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -1304,9 +1304,9 @@ std::vector BlockchainLMDB::get_tx_output_indices(const crypto::hash& for (uint64_t i = 0; i < num_elems; ++i) { - mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); index_vec.push_back(*(const uint64_t *)v.mv_data); + mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); } cur.close(); -- cgit v1.2.3 From c5c100c69b83242dae2ebfd88eaff2ff743e9b30 Mon Sep 17 00:00:00 2001 From: warptangent Date: Sat, 27 Dec 2014 06:40:17 -0800 Subject: Obtain tx hash and tx output index from amount and output offset Fixes problem of obtaining incorrect outputs used for tx input. Reverts to earlier intended behavior that was fixed in previous commit's split of get_output_tx_and_index into two functions. --- src/cryptonote_core/blockchain.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index c4767765d..87be9d566 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -170,7 +170,7 @@ bool Blockchain::scan_outputkeys_for_indexes(const txin_to_key& tx_in_to_key, vi try { // get tx hash and output index for output - auto output_index = m_db->get_output_tx_and_index_from_global(i); + auto output_index = m_db->get_output_tx_and_index(tx_in_to_key.amount, i); // get tx that output is from auto tx = m_db->get_tx(output_index.first); -- cgit v1.2.3 From 14555eefd5d2163fbd0cd892cf873b10d20b64b6 Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Fri, 9 Jan 2015 05:56:51 -0500 Subject: Fixes segfault in Blockchain::handle_alternative_block This commit should fix the segfault in Blockchain::handle_alternative_block, and also updates a few comments that were either incorrect or incomplete. --- src/cryptonote_core/blockchain.cpp | 14 ++++++++------ src/cryptonote_core/checkpoints.cpp | 4 ++++ 2 files changed, 12 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 87be9d566..bc12fa034 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -1072,13 +1072,14 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id uint64_t block_height = get_block_height(b); if(0 == block_height) { - LOG_ERROR("Block with id: " << epee::string_tools::pod_to_hex(id) << " (as alternative) have wrong miner transaction"); + LOG_ERROR("Block with id: " << epee::string_tools::pod_to_hex(id) << " (as alternative), but miner tx says height is 0."); bvc.m_verifivation_failed = true; return false; } - // TODO: this basically says if the blockchain is smaller than the first - // checkpoint then alternate blocks are allowed...this seems backwards, but - // I'm not sure. Needs further investigating. + // this basically says if the blockchain is smaller than the first + // checkpoint then alternate blocks are allowed. Alternatively, if the + // last checkpoint *before* the end of the current chain is also before + // the block to be added, then this is fine. if (!m_checkpoints.is_alternative_block_allowed(get_current_blockchain_height(), block_height)) { LOG_PRINT_RED_L0("Block with id: " << id @@ -1187,7 +1188,8 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id // FIXME: // this brings up an interesting point: consider allowing to get block // difficulty both by height OR by hash, not just height. - bei.cumulative_difficulty = alt_chain.size() ? it_prev->second.cumulative_difficulty : m_db->get_block_cumulative_difficulty(m_db->get_block_height(b.prev_id)); + auto main_chain_cumulative_difficulty = m_db->get_block_cumulative_difficulty(m_db->get_block_height(b.prev_id)); + bei.cumulative_difficulty = alt_chain.size() ? it_prev->second.cumulative_difficulty : main_chain_cumulative_difficulty; bei.cumulative_difficulty += current_diff; // add block to alternate blocks storage, @@ -1210,7 +1212,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id return r; } - else if(m_blocks.back().cumulative_difficulty < bei.cumulative_difficulty) //check if difficulty bigger then in main chain + else if(main_chain_cumulative_difficulty < bei.cumulative_difficulty) //check if difficulty bigger then in main chain { //do reorganize! LOG_PRINT_GREEN("###### REORGANIZE on height: " diff --git a/src/cryptonote_core/checkpoints.cpp b/src/cryptonote_core/checkpoints.cpp index 58edda7c9..e4223afb5 100644 --- a/src/cryptonote_core/checkpoints.cpp +++ b/src/cryptonote_core/checkpoints.cpp @@ -84,6 +84,10 @@ namespace cryptonote return check_block(height, h, ignored); } //--------------------------------------------------------------------------- + // this basically says if the blockchain is smaller than the first + // checkpoint then alternate blocks are allowed. Alternatively, if the + // last checkpoint *before* the end of the current chain is also before + // the block to be added, then this is fine. bool checkpoints::is_alternative_block_allowed(uint64_t blockchain_height, uint64_t block_height) const { if (0 == block_height) -- cgit v1.2.3 From 429a7405622ef84c17ecc851d0a6ac0086218837 Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Fri, 9 Jan 2015 07:29:05 -0500 Subject: throw inline functions need to keep exception type As it is useful for functions calling BlockchainDB functions to know whether an exception is expected (attempting to get a block that doesn't exist and counting it missing if not, to save time checking if it does, for example), the inline functions throw{0,1} need to keep the exception type information. Slight comment update due to copy/paste failure. --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index 5b71dcf08..0184dbc58 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -39,13 +39,15 @@ using epee::string_tools::pod_to_hex; namespace { -inline void throw0(const std::exception &e) +template +inline void throw0(const T &e) { LOG_PRINT_L0(e.what()); throw e; } -inline void throw1(const std::exception &e) +template +inline void throw1(const T &e) { LOG_PRINT_L1(e.what()); throw e; @@ -814,7 +816,7 @@ difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& auto get_result = mdb_get(txn, m_block_diffs, &key, &result); if (get_result == MDB_NOTFOUND) { - throw0(DB_ERROR(std::string("Attempt to get cumulative difficulty from height ").append(boost::lexical_cast(height)).append(" failed -- block size not in db").c_str())); + throw0(DB_ERROR(std::string("Attempt to get cumulative difficulty from height ").append(boost::lexical_cast(height)).append(" failed -- difficulty not in db").c_str())); } else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a cumulative difficulty from the db")); -- cgit v1.2.3 From d045dfa7ce0bf131681193c97560da26f9f37900 Mon Sep 17 00:00:00 2001 From: warptangent Date: Fri, 9 Jan 2015 12:57:33 -0800 Subject: Fix transfers (without mixins) Fix Blockchain::get_tx_outputs_gindexs() to return amount output indices. Implement BlockchainLMDB::get_tx_amount_output_indices() and call it from the function instead of BlockchainLMDB::get_tx_output_indices() Previously, Blockchain::get_tx_outputs_gindexs() was instead returning global output indices, which are internal to LMDB databases. Allows bitmonerod RPC /get_o_indexes.bin to return the amount output indices as expected. Allows simplewallet refresh to set correct amount output indices for incoming transfers. simplewallet can now construct and send valid transactions (currently only without mixins). This is a fix that doesn't require altering the structure of the current LMDB databases. TODO: This can be done more efficiently by adding another LMDB database (key-value table). It's not used during regular transaction validation by bitmonerod. I think it's currently used only or mainly by simplewallet for just its own incoming transactions. So the current behavior is not a primary bottleneck. Currently, it's using the "output_amounts" database, walking through a given amount's list of values, comparing each one to a given global output index. The iteration number of the match is the desired result: the amount output index. This is done for each global output index of the transaction. A tx's amount output indices can be stored in various other ways allowing for faster lookup. Since a tx is only written once, there are no special future write requirements for its list of indices. --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 79 +++++++++++++++++++++++ src/cryptonote_core/BlockchainDB_impl/db_lmdb.h | 1 + src/cryptonote_core/blockchain.cpp | 3 +- src/cryptonote_core/blockchain_db.h | 3 + 4 files changed, 85 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index 0184dbc58..4c5d310f1 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -1317,6 +1317,85 @@ std::vector BlockchainLMDB::get_tx_output_indices(const crypto::hash& return index_vec; } +std::vector BlockchainLMDB::get_tx_amount_output_indices(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + std::vector index_vec; + std::vector index_vec2; + + // get the transaction's global output indices first + index_vec = get_tx_output_indices(h); + // these are next used to obtain the amount output indices + + transaction tx = get_tx(h); + + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + uint64_t i = 0; + uint64_t global_index; + BOOST_FOREACH(const auto& vout, tx.vout) + { + uint64_t amount = vout.amount; + + global_index = index_vec[i]; + + lmdb_cur cur(txn, m_output_amounts); + + MDB_val_copy k(amount); + MDB_val v; + + auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + size_t num_elems = 0; + mdb_cursor_count(cur, &num_elems); + + mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); + + uint64_t amount_output_index = 0; + uint64_t output_index = 0; + bool found_index = false; + for (uint64_t j = 0; j < num_elems; ++j) + { + mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); + output_index = *(const uint64_t *)v.mv_data; + if (output_index == global_index) + { + amount_output_index = j; + found_index = true; + break; + } + mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); + } + if (found_index) + { + index_vec2.push_back(amount_output_index); + } + else + { + // not found + cur.close(); + txn.commit(); + throw1(OUTPUT_DNE("specified output not found in db")); + } + + cur.close(); + ++i; + } + + txn.commit(); + + return index_vec2; +} + + + bool BlockchainLMDB::has_key_image(const crypto::key_image& img) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h index 6696ef492..d95d19019 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h @@ -166,6 +166,7 @@ public: virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const; virtual std::vector get_tx_output_indices(const crypto::hash& h) const; + virtual std::vector get_tx_amount_output_indices(const crypto::hash& h) const; virtual bool has_key_image(const crypto::key_image& img) const; diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index bc12fa034..a032a628b 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -1789,7 +1789,8 @@ bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vectorget_tx_output_indices(tx_id); + // get amount output indexes, currently referred to in parts as "output global indices", but they are actually specific to amounts + indexs = m_db->get_tx_amount_output_indices(tx_id); return true; } //------------------------------------------------------------------ diff --git a/src/cryptonote_core/blockchain_db.h b/src/cryptonote_core/blockchain_db.h index a3c7bc26b..b498320ae 100644 --- a/src/cryptonote_core/blockchain_db.h +++ b/src/cryptonote_core/blockchain_db.h @@ -452,6 +452,9 @@ public: // return a vector of indices corresponding to the global output index for // each output in the transaction with hash virtual std::vector get_tx_output_indices(const crypto::hash& h) const = 0; + // return a vector of indices corresponding to the amount output index for + // each output in the transaction with hash + virtual std::vector get_tx_amount_output_indices(const crypto::hash& h) const = 0; // returns true if key image is present in spent key images storage virtual bool has_key_image(const crypto::key_image& img) const = 0; -- cgit v1.2.3 From 4eba21fd48c245fc137630341738c1edfd35a230 Mon Sep 17 00:00:00 2001 From: warptangent Date: Fri, 9 Jan 2015 13:01:22 -0800 Subject: Fix transfers to support mixins Implement BlockchainLMDB::get_output_global_index() - returns global output index for a given amount and amount output index. Add information to debug statement for failed ring signature check within Blockchain::check_tx_inputs() Fixes bitmonerod RPC call "/getrandom_outs.bin" to return correct output keys, used in creating a transaction with mixins. TODO: get_output_global_index() could be refactored with part of get_output_tx_and_index() as the latter uses the former's functionality. Keep track of LMDB read transaction. --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 43 +++++++++++++++++++++-- src/cryptonote_core/blockchain.cpp | 2 +- 2 files changed, 42 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index 4c5d310f1..835b9248f 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -477,7 +477,44 @@ tx_out BlockchainLMDB::output_from_blob(const blobdata& blob) const uint64_t BlockchainLMDB::get_output_global_index(const uint64_t& amount, const uint64_t& index) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); - return 0; + check_open(); + + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + lmdb_cur cur(txn, m_output_amounts); + + MDB_val_copy k(amount); + MDB_val v; + + auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + size_t num_elems = 0; + mdb_cursor_count(cur, &num_elems); + if (num_elems <= index) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found")); + + mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); + + for (uint64_t i = 0; i < index; ++i) + { + mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); + } + + mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); + + uint64_t glob_index = *(const uint64_t*)v.mv_data; + + cur.close(); + + txn.commit(); + + return glob_index; } void BlockchainLMDB::check_open() const @@ -1123,11 +1160,13 @@ crypto::public_key BlockchainLMDB::get_output_key(const uint64_t& amount, const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); + uint64_t glob_index = get_output_global_index(amount, index); + txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) throw0(DB_ERROR("Failed to create a transaction for the db")); - MDB_val_copy k(get_output_global_index(amount, index)); + MDB_val_copy k(glob_index); MDB_val v; auto get_result = mdb_get(txn, m_output_keys, &k, &v); if (get_result == MDB_NOTFOUND) diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index a032a628b..48e6543ed 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -1849,7 +1849,7 @@ bool Blockchain::check_tx_inputs(const transaction& tx, uint64_t* pmax_used_bloc // signature spending it. if(!check_tx_input(in_to_key, tx_prefix_hash, tx.signatures[sig_index], pmax_used_block_height)) { - LOG_PRINT_L0("Failed to check ring signature for tx " << get_transaction_hash(tx)); + LOG_PRINT_L0("Failed to check ring signature for tx " << get_transaction_hash(tx) << " vin key with k_image: " << in_to_key.k_image << " sig_index: " << sig_index << " *pmax_used_block_height: " << *pmax_used_block_height); return false; } -- cgit v1.2.3 From 1701c267502c32af64212d8e496cbdb6f3bc00df Mon Sep 17 00:00:00 2001 From: warptangent Date: Sun, 11 Jan 2015 16:59:59 -0800 Subject: Use block index when obtaining block's difficulty for log statement Use last block id, not number of blocks (off-by-one error). Fixes error at start of blockchain reorganization: "Attempt to get cumulative difficulty from height failed -- difficulty not in db" --- src/cryptonote_core/blockchain.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 48e6543ed..2a8eb6721 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -1216,8 +1216,8 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id { //do reorganize! LOG_PRINT_GREEN("###### REORGANIZE on height: " - << alt_chain.front()->second.height << " of " << m_db->height() - << " with cum_difficulty " << m_db->get_block_cumulative_difficulty(m_db->height()) + << alt_chain.front()->second.height << " of " << m_db->height() - 1 + << " with cum_difficulty " << m_db->get_block_cumulative_difficulty(m_db->height() - 1) << std::endl << " alternative blockchain size: " << alt_chain.size() << " with cum_difficulty " << bei.cumulative_difficulty, LOG_LEVEL_0 ); -- cgit v1.2.3 From 4d0a94b20cd6e3b03657dc4ca49bc248a1e7f230 Mon Sep 17 00:00:00 2001 From: warptangent Date: Sun, 11 Jan 2015 18:04:04 -0800 Subject: Complete implementation of transaction removal Complete method BlockchainLMDB::remove_output() - use output index as the key for: m_output_indices, m_output_txs, m_output_keys - call new method BlockchainLMDB::remove_amount_output_index() Add method to remove amount output index. - BlockchainLMDB::remove_amount_output_index() - for m_output_amounts This also fixes the segfault when blockchain reorganization is attempted. --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 105 ++++++++++++++++++---- src/cryptonote_core/BlockchainDB_impl/db_lmdb.h | 7 +- src/cryptonote_core/blockchain_db.cpp | 6 +- src/cryptonote_core/blockchain_db.h | 2 +- 4 files changed, 97 insertions(+), 23 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index 835b9248f..4fa16b23e 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -262,7 +262,7 @@ void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const tr throw0(DB_ERROR("Failed to add tx unlock time to db transaction")); } -void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash) +void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -279,7 +279,7 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash) if (mdb_del(*m_write_txn, m_tx_heights, &val_h, NULL)) throw1(DB_ERROR("Failed to add removal of tx block height to db transaction")); - remove_tx_outputs(tx_hash); + remove_tx_outputs(tx_hash, tx); if (mdb_del(*m_write_txn, m_tx_outputs, &val_h, NULL)) throw1(DB_ERROR("Failed to add removal of tx outputs to db transaction")); @@ -330,8 +330,9 @@ void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_ou m_num_outputs++; } -void BlockchainLMDB::remove_tx_outputs(const crypto::hash& tx_hash) +void BlockchainLMDB::remove_tx_outputs(const crypto::hash& tx_hash, const transaction& tx) { + LOG_PRINT_L3("BlockchainLMDB::" << __func__); lmdb_cur cur(*m_write_txn, m_tx_outputs); @@ -356,7 +357,8 @@ void BlockchainLMDB::remove_tx_outputs(const crypto::hash& tx_hash) for (uint64_t i = 0; i < num_elems; ++i) { - remove_output(*(const uint64_t*)v.mv_data); + const tx_out tx_output = tx.vout[i]; + remove_output(*(const uint64_t*)v.mv_data, tx_output.amount); if (i < num_elems - 1) { mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); @@ -367,17 +369,19 @@ void BlockchainLMDB::remove_tx_outputs(const crypto::hash& tx_hash) cur.close(); } +// TODO: probably remove this function void BlockchainLMDB::remove_output(const tx_out& tx_output) { + LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " (unused version - does nothing)"); return; } -void BlockchainLMDB::remove_output(const uint64_t& out_index) +void BlockchainLMDB::remove_output(const uint64_t& out_index, const uint64_t amount) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - MDB_val k; + MDB_val_copy k(out_index); MDB_val v; /****** Uncomment if ever outputs actually need to be stored in this manner @@ -400,29 +404,94 @@ void BlockchainLMDB::remove_output(const uint64_t& out_index) throw1(DB_ERROR("Error adding removal of output to db transaction")); *********************************************************************/ - auto result = mdb_del(*m_write_txn, m_output_indices, &v, NULL); - if (result != 0 && result != MDB_NOTFOUND) - throw1(DB_ERROR("Error adding removal of output tx index to db transaction")); - - result = mdb_del(*m_write_txn, m_output_txs, &v, NULL); - if (result != 0 && result != MDB_NOTFOUND) - throw1(DB_ERROR("Error adding removal of output tx hash to db transaction")); + auto result = mdb_del(*m_write_txn, m_output_indices, &k, NULL); + if (result == MDB_NOTFOUND) + { + LOG_PRINT_L0("Unexpected: global output index not found in m_output_indices"); + } + else if (result) + { + throw1(DB_ERROR("Error adding removal of output tx index to db transaction")); + } - result = mdb_del(*m_write_txn, m_output_amounts, &v, NULL); - if (result != 0 && result != MDB_NOTFOUND) - throw1(DB_ERROR("Error adding removal of output amount to db transaction")); + result = mdb_del(*m_write_txn, m_output_txs, &k, NULL); + // if (result != 0 && result != MDB_NOTFOUND) + // throw1(DB_ERROR("Error adding removal of output tx hash to db transaction")); + if (result == MDB_NOTFOUND) + { + LOG_PRINT_L0("Unexpected: global output index not found in m_output_txs"); + } + else if (result) + { + throw1(DB_ERROR("Error adding removal of output tx hash to db transaction")); + } - result = mdb_del(*m_write_txn, m_output_keys, &v, NULL); + result = mdb_del(*m_write_txn, m_output_keys, &k, NULL); if (result == MDB_NOTFOUND) { - LOG_PRINT_L2("Removing output, no public key found."); + LOG_PRINT_L0("Unexpected: global output index not found in m_output_keys"); } else if (result) throw1(DB_ERROR("Error adding removal of output pubkey to db transaction")); + remove_amount_output_index(amount, out_index); + m_num_outputs--; } +void BlockchainLMDB::remove_amount_output_index(const uint64_t amount, const uint64_t global_output_index) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + lmdb_cur cur(*m_write_txn, m_output_amounts); + + MDB_val_copy k(amount); + MDB_val v; + + auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + size_t num_elems = 0; + mdb_cursor_count(cur, &num_elems); + + mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); + + uint64_t amount_output_index = 0; + uint64_t goi = 0; + bool found_index = false; + for (uint64_t i = 0; i < num_elems; ++i) + { + mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); + goi = *(const uint64_t *)v.mv_data; + if (goi == global_output_index) + { + amount_output_index = i; + found_index = true; + break; + } + mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); + } + if (found_index) + { + // found the amount output index + // now delete it + result = mdb_cursor_del(cur, 0); + if (result) + throw0(DB_ERROR(std::string("Error deleting amount output index ").append(boost::lexical_cast(amount_output_index)).c_str())); + } + else + { + // not found + cur.close(); + throw1(OUTPUT_DNE("Failed to find amount output index")); + } + cur.close(); +} + void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h index d95d19019..2513aa48a 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h @@ -190,15 +190,16 @@ private: virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx); - virtual void remove_transaction_data(const crypto::hash& tx_hash); + virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx); virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index); virtual void remove_output(const tx_out& tx_output); - void remove_tx_outputs(const crypto::hash& tx_hash); + void remove_tx_outputs(const crypto::hash& tx_hash, const transaction& tx); - void remove_output(const uint64_t& out_index); + void remove_output(const uint64_t& out_index, const uint64_t amount); + void remove_amount_output_index(const uint64_t amount, const uint64_t global_output_index); virtual void add_spent_key(const crypto::key_image& k_image); diff --git a/src/cryptonote_core/blockchain_db.cpp b/src/cryptonote_core/blockchain_db.cpp index 439cf5ded..5e781af79 100644 --- a/src/cryptonote_core/blockchain_db.cpp +++ b/src/cryptonote_core/blockchain_db.cpp @@ -107,6 +107,9 @@ void BlockchainDB::remove_transaction(const crypto::hash& tx_hash) { transaction tx = get_tx(tx_hash); + // TODO: This loop calling remove_output() should be removed. It doesn't do + // anything (function is empty), and the outputs were already intended to be + // removed later as part of remove_transaction_data(). for (const tx_out& tx_output : tx.vout) { remove_output(tx_output); @@ -120,7 +123,8 @@ void BlockchainDB::remove_transaction(const crypto::hash& tx_hash) } } - remove_transaction_data(tx_hash); + // need tx as tx.vout has the tx outputs, and the output amounts are needed + remove_transaction_data(tx_hash, tx); } } // namespace cryptonote diff --git a/src/cryptonote_core/blockchain_db.h b/src/cryptonote_core/blockchain_db.h index b498320ae..db56c7c07 100644 --- a/src/cryptonote_core/blockchain_db.h +++ b/src/cryptonote_core/blockchain_db.h @@ -274,7 +274,7 @@ private: virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx) = 0; // tells the subclass to remove data about a transaction - virtual void remove_transaction_data(const crypto::hash& tx_hash) = 0; + virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) = 0; // tells the subclass to store an output virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index) = 0; -- cgit v1.2.3 From 909ea810671e8da74b0c0d92641eee8dd798e0b3 Mon Sep 17 00:00:00 2001 From: warptangent Date: Sun, 11 Jan 2015 18:19:01 -0800 Subject: Remove a have_block() check so alternate block can be processed Remove have_block() check from Blockchain::handle_block_to_main_chain(). Add logging to have_block(). This allows blockchain reorganization to proceed further. have_block() check here causes an error after a blockchain reorganize begins with error: "Attempting to add block to main chain, but it's already either there or in an alternate chain." While reorganizing to become the main chain, a block in the alternative chain would be refused due to have_block() rightfully finding it in the alternative chain. The reorganization would end in rollback, restoring to previous blockchain. Original implementation didn't call it here, and it doesn't appear necessary to be called from here in this implementation either. When needed, it appears it's called prior to handle_block_to_main_chain(). --- src/cryptonote_core/blockchain.cpp | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 2a8eb6721..7015383fa 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -1699,13 +1699,22 @@ bool Blockchain::have_block(const crypto::hash& id) const CRITICAL_REGION_LOCAL(m_blockchain_lock); if(m_db->block_exists(id)) + { + LOG_PRINT_L3("block exists in main chain"); return true; + } if(m_alternative_chains.count(id)) + { + LOG_PRINT_L3("block found in m_alternative_chains"); return true; + } if(m_invalid_blocks.count(id)) + { + LOG_PRINT_L3("block found in m_invalid_blocks"); return true; + } return false; } @@ -2010,14 +2019,25 @@ bool Blockchain::check_block_timestamp(const block& b) const bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& id, block_verification_context& bvc) { LOG_PRINT_L3("Blockchain::" << __func__); + + // NOTE: Omitting check below with have_block() It causes an error after a + // blockchain reorganize begins with error: "Attempting to add block to main + // chain, but it's already either there or in an alternate" + // + // A block in the alternative chain, desired to become the main chain, never + // makes it due to have_block finding it in he alternative chain. + // + // Original implementation didn't use it here, and it doesn't appear + // necessary to be called from here in this implementation either. + // if we already have the block, return false - if (have_block(id)) - { - LOG_PRINT_L0("Attempting to add block to main chain, but it's already either there or in an alternate chain. hash: " << id); - bvc.m_verifivation_failed = true; - return false; - } - + // if (have_block(id)) + // { + // LOG_PRINT_L0("Attempting to add block to main chain, but it's already either there or in an alternate chain. hash: " << id); + // bvc.m_verifivation_failed = true; + // return false; + // } + TIME_MEASURE_START(block_processing_time); CRITICAL_REGION_LOCAL(m_blockchain_lock); if(bl.prev_id != get_tail_id()) -- cgit v1.2.3 From 63051bea1c5f37b922a50af444e039d5ea33c09d Mon Sep 17 00:00:00 2001 From: warptangent Date: Sun, 11 Jan 2015 18:46:08 -0800 Subject: Fix comparison between main and alternate chain's cumulative difficulty. This fixes the continual reorganization between a main and alternate chain, using the same two latest blocks from each. The check that cumulative difficulty of the alternate chain is bigger than main's was not using main's last block, but incorrectly using the passed-in block's previous block. main_chain_cumulative_difficulty was being used in two different ways. This has been split up to keep use of main_chain_cumulative_difficulty consistent. --- src/cryptonote_core/blockchain.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 7015383fa..c052e3944 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -1188,8 +1188,16 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id // FIXME: // this brings up an interesting point: consider allowing to get block // difficulty both by height OR by hash, not just height. - auto main_chain_cumulative_difficulty = m_db->get_block_cumulative_difficulty(m_db->get_block_height(b.prev_id)); - bei.cumulative_difficulty = alt_chain.size() ? it_prev->second.cumulative_difficulty : main_chain_cumulative_difficulty; + difficulty_type main_chain_cumulative_difficulty = m_db->get_block_cumulative_difficulty(m_db->height() - 1); + if (alt_chain.size()) + { + bei.cumulative_difficulty = it_prev->second.cumulative_difficulty; + } + else + { + // passed-in block's previous block's cumulative difficulty, found on the main chain + bei.cumulative_difficulty = m_db->get_block_cumulative_difficulty(m_db->get_block_height(b.prev_id)); + } bei.cumulative_difficulty += current_diff; // add block to alternate blocks storage, -- cgit v1.2.3 From 0840c2fd7eb28aabcf793c2ec59824fd354a936c Mon Sep 17 00:00:00 2001 From: warptangent Date: Sun, 11 Jan 2015 19:07:27 -0800 Subject: Fix height assertion in Blockchain::handle_alternative_block() It expects the total number of blocks of main chain, not last block id (off-by-one error). This again behaves like the same height assertion done in original implementation in blockchain_storage::handle_alternative_block(). This allows a reorganization to proceed after an alternative block has been added. --- src/cryptonote_core/blockchain.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index c052e3944..4fa868abe 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -1113,7 +1113,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id if(alt_chain.size()) { // make sure alt chain doesn't somehow start past the end of the main chain - CHECK_AND_ASSERT_MES(m_db->height() - 1 > alt_chain.front()->second.height, false, "main blockchain wrong height"); + CHECK_AND_ASSERT_MES(m_db->height() > alt_chain.front()->second.height, false, "main blockchain wrong height"); // make sure that the blockchain contains the block that should connect // this alternate chain with it. -- cgit v1.2.3 From 800d9b9247530e3810a9e36699bdb96d782e51e4 Mon Sep 17 00:00:00 2001 From: warptangent Date: Wed, 14 Jan 2015 13:41:37 -0800 Subject: Remove code previously made unused and marked unused --- src/cryptonote_core/blockchain.cpp | 18 ------------------ src/cryptonote_core/blockchain_db.cpp | 8 -------- 2 files changed, 26 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 4fa868abe..88ae9d136 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -2028,24 +2028,6 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& { LOG_PRINT_L3("Blockchain::" << __func__); - // NOTE: Omitting check below with have_block() It causes an error after a - // blockchain reorganize begins with error: "Attempting to add block to main - // chain, but it's already either there or in an alternate" - // - // A block in the alternative chain, desired to become the main chain, never - // makes it due to have_block finding it in he alternative chain. - // - // Original implementation didn't use it here, and it doesn't appear - // necessary to be called from here in this implementation either. - - // if we already have the block, return false - // if (have_block(id)) - // { - // LOG_PRINT_L0("Attempting to add block to main chain, but it's already either there or in an alternate chain. hash: " << id); - // bvc.m_verifivation_failed = true; - // return false; - // } - TIME_MEASURE_START(block_processing_time); CRITICAL_REGION_LOCAL(m_blockchain_lock); if(bl.prev_id != get_tail_id()) diff --git a/src/cryptonote_core/blockchain_db.cpp b/src/cryptonote_core/blockchain_db.cpp index 5e781af79..0ee10bd4d 100644 --- a/src/cryptonote_core/blockchain_db.cpp +++ b/src/cryptonote_core/blockchain_db.cpp @@ -107,14 +107,6 @@ void BlockchainDB::remove_transaction(const crypto::hash& tx_hash) { transaction tx = get_tx(tx_hash); - // TODO: This loop calling remove_output() should be removed. It doesn't do - // anything (function is empty), and the outputs were already intended to be - // removed later as part of remove_transaction_data(). - for (const tx_out& tx_output : tx.vout) - { - remove_output(tx_output); - } - for (const txin_v& tx_input : tx.vin) { if (tx_input.type() == typeid(txin_to_key)) -- cgit v1.2.3 From acd4c369e4330562f7dbb22b665b3a1222835b55 Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Mon, 19 Jan 2015 17:39:38 -0500 Subject: Should fix std::min issues related to size_t --- src/cryptonote_core/blockchain.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 88ae9d136..c5771882d 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -616,7 +616,7 @@ difficulty_type Blockchain::get_difficulty_for_next_block() const std::vector cumulative_difficulties; auto h = m_db->height(); - size_t offset = h - std::min(h, static_cast(DIFFICULTY_BLOCKS_COUNT)); + size_t offset = h - std::min(h, static_cast(DIFFICULTY_BLOCKS_COUNT)); if (offset == 0) { @@ -892,7 +892,7 @@ void Blockchain::get_last_n_blocks_sizes(std::vector& sz, size_t count) return; // add size of last blocks to vector (or less, if blockchain size < count) - size_t start_offset = h - std::min(h, count); + size_t start_offset = h - std::min(h, count); for(size_t i = start_offset; i < h; i++) { sz.push_back(m_db->get_block_size(i)); -- cgit v1.2.3 From d00ee784db647ecb1be454a3c5939fac619e7a54 Mon Sep 17 00:00:00 2001 From: warptangent Date: Wed, 28 Jan 2015 09:46:04 -0800 Subject: Update recently added log statement to fix possible null dereference This would have been triggered if function was called without fourth parameter and ring signature check failed. --- src/cryptonote_core/blockchain.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index c5771882d..8740efb9f 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -1866,7 +1866,11 @@ bool Blockchain::check_tx_inputs(const transaction& tx, uint64_t* pmax_used_bloc // signature spending it. if(!check_tx_input(in_to_key, tx_prefix_hash, tx.signatures[sig_index], pmax_used_block_height)) { - LOG_PRINT_L0("Failed to check ring signature for tx " << get_transaction_hash(tx) << " vin key with k_image: " << in_to_key.k_image << " sig_index: " << sig_index << " *pmax_used_block_height: " << *pmax_used_block_height); + LOG_PRINT_L0("Failed to check ring signature for tx " << get_transaction_hash(tx) << " vin key with k_image: " << in_to_key.k_image << " sig_index: " << sig_index); + if (pmax_used_block_height) // a default value of NULL is used when called from Blockchain::handle_block_to_main_chain() + { + LOG_PRINT_L0(" *pmax_used_block_height: " << *pmax_used_block_height); + } return false; } -- cgit v1.2.3 From c8d27fb38df4a3170755390959e388cebd229bd2 Mon Sep 17 00:00:00 2001 From: warptangent Date: Fri, 30 Jan 2015 17:28:38 -0800 Subject: Blockchain: reflect assert behavior of blockchain_storage for get_tx_outputs_gindexs() --- src/cryptonote_core/blockchain.cpp | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src') diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 8740efb9f..7225d5fec 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -1808,6 +1808,8 @@ bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vectorget_tx_amount_output_indices(tx_id); + CHECK_AND_ASSERT_MES(indexs.size(), false, "internal error: global indexes for transaction " << tx_id << " is empty"); + return true; } //------------------------------------------------------------------ -- cgit v1.2.3 From 70342ecadaa1bacbb60be3332894738798d2871b Mon Sep 17 00:00:00 2001 From: warptangent Date: Fri, 30 Jan 2015 17:33:00 -0800 Subject: Blockchain: reflect log level of blockchain_storage Update to match LOG_PRINT_RED_Lx statements. See commit cf5a8b1d6c3df615641e81328bb3d8cf80cd70e3 --- src/cryptonote_core/blockchain.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 7225d5fec..2bd1f2ce9 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -830,7 +830,7 @@ bool Blockchain::prevalidate_miner_transaction(const block& b, uint64_t height) CHECK_AND_ASSERT_MES(b.miner_tx.vin[0].type() == typeid(txin_gen), false, "coinbase transaction in the block has the wrong type"); if(boost::get(b.miner_tx.vin[0]).height != height) { - LOG_PRINT_RED_L0("The miner transaction in block has invalid height: " << boost::get(b.miner_tx.vin[0]).height << ", expected: " << height); + LOG_PRINT_RED_L1("The miner transaction in block has invalid height: " << boost::get(b.miner_tx.vin[0]).height << ", expected: " << height); return false; } CHECK_AND_ASSERT_MES(b.miner_tx.unlock_time == height + CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW, @@ -843,7 +843,7 @@ bool Blockchain::prevalidate_miner_transaction(const block& b, uint64_t height) // does not overflow a uint64_t, and this transaction *is* a uint64_t... if(!check_outs_overflow(b.miner_tx)) { - LOG_PRINT_RED_L0("miner transaction have money overflow in block " << get_block_hash(b)); + LOG_PRINT_RED_L1("miner transaction have money overflow in block " << get_block_hash(b)); return false; } @@ -1082,7 +1082,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id // the block to be added, then this is fine. if (!m_checkpoints.is_alternative_block_allowed(get_current_blockchain_height(), block_height)) { - LOG_PRINT_RED_L0("Block with id: " << id + LOG_PRINT_RED_L1("Block with id: " << id << std::endl << " can't be accepted for alternative chain, block height: " << block_height << std::endl << " blockchain height: " << get_current_blockchain_height()); bvc.m_verifivation_failed = true; @@ -1142,7 +1142,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id // (not earlier than the median of the last X blocks) if(!check_block_timestamp(timestamps, b)) { - LOG_PRINT_RED_L0("Block with id: " << id + LOG_PRINT_RED_L1("Block with id: " << id << std::endl << " for alternative chain, have invalid timestamp: " << b.timestamp); bvc.m_verifivation_failed = true; return false; @@ -1169,7 +1169,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id get_block_longhash(bei.bl, proof_of_work, bei.height); if(!check_hash(proof_of_work, current_diff)) { - LOG_PRINT_RED_L0("Block with id: " << id + LOG_PRINT_RED_L1("Block with id: " << id << std::endl << " for alternative chain, have not enough proof of work: " << proof_of_work << std::endl << " expected difficulty: " << current_diff); bvc.m_verifivation_failed = true; @@ -1178,7 +1178,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id if(!prevalidate_miner_transaction(b, bei.height)) { - LOG_PRINT_RED_L0("Block with id: " << epee::string_tools::pod_to_hex(id) + LOG_PRINT_RED_L1("Block with id: " << epee::string_tools::pod_to_hex(id) << " (as alternative) have wrong miner transaction."); bvc.m_verifivation_failed = true; return false; @@ -1248,7 +1248,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id { //block orphaned bvc.m_marked_as_orphaned = true; - LOG_PRINT_RED_L0("Block recognized as orphaned and rejected, id = " << id); + LOG_PRINT_RED_L1("Block recognized as orphaned and rejected, id = " << id); } return true; @@ -1802,7 +1802,7 @@ bool Blockchain::get_tx_outputs_gindexs(const crypto::hash& tx_id, std::vectortx_exists(tx_id)) { - LOG_PRINT_L0("warning: get_tx_outputs_gindexs failed to find transaction with id = " << tx_id); + LOG_PRINT_RED_L1("warning: get_tx_outputs_gindexs failed to find transaction with id = " << tx_id); return false; } -- cgit v1.2.3 From 7f9b0701652d356b2651214011d6415c70ba2bbb Mon Sep 17 00:00:00 2001 From: warptangent Date: Sun, 1 Feb 2015 18:34:04 -0800 Subject: Blockchain: reflect log and assert updates from blockchain_storage See commit cf5a8b1d6c3df615641e81328bb3d8cf80cd70e3 --- src/cryptonote_core/blockchain.cpp | 75 +++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 37 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 2bd1f2ce9..384a7da7d 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -650,7 +650,7 @@ bool Blockchain::rollback_blockchain_switching(std::list& original_chain, { block_verification_context bvc = boost::value_initialized(); bool r = handle_block_to_main_chain(bl, bvc); - CHECK_AND_ASSERT_MES(r && bvc.m_added_to_main_chain, false, "PANIC!!! failed to add (again) block while chain switching during the rollback!"); + CHECK_AND_ASSERT_MES(r && bvc.m_added_to_main_chain, false, "PANIC! failed to add (again) block while chain switching during the rollback!"); } LOG_PRINT_L1("Rollback to height " << rollback_height << " was successful."); @@ -702,7 +702,7 @@ bool Blockchain::switch_to_alternative_blockchain(std::listsecond, get_block_hash(ch_ent->second.bl)); - LOG_PRINT_L0("The block was inserted as invalid while connecting new alternative chain, block_id: " << get_block_hash(ch_ent->second.bl)); + LOG_PRINT_L1("The block was inserted as invalid while connecting new alternative chain, block_id: " << get_block_hash(ch_ent->second.bl)); m_alternative_chains.erase(ch_ent); for(auto alt_ch_to_orph_iter = ++alt_ch_iter; alt_ch_to_orph_iter != alt_chain.end(); alt_ch_to_orph_iter++) @@ -835,7 +835,8 @@ bool Blockchain::prevalidate_miner_transaction(const block& b, uint64_t height) } CHECK_AND_ASSERT_MES(b.miner_tx.unlock_time == height + CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW, false, - "coinbase transaction transaction have wrong unlock time=" << b.miner_tx.unlock_time << ", expected " << height + CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW); + "coinbase transaction transaction has the wrong unlock time=" << b.miner_tx.unlock_time << ", expected " << height + CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW); + //check outs overflow //NOTE: not entirely sure this is necessary, given that this function is @@ -843,7 +844,7 @@ bool Blockchain::prevalidate_miner_transaction(const block& b, uint64_t height) // does not overflow a uint64_t, and this transaction *is* a uint64_t... if(!check_outs_overflow(b.miner_tx)) { - LOG_PRINT_RED_L1("miner transaction have money overflow in block " << get_block_hash(b)); + LOG_PRINT_RED_L1("miner transaction has money overflow in block " << get_block_hash(b)); return false; } @@ -862,7 +863,7 @@ bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_bl std::vector last_blocks_sizes; get_last_n_blocks_sizes(last_blocks_sizes, CRYPTONOTE_REWARD_BLOCKS_WINDOW); if (!get_block_reward(epee::misc_utils::median(last_blocks_sizes), cumulative_block_size, already_generated_coins, base_reward)) { - LOG_PRINT_L0("block size " << cumulative_block_size << " is bigger than allowed for this blockchain"); + LOG_PRINT_L1("block size " << cumulative_block_size << " is bigger than allowed for this blockchain"); return false; } if(base_reward + fee < money_in_use) @@ -1019,7 +1020,7 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m b.miner_tx.extra.resize(b.miner_tx.extra.size() - 1); if (cumulative_size != txs_size + get_object_blobsize(b.miner_tx)) { //fuck, not lucky, -1 makes varint-counter size smaller, in that case we continue to grow with cumulative_size - LOG_PRINT_RED("Miner tx creation have no luck with delta_extra size = " << delta << " and " << delta - 1 , LOG_LEVEL_2); + LOG_PRINT_RED("Miner tx creation has no luck with delta_extra size = " << delta << " and " << delta - 1 , LOG_LEVEL_2); cumulative_size += delta - 1; continue; } @@ -1125,7 +1126,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id // make sure block connects correctly to the main chain auto h = m_db->get_block_hash_from_height(alt_chain.front()->second.height - 1); - CHECK_AND_ASSERT_MES(h == alt_chain.front()->second.bl.prev_id, false, "alternative chain have wrong connection to main chain"); + CHECK_AND_ASSERT_MES(h == alt_chain.front()->second.bl.prev_id, false, "alternative chain has wrong connection to main chain"); complete_timestamps_vector(m_db->get_block_height(alt_chain.front()->second.bl.prev_id), timestamps); } // if block not associated with known alternate chain @@ -1143,7 +1144,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id if(!check_block_timestamp(timestamps, b)) { LOG_PRINT_RED_L1("Block with id: " << id - << std::endl << " for alternative chain, have invalid timestamp: " << b.timestamp); + << std::endl << " for alternative chain, has invalid timestamp: " << b.timestamp); bvc.m_verifivation_failed = true; return false; } @@ -1170,7 +1171,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id if(!check_hash(proof_of_work, current_diff)) { LOG_PRINT_RED_L1("Block with id: " << id - << std::endl << " for alternative chain, have not enough proof of work: " << proof_of_work + << std::endl << " for alternative chain, does not have enough proof of work: " << proof_of_work << std::endl << " expected difficulty: " << current_diff); bvc.m_verifivation_failed = true; return false; @@ -1179,7 +1180,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id if(!prevalidate_miner_transaction(b, bei.height)) { LOG_PRINT_RED_L1("Block with id: " << epee::string_tools::pod_to_hex(id) - << " (as alternative) have wrong miner transaction."); + << " (as alternative) has incorrect miner transaction."); bvc.m_verifivation_failed = true; return false; @@ -1270,7 +1271,7 @@ bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::list missed_ids; get_transactions(blk.tx_hashes, txs, missed_ids); - CHECK_AND_ASSERT_MES(!missed_ids.size(), false, "have missed transactions in own block in main blockchain"); + CHECK_AND_ASSERT_MES(!missed_ids.size(), false, "has missed transactions in own block in main blockchain"); } return true; @@ -1306,7 +1307,7 @@ bool Blockchain::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NO std::list missed_tx_id; std::list txs; get_transactions(bl.tx_hashes, txs, rsp.missed_ids); - CHECK_AND_ASSERT_MES(!missed_tx_id.size(), false, "Internal error: have missed missed_tx_id.size()=" << missed_tx_id.size() + CHECK_AND_ASSERT_MES(!missed_tx_id.size(), false, "Internal error: has missed missed_tx_id.size()=" << missed_tx_id.size() << std::endl << "for block id = " << get_block_hash(bl)); rsp.blocks.push_back(block_complete_entry()); block_complete_entry& e = rsp.blocks.back(); @@ -1574,7 +1575,7 @@ void Blockchain::print_blockchain(uint64_t start_index, uint64_t end_index) auto h = m_db->height(); if(start_index > h) { - LOG_PRINT_L0("Wrong starter index set: " << start_index << ", expected max index " << h); + LOG_PRINT_L1("Wrong starter index set: " << start_index << ", expected max index " << h); return; } @@ -1697,7 +1698,7 @@ bool Blockchain::add_block_as_invalid(const block_extended_info& bei, const cryp CRITICAL_REGION_LOCAL(m_blockchain_lock); auto i_res = m_invalid_blocks.insert(std::map::value_type(h, bei)); CHECK_AND_ASSERT_MES(i_res.second, false, "at insertion invalid by tx returned status existed"); - LOG_PRINT_L0("BLOCK ADDED AS INVALID: " << h << std::endl << ", prev_id=" << bei.bl.prev_id << ", m_invalid_blocks count=" << m_invalid_blocks.size()); + LOG_PRINT_L1("BLOCK ADDED AS INVALID: " << h << std::endl << ", prev_id=" << bei.bl.prev_id << ", m_invalid_blocks count=" << m_invalid_blocks.size()); return true; } //------------------------------------------------------------------ @@ -1868,10 +1869,10 @@ bool Blockchain::check_tx_inputs(const transaction& tx, uint64_t* pmax_used_bloc // signature spending it. if(!check_tx_input(in_to_key, tx_prefix_hash, tx.signatures[sig_index], pmax_used_block_height)) { - LOG_PRINT_L0("Failed to check ring signature for tx " << get_transaction_hash(tx) << " vin key with k_image: " << in_to_key.k_image << " sig_index: " << sig_index); + LOG_PRINT_L1("Failed to check ring signature for tx " << get_transaction_hash(tx) << " vin key with k_image: " << in_to_key.k_image << " sig_index: " << sig_index); if (pmax_used_block_height) // a default value of NULL is used when called from Blockchain::handle_block_to_main_chain() { - LOG_PRINT_L0(" *pmax_used_block_height: " << *pmax_used_block_height); + LOG_PRINT_L1(" *pmax_used_block_height: " << *pmax_used_block_height); } return false; } @@ -1926,13 +1927,13 @@ bool Blockchain::check_tx_input(const txin_to_key& txin, const crypto::hash& tx_ //check tx unlock time if(!m_bch.is_tx_spendtime_unlocked(tx.unlock_time)) { - LOG_PRINT_L0("One of outputs for one of inputs have wrong tx.unlock_time = " << tx.unlock_time); + LOG_PRINT_L1("One of outputs for one of inputs has wrong tx.unlock_time = " << tx.unlock_time); return false; } if(out.target.type() != typeid(txout_to_key)) { - LOG_PRINT_L0("Output have wrong type id, which=" << out.target.which()); + LOG_PRINT_L1("Output has wrong type id, which=" << out.target.which()); return false; } @@ -1947,7 +1948,7 @@ bool Blockchain::check_tx_input(const txin_to_key& txin, const crypto::hash& tx_ outputs_visitor vi(output_keys, p_output_keys, *this); if(!scan_outputkeys_for_indexes(txin, vi, pmax_related_block_height)) { - LOG_PRINT_L0("Failed to get output keys for tx with amount = " << print_money(txin.amount) << " and count indexes " << txin.key_offsets.size()); + LOG_PRINT_L1("Failed to get output keys for tx with amount = " << print_money(txin.amount) << " and count indexes " << txin.key_offsets.size()); return false; } @@ -1958,7 +1959,7 @@ bool Blockchain::check_tx_input(const txin_to_key& txin, const crypto::hash& tx_ if(txin.key_offsets.size() != output_keys.size()) { - LOG_PRINT_L0("Output keys for tx with amount = " << txin.amount << " and count indexes " << txin.key_offsets.size() << " returned wrong keys count " << output_keys.size()); + LOG_PRINT_L1("Output keys for tx with amount = " << txin.amount << " and count indexes " << txin.key_offsets.size() << " returned wrong keys count " << output_keys.size()); return false; } CHECK_AND_ASSERT_MES(sig.size() == output_keys.size(), false, "internal error: tx signatures count=" << sig.size() << " mismatch with outputs keys count for inputs=" << output_keys.size()); @@ -1983,7 +1984,7 @@ bool Blockchain::check_block_timestamp(std::vector& timestamps, const if(b.timestamp < median_ts) { - LOG_PRINT_L0("Timestamp of block with id: " << get_block_hash(b) << ", " << b.timestamp << ", less than median of last " << BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW << " blocks, " << median_ts); + LOG_PRINT_L1("Timestamp of block with id: " << get_block_hash(b) << ", " << b.timestamp << ", less than median of last " << BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW << " blocks, " << median_ts); return false; } @@ -2002,7 +2003,7 @@ bool Blockchain::check_block_timestamp(const block& b) const LOG_PRINT_L3("Blockchain::" << __func__); if(b.timestamp > get_adjusted_time() + CRYPTONOTE_BLOCK_FUTURE_TIME_LIMIT) { - LOG_PRINT_L0("Timestamp of block with id: " << get_block_hash(b) << ", " << b.timestamp << ", bigger than adjusted time + 2 hours"); + LOG_PRINT_L1("Timestamp of block with id: " << get_block_hash(b) << ", " << b.timestamp << ", bigger than adjusted time + 2 hours"); return false; } @@ -2038,8 +2039,8 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& CRITICAL_REGION_LOCAL(m_blockchain_lock); if(bl.prev_id != get_tail_id()) { - LOG_PRINT_L0("Block with id: " << id << std::endl - << "have wrong prev_id: " << bl.prev_id << std::endl + LOG_PRINT_L1("Block with id: " << id << std::endl + << "has wrong prev_id: " << bl.prev_id << std::endl << "expected: " << get_tail_id()); return false; } @@ -2048,8 +2049,8 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& // of a set number of the most recent blocks. if(!check_block_timestamp(bl)) { - LOG_PRINT_L0("Block with id: " << id << std::endl - << "have invalid timestamp: " << bl.timestamp); + LOG_PRINT_L1("Block with id: " << id << std::endl + << "has invalid timestamp: " << bl.timestamp); bvc.m_verifivation_failed = true; return false; } @@ -2084,9 +2085,9 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& // validate proof_of_work versus difficulty target if(!check_hash(proof_of_work, current_diffic)) { - LOG_PRINT_L0("Block with id: " << id << std::endl - << "have not enough proof of work: " << proof_of_work << std::endl - << "nexpected difficulty: " << current_diffic ); + LOG_PRINT_L1("Block with id: " << id << std::endl + << "does not have enough proof of work: " << proof_of_work << std::endl + << "unexpected difficulty: " << current_diffic ); bvc.m_verifivation_failed = true; return false; } @@ -2108,7 +2109,7 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& // sanity check basic miner tx properties if(!prevalidate_miner_transaction(bl, m_db->height())) { - LOG_PRINT_L0("Block with id: " << id + LOG_PRINT_L1("Block with id: " << id << " failed to pass prevalidation"); bvc.m_verifivation_failed = true; return false; @@ -2133,7 +2134,7 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& if (m_db->tx_exists(tx_id)) { - LOG_PRINT_L0("Block with id: " << id << " attempting to add transaction already in blockchain with id: " << tx_id); + LOG_PRINT_L1("Block with id: " << id << " attempting to add transaction already in blockchain with id: " << tx_id); bvc.m_verifivation_failed = true; break; } @@ -2141,7 +2142,7 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& // get transaction with hash from tx_pool if(!m_tx_pool.take_tx(tx_id, tx, blob_size, fee)) { - LOG_PRINT_L0("Block with id: " << id << "have at least one unknown transaction with id: " << tx_id); + LOG_PRINT_L1("Block with id: " << id << " has at least one unknown transaction with id: " << tx_id); bvc.m_verifivation_failed = true; break; } @@ -2154,11 +2155,11 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& // validate that transaction inputs and the keys spending them are correct. if(!check_tx_inputs(tx)) { - LOG_PRINT_L0("Block with id: " << id << "have at least one transaction (id: " << tx_id << ") with wrong inputs."); + LOG_PRINT_L1("Block with id: " << id << " has at least one transaction (id: " << tx_id << ") with wrong inputs."); //TODO: why is this done? make sure that keeping invalid blocks makes sense. add_block_as_invalid(bl, id); - LOG_PRINT_L0("Block with id " << id << " added as invalid becouse of wrong inputs in transactions"); + LOG_PRINT_L1("Block with id " << id << " added as invalid becouse of wrong inputs in transactions"); bvc.m_verifivation_failed = true; break; } @@ -2178,8 +2179,8 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& uint64_t already_generated_coins = m_db->height() ? m_db->get_block_already_generated_coins(m_db->height() - 1) : 0; if(!validate_miner_transaction(bl, cumulative_block_size, fee_summary, base_reward, already_generated_coins)) { - LOG_PRINT_L0("Block with id: " << id - << " have wrong miner transaction"); + LOG_PRINT_L1("Block with id: " << id + << " has incorrect miner transaction"); bvc.m_verifivation_failed = true; } -- cgit v1.2.3 From 8bd1983cdc03653912fbe27f7fe85d0135435d12 Mon Sep 17 00:00:00 2001 From: warptangent Date: Sun, 1 Feb 2015 17:51:37 -0800 Subject: Blockchain: reflect log updates from blockchain_storage See commit 4ba680f2946966df2030e5765e40ee0a36b112c4 --- src/cryptonote_core/blockchain.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 384a7da7d..b671fa71a 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -735,7 +735,7 @@ bool Blockchain::switch_to_alternative_blockchain(std::list& qbloc // how can we expect to sync from the client that the block list came from? if(!qblock_ids.size() /*|| !req.m_total_height*/) { - LOG_ERROR("Client sent wrong NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << qblock_ids.size() << /*", m_height=" << req.m_total_height <<*/ ", dropping connection"); + LOG_PRINT_L1("Client sent wrong NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << qblock_ids.size() << /*", m_height=" << req.m_total_height <<*/ ", dropping connection"); return false; } @@ -1454,7 +1454,7 @@ bool Blockchain::find_blockchain_supplement(const std::list& qbloc auto gen_hash = m_db->get_block_hash_from_height(0); if(qblock_ids.back() != gen_hash) { - LOG_ERROR("Client sent wrong NOTIFY_REQUEST_CHAIN: genesis block missmatch: " << std::endl << "id: " + LOG_PRINT_L1("Client sent wrong NOTIFY_REQUEST_CHAIN: genesis block missmatch: " << std::endl << "id: " << qblock_ids.back() << ", " << std::endl << "expected: " << gen_hash << "," << std::endl << " dropping connection"); return false; @@ -1486,7 +1486,7 @@ bool Blockchain::find_blockchain_supplement(const std::list& qbloc // but just in case... if(bl_it == qblock_ids.end()) { - LOG_ERROR("Internal error handling connection, can't find split point"); + LOG_PRINT_L1("Internal error handling connection, can't find split point"); return false; } -- cgit v1.2.3 From 84fe5fbd65100a731608471ef0a4c462ea8f5626 Mon Sep 17 00:00:00 2001 From: warptangent Date: Sun, 25 Jan 2015 21:36:09 -0800 Subject: Add compile-time support for both db implementations: in-memory and LMDB Usage: default is lmdb for blockchain branch: $ make release same as: $ DATABASE=lmdb make release for original in-memory implementation: $ DATABASE=memory make release --- src/blockchain_converter/CMakeLists.txt | 2 ++ src/cryptonote_core/blockchain_storage.cpp | 2 +- src/cryptonote_core/blockchain_storage.h | 2 +- src/cryptonote_core/cryptonote_basic.h | 2 ++ src/cryptonote_core/cryptonote_core.cpp | 8 ++++++++ src/cryptonote_core/cryptonote_core.h | 12 ++++++++++++ src/cryptonote_core/tx_pool.cpp | 13 ++++++++++++- src/cryptonote_core/tx_pool.h | 19 ++++++++++++++++++- 8 files changed, 56 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/blockchain_converter/CMakeLists.txt b/src/blockchain_converter/CMakeLists.txt index 713ba18ef..fa2c7bafc 100644 --- a/src/blockchain_converter/CMakeLists.txt +++ b/src/blockchain_converter/CMakeLists.txt @@ -35,6 +35,7 @@ set(blockchain_converter_private_headers) bitmonero_private_headers(blockchain_converter ${blockchain_converter_private_headers}) +if (BLOCKCHAIN_DB STREQUAL DB_LMDB) bitmonero_add_executable(blockchain_converter ${blockchain_converter_sources} ${blockchain_converter_private_headers}) @@ -49,3 +50,4 @@ add_dependencies(blockchain_converter set_property(TARGET blockchain_converter PROPERTY OUTPUT_NAME "blockchain_converter") +endif () diff --git a/src/cryptonote_core/blockchain_storage.cpp b/src/cryptonote_core/blockchain_storage.cpp index 11bd1f2ac..564342444 100644 --- a/src/cryptonote_core/blockchain_storage.cpp +++ b/src/cryptonote_core/blockchain_storage.cpp @@ -640,7 +640,7 @@ bool blockchain_storage::get_last_n_blocks_sizes(std::vector& sz, size_t return get_backward_blocks_sizes(m_blocks.size() -1, sz, count); } //------------------------------------------------------------------ -uint64_t blockchain_storage::get_current_comulative_blocksize_limit() const +uint64_t blockchain_storage::get_current_cumulative_blocksize_limit() const { return m_current_block_cumul_sz_limit; } diff --git a/src/cryptonote_core/blockchain_storage.h b/src/cryptonote_core/blockchain_storage.h index e26d55b64..4846177ef 100644 --- a/src/cryptonote_core/blockchain_storage.h +++ b/src/cryptonote_core/blockchain_storage.h @@ -131,7 +131,7 @@ namespace cryptonote bool check_tx_inputs(const transaction& tx, const crypto::hash& tx_prefix_hash, uint64_t* pmax_used_block_height = NULL) const; bool check_tx_inputs(const transaction& tx, uint64_t* pmax_used_block_height = NULL) const; bool check_tx_inputs(const transaction& tx, uint64_t& pmax_used_block_height, crypto::hash& max_used_block_id) const; - uint64_t get_current_comulative_blocksize_limit() const; + uint64_t get_current_cumulative_blocksize_limit() const; bool is_storing_blockchain()const{return m_is_blockchain_storing;} uint64_t block_difficulty(size_t i) const; diff --git a/src/cryptonote_core/cryptonote_basic.h b/src/cryptonote_core/cryptonote_basic.h index f50a19f9e..2be76c0de 100644 --- a/src/cryptonote_core/cryptonote_basic.h +++ b/src/cryptonote_core/cryptonote_basic.h @@ -50,6 +50,8 @@ #include "misc_language.h" #include "tx_extra.h" +#define DB_MEMORY 1 +#define DB_LMDB 2 namespace cryptonote { diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index e2c533fe5..3a6b84b74 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -51,7 +51,11 @@ namespace cryptonote //----------------------------------------------------------------------------------------------- core::core(i_cryptonote_protocol* pprotocol): m_mempool(m_blockchain_storage), +#if BLOCKCHAIN_DB == DB_LMDB m_blockchain_storage(m_mempool), +#else + m_blockchain_storage(&m_mempool), +#endif m_miner(this), m_miner_address(boost::value_initialized()), m_starter_message_showed(false), @@ -577,7 +581,11 @@ namespace cryptonote m_starter_message_showed = true; } +#if BLOCKCHAIN_DB == DB_LMDB m_store_blockchain_interval.do_call(boost::bind(&Blockchain::store_blockchain, &m_blockchain_storage)); +#else + m_store_blockchain_interval.do_call(boost::bind(&blockchain_storage::store_blockchain, &m_blockchain_storage)); +#endif m_miner.on_idle(); m_mempool.on_idle(); return true; diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 8ee0d8a8d..bf4d7d49f 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -39,7 +39,11 @@ #include "cryptonote_protocol/cryptonote_protocol_handler_common.h" #include "storages/portable_storage_template_helper.h" #include "tx_pool.h" +#if BLOCKCHAIN_DB == DB_LMDB #include "blockchain.h" +#else +#include "blockchain_storage.h" +#endif #include "miner.h" #include "connection_context.h" #include "cryptonote_core/cryptonote_stat_info.h" @@ -112,7 +116,11 @@ namespace cryptonote bool get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res); void pause_mine(); void resume_mine(); +#if BLOCKCHAIN_DB == DB_LMDB Blockchain& get_blockchain_storage(){return m_blockchain_storage;} +#else + blockchain_storage& get_blockchain_storage(){return m_blockchain_storage;} +#endif //debug functions void print_blockchain(uint64_t start_index, uint64_t end_index); void print_blockchain_index(); @@ -149,7 +157,11 @@ namespace cryptonote tx_memory_pool m_mempool; +#if BLOCKCHAIN_DB == DB_LMDB Blockchain m_blockchain_storage; +#else + blockchain_storage m_blockchain_storage; +#endif i_cryptonote_protocol* m_pprotocol; epee::critical_section m_incoming_tx_lock; //m_miner and m_miner_addres are probably temporary here diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index e6c20d814..03ced2c2e 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -37,7 +37,11 @@ #include "cryptonote_format_utils.h" #include "cryptonote_boost_serialization.h" #include "cryptonote_config.h" +#if BLOCKCHAIN_DB == DB_LMDB #include "blockchain.h" +#else +#include "blockchain_storage.h" +#endif #include "common/boost_serialization_helper.h" #include "common/int-util.h" #include "misc_language.h" @@ -52,12 +56,19 @@ namespace cryptonote { size_t const TRANSACTION_SIZE_LIMIT = (((CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE * 125) / 100) - CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE); } - + //--------------------------------------------------------------------------------- +#if BLOCKCHAIN_DB == DB_LMDB //--------------------------------------------------------------------------------- tx_memory_pool::tx_memory_pool(Blockchain& bchs): m_blockchain(bchs) { } +#else + tx_memory_pool::tx_memory_pool(blockchain_storage& bchs): m_blockchain(bchs) + { + + } +#endif //--------------------------------------------------------------------------------- bool tx_memory_pool::add_tx(const transaction &tx, /*const crypto::hash& tx_prefix_hash,*/ const crypto::hash &id, size_t blob_size, tx_verification_context& tvc, bool kept_by_block) { diff --git a/src/cryptonote_core/tx_pool.h b/src/cryptonote_core/tx_pool.h index 7ff8c5e1c..b867a1a7d 100644 --- a/src/cryptonote_core/tx_pool.h +++ b/src/cryptonote_core/tx_pool.h @@ -44,10 +44,13 @@ #include "verification_context.h" #include "crypto/hash.h" - namespace cryptonote { +#if BLOCKCHAIN_DB == DB_LMDB class Blockchain; +#else + class blockchain_storage; +#endif /************************************************************************/ /* */ /************************************************************************/ @@ -55,7 +58,11 @@ namespace cryptonote class tx_memory_pool: boost::noncopyable { public: +#if BLOCKCHAIN_DB == DB_LMDB tx_memory_pool(Blockchain& bchs); +#else + tx_memory_pool(blockchain_storage& bchs); +#endif bool add_tx(const transaction &tx, const crypto::hash &id, size_t blob_size, tx_verification_context& tvc, bool keeped_by_block); bool add_tx(const transaction &tx, tx_verification_context& tvc, bool keeped_by_block); //gets tx and remove it from pool @@ -127,7 +134,11 @@ namespace cryptonote //transactions_container m_alternative_transactions; std::string m_config_folder; +#if BLOCKCHAIN_DB == DB_LMDB Blockchain& m_blockchain; +#else + blockchain_storage& m_blockchain; +#endif /************************************************************************/ /* */ /************************************************************************/ @@ -170,6 +181,12 @@ namespace cryptonote uint64_t operator()(const txin_to_scripthash& tx) const {return 0;} }; +#if BLOCKCHAIN_DB == DB_LMDB +#else +#if defined(DEBUG_CREATE_BLOCK_TEMPLATE) + friend class blockchain_storage; +#endif +#endif }; } -- cgit v1.2.3 From 6f1c4b4c2c78c930fe30ed648e855a6ce55f7dcd Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Fri, 20 Feb 2015 21:09:32 -0500 Subject: Bounds error, should fix #27 --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index 4fa16b23e..ce568151c 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -1053,7 +1053,7 @@ uint64_t BlockchainLMDB::height() const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - return m_height; + return m_height - 1; } -- cgit v1.2.3 From 963bc09087ecf50d1043e62876a8878dae258385 Mon Sep 17 00:00:00 2001 From: warptangent Date: Sun, 22 Feb 2015 10:31:11 -0800 Subject: Revert "Bounds error, should fix #27" This reverts commit 6f1c4b4c2c78c930fe30ed648e855a6ce55f7dcd. --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index ce568151c..4fa16b23e 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -1053,7 +1053,7 @@ uint64_t BlockchainLMDB::height() const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - return m_height - 1; + return m_height; } -- cgit v1.2.3 From b88ab643ca576b450944b0438ae1f606520204fe Mon Sep 17 00:00:00 2001 From: warptangent Date: Sun, 22 Feb 2015 10:38:23 -0800 Subject: Fix Blockchain::get_tail_id() to set parameter to last block number instead of height This reflects the behavior of blockchain_storage::get_tail_id(). Fixes #27 so that RPC method getlastblockheader works. --- src/cryptonote_core/blockchain.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index b671fa71a..525c7f36e 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -463,7 +463,7 @@ crypto::hash Blockchain::get_tail_id(uint64_t& height) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); - height = m_db->height(); + height = m_db->height() - 1; return get_tail_id(); } //------------------------------------------------------------------ -- cgit v1.2.3 From cd972bdcc25e9ed1d4c5bc7663242afa6a61125e Mon Sep 17 00:00:00 2001 From: warptangent Date: Tue, 10 Feb 2015 19:54:27 -0800 Subject: Update year and formatting in license --- src/blockchain_converter/blockchain_converter.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/blockchain_converter/blockchain_converter.cpp b/src/blockchain_converter/blockchain_converter.cpp index 57822700f..0dfc1a2c3 100644 --- a/src/blockchain_converter/blockchain_converter.cpp +++ b/src/blockchain_converter/blockchain_converter.cpp @@ -1,21 +1,21 @@ -// Copyright (c) 2014, The Monero Project -// +// Copyright (c) 2014-2015, The Monero Project +// // All rights reserved. -// +// // Redistribution and use in source and binary forms, with or without modification, are // permitted provided that the following conditions are met: -// +// // 1. Redistributions of source code must retain the above copyright notice, this list of // conditions and the following disclaimer. -// +// // 2. Redistributions in binary form must reproduce the above copyright notice, this list // of conditions and the following disclaimer in the documentation and/or other // materials provided with the distribution. -// +// // 3. Neither the name of the copyright holder nor the names of its contributors may be // used to endorse or promote products derived from this software without specific // prior written permission. -// +// // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL -- cgit v1.2.3 From 59305d3137cbf477c2dc07c26b757d23729ad72b Mon Sep 17 00:00:00 2001 From: warptangent Date: Tue, 10 Feb 2015 15:25:15 -0800 Subject: Blockchain: match original function declaration from blockchain_storage --- src/cryptonote_core/blockchain.cpp | 2 +- src/cryptonote_core/blockchain.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 525c7f36e..b4b1a4c86 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -2288,7 +2288,7 @@ bool Blockchain::add_new_block(const block& bl_, block_verification_context& bvc return handle_block_to_main_chain(bl, id, bvc); } //------------------------------------------------------------------ -void Blockchain::check_against_checkpoints(checkpoints& points, bool enforce) +void Blockchain::check_against_checkpoints(const checkpoints& points, bool enforce) { const auto& pts = points.get_points(); diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index 6ab963ac7..3e8a2de31 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -141,7 +141,7 @@ namespace cryptonote void print_blockchain_index(); void print_blockchain_outs(const std::string& file); - void check_against_checkpoints(checkpoints& points, bool enforce); + void check_against_checkpoints(const checkpoints& points, bool enforce); void set_enforce_dns_checkpoints(bool enforce); bool update_checkpoints(const std::string& file_path, bool check_dns); -- cgit v1.2.3 From 2531aa31f80dff871b08371324a43e1363218c59 Mon Sep 17 00:00:00 2001 From: warptangent Date: Wed, 11 Feb 2015 15:55:53 -0800 Subject: Add and extend log statements --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index 4fa16b23e..f5368d6e5 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -167,8 +167,11 @@ void BlockchainLMDB::add_block( const block& blk MDB_val_copy parent_key(blk.prev_id); MDB_val parent_h; if (mdb_get(*m_write_txn, m_block_heights, &parent_key, &parent_h)) + { + LOG_PRINT_L3("m_height: " << m_height); + LOG_PRINT_L3("parent_key: " << blk.prev_id); throw0(DB_ERROR("Failed to get top block hash to check for new block's parent")); - + } uint64_t parent_height = *(const uint64_t *)parent_h.mv_data; if (parent_height != m_height - 1) throw0(BLOCK_PARENT_DNE("Top block is not new block's parent")); @@ -177,8 +180,9 @@ void BlockchainLMDB::add_block( const block& blk MDB_val_copy key(m_height); MDB_val_copy blob(block_to_blob(blk)); - if (mdb_put(*m_write_txn, m_blocks, &key, &blob, 0)) - throw0(DB_ERROR("Failed to add block blob to db transaction")); + auto res = mdb_put(*m_write_txn, m_blocks, &key, &blob, 0); + if (res) + throw0(DB_ERROR(std::string("Failed to add block blob to db transaction: ").append(mdb_strerror(res)).c_str())); MDB_val_copy sz(block_size); if (mdb_put(*m_write_txn, m_block_sizes, &key, &sz, 0)) @@ -505,8 +509,8 @@ void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image) char anything = '\0'; unused.mv_size = sizeof(char); unused.mv_data = &anything; - if (mdb_put(*m_write_txn, m_spent_keys, &val_key, &unused, 0)) - throw1(DB_ERROR("Error adding spent key image to db transaction")); + if (auto result = mdb_put(*m_write_txn, m_spent_keys, &val_key, &unused, 0)) + throw1(DB_ERROR(std::string("Error adding spent key image to db transaction: ").append(mdb_strerror(result)).c_str())); } void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image) @@ -910,7 +914,7 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) const difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& height) const { - LOG_PRINT_L3("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " height: " << height); check_open(); txn_safe txn; -- cgit v1.2.3 From 4b90fd389de915767b89c558d7ec8187a5d61d14 Mon Sep 17 00:00:00 2001 From: warptangent Date: Wed, 11 Feb 2015 15:55:53 -0800 Subject: Add log statement --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 1 + 1 file changed, 1 insertion(+) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index f5368d6e5..b748baddb 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -1563,6 +1563,7 @@ uint64_t BlockchainLMDB::add_block( const block& blk void BlockchainLMDB::pop_block(block& blk, std::vector& txs) { + LOG_PRINT_L3("BlockchainLMDB::" << __func__); txn_safe txn; if (mdb_txn_begin(m_env, NULL, 0, txn)) throw0(DB_ERROR("Failed to create a transaction for the db")); -- cgit v1.2.3 From 26873db199198bad1892b17a9e7bc0a72c16d5b5 Mon Sep 17 00:00:00 2001 From: warptangent Date: Wed, 11 Feb 2015 15:55:53 -0800 Subject: Remove unused variable --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 1 - 1 file changed, 1 deletion(-) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index b748baddb..c9741c81d 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -386,7 +386,6 @@ void BlockchainLMDB::remove_output(const uint64_t& out_index, const uint64_t amo check_open(); MDB_val_copy k(out_index); - MDB_val v; /****** Uncomment if ever outputs actually need to be stored in this manner blobdata b; -- cgit v1.2.3 From aa82f786c730cb27a69bb70d0806ccc384a5890a Mon Sep 17 00:00:00 2001 From: warptangent Date: Wed, 11 Feb 2015 15:55:53 -0800 Subject: Fix log statement --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index c9741c81d..a11bfdc8c 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -675,7 +675,7 @@ void BlockchainLMDB::open(const std::string& filename) lmdb_db_open(txn, LMDB_OUTPUT_GINDICES, MDB_CREATE, m_output_gindices, "Failed to open db handle for m_output_gindices"); *************************************************/ - lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_CREATE, m_spent_keys, "Failed to open db handle for m_outputs"); + lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_CREATE, m_spent_keys, "Failed to open db handle for m_spent_keys"); mdb_set_dupsort(txn, m_output_amounts, compare_uint64); mdb_set_dupsort(txn, m_tx_outputs, compare_uint64); -- cgit v1.2.3 From 42f8fe5c7f5c3ed7b0d0090a943bffd518c6554d Mon Sep 17 00:00:00 2001 From: warptangent Date: Wed, 11 Feb 2015 15:55:53 -0800 Subject: Fix formatting --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index a11bfdc8c..9f6bc6b31 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -756,7 +756,6 @@ void BlockchainLMDB::unlock() check_open(); } - bool BlockchainLMDB::block_exists(const crypto::hash& h) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -772,7 +771,7 @@ bool BlockchainLMDB::block_exists(const crypto::hash& h) const if (get_result == MDB_NOTFOUND) { txn.commit(); - LOG_PRINT_L1("Block with hash " << epee::string_tools::pod_to_hex(h) << "not found in db"); + LOG_PRINT_L1("Block with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); return false; } else if (get_result) @@ -1075,7 +1074,7 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) const if (get_result == MDB_NOTFOUND) { txn.commit(); - LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << "not found in db"); + LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); return false; } else if (get_result) @@ -1097,7 +1096,7 @@ uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) const MDB_val result; auto get_result = mdb_get(txn, m_tx_unlocks, &key, &result); if (get_result == MDB_NOTFOUND) - throw1(TX_DNE(std::string("tx unlock time with hash ").append(epee::string_tools::pod_to_hex(h)).append("not found in db").c_str())); + throw1(TX_DNE(std::string("tx unlock time with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); else if (get_result) throw0(DB_ERROR("DB error attempting to fetch tx unlock time from hash")); @@ -1117,7 +1116,7 @@ transaction BlockchainLMDB::get_tx(const crypto::hash& h) const MDB_val result; auto get_result = mdb_get(txn, m_txs, &key, &result); if (get_result == MDB_NOTFOUND) - throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(h)).append("not found in db").c_str())); + throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); else if (get_result) throw0(DB_ERROR("DB error attempting to fetch tx from hash")); @@ -1177,7 +1176,7 @@ uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const auto get_result = mdb_get(txn, m_tx_heights, &key, &result); if (get_result == MDB_NOTFOUND) { - throw1(TX_DNE(std::string("tx height with hash ").append(epee::string_tools::pod_to_hex(h)).append("not found in db").c_str())); + throw1(TX_DNE(std::string("tx height with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); } else if (get_result) throw0(DB_ERROR("DB error attempting to fetch tx height from hash")); -- cgit v1.2.3 From ce71abd0fe9739c3557ba5fce446d1244670976f Mon Sep 17 00:00:00 2001 From: warptangent Date: Thu, 19 Feb 2015 06:37:00 -0800 Subject: Move LMDB storage to subfolder --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 10 ++++++++++ src/cryptonote_core/blockchain.cpp | 3 ++- 2 files changed, 12 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index 9f6bc6b31..f7e8c5dda 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -630,6 +630,16 @@ void BlockchainLMDB::open(const std::string& filename) throw0(DB_OPEN_FAILURE(std::string("Failed to create directory ").append(filename).c_str())); } + // check for existing LMDB files in base directory + boost::filesystem::path old_files = direc.parent_path(); + if (boost::filesystem::exists(old_files / "data.mdb") || + boost::filesystem::exists(old_files / "lock.mdb")) + { + LOG_PRINT_L0("Found existing LMDB files in " << old_files.c_str()); + LOG_PRINT_L0("Move data.mdb and/or lock.mdb to " << filename << ", or delete them, and then restart"); + throw DB_ERROR("Database could not be opened"); + } + m_folder = filename; // set up lmdb environment diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index b4b1a4c86..b2d979432 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -237,8 +237,9 @@ bool Blockchain::init(const std::string& config_folder, bool testnet) m_testnet = testnet; boost::filesystem::path folder(m_config_folder); + folder /= "lmdb"; - LOG_PRINT_L0("Loading blockchain..."); + LOG_PRINT_L0("Loading blockchain from folder " << folder.c_str() << " ..."); //FIXME: update filename for BlockchainDB const std::string filename = folder.string(); -- cgit v1.2.3 From 3676ac5841b1e433830323b3c51bf78e96947054 Mon Sep 17 00:00:00 2001 From: warptangent Date: Wed, 11 Feb 2015 15:55:53 -0800 Subject: Add profiling to block and tx processing --- src/cryptonote_core/blockchain_db.cpp | 38 +++++++++++++++++++++++++++++++++++ src/cryptonote_core/blockchain_db.h | 10 +++++++++ 2 files changed, 48 insertions(+) (limited to 'src') diff --git a/src/cryptonote_core/blockchain_db.cpp b/src/cryptonote_core/blockchain_db.cpp index 0ee10bd4d..615b08814 100644 --- a/src/cryptonote_core/blockchain_db.cpp +++ b/src/cryptonote_core/blockchain_db.cpp @@ -28,6 +28,7 @@ #include "cryptonote_core/blockchain_db.h" #include "cryptonote_format_utils.h" +#include "profile_tools.h" using epee::string_tools::pod_to_hex; @@ -73,18 +74,29 @@ uint64_t BlockchainDB::add_block( const block& blk , const std::vector& txs ) { + TIME_MEASURE_START(time1); crypto::hash blk_hash = get_block_hash(blk); + TIME_MEASURE_FINISH(time1); + time_blk_hash += time1; // call out to subclass implementation to add the block & metadata + time1 = epee::misc_utils::get_tick_count(); add_block(blk, block_size, cumulative_difficulty, coins_generated); + TIME_MEASURE_FINISH(time1); + time_add_block1 += time1; // call out to add the transactions + time1 = epee::misc_utils::get_tick_count(); add_transaction(blk_hash, blk.miner_tx); for (const transaction& tx : txs) { add_transaction(blk_hash, tx); } + TIME_MEASURE_FINISH(time1); + time_add_transaction += time1; + + ++num_calls; return height(); } @@ -119,4 +131,30 @@ void BlockchainDB::remove_transaction(const crypto::hash& tx_hash) remove_transaction_data(tx_hash, tx); } +void BlockchainDB::reset_stats() +{ + num_calls = 0; + time_blk_hash = 0; + time_add_block1 = 0; + time_add_transaction = 0; +} + +void BlockchainDB::show_stats() +{ + LOG_PRINT_L1(ENDL + << "*********************************" + << ENDL + << "num_calls: " << num_calls + << ENDL + << "time_blk_hash: " << time_blk_hash << "ms" + << ENDL + << "time_add_block1: " << time_add_block1 << "ms" + << ENDL + << "time_add_transaction: " << time_add_transaction << "ms" + << ENDL + << "*********************************" + << ENDL + ); +} + } // namespace cryptonote diff --git a/src/cryptonote_core/blockchain_db.h b/src/cryptonote_core/blockchain_db.h index db56c7c07..531de04bd 100644 --- a/src/cryptonote_core/blockchain_db.h +++ b/src/cryptonote_core/blockchain_db.h @@ -301,12 +301,22 @@ private: // helper function to remove transaction from blockchain void remove_transaction(const crypto::hash& tx_hash); + uint64_t num_calls = 0; + uint64_t time_blk_hash = 0; + uint64_t time_add_block1 = 0; + uint64_t time_add_transaction = 0; + public: // virtual dtor virtual ~BlockchainDB() { }; + // reset profiling stats + void reset_stats(); + + // show profiling stats + void show_stats(); // open the db at location , or create it if there isn't one. virtual void open(const std::string& filename) = 0; -- cgit v1.2.3 From 8909d7d82eaf2b130b7d36fb5db7863170356b15 Mon Sep 17 00:00:00 2001 From: warptangent Date: Wed, 11 Feb 2015 15:55:53 -0800 Subject: Improve block and tx processing efficiency by less repeat hashing BlockchainLMDB::add_block() BlockchainLMDB::add_transaction_data() BlockchainDB::add_transaction() --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 7 ++++--- src/cryptonote_core/BlockchainDB_impl/db_lmdb.h | 3 ++- src/cryptonote_core/blockchain_db.cpp | 24 ++++++++++++++++++----- src/cryptonote_core/blockchain_db.h | 5 +++-- 4 files changed, 28 insertions(+), 11 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index f7e8c5dda..849ec8f6d 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -152,12 +152,13 @@ void BlockchainLMDB::add_block( const block& blk , const size_t& block_size , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated + , const crypto::hash& blk_hash ) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - MDB_val_copy val_h(get_block_hash(blk)); + MDB_val_copy val_h(blk_hash); MDB_val unused; if (mdb_get(*m_write_txn, m_block_heights, &val_h, &unused) == 0) throw1(BLOCK_EXISTS("Attempting to add block that's already in the db")); @@ -243,12 +244,12 @@ void BlockchainLMDB::remove_block() throw1(DB_ERROR("Failed to add removal of block hash to db transaction")); } -void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx) +void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - MDB_val_copy val_h(get_transaction_hash(tx)); + MDB_val_copy val_h(tx_hash); MDB_val unused; if (mdb_get(*m_write_txn, m_txs, &val_h, &unused) == 0) throw1(TX_EXISTS("Attempting to add transaction that's already in the db")); diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h index 2513aa48a..29fc52fbd 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h @@ -184,11 +184,12 @@ private: , const size_t& block_size , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated + , const crypto::hash& block_hash ); virtual void remove_block(); - virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx); + virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash); virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx); diff --git a/src/cryptonote_core/blockchain_db.cpp b/src/cryptonote_core/blockchain_db.cpp index 615b08814..af0c9487f 100644 --- a/src/cryptonote_core/blockchain_db.cpp +++ b/src/cryptonote_core/blockchain_db.cpp @@ -42,11 +42,21 @@ void BlockchainDB::pop_block() pop_block(blk, txs); } -void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transaction& tx) +void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr) { - crypto::hash tx_hash = get_transaction_hash(tx); + crypto::hash tx_hash; + if (!tx_hash_ptr) + { + // should only need to compute hash for miner transactions + tx_hash = get_transaction_hash(tx); + LOG_PRINT_L3("null tx_hash_ptr - needed to compute: " << tx_hash); + } + else + { + tx_hash = *tx_hash_ptr; + } - add_transaction_data(blk_hash, tx); + add_transaction_data(blk_hash, tx, tx_hash); // iterate tx.vout using indices instead of C++11 foreach syntax because // we need the index @@ -81,7 +91,7 @@ uint64_t BlockchainDB::add_block( const block& blk // call out to subclass implementation to add the block & metadata time1 = epee::misc_utils::get_tick_count(); - add_block(blk, block_size, cumulative_difficulty, coins_generated); + add_block(blk, block_size, cumulative_difficulty, coins_generated, blk_hash); TIME_MEASURE_FINISH(time1); time_add_block1 += time1; @@ -89,9 +99,13 @@ uint64_t BlockchainDB::add_block( const block& blk time1 = epee::misc_utils::get_tick_count(); add_transaction(blk_hash, blk.miner_tx); + int tx_i = 0; + crypto::hash tx_hash = null_hash; for (const transaction& tx : txs) { - add_transaction(blk_hash, tx); + tx_hash = blk.tx_hashes[tx_i]; + add_transaction(blk_hash, tx, &tx_hash); + ++tx_i; } TIME_MEASURE_FINISH(time1); time_add_transaction += time1; diff --git a/src/cryptonote_core/blockchain_db.h b/src/cryptonote_core/blockchain_db.h index 531de04bd..02620c406 100644 --- a/src/cryptonote_core/blockchain_db.h +++ b/src/cryptonote_core/blockchain_db.h @@ -265,13 +265,14 @@ private: , const size_t& block_size , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated + , const crypto::hash& blk_hash ) = 0; // tells the subclass to remove data about the top block virtual void remove_block() = 0; // tells the subclass to store the transaction and its metadata - virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx) = 0; + virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) = 0; // tells the subclass to remove data about a transaction virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) = 0; @@ -296,7 +297,7 @@ private: void pop_block(); // helper function for add_transactions, to add each individual tx - void add_transaction(const crypto::hash& blk_hash, const transaction& tx); + void add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr = NULL); // helper function to remove transaction from blockchain void remove_transaction(const crypto::hash& tx_hash); -- cgit v1.2.3 From 58ecc58be13befb8b5d0e6b847987b2fd9635d2c Mon Sep 17 00:00:00 2001 From: warptangent Date: Wed, 11 Feb 2015 15:55:53 -0800 Subject: BlockchainLMDB: Add support for batch transactions --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 294 ++++++++++++++++++---- src/cryptonote_core/BlockchainDB_impl/db_lmdb.h | 50 +++- 2 files changed, 289 insertions(+), 55 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index 849ec8f6d..c30a785ed 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -600,15 +600,23 @@ void BlockchainLMDB::check_open() const BlockchainLMDB::~BlockchainLMDB() { LOG_PRINT_L3("BlockchainLMDB::" << __func__); + + // batch transaction shouldn't be active at this point. If it is, consider it aborted. + if (m_batch_active) + batch_abort(); } -BlockchainLMDB::BlockchainLMDB() +BlockchainLMDB::BlockchainLMDB(bool batch_transactions) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); // initialize folder to something "safe" just in case // someone accidentally misuses this class... m_folder = "thishsouldnotexistbecauseitisgibberish"; m_open = false; + + m_batch_transactions = batch_transactions; + m_write_txn = nullptr; + m_batch_active = false; m_height = 0; } @@ -720,6 +728,13 @@ void BlockchainLMDB::create(const std::string& filename) void BlockchainLMDB::close() { LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if (m_batch_active) + { + LOG_PRINT_L3("close() first calling batch_abort() due to active batch transaction"); + batch_abort(); + } + this->sync(); + // FIXME: not yet thread safe!!! Use with care. mdb_env_close(m_env); } @@ -727,7 +742,13 @@ void BlockchainLMDB::close() void BlockchainLMDB::sync() { LOG_PRINT_L3("BlockchainLMDB::" << __func__); - // LMDB documentation leads me to believe this is unnecessary + + // Does nothing unless LMDB environment was opened with MDB_NOSYNC or in part + // MDB_NOMETASYNC. Force flush to be synchronous. + if (auto result = mdb_env_sync(m_env, true)) + { + throw0(DB_ERROR(std::string("Failed to sync database").append(mdb_strerror(result)).c_str())); + } } void BlockchainLMDB::reset() @@ -867,12 +888,17 @@ uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const check_open(); txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } MDB_val_copy key(height); MDB_val result; - auto get_result = mdb_get(txn, m_block_timestamps, &key, &result); + auto get_result = mdb_get(*txn_ptr, m_block_timestamps, &key, &result); if (get_result == MDB_NOTFOUND) { throw0(DB_ERROR(std::string("Attempt to get timestamp from height ").append(boost::lexical_cast(height)).append(" failed -- timestamp not in db").c_str())); @@ -880,7 +906,8 @@ uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a timestamp from the db")); - txn.commit(); + if (! m_batch_active) + txn.commit(); return *(const uint64_t*)result.mv_data; } @@ -904,12 +931,18 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) const check_open(); txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } MDB_val_copy key(height); MDB_val result; - auto get_result = mdb_get(txn, m_block_sizes, &key, &result); + auto get_result = mdb_get(*txn_ptr, m_block_sizes, &key, &result); if (get_result == MDB_NOTFOUND) { throw0(DB_ERROR(std::string("Attempt to get block size from height ").append(boost::lexical_cast(height)).append(" failed -- block size not in db").c_str())); @@ -917,7 +950,8 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) const else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a block size from the db")); - txn.commit(); + if (! m_batch_active) + txn.commit(); return *(const size_t*)result.mv_data; } @@ -927,12 +961,17 @@ difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& check_open(); txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } MDB_val_copy key(height); MDB_val result; - auto get_result = mdb_get(txn, m_block_diffs, &key, &result); + auto get_result = mdb_get(*txn_ptr, m_block_diffs, &key, &result); if (get_result == MDB_NOTFOUND) { throw0(DB_ERROR(std::string("Attempt to get cumulative difficulty from height ").append(boost::lexical_cast(height)).append(" failed -- difficulty not in db").c_str())); @@ -940,7 +979,8 @@ difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a cumulative difficulty from the db")); - txn.commit(); + if (! m_batch_active) + txn.commit(); return *(difficulty_type*)result.mv_data; } @@ -967,12 +1007,18 @@ uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& heigh check_open(); txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } MDB_val_copy key(height); MDB_val result; - auto get_result = mdb_get(txn, m_block_coins, &key, &result); + auto get_result = mdb_get(*txn_ptr, m_block_coins, &key, &result); if (get_result == MDB_NOTFOUND) { throw0(DB_ERROR(std::string("Attempt to get generated coins from height ").append(boost::lexical_cast(height)).append(" failed -- block size not in db").c_str())); @@ -980,7 +1026,8 @@ uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& heigh else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a total generated coins from the db")); - txn.commit(); + if (! m_batch_active) + txn.commit(); return *(const uint64_t*)result.mv_data; } @@ -990,20 +1037,28 @@ crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height) check_open(); txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } MDB_val_copy key(height); MDB_val result; - auto get_result = mdb_get(txn, m_block_hashes, &key, &result); + auto get_result = mdb_get(*txn_ptr, m_block_hashes, &key, &result); if (get_result == MDB_NOTFOUND) { throw0(BLOCK_DNE(std::string("Attempt to get hash from height ").append(boost::lexical_cast(height)).append(" failed -- hash not in db").c_str())); } else if (get_result) - throw0(DB_ERROR("Error attempting to retrieve a block hash from the db")); + throw0(DB_ERROR(std::string("Error attempting to retrieve a block hash from the db: "). + append(mdb_strerror(get_result)).c_str())); - txn.commit(); + if (! m_batch_active) + txn.commit(); return *(crypto::hash*)result.mv_data; } @@ -1075,6 +1130,10 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); + if (m_batch_active) + { + LOG_PRINT_L0("WARNING: active batch transaction while creating a read-only txn in tx_exists()"); + } txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) throw0(DB_ERROR("Failed to create a transaction for the db")); @@ -1120,12 +1179,18 @@ transaction BlockchainLMDB::get_tx(const crypto::hash& h) const check_open(); txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } MDB_val_copy key(h); MDB_val result; - auto get_result = mdb_get(txn, m_txs, &key, &result); + auto get_result = mdb_get(*txn_ptr, m_txs, &key, &result); if (get_result == MDB_NOTFOUND) throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); else if (get_result) @@ -1137,6 +1202,8 @@ transaction BlockchainLMDB::get_tx(const crypto::hash& h) const transaction tx; if (!parse_and_validate_tx_from_blob(bd, tx)) throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); + if (! m_batch_active) + txn.commit(); return tx; } @@ -1178,13 +1245,27 @@ uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); + // If m_batch_active is set, a batch transaction exists beyond this class, + // such as a batch import with verification enabled, or possibly (later) a + // batch network sync. + // + // A regular network sync without batching would be expected to open a new + // read transaction here, as validation is done prior to the write for block + // and tx data. + txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } MDB_val_copy key(h); MDB_val result; - auto get_result = mdb_get(txn, m_tx_heights, &key, &result); + auto get_result = mdb_get(*txn_ptr, m_tx_heights, &key, &result); if (get_result == MDB_NOTFOUND) { throw1(TX_DNE(std::string("tx height with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); @@ -1192,6 +1273,9 @@ uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const else if (get_result) throw0(DB_ERROR("DB error attempting to fetch tx height from hash")); + if (! m_batch_active) + txn.commit(); + return *(const uint64_t*)result.mv_data; } @@ -1334,13 +1418,18 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index_from_global(const uint64_t& check_open(); txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } MDB_val_copy k(index); MDB_val v; - auto get_result = mdb_get(txn, m_output_txs, &k, &v); + auto get_result = mdb_get(*txn_ptr, m_output_txs, &k, &v); if (get_result == MDB_NOTFOUND) throw1(OUTPUT_DNE("output with given index not in db")); else if (get_result) @@ -1348,11 +1437,13 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index_from_global(const uint64_t& crypto::hash tx_hash = *(crypto::hash*)v.mv_data; - get_result = mdb_get(txn, m_output_indices, &k, &v); + get_result = mdb_get(*txn_ptr, m_output_indices, &k, &v); if (get_result == MDB_NOTFOUND) throw1(OUTPUT_DNE("output with given index not in db")); else if (get_result) throw0(DB_ERROR("DB error attempting to fetch output tx index")); + if (! m_batch_active) + txn.commit(); return tx_out_index(tx_hash, *(const uint64_t *)v.mv_data); } @@ -1363,10 +1454,15 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con check_open(); txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - - lmdb_cur cur(txn, m_output_amounts); + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } + lmdb_cur cur(*txn_ptr, m_output_amounts); MDB_val_copy k(amount); MDB_val v; @@ -1395,7 +1491,8 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con cur.close(); - txn.commit(); + if (! m_batch_active) + txn.commit(); return get_output_tx_and_index_from_global(glob_index); } @@ -1538,6 +1635,85 @@ bool BlockchainLMDB::has_key_image(const crypto::key_image& img) const return false; } +void BlockchainLMDB::batch_start() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if (! m_batch_transactions) + throw0(DB_ERROR("batch transactions not enabled")); + if (m_batch_active) + throw0(DB_ERROR("batch transaction already in progress")); + if (m_write_txn) + throw0(DB_ERROR("batch transaction attempted, but m_write_txn already in use")); + check_open(); + // NOTE: need to make sure it's destroyed properly when done + if (mdb_txn_begin(m_env, NULL, 0, m_write_batch_txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + // indicates this transaction is for batch transactions, but not whether it's + // active + m_write_batch_txn.m_batch_txn = true; + m_write_txn = &m_write_batch_txn; + m_batch_active = true; + LOG_PRINT_L3("batch transaction: begin"); +} + +void BlockchainLMDB::batch_commit() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if (! m_batch_transactions) + throw0(DB_ERROR("batch transactions not enabled")); + if (! m_batch_active) + throw0(DB_ERROR("batch transaction not in progress")); + check_open(); + LOG_PRINT_L3("batch transaction: committing..."); + m_write_txn->commit(); + LOG_PRINT_L3("batch transaction: committed"); + + if (mdb_txn_begin(m_env, NULL, 0, m_write_batch_txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + if (! m_write_batch_txn.m_batch_txn) + throw0(DB_ERROR("m_write_batch_txn not marked as a batch transaction")); + m_write_txn = &m_write_batch_txn; +} + +void BlockchainLMDB::batch_stop() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if (! m_batch_transactions) + throw0(DB_ERROR("batch transactions not enabled")); + if (! m_batch_active) + throw0(DB_ERROR("batch transaction not in progress")); + check_open(); + LOG_PRINT_L3("batch transaction: committing..."); + m_write_txn->commit(); + // for destruction of batch transaction + m_write_txn = nullptr; + m_batch_active = false; + LOG_PRINT_L3("batch transaction: end"); +} + +void BlockchainLMDB::batch_abort() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if (! m_batch_transactions) + throw0(DB_ERROR("batch transactions not enabled")); + if (! m_batch_active) + throw0(DB_ERROR("batch transaction not in progress")); + check_open(); + // for destruction of batch transaction + m_write_txn = nullptr; + // explicitly call in case mdb_env_close() (BlockchainLMDB::close()) called before BlockchainLMDB destructor called. + m_write_batch_txn.abort(); + m_batch_active = false; + LOG_PRINT_L3("batch transaction: aborted"); +} + +void BlockchainLMDB::set_batch_transactions(bool batch_transactions) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + m_batch_transactions = batch_transactions; + LOG_PRINT_L3("batch transactions " << (m_batch_transactions ? "enabled" : "disabled")); +} + uint64_t BlockchainLMDB::add_block( const block& blk , const size_t& block_size , const difficulty_type& cumulative_difficulty @@ -1547,23 +1723,31 @@ uint64_t BlockchainLMDB::add_block( const block& blk { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); + txn_safe txn; - if (mdb_txn_begin(m_env, NULL, 0, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - m_write_txn = &txn; + if (! m_batch_active) + { + if (mdb_txn_begin(m_env, NULL, 0, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + m_write_txn = &txn; + } uint64_t num_outputs = m_num_outputs; try { BlockchainDB::add_block(blk, block_size, cumulative_difficulty, coins_generated, txs); - m_write_txn = NULL; + if (! m_batch_active) + { + m_write_txn = NULL; - txn.commit(); + txn.commit(); + } } catch (...) { m_num_outputs = num_outputs; - m_write_txn = NULL; + if (! m_batch_active) + m_write_txn = NULL; throw; } @@ -1574,17 +1758,23 @@ void BlockchainLMDB::pop_block(block& blk, std::vector& txs) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); txn_safe txn; - if (mdb_txn_begin(m_env, NULL, 0, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - m_write_txn = &txn; + if (! m_batch_active) + { + if (mdb_txn_begin(m_env, NULL, 0, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + m_write_txn = &txn; + } uint64_t num_outputs = m_num_outputs; try { BlockchainDB::pop_block(blk, txs); - m_write_txn = NULL; + if (! m_batch_active) + { + m_write_txn = NULL; - txn.commit(); + txn.commit(); + } } catch (...) { diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h index 29fc52fbd..0b4b5ec1a 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h @@ -38,8 +38,23 @@ struct txn_safe txn_safe() : m_txn(NULL) { } ~txn_safe() { - if(m_txn != NULL) + LOG_PRINT_L3("txn_safe: destructor"); + if (m_txn != NULL) { + if (m_batch_txn) // this is a batch txn and should have been handled before this point for safety + { + LOG_PRINT_L0("WARNING: txn_safe: m_txn is a batch txn and it's not NULL in destructor - calling mdb_txn_abort()"); + } + else + { + // Example of when this occurs: a lookup fails, so a read-only txn is + // aborted through this destructor. However, successful read-only txns + // ideally should have been committed when done and not end up here. + // + // NOTE: not sure if this is ever reached for a non-batch write + // transaction, but it's probably not ideal if it did. + LOG_PRINT_L3("txn_safe: m_txn not NULL in destructor - calling mdb_txn_abort()"); + } mdb_txn_abort(m_txn); } } @@ -60,6 +75,24 @@ struct txn_safe m_txn = NULL; } + // This should only be needed for batch transaction which must be ensured to + // be aborted before mdb_env_close, not after. So we can't rely on + // BlockchainLMDB destructor to call txn_safe destructor, as that's too late + // to properly abort, since mdb_env_close would have been called earlier. + void abort() + { + LOG_PRINT_L3("txn_safe: abort()"); + if(m_txn != NULL) + { + mdb_txn_abort(m_txn); + m_txn = NULL; + } + else + { + LOG_PRINT_L0("WARNING: txn_safe: abort() called, but m_txn is NULL"); + } + } + operator MDB_txn*() { return m_txn; @@ -71,13 +104,14 @@ struct txn_safe } MDB_txn* m_txn; + bool m_batch_txn = false; }; class BlockchainLMDB : public BlockchainDB { public: - BlockchainLMDB(); + BlockchainLMDB(bool batch_transactions=false); ~BlockchainLMDB(); virtual void open(const std::string& filename); @@ -177,6 +211,12 @@ public: , const std::vector& txs ); + virtual void set_batch_transactions(bool batch_transactions); + virtual void batch_start(); + virtual void batch_commit(); + virtual void batch_stop(); + virtual void batch_abort(); + virtual void pop_block(block& blk, std::vector& txs); private: @@ -264,7 +304,11 @@ private: uint64_t m_height; uint64_t m_num_outputs; std::string m_folder; - txn_safe* m_write_txn; + txn_safe* m_write_txn; // may point to either a short-lived txn or a batch txn + txn_safe m_write_batch_txn; // persist batch txn outside of BlockchainLMDB + + bool m_batch_transactions; // support for batch transactions + bool m_batch_active; // whether batch transaction is in progress }; } // namespace cryptonote -- cgit v1.2.3 From b7a2d84919bc28b3747974509f932e64907e357a Mon Sep 17 00:00:00 2001 From: warptangent Date: Wed, 18 Feb 2015 20:52:44 -0800 Subject: BlockchainLMDB: Add check for open database to two functions --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index c30a785ed..c3c0724ce 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -742,6 +742,7 @@ void BlockchainLMDB::close() void BlockchainLMDB::sync() { LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); // Does nothing unless LMDB environment was opened with MDB_NOSYNC or in part // MDB_NOMETASYNC. Force flush to be synchronous. @@ -1757,6 +1758,8 @@ uint64_t BlockchainLMDB::add_block( const block& blk void BlockchainLMDB::pop_block(block& blk, std::vector& txs) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + txn_safe txn; if (! m_batch_active) { -- cgit v1.2.3 From 7a66b8bbcfb3f8084141d8ffb1154a3a514a9ee5 Mon Sep 17 00:00:00 2001 From: warptangent Date: Wed, 11 Feb 2015 15:55:53 -0800 Subject: BlockchainDB: Add virtual function declarations for batch transactions --- src/cryptonote_core/blockchain_db.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src') diff --git a/src/cryptonote_core/blockchain_db.h b/src/cryptonote_core/blockchain_db.h index 02620c406..25e9781f8 100644 --- a/src/cryptonote_core/blockchain_db.h +++ b/src/cryptonote_core/blockchain_db.h @@ -347,6 +347,9 @@ public: // release db lock virtual void unlock() = 0; + virtual void batch_start() = 0; + virtual void batch_stop() = 0; + virtual void set_batch_transactions(bool) = 0; // adds a block with the given metadata to the top of the blockchain, returns the new height // NOTE: subclass implementations of this (or the functions it calls) need -- cgit v1.2.3 From 8529c0ea9aa305f097d4a2c89d581163e208e04f Mon Sep 17 00:00:00 2001 From: warptangent Date: Wed, 11 Feb 2015 15:55:53 -0800 Subject: BlockchainDB, BlockchainLMDB: Add profiling for DB commits --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 10 ++++++++++ src/cryptonote_core/blockchain_db.cpp | 3 +++ src/cryptonote_core/blockchain_db.h | 5 +++++ 3 files changed, 18 insertions(+) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index c3c0724ce..25c25f399 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -33,6 +33,7 @@ #include "cryptonote_core/cryptonote_format_utils.h" #include "crypto/crypto.h" +#include "profile_tools.h" using epee::string_tools::pod_to_hex; @@ -1666,7 +1667,10 @@ void BlockchainLMDB::batch_commit() throw0(DB_ERROR("batch transaction not in progress")); check_open(); LOG_PRINT_L3("batch transaction: committing..."); + TIME_MEASURE_START(time1); m_write_txn->commit(); + TIME_MEASURE_FINISH(time1); + time_commit1 += time1; LOG_PRINT_L3("batch transaction: committed"); if (mdb_txn_begin(m_env, NULL, 0, m_write_batch_txn)) @@ -1685,7 +1689,10 @@ void BlockchainLMDB::batch_stop() throw0(DB_ERROR("batch transaction not in progress")); check_open(); LOG_PRINT_L3("batch transaction: committing..."); + TIME_MEASURE_START(time1); m_write_txn->commit(); + TIME_MEASURE_FINISH(time1); + time_commit1 += time1; // for destruction of batch transaction m_write_txn = nullptr; m_batch_active = false; @@ -1741,7 +1748,10 @@ uint64_t BlockchainLMDB::add_block( const block& blk { m_write_txn = NULL; + TIME_MEASURE_START(time1); txn.commit(); + TIME_MEASURE_FINISH(time1); + time_commit1 += time1; } } catch (...) diff --git a/src/cryptonote_core/blockchain_db.cpp b/src/cryptonote_core/blockchain_db.cpp index af0c9487f..74de19585 100644 --- a/src/cryptonote_core/blockchain_db.cpp +++ b/src/cryptonote_core/blockchain_db.cpp @@ -151,6 +151,7 @@ void BlockchainDB::reset_stats() time_blk_hash = 0; time_add_block1 = 0; time_add_transaction = 0; + time_commit1 = 0; } void BlockchainDB::show_stats() @@ -166,6 +167,8 @@ void BlockchainDB::show_stats() << ENDL << "time_add_transaction: " << time_add_transaction << "ms" << ENDL + << "time_commit1: " << time_commit1 << "ms" + << ENDL << "*********************************" << ENDL ); diff --git a/src/cryptonote_core/blockchain_db.h b/src/cryptonote_core/blockchain_db.h index 25e9781f8..ba0e5738b 100644 --- a/src/cryptonote_core/blockchain_db.h +++ b/src/cryptonote_core/blockchain_db.h @@ -308,6 +308,11 @@ private: uint64_t time_add_transaction = 0; +protected: + + uint64_t time_commit1 = 0; + + public: // virtual dtor -- cgit v1.2.3 From 83fb6d8d07c0b473e7b53a35d8e69f586bd1945e Mon Sep 17 00:00:00 2001 From: warptangent Date: Tue, 10 Feb 2015 20:01:02 -0800 Subject: BlockchainLMDB: Add batch transaction support to tx_exists() --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index 25c25f399..4e9f6cb28 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -1132,20 +1132,23 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); + txn_safe txn; + txn_safe* txn_ptr = &txn; if (m_batch_active) + txn_ptr = m_write_txn; + else { - LOG_PRINT_L0("WARNING: active batch transaction while creating a read-only txn in tx_exists()"); + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); } - txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); MDB_val_copy key(h); MDB_val result; - auto get_result = mdb_get(txn, m_txs, &key, &result); + auto get_result = mdb_get(*txn_ptr, m_txs, &key, &result); if (get_result == MDB_NOTFOUND) { - txn.commit(); + if (! m_batch_active) + txn.commit(); LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); return false; } -- cgit v1.2.3 From 6485dacc2fcaab7eb6db63c46716ffdfa0c568c4 Mon Sep 17 00:00:00 2001 From: warptangent Date: Tue, 10 Feb 2015 20:01:02 -0800 Subject: BlockchainLMDB: Add profiling to tx_exists() --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 5 ++++- src/cryptonote_core/blockchain_db.cpp | 3 +++ src/cryptonote_core/blockchain_db.h | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index 4e9f6cb28..a09d9331c 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -1126,7 +1126,6 @@ uint64_t BlockchainLMDB::height() const return m_height; } - bool BlockchainLMDB::tx_exists(const crypto::hash& h) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -1144,7 +1143,11 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) const MDB_val_copy key(h); MDB_val result; + + TIME_MEASURE_START(time1); auto get_result = mdb_get(*txn_ptr, m_txs, &key, &result); + TIME_MEASURE_FINISH(time1); + time_tx_exists += time1; if (get_result == MDB_NOTFOUND) { if (! m_batch_active) diff --git a/src/cryptonote_core/blockchain_db.cpp b/src/cryptonote_core/blockchain_db.cpp index 74de19585..fc8aeef4e 100644 --- a/src/cryptonote_core/blockchain_db.cpp +++ b/src/cryptonote_core/blockchain_db.cpp @@ -149,6 +149,7 @@ void BlockchainDB::reset_stats() { num_calls = 0; time_blk_hash = 0; + time_tx_exists = 0; time_add_block1 = 0; time_add_transaction = 0; time_commit1 = 0; @@ -163,6 +164,8 @@ void BlockchainDB::show_stats() << ENDL << "time_blk_hash: " << time_blk_hash << "ms" << ENDL + << "time_tx_exists: " << time_tx_exists << "ms" + << ENDL << "time_add_block1: " << time_add_block1 << "ms" << ENDL << "time_add_transaction: " << time_add_transaction << "ms" diff --git a/src/cryptonote_core/blockchain_db.h b/src/cryptonote_core/blockchain_db.h index ba0e5738b..2a7fa8f82 100644 --- a/src/cryptonote_core/blockchain_db.h +++ b/src/cryptonote_core/blockchain_db.h @@ -310,6 +310,7 @@ private: protected: + mutable uint64_t time_tx_exists = 0; uint64_t time_commit1 = 0; -- cgit v1.2.3 From 0ad0784f46cacfbd5ac00a9de4f1dd3e74ca8f95 Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Mon, 23 Feb 2015 18:28:20 -0500 Subject: Changed log level of debug message -- too spammy --- src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp index a09d9331c..feff4a272 100644 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp @@ -805,7 +805,7 @@ bool BlockchainLMDB::block_exists(const crypto::hash& h) const if (get_result == MDB_NOTFOUND) { txn.commit(); - LOG_PRINT_L1("Block with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); + LOG_PRINT_L3("Block with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); return false; } else if (get_result) -- cgit v1.2.3 From 5eab480cb116b7c58fe020951995815baa7ccd8f Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Fri, 6 Mar 2015 15:20:45 -0500 Subject: Moved BlockchainDB into its own src/ subfolder Ostensibly janitorial work, but should be more relevant later down the line. Things that depend on core cryptonote things (i.e. cryptonote_core) don't necessarily depend on BlockchainDB and thus have no need to have BlockchainDB baked in with them. --- src/CMakeLists.txt | 1 + src/blockchain_converter/CMakeLists.txt | 1 + src/blockchain_converter/blockchain_converter.cpp | 4 +- src/blockchain_db/CMakeLists.txt | 60 + src/blockchain_db/blockchain_db.cpp | 180 ++ src/blockchain_db/blockchain_db.h | 487 ++++++ src/blockchain_db/lmdb/db_lmdb.cpp | 1808 +++++++++++++++++++++ src/blockchain_db/lmdb/db_lmdb.h | 314 ++++ src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp | 1808 --------------------- src/cryptonote_core/BlockchainDB_impl/db_lmdb.h | 314 ---- src/cryptonote_core/CMakeLists.txt | 6 +- src/cryptonote_core/blockchain.cpp | 4 +- src/cryptonote_core/blockchain.h | 2 +- src/cryptonote_core/blockchain_db.cpp | 180 -- src/cryptonote_core/blockchain_db.h | 487 ------ src/daemon/CMakeLists.txt | 1 + 16 files changed, 2858 insertions(+), 2799 deletions(-) create mode 100644 src/blockchain_db/CMakeLists.txt create mode 100644 src/blockchain_db/blockchain_db.cpp create mode 100644 src/blockchain_db/blockchain_db.h create mode 100644 src/blockchain_db/lmdb/db_lmdb.cpp create mode 100644 src/blockchain_db/lmdb/db_lmdb.h delete mode 100644 src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp delete mode 100644 src/cryptonote_core/BlockchainDB_impl/db_lmdb.h delete mode 100644 src/cryptonote_core/blockchain_db.cpp delete mode 100644 src/cryptonote_core/blockchain_db.h (limited to 'src') diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7c4492b01..5242d261e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -89,6 +89,7 @@ endfunction () add_subdirectory(common) add_subdirectory(crypto) add_subdirectory(cryptonote_core) +add_subdirectory(blockchain_db) add_subdirectory(mnemonics) add_subdirectory(rpc) add_subdirectory(wallet) diff --git a/src/blockchain_converter/CMakeLists.txt b/src/blockchain_converter/CMakeLists.txt index fa2c7bafc..a91624f4f 100644 --- a/src/blockchain_converter/CMakeLists.txt +++ b/src/blockchain_converter/CMakeLists.txt @@ -43,6 +43,7 @@ bitmonero_add_executable(blockchain_converter target_link_libraries(blockchain_converter LINK_PRIVATE cryptonote_core + blockchain_db ${CMAKE_THREAD_LIBS_INIT}) add_dependencies(blockchain_converter diff --git a/src/blockchain_converter/blockchain_converter.cpp b/src/blockchain_converter/blockchain_converter.cpp index 0dfc1a2c3..aae569e58 100644 --- a/src/blockchain_converter/blockchain_converter.cpp +++ b/src/blockchain_converter/blockchain_converter.cpp @@ -34,9 +34,9 @@ #include "cryptonote_core/cryptonote_format_utils.h" #include "misc_language.h" #include "cryptonote_core/blockchain_storage.h" -#include "cryptonote_core/blockchain_db.h" +#include "blockchain_db/blockchain_db.h" #include "cryptonote_core/blockchain.h" -#include "cryptonote_core/BlockchainDB_impl/db_lmdb.h" +#include "blockchain_db/lmdb/db_lmdb.h" #include "cryptonote_core/tx_pool.h" #include diff --git a/src/blockchain_db/CMakeLists.txt b/src/blockchain_db/CMakeLists.txt new file mode 100644 index 000000000..84b2d6a74 --- /dev/null +++ b/src/blockchain_db/CMakeLists.txt @@ -0,0 +1,60 @@ +# Copyright (c) 2014-2015, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set(blockchain_db_sources + blockchain_db.cpp + lmdb/db_lmdb.cpp + ) + +set(blockchain_db_headers) + +set(blockchain_db_private_headers + blockchain_db.h + lmdb/db_lmdb.h + ) + +bitmonero_private_headers(blockchain_db + ${crypto_private_headers}) +bitmonero_add_library(blockchain_db + ${blockchain_db_sources} + ${blockchain_db_headers} + ${blockchain_db_private_headers}) +target_link_libraries(blockchain_db + LINK_PUBLIC + common + crypto + cryptonote_core + ${Boost_DATE_TIME_LIBRARY} + ${Boost_PROGRAM_OPTIONS_LIBRARY} + ${Boost_SERIALIZATION_LIBRARY} + LINK_PRIVATE + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${Boost_THREAD_LIBRARY} + ${LMDB_LIBRARY} + ${EXTRA_LIBRARIES}) diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp new file mode 100644 index 000000000..8b08d3805 --- /dev/null +++ b/src/blockchain_db/blockchain_db.cpp @@ -0,0 +1,180 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "blockchain_db.h" +#include "cryptonote_core/cryptonote_format_utils.h" +#include "profile_tools.h" + +using epee::string_tools::pod_to_hex; + +namespace cryptonote +{ + +void BlockchainDB::pop_block() +{ + block blk; + std::vector txs; + pop_block(blk, txs); +} + +void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr) +{ + crypto::hash tx_hash; + if (!tx_hash_ptr) + { + // should only need to compute hash for miner transactions + tx_hash = get_transaction_hash(tx); + LOG_PRINT_L3("null tx_hash_ptr - needed to compute: " << tx_hash); + } + else + { + tx_hash = *tx_hash_ptr; + } + + add_transaction_data(blk_hash, tx, tx_hash); + + // iterate tx.vout using indices instead of C++11 foreach syntax because + // we need the index + if (tx.vout.size() != 0) // it may be technically possible for a tx to have no outputs + { + for (uint64_t i = 0; i < tx.vout.size(); ++i) + { + add_output(tx_hash, tx.vout[i], i); + } + + for (const txin_v& tx_input : tx.vin) + { + if (tx_input.type() == typeid(txin_to_key)) + { + add_spent_key(boost::get(tx_input).k_image); + } + } + } +} + +uint64_t BlockchainDB::add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const std::vector& txs + ) +{ + TIME_MEASURE_START(time1); + crypto::hash blk_hash = get_block_hash(blk); + TIME_MEASURE_FINISH(time1); + time_blk_hash += time1; + + // call out to subclass implementation to add the block & metadata + time1 = epee::misc_utils::get_tick_count(); + add_block(blk, block_size, cumulative_difficulty, coins_generated, blk_hash); + TIME_MEASURE_FINISH(time1); + time_add_block1 += time1; + + // call out to add the transactions + + time1 = epee::misc_utils::get_tick_count(); + add_transaction(blk_hash, blk.miner_tx); + int tx_i = 0; + crypto::hash tx_hash = null_hash; + for (const transaction& tx : txs) + { + tx_hash = blk.tx_hashes[tx_i]; + add_transaction(blk_hash, tx, &tx_hash); + ++tx_i; + } + TIME_MEASURE_FINISH(time1); + time_add_transaction += time1; + + ++num_calls; + + return height(); +} + +void BlockchainDB::pop_block(block& blk, std::vector& txs) +{ + blk = get_top_block(); + + remove_block(); + + remove_transaction(get_transaction_hash(blk.miner_tx)); + for (const auto& h : blk.tx_hashes) + { + txs.push_back(get_tx(h)); + remove_transaction(h); + } +} + +void BlockchainDB::remove_transaction(const crypto::hash& tx_hash) +{ + transaction tx = get_tx(tx_hash); + + for (const txin_v& tx_input : tx.vin) + { + if (tx_input.type() == typeid(txin_to_key)) + { + remove_spent_key(boost::get(tx_input).k_image); + } + } + + // need tx as tx.vout has the tx outputs, and the output amounts are needed + remove_transaction_data(tx_hash, tx); +} + +void BlockchainDB::reset_stats() +{ + num_calls = 0; + time_blk_hash = 0; + time_tx_exists = 0; + time_add_block1 = 0; + time_add_transaction = 0; + time_commit1 = 0; +} + +void BlockchainDB::show_stats() +{ + LOG_PRINT_L1(ENDL + << "*********************************" + << ENDL + << "num_calls: " << num_calls + << ENDL + << "time_blk_hash: " << time_blk_hash << "ms" + << ENDL + << "time_tx_exists: " << time_tx_exists << "ms" + << ENDL + << "time_add_block1: " << time_add_block1 << "ms" + << ENDL + << "time_add_transaction: " << time_add_transaction << "ms" + << ENDL + << "time_commit1: " << time_commit1 << "ms" + << ENDL + << "*********************************" + << ENDL + ); +} + +} // namespace cryptonote diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h new file mode 100644 index 000000000..2a7fa8f82 --- /dev/null +++ b/src/blockchain_db/blockchain_db.h @@ -0,0 +1,487 @@ +// Copyright (c) 2014, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#ifndef BLOCKCHAIN_DB_H +#define BLOCKCHAIN_DB_H + +#include +#include +#include +#include "crypto/hash.h" +#include "cryptonote_core/cryptonote_basic.h" +#include "cryptonote_core/difficulty.h" + +/* DB Driver Interface + * + * The DB interface is a store for the canonical block chain. + * It serves as a persistent storage for the blockchain. + * + * For the sake of efficiency, the reference implementation will also + * store some blockchain data outside of the blocks, such as spent + * transfer key images, unspent transaction outputs, etc. + * + * Transactions are duplicated so that we don't have to fetch a whole block + * in order to fetch a transaction from that block. If this is deemed + * unnecessary later, this can change. + * + * Spent key images are duplicated outside of the blocks so it is quick + * to verify an output hasn't already been spent + * + * Unspent transaction outputs are duplicated to quickly gather random + * outputs to use for mixins + * + * IMPORTANT: + * A concrete implementation of this interface should populate these + * duplicated members! It is possible to have a partial implementation + * of this interface call to private members of the interface to be added + * later that will then populate as needed. + * + * General: + * open() + * close() + * sync() + * reset() + * + * Lock and unlock provided for reorg externally, and for block + * additions internally, this way threaded reads are completely fine + * unless the blockchain is changing. + * bool lock() + * unlock() + * + * vector get_filenames() + * + * Blocks: + * bool block_exists(hash) + * height add_block(block, block_size, cumulative_difficulty, coins_generated, transactions) + * block get_block(hash) + * height get_block_height(hash) + * header get_block_header(hash) + * block get_block_from_height(height) + * size_t get_block_size(height) + * difficulty get_block_cumulative_difficulty(height) + * uint64_t get_block_already_generated_coins(height) + * uint64_t get_block_timestamp(height) + * uint64_t get_top_block_timestamp() + * hash get_block_hash_from_height(height) + * blocks get_blocks_range(height1, height2) + * hashes get_hashes_range(height1, height2) + * hash top_block_hash() + * block get_top_block() + * height height() + * void pop_block(block&, tx_list&) + * + * Transactions: + * bool tx_exists(hash) + * uint64_t get_tx_unlock_time(hash) + * tx get_tx(hash) + * uint64_t get_tx_count() + * tx_list get_tx_list(hash_list) + * height get_tx_block_height(hash) + * + * Outputs: + * index get_random_output(amount) + * uint64_t get_num_outputs(amount) + * pub_key get_output_key(amount, index) + * tx_out get_output(tx_hash, index) + * hash,index get_output_tx_and_index_from_global(index) + * hash,index get_output_tx_and_index(amount, index) + * vec get_tx_output_indices(tx_hash) + * + * + * Spent Output Key Images: + * bool has_key_image(key_image) + * + * Exceptions: + * DB_ERROR -- generic + * DB_OPEN_FAILURE + * DB_CREATE_FAILURE + * DB_SYNC_FAILURE + * BLOCK_DNE + * BLOCK_PARENT_DNE + * BLOCK_EXISTS + * BLOCK_INVALID -- considering making this multiple errors + * TX_DNE + * TX_EXISTS + * OUTPUT_DNE + * OUTPUT_EXISTS + * KEY_IMAGE_EXISTS + */ + +namespace cryptonote +{ + +// typedef for convenience +typedef std::pair tx_out_index; + +/*********************************** + * Exception Definitions + ***********************************/ +class DB_EXCEPTION : public std::exception +{ + private: + std::string m; + + protected: + DB_EXCEPTION(const char *s) : m(s) { } + + public: + virtual ~DB_EXCEPTION() { } + + const char* what() const throw() + { + return m.c_str(); + } +}; + +class DB_ERROR : public DB_EXCEPTION +{ + public: + DB_ERROR() : DB_EXCEPTION("Generic DB Error") { } + DB_ERROR(const char* s) : DB_EXCEPTION(s) { } +}; + +class DB_OPEN_FAILURE : public DB_EXCEPTION +{ + public: + DB_OPEN_FAILURE() : DB_EXCEPTION("Failed to open the db") { } + DB_OPEN_FAILURE(const char* s) : DB_EXCEPTION(s) { } +}; + +class DB_CREATE_FAILURE : public DB_EXCEPTION +{ + public: + DB_CREATE_FAILURE() : DB_EXCEPTION("Failed to create the db") { } + DB_CREATE_FAILURE(const char* s) : DB_EXCEPTION(s) { } +}; + +class DB_SYNC_FAILURE : public DB_EXCEPTION +{ + public: + DB_SYNC_FAILURE() : DB_EXCEPTION("Failed to sync the db") { } + DB_SYNC_FAILURE(const char* s) : DB_EXCEPTION(s) { } +}; + +class BLOCK_DNE : public DB_EXCEPTION +{ + public: + BLOCK_DNE() : DB_EXCEPTION("The block requested does not exist") { } + BLOCK_DNE(const char* s) : DB_EXCEPTION(s) { } +}; + +class BLOCK_PARENT_DNE : public DB_EXCEPTION +{ + public: + BLOCK_PARENT_DNE() : DB_EXCEPTION("The parent of the block does not exist") { } + BLOCK_PARENT_DNE(const char* s) : DB_EXCEPTION(s) { } +}; + +class BLOCK_EXISTS : public DB_EXCEPTION +{ + public: + BLOCK_EXISTS() : DB_EXCEPTION("The block to be added already exists!") { } + BLOCK_EXISTS(const char* s) : DB_EXCEPTION(s) { } +}; + +class BLOCK_INVALID : public DB_EXCEPTION +{ + public: + BLOCK_INVALID() : DB_EXCEPTION("The block to be added did not pass validation!") { } + BLOCK_INVALID(const char* s) : DB_EXCEPTION(s) { } +}; + +class TX_DNE : public DB_EXCEPTION +{ + public: + TX_DNE() : DB_EXCEPTION("The transaction requested does not exist") { } + TX_DNE(const char* s) : DB_EXCEPTION(s) { } +}; + +class TX_EXISTS : public DB_EXCEPTION +{ + public: + TX_EXISTS() : DB_EXCEPTION("The transaction to be added already exists!") { } + TX_EXISTS(const char* s) : DB_EXCEPTION(s) { } +}; + +class OUTPUT_DNE : public DB_EXCEPTION +{ + public: + OUTPUT_DNE() : DB_EXCEPTION("The output requested does not exist!") { } + OUTPUT_DNE(const char* s) : DB_EXCEPTION(s) { } +}; + +class OUTPUT_EXISTS : public DB_EXCEPTION +{ + public: + OUTPUT_EXISTS() : DB_EXCEPTION("The output to be added already exists!") { } + OUTPUT_EXISTS(const char* s) : DB_EXCEPTION(s) { } +}; + +class KEY_IMAGE_EXISTS : public DB_EXCEPTION +{ + public: + KEY_IMAGE_EXISTS() : DB_EXCEPTION("The spent key image to be added already exists!") { } + KEY_IMAGE_EXISTS(const char* s) : DB_EXCEPTION(s) { } +}; + +/*********************************** + * End of Exception Definitions + ***********************************/ + + +class BlockchainDB +{ +private: + /********************************************************************* + * private virtual members + *********************************************************************/ + + // tells the subclass to add the block and metadata to storage + virtual void add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const crypto::hash& blk_hash + ) = 0; + + // tells the subclass to remove data about the top block + virtual void remove_block() = 0; + + // tells the subclass to store the transaction and its metadata + virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) = 0; + + // tells the subclass to remove data about a transaction + virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) = 0; + + // tells the subclass to store an output + virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index) = 0; + + // tells the subclass to remove an output + virtual void remove_output(const tx_out& tx_output) = 0; + + // tells the subclass to store a spent key + virtual void add_spent_key(const crypto::key_image& k_image) = 0; + + // tells the subclass to remove a spent key + virtual void remove_spent_key(const crypto::key_image& k_image) = 0; + + + /********************************************************************* + * private concrete members + *********************************************************************/ + // private version of pop_block, for undoing if an add_block goes tits up + void pop_block(); + + // helper function for add_transactions, to add each individual tx + void add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr = NULL); + + // helper function to remove transaction from blockchain + void remove_transaction(const crypto::hash& tx_hash); + + uint64_t num_calls = 0; + uint64_t time_blk_hash = 0; + uint64_t time_add_block1 = 0; + uint64_t time_add_transaction = 0; + + +protected: + + mutable uint64_t time_tx_exists = 0; + uint64_t time_commit1 = 0; + + +public: + + // virtual dtor + virtual ~BlockchainDB() { }; + + // reset profiling stats + void reset_stats(); + + // show profiling stats + void show_stats(); + + // open the db at location , or create it if there isn't one. + virtual void open(const std::string& filename) = 0; + + // make sure implementation has a create function as well + virtual void create(const std::string& filename) = 0; + + // close and sync the db + virtual void close() = 0; + + // sync the db + virtual void sync() = 0; + + // reset the db -- USE WITH CARE + virtual void reset() = 0; + + // get all files used by this db (if any) + virtual std::vector get_filenames() const = 0; + + + // FIXME: these are just for functionality mocking, need to implement + // RAII-friendly and multi-read one-write friendly locking mechanism + // + // acquire db lock + virtual bool lock() = 0; + + // release db lock + virtual void unlock() = 0; + + virtual void batch_start() = 0; + virtual void batch_stop() = 0; + virtual void set_batch_transactions(bool) = 0; + + // adds a block with the given metadata to the top of the blockchain, returns the new height + // NOTE: subclass implementations of this (or the functions it calls) need + // to handle undoing any partially-added blocks in the event of a failure. + virtual uint64_t add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const std::vector& txs + ); + + // return true if a block with hash exists in the blockchain + virtual bool block_exists(const crypto::hash& h) const = 0; + + // return block with hash + virtual block get_block(const crypto::hash& h) const = 0; + + // return the height of the block with hash on the blockchain, + // throw if it doesn't exist + virtual uint64_t get_block_height(const crypto::hash& h) const = 0; + + // return header for block with hash + virtual block_header get_block_header(const crypto::hash& h) const = 0; + + // return block at height + virtual block get_block_from_height(const uint64_t& height) const = 0; + + // return timestamp of block at height + virtual uint64_t get_block_timestamp(const uint64_t& height) const = 0; + + // return timestamp of most recent block + virtual uint64_t get_top_block_timestamp() const = 0; + + // return block size of block at height + virtual size_t get_block_size(const uint64_t& height) const = 0; + + // return cumulative difficulty up to and including block at height + virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const = 0; + + // return difficulty of block at height + virtual difficulty_type get_block_difficulty(const uint64_t& height) const = 0; + + // return number of coins generated up to and including block at height + virtual uint64_t get_block_already_generated_coins(const uint64_t& height) const = 0; + + // return hash of block at height + virtual crypto::hash get_block_hash_from_height(const uint64_t& height) const = 0; + + // return vector of blocks in range of height (inclusively) + virtual std::vector get_blocks_range(const uint64_t& h1, const uint64_t& h2) const = 0; + + // return vector of block hashes in range of height (inclusively) + virtual std::vector get_hashes_range(const uint64_t& h1, const uint64_t& h2) const = 0; + + // return the hash of the top block on the chain + virtual crypto::hash top_block_hash() const = 0; + + // return the block at the top of the blockchain + virtual block get_top_block() const = 0; + + // return the height of the chain + virtual uint64_t height() const = 0; + + // pops the top block off the blockchain. + // Returns by reference the popped block and its associated transactions + // + // IMPORTANT: + // When a block is popped, the transactions associated with it need to be + // removed, as well as outputs and spent key images associated with + // those transactions. + virtual void pop_block(block& blk, std::vector& txs); + + + // return true if a transaction with hash exists + virtual bool tx_exists(const crypto::hash& h) const = 0; + + // return unlock time of tx with hash + virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const = 0; + + // return tx with hash + // throw if no such tx exists + virtual transaction get_tx(const crypto::hash& h) const = 0; + + // returns the total number of transactions in all blocks + virtual uint64_t get_tx_count() const = 0; + + // return list of tx with hashes . + // TODO: decide if a missing hash means return empty list + // or just skip that hash + virtual std::vector get_tx_list(const std::vector& hlist) const = 0; + + // returns height of block that contains transaction with hash + virtual uint64_t get_tx_block_height(const crypto::hash& h) const = 0; + + // return global output index of a random output of amount + virtual uint64_t get_random_output(const uint64_t& amount) const = 0; + + // returns the total number of outputs of amount + virtual uint64_t get_num_outputs(const uint64_t& amount) const = 0; + + // return public key for output with global output amount and index + virtual crypto::public_key get_output_key(const uint64_t& amount, const uint64_t& index) const = 0; + + // returns the output indexed by in the transaction with hash + virtual tx_out get_output(const crypto::hash& h, const uint64_t& index) const = 0; + + // returns the tx hash associated with an output, referenced by global output index + virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const = 0; + + // returns the transaction-local reference for the output with at + // return type is pair of tx hash and index + virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const = 0; + + // return a vector of indices corresponding to the global output index for + // each output in the transaction with hash + virtual std::vector get_tx_output_indices(const crypto::hash& h) const = 0; + // return a vector of indices corresponding to the amount output index for + // each output in the transaction with hash + virtual std::vector get_tx_amount_output_indices(const crypto::hash& h) const = 0; + + // returns true if key image is present in spent key images storage + virtual bool has_key_image(const crypto::key_image& img) const = 0; + +}; // class BlockchainDB + + +} // namespace cryptonote + +#endif // BLOCKCHAIN_DB_H diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp new file mode 100644 index 000000000..feff4a272 --- /dev/null +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -0,0 +1,1808 @@ +// Copyright (c) 2014, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "db_lmdb.h" + +#include +#include // std::unique_ptr +#include // memcpy + +#include "cryptonote_core/cryptonote_format_utils.h" +#include "crypto/crypto.h" +#include "profile_tools.h" + +using epee::string_tools::pod_to_hex; + +namespace +{ + +template +inline void throw0(const T &e) +{ + LOG_PRINT_L0(e.what()); + throw e; +} + +template +inline void throw1(const T &e) +{ + LOG_PRINT_L1(e.what()); + throw e; +} + +// cursor needs to be closed when it goes out of scope, +// this helps if the function using it throws +struct lmdb_cur +{ + lmdb_cur(MDB_txn* txn, MDB_dbi dbi) + { + if (mdb_cursor_open(txn, dbi, &m_cur)) + throw0(cryptonote::DB_ERROR("Error opening db cursor")); + done = false; + } + + ~lmdb_cur() { close(); } + + operator MDB_cursor*() { return m_cur; } + operator MDB_cursor**() { return &m_cur; } + + void close() + { + if (!done) + { + mdb_cursor_close(m_cur); + done = true; + } + } + +private: + MDB_cursor* m_cur; + bool done; +}; + +template +struct MDB_val_copy: public MDB_val +{ + MDB_val_copy(const T &t): t_copy(t) + { + mv_size = sizeof (T); + mv_data = &t_copy; + } +private: + T t_copy; +}; + +template<> +struct MDB_val_copy: public MDB_val +{ + MDB_val_copy(const cryptonote::blobdata &bd): data(new char[bd.size()]) + { + memcpy(data.get(), bd.data(), bd.size()); + mv_size = bd.size(); + mv_data = data.get(); + } +private: + std::unique_ptr data; +}; + +auto compare_uint64 = [](const MDB_val *a, const MDB_val *b) { + const uint64_t va = *(const uint64_t*)a->mv_data; + const uint64_t vb = *(const uint64_t*)b->mv_data; + if (va < vb) return -1; + else if (va == vb) return 0; + else return 1; +}; + +const char* const LMDB_BLOCKS = "blocks"; +const char* const LMDB_BLOCK_TIMESTAMPS = "block_timestamps"; +const char* const LMDB_BLOCK_HEIGHTS = "block_heights"; +const char* const LMDB_BLOCK_HASHES = "block_hashes"; +const char* const LMDB_BLOCK_SIZES = "block_sizes"; +const char* const LMDB_BLOCK_DIFFS = "block_diffs"; +const char* const LMDB_BLOCK_COINS = "block_coins"; + +const char* const LMDB_TXS = "txs"; +const char* const LMDB_TX_UNLOCKS = "tx_unlocks"; +const char* const LMDB_TX_HEIGHTS = "tx_heights"; +const char* const LMDB_TX_OUTPUTS = "tx_outputs"; + +const char* const LMDB_OUTPUT_TXS = "output_txs"; +const char* const LMDB_OUTPUT_INDICES = "output_indices"; +const char* const LMDB_OUTPUT_AMOUNTS = "output_amounts"; +const char* const LMDB_OUTPUT_KEYS = "output_keys"; +const char* const LMDB_OUTPUTS = "outputs"; +const char* const LMDB_OUTPUT_GINDICES = "output_gindices"; +const char* const LMDB_SPENT_KEYS = "spent_keys"; + +inline void lmdb_db_open(MDB_txn* txn, const char* name, int flags, MDB_dbi& dbi, const std::string& error_string) +{ + if (mdb_dbi_open(txn, name, flags, &dbi)) + throw0(cryptonote::DB_OPEN_FAILURE(error_string.c_str())); +} + +} // anonymous namespace + +namespace cryptonote +{ + +void BlockchainLMDB::add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const crypto::hash& blk_hash + ) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + MDB_val_copy val_h(blk_hash); + MDB_val unused; + if (mdb_get(*m_write_txn, m_block_heights, &val_h, &unused) == 0) + throw1(BLOCK_EXISTS("Attempting to add block that's already in the db")); + + if (m_height > 0) + { + MDB_val_copy parent_key(blk.prev_id); + MDB_val parent_h; + if (mdb_get(*m_write_txn, m_block_heights, &parent_key, &parent_h)) + { + LOG_PRINT_L3("m_height: " << m_height); + LOG_PRINT_L3("parent_key: " << blk.prev_id); + throw0(DB_ERROR("Failed to get top block hash to check for new block's parent")); + } + uint64_t parent_height = *(const uint64_t *)parent_h.mv_data; + if (parent_height != m_height - 1) + throw0(BLOCK_PARENT_DNE("Top block is not new block's parent")); + } + + MDB_val_copy key(m_height); + + MDB_val_copy blob(block_to_blob(blk)); + auto res = mdb_put(*m_write_txn, m_blocks, &key, &blob, 0); + if (res) + throw0(DB_ERROR(std::string("Failed to add block blob to db transaction: ").append(mdb_strerror(res)).c_str())); + + MDB_val_copy sz(block_size); + if (mdb_put(*m_write_txn, m_block_sizes, &key, &sz, 0)) + throw0(DB_ERROR("Failed to add block size to db transaction")); + + MDB_val_copy ts(blk.timestamp); + if (mdb_put(*m_write_txn, m_block_timestamps, &key, &ts, 0)) + throw0(DB_ERROR("Failed to add block timestamp to db transaction")); + + MDB_val_copy diff(cumulative_difficulty); + if (mdb_put(*m_write_txn, m_block_diffs, &key, &diff, 0)) + throw0(DB_ERROR("Failed to add block cumulative difficulty to db transaction")); + + MDB_val_copy coinsgen(coins_generated); + if (mdb_put(*m_write_txn, m_block_coins, &key, &coinsgen, 0)) + throw0(DB_ERROR("Failed to add block total generated coins to db transaction")); + + if (mdb_put(*m_write_txn, m_block_heights, &val_h, &key, 0)) + throw0(DB_ERROR("Failed to add block height by hash to db transaction")); + + if (mdb_put(*m_write_txn, m_block_hashes, &key, &val_h, 0)) + throw0(DB_ERROR("Failed to add block hash to db transaction")); + +} + +void BlockchainLMDB::remove_block() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + if (m_height == 0) + throw0(BLOCK_DNE ("Attempting to remove block from an empty blockchain")); + + MDB_val_copy k(m_height - 1); + MDB_val h; + if (mdb_get(*m_write_txn, m_block_hashes, &k, &h)) + throw1(BLOCK_DNE("Attempting to remove block that's not in the db")); + + if (mdb_del(*m_write_txn, m_blocks, &k, NULL)) + throw1(DB_ERROR("Failed to add removal of block to db transaction")); + + if (mdb_del(*m_write_txn, m_block_sizes, &k, NULL)) + throw1(DB_ERROR("Failed to add removal of block size to db transaction")); + + if (mdb_del(*m_write_txn, m_block_diffs, &k, NULL)) + throw1(DB_ERROR("Failed to add removal of block cumulative difficulty to db transaction")); + + if (mdb_del(*m_write_txn, m_block_coins, &k, NULL)) + throw1(DB_ERROR("Failed to add removal of block total generated coins to db transaction")); + + if (mdb_del(*m_write_txn, m_block_timestamps, &k, NULL)) + throw1(DB_ERROR("Failed to add removal of block timestamp to db transaction")); + + if (mdb_del(*m_write_txn, m_block_heights, &h, NULL)) + throw1(DB_ERROR("Failed to add removal of block height by hash to db transaction")); + + if (mdb_del(*m_write_txn, m_block_hashes, &k, NULL)) + throw1(DB_ERROR("Failed to add removal of block hash to db transaction")); +} + +void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + MDB_val_copy val_h(tx_hash); + MDB_val unused; + if (mdb_get(*m_write_txn, m_txs, &val_h, &unused) == 0) + throw1(TX_EXISTS("Attempting to add transaction that's already in the db")); + + MDB_val_copy blob(tx_to_blob(tx)); + if (mdb_put(*m_write_txn, m_txs, &val_h, &blob, 0)) + throw0(DB_ERROR("Failed to add tx blob to db transaction")); + + MDB_val_copy height(m_height); + if (mdb_put(*m_write_txn, m_tx_heights, &val_h, &height, 0)) + throw0(DB_ERROR("Failed to add tx block height to db transaction")); + + MDB_val_copy unlock_time(tx.unlock_time); + if (mdb_put(*m_write_txn, m_tx_unlocks, &val_h, &unlock_time, 0)) + throw0(DB_ERROR("Failed to add tx unlock time to db transaction")); +} + +void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + MDB_val_copy val_h(tx_hash); + MDB_val unused; + if (mdb_get(*m_write_txn, m_txs, &val_h, &unused)) + throw1(TX_DNE("Attempting to remove transaction that isn't in the db")); + + if (mdb_del(*m_write_txn, m_txs, &val_h, NULL)) + throw1(DB_ERROR("Failed to add removal of tx to db transaction")); + if (mdb_del(*m_write_txn, m_tx_unlocks, &val_h, NULL)) + throw1(DB_ERROR("Failed to add removal of tx unlock time to db transaction")); + if (mdb_del(*m_write_txn, m_tx_heights, &val_h, NULL)) + throw1(DB_ERROR("Failed to add removal of tx block height to db transaction")); + + remove_tx_outputs(tx_hash, tx); + + if (mdb_del(*m_write_txn, m_tx_outputs, &val_h, NULL)) + throw1(DB_ERROR("Failed to add removal of tx outputs to db transaction")); + +} + +void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + MDB_val_copy k(m_num_outputs); + MDB_val_copy v(tx_hash); + + if (mdb_put(*m_write_txn, m_output_txs, &k, &v, 0)) + throw0(DB_ERROR("Failed to add output tx hash to db transaction")); + if (mdb_put(*m_write_txn, m_tx_outputs, &v, &k, 0)) + throw0(DB_ERROR("Failed to add tx output index to db transaction")); + + MDB_val_copy val_local_index(local_index); + if (mdb_put(*m_write_txn, m_output_indices, &k, &val_local_index, 0)) + throw0(DB_ERROR("Failed to add tx output index to db transaction")); + + MDB_val_copy val_amount(tx_output.amount); + if (auto result = mdb_put(*m_write_txn, m_output_amounts, &val_amount, &k, 0)) + throw0(DB_ERROR(std::string("Failed to add output amount to db transaction").append(mdb_strerror(result)).c_str())); + + if (tx_output.target.type() == typeid(txout_to_key)) + { + MDB_val_copy val_pubkey(boost::get(tx_output.target).key); + if (mdb_put(*m_write_txn, m_output_keys, &k, &val_pubkey, 0)) + throw0(DB_ERROR("Failed to add output pubkey to db transaction")); + } + + +/****** Uncomment if ever outputs actually need to be stored in this manner + * + blobdata b = output_to_blob(tx_output); + + v.mv_size = b.size(); + v.mv_data = &b; + if (mdb_put(*m_write_txn, m_outputs, &k, &v, 0)) + throw0(DB_ERROR("Failed to add output to db transaction")); + if (mdb_put(*m_write_txn, m_output_gindices, &v, &k, 0)) + throw0(DB_ERROR("Failed to add output global index to db transaction")); +************************************************************************/ + + m_num_outputs++; +} + +void BlockchainLMDB::remove_tx_outputs(const crypto::hash& tx_hash, const transaction& tx) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + + lmdb_cur cur(*m_write_txn, m_tx_outputs); + + MDB_val_copy k(tx_hash); + MDB_val v; + + auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + { + LOG_ERROR("Attempting to remove a tx's outputs, but none found. Continuing, but...be wary, because that's weird."); + } + else if (result) + { + throw0(DB_ERROR("DB error attempting to get an output")); + } + else + { + size_t num_elems = 0; + mdb_cursor_count(cur, &num_elems); + + mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); + + for (uint64_t i = 0; i < num_elems; ++i) + { + const tx_out tx_output = tx.vout[i]; + remove_output(*(const uint64_t*)v.mv_data, tx_output.amount); + if (i < num_elems - 1) + { + mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); + } + } + } + + cur.close(); +} + +// TODO: probably remove this function +void BlockchainLMDB::remove_output(const tx_out& tx_output) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " (unused version - does nothing)"); + return; +} + +void BlockchainLMDB::remove_output(const uint64_t& out_index, const uint64_t amount) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + MDB_val_copy k(out_index); + +/****** Uncomment if ever outputs actually need to be stored in this manner + blobdata b; + t_serializable_object_to_blob(tx_output, b); + k.mv_size = b.size(); + k.mv_data = &b; + + if (mdb_get(*m_write_txn, m_output_gindices, &k, &v)) + throw1(OUTPUT_DNE("Attempting to remove output that does not exist")); + + uint64_t gindex = *(uint64_t*)v.mv_data; + + auto result = mdb_del(*m_write_txn, m_output_gindices, &k, NULL); + if (result != 0 && result != MDB_NOTFOUND) + throw1(DB_ERROR("Error adding removal of output global index to db transaction")); + + result = mdb_del(*m_write_txn, m_outputs, &v, NULL); + if (result != 0 && result != MDB_NOTFOUND) + throw1(DB_ERROR("Error adding removal of output to db transaction")); +*********************************************************************/ + + auto result = mdb_del(*m_write_txn, m_output_indices, &k, NULL); + if (result == MDB_NOTFOUND) + { + LOG_PRINT_L0("Unexpected: global output index not found in m_output_indices"); + } + else if (result) + { + throw1(DB_ERROR("Error adding removal of output tx index to db transaction")); + } + + result = mdb_del(*m_write_txn, m_output_txs, &k, NULL); + // if (result != 0 && result != MDB_NOTFOUND) + // throw1(DB_ERROR("Error adding removal of output tx hash to db transaction")); + if (result == MDB_NOTFOUND) + { + LOG_PRINT_L0("Unexpected: global output index not found in m_output_txs"); + } + else if (result) + { + throw1(DB_ERROR("Error adding removal of output tx hash to db transaction")); + } + + result = mdb_del(*m_write_txn, m_output_keys, &k, NULL); + if (result == MDB_NOTFOUND) + { + LOG_PRINT_L0("Unexpected: global output index not found in m_output_keys"); + } + else if (result) + throw1(DB_ERROR("Error adding removal of output pubkey to db transaction")); + + remove_amount_output_index(amount, out_index); + + m_num_outputs--; +} + +void BlockchainLMDB::remove_amount_output_index(const uint64_t amount, const uint64_t global_output_index) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + lmdb_cur cur(*m_write_txn, m_output_amounts); + + MDB_val_copy k(amount); + MDB_val v; + + auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + size_t num_elems = 0; + mdb_cursor_count(cur, &num_elems); + + mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); + + uint64_t amount_output_index = 0; + uint64_t goi = 0; + bool found_index = false; + for (uint64_t i = 0; i < num_elems; ++i) + { + mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); + goi = *(const uint64_t *)v.mv_data; + if (goi == global_output_index) + { + amount_output_index = i; + found_index = true; + break; + } + mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); + } + if (found_index) + { + // found the amount output index + // now delete it + result = mdb_cursor_del(cur, 0); + if (result) + throw0(DB_ERROR(std::string("Error deleting amount output index ").append(boost::lexical_cast(amount_output_index)).c_str())); + } + else + { + // not found + cur.close(); + throw1(OUTPUT_DNE("Failed to find amount output index")); + } + cur.close(); +} + +void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + MDB_val_copy val_key(k_image); + MDB_val unused; + if (mdb_get(*m_write_txn, m_spent_keys, &val_key, &unused) == 0) + throw1(KEY_IMAGE_EXISTS("Attempting to add spent key image that's already in the db")); + + char anything = '\0'; + unused.mv_size = sizeof(char); + unused.mv_data = &anything; + if (auto result = mdb_put(*m_write_txn, m_spent_keys, &val_key, &unused, 0)) + throw1(DB_ERROR(std::string("Error adding spent key image to db transaction: ").append(mdb_strerror(result)).c_str())); +} + +void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + MDB_val_copy k(k_image); + auto result = mdb_del(*m_write_txn, m_spent_keys, &k, NULL); + if (result != 0 && result != MDB_NOTFOUND) + throw1(DB_ERROR("Error adding removal of key image to db transaction")); +} + +blobdata BlockchainLMDB::output_to_blob(const tx_out& output) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + blobdata b; + if (!t_serializable_object_to_blob(output, b)) + throw1(DB_ERROR("Error serializing output to blob")); + return b; +} + +tx_out BlockchainLMDB::output_from_blob(const blobdata& blob) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + std::stringstream ss; + ss << blob; + binary_archive ba(ss); + tx_out o; + + if (!(::serialization::serialize(ba, o))) + throw1(DB_ERROR("Error deserializing tx output blob")); + + return o; +} + +uint64_t BlockchainLMDB::get_output_global_index(const uint64_t& amount, const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + lmdb_cur cur(txn, m_output_amounts); + + MDB_val_copy k(amount); + MDB_val v; + + auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + size_t num_elems = 0; + mdb_cursor_count(cur, &num_elems); + if (num_elems <= index) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found")); + + mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); + + for (uint64_t i = 0; i < index; ++i) + { + mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); + } + + mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); + + uint64_t glob_index = *(const uint64_t*)v.mv_data; + + cur.close(); + + txn.commit(); + + return glob_index; +} + +void BlockchainLMDB::check_open() const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if (!m_open) + throw0(DB_ERROR("DB operation attempted on a not-open DB instance")); +} + +BlockchainLMDB::~BlockchainLMDB() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + + // batch transaction shouldn't be active at this point. If it is, consider it aborted. + if (m_batch_active) + batch_abort(); +} + +BlockchainLMDB::BlockchainLMDB(bool batch_transactions) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + // initialize folder to something "safe" just in case + // someone accidentally misuses this class... + m_folder = "thishsouldnotexistbecauseitisgibberish"; + m_open = false; + + m_batch_transactions = batch_transactions; + m_write_txn = nullptr; + m_batch_active = false; + m_height = 0; +} + +void BlockchainLMDB::open(const std::string& filename) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + + if (m_open) + throw0(DB_OPEN_FAILURE("Attempted to open db, but it's already open")); + + boost::filesystem::path direc(filename); + if (boost::filesystem::exists(direc)) + { + if (!boost::filesystem::is_directory(direc)) + throw0(DB_OPEN_FAILURE("LMDB needs a directory path, but a file was passed")); + } + else + { + if (!boost::filesystem::create_directory(direc)) + throw0(DB_OPEN_FAILURE(std::string("Failed to create directory ").append(filename).c_str())); + } + + // check for existing LMDB files in base directory + boost::filesystem::path old_files = direc.parent_path(); + if (boost::filesystem::exists(old_files / "data.mdb") || + boost::filesystem::exists(old_files / "lock.mdb")) + { + LOG_PRINT_L0("Found existing LMDB files in " << old_files.c_str()); + LOG_PRINT_L0("Move data.mdb and/or lock.mdb to " << filename << ", or delete them, and then restart"); + throw DB_ERROR("Database could not be opened"); + } + + m_folder = filename; + + // set up lmdb environment + if (mdb_env_create(&m_env)) + throw0(DB_ERROR("Failed to create lmdb environment")); + if (mdb_env_set_maxdbs(m_env, 20)) + throw0(DB_ERROR("Failed to set max number of dbs")); + + size_t mapsize = 1LL << 34; + if (auto result = mdb_env_set_mapsize(m_env, mapsize)) + throw0(DB_ERROR(std::string("Failed to set max memory map size: ").append(mdb_strerror(result)).c_str())); + if (auto result = mdb_env_open(m_env, filename.c_str(), 0, 0644)) + throw0(DB_ERROR(std::string("Failed to open lmdb environment: ").append(mdb_strerror(result)).c_str())); + + // get a read/write MDB_txn + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, 0, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + // open necessary databases, and set properties as needed + // uses macros to avoid having to change things too many places + lmdb_db_open(txn, LMDB_BLOCKS, MDB_INTEGERKEY | MDB_CREATE, m_blocks, "Failed to open db handle for m_blocks"); + + lmdb_db_open(txn, LMDB_BLOCK_TIMESTAMPS, MDB_INTEGERKEY | MDB_CREATE, m_block_timestamps, "Failed to open db handle for m_block_timestamps"); + lmdb_db_open(txn, LMDB_BLOCK_HEIGHTS, MDB_CREATE, m_block_heights, "Failed to open db handle for m_block_heights"); + lmdb_db_open(txn, LMDB_BLOCK_HASHES, MDB_INTEGERKEY | MDB_CREATE, m_block_hashes, "Failed to open db handle for m_block_hashes"); + lmdb_db_open(txn, LMDB_BLOCK_SIZES, MDB_INTEGERKEY | MDB_CREATE, m_block_sizes, "Failed to open db handle for m_block_sizes"); + lmdb_db_open(txn, LMDB_BLOCK_DIFFS, MDB_INTEGERKEY | MDB_CREATE, m_block_diffs, "Failed to open db handle for m_block_diffs"); + lmdb_db_open(txn, LMDB_BLOCK_COINS, MDB_INTEGERKEY | MDB_CREATE, m_block_coins, "Failed to open db handle for m_block_coins"); + + lmdb_db_open(txn, LMDB_TXS, MDB_CREATE, m_txs, "Failed to open db handle for m_txs"); + lmdb_db_open(txn, LMDB_TX_UNLOCKS, MDB_CREATE, m_tx_unlocks, "Failed to open db handle for m_tx_unlocks"); + lmdb_db_open(txn, LMDB_TX_HEIGHTS, MDB_CREATE, m_tx_heights, "Failed to open db handle for m_tx_heights"); + lmdb_db_open(txn, LMDB_TX_OUTPUTS, MDB_DUPSORT | MDB_CREATE, m_tx_outputs, "Failed to open db handle for m_tx_outputs"); + + lmdb_db_open(txn, LMDB_OUTPUT_TXS, MDB_INTEGERKEY | MDB_CREATE, m_output_txs, "Failed to open db handle for m_output_txs"); + lmdb_db_open(txn, LMDB_OUTPUT_INDICES, MDB_INTEGERKEY | MDB_CREATE, m_output_indices, "Failed to open db handle for m_output_indices"); + lmdb_db_open(txn, LMDB_OUTPUT_AMOUNTS, MDB_INTEGERKEY | MDB_DUPSORT | MDB_CREATE, m_output_amounts, "Failed to open db handle for m_output_amounts"); + lmdb_db_open(txn, LMDB_OUTPUT_KEYS, MDB_INTEGERKEY | MDB_CREATE, m_output_keys, "Failed to open db handle for m_output_keys"); + +/*************** not used, but kept for posterity + lmdb_db_open(txn, LMDB_OUTPUTS, MDB_INTEGERKEY | MDB_CREATE, m_outputs, "Failed to open db handle for m_outputs"); + lmdb_db_open(txn, LMDB_OUTPUT_GINDICES, MDB_CREATE, m_output_gindices, "Failed to open db handle for m_output_gindices"); +*************************************************/ + + lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_CREATE, m_spent_keys, "Failed to open db handle for m_spent_keys"); + + mdb_set_dupsort(txn, m_output_amounts, compare_uint64); + mdb_set_dupsort(txn, m_tx_outputs, compare_uint64); + + // get and keep current height + MDB_stat db_stats; + if (mdb_stat(txn, m_blocks, &db_stats)) + throw0(DB_ERROR("Failed to query m_blocks")); + LOG_PRINT_L2("Setting m_height to: " << db_stats.ms_entries); + m_height = db_stats.ms_entries; + + // get and keep current number of outputs + if (mdb_stat(txn, m_output_indices, &db_stats)) + throw0(DB_ERROR("Failed to query m_output_indices")); + m_num_outputs = db_stats.ms_entries; + + // commit the transaction + txn.commit(); + + m_open = true; + // from here, init should be finished +} + +// unused for now, create will happen on open if doesn't exist +void BlockchainLMDB::create(const std::string& filename) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + throw DB_CREATE_FAILURE("create() is not implemented for this BlockchainDB, open() will create files if needed."); +} + +void BlockchainLMDB::close() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if (m_batch_active) + { + LOG_PRINT_L3("close() first calling batch_abort() due to active batch transaction"); + batch_abort(); + } + this->sync(); + + // FIXME: not yet thread safe!!! Use with care. + mdb_env_close(m_env); +} + +void BlockchainLMDB::sync() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + // Does nothing unless LMDB environment was opened with MDB_NOSYNC or in part + // MDB_NOMETASYNC. Force flush to be synchronous. + if (auto result = mdb_env_sync(m_env, true)) + { + throw0(DB_ERROR(std::string("Failed to sync database").append(mdb_strerror(result)).c_str())); + } +} + +void BlockchainLMDB::reset() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + // TODO: this +} + +std::vector BlockchainLMDB::get_filenames() const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + std::vector filenames; + + boost::filesystem::path datafile(m_folder); + datafile /= "data.mdb"; + boost::filesystem::path lockfile(m_folder); + lockfile /= "lock.mdb"; + + filenames.push_back(datafile.string()); + filenames.push_back(lockfile.string()); + + return filenames; +} + +// TODO: this? +bool BlockchainLMDB::lock() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + return false; +} + +// TODO: this? +void BlockchainLMDB::unlock() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); +} + +bool BlockchainLMDB::block_exists(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + MDB_val_copy key(h); + MDB_val result; + auto get_result = mdb_get(txn, m_block_heights, &key, &result); + if (get_result == MDB_NOTFOUND) + { + txn.commit(); + LOG_PRINT_L3("Block with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); + return false; + } + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch block index from hash")); + + txn.commit(); + return true; +} + +block BlockchainLMDB::get_block(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + return get_block_from_height(get_block_height(h)); +} + +uint64_t BlockchainLMDB::get_block_height(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + MDB_val_copy key(h); + MDB_val result; + auto get_result = mdb_get(txn, m_block_heights, &key, &result); + if (get_result == MDB_NOTFOUND) + throw1(BLOCK_DNE("Attempted to retrieve non-existent block height")); + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a block height from the db")); + + txn.commit(); + return *(const uint64_t*)result.mv_data; +} + +block_header BlockchainLMDB::get_block_header(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + // block_header object is automatically cast from block object + return get_block(h); +} + +block BlockchainLMDB::get_block_from_height(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + MDB_val_copy key(height); + MDB_val result; + auto get_result = mdb_get(txn, m_blocks, &key, &result); + if (get_result == MDB_NOTFOUND) + { + throw0(DB_ERROR(std::string("Attempt to get block from height ").append(boost::lexical_cast(height)).append(" failed -- block not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a block from the db")); + + txn.commit(); + + blobdata bd; + bd.assign(reinterpret_cast(result.mv_data), result.mv_size); + + block b; + if (!parse_and_validate_block_from_blob(bd, b)) + throw0(DB_ERROR("Failed to parse block from blob retrieved from the db")); + + return b; +} + +uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + txn_safe txn; + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } + MDB_val_copy key(height); + MDB_val result; + auto get_result = mdb_get(*txn_ptr, m_block_timestamps, &key, &result); + if (get_result == MDB_NOTFOUND) + { + throw0(DB_ERROR(std::string("Attempt to get timestamp from height ").append(boost::lexical_cast(height)).append(" failed -- timestamp not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a timestamp from the db")); + + if (! m_batch_active) + txn.commit(); + return *(const uint64_t*)result.mv_data; +} + +uint64_t BlockchainLMDB::get_top_block_timestamp() const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + // if no blocks, return 0 + if (m_height == 0) + { + return 0; + } + + return get_block_timestamp(m_height - 1); +} + +size_t BlockchainLMDB::get_block_size(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + txn_safe txn; + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } + + MDB_val_copy key(height); + MDB_val result; + auto get_result = mdb_get(*txn_ptr, m_block_sizes, &key, &result); + if (get_result == MDB_NOTFOUND) + { + throw0(DB_ERROR(std::string("Attempt to get block size from height ").append(boost::lexical_cast(height)).append(" failed -- block size not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a block size from the db")); + + if (! m_batch_active) + txn.commit(); + return *(const size_t*)result.mv_data; +} + +difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " height: " << height); + check_open(); + + txn_safe txn; + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } + MDB_val_copy key(height); + MDB_val result; + auto get_result = mdb_get(*txn_ptr, m_block_diffs, &key, &result); + if (get_result == MDB_NOTFOUND) + { + throw0(DB_ERROR(std::string("Attempt to get cumulative difficulty from height ").append(boost::lexical_cast(height)).append(" failed -- difficulty not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a cumulative difficulty from the db")); + + if (! m_batch_active) + txn.commit(); + return *(difficulty_type*)result.mv_data; +} + +difficulty_type BlockchainLMDB::get_block_difficulty(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + difficulty_type diff1 = 0; + difficulty_type diff2 = 0; + + diff1 = get_block_cumulative_difficulty(height); + if (height != 0) + { + diff2 = get_block_cumulative_difficulty(height - 1); + } + + return diff1 - diff2; +} + +uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + txn_safe txn; + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } + + MDB_val_copy key(height); + MDB_val result; + auto get_result = mdb_get(*txn_ptr, m_block_coins, &key, &result); + if (get_result == MDB_NOTFOUND) + { + throw0(DB_ERROR(std::string("Attempt to get generated coins from height ").append(boost::lexical_cast(height)).append(" failed -- block size not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a total generated coins from the db")); + + if (! m_batch_active) + txn.commit(); + return *(const uint64_t*)result.mv_data; +} + +crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + txn_safe txn; + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } + + MDB_val_copy key(height); + MDB_val result; + auto get_result = mdb_get(*txn_ptr, m_block_hashes, &key, &result); + if (get_result == MDB_NOTFOUND) + { + throw0(BLOCK_DNE(std::string("Attempt to get hash from height ").append(boost::lexical_cast(height)).append(" failed -- hash not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR(std::string("Error attempting to retrieve a block hash from the db: "). + append(mdb_strerror(get_result)).c_str())); + + if (! m_batch_active) + txn.commit(); + return *(crypto::hash*)result.mv_data; +} + +std::vector BlockchainLMDB::get_blocks_range(const uint64_t& h1, const uint64_t& h2) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + std::vector v; + + for (uint64_t height = h1; height <= h2; ++height) + { + v.push_back(get_block_from_height(height)); + } + + return v; +} + +std::vector BlockchainLMDB::get_hashes_range(const uint64_t& h1, const uint64_t& h2) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + std::vector v; + + for (uint64_t height = h1; height <= h2; ++height) + { + v.push_back(get_block_hash_from_height(height)); + } + + return v; +} + +crypto::hash BlockchainLMDB::top_block_hash() const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + if (m_height != 0) + { + return get_block_hash_from_height(m_height - 1); + } + + return null_hash; +} + +block BlockchainLMDB::get_top_block() const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + if (m_height != 0) + { + return get_block_from_height(m_height - 1); + } + + block b; + return b; +} + +uint64_t BlockchainLMDB::height() const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + return m_height; +} + +bool BlockchainLMDB::tx_exists(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + txn_safe txn; + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } + + MDB_val_copy key(h); + MDB_val result; + + TIME_MEASURE_START(time1); + auto get_result = mdb_get(*txn_ptr, m_txs, &key, &result); + TIME_MEASURE_FINISH(time1); + time_tx_exists += time1; + if (get_result == MDB_NOTFOUND) + { + if (! m_batch_active) + txn.commit(); + LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); + return false; + } + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch transaction from hash")); + + return true; +} + +uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + MDB_val_copy key(h); + MDB_val result; + auto get_result = mdb_get(txn, m_tx_unlocks, &key, &result); + if (get_result == MDB_NOTFOUND) + throw1(TX_DNE(std::string("tx unlock time with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch tx unlock time from hash")); + + return *(const uint64_t*)result.mv_data; +} + +transaction BlockchainLMDB::get_tx(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + txn_safe txn; + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } + + MDB_val_copy key(h); + MDB_val result; + auto get_result = mdb_get(*txn_ptr, m_txs, &key, &result); + if (get_result == MDB_NOTFOUND) + throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch tx from hash")); + + blobdata bd; + bd.assign(reinterpret_cast(result.mv_data), result.mv_size); + + transaction tx; + if (!parse_and_validate_tx_from_blob(bd, tx)) + throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); + if (! m_batch_active) + txn.commit(); + + return tx; +} + +uint64_t BlockchainLMDB::get_tx_count() const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + MDB_stat db_stats; + if (mdb_stat(txn, m_txs, &db_stats)) + throw0(DB_ERROR("Failed to query m_txs")); + + txn.commit(); + + return db_stats.ms_entries; +} + +std::vector BlockchainLMDB::get_tx_list(const std::vector& hlist) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + std::vector v; + + for (auto& h : hlist) + { + v.push_back(get_tx(h)); + } + + return v; +} + +uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + // If m_batch_active is set, a batch transaction exists beyond this class, + // such as a batch import with verification enabled, or possibly (later) a + // batch network sync. + // + // A regular network sync without batching would be expected to open a new + // read transaction here, as validation is done prior to the write for block + // and tx data. + + txn_safe txn; + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } + + MDB_val_copy key(h); + MDB_val result; + auto get_result = mdb_get(*txn_ptr, m_tx_heights, &key, &result); + if (get_result == MDB_NOTFOUND) + { + throw1(TX_DNE(std::string("tx height with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch tx height from hash")); + + if (! m_batch_active) + txn.commit(); + + return *(const uint64_t*)result.mv_data; +} + +//FIXME: make sure the random method used here is appropriate +uint64_t BlockchainLMDB::get_random_output(const uint64_t& amount) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + uint64_t num_outputs = get_num_outputs(amount); + if (num_outputs == 0) + throw1(OUTPUT_DNE("Attempting to get a random output for an amount, but none exist")); + + return crypto::rand() % num_outputs; +} + +uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + lmdb_cur cur(txn, m_output_amounts); + + MDB_val_copy k(amount); + MDB_val v; + auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + { + return 0; + } + else if (result) + throw0(DB_ERROR("DB error attempting to get number of outputs of an amount")); + + size_t num_elems = 0; + mdb_cursor_count(cur, &num_elems); + + txn.commit(); + + return num_elems; +} + +crypto::public_key BlockchainLMDB::get_output_key(const uint64_t& amount, const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + uint64_t glob_index = get_output_global_index(amount, index); + + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + MDB_val_copy k(glob_index); + MDB_val v; + auto get_result = mdb_get(txn, m_output_keys, &k, &v); + if (get_result == MDB_NOTFOUND) + throw0(DB_ERROR("Attempting to get output pubkey by global index, but key does not exist")); + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve an output pubkey from the db")); + + return *(crypto::public_key*)v.mv_data; +} + +tx_out BlockchainLMDB::get_output(const crypto::hash& h, const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + lmdb_cur cur(txn, m_tx_outputs); + + MDB_val_copy k(h); + MDB_val v; + auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output by tx hash and tx index, but output not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + size_t num_elems = 0; + mdb_cursor_count(cur, &num_elems); + if (num_elems <= index) + throw1(OUTPUT_DNE("Attempting to get an output by tx hash and tx index, but output not found")); + + mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); + + for (uint64_t i = 0; i < index; ++i) + { + mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); + } + + mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); + + blobdata b; + b = *(blobdata*)v.mv_data; + + cur.close(); + txn.commit(); + + return output_from_blob(b); +} + +// As this is not used, its return is now a blank output. +// This will save on space in the db. +tx_out BlockchainLMDB::get_output(const uint64_t& index) const +{ + return tx_out(); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + MDB_val_copy k(index); + MDB_val v; + auto get_result = mdb_get(txn, m_outputs, &k, &v); + if (get_result == MDB_NOTFOUND) + { + throw OUTPUT_DNE("Attempting to get output by global index, but output does not exist"); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve an output from the db")); + + blobdata b = *(blobdata*)v.mv_data; + + return output_from_blob(b); +} + +tx_out_index BlockchainLMDB::get_output_tx_and_index_from_global(const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + txn_safe txn; + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } + MDB_val_copy k(index); + MDB_val v; + + auto get_result = mdb_get(*txn_ptr, m_output_txs, &k, &v); + if (get_result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("output with given index not in db")); + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch output tx hash")); + + crypto::hash tx_hash = *(crypto::hash*)v.mv_data; + + get_result = mdb_get(*txn_ptr, m_output_indices, &k, &v); + if (get_result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("output with given index not in db")); + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch output tx index")); + if (! m_batch_active) + txn.commit(); + + return tx_out_index(tx_hash, *(const uint64_t *)v.mv_data); +} + +tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + txn_safe txn; + txn_safe* txn_ptr = &txn; + if (m_batch_active) + txn_ptr = m_write_txn; + else + { + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } + lmdb_cur cur(*txn_ptr, m_output_amounts); + + MDB_val_copy k(amount); + MDB_val v; + + auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + size_t num_elems = 0; + mdb_cursor_count(cur, &num_elems); + if (num_elems <= index) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found")); + + mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); + + for (uint64_t i = 0; i < index; ++i) + { + mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); + } + + mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); + + uint64_t glob_index = *(const uint64_t*)v.mv_data; + + cur.close(); + + if (! m_batch_active) + txn.commit(); + + return get_output_tx_and_index_from_global(glob_index); +} + +std::vector BlockchainLMDB::get_tx_output_indices(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + std::vector index_vec; + + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + lmdb_cur cur(txn, m_tx_outputs); + + MDB_val_copy k(h); + MDB_val v; + auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output by tx hash and tx index, but output not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + size_t num_elems = 0; + mdb_cursor_count(cur, &num_elems); + + mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); + + for (uint64_t i = 0; i < num_elems; ++i) + { + mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); + index_vec.push_back(*(const uint64_t *)v.mv_data); + mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); + } + + cur.close(); + txn.commit(); + + return index_vec; +} + +std::vector BlockchainLMDB::get_tx_amount_output_indices(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + std::vector index_vec; + std::vector index_vec2; + + // get the transaction's global output indices first + index_vec = get_tx_output_indices(h); + // these are next used to obtain the amount output indices + + transaction tx = get_tx(h); + + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + uint64_t i = 0; + uint64_t global_index; + BOOST_FOREACH(const auto& vout, tx.vout) + { + uint64_t amount = vout.amount; + + global_index = index_vec[i]; + + lmdb_cur cur(txn, m_output_amounts); + + MDB_val_copy k(amount); + MDB_val v; + + auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + size_t num_elems = 0; + mdb_cursor_count(cur, &num_elems); + + mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); + + uint64_t amount_output_index = 0; + uint64_t output_index = 0; + bool found_index = false; + for (uint64_t j = 0; j < num_elems; ++j) + { + mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); + output_index = *(const uint64_t *)v.mv_data; + if (output_index == global_index) + { + amount_output_index = j; + found_index = true; + break; + } + mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); + } + if (found_index) + { + index_vec2.push_back(amount_output_index); + } + else + { + // not found + cur.close(); + txn.commit(); + throw1(OUTPUT_DNE("specified output not found in db")); + } + + cur.close(); + ++i; + } + + txn.commit(); + + return index_vec2; +} + + + +bool BlockchainLMDB::has_key_image(const crypto::key_image& img) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + txn_safe txn; + if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + MDB_val_copy val_key(img); + MDB_val unused; + if (mdb_get(txn, m_spent_keys, &val_key, &unused) == 0) + { + txn.commit(); + return true; + } + + txn.commit(); + return false; +} + +void BlockchainLMDB::batch_start() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if (! m_batch_transactions) + throw0(DB_ERROR("batch transactions not enabled")); + if (m_batch_active) + throw0(DB_ERROR("batch transaction already in progress")); + if (m_write_txn) + throw0(DB_ERROR("batch transaction attempted, but m_write_txn already in use")); + check_open(); + // NOTE: need to make sure it's destroyed properly when done + if (mdb_txn_begin(m_env, NULL, 0, m_write_batch_txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + // indicates this transaction is for batch transactions, but not whether it's + // active + m_write_batch_txn.m_batch_txn = true; + m_write_txn = &m_write_batch_txn; + m_batch_active = true; + LOG_PRINT_L3("batch transaction: begin"); +} + +void BlockchainLMDB::batch_commit() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if (! m_batch_transactions) + throw0(DB_ERROR("batch transactions not enabled")); + if (! m_batch_active) + throw0(DB_ERROR("batch transaction not in progress")); + check_open(); + LOG_PRINT_L3("batch transaction: committing..."); + TIME_MEASURE_START(time1); + m_write_txn->commit(); + TIME_MEASURE_FINISH(time1); + time_commit1 += time1; + LOG_PRINT_L3("batch transaction: committed"); + + if (mdb_txn_begin(m_env, NULL, 0, m_write_batch_txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + if (! m_write_batch_txn.m_batch_txn) + throw0(DB_ERROR("m_write_batch_txn not marked as a batch transaction")); + m_write_txn = &m_write_batch_txn; +} + +void BlockchainLMDB::batch_stop() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if (! m_batch_transactions) + throw0(DB_ERROR("batch transactions not enabled")); + if (! m_batch_active) + throw0(DB_ERROR("batch transaction not in progress")); + check_open(); + LOG_PRINT_L3("batch transaction: committing..."); + TIME_MEASURE_START(time1); + m_write_txn->commit(); + TIME_MEASURE_FINISH(time1); + time_commit1 += time1; + // for destruction of batch transaction + m_write_txn = nullptr; + m_batch_active = false; + LOG_PRINT_L3("batch transaction: end"); +} + +void BlockchainLMDB::batch_abort() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + if (! m_batch_transactions) + throw0(DB_ERROR("batch transactions not enabled")); + if (! m_batch_active) + throw0(DB_ERROR("batch transaction not in progress")); + check_open(); + // for destruction of batch transaction + m_write_txn = nullptr; + // explicitly call in case mdb_env_close() (BlockchainLMDB::close()) called before BlockchainLMDB destructor called. + m_write_batch_txn.abort(); + m_batch_active = false; + LOG_PRINT_L3("batch transaction: aborted"); +} + +void BlockchainLMDB::set_batch_transactions(bool batch_transactions) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + m_batch_transactions = batch_transactions; + LOG_PRINT_L3("batch transactions " << (m_batch_transactions ? "enabled" : "disabled")); +} + +uint64_t BlockchainLMDB::add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const std::vector& txs + ) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + txn_safe txn; + if (! m_batch_active) + { + if (mdb_txn_begin(m_env, NULL, 0, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + m_write_txn = &txn; + } + + uint64_t num_outputs = m_num_outputs; + try + { + BlockchainDB::add_block(blk, block_size, cumulative_difficulty, coins_generated, txs); + if (! m_batch_active) + { + m_write_txn = NULL; + + TIME_MEASURE_START(time1); + txn.commit(); + TIME_MEASURE_FINISH(time1); + time_commit1 += time1; + } + } + catch (...) + { + m_num_outputs = num_outputs; + if (! m_batch_active) + m_write_txn = NULL; + throw; + } + + return ++m_height; +} + +void BlockchainLMDB::pop_block(block& blk, std::vector& txs) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + txn_safe txn; + if (! m_batch_active) + { + if (mdb_txn_begin(m_env, NULL, 0, txn)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + m_write_txn = &txn; + } + + uint64_t num_outputs = m_num_outputs; + try + { + BlockchainDB::pop_block(blk, txs); + if (! m_batch_active) + { + m_write_txn = NULL; + + txn.commit(); + } + } + catch (...) + { + m_num_outputs = num_outputs; + m_write_txn = NULL; + throw; + } + + --m_height; +} + +} // namespace cryptonote diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h new file mode 100644 index 000000000..f6582fce4 --- /dev/null +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -0,0 +1,314 @@ +// Copyright (c) 2014, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "blockchain_db/blockchain_db.h" +#include "cryptonote_protocol/blobdatatype.h" // for type blobdata + +#include + +namespace cryptonote +{ + +struct txn_safe +{ + txn_safe() : m_txn(NULL) { } + ~txn_safe() + { + LOG_PRINT_L3("txn_safe: destructor"); + if (m_txn != NULL) + { + if (m_batch_txn) // this is a batch txn and should have been handled before this point for safety + { + LOG_PRINT_L0("WARNING: txn_safe: m_txn is a batch txn and it's not NULL in destructor - calling mdb_txn_abort()"); + } + else + { + // Example of when this occurs: a lookup fails, so a read-only txn is + // aborted through this destructor. However, successful read-only txns + // ideally should have been committed when done and not end up here. + // + // NOTE: not sure if this is ever reached for a non-batch write + // transaction, but it's probably not ideal if it did. + LOG_PRINT_L3("txn_safe: m_txn not NULL in destructor - calling mdb_txn_abort()"); + } + mdb_txn_abort(m_txn); + } + } + + void commit(std::string message = "") + { + if (message.size() == 0) + { + message = "Failed to commit a transaction to the db"; + } + + if (mdb_txn_commit(m_txn)) + { + m_txn = NULL; + LOG_PRINT_L0(message); + throw DB_ERROR(message.c_str()); + } + m_txn = NULL; + } + + // This should only be needed for batch transaction which must be ensured to + // be aborted before mdb_env_close, not after. So we can't rely on + // BlockchainLMDB destructor to call txn_safe destructor, as that's too late + // to properly abort, since mdb_env_close would have been called earlier. + void abort() + { + LOG_PRINT_L3("txn_safe: abort()"); + if(m_txn != NULL) + { + mdb_txn_abort(m_txn); + m_txn = NULL; + } + else + { + LOG_PRINT_L0("WARNING: txn_safe: abort() called, but m_txn is NULL"); + } + } + + operator MDB_txn*() + { + return m_txn; + } + + operator MDB_txn**() + { + return &m_txn; + } + + MDB_txn* m_txn; + bool m_batch_txn = false; +}; + + +class BlockchainLMDB : public BlockchainDB +{ +public: + BlockchainLMDB(bool batch_transactions=false); + ~BlockchainLMDB(); + + virtual void open(const std::string& filename); + + virtual void create(const std::string& filename); + + virtual void close(); + + virtual void sync(); + + virtual void reset(); + + virtual std::vector get_filenames() const; + + virtual bool lock(); + + virtual void unlock(); + + virtual bool block_exists(const crypto::hash& h) const; + + virtual block get_block(const crypto::hash& h) const; + + virtual uint64_t get_block_height(const crypto::hash& h) const; + + virtual block_header get_block_header(const crypto::hash& h) const; + + virtual block get_block_from_height(const uint64_t& height) const; + + virtual uint64_t get_block_timestamp(const uint64_t& height) const; + + virtual uint64_t get_top_block_timestamp() const; + + virtual size_t get_block_size(const uint64_t& height) const; + + virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const; + + virtual difficulty_type get_block_difficulty(const uint64_t& height) const; + + virtual uint64_t get_block_already_generated_coins(const uint64_t& height) const; + + virtual crypto::hash get_block_hash_from_height(const uint64_t& height) const; + + virtual std::vector get_blocks_range(const uint64_t& h1, const uint64_t& h2) const; + + virtual std::vector get_hashes_range(const uint64_t& h1, const uint64_t& h2) const; + + virtual crypto::hash top_block_hash() const; + + virtual block get_top_block() const; + + virtual uint64_t height() const; + + virtual bool tx_exists(const crypto::hash& h) const; + + virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const; + + virtual transaction get_tx(const crypto::hash& h) const; + + virtual uint64_t get_tx_count() const; + + virtual std::vector get_tx_list(const std::vector& hlist) const; + + virtual uint64_t get_tx_block_height(const crypto::hash& h) const; + + virtual uint64_t get_random_output(const uint64_t& amount) const; + + virtual uint64_t get_num_outputs(const uint64_t& amount) const; + + virtual crypto::public_key get_output_key(const uint64_t& amount, const uint64_t& index) const; + + virtual tx_out get_output(const crypto::hash& h, const uint64_t& index) const; + + /** + * @brief get an output from its global index + * + * @param index global index of the output desired + * + * @return the output associated with the index. + * Will throw OUTPUT_DNE if not output has that global index. + * Will throw DB_ERROR if there is a non-specific LMDB error in fetching + */ + tx_out get_output(const uint64_t& index) const; + + virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const; + + virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const; + + virtual std::vector get_tx_output_indices(const crypto::hash& h) const; + virtual std::vector get_tx_amount_output_indices(const crypto::hash& h) const; + + virtual bool has_key_image(const crypto::key_image& img) const; + + virtual uint64_t add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const std::vector& txs + ); + + virtual void set_batch_transactions(bool batch_transactions); + virtual void batch_start(); + virtual void batch_commit(); + virtual void batch_stop(); + virtual void batch_abort(); + + virtual void pop_block(block& blk, std::vector& txs); + +private: + virtual void add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const crypto::hash& block_hash + ); + + virtual void remove_block(); + + virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash); + + virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx); + + virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index); + + virtual void remove_output(const tx_out& tx_output); + + void remove_tx_outputs(const crypto::hash& tx_hash, const transaction& tx); + + void remove_output(const uint64_t& out_index, const uint64_t amount); + void remove_amount_output_index(const uint64_t amount, const uint64_t global_output_index); + + virtual void add_spent_key(const crypto::key_image& k_image); + + virtual void remove_spent_key(const crypto::key_image& k_image); + + /** + * @brief convert a tx output to a blob for storage + * + * @param output the output to convert + * + * @return the resultant blob + */ + blobdata output_to_blob(const tx_out& output); + + /** + * @brief convert a tx output blob to a tx output + * + * @param blob the blob to convert + * + * @return the resultant tx output + */ + tx_out output_from_blob(const blobdata& blob) const; + + /** + * @brief get the global index of the index-th output of the given amount + * + * @param amount the output amount + * @param index the index into the set of outputs of that amount + * + * @return the global index of the desired output + */ + uint64_t get_output_global_index(const uint64_t& amount, const uint64_t& index) const; + + void check_open() const; + + MDB_env* m_env; + + MDB_dbi m_blocks; + MDB_dbi m_block_heights; + MDB_dbi m_block_hashes; + MDB_dbi m_block_timestamps; + MDB_dbi m_block_sizes; + MDB_dbi m_block_diffs; + MDB_dbi m_block_coins; + + MDB_dbi m_txs; + MDB_dbi m_tx_unlocks; + MDB_dbi m_tx_heights; + MDB_dbi m_tx_outputs; + + MDB_dbi m_output_txs; + MDB_dbi m_output_indices; + MDB_dbi m_output_gindices; + MDB_dbi m_output_amounts; + MDB_dbi m_output_keys; + MDB_dbi m_outputs; + + MDB_dbi m_spent_keys; + + bool m_open; + uint64_t m_height; + uint64_t m_num_outputs; + std::string m_folder; + txn_safe* m_write_txn; // may point to either a short-lived txn or a batch txn + txn_safe m_write_batch_txn; // persist batch txn outside of BlockchainLMDB + + bool m_batch_transactions; // support for batch transactions + bool m_batch_active; // whether batch transaction is in progress +}; + +} // namespace cryptonote diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp deleted file mode 100644 index feff4a272..000000000 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.cpp +++ /dev/null @@ -1,1808 +0,0 @@ -// Copyright (c) 2014, The Monero Project -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without modification, are -// permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this list of -// conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright notice, this list -// of conditions and the following disclaimer in the documentation and/or other -// materials provided with the distribution. -// -// 3. Neither the name of the copyright holder nor the names of its contributors may be -// used to endorse or promote products derived from this software without specific -// prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL -// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF -// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -#include "db_lmdb.h" - -#include -#include // std::unique_ptr -#include // memcpy - -#include "cryptonote_core/cryptonote_format_utils.h" -#include "crypto/crypto.h" -#include "profile_tools.h" - -using epee::string_tools::pod_to_hex; - -namespace -{ - -template -inline void throw0(const T &e) -{ - LOG_PRINT_L0(e.what()); - throw e; -} - -template -inline void throw1(const T &e) -{ - LOG_PRINT_L1(e.what()); - throw e; -} - -// cursor needs to be closed when it goes out of scope, -// this helps if the function using it throws -struct lmdb_cur -{ - lmdb_cur(MDB_txn* txn, MDB_dbi dbi) - { - if (mdb_cursor_open(txn, dbi, &m_cur)) - throw0(cryptonote::DB_ERROR("Error opening db cursor")); - done = false; - } - - ~lmdb_cur() { close(); } - - operator MDB_cursor*() { return m_cur; } - operator MDB_cursor**() { return &m_cur; } - - void close() - { - if (!done) - { - mdb_cursor_close(m_cur); - done = true; - } - } - -private: - MDB_cursor* m_cur; - bool done; -}; - -template -struct MDB_val_copy: public MDB_val -{ - MDB_val_copy(const T &t): t_copy(t) - { - mv_size = sizeof (T); - mv_data = &t_copy; - } -private: - T t_copy; -}; - -template<> -struct MDB_val_copy: public MDB_val -{ - MDB_val_copy(const cryptonote::blobdata &bd): data(new char[bd.size()]) - { - memcpy(data.get(), bd.data(), bd.size()); - mv_size = bd.size(); - mv_data = data.get(); - } -private: - std::unique_ptr data; -}; - -auto compare_uint64 = [](const MDB_val *a, const MDB_val *b) { - const uint64_t va = *(const uint64_t*)a->mv_data; - const uint64_t vb = *(const uint64_t*)b->mv_data; - if (va < vb) return -1; - else if (va == vb) return 0; - else return 1; -}; - -const char* const LMDB_BLOCKS = "blocks"; -const char* const LMDB_BLOCK_TIMESTAMPS = "block_timestamps"; -const char* const LMDB_BLOCK_HEIGHTS = "block_heights"; -const char* const LMDB_BLOCK_HASHES = "block_hashes"; -const char* const LMDB_BLOCK_SIZES = "block_sizes"; -const char* const LMDB_BLOCK_DIFFS = "block_diffs"; -const char* const LMDB_BLOCK_COINS = "block_coins"; - -const char* const LMDB_TXS = "txs"; -const char* const LMDB_TX_UNLOCKS = "tx_unlocks"; -const char* const LMDB_TX_HEIGHTS = "tx_heights"; -const char* const LMDB_TX_OUTPUTS = "tx_outputs"; - -const char* const LMDB_OUTPUT_TXS = "output_txs"; -const char* const LMDB_OUTPUT_INDICES = "output_indices"; -const char* const LMDB_OUTPUT_AMOUNTS = "output_amounts"; -const char* const LMDB_OUTPUT_KEYS = "output_keys"; -const char* const LMDB_OUTPUTS = "outputs"; -const char* const LMDB_OUTPUT_GINDICES = "output_gindices"; -const char* const LMDB_SPENT_KEYS = "spent_keys"; - -inline void lmdb_db_open(MDB_txn* txn, const char* name, int flags, MDB_dbi& dbi, const std::string& error_string) -{ - if (mdb_dbi_open(txn, name, flags, &dbi)) - throw0(cryptonote::DB_OPEN_FAILURE(error_string.c_str())); -} - -} // anonymous namespace - -namespace cryptonote -{ - -void BlockchainLMDB::add_block( const block& blk - , const size_t& block_size - , const difficulty_type& cumulative_difficulty - , const uint64_t& coins_generated - , const crypto::hash& blk_hash - ) -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - MDB_val_copy val_h(blk_hash); - MDB_val unused; - if (mdb_get(*m_write_txn, m_block_heights, &val_h, &unused) == 0) - throw1(BLOCK_EXISTS("Attempting to add block that's already in the db")); - - if (m_height > 0) - { - MDB_val_copy parent_key(blk.prev_id); - MDB_val parent_h; - if (mdb_get(*m_write_txn, m_block_heights, &parent_key, &parent_h)) - { - LOG_PRINT_L3("m_height: " << m_height); - LOG_PRINT_L3("parent_key: " << blk.prev_id); - throw0(DB_ERROR("Failed to get top block hash to check for new block's parent")); - } - uint64_t parent_height = *(const uint64_t *)parent_h.mv_data; - if (parent_height != m_height - 1) - throw0(BLOCK_PARENT_DNE("Top block is not new block's parent")); - } - - MDB_val_copy key(m_height); - - MDB_val_copy blob(block_to_blob(blk)); - auto res = mdb_put(*m_write_txn, m_blocks, &key, &blob, 0); - if (res) - throw0(DB_ERROR(std::string("Failed to add block blob to db transaction: ").append(mdb_strerror(res)).c_str())); - - MDB_val_copy sz(block_size); - if (mdb_put(*m_write_txn, m_block_sizes, &key, &sz, 0)) - throw0(DB_ERROR("Failed to add block size to db transaction")); - - MDB_val_copy ts(blk.timestamp); - if (mdb_put(*m_write_txn, m_block_timestamps, &key, &ts, 0)) - throw0(DB_ERROR("Failed to add block timestamp to db transaction")); - - MDB_val_copy diff(cumulative_difficulty); - if (mdb_put(*m_write_txn, m_block_diffs, &key, &diff, 0)) - throw0(DB_ERROR("Failed to add block cumulative difficulty to db transaction")); - - MDB_val_copy coinsgen(coins_generated); - if (mdb_put(*m_write_txn, m_block_coins, &key, &coinsgen, 0)) - throw0(DB_ERROR("Failed to add block total generated coins to db transaction")); - - if (mdb_put(*m_write_txn, m_block_heights, &val_h, &key, 0)) - throw0(DB_ERROR("Failed to add block height by hash to db transaction")); - - if (mdb_put(*m_write_txn, m_block_hashes, &key, &val_h, 0)) - throw0(DB_ERROR("Failed to add block hash to db transaction")); - -} - -void BlockchainLMDB::remove_block() -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - if (m_height == 0) - throw0(BLOCK_DNE ("Attempting to remove block from an empty blockchain")); - - MDB_val_copy k(m_height - 1); - MDB_val h; - if (mdb_get(*m_write_txn, m_block_hashes, &k, &h)) - throw1(BLOCK_DNE("Attempting to remove block that's not in the db")); - - if (mdb_del(*m_write_txn, m_blocks, &k, NULL)) - throw1(DB_ERROR("Failed to add removal of block to db transaction")); - - if (mdb_del(*m_write_txn, m_block_sizes, &k, NULL)) - throw1(DB_ERROR("Failed to add removal of block size to db transaction")); - - if (mdb_del(*m_write_txn, m_block_diffs, &k, NULL)) - throw1(DB_ERROR("Failed to add removal of block cumulative difficulty to db transaction")); - - if (mdb_del(*m_write_txn, m_block_coins, &k, NULL)) - throw1(DB_ERROR("Failed to add removal of block total generated coins to db transaction")); - - if (mdb_del(*m_write_txn, m_block_timestamps, &k, NULL)) - throw1(DB_ERROR("Failed to add removal of block timestamp to db transaction")); - - if (mdb_del(*m_write_txn, m_block_heights, &h, NULL)) - throw1(DB_ERROR("Failed to add removal of block height by hash to db transaction")); - - if (mdb_del(*m_write_txn, m_block_hashes, &k, NULL)) - throw1(DB_ERROR("Failed to add removal of block hash to db transaction")); -} - -void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - MDB_val_copy val_h(tx_hash); - MDB_val unused; - if (mdb_get(*m_write_txn, m_txs, &val_h, &unused) == 0) - throw1(TX_EXISTS("Attempting to add transaction that's already in the db")); - - MDB_val_copy blob(tx_to_blob(tx)); - if (mdb_put(*m_write_txn, m_txs, &val_h, &blob, 0)) - throw0(DB_ERROR("Failed to add tx blob to db transaction")); - - MDB_val_copy height(m_height); - if (mdb_put(*m_write_txn, m_tx_heights, &val_h, &height, 0)) - throw0(DB_ERROR("Failed to add tx block height to db transaction")); - - MDB_val_copy unlock_time(tx.unlock_time); - if (mdb_put(*m_write_txn, m_tx_unlocks, &val_h, &unlock_time, 0)) - throw0(DB_ERROR("Failed to add tx unlock time to db transaction")); -} - -void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - MDB_val_copy val_h(tx_hash); - MDB_val unused; - if (mdb_get(*m_write_txn, m_txs, &val_h, &unused)) - throw1(TX_DNE("Attempting to remove transaction that isn't in the db")); - - if (mdb_del(*m_write_txn, m_txs, &val_h, NULL)) - throw1(DB_ERROR("Failed to add removal of tx to db transaction")); - if (mdb_del(*m_write_txn, m_tx_unlocks, &val_h, NULL)) - throw1(DB_ERROR("Failed to add removal of tx unlock time to db transaction")); - if (mdb_del(*m_write_txn, m_tx_heights, &val_h, NULL)) - throw1(DB_ERROR("Failed to add removal of tx block height to db transaction")); - - remove_tx_outputs(tx_hash, tx); - - if (mdb_del(*m_write_txn, m_tx_outputs, &val_h, NULL)) - throw1(DB_ERROR("Failed to add removal of tx outputs to db transaction")); - -} - -void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index) -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - MDB_val_copy k(m_num_outputs); - MDB_val_copy v(tx_hash); - - if (mdb_put(*m_write_txn, m_output_txs, &k, &v, 0)) - throw0(DB_ERROR("Failed to add output tx hash to db transaction")); - if (mdb_put(*m_write_txn, m_tx_outputs, &v, &k, 0)) - throw0(DB_ERROR("Failed to add tx output index to db transaction")); - - MDB_val_copy val_local_index(local_index); - if (mdb_put(*m_write_txn, m_output_indices, &k, &val_local_index, 0)) - throw0(DB_ERROR("Failed to add tx output index to db transaction")); - - MDB_val_copy val_amount(tx_output.amount); - if (auto result = mdb_put(*m_write_txn, m_output_amounts, &val_amount, &k, 0)) - throw0(DB_ERROR(std::string("Failed to add output amount to db transaction").append(mdb_strerror(result)).c_str())); - - if (tx_output.target.type() == typeid(txout_to_key)) - { - MDB_val_copy val_pubkey(boost::get(tx_output.target).key); - if (mdb_put(*m_write_txn, m_output_keys, &k, &val_pubkey, 0)) - throw0(DB_ERROR("Failed to add output pubkey to db transaction")); - } - - -/****** Uncomment if ever outputs actually need to be stored in this manner - * - blobdata b = output_to_blob(tx_output); - - v.mv_size = b.size(); - v.mv_data = &b; - if (mdb_put(*m_write_txn, m_outputs, &k, &v, 0)) - throw0(DB_ERROR("Failed to add output to db transaction")); - if (mdb_put(*m_write_txn, m_output_gindices, &v, &k, 0)) - throw0(DB_ERROR("Failed to add output global index to db transaction")); -************************************************************************/ - - m_num_outputs++; -} - -void BlockchainLMDB::remove_tx_outputs(const crypto::hash& tx_hash, const transaction& tx) -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - - lmdb_cur cur(*m_write_txn, m_tx_outputs); - - MDB_val_copy k(tx_hash); - MDB_val v; - - auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); - if (result == MDB_NOTFOUND) - { - LOG_ERROR("Attempting to remove a tx's outputs, but none found. Continuing, but...be wary, because that's weird."); - } - else if (result) - { - throw0(DB_ERROR("DB error attempting to get an output")); - } - else - { - size_t num_elems = 0; - mdb_cursor_count(cur, &num_elems); - - mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); - - for (uint64_t i = 0; i < num_elems; ++i) - { - const tx_out tx_output = tx.vout[i]; - remove_output(*(const uint64_t*)v.mv_data, tx_output.amount); - if (i < num_elems - 1) - { - mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); - } - } - } - - cur.close(); -} - -// TODO: probably remove this function -void BlockchainLMDB::remove_output(const tx_out& tx_output) -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " (unused version - does nothing)"); - return; -} - -void BlockchainLMDB::remove_output(const uint64_t& out_index, const uint64_t amount) -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - MDB_val_copy k(out_index); - -/****** Uncomment if ever outputs actually need to be stored in this manner - blobdata b; - t_serializable_object_to_blob(tx_output, b); - k.mv_size = b.size(); - k.mv_data = &b; - - if (mdb_get(*m_write_txn, m_output_gindices, &k, &v)) - throw1(OUTPUT_DNE("Attempting to remove output that does not exist")); - - uint64_t gindex = *(uint64_t*)v.mv_data; - - auto result = mdb_del(*m_write_txn, m_output_gindices, &k, NULL); - if (result != 0 && result != MDB_NOTFOUND) - throw1(DB_ERROR("Error adding removal of output global index to db transaction")); - - result = mdb_del(*m_write_txn, m_outputs, &v, NULL); - if (result != 0 && result != MDB_NOTFOUND) - throw1(DB_ERROR("Error adding removal of output to db transaction")); -*********************************************************************/ - - auto result = mdb_del(*m_write_txn, m_output_indices, &k, NULL); - if (result == MDB_NOTFOUND) - { - LOG_PRINT_L0("Unexpected: global output index not found in m_output_indices"); - } - else if (result) - { - throw1(DB_ERROR("Error adding removal of output tx index to db transaction")); - } - - result = mdb_del(*m_write_txn, m_output_txs, &k, NULL); - // if (result != 0 && result != MDB_NOTFOUND) - // throw1(DB_ERROR("Error adding removal of output tx hash to db transaction")); - if (result == MDB_NOTFOUND) - { - LOG_PRINT_L0("Unexpected: global output index not found in m_output_txs"); - } - else if (result) - { - throw1(DB_ERROR("Error adding removal of output tx hash to db transaction")); - } - - result = mdb_del(*m_write_txn, m_output_keys, &k, NULL); - if (result == MDB_NOTFOUND) - { - LOG_PRINT_L0("Unexpected: global output index not found in m_output_keys"); - } - else if (result) - throw1(DB_ERROR("Error adding removal of output pubkey to db transaction")); - - remove_amount_output_index(amount, out_index); - - m_num_outputs--; -} - -void BlockchainLMDB::remove_amount_output_index(const uint64_t amount, const uint64_t global_output_index) -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - lmdb_cur cur(*m_write_txn, m_output_amounts); - - MDB_val_copy k(amount); - MDB_val v; - - auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); - if (result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); - else if (result) - throw0(DB_ERROR("DB error attempting to get an output")); - - size_t num_elems = 0; - mdb_cursor_count(cur, &num_elems); - - mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); - - uint64_t amount_output_index = 0; - uint64_t goi = 0; - bool found_index = false; - for (uint64_t i = 0; i < num_elems; ++i) - { - mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); - goi = *(const uint64_t *)v.mv_data; - if (goi == global_output_index) - { - amount_output_index = i; - found_index = true; - break; - } - mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); - } - if (found_index) - { - // found the amount output index - // now delete it - result = mdb_cursor_del(cur, 0); - if (result) - throw0(DB_ERROR(std::string("Error deleting amount output index ").append(boost::lexical_cast(amount_output_index)).c_str())); - } - else - { - // not found - cur.close(); - throw1(OUTPUT_DNE("Failed to find amount output index")); - } - cur.close(); -} - -void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image) -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - MDB_val_copy val_key(k_image); - MDB_val unused; - if (mdb_get(*m_write_txn, m_spent_keys, &val_key, &unused) == 0) - throw1(KEY_IMAGE_EXISTS("Attempting to add spent key image that's already in the db")); - - char anything = '\0'; - unused.mv_size = sizeof(char); - unused.mv_data = &anything; - if (auto result = mdb_put(*m_write_txn, m_spent_keys, &val_key, &unused, 0)) - throw1(DB_ERROR(std::string("Error adding spent key image to db transaction: ").append(mdb_strerror(result)).c_str())); -} - -void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image) -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - MDB_val_copy k(k_image); - auto result = mdb_del(*m_write_txn, m_spent_keys, &k, NULL); - if (result != 0 && result != MDB_NOTFOUND) - throw1(DB_ERROR("Error adding removal of key image to db transaction")); -} - -blobdata BlockchainLMDB::output_to_blob(const tx_out& output) -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - blobdata b; - if (!t_serializable_object_to_blob(output, b)) - throw1(DB_ERROR("Error serializing output to blob")); - return b; -} - -tx_out BlockchainLMDB::output_from_blob(const blobdata& blob) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - std::stringstream ss; - ss << blob; - binary_archive ba(ss); - tx_out o; - - if (!(::serialization::serialize(ba, o))) - throw1(DB_ERROR("Error deserializing tx output blob")); - - return o; -} - -uint64_t BlockchainLMDB::get_output_global_index(const uint64_t& amount, const uint64_t& index) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - - lmdb_cur cur(txn, m_output_amounts); - - MDB_val_copy k(amount); - MDB_val v; - - auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); - if (result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); - else if (result) - throw0(DB_ERROR("DB error attempting to get an output")); - - size_t num_elems = 0; - mdb_cursor_count(cur, &num_elems); - if (num_elems <= index) - throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found")); - - mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); - - for (uint64_t i = 0; i < index; ++i) - { - mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); - } - - mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); - - uint64_t glob_index = *(const uint64_t*)v.mv_data; - - cur.close(); - - txn.commit(); - - return glob_index; -} - -void BlockchainLMDB::check_open() const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - if (!m_open) - throw0(DB_ERROR("DB operation attempted on a not-open DB instance")); -} - -BlockchainLMDB::~BlockchainLMDB() -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - - // batch transaction shouldn't be active at this point. If it is, consider it aborted. - if (m_batch_active) - batch_abort(); -} - -BlockchainLMDB::BlockchainLMDB(bool batch_transactions) -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - // initialize folder to something "safe" just in case - // someone accidentally misuses this class... - m_folder = "thishsouldnotexistbecauseitisgibberish"; - m_open = false; - - m_batch_transactions = batch_transactions; - m_write_txn = nullptr; - m_batch_active = false; - m_height = 0; -} - -void BlockchainLMDB::open(const std::string& filename) -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - - if (m_open) - throw0(DB_OPEN_FAILURE("Attempted to open db, but it's already open")); - - boost::filesystem::path direc(filename); - if (boost::filesystem::exists(direc)) - { - if (!boost::filesystem::is_directory(direc)) - throw0(DB_OPEN_FAILURE("LMDB needs a directory path, but a file was passed")); - } - else - { - if (!boost::filesystem::create_directory(direc)) - throw0(DB_OPEN_FAILURE(std::string("Failed to create directory ").append(filename).c_str())); - } - - // check for existing LMDB files in base directory - boost::filesystem::path old_files = direc.parent_path(); - if (boost::filesystem::exists(old_files / "data.mdb") || - boost::filesystem::exists(old_files / "lock.mdb")) - { - LOG_PRINT_L0("Found existing LMDB files in " << old_files.c_str()); - LOG_PRINT_L0("Move data.mdb and/or lock.mdb to " << filename << ", or delete them, and then restart"); - throw DB_ERROR("Database could not be opened"); - } - - m_folder = filename; - - // set up lmdb environment - if (mdb_env_create(&m_env)) - throw0(DB_ERROR("Failed to create lmdb environment")); - if (mdb_env_set_maxdbs(m_env, 20)) - throw0(DB_ERROR("Failed to set max number of dbs")); - - size_t mapsize = 1LL << 34; - if (auto result = mdb_env_set_mapsize(m_env, mapsize)) - throw0(DB_ERROR(std::string("Failed to set max memory map size: ").append(mdb_strerror(result)).c_str())); - if (auto result = mdb_env_open(m_env, filename.c_str(), 0, 0644)) - throw0(DB_ERROR(std::string("Failed to open lmdb environment: ").append(mdb_strerror(result)).c_str())); - - // get a read/write MDB_txn - txn_safe txn; - if (mdb_txn_begin(m_env, NULL, 0, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - - // open necessary databases, and set properties as needed - // uses macros to avoid having to change things too many places - lmdb_db_open(txn, LMDB_BLOCKS, MDB_INTEGERKEY | MDB_CREATE, m_blocks, "Failed to open db handle for m_blocks"); - - lmdb_db_open(txn, LMDB_BLOCK_TIMESTAMPS, MDB_INTEGERKEY | MDB_CREATE, m_block_timestamps, "Failed to open db handle for m_block_timestamps"); - lmdb_db_open(txn, LMDB_BLOCK_HEIGHTS, MDB_CREATE, m_block_heights, "Failed to open db handle for m_block_heights"); - lmdb_db_open(txn, LMDB_BLOCK_HASHES, MDB_INTEGERKEY | MDB_CREATE, m_block_hashes, "Failed to open db handle for m_block_hashes"); - lmdb_db_open(txn, LMDB_BLOCK_SIZES, MDB_INTEGERKEY | MDB_CREATE, m_block_sizes, "Failed to open db handle for m_block_sizes"); - lmdb_db_open(txn, LMDB_BLOCK_DIFFS, MDB_INTEGERKEY | MDB_CREATE, m_block_diffs, "Failed to open db handle for m_block_diffs"); - lmdb_db_open(txn, LMDB_BLOCK_COINS, MDB_INTEGERKEY | MDB_CREATE, m_block_coins, "Failed to open db handle for m_block_coins"); - - lmdb_db_open(txn, LMDB_TXS, MDB_CREATE, m_txs, "Failed to open db handle for m_txs"); - lmdb_db_open(txn, LMDB_TX_UNLOCKS, MDB_CREATE, m_tx_unlocks, "Failed to open db handle for m_tx_unlocks"); - lmdb_db_open(txn, LMDB_TX_HEIGHTS, MDB_CREATE, m_tx_heights, "Failed to open db handle for m_tx_heights"); - lmdb_db_open(txn, LMDB_TX_OUTPUTS, MDB_DUPSORT | MDB_CREATE, m_tx_outputs, "Failed to open db handle for m_tx_outputs"); - - lmdb_db_open(txn, LMDB_OUTPUT_TXS, MDB_INTEGERKEY | MDB_CREATE, m_output_txs, "Failed to open db handle for m_output_txs"); - lmdb_db_open(txn, LMDB_OUTPUT_INDICES, MDB_INTEGERKEY | MDB_CREATE, m_output_indices, "Failed to open db handle for m_output_indices"); - lmdb_db_open(txn, LMDB_OUTPUT_AMOUNTS, MDB_INTEGERKEY | MDB_DUPSORT | MDB_CREATE, m_output_amounts, "Failed to open db handle for m_output_amounts"); - lmdb_db_open(txn, LMDB_OUTPUT_KEYS, MDB_INTEGERKEY | MDB_CREATE, m_output_keys, "Failed to open db handle for m_output_keys"); - -/*************** not used, but kept for posterity - lmdb_db_open(txn, LMDB_OUTPUTS, MDB_INTEGERKEY | MDB_CREATE, m_outputs, "Failed to open db handle for m_outputs"); - lmdb_db_open(txn, LMDB_OUTPUT_GINDICES, MDB_CREATE, m_output_gindices, "Failed to open db handle for m_output_gindices"); -*************************************************/ - - lmdb_db_open(txn, LMDB_SPENT_KEYS, MDB_CREATE, m_spent_keys, "Failed to open db handle for m_spent_keys"); - - mdb_set_dupsort(txn, m_output_amounts, compare_uint64); - mdb_set_dupsort(txn, m_tx_outputs, compare_uint64); - - // get and keep current height - MDB_stat db_stats; - if (mdb_stat(txn, m_blocks, &db_stats)) - throw0(DB_ERROR("Failed to query m_blocks")); - LOG_PRINT_L2("Setting m_height to: " << db_stats.ms_entries); - m_height = db_stats.ms_entries; - - // get and keep current number of outputs - if (mdb_stat(txn, m_output_indices, &db_stats)) - throw0(DB_ERROR("Failed to query m_output_indices")); - m_num_outputs = db_stats.ms_entries; - - // commit the transaction - txn.commit(); - - m_open = true; - // from here, init should be finished -} - -// unused for now, create will happen on open if doesn't exist -void BlockchainLMDB::create(const std::string& filename) -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - throw DB_CREATE_FAILURE("create() is not implemented for this BlockchainDB, open() will create files if needed."); -} - -void BlockchainLMDB::close() -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - if (m_batch_active) - { - LOG_PRINT_L3("close() first calling batch_abort() due to active batch transaction"); - batch_abort(); - } - this->sync(); - - // FIXME: not yet thread safe!!! Use with care. - mdb_env_close(m_env); -} - -void BlockchainLMDB::sync() -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - // Does nothing unless LMDB environment was opened with MDB_NOSYNC or in part - // MDB_NOMETASYNC. Force flush to be synchronous. - if (auto result = mdb_env_sync(m_env, true)) - { - throw0(DB_ERROR(std::string("Failed to sync database").append(mdb_strerror(result)).c_str())); - } -} - -void BlockchainLMDB::reset() -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - // TODO: this -} - -std::vector BlockchainLMDB::get_filenames() const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - std::vector filenames; - - boost::filesystem::path datafile(m_folder); - datafile /= "data.mdb"; - boost::filesystem::path lockfile(m_folder); - lockfile /= "lock.mdb"; - - filenames.push_back(datafile.string()); - filenames.push_back(lockfile.string()); - - return filenames; -} - -// TODO: this? -bool BlockchainLMDB::lock() -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - return false; -} - -// TODO: this? -void BlockchainLMDB::unlock() -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); -} - -bool BlockchainLMDB::block_exists(const crypto::hash& h) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - - MDB_val_copy key(h); - MDB_val result; - auto get_result = mdb_get(txn, m_block_heights, &key, &result); - if (get_result == MDB_NOTFOUND) - { - txn.commit(); - LOG_PRINT_L3("Block with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); - return false; - } - else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch block index from hash")); - - txn.commit(); - return true; -} - -block BlockchainLMDB::get_block(const crypto::hash& h) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - return get_block_from_height(get_block_height(h)); -} - -uint64_t BlockchainLMDB::get_block_height(const crypto::hash& h) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - - MDB_val_copy key(h); - MDB_val result; - auto get_result = mdb_get(txn, m_block_heights, &key, &result); - if (get_result == MDB_NOTFOUND) - throw1(BLOCK_DNE("Attempted to retrieve non-existent block height")); - else if (get_result) - throw0(DB_ERROR("Error attempting to retrieve a block height from the db")); - - txn.commit(); - return *(const uint64_t*)result.mv_data; -} - -block_header BlockchainLMDB::get_block_header(const crypto::hash& h) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - // block_header object is automatically cast from block object - return get_block(h); -} - -block BlockchainLMDB::get_block_from_height(const uint64_t& height) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - - MDB_val_copy key(height); - MDB_val result; - auto get_result = mdb_get(txn, m_blocks, &key, &result); - if (get_result == MDB_NOTFOUND) - { - throw0(DB_ERROR(std::string("Attempt to get block from height ").append(boost::lexical_cast(height)).append(" failed -- block not in db").c_str())); - } - else if (get_result) - throw0(DB_ERROR("Error attempting to retrieve a block from the db")); - - txn.commit(); - - blobdata bd; - bd.assign(reinterpret_cast(result.mv_data), result.mv_size); - - block b; - if (!parse_and_validate_block_from_blob(bd, b)) - throw0(DB_ERROR("Failed to parse block from blob retrieved from the db")); - - return b; -} - -uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - txn_safe txn; - txn_safe* txn_ptr = &txn; - if (m_batch_active) - txn_ptr = m_write_txn; - else - { - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - } - MDB_val_copy key(height); - MDB_val result; - auto get_result = mdb_get(*txn_ptr, m_block_timestamps, &key, &result); - if (get_result == MDB_NOTFOUND) - { - throw0(DB_ERROR(std::string("Attempt to get timestamp from height ").append(boost::lexical_cast(height)).append(" failed -- timestamp not in db").c_str())); - } - else if (get_result) - throw0(DB_ERROR("Error attempting to retrieve a timestamp from the db")); - - if (! m_batch_active) - txn.commit(); - return *(const uint64_t*)result.mv_data; -} - -uint64_t BlockchainLMDB::get_top_block_timestamp() const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - // if no blocks, return 0 - if (m_height == 0) - { - return 0; - } - - return get_block_timestamp(m_height - 1); -} - -size_t BlockchainLMDB::get_block_size(const uint64_t& height) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - txn_safe txn; - txn_safe* txn_ptr = &txn; - if (m_batch_active) - txn_ptr = m_write_txn; - else - { - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - } - - MDB_val_copy key(height); - MDB_val result; - auto get_result = mdb_get(*txn_ptr, m_block_sizes, &key, &result); - if (get_result == MDB_NOTFOUND) - { - throw0(DB_ERROR(std::string("Attempt to get block size from height ").append(boost::lexical_cast(height)).append(" failed -- block size not in db").c_str())); - } - else if (get_result) - throw0(DB_ERROR("Error attempting to retrieve a block size from the db")); - - if (! m_batch_active) - txn.commit(); - return *(const size_t*)result.mv_data; -} - -difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& height) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " height: " << height); - check_open(); - - txn_safe txn; - txn_safe* txn_ptr = &txn; - if (m_batch_active) - txn_ptr = m_write_txn; - else - { - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - } - MDB_val_copy key(height); - MDB_val result; - auto get_result = mdb_get(*txn_ptr, m_block_diffs, &key, &result); - if (get_result == MDB_NOTFOUND) - { - throw0(DB_ERROR(std::string("Attempt to get cumulative difficulty from height ").append(boost::lexical_cast(height)).append(" failed -- difficulty not in db").c_str())); - } - else if (get_result) - throw0(DB_ERROR("Error attempting to retrieve a cumulative difficulty from the db")); - - if (! m_batch_active) - txn.commit(); - return *(difficulty_type*)result.mv_data; -} - -difficulty_type BlockchainLMDB::get_block_difficulty(const uint64_t& height) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - difficulty_type diff1 = 0; - difficulty_type diff2 = 0; - - diff1 = get_block_cumulative_difficulty(height); - if (height != 0) - { - diff2 = get_block_cumulative_difficulty(height - 1); - } - - return diff1 - diff2; -} - -uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& height) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - txn_safe txn; - txn_safe* txn_ptr = &txn; - if (m_batch_active) - txn_ptr = m_write_txn; - else - { - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - } - - MDB_val_copy key(height); - MDB_val result; - auto get_result = mdb_get(*txn_ptr, m_block_coins, &key, &result); - if (get_result == MDB_NOTFOUND) - { - throw0(DB_ERROR(std::string("Attempt to get generated coins from height ").append(boost::lexical_cast(height)).append(" failed -- block size not in db").c_str())); - } - else if (get_result) - throw0(DB_ERROR("Error attempting to retrieve a total generated coins from the db")); - - if (! m_batch_active) - txn.commit(); - return *(const uint64_t*)result.mv_data; -} - -crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - txn_safe txn; - txn_safe* txn_ptr = &txn; - if (m_batch_active) - txn_ptr = m_write_txn; - else - { - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - } - - MDB_val_copy key(height); - MDB_val result; - auto get_result = mdb_get(*txn_ptr, m_block_hashes, &key, &result); - if (get_result == MDB_NOTFOUND) - { - throw0(BLOCK_DNE(std::string("Attempt to get hash from height ").append(boost::lexical_cast(height)).append(" failed -- hash not in db").c_str())); - } - else if (get_result) - throw0(DB_ERROR(std::string("Error attempting to retrieve a block hash from the db: "). - append(mdb_strerror(get_result)).c_str())); - - if (! m_batch_active) - txn.commit(); - return *(crypto::hash*)result.mv_data; -} - -std::vector BlockchainLMDB::get_blocks_range(const uint64_t& h1, const uint64_t& h2) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - std::vector v; - - for (uint64_t height = h1; height <= h2; ++height) - { - v.push_back(get_block_from_height(height)); - } - - return v; -} - -std::vector BlockchainLMDB::get_hashes_range(const uint64_t& h1, const uint64_t& h2) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - std::vector v; - - for (uint64_t height = h1; height <= h2; ++height) - { - v.push_back(get_block_hash_from_height(height)); - } - - return v; -} - -crypto::hash BlockchainLMDB::top_block_hash() const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - if (m_height != 0) - { - return get_block_hash_from_height(m_height - 1); - } - - return null_hash; -} - -block BlockchainLMDB::get_top_block() const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - if (m_height != 0) - { - return get_block_from_height(m_height - 1); - } - - block b; - return b; -} - -uint64_t BlockchainLMDB::height() const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - return m_height; -} - -bool BlockchainLMDB::tx_exists(const crypto::hash& h) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - txn_safe txn; - txn_safe* txn_ptr = &txn; - if (m_batch_active) - txn_ptr = m_write_txn; - else - { - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - } - - MDB_val_copy key(h); - MDB_val result; - - TIME_MEASURE_START(time1); - auto get_result = mdb_get(*txn_ptr, m_txs, &key, &result); - TIME_MEASURE_FINISH(time1); - time_tx_exists += time1; - if (get_result == MDB_NOTFOUND) - { - if (! m_batch_active) - txn.commit(); - LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); - return false; - } - else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch transaction from hash")); - - return true; -} - -uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - - MDB_val_copy key(h); - MDB_val result; - auto get_result = mdb_get(txn, m_tx_unlocks, &key, &result); - if (get_result == MDB_NOTFOUND) - throw1(TX_DNE(std::string("tx unlock time with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); - else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch tx unlock time from hash")); - - return *(const uint64_t*)result.mv_data; -} - -transaction BlockchainLMDB::get_tx(const crypto::hash& h) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - txn_safe txn; - txn_safe* txn_ptr = &txn; - if (m_batch_active) - txn_ptr = m_write_txn; - else - { - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - } - - MDB_val_copy key(h); - MDB_val result; - auto get_result = mdb_get(*txn_ptr, m_txs, &key, &result); - if (get_result == MDB_NOTFOUND) - throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); - else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch tx from hash")); - - blobdata bd; - bd.assign(reinterpret_cast(result.mv_data), result.mv_size); - - transaction tx; - if (!parse_and_validate_tx_from_blob(bd, tx)) - throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); - if (! m_batch_active) - txn.commit(); - - return tx; -} - -uint64_t BlockchainLMDB::get_tx_count() const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - - MDB_stat db_stats; - if (mdb_stat(txn, m_txs, &db_stats)) - throw0(DB_ERROR("Failed to query m_txs")); - - txn.commit(); - - return db_stats.ms_entries; -} - -std::vector BlockchainLMDB::get_tx_list(const std::vector& hlist) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - std::vector v; - - for (auto& h : hlist) - { - v.push_back(get_tx(h)); - } - - return v; -} - -uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - // If m_batch_active is set, a batch transaction exists beyond this class, - // such as a batch import with verification enabled, or possibly (later) a - // batch network sync. - // - // A regular network sync without batching would be expected to open a new - // read transaction here, as validation is done prior to the write for block - // and tx data. - - txn_safe txn; - txn_safe* txn_ptr = &txn; - if (m_batch_active) - txn_ptr = m_write_txn; - else - { - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - } - - MDB_val_copy key(h); - MDB_val result; - auto get_result = mdb_get(*txn_ptr, m_tx_heights, &key, &result); - if (get_result == MDB_NOTFOUND) - { - throw1(TX_DNE(std::string("tx height with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); - } - else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch tx height from hash")); - - if (! m_batch_active) - txn.commit(); - - return *(const uint64_t*)result.mv_data; -} - -//FIXME: make sure the random method used here is appropriate -uint64_t BlockchainLMDB::get_random_output(const uint64_t& amount) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - uint64_t num_outputs = get_num_outputs(amount); - if (num_outputs == 0) - throw1(OUTPUT_DNE("Attempting to get a random output for an amount, but none exist")); - - return crypto::rand() % num_outputs; -} - -uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - - lmdb_cur cur(txn, m_output_amounts); - - MDB_val_copy k(amount); - MDB_val v; - auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); - if (result == MDB_NOTFOUND) - { - return 0; - } - else if (result) - throw0(DB_ERROR("DB error attempting to get number of outputs of an amount")); - - size_t num_elems = 0; - mdb_cursor_count(cur, &num_elems); - - txn.commit(); - - return num_elems; -} - -crypto::public_key BlockchainLMDB::get_output_key(const uint64_t& amount, const uint64_t& index) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - uint64_t glob_index = get_output_global_index(amount, index); - - txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - - MDB_val_copy k(glob_index); - MDB_val v; - auto get_result = mdb_get(txn, m_output_keys, &k, &v); - if (get_result == MDB_NOTFOUND) - throw0(DB_ERROR("Attempting to get output pubkey by global index, but key does not exist")); - else if (get_result) - throw0(DB_ERROR("Error attempting to retrieve an output pubkey from the db")); - - return *(crypto::public_key*)v.mv_data; -} - -tx_out BlockchainLMDB::get_output(const crypto::hash& h, const uint64_t& index) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - - lmdb_cur cur(txn, m_tx_outputs); - - MDB_val_copy k(h); - MDB_val v; - auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); - if (result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("Attempting to get an output by tx hash and tx index, but output not found")); - else if (result) - throw0(DB_ERROR("DB error attempting to get an output")); - - size_t num_elems = 0; - mdb_cursor_count(cur, &num_elems); - if (num_elems <= index) - throw1(OUTPUT_DNE("Attempting to get an output by tx hash and tx index, but output not found")); - - mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); - - for (uint64_t i = 0; i < index; ++i) - { - mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); - } - - mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); - - blobdata b; - b = *(blobdata*)v.mv_data; - - cur.close(); - txn.commit(); - - return output_from_blob(b); -} - -// As this is not used, its return is now a blank output. -// This will save on space in the db. -tx_out BlockchainLMDB::get_output(const uint64_t& index) const -{ - return tx_out(); - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - - MDB_val_copy k(index); - MDB_val v; - auto get_result = mdb_get(txn, m_outputs, &k, &v); - if (get_result == MDB_NOTFOUND) - { - throw OUTPUT_DNE("Attempting to get output by global index, but output does not exist"); - } - else if (get_result) - throw0(DB_ERROR("Error attempting to retrieve an output from the db")); - - blobdata b = *(blobdata*)v.mv_data; - - return output_from_blob(b); -} - -tx_out_index BlockchainLMDB::get_output_tx_and_index_from_global(const uint64_t& index) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - txn_safe txn; - txn_safe* txn_ptr = &txn; - if (m_batch_active) - txn_ptr = m_write_txn; - else - { - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - } - MDB_val_copy k(index); - MDB_val v; - - auto get_result = mdb_get(*txn_ptr, m_output_txs, &k, &v); - if (get_result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("output with given index not in db")); - else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch output tx hash")); - - crypto::hash tx_hash = *(crypto::hash*)v.mv_data; - - get_result = mdb_get(*txn_ptr, m_output_indices, &k, &v); - if (get_result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("output with given index not in db")); - else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch output tx index")); - if (! m_batch_active) - txn.commit(); - - return tx_out_index(tx_hash, *(const uint64_t *)v.mv_data); -} - -tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - txn_safe txn; - txn_safe* txn_ptr = &txn; - if (m_batch_active) - txn_ptr = m_write_txn; - else - { - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - } - lmdb_cur cur(*txn_ptr, m_output_amounts); - - MDB_val_copy k(amount); - MDB_val v; - - auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); - if (result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); - else if (result) - throw0(DB_ERROR("DB error attempting to get an output")); - - size_t num_elems = 0; - mdb_cursor_count(cur, &num_elems); - if (num_elems <= index) - throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found")); - - mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); - - for (uint64_t i = 0; i < index; ++i) - { - mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); - } - - mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); - - uint64_t glob_index = *(const uint64_t*)v.mv_data; - - cur.close(); - - if (! m_batch_active) - txn.commit(); - - return get_output_tx_and_index_from_global(glob_index); -} - -std::vector BlockchainLMDB::get_tx_output_indices(const crypto::hash& h) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - std::vector index_vec; - - txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - - lmdb_cur cur(txn, m_tx_outputs); - - MDB_val_copy k(h); - MDB_val v; - auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); - if (result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("Attempting to get an output by tx hash and tx index, but output not found")); - else if (result) - throw0(DB_ERROR("DB error attempting to get an output")); - - size_t num_elems = 0; - mdb_cursor_count(cur, &num_elems); - - mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); - - for (uint64_t i = 0; i < num_elems; ++i) - { - mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); - index_vec.push_back(*(const uint64_t *)v.mv_data); - mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); - } - - cur.close(); - txn.commit(); - - return index_vec; -} - -std::vector BlockchainLMDB::get_tx_amount_output_indices(const crypto::hash& h) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - std::vector index_vec; - std::vector index_vec2; - - // get the transaction's global output indices first - index_vec = get_tx_output_indices(h); - // these are next used to obtain the amount output indices - - transaction tx = get_tx(h); - - txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - - uint64_t i = 0; - uint64_t global_index; - BOOST_FOREACH(const auto& vout, tx.vout) - { - uint64_t amount = vout.amount; - - global_index = index_vec[i]; - - lmdb_cur cur(txn, m_output_amounts); - - MDB_val_copy k(amount); - MDB_val v; - - auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); - if (result == MDB_NOTFOUND) - throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); - else if (result) - throw0(DB_ERROR("DB error attempting to get an output")); - - size_t num_elems = 0; - mdb_cursor_count(cur, &num_elems); - - mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); - - uint64_t amount_output_index = 0; - uint64_t output_index = 0; - bool found_index = false; - for (uint64_t j = 0; j < num_elems; ++j) - { - mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); - output_index = *(const uint64_t *)v.mv_data; - if (output_index == global_index) - { - amount_output_index = j; - found_index = true; - break; - } - mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); - } - if (found_index) - { - index_vec2.push_back(amount_output_index); - } - else - { - // not found - cur.close(); - txn.commit(); - throw1(OUTPUT_DNE("specified output not found in db")); - } - - cur.close(); - ++i; - } - - txn.commit(); - - return index_vec2; -} - - - -bool BlockchainLMDB::has_key_image(const crypto::key_image& img) const -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - txn_safe txn; - if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - - MDB_val_copy val_key(img); - MDB_val unused; - if (mdb_get(txn, m_spent_keys, &val_key, &unused) == 0) - { - txn.commit(); - return true; - } - - txn.commit(); - return false; -} - -void BlockchainLMDB::batch_start() -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - if (! m_batch_transactions) - throw0(DB_ERROR("batch transactions not enabled")); - if (m_batch_active) - throw0(DB_ERROR("batch transaction already in progress")); - if (m_write_txn) - throw0(DB_ERROR("batch transaction attempted, but m_write_txn already in use")); - check_open(); - // NOTE: need to make sure it's destroyed properly when done - if (mdb_txn_begin(m_env, NULL, 0, m_write_batch_txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - // indicates this transaction is for batch transactions, but not whether it's - // active - m_write_batch_txn.m_batch_txn = true; - m_write_txn = &m_write_batch_txn; - m_batch_active = true; - LOG_PRINT_L3("batch transaction: begin"); -} - -void BlockchainLMDB::batch_commit() -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - if (! m_batch_transactions) - throw0(DB_ERROR("batch transactions not enabled")); - if (! m_batch_active) - throw0(DB_ERROR("batch transaction not in progress")); - check_open(); - LOG_PRINT_L3("batch transaction: committing..."); - TIME_MEASURE_START(time1); - m_write_txn->commit(); - TIME_MEASURE_FINISH(time1); - time_commit1 += time1; - LOG_PRINT_L3("batch transaction: committed"); - - if (mdb_txn_begin(m_env, NULL, 0, m_write_batch_txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - if (! m_write_batch_txn.m_batch_txn) - throw0(DB_ERROR("m_write_batch_txn not marked as a batch transaction")); - m_write_txn = &m_write_batch_txn; -} - -void BlockchainLMDB::batch_stop() -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - if (! m_batch_transactions) - throw0(DB_ERROR("batch transactions not enabled")); - if (! m_batch_active) - throw0(DB_ERROR("batch transaction not in progress")); - check_open(); - LOG_PRINT_L3("batch transaction: committing..."); - TIME_MEASURE_START(time1); - m_write_txn->commit(); - TIME_MEASURE_FINISH(time1); - time_commit1 += time1; - // for destruction of batch transaction - m_write_txn = nullptr; - m_batch_active = false; - LOG_PRINT_L3("batch transaction: end"); -} - -void BlockchainLMDB::batch_abort() -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - if (! m_batch_transactions) - throw0(DB_ERROR("batch transactions not enabled")); - if (! m_batch_active) - throw0(DB_ERROR("batch transaction not in progress")); - check_open(); - // for destruction of batch transaction - m_write_txn = nullptr; - // explicitly call in case mdb_env_close() (BlockchainLMDB::close()) called before BlockchainLMDB destructor called. - m_write_batch_txn.abort(); - m_batch_active = false; - LOG_PRINT_L3("batch transaction: aborted"); -} - -void BlockchainLMDB::set_batch_transactions(bool batch_transactions) -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - m_batch_transactions = batch_transactions; - LOG_PRINT_L3("batch transactions " << (m_batch_transactions ? "enabled" : "disabled")); -} - -uint64_t BlockchainLMDB::add_block( const block& blk - , const size_t& block_size - , const difficulty_type& cumulative_difficulty - , const uint64_t& coins_generated - , const std::vector& txs - ) -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - txn_safe txn; - if (! m_batch_active) - { - if (mdb_txn_begin(m_env, NULL, 0, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - m_write_txn = &txn; - } - - uint64_t num_outputs = m_num_outputs; - try - { - BlockchainDB::add_block(blk, block_size, cumulative_difficulty, coins_generated, txs); - if (! m_batch_active) - { - m_write_txn = NULL; - - TIME_MEASURE_START(time1); - txn.commit(); - TIME_MEASURE_FINISH(time1); - time_commit1 += time1; - } - } - catch (...) - { - m_num_outputs = num_outputs; - if (! m_batch_active) - m_write_txn = NULL; - throw; - } - - return ++m_height; -} - -void BlockchainLMDB::pop_block(block& blk, std::vector& txs) -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - check_open(); - - txn_safe txn; - if (! m_batch_active) - { - if (mdb_txn_begin(m_env, NULL, 0, txn)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - m_write_txn = &txn; - } - - uint64_t num_outputs = m_num_outputs; - try - { - BlockchainDB::pop_block(blk, txs); - if (! m_batch_active) - { - m_write_txn = NULL; - - txn.commit(); - } - } - catch (...) - { - m_num_outputs = num_outputs; - m_write_txn = NULL; - throw; - } - - --m_height; -} - -} // namespace cryptonote diff --git a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h b/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h deleted file mode 100644 index 0b4b5ec1a..000000000 --- a/src/cryptonote_core/BlockchainDB_impl/db_lmdb.h +++ /dev/null @@ -1,314 +0,0 @@ -// Copyright (c) 2014, The Monero Project -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without modification, are -// permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this list of -// conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright notice, this list -// of conditions and the following disclaimer in the documentation and/or other -// materials provided with the distribution. -// -// 3. Neither the name of the copyright holder nor the names of its contributors may be -// used to endorse or promote products derived from this software without specific -// prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL -// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF -// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -#include "cryptonote_core/blockchain_db.h" -#include "cryptonote_protocol/blobdatatype.h" // for type blobdata - -#include - -namespace cryptonote -{ - -struct txn_safe -{ - txn_safe() : m_txn(NULL) { } - ~txn_safe() - { - LOG_PRINT_L3("txn_safe: destructor"); - if (m_txn != NULL) - { - if (m_batch_txn) // this is a batch txn and should have been handled before this point for safety - { - LOG_PRINT_L0("WARNING: txn_safe: m_txn is a batch txn and it's not NULL in destructor - calling mdb_txn_abort()"); - } - else - { - // Example of when this occurs: a lookup fails, so a read-only txn is - // aborted through this destructor. However, successful read-only txns - // ideally should have been committed when done and not end up here. - // - // NOTE: not sure if this is ever reached for a non-batch write - // transaction, but it's probably not ideal if it did. - LOG_PRINT_L3("txn_safe: m_txn not NULL in destructor - calling mdb_txn_abort()"); - } - mdb_txn_abort(m_txn); - } - } - - void commit(std::string message = "") - { - if (message.size() == 0) - { - message = "Failed to commit a transaction to the db"; - } - - if (mdb_txn_commit(m_txn)) - { - m_txn = NULL; - LOG_PRINT_L0(message); - throw DB_ERROR(message.c_str()); - } - m_txn = NULL; - } - - // This should only be needed for batch transaction which must be ensured to - // be aborted before mdb_env_close, not after. So we can't rely on - // BlockchainLMDB destructor to call txn_safe destructor, as that's too late - // to properly abort, since mdb_env_close would have been called earlier. - void abort() - { - LOG_PRINT_L3("txn_safe: abort()"); - if(m_txn != NULL) - { - mdb_txn_abort(m_txn); - m_txn = NULL; - } - else - { - LOG_PRINT_L0("WARNING: txn_safe: abort() called, but m_txn is NULL"); - } - } - - operator MDB_txn*() - { - return m_txn; - } - - operator MDB_txn**() - { - return &m_txn; - } - - MDB_txn* m_txn; - bool m_batch_txn = false; -}; - - -class BlockchainLMDB : public BlockchainDB -{ -public: - BlockchainLMDB(bool batch_transactions=false); - ~BlockchainLMDB(); - - virtual void open(const std::string& filename); - - virtual void create(const std::string& filename); - - virtual void close(); - - virtual void sync(); - - virtual void reset(); - - virtual std::vector get_filenames() const; - - virtual bool lock(); - - virtual void unlock(); - - virtual bool block_exists(const crypto::hash& h) const; - - virtual block get_block(const crypto::hash& h) const; - - virtual uint64_t get_block_height(const crypto::hash& h) const; - - virtual block_header get_block_header(const crypto::hash& h) const; - - virtual block get_block_from_height(const uint64_t& height) const; - - virtual uint64_t get_block_timestamp(const uint64_t& height) const; - - virtual uint64_t get_top_block_timestamp() const; - - virtual size_t get_block_size(const uint64_t& height) const; - - virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const; - - virtual difficulty_type get_block_difficulty(const uint64_t& height) const; - - virtual uint64_t get_block_already_generated_coins(const uint64_t& height) const; - - virtual crypto::hash get_block_hash_from_height(const uint64_t& height) const; - - virtual std::vector get_blocks_range(const uint64_t& h1, const uint64_t& h2) const; - - virtual std::vector get_hashes_range(const uint64_t& h1, const uint64_t& h2) const; - - virtual crypto::hash top_block_hash() const; - - virtual block get_top_block() const; - - virtual uint64_t height() const; - - virtual bool tx_exists(const crypto::hash& h) const; - - virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const; - - virtual transaction get_tx(const crypto::hash& h) const; - - virtual uint64_t get_tx_count() const; - - virtual std::vector get_tx_list(const std::vector& hlist) const; - - virtual uint64_t get_tx_block_height(const crypto::hash& h) const; - - virtual uint64_t get_random_output(const uint64_t& amount) const; - - virtual uint64_t get_num_outputs(const uint64_t& amount) const; - - virtual crypto::public_key get_output_key(const uint64_t& amount, const uint64_t& index) const; - - virtual tx_out get_output(const crypto::hash& h, const uint64_t& index) const; - - /** - * @brief get an output from its global index - * - * @param index global index of the output desired - * - * @return the output associated with the index. - * Will throw OUTPUT_DNE if not output has that global index. - * Will throw DB_ERROR if there is a non-specific LMDB error in fetching - */ - tx_out get_output(const uint64_t& index) const; - - virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const; - - virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const; - - virtual std::vector get_tx_output_indices(const crypto::hash& h) const; - virtual std::vector get_tx_amount_output_indices(const crypto::hash& h) const; - - virtual bool has_key_image(const crypto::key_image& img) const; - - virtual uint64_t add_block( const block& blk - , const size_t& block_size - , const difficulty_type& cumulative_difficulty - , const uint64_t& coins_generated - , const std::vector& txs - ); - - virtual void set_batch_transactions(bool batch_transactions); - virtual void batch_start(); - virtual void batch_commit(); - virtual void batch_stop(); - virtual void batch_abort(); - - virtual void pop_block(block& blk, std::vector& txs); - -private: - virtual void add_block( const block& blk - , const size_t& block_size - , const difficulty_type& cumulative_difficulty - , const uint64_t& coins_generated - , const crypto::hash& block_hash - ); - - virtual void remove_block(); - - virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash); - - virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx); - - virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index); - - virtual void remove_output(const tx_out& tx_output); - - void remove_tx_outputs(const crypto::hash& tx_hash, const transaction& tx); - - void remove_output(const uint64_t& out_index, const uint64_t amount); - void remove_amount_output_index(const uint64_t amount, const uint64_t global_output_index); - - virtual void add_spent_key(const crypto::key_image& k_image); - - virtual void remove_spent_key(const crypto::key_image& k_image); - - /** - * @brief convert a tx output to a blob for storage - * - * @param output the output to convert - * - * @return the resultant blob - */ - blobdata output_to_blob(const tx_out& output); - - /** - * @brief convert a tx output blob to a tx output - * - * @param blob the blob to convert - * - * @return the resultant tx output - */ - tx_out output_from_blob(const blobdata& blob) const; - - /** - * @brief get the global index of the index-th output of the given amount - * - * @param amount the output amount - * @param index the index into the set of outputs of that amount - * - * @return the global index of the desired output - */ - uint64_t get_output_global_index(const uint64_t& amount, const uint64_t& index) const; - - void check_open() const; - - MDB_env* m_env; - - MDB_dbi m_blocks; - MDB_dbi m_block_heights; - MDB_dbi m_block_hashes; - MDB_dbi m_block_timestamps; - MDB_dbi m_block_sizes; - MDB_dbi m_block_diffs; - MDB_dbi m_block_coins; - - MDB_dbi m_txs; - MDB_dbi m_tx_unlocks; - MDB_dbi m_tx_heights; - MDB_dbi m_tx_outputs; - - MDB_dbi m_output_txs; - MDB_dbi m_output_indices; - MDB_dbi m_output_gindices; - MDB_dbi m_output_amounts; - MDB_dbi m_output_keys; - MDB_dbi m_outputs; - - MDB_dbi m_spent_keys; - - bool m_open; - uint64_t m_height; - uint64_t m_num_outputs; - std::string m_folder; - txn_safe* m_write_txn; // may point to either a short-lived txn or a batch txn - txn_safe m_write_batch_txn; // persist batch txn outside of BlockchainLMDB - - bool m_batch_transactions; // support for batch transactions - bool m_batch_active; // whether batch transaction is in progress -}; - -} // namespace cryptonote diff --git a/src/cryptonote_core/CMakeLists.txt b/src/cryptonote_core/CMakeLists.txt index fc9cc629b..26122e367 100644 --- a/src/cryptonote_core/CMakeLists.txt +++ b/src/cryptonote_core/CMakeLists.txt @@ -30,8 +30,6 @@ set(cryptonote_core_sources account.cpp blockchain_storage.cpp blockchain.cpp - blockchain_db.cpp - BlockchainDB_impl/db_lmdb.cpp checkpoints.cpp checkpoints_create.cpp cryptonote_basic_impl.cpp @@ -49,8 +47,6 @@ set(cryptonote_core_private_headers blockchain_storage.h blockchain_storage_boost_serialization.h blockchain.h - blockchain_db.h - BlockchainDB_impl/db_lmdb.h checkpoints.h checkpoints_create.h connection_context.h @@ -76,6 +72,7 @@ target_link_libraries(cryptonote_core LINK_PUBLIC common crypto + blockchain_db ${Boost_DATE_TIME_LIBRARY} ${Boost_PROGRAM_OPTIONS_LIBRARY} ${Boost_SERIALIZATION_LIBRARY} @@ -83,5 +80,4 @@ target_link_libraries(cryptonote_core ${Boost_FILESYSTEM_LIBRARY} ${Boost_SYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY} - ${LMDB_LIBRARY} ${EXTRA_LIBRARIES}) diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index b2d979432..9f4895736 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -38,8 +38,8 @@ #include "cryptonote_basic_impl.h" #include "tx_pool.h" #include "blockchain.h" -#include "cryptonote_core/blockchain_db.h" -#include "cryptonote_core/BlockchainDB_impl/db_lmdb.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/lmdb/db_lmdb.h" #include "cryptonote_format_utils.h" #include "cryptonote_boost_serialization.h" #include "cryptonote_config.h" diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index 3e8a2de31..477aa4964 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -50,7 +50,7 @@ #include "verification_context.h" #include "crypto/hash.h" #include "checkpoints.h" -#include "blockchain_db.h" +#include "blockchain_db/blockchain_db.h" namespace cryptonote { diff --git a/src/cryptonote_core/blockchain_db.cpp b/src/cryptonote_core/blockchain_db.cpp deleted file mode 100644 index fc8aeef4e..000000000 --- a/src/cryptonote_core/blockchain_db.cpp +++ /dev/null @@ -1,180 +0,0 @@ -// Copyright (c) 2014, The Monero Project -// -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without modification, are -// permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this list of -// conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright notice, this list -// of conditions and the following disclaimer in the documentation and/or other -// materials provided with the distribution. -// -// 3. Neither the name of the copyright holder nor the names of its contributors may be -// used to endorse or promote products derived from this software without specific -// prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL -// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF -// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -#include "cryptonote_core/blockchain_db.h" -#include "cryptonote_format_utils.h" -#include "profile_tools.h" - -using epee::string_tools::pod_to_hex; - -namespace cryptonote -{ - -void BlockchainDB::pop_block() -{ - block blk; - std::vector txs; - pop_block(blk, txs); -} - -void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr) -{ - crypto::hash tx_hash; - if (!tx_hash_ptr) - { - // should only need to compute hash for miner transactions - tx_hash = get_transaction_hash(tx); - LOG_PRINT_L3("null tx_hash_ptr - needed to compute: " << tx_hash); - } - else - { - tx_hash = *tx_hash_ptr; - } - - add_transaction_data(blk_hash, tx, tx_hash); - - // iterate tx.vout using indices instead of C++11 foreach syntax because - // we need the index - if (tx.vout.size() != 0) // it may be technically possible for a tx to have no outputs - { - for (uint64_t i = 0; i < tx.vout.size(); ++i) - { - add_output(tx_hash, tx.vout[i], i); - } - - for (const txin_v& tx_input : tx.vin) - { - if (tx_input.type() == typeid(txin_to_key)) - { - add_spent_key(boost::get(tx_input).k_image); - } - } - } -} - -uint64_t BlockchainDB::add_block( const block& blk - , const size_t& block_size - , const difficulty_type& cumulative_difficulty - , const uint64_t& coins_generated - , const std::vector& txs - ) -{ - TIME_MEASURE_START(time1); - crypto::hash blk_hash = get_block_hash(blk); - TIME_MEASURE_FINISH(time1); - time_blk_hash += time1; - - // call out to subclass implementation to add the block & metadata - time1 = epee::misc_utils::get_tick_count(); - add_block(blk, block_size, cumulative_difficulty, coins_generated, blk_hash); - TIME_MEASURE_FINISH(time1); - time_add_block1 += time1; - - // call out to add the transactions - - time1 = epee::misc_utils::get_tick_count(); - add_transaction(blk_hash, blk.miner_tx); - int tx_i = 0; - crypto::hash tx_hash = null_hash; - for (const transaction& tx : txs) - { - tx_hash = blk.tx_hashes[tx_i]; - add_transaction(blk_hash, tx, &tx_hash); - ++tx_i; - } - TIME_MEASURE_FINISH(time1); - time_add_transaction += time1; - - ++num_calls; - - return height(); -} - -void BlockchainDB::pop_block(block& blk, std::vector& txs) -{ - blk = get_top_block(); - - remove_block(); - - remove_transaction(get_transaction_hash(blk.miner_tx)); - for (const auto& h : blk.tx_hashes) - { - txs.push_back(get_tx(h)); - remove_transaction(h); - } -} - -void BlockchainDB::remove_transaction(const crypto::hash& tx_hash) -{ - transaction tx = get_tx(tx_hash); - - for (const txin_v& tx_input : tx.vin) - { - if (tx_input.type() == typeid(txin_to_key)) - { - remove_spent_key(boost::get(tx_input).k_image); - } - } - - // need tx as tx.vout has the tx outputs, and the output amounts are needed - remove_transaction_data(tx_hash, tx); -} - -void BlockchainDB::reset_stats() -{ - num_calls = 0; - time_blk_hash = 0; - time_tx_exists = 0; - time_add_block1 = 0; - time_add_transaction = 0; - time_commit1 = 0; -} - -void BlockchainDB::show_stats() -{ - LOG_PRINT_L1(ENDL - << "*********************************" - << ENDL - << "num_calls: " << num_calls - << ENDL - << "time_blk_hash: " << time_blk_hash << "ms" - << ENDL - << "time_tx_exists: " << time_tx_exists << "ms" - << ENDL - << "time_add_block1: " << time_add_block1 << "ms" - << ENDL - << "time_add_transaction: " << time_add_transaction << "ms" - << ENDL - << "time_commit1: " << time_commit1 << "ms" - << ENDL - << "*********************************" - << ENDL - ); -} - -} // namespace cryptonote diff --git a/src/cryptonote_core/blockchain_db.h b/src/cryptonote_core/blockchain_db.h deleted file mode 100644 index 2a7fa8f82..000000000 --- a/src/cryptonote_core/blockchain_db.h +++ /dev/null @@ -1,487 +0,0 @@ -// Copyright (c) 2014, The Monero Project -// -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without modification, are -// permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this list of -// conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright notice, this list -// of conditions and the following disclaimer in the documentation and/or other -// materials provided with the distribution. -// -// 3. Neither the name of the copyright holder nor the names of its contributors may be -// used to endorse or promote products derived from this software without specific -// prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL -// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF -// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#ifndef BLOCKCHAIN_DB_H -#define BLOCKCHAIN_DB_H - -#include -#include -#include -#include "crypto/hash.h" -#include "cryptonote_core/cryptonote_basic.h" -#include "cryptonote_core/difficulty.h" - -/* DB Driver Interface - * - * The DB interface is a store for the canonical block chain. - * It serves as a persistent storage for the blockchain. - * - * For the sake of efficiency, the reference implementation will also - * store some blockchain data outside of the blocks, such as spent - * transfer key images, unspent transaction outputs, etc. - * - * Transactions are duplicated so that we don't have to fetch a whole block - * in order to fetch a transaction from that block. If this is deemed - * unnecessary later, this can change. - * - * Spent key images are duplicated outside of the blocks so it is quick - * to verify an output hasn't already been spent - * - * Unspent transaction outputs are duplicated to quickly gather random - * outputs to use for mixins - * - * IMPORTANT: - * A concrete implementation of this interface should populate these - * duplicated members! It is possible to have a partial implementation - * of this interface call to private members of the interface to be added - * later that will then populate as needed. - * - * General: - * open() - * close() - * sync() - * reset() - * - * Lock and unlock provided for reorg externally, and for block - * additions internally, this way threaded reads are completely fine - * unless the blockchain is changing. - * bool lock() - * unlock() - * - * vector get_filenames() - * - * Blocks: - * bool block_exists(hash) - * height add_block(block, block_size, cumulative_difficulty, coins_generated, transactions) - * block get_block(hash) - * height get_block_height(hash) - * header get_block_header(hash) - * block get_block_from_height(height) - * size_t get_block_size(height) - * difficulty get_block_cumulative_difficulty(height) - * uint64_t get_block_already_generated_coins(height) - * uint64_t get_block_timestamp(height) - * uint64_t get_top_block_timestamp() - * hash get_block_hash_from_height(height) - * blocks get_blocks_range(height1, height2) - * hashes get_hashes_range(height1, height2) - * hash top_block_hash() - * block get_top_block() - * height height() - * void pop_block(block&, tx_list&) - * - * Transactions: - * bool tx_exists(hash) - * uint64_t get_tx_unlock_time(hash) - * tx get_tx(hash) - * uint64_t get_tx_count() - * tx_list get_tx_list(hash_list) - * height get_tx_block_height(hash) - * - * Outputs: - * index get_random_output(amount) - * uint64_t get_num_outputs(amount) - * pub_key get_output_key(amount, index) - * tx_out get_output(tx_hash, index) - * hash,index get_output_tx_and_index_from_global(index) - * hash,index get_output_tx_and_index(amount, index) - * vec get_tx_output_indices(tx_hash) - * - * - * Spent Output Key Images: - * bool has_key_image(key_image) - * - * Exceptions: - * DB_ERROR -- generic - * DB_OPEN_FAILURE - * DB_CREATE_FAILURE - * DB_SYNC_FAILURE - * BLOCK_DNE - * BLOCK_PARENT_DNE - * BLOCK_EXISTS - * BLOCK_INVALID -- considering making this multiple errors - * TX_DNE - * TX_EXISTS - * OUTPUT_DNE - * OUTPUT_EXISTS - * KEY_IMAGE_EXISTS - */ - -namespace cryptonote -{ - -// typedef for convenience -typedef std::pair tx_out_index; - -/*********************************** - * Exception Definitions - ***********************************/ -class DB_EXCEPTION : public std::exception -{ - private: - std::string m; - - protected: - DB_EXCEPTION(const char *s) : m(s) { } - - public: - virtual ~DB_EXCEPTION() { } - - const char* what() const throw() - { - return m.c_str(); - } -}; - -class DB_ERROR : public DB_EXCEPTION -{ - public: - DB_ERROR() : DB_EXCEPTION("Generic DB Error") { } - DB_ERROR(const char* s) : DB_EXCEPTION(s) { } -}; - -class DB_OPEN_FAILURE : public DB_EXCEPTION -{ - public: - DB_OPEN_FAILURE() : DB_EXCEPTION("Failed to open the db") { } - DB_OPEN_FAILURE(const char* s) : DB_EXCEPTION(s) { } -}; - -class DB_CREATE_FAILURE : public DB_EXCEPTION -{ - public: - DB_CREATE_FAILURE() : DB_EXCEPTION("Failed to create the db") { } - DB_CREATE_FAILURE(const char* s) : DB_EXCEPTION(s) { } -}; - -class DB_SYNC_FAILURE : public DB_EXCEPTION -{ - public: - DB_SYNC_FAILURE() : DB_EXCEPTION("Failed to sync the db") { } - DB_SYNC_FAILURE(const char* s) : DB_EXCEPTION(s) { } -}; - -class BLOCK_DNE : public DB_EXCEPTION -{ - public: - BLOCK_DNE() : DB_EXCEPTION("The block requested does not exist") { } - BLOCK_DNE(const char* s) : DB_EXCEPTION(s) { } -}; - -class BLOCK_PARENT_DNE : public DB_EXCEPTION -{ - public: - BLOCK_PARENT_DNE() : DB_EXCEPTION("The parent of the block does not exist") { } - BLOCK_PARENT_DNE(const char* s) : DB_EXCEPTION(s) { } -}; - -class BLOCK_EXISTS : public DB_EXCEPTION -{ - public: - BLOCK_EXISTS() : DB_EXCEPTION("The block to be added already exists!") { } - BLOCK_EXISTS(const char* s) : DB_EXCEPTION(s) { } -}; - -class BLOCK_INVALID : public DB_EXCEPTION -{ - public: - BLOCK_INVALID() : DB_EXCEPTION("The block to be added did not pass validation!") { } - BLOCK_INVALID(const char* s) : DB_EXCEPTION(s) { } -}; - -class TX_DNE : public DB_EXCEPTION -{ - public: - TX_DNE() : DB_EXCEPTION("The transaction requested does not exist") { } - TX_DNE(const char* s) : DB_EXCEPTION(s) { } -}; - -class TX_EXISTS : public DB_EXCEPTION -{ - public: - TX_EXISTS() : DB_EXCEPTION("The transaction to be added already exists!") { } - TX_EXISTS(const char* s) : DB_EXCEPTION(s) { } -}; - -class OUTPUT_DNE : public DB_EXCEPTION -{ - public: - OUTPUT_DNE() : DB_EXCEPTION("The output requested does not exist!") { } - OUTPUT_DNE(const char* s) : DB_EXCEPTION(s) { } -}; - -class OUTPUT_EXISTS : public DB_EXCEPTION -{ - public: - OUTPUT_EXISTS() : DB_EXCEPTION("The output to be added already exists!") { } - OUTPUT_EXISTS(const char* s) : DB_EXCEPTION(s) { } -}; - -class KEY_IMAGE_EXISTS : public DB_EXCEPTION -{ - public: - KEY_IMAGE_EXISTS() : DB_EXCEPTION("The spent key image to be added already exists!") { } - KEY_IMAGE_EXISTS(const char* s) : DB_EXCEPTION(s) { } -}; - -/*********************************** - * End of Exception Definitions - ***********************************/ - - -class BlockchainDB -{ -private: - /********************************************************************* - * private virtual members - *********************************************************************/ - - // tells the subclass to add the block and metadata to storage - virtual void add_block( const block& blk - , const size_t& block_size - , const difficulty_type& cumulative_difficulty - , const uint64_t& coins_generated - , const crypto::hash& blk_hash - ) = 0; - - // tells the subclass to remove data about the top block - virtual void remove_block() = 0; - - // tells the subclass to store the transaction and its metadata - virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) = 0; - - // tells the subclass to remove data about a transaction - virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) = 0; - - // tells the subclass to store an output - virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index) = 0; - - // tells the subclass to remove an output - virtual void remove_output(const tx_out& tx_output) = 0; - - // tells the subclass to store a spent key - virtual void add_spent_key(const crypto::key_image& k_image) = 0; - - // tells the subclass to remove a spent key - virtual void remove_spent_key(const crypto::key_image& k_image) = 0; - - - /********************************************************************* - * private concrete members - *********************************************************************/ - // private version of pop_block, for undoing if an add_block goes tits up - void pop_block(); - - // helper function for add_transactions, to add each individual tx - void add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr = NULL); - - // helper function to remove transaction from blockchain - void remove_transaction(const crypto::hash& tx_hash); - - uint64_t num_calls = 0; - uint64_t time_blk_hash = 0; - uint64_t time_add_block1 = 0; - uint64_t time_add_transaction = 0; - - -protected: - - mutable uint64_t time_tx_exists = 0; - uint64_t time_commit1 = 0; - - -public: - - // virtual dtor - virtual ~BlockchainDB() { }; - - // reset profiling stats - void reset_stats(); - - // show profiling stats - void show_stats(); - - // open the db at location , or create it if there isn't one. - virtual void open(const std::string& filename) = 0; - - // make sure implementation has a create function as well - virtual void create(const std::string& filename) = 0; - - // close and sync the db - virtual void close() = 0; - - // sync the db - virtual void sync() = 0; - - // reset the db -- USE WITH CARE - virtual void reset() = 0; - - // get all files used by this db (if any) - virtual std::vector get_filenames() const = 0; - - - // FIXME: these are just for functionality mocking, need to implement - // RAII-friendly and multi-read one-write friendly locking mechanism - // - // acquire db lock - virtual bool lock() = 0; - - // release db lock - virtual void unlock() = 0; - - virtual void batch_start() = 0; - virtual void batch_stop() = 0; - virtual void set_batch_transactions(bool) = 0; - - // adds a block with the given metadata to the top of the blockchain, returns the new height - // NOTE: subclass implementations of this (or the functions it calls) need - // to handle undoing any partially-added blocks in the event of a failure. - virtual uint64_t add_block( const block& blk - , const size_t& block_size - , const difficulty_type& cumulative_difficulty - , const uint64_t& coins_generated - , const std::vector& txs - ); - - // return true if a block with hash exists in the blockchain - virtual bool block_exists(const crypto::hash& h) const = 0; - - // return block with hash - virtual block get_block(const crypto::hash& h) const = 0; - - // return the height of the block with hash on the blockchain, - // throw if it doesn't exist - virtual uint64_t get_block_height(const crypto::hash& h) const = 0; - - // return header for block with hash - virtual block_header get_block_header(const crypto::hash& h) const = 0; - - // return block at height - virtual block get_block_from_height(const uint64_t& height) const = 0; - - // return timestamp of block at height - virtual uint64_t get_block_timestamp(const uint64_t& height) const = 0; - - // return timestamp of most recent block - virtual uint64_t get_top_block_timestamp() const = 0; - - // return block size of block at height - virtual size_t get_block_size(const uint64_t& height) const = 0; - - // return cumulative difficulty up to and including block at height - virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const = 0; - - // return difficulty of block at height - virtual difficulty_type get_block_difficulty(const uint64_t& height) const = 0; - - // return number of coins generated up to and including block at height - virtual uint64_t get_block_already_generated_coins(const uint64_t& height) const = 0; - - // return hash of block at height - virtual crypto::hash get_block_hash_from_height(const uint64_t& height) const = 0; - - // return vector of blocks in range of height (inclusively) - virtual std::vector get_blocks_range(const uint64_t& h1, const uint64_t& h2) const = 0; - - // return vector of block hashes in range of height (inclusively) - virtual std::vector get_hashes_range(const uint64_t& h1, const uint64_t& h2) const = 0; - - // return the hash of the top block on the chain - virtual crypto::hash top_block_hash() const = 0; - - // return the block at the top of the blockchain - virtual block get_top_block() const = 0; - - // return the height of the chain - virtual uint64_t height() const = 0; - - // pops the top block off the blockchain. - // Returns by reference the popped block and its associated transactions - // - // IMPORTANT: - // When a block is popped, the transactions associated with it need to be - // removed, as well as outputs and spent key images associated with - // those transactions. - virtual void pop_block(block& blk, std::vector& txs); - - - // return true if a transaction with hash exists - virtual bool tx_exists(const crypto::hash& h) const = 0; - - // return unlock time of tx with hash - virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const = 0; - - // return tx with hash - // throw if no such tx exists - virtual transaction get_tx(const crypto::hash& h) const = 0; - - // returns the total number of transactions in all blocks - virtual uint64_t get_tx_count() const = 0; - - // return list of tx with hashes . - // TODO: decide if a missing hash means return empty list - // or just skip that hash - virtual std::vector get_tx_list(const std::vector& hlist) const = 0; - - // returns height of block that contains transaction with hash - virtual uint64_t get_tx_block_height(const crypto::hash& h) const = 0; - - // return global output index of a random output of amount - virtual uint64_t get_random_output(const uint64_t& amount) const = 0; - - // returns the total number of outputs of amount - virtual uint64_t get_num_outputs(const uint64_t& amount) const = 0; - - // return public key for output with global output amount and index - virtual crypto::public_key get_output_key(const uint64_t& amount, const uint64_t& index) const = 0; - - // returns the output indexed by in the transaction with hash - virtual tx_out get_output(const crypto::hash& h, const uint64_t& index) const = 0; - - // returns the tx hash associated with an output, referenced by global output index - virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const = 0; - - // returns the transaction-local reference for the output with at - // return type is pair of tx hash and index - virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const = 0; - - // return a vector of indices corresponding to the global output index for - // each output in the transaction with hash - virtual std::vector get_tx_output_indices(const crypto::hash& h) const = 0; - // return a vector of indices corresponding to the amount output index for - // each output in the transaction with hash - virtual std::vector get_tx_amount_output_indices(const crypto::hash& h) const = 0; - - // returns true if key image is present in spent key images storage - virtual bool has_key_image(const crypto::key_image& img) const = 0; - -}; // class BlockchainDB - - -} // namespace cryptonote - -#endif // BLOCKCHAIN_DB_H diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt index 5f60857d6..1ad1949d3 100644 --- a/src/daemon/CMakeLists.txt +++ b/src/daemon/CMakeLists.txt @@ -59,6 +59,7 @@ bitmonero_add_executable(daemon target_link_libraries(daemon LINK_PRIVATE rpc + blockchain_db cryptonote_core crypto common -- cgit v1.2.3 From eee3ee70730bc3bf7874e5f22139a55dac3a2cdd Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Fri, 13 Mar 2015 21:39:27 -0400 Subject: BlockchainDB implementations have names now In order to make things more general, BlockchainDB now has get_db_name() which should return a string with the "name" of that type of db. This "name" will be the subfolder name that holds that db type's files within the monero folder. Small bugfix: blockchain_converter was not correctly appending this in the prior hard-coded-string implementation of the subfolder data directory concept. --- src/blockchain_converter/blockchain_converter.cpp | 6 +++++- src/blockchain_db/blockchain_db.h | 3 +++ src/blockchain_db/lmdb/db_lmdb.cpp | 7 +++++++ src/blockchain_db/lmdb/db_lmdb.h | 2 ++ src/cryptonote_core/blockchain.cpp | 5 +++-- 5 files changed, 20 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/blockchain_converter/blockchain_converter.cpp b/src/blockchain_converter/blockchain_converter.cpp index aae569e58..8ac9b81bb 100644 --- a/src/blockchain_converter/blockchain_converter.cpp +++ b/src/blockchain_converter/blockchain_converter.cpp @@ -71,7 +71,11 @@ int main(int argc, char* argv[]) blockchain = new BlockchainLMDB(); - blockchain->open(default_data_path.string()); + boost::filesystem::path db_path(default_data_path); + + db_path /= blockchain->get_db_name(); + + blockchain->open(db_path.string()); for (uint64_t height, i = 0; i < (height = c.m_storage.get_current_blockchain_height()); ++i) { diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 2a7fa8f82..d2b4a07a7 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -343,6 +343,9 @@ public: // get all files used by this db (if any) virtual std::vector get_filenames() const = 0; + // return the name of the folder the db's file(s) should reside in + virtual std::string get_db_name() const = 0; + // FIXME: these are just for functionality mocking, need to implement // RAII-friendly and multi-read one-write friendly locking mechanism diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index feff4a272..ee49e1827 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -775,6 +775,13 @@ std::vector BlockchainLMDB::get_filenames() const return filenames; } +std::string BlockchainLMDB::get_db_name() const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + + return std::string("lmdb"); +} + // TODO: this? bool BlockchainLMDB::lock() { diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index f6582fce4..1a684548e 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -126,6 +126,8 @@ public: virtual std::vector get_filenames() const; + virtual std::string get_db_name() const; + virtual bool lock(); virtual void unlock(); diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 9f4895736..b7920c99c 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -231,17 +231,18 @@ bool Blockchain::init(const std::string& config_folder, bool testnet) LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); + // TODO: make this configurable m_db = new BlockchainLMDB(); m_config_folder = config_folder; m_testnet = testnet; boost::filesystem::path folder(m_config_folder); - folder /= "lmdb"; + + folder /= m_db->get_db_name(); LOG_PRINT_L0("Loading blockchain from folder " << folder.c_str() << " ..."); - //FIXME: update filename for BlockchainDB const std::string filename = folder.string(); try { -- cgit v1.2.3 From acb5d291b893bf2b3e6378ea516690d4dc04fecc Mon Sep 17 00:00:00 2001 From: warptangent Date: Tue, 3 Mar 2015 13:09:49 -0800 Subject: Update and relocate comment that applies class wide --- src/blockchain_db/lmdb/db_lmdb.cpp | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) (limited to 'src') diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index ee49e1827..495f93bdc 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -149,6 +149,20 @@ inline void lmdb_db_open(MDB_txn* txn, const char* name, int flags, MDB_dbi& dbi namespace cryptonote { +// If m_batch_active is set, a batch transaction exists beyond this class, such +// as a batch import with verification enabled, or possibly (later) a batch +// network sync. +// +// For some of the lookup methods, such as get_block_timestamp(), tx_exists(), +// and get_tx(), when m_batch_active is set, the lookup uses the batch +// transaction. This isn't only because the transaction is available, but it's +// necessary so that lookups include the database updates only present in the +// current batch write. +// +// A regular network sync without batch writes is expected to open a new read +// transaction, as those lookups are part of the validation done prior to the +// write for block and tx data, so no write transaction is open at the time. + void BlockchainLMDB::add_block( const block& blk , const size_t& block_size , const difficulty_type& cumulative_difficulty @@ -1260,14 +1274,6 @@ uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - // If m_batch_active is set, a batch transaction exists beyond this class, - // such as a batch import with verification enabled, or possibly (later) a - // batch network sync. - // - // A regular network sync without batching would be expected to open a new - // read transaction here, as validation is done prior to the write for block - // and tx data. - txn_safe txn; txn_safe* txn_ptr = &txn; if (m_batch_active) -- cgit v1.2.3 From a3dd9d10f39bddfb386370eb01e76cbbb0fe6fbb Mon Sep 17 00:00:00 2001 From: warptangent Date: Tue, 3 Mar 2015 17:20:27 -0800 Subject: blockchain_converter: Add support for batch transactions Add log level support. Add testnet support. Add command-line options: --help --data-dir --testnet-data-dir --testnet --log-level --batch --batch-size --block-number See help for usage. Run at log level 1 to see profiling stats. --- src/blockchain_converter/blockchain_converter.cpp | 196 +++++++++++++++++++--- 1 file changed, 173 insertions(+), 23 deletions(-) (limited to 'src') diff --git a/src/blockchain_converter/blockchain_converter.cpp b/src/blockchain_converter/blockchain_converter.cpp index 8ac9b81bb..eaf799a36 100644 --- a/src/blockchain_converter/blockchain_converter.cpp +++ b/src/blockchain_converter/blockchain_converter.cpp @@ -38,52 +38,179 @@ #include "cryptonote_core/blockchain.h" #include "blockchain_db/lmdb/db_lmdb.h" #include "cryptonote_core/tx_pool.h" +#include "common/command_line.h" +#include "serialization/json_utils.h" +#include "include_base_utils.h" +#include "version.h" #include +// CONFIG +static bool opt_batch = true; +static bool opt_testnet = false; + +// number of blocks per batch transaction +// adjustable through command-line argument according to available RAM +static uint64_t db_batch_size = 20000; + + +namespace po = boost::program_options; + using namespace cryptonote; +using namespace epee; struct fake_core { - tx_memory_pool m_pool; Blockchain dummy; + tx_memory_pool m_pool; blockchain_storage m_storage; - - fake_core(const boost::filesystem::path &path) : m_pool(dummy), dummy(m_pool), m_storage(&m_pool) +#if !defined(BLOCKCHAIN_DB) + // for multi_db_runtime: + fake_core(const boost::filesystem::path &path, const bool use_testnet) : dummy(m_pool), m_pool(&dummy), m_storage(m_pool) +#else + // for multi_db_compile: + fake_core(const boost::filesystem::path &path, const bool use_testnet) : dummy(m_pool), m_pool(dummy), m_storage(&m_pool) +#endif { m_pool.init(path.string()); - m_storage.init(path.string(), false); + m_storage.init(path.string(), use_testnet); } }; int main(int argc, char* argv[]) { - std::string dir = tools::get_default_data_dir(); - boost::filesystem::path default_data_path {dir}; - if (argc >= 2 && !strcmp(argv[1], "--testnet")) { - default_data_path /= "testnet"; + uint64_t height = 0; + uint64_t start_block = 0; + uint64_t end_block = 0; + uint64_t num_blocks = 0; + + boost::filesystem::path default_data_path {tools::get_default_data_dir()}; + boost::filesystem::path default_testnet_data_path {default_data_path / "testnet"}; + + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor arg_log_level = {"log-level", "", LOG_LEVEL_0}; + const command_line::arg_descriptor arg_batch_size = {"batch-size", "", db_batch_size}; + const command_line::arg_descriptor arg_testnet_on = { + "testnet" + , "Run on testnet." + , opt_testnet + }; + const command_line::arg_descriptor arg_block_number = + {"block-number", "Number of blocks (default: use entire source blockchain)", + 0}; + + command_line::add_arg(desc_cmd_sett, command_line::arg_data_dir, default_data_path.string()); + command_line::add_arg(desc_cmd_sett, command_line::arg_testnet_data_dir, default_testnet_data_path.string()); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_batch_size); + command_line::add_arg(desc_cmd_sett, arg_testnet_on); + command_line::add_arg(desc_cmd_sett, arg_block_number); + + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + const command_line::arg_descriptor arg_batch = {"batch", + "Batch transactions for faster import", true}; + + // call add_options() directly for these arguments since command_line helpers + // support only boolean switch, not boolean argument + desc_cmd_sett.add_options() + (arg_batch.name, make_semantic(arg_batch), arg_batch.description) + ; + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + po::store(po::parse_command_line(argc, argv, desc_options), vm); + po::notify(vm); + + return true; + }); + if (!r) + return 1; + + int log_level = command_line::get_arg(vm, arg_log_level); + opt_batch = command_line::get_arg(vm, arg_batch); + db_batch_size = command_line::get_arg(vm, arg_batch_size); + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << CRYPTONOTE_NAME << " v" << MONERO_VERSION_FULL << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; } - fake_core c(default_data_path); + if (! opt_batch && ! vm["batch-size"].defaulted()) + { + std::cerr << "Error: batch-size set, but batch option not enabled" << ENDL; + return 1; + } + if (! db_batch_size) + { + std::cerr << "Error: batch-size must be > 0" << ENDL; + return 1; + } - BlockchainDB *blockchain; + log_space::get_set_log_detalisation_level(true, log_level); + log_space::log_singletone::add_logger(LOGGER_CONSOLE, NULL, NULL); + LOG_PRINT_L0("Starting..."); - blockchain = new BlockchainLMDB(); + std::string src_folder; + opt_testnet = command_line::get_arg(vm, arg_testnet_on); + auto data_dir_arg = opt_testnet ? command_line::arg_testnet_data_dir : command_line::arg_data_dir; + src_folder = command_line::get_arg(vm, data_dir_arg); + boost::filesystem::path dest_folder(src_folder); - boost::filesystem::path db_path(default_data_path); + num_blocks = command_line::get_arg(vm, arg_block_number); - db_path /= blockchain->get_db_name(); + if (opt_batch) + { + LOG_PRINT_L0("batch: " << std::boolalpha << opt_batch << std::noboolalpha + << " batch size: " << db_batch_size); + } + else + { + LOG_PRINT_L0("batch: " << std::boolalpha << opt_batch << std::noboolalpha); + } + LOG_PRINT_L0("testnet: " << std::boolalpha << opt_testnet << std::noboolalpha); + + fake_core c(src_folder, opt_testnet); - blockchain->open(db_path.string()); + height = c.m_storage.get_current_blockchain_height(); + if (! num_blocks || num_blocks > height) + end_block = height - 1; + else + end_block = start_block + num_blocks - 1; + + BlockchainDB *blockchain; + blockchain = new BlockchainLMDB(opt_batch); + dest_folder /= blockchain->get_db_name(); + LOG_PRINT_L0("Source blockchain: " << src_folder); + LOG_PRINT_L0("Dest blockchain: " << dest_folder.string()); + LOG_PRINT_L0("Opening LMDB: " << dest_folder.string()); + blockchain->open(dest_folder.string()); - for (uint64_t height, i = 0; i < (height = c.m_storage.get_current_blockchain_height()); ++i) + if (opt_batch) + blockchain->batch_start(); + uint64_t i = 0; + for (i = start_block; i < end_block + 1; ++i) { - if (i % 10 == 0) + // block: i height: i+1 end height: end_block + 1 + if ((i+1) % 10 == 0) { - std::cout << "\r \r" << "block " << i << "/" << height - << " (" << (i+1)*100/height<< "%)" << std::flush; + std::cout << "\r \r" << "height " << i+1 << "/" << + end_block+1 << " (" << (i+1)*100/(end_block+1)<< "%)" << std::flush; } + // for debugging: + // std::cout << "height " << i+1 << "/" << end_block+1 + // << " ((" << i+1 << ")*100/(end_block+1))" << "%)" << ENDL; + block b = c.m_storage.get_block(i); size_t bsize = c.m_storage.get_block_size(i); difficulty_type bdiff = c.m_storage.get_block_cumulative_difficulty(i); @@ -94,8 +221,8 @@ int main(int argc, char* argv[]) c.m_storage.get_transactions(b.tx_hashes, txs, missed); if (missed.size()) { - std::cout << std::endl; - std::cerr << "Missed transaction(s) for block at height " << i << ", exiting" << std::endl; + std::cout << ENDL; + std::cerr << "Missed transaction(s) for block at height " << i + 1 << ", exiting" << ENDL; delete blockchain; return 1; } @@ -103,17 +230,40 @@ int main(int argc, char* argv[]) try { blockchain->add_block(b, bsize, bdiff, bcoins, txs); + + if (opt_batch) + { + if ((i < end_block) && ((i + 1) % db_batch_size == 0)) + { + std::cout << "\r \r"; + std::cout << "[- batch commit at height " << i + 1 << " -]" << ENDL; + blockchain->batch_stop(); + blockchain->batch_start(); + std::cout << ENDL; + blockchain->show_stats(); + } + } } catch (const std::exception& e) { - std::cout << std::endl; - std::cerr << "Error adding block to new blockchain: " << e.what() << std::endl; + std::cout << ENDL; + std::cerr << "Error adding block to new blockchain: " << e.what() << ENDL; delete blockchain; return 2; } } + if (opt_batch) + { + std::cout << "\r \r" << "height " << i << "/" << + end_block+1 << " (" << (i)*100/(end_block+1)<< "%)" << std::flush; + std::cout << ENDL; + std::cout << "[- batch commit at height " << i << " -]" << ENDL; + blockchain->batch_stop(); + } + std::cout << ENDL; + blockchain->show_stats(); + std::cout << "Finished at height: " << i << " block: " << i-1 << ENDL; - std::cout << std::endl; delete blockchain; return 0; } -- cgit v1.2.3 From ca75b4789cf6ff379a3d8191e8a8a03cc669de31 Mon Sep 17 00:00:00 2001 From: warptangent Date: Tue, 10 Feb 2015 16:44:32 -0800 Subject: Blockchain: add get_db() accessor, needed for blockchain_import This handling may be changed in the future. --- src/cryptonote_core/blockchain.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'src') diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index 477aa4964..e86b3887e 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -145,6 +145,11 @@ namespace cryptonote void set_enforce_dns_checkpoints(bool enforce); bool update_checkpoints(const std::string& file_path, bool check_dns); + BlockchainDB* get_db() + { + return m_db; + } + private: typedef std::unordered_map blocks_by_id_index; typedef std::unordered_map transactions_container; -- cgit v1.2.3 From 260cc56faed3ef9f9c44c5d021dd88d591e33a9e Mon Sep 17 00:00:00 2001 From: warptangent Date: Tue, 10 Feb 2015 15:13:32 -0800 Subject: Add blockchain_import utility This imports to the blockchain database from an exported blockchain file. It can be used to bootstrap a new database or to add blocks to an existing one. Supports: - both the in-memory and LMDB implementations - optional: batching, verification, testnet See help for usage. Based on work by tomerkon. See https://github.com/tomerkon src/cryptonote_core/bootfileloader.{h,cpp} --- src/blockchain_converter/CMakeLists.txt | 27 + src/blockchain_converter/blockchain_import.cpp | 687 +++++++++++++++++++++++++ src/blockchain_converter/fake_core.h | 143 +++++ src/blockchain_converter/import.h | 37 ++ 4 files changed, 894 insertions(+) create mode 100644 src/blockchain_converter/blockchain_import.cpp create mode 100644 src/blockchain_converter/fake_core.h create mode 100644 src/blockchain_converter/import.h (limited to 'src') diff --git a/src/blockchain_converter/CMakeLists.txt b/src/blockchain_converter/CMakeLists.txt index a91624f4f..cc400d927 100644 --- a/src/blockchain_converter/CMakeLists.txt +++ b/src/blockchain_converter/CMakeLists.txt @@ -35,6 +35,18 @@ set(blockchain_converter_private_headers) bitmonero_private_headers(blockchain_converter ${blockchain_converter_private_headers}) +set(blockchain_import_sources + blockchain_import.cpp + ) + +set(blockchain_import_private_headers + import.h + fake_core.h + ) + +bitmonero_private_headers(blockchain_import + ${blockchain_import_private_headers}) + if (BLOCKCHAIN_DB STREQUAL DB_LMDB) bitmonero_add_executable(blockchain_converter ${blockchain_converter_sources} @@ -52,3 +64,18 @@ set_property(TARGET blockchain_converter PROPERTY OUTPUT_NAME "blockchain_converter") endif () + +bitmonero_add_executable(blockchain_import + ${blockchain_import_sources} + ${blockchain_import_private_headers}) + +target_link_libraries(blockchain_import + LINK_PRIVATE + cryptonote_core + ${CMAKE_THREAD_LIBS_INIT}) + +add_dependencies(blockchain_import + version) +set_property(TARGET blockchain_import + PROPERTY + OUTPUT_NAME "blockchain_import") diff --git a/src/blockchain_converter/blockchain_import.cpp b/src/blockchain_converter/blockchain_import.cpp new file mode 100644 index 000000000..ce2abf4ac --- /dev/null +++ b/src/blockchain_converter/blockchain_import.cpp @@ -0,0 +1,687 @@ +// Copyright (c) 2014-2015, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include +#include +#include + +#include +#include +#include +#include "cryptonote_core/cryptonote_basic.h" +#include "cryptonote_core/cryptonote_format_utils.h" +#include "cryptonote_core/cryptonote_boost_serialization.h" +#include "serialization/json_utils.h" // dump_json() +#include "include_base_utils.h" +#include "common/command_line.h" +#include "version.h" + + +#include "import.h" +#include "fake_core.h" + +// CONFIG +static bool opt_batch = true; +static bool opt_verify = true; // use add_new_block, which does verification before calling add_block +static bool opt_resume = true; +static bool opt_testnet = true; + +// number of blocks per batch transaction +// adjustable through command-line argument according to available RAM +static uint64_t db_batch_size = 20000; + + +namespace po = boost::program_options; + +using namespace cryptonote; +using namespace epee; + +int count_blocks(std::string& import_file_path) +{ + boost::filesystem::path raw_file_path(import_file_path); + boost::system::error_code ec; + if (!boost::filesystem::exists(raw_file_path, ec)) + { + LOG_PRINT_L0("import file not found: " << raw_file_path); + throw std::runtime_error("Aborting"); + } + std::ifstream import_file; + import_file.open(import_file_path, std::ios_base::binary | std::ifstream::in); + + uint64_t h = 0; + if (import_file.fail()) + { + LOG_PRINT_L0("import_file.open() fail"); + throw std::runtime_error("Aborting"); + } + LOG_PRINT_L0("Scanning blockchain from import file..."); + char buffer1[STR_LENGTH_OF_INT + 1]; + block b; + transaction tx; + bool quit = false; + uint64_t bytes_read = 0; + int progress_interval = 10; + + while (! quit) + { + int chunk_size; + import_file.read(buffer1, STR_LENGTH_OF_INT); + if (!import_file) { + std::cout << "\r \r"; + LOG_PRINT_L1("End of import file reached"); + quit = true; + break; + } + h += NUM_BLOCKS_PER_CHUNK; + if (h % progress_interval == 0) + { + std::cout << "\r \r" << "block height: " << h << + std::flush; + } + bytes_read += STR_LENGTH_OF_INT; + buffer1[STR_LENGTH_OF_INT] = '\0'; + chunk_size = atoi(buffer1); + if (chunk_size > BUFFER_SIZE) + { + std::cout << "\r \r"; + LOG_PRINT_L0("WARNING: chunk_size " << chunk_size << " > BUFFER_SIZE " << BUFFER_SIZE + << " height: " << h); + throw std::runtime_error("Aborting: chunk size exceeds buffer size"); + } + if (chunk_size > 100000) + { + std::cout << "\r \r"; + LOG_PRINT_L0("WARNING: chunk_size " << chunk_size << " > 100000" << " height: " + << h); + } + else if (chunk_size <= 0) { + std::cout << "\r \r"; + LOG_PRINT_L0("ERROR: chunk_size " << chunk_size << " <= 0" << " height: " << h); + throw std::runtime_error("Aborting"); + } + // skip to next expected block size value + import_file.seekg(chunk_size, std::ios_base::cur); + if (! import_file) { + std::cout << "\r \r"; + LOG_PRINT_L0("ERROR: unexpected end of import file: bytes read before error: " + << import_file.gcount() << " of chunk_size " << chunk_size); + throw std::runtime_error("Aborting"); + } + bytes_read += chunk_size; + std::cout << "\r \r"; + LOG_PRINT_L3("Total bytes scanned: " << bytes_read); + } + + import_file.close(); + + std::cout << ENDL; + std::cout << "Done scanning import file" << ENDL; + std::cout << "Total bytes scanned: " << bytes_read << ENDL; + std::cout << "Height: " << h << ENDL; + + return h; +} + +template +int import_from_file(FakeCore& simple_core, std::string& import_file_path) +{ +#if !defined(BLOCKCHAIN_DB) + static_assert(std::is_same::value || std::is_same::value, + "FakeCore constraint error"); +#endif +#if !defined(BLOCKCHAIN_DB) || (BLOCKCHAIN_DB == DB_LMDB) + if (std::is_same::value) + { + // Reset stats, in case we're using newly created db, accumulating stats + // from addition of genesis block. + // This aligns internal db counts with importer counts. + simple_core.m_storage.get_db()->reset_stats(); + } +#endif + boost::filesystem::path raw_file_path(import_file_path); + boost::system::error_code ec; + if (!boost::filesystem::exists(raw_file_path, ec)) + { + LOG_PRINT_L0("import file not found: " << raw_file_path); + return false; + } + + uint64_t source_height = count_blocks(import_file_path); + LOG_PRINT_L0("import file blockchain height: " << source_height); + + std::ifstream import_file; + import_file.open(import_file_path, std::ios_base::binary | std::ifstream::in); + + uint64_t h = 0; + if (import_file.fail()) + { + LOG_PRINT_L0("import_file.open() fail"); + return false; + } + char buffer1[STR_LENGTH_OF_INT + 1]; + char buffer_block[BUFFER_SIZE]; + block b; + transaction tx; + int quit = 0; + uint64_t bytes_read = 0; + + uint64_t start_height = 1; + if (opt_resume) + start_height = simple_core.m_storage.get_current_blockchain_height(); + + // Note that a new blockchain will start with a height of 1 (block number 0) + // due to genesis block being added at initialization. + + // CONFIG + // TODO: can expand on this, e.g. with --block-number option + uint64_t stop_height = source_height; + + // These are what we'll try to use, and they don't have to be a determination + // from source and destination blockchains, but those are the current + // defaults. + LOG_PRINT_L0("start height: " << start_height << " stop height: " << + stop_height); + + bool use_batch = false; + if (opt_batch) + { + if (simple_core.support_batch) + use_batch = true; + else + LOG_PRINT_L0("WARNING: batch transactions enabled but unsupported or unnecessary for this database engine - ignoring"); + } + + if (use_batch) + simple_core.batch_start(); + + LOG_PRINT_L0("Reading blockchain from import file..."); + std::cout << ENDL; + + // Within the loop, we skip to start_height before we start adding. + // TODO: Not a bottleneck, but we can use what's done in count_blocks() and + // only do the chunk size reads, skipping the chunk content reads until we're + // at start_height. + while (! quit) + { + int chunk_size; + import_file.read(buffer1, STR_LENGTH_OF_INT); + if (! import_file) { + std::cout << "\r \r"; + LOG_PRINT_L0("End of import file reached"); + quit = 1; + break; + } + bytes_read += STR_LENGTH_OF_INT; + buffer1[STR_LENGTH_OF_INT] = '\0'; + chunk_size = atoi(buffer1); + if (chunk_size > BUFFER_SIZE) + { + LOG_PRINT_L0("WARNING: chunk_size " << chunk_size << " > BUFFER_SIZE " << BUFFER_SIZE); + throw std::runtime_error("Aborting: chunk size exceeds buffer size"); + } + if (chunk_size > 100000) + { + LOG_PRINT_L0("WARNING: chunk_size " << chunk_size << " > 100000"); + } + else if (chunk_size < 0) { + LOG_PRINT_L0("ERROR: chunk_size " << chunk_size << " < 0"); + return 2; + } + import_file.read(buffer_block, chunk_size); + if (! import_file) { + LOG_PRINT_L0("ERROR: unexpected end of import file: bytes read before error: " + << import_file.gcount() << " of chunk_size " << chunk_size); + return 2; + } + bytes_read += chunk_size; + LOG_PRINT_L3("Total bytes read: " << bytes_read); + + if (h + NUM_BLOCKS_PER_CHUNK < start_height + 1) + { + h += NUM_BLOCKS_PER_CHUNK; + continue; + } + if (h > stop_height) + { + LOG_PRINT_L0("Specified height reached - stopping. height: " << h << " block: " << h-1); + quit = 1; + break; + } + + try + { + boost::iostreams::basic_array_source device(buffer_block, chunk_size); + boost::iostreams::stream> s(device); + boost::archive::binary_iarchive a(s); + + int display_interval = 1000; + int progress_interval = 10; + for (int chunk_ind = 0; chunk_ind < NUM_BLOCKS_PER_CHUNK; chunk_ind++) + { + h++; + if (h % display_interval == 0) + { + std::cout << "\r \r"; + LOG_PRINT_L0("loading block height " << h); + } + else + { + LOG_PRINT_L3("loading block height " << h); + } + try { + a >> b; + } + catch (const std::exception& e) + { + std::cout << "\r \r"; + LOG_PRINT_RED_L0("exception while de-archiving block, height=" << h); + quit = 1; + break; + } + LOG_PRINT_L2("block prev_id: " << b.prev_id << ENDL); + + if (h % progress_interval == 0) + { + std::cout << "\r \r" << "block " << h-1 + << std::flush; + } + + std::vector txs; + + int num_txs; + try + { + a >> num_txs; + } + catch (const std::exception& e) + { + std::cout << "\r \r"; + LOG_PRINT_RED_L0("exception while de-archiving tx-num, height=" << h); + quit = 1; + break; + } + for(int tx_num = 1; tx_num <= num_txs; tx_num++) + { + try { + a >> tx; + } + catch (const std::exception& e) + { + LOG_PRINT_RED_L0("exception while de-archiving tx, height=" << h <<", tx_num=" << tx_num); + quit = 1; + break; + } + // if (tx_num == 1) { + // std::cout << "coinbase transaction" << ENDL; + // } + // crypto::hash hsh = null_hash; + // size_t blob_size = 0; + // NOTE: all tx hashes except for coinbase tx are available in the block data + // get_transaction_hash(tx, hsh, blob_size); + // LOG_PRINT_L0("tx " << tx_num << " " << hsh << " : " << ENDL); + // LOG_PRINT_L0(obj_to_json_str(tx) << ENDL); + + // add blocks with verification. + // for Blockchain and blockchain_storage add_new_block(). + if (opt_verify) + { + if (tx_num == 1) { + continue; // coinbase transaction. no need to insert to tx_pool. + } + // crypto::hash hsh = null_hash; + // size_t blob_size = 0; + // get_transaction_hash(tx, hsh, blob_size); + tx_verification_context tvc = AUTO_VAL_INIT(tvc); + bool r = true; + r = simple_core.m_pool.add_tx(tx, tvc, true); + if (!r) + { + LOG_PRINT_RED_L0("failed to add transaction to transaction pool, height=" << h <<", tx_num=" << tx_num); + quit = 1; + break; + } + } + else + { + // for add_block() method, without (much) processing. + // don't add coinbase transaction to txs. + // + // because add_block() calls + // add_transaction(blk_hash, blk.miner_tx) first, and + // then a for loop for the transactions in txs. + if (tx_num > 1) + { + txs.push_back(tx); + } + } + } + + if (opt_verify) + { + block_verification_context bvc = boost::value_initialized(); + simple_core.m_storage.add_new_block(b, bvc); + + if (bvc.m_verifivation_failed) + { + LOG_PRINT_L0("Failed to add block to blockchain, verification failed, height = " << h); + LOG_PRINT_L0("skipping rest of import file"); + // ok to commit previously batched data because it failed only in + // verification of potential new block with nothing added to batch + // yet + quit = 1; + break; + } + if (! bvc.m_added_to_main_chain) + { + LOG_PRINT_L0("Failed to add block to blockchain, height = " << h); + LOG_PRINT_L0("skipping rest of import file"); + // make sure we don't commit partial block data + quit = 2; + break; + } + } + else + { + size_t block_size; + difficulty_type cumulative_difficulty; + uint64_t coins_generated; + + a >> block_size; + a >> cumulative_difficulty; + a >> coins_generated; + + std::cout << "\r \r"; + LOG_PRINT_L2("block_size: " << block_size); + LOG_PRINT_L2("cumulative_difficulty: " << cumulative_difficulty); + LOG_PRINT_L2("coins_generated: " << coins_generated); + + try + { + simple_core.add_block(b, block_size, cumulative_difficulty, coins_generated, txs); + } + catch (const std::exception& e) + { + std::cout << "\r \r"; + LOG_PRINT_RED_L0("Error adding block to blockchain: " << e.what()); + quit = 2; // make sure we don't commit partial block data + break; + } + } + + if (use_batch) + { + if (h % db_batch_size == 0) + { + std::cout << "\r \r"; + std::cout << ENDL << "[- batch commit at height " << h << " -]" << ENDL; + simple_core.batch_stop(); + simple_core.batch_start(); + std::cout << ENDL; +#if !defined(BLOCKCHAIN_DB) || (BLOCKCHAIN_DB == DB_LMDB) + simple_core.m_storage.get_db()->show_stats(); +#endif + } + } + } + } + catch (const std::exception& e) + { + std::cout << "\r \r"; + LOG_PRINT_RED_L0("exception while reading from import file, height=" << h); + return 2; + } + } // while + + import_file.close(); + + if (use_batch) + { + if (quit > 1) + { + // There was an error, so don't commit pending data. + // Destructor will abort write txn. + } + else + { + simple_core.batch_stop(); + } +#if !defined(BLOCKCHAIN_DB) || (BLOCKCHAIN_DB == DB_LMDB) + simple_core.m_storage.get_db()->show_stats(); +#endif + if (h > 0) + LOG_PRINT_L0("Finished at height: " << h << " block: " << h-1); + } + std::cout << ENDL; + return 0; +} + +int main(int argc, char* argv[]) +{ + std::string import_filename = BLOCKCHAIN_RAW; +#if defined(BLOCKCHAIN_DB) && (BLOCKCHAIN_DB == DB_MEMORY) + std::string default_db_engine = "memory"; +#else + std::string default_db_engine = "lmdb"; +#endif + + uint32_t log_level = LOG_LEVEL_0; + std::string dirname; + std::string db_engine; + + boost::filesystem::path default_data_path {tools::get_default_data_dir()}; + boost::filesystem::path default_testnet_data_path {default_data_path / "testnet"}; + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor arg_log_level = {"log-level", "", log_level}; + const command_line::arg_descriptor arg_batch_size = {"batch-size", "", db_batch_size}; + const command_line::arg_descriptor arg_testnet_on = { + "testnet" + , "Run on testnet." + , false + }; + const command_line::arg_descriptor arg_count_blocks = { + "count-blocks" + , "Count blocks in import file and exit" + , false + }; + const command_line::arg_descriptor arg_database = { + "database", "available: memory, lmdb" + , default_db_engine + }; + const command_line::arg_descriptor arg_verify = {"verify", + "Verify blocks and transactions during import", true}; + const command_line::arg_descriptor arg_batch = {"batch", + "Batch transactions for faster import", true}; + const command_line::arg_descriptor arg_resume = {"resume", + "Resume from current height if output database already exists", true}; + + command_line::add_arg(desc_cmd_sett, command_line::arg_data_dir, default_data_path.string()); + command_line::add_arg(desc_cmd_sett, command_line::arg_testnet_data_dir, default_testnet_data_path.string()); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_batch_size); + command_line::add_arg(desc_cmd_sett, arg_testnet_on); + command_line::add_arg(desc_cmd_sett, arg_database); + + command_line::add_arg(desc_cmd_only, arg_count_blocks); + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + // call add_options() directly for these arguments since + // command_line helpers support only boolean switch, not boolean argument + desc_cmd_sett.add_options() + (arg_verify.name, make_semantic(arg_verify), arg_verify.description) + (arg_batch.name, make_semantic(arg_batch), arg_batch.description) + (arg_resume.name, make_semantic(arg_resume), arg_resume.description) + ; + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + po::store(po::parse_command_line(argc, argv, desc_options), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + log_level = command_line::get_arg(vm, arg_log_level); + opt_verify = command_line::get_arg(vm, arg_verify); + opt_batch = command_line::get_arg(vm, arg_batch); + opt_resume = command_line::get_arg(vm, arg_resume); + db_batch_size = command_line::get_arg(vm, arg_batch_size); + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << CRYPTONOTE_NAME << " v" << MONERO_VERSION_FULL << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + if (! opt_batch && ! vm["batch-size"].defaulted()) + { + std::cerr << "Error: batch-size set, but batch option not enabled" << ENDL; + exit(1); + } + if (! db_batch_size) + { + std::cerr << "Error: batch-size must be > 0" << ENDL; + exit(1); + } + + std::vector db_engines {"memory", "lmdb"}; + + opt_testnet = command_line::get_arg(vm, arg_testnet_on); + auto data_dir_arg = opt_testnet ? command_line::arg_testnet_data_dir : command_line::arg_data_dir; + dirname = command_line::get_arg(vm, data_dir_arg); + db_engine = command_line::get_arg(vm, arg_database); + + log_space::get_set_log_detalisation_level(true, log_level); + log_space::log_singletone::add_logger(LOGGER_CONSOLE, NULL, NULL); + LOG_PRINT_L0("Starting..."); + LOG_PRINT_L0("Setting log level = " << log_level); + + boost::filesystem::path file_path {dirname}; + std::string import_file_path; + + import_file_path = (file_path / "export" / import_filename).string(); + + if (command_line::has_arg(vm, arg_count_blocks)) + { + count_blocks(import_file_path); + exit(0); + } + + if (std::find(db_engines.begin(), db_engines.end(), db_engine) == db_engines.end()) + { + std::cerr << "Invalid database engine: " << db_engine << std::endl; + exit(1); + } + + LOG_PRINT_L0("database: " << db_engine); + LOG_PRINT_L0("verify: " << std::boolalpha << opt_verify << std::noboolalpha); + if (opt_batch) + { + LOG_PRINT_L0("batch: " << std::boolalpha << opt_batch << std::noboolalpha + << " batch size: " << db_batch_size); + } + else + { + LOG_PRINT_L0("batch: " << std::boolalpha << opt_batch << std::noboolalpha); + } + LOG_PRINT_L0("resume: " << std::boolalpha << opt_resume << std::noboolalpha); + LOG_PRINT_L0("testnet: " << std::boolalpha << opt_testnet << std::noboolalpha); + + std::cout << "import file path: " << import_file_path << ENDL; + std::cout << "database path: " << file_path.string() << ENDL; + + try + { + + // fake_core needed for verification to work when enabled. + // + // NOTE: don't need fake_core method of doing things when we're going to call + // BlockchainDB add_block() directly and have available the 3 block + // properties to do so. Both ways work, but fake core isn't necessary in that + // circumstance. + + // for multi_db_runtime: +#if !defined(BLOCKCHAIN_DB) + if (db_engine == "lmdb") + { + fake_core_lmdb simple_core(dirname, opt_testnet, opt_batch); + import_from_file(simple_core, import_file_path); + } + else if (db_engine == "memory") + { + fake_core_memory simple_core(dirname, opt_testnet); + import_from_file(simple_core, import_file_path); + } + else + { + std::cerr << "database engine unrecognized" << ENDL; + exit(1); + } + + // for multi_db_compile: +#else + if (db_engine != default_db_engine) + { + std::cerr << "Invalid database engine for compiled version: " << db_engine << std::endl; + exit(1); + } +#if BLOCKCHAIN_DB == DB_LMDB + fake_core_lmdb simple_core(dirname, opt_testnet, opt_batch); +#else + fake_core_memory simple_core(dirname, opt_testnet); +#endif + + import_from_file(simple_core, import_file_path); +#endif + + } + catch (const DB_ERROR& e) + { + std::cout << std::string("Error loading blockchain db: ") + e.what() + " -- shutting down now" << ENDL; + exit(1); + } + + // destructors called at exit: + // + // ensure db closed + // - transactions properly checked and handled + // - disk sync if needed + // + // fake_core object's destructor is called when it goes out of scope. For an + // LMDB fake_core, it calls Blockchain::deinit() on its object, which in turn + // calls delete on its BlockchainDB derived class' object, which closes its + // files. +} diff --git a/src/blockchain_converter/fake_core.h b/src/blockchain_converter/fake_core.h new file mode 100644 index 000000000..68021a2bd --- /dev/null +++ b/src/blockchain_converter/fake_core.h @@ -0,0 +1,143 @@ +// Copyright (c) 2014-2015, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include +#include "cryptonote_core/blockchain.h" // BlockchainDB and LMDB +#include "cryptonote_core/blockchain_storage.h" // in-memory DB + +using namespace cryptonote; + + +#if !defined(BLOCKCHAIN_DB) || BLOCKCHAIN_DB == DB_LMDB + +struct fake_core_lmdb +{ + Blockchain m_storage; + tx_memory_pool m_pool; + bool support_batch; + bool support_add_block; + + // for multi_db_runtime: +#if !defined(BLOCKCHAIN_DB) + fake_core_lmdb(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true) : m_pool(&m_storage), m_storage(m_pool) + // for multi_db_compile: +#else + fake_core_lmdb(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true) : m_pool(m_storage), m_storage(m_pool) +#endif + { + m_pool.init(path.string()); + m_storage.init(path.string(), use_testnet); + if (do_batch) + m_storage.get_db()->set_batch_transactions(do_batch); + support_batch = true; + support_add_block = true; + } + ~fake_core_lmdb() + { + m_storage.deinit(); + } + + uint64_t add_block(const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const std::vector& txs + ) + { + return m_storage.get_db()->add_block(blk, block_size, cumulative_difficulty, coins_generated, txs); + } + + void batch_start() + { + m_storage.get_db()->batch_start(); + } + + void batch_stop() + { + m_storage.get_db()->batch_stop(); + } + +}; +#endif + +#if !defined(BLOCKCHAIN_DB) || BLOCKCHAIN_DB == DB_MEMORY + +struct fake_core_memory +{ + blockchain_storage m_storage; + tx_memory_pool m_pool; + bool support_batch; + bool support_add_block; + + // for multi_db_runtime: +#if !defined(BLOCKCHAIN_DB) + fake_core_memory(const boost::filesystem::path &path, const bool use_testnet=false) : m_pool(&m_storage), m_storage(m_pool) +#else + // for multi_db_compile: + fake_core_memory(const boost::filesystem::path &path, const bool use_testnet=false) : m_pool(m_storage), m_storage(&m_pool) +#endif + { + m_pool.init(path.string()); + m_storage.init(path.string(), use_testnet); + support_batch = false; + support_add_block = false; + } + ~fake_core_memory() + { + LOG_PRINT_L3("fake_core_memory() destructor called - want to see it ripple down"); + m_storage.deinit(); + } + + uint64_t add_block(const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const std::vector& txs + ) + { + // TODO: + // would need to refactor handle_block_to_main_chain() to have a direct add_block() method like Blockchain class + throw std::runtime_error("direct add_block() method not implemented for in-memory db"); + return 2; + } + + void batch_start() + { + LOG_PRINT_L0("WARNING: [batch_start] opt_batch set, but this database doesn't support/need transactions - ignoring"); + } + + void batch_stop() + { + LOG_PRINT_L0("WARNING: [batch_stop] opt_batch set, but this database doesn't support/need transactions - ignoring"); + } + +}; + +#endif diff --git a/src/blockchain_converter/import.h b/src/blockchain_converter/import.h new file mode 100644 index 000000000..632b4c0d9 --- /dev/null +++ b/src/blockchain_converter/import.h @@ -0,0 +1,37 @@ +// Copyright (c) 2014-2015, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +// TODO: bounds checking is done before writing to buffer, but buffer size +// should be a sensible maximum +#define BUFFER_SIZE 1000000 +#define NUM_BLOCKS_PER_CHUNK 1 +#define STR_LENGTH_OF_INT 9 +#define STR_FORMAT_OF_INT "%09d" +#define BLOCKCHAIN_RAW "blockchain.raw" -- cgit v1.2.3 From cb862cb81a664f8c98ed27ba281b8969cbe48e39 Mon Sep 17 00:00:00 2001 From: warptangent Date: Wed, 11 Feb 2015 15:55:53 -0800 Subject: Add mdb_flags variable to LMDB database open --- src/blockchain_db/lmdb/db_lmdb.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 495f93bdc..6d815c997 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -637,6 +637,7 @@ BlockchainLMDB::BlockchainLMDB(bool batch_transactions) void BlockchainLMDB::open(const std::string& filename) { + int mdb_flags = 0; LOG_PRINT_L3("BlockchainLMDB::" << __func__); if (m_open) @@ -675,7 +676,7 @@ void BlockchainLMDB::open(const std::string& filename) size_t mapsize = 1LL << 34; if (auto result = mdb_env_set_mapsize(m_env, mapsize)) throw0(DB_ERROR(std::string("Failed to set max memory map size: ").append(mdb_strerror(result)).c_str())); - if (auto result = mdb_env_open(m_env, filename.c_str(), 0, 0644)) + if (auto result = mdb_env_open(m_env, filename.c_str(), mdb_flags, 0644)) throw0(DB_ERROR(std::string("Failed to open lmdb environment: ").append(mdb_strerror(result)).c_str())); // get a read/write MDB_txn -- cgit v1.2.3 From 275cbd4348975371988c79f67d68ceede3f47a1d Mon Sep 17 00:00:00 2001 From: warptangent Date: Wed, 11 Feb 2015 16:02:20 -0800 Subject: Add support for database open with flags Add support to: - BlockchainDB, BlockchainLMDB - blockchain_import utility to open LMDB database with one or more LMDB flags. Sample use: $ blockchain_import --database lmdb#nosync $ blockchain_import --database lmdb#nosync,nometasync --- src/blockchain_converter/blockchain_import.cpp | 72 ++++++++++++++++++++++++-- src/blockchain_converter/fake_core.h | 6 +-- src/blockchain_db/blockchain_db.h | 2 +- src/blockchain_db/lmdb/db_lmdb.cpp | 3 +- src/blockchain_db/lmdb/db_lmdb.h | 2 +- src/cryptonote_core/blockchain.cpp | 4 +- src/cryptonote_core/blockchain.h | 2 +- 7 files changed, 77 insertions(+), 14 deletions(-) (limited to 'src') diff --git a/src/blockchain_converter/blockchain_import.cpp b/src/blockchain_converter/blockchain_import.cpp index ce2abf4ac..060fe8481 100644 --- a/src/blockchain_converter/blockchain_import.cpp +++ b/src/blockchain_converter/blockchain_import.cpp @@ -42,6 +42,7 @@ #include "common/command_line.h" #include "version.h" +#include // for db flag arguments #include "import.h" #include "fake_core.h" @@ -62,6 +63,58 @@ namespace po = boost::program_options; using namespace cryptonote; using namespace epee; + +int parse_db_arguments(const std::string& db_arg_str, std::string& db_engine, int& mdb_flags) +{ + std::vector db_args; + boost::split(db_args, db_arg_str, boost::is_any_of("#")); + db_engine = db_args.front(); + boost::algorithm::trim(db_engine); + + if (db_args.size() == 1) + { + return 0; + } + else if (db_args.size() > 2) + { + std::cerr << "unrecognized database argument format: " << db_arg_str << ENDL; + return 1; + } + + std::string db_arg_str2 = db_args[1]; + boost::split(db_args, db_arg_str2, boost::is_any_of(",")); + for (auto& it : db_args) + { + boost::algorithm::trim(it); + if (it.empty()) + continue; + LOG_PRINT_L1("LMDB flag: " << it); + if (it == "nosync") + { + mdb_flags |= MDB_NOSYNC; + } + else if (it == "nometasync") + { + mdb_flags |= MDB_NOMETASYNC; + } + else if (it == "writemap") + { + mdb_flags |= MDB_WRITEMAP; + } + else if (it == "mapasync") + { + mdb_flags |= MDB_MAPASYNC; + } + else + { + std::cerr << "unrecognized database flag: " << it << ENDL; + return 1; + } + } + return 0; +} + + int count_blocks(std::string& import_file_path) { boost::filesystem::path raw_file_path(import_file_path); @@ -492,7 +545,7 @@ int main(int argc, char* argv[]) uint32_t log_level = LOG_LEVEL_0; std::string dirname; - std::string db_engine; + std::string db_arg_str; boost::filesystem::path default_data_path {tools::get_default_data_dir()}; boost::filesystem::path default_testnet_data_path {default_data_path / "testnet"}; @@ -582,7 +635,7 @@ int main(int argc, char* argv[]) opt_testnet = command_line::get_arg(vm, arg_testnet_on); auto data_dir_arg = opt_testnet ? command_line::arg_testnet_data_dir : command_line::arg_data_dir; dirname = command_line::get_arg(vm, data_dir_arg); - db_engine = command_line::get_arg(vm, arg_database); + db_arg_str = command_line::get_arg(vm, arg_database); log_space::get_set_log_detalisation_level(true, log_level); log_space::log_singletone::add_logger(LOGGER_CONSOLE, NULL, NULL); @@ -600,6 +653,17 @@ int main(int argc, char* argv[]) exit(0); } + + std::string db_engine; + int mdb_flags = 0; + int res = 0; + res = parse_db_arguments(db_arg_str, db_engine, mdb_flags); + if (res) + { + std::cerr << "Error parsing database argument(s)" << ENDL; + exit(1); + } + if (std::find(db_engines.begin(), db_engines.end(), db_engine) == db_engines.end()) { std::cerr << "Invalid database engine: " << db_engine << std::endl; @@ -637,7 +701,7 @@ int main(int argc, char* argv[]) #if !defined(BLOCKCHAIN_DB) if (db_engine == "lmdb") { - fake_core_lmdb simple_core(dirname, opt_testnet, opt_batch); + fake_core_lmdb simple_core(dirname, opt_testnet, opt_batch, mdb_flags); import_from_file(simple_core, import_file_path); } else if (db_engine == "memory") @@ -659,7 +723,7 @@ int main(int argc, char* argv[]) exit(1); } #if BLOCKCHAIN_DB == DB_LMDB - fake_core_lmdb simple_core(dirname, opt_testnet, opt_batch); + fake_core_lmdb simple_core(dirname, opt_testnet, opt_batch, mdb_flags); #else fake_core_memory simple_core(dirname, opt_testnet); #endif diff --git a/src/blockchain_converter/fake_core.h b/src/blockchain_converter/fake_core.h index 68021a2bd..aff249cd9 100644 --- a/src/blockchain_converter/fake_core.h +++ b/src/blockchain_converter/fake_core.h @@ -46,14 +46,14 @@ struct fake_core_lmdb // for multi_db_runtime: #if !defined(BLOCKCHAIN_DB) - fake_core_lmdb(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true) : m_pool(&m_storage), m_storage(m_pool) + fake_core_lmdb(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const int mdb_flags=0) : m_pool(&m_storage), m_storage(m_pool) // for multi_db_compile: #else - fake_core_lmdb(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true) : m_pool(m_storage), m_storage(m_pool) + fake_core_lmdb(const boost::filesystem::path &path, const bool use_testnet=false, const bool do_batch=true, const int mdb_flags=0) : m_pool(m_storage), m_storage(m_pool) #endif { m_pool.init(path.string()); - m_storage.init(path.string(), use_testnet); + m_storage.init(path.string(), use_testnet, mdb_flags); if (do_batch) m_storage.get_db()->set_batch_transactions(do_batch); support_batch = true; diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index d2b4a07a7..7b6b55a40 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -326,7 +326,7 @@ public: void show_stats(); // open the db at location , or create it if there isn't one. - virtual void open(const std::string& filename) = 0; + virtual void open(const std::string& filename, const int db_flags = 0) = 0; // make sure implementation has a create function as well virtual void create(const std::string& filename) = 0; diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 6d815c997..b1da6308f 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -635,9 +635,8 @@ BlockchainLMDB::BlockchainLMDB(bool batch_transactions) m_height = 0; } -void BlockchainLMDB::open(const std::string& filename) +void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) { - int mdb_flags = 0; LOG_PRINT_L3("BlockchainLMDB::" << __func__); if (m_open) diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 1a684548e..fa7cb988a 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -114,7 +114,7 @@ public: BlockchainLMDB(bool batch_transactions=false); ~BlockchainLMDB(); - virtual void open(const std::string& filename); + virtual void open(const std::string& filename, const int mdb_flags=0); virtual void create(const std::string& filename); diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index b7920c99c..57934b3fc 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -226,7 +226,7 @@ uint64_t Blockchain::get_current_blockchain_height() const //------------------------------------------------------------------ //FIXME: possibly move this into the constructor, to avoid accidentally // dereferencing a null BlockchainDB pointer -bool Blockchain::init(const std::string& config_folder, bool testnet) +bool Blockchain::init(const std::string& config_folder, const bool testnet, const int db_flags) { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -246,7 +246,7 @@ bool Blockchain::init(const std::string& config_folder, bool testnet) const std::string filename = folder.string(); try { - m_db->open(filename); + m_db->open(filename, db_flags); } catch (const DB_OPEN_FAILURE& e) { diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index e86b3887e..da4da075b 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -81,7 +81,7 @@ namespace cryptonote Blockchain(tx_memory_pool& tx_pool); - bool init(const std::string& config_folder, bool testnet = false); + bool init(const std::string& config_folder, const bool testnet = false, const int db_flags = 0); bool deinit(); void set_checkpoints(checkpoints&& chk_pts) { m_checkpoints = chk_pts; } -- cgit v1.2.3 From f6cbfb623bd2e7f75b8380b43e29ae6d73442330 Mon Sep 17 00:00:00 2001 From: warptangent Date: Mon, 16 Feb 2015 04:20:55 -0800 Subject: Add blockchain_export utility Based on work by tomerkon. See https://github.com/tomerkon/bitmonero src/cryptonote_core/bootfilesaver.{h,cpp} src/bootfilegen/bootfilegen.cpp --- src/blockchain_converter/CMakeLists.txt | 29 ++ src/blockchain_converter/blockchain_export.cpp | 367 +++++++++++++++++++++++++ src/blockchain_converter/blockchain_export.h | 74 +++++ 3 files changed, 470 insertions(+) create mode 100644 src/blockchain_converter/blockchain_export.cpp create mode 100644 src/blockchain_converter/blockchain_export.h (limited to 'src') diff --git a/src/blockchain_converter/CMakeLists.txt b/src/blockchain_converter/CMakeLists.txt index cc400d927..e0f3be901 100644 --- a/src/blockchain_converter/CMakeLists.txt +++ b/src/blockchain_converter/CMakeLists.txt @@ -47,6 +47,20 @@ set(blockchain_import_private_headers bitmonero_private_headers(blockchain_import ${blockchain_import_private_headers}) +set(blockchain_export_sources + blockchain_export.cpp + ) + +set(blockchain_export_private_headers + import.h + blockchain_export.h + ) + +bitmonero_private_headers(blockchain_export + ${blockchain_export_private_headers}) + + + if (BLOCKCHAIN_DB STREQUAL DB_LMDB) bitmonero_add_executable(blockchain_converter ${blockchain_converter_sources} @@ -79,3 +93,18 @@ add_dependencies(blockchain_import set_property(TARGET blockchain_import PROPERTY OUTPUT_NAME "blockchain_import") + +bitmonero_add_executable(blockchain_export + ${blockchain_export_sources} + ${blockchain_export_private_headers}) + +target_link_libraries(blockchain_export + LINK_PRIVATE + cryptonote_core + ${CMAKE_THREAD_LIBS_INIT}) + +add_dependencies(blockchain_export + version) +set_property(TARGET blockchain_export + PROPERTY + OUTPUT_NAME "blockchain_export") diff --git a/src/blockchain_converter/blockchain_export.cpp b/src/blockchain_converter/blockchain_export.cpp new file mode 100644 index 000000000..33944dba6 --- /dev/null +++ b/src/blockchain_converter/blockchain_export.cpp @@ -0,0 +1,367 @@ +// Copyright (c) 2014-2015, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include "common/command_line.h" +#include "version.h" +#include "blockchain_export.h" +#include "cryptonote_core/cryptonote_boost_serialization.h" + +#include "import.h" +// #include "fake_core.h" + +#define SOURCE_DB DB_MEMORY +// currently not supported: +// to use what was set at compile time (DB_MEMORY or DB_LMDB) +// #define SOURCE_DB BLOCKCHAIN_DB + +static int max_chunk = 0; +static size_t height; + +namespace po = boost::program_options; + +using namespace cryptonote; +using namespace epee; + +bool BlockchainExport::open(const boost::filesystem::path& dir_path) +{ + if (boost::filesystem::exists(dir_path)) + { + if (!boost::filesystem::is_directory(dir_path)) + { + LOG_PRINT_RED_L0("export directory path is a file: " << dir_path); + return false; + } + } + else + { + if (!boost::filesystem::create_directory(dir_path)) + { + LOG_PRINT_RED_L0("Failed to create directory " << dir_path); + return false; + } + } + + std::string file_path = (dir_path / BLOCKCHAIN_RAW).string(); + m_raw_data_file = new std::ofstream(); + m_raw_data_file->open(file_path , std::ios_base::binary | std::ios_base::out| std::ios::trunc); + if (m_raw_data_file->fail()) + return false; + + m_output_stream = new boost::iostreams::stream>(m_buffer); + m_raw_archive = new boost::archive::binary_oarchive(*m_output_stream); + if (m_raw_archive == NULL) + return false; + + return true; +} + +void BlockchainExport::flush_chunk() +{ + m_output_stream->flush(); + char buffer[STR_LENGTH_OF_INT + 1]; + int chunk_size = (int) m_buffer.size(); + if (chunk_size > BUFFER_SIZE) + { + LOG_PRINT_L0("WARNING: chunk_size " << chunk_size << " > BUFFER_SIZE " << BUFFER_SIZE); + } + sprintf(buffer, STR_FORMAT_OF_INT, chunk_size); + m_raw_data_file->write(buffer, STR_LENGTH_OF_INT); + if (max_chunk < chunk_size) + { + max_chunk = chunk_size; + } + long pos_before = m_raw_data_file->tellp(); + std::copy(m_buffer.begin(), m_buffer.end(), std::ostreambuf_iterator(*m_raw_data_file)); + m_raw_data_file->flush(); + long pos_after = m_raw_data_file->tellp(); + long num_chars_written = pos_after - pos_before; + if ((int) num_chars_written != chunk_size) + { + LOG_PRINT_RED_L0("INTERNAL ERROR: num chars wrote NEQ buffer size. height = " << height); + } + + m_buffer.clear(); + delete m_raw_archive; + delete m_output_stream; + m_output_stream = new boost::iostreams::stream>(m_buffer); + m_raw_archive = new boost::archive::binary_oarchive(*m_output_stream); +} + +void BlockchainExport::serialize_block_to_text_buffer(const block& block) +{ + *m_raw_archive << block; +} + +void BlockchainExport::buffer_serialize_tx(const transaction& tx) +{ + *m_raw_archive << tx; +} + +void BlockchainExport::buffer_write_num_txs(const std::list txs) +{ + int n = txs.size(); + *m_raw_archive << n; +} + +void BlockchainExport::write_block(block& block) +{ + serialize_block_to_text_buffer(block); + std::list txs; + + uint64_t block_height = boost::get(block.miner_tx.vin.front()).height; + + // put coinbase transaction first + transaction coinbase_tx = block.miner_tx; + crypto::hash coinbase_tx_hash = get_transaction_hash(coinbase_tx); + // blockchain_storage: + const transaction* cb_tx_full = m_blockchain_storage->get_tx(coinbase_tx_hash); + // + // BlockchainDB: + // transaction cb_tx_full = m_blockchain_storage->get_tx(coinbase_tx_hash); + + + // blockchain_storage: + if (cb_tx_full != NULL) + { + txs.push_back(*cb_tx_full); + } + // + // BlockchainDB: + // txs.push_back(cb_tx_full); + + // now add all regular transactions + BOOST_FOREACH(const auto& tx_id, block.tx_hashes) + { +#if SOURCE_DB == DB_MEMORY + const transaction* tx = m_blockchain_storage->get_tx(tx_id); +#else + transaction tx = m_blockchain_storage->get_tx(tx_id); +#endif + +#if SOURCE_DB == DB_MEMORY + if(tx == NULL) + { + if (! m_tx_pool) + throw std::runtime_error("Aborting: tx == NULL, so memory pool required to get tx, but memory pool isn't enabled"); + else + { + transaction tx; + if(m_tx_pool->get_transaction(tx_id, tx)) + txs.push_back(tx); + else + throw std::runtime_error("Aborting: tx not found in pool"); + } + } + else + txs.push_back(*tx); +#else + txs.push_back(tx); +#endif + } + + // serialize all txs to the persistant storage + buffer_write_num_txs(txs); + BOOST_FOREACH(const auto& tx, txs) + { + buffer_serialize_tx(tx); + } + + // These three attributes are currently necessary for a fast import that adds blocks without verification. + bool include_extra_block_data = true; + if (include_extra_block_data) + { + size_t block_size = m_blockchain_storage->get_block_size(block_height); + difficulty_type cumulative_difficulty = m_blockchain_storage->get_block_cumulative_difficulty(block_height); + uint64_t coins_generated = m_blockchain_storage->get_block_coins_generated(block_height); + + *m_raw_archive << block_size; + *m_raw_archive << cumulative_difficulty; + *m_raw_archive << coins_generated; + } +} + +bool BlockchainExport::BlockchainExport::close() +{ + if (m_raw_data_file->fail()) + return false; + + m_raw_data_file->flush(); + delete m_raw_archive; + delete m_output_stream; + delete m_raw_data_file; + return true; +} + + +#if SOURCE_DB == DB_MEMORY +bool BlockchainExport::store_blockchain_raw(blockchain_storage* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_dir, uint64_t use_block_height) +#else +bool BlockchainExport::store_blockchain_raw(BlockchainDB* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_dir, uint64_t use_block_height) +#endif +{ + uint64_t use_block_height2 = 0; + m_blockchain_storage = _blockchain_storage; + m_tx_pool = _tx_pool; + LOG_PRINT_L0("Storing blocks raw data..."); + if (!BlockchainExport::open(output_dir)) + { + LOG_PRINT_RED_L0("failed to open raw file for write"); + return false; + } + block b; + LOG_PRINT_L0("source blockchain height: " << m_blockchain_storage->get_current_blockchain_height()); + LOG_PRINT_L0("requested block height: " << use_block_height); + if ((use_block_height > 0) && (use_block_height < m_blockchain_storage->get_current_blockchain_height())) + use_block_height2 = use_block_height; + else + { + use_block_height2 = m_blockchain_storage->get_current_blockchain_height(); + LOG_PRINT_L0("using block height: " << use_block_height2); + } + for (height=0; height < use_block_height2; ++height) + { + crypto::hash hash = m_blockchain_storage->get_block_id_by_height(height); + m_blockchain_storage->get_block_by_hash(hash, b); + write_block(b); + if (height % NUM_BLOCKS_PER_CHUNK == 0) { + flush_chunk(); + } + } + if (height % NUM_BLOCKS_PER_CHUNK != 0) + { + flush_chunk(); + } + + LOG_PRINT_L0("longest chunk was " << max_chunk << " bytes"); + return BlockchainExport::close(); +} + + +int main(int argc, char* argv[]) +{ + uint32_t log_level = 0; + uint64_t block_height = 0; + std::string import_filename = BLOCKCHAIN_RAW; + + boost::filesystem::path default_data_path {tools::get_default_data_dir()}; + boost::filesystem::path default_testnet_data_path {default_data_path / "testnet"}; + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor arg_log_level = {"log-level", "", log_level}; + const command_line::arg_descriptor arg_block_height = {"block-number", "", block_height}; + const command_line::arg_descriptor arg_testnet_on = { + "testnet" + , "Run on testnet." + , false + }; + + + command_line::add_arg(desc_cmd_sett, command_line::arg_data_dir, default_data_path.string()); + command_line::add_arg(desc_cmd_sett, command_line::arg_testnet_data_dir, default_testnet_data_path.string()); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_block_height); + command_line::add_arg(desc_cmd_sett, arg_testnet_on); + + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + po::store(po::parse_command_line(argc, argv, desc_options), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << CRYPTONOTE_NAME << " v" << MONERO_VERSION_FULL << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + log_level = command_line::get_arg(vm, arg_log_level); + block_height = command_line::get_arg(vm, arg_block_height); + + log_space::get_set_log_detalisation_level(true, log_level); + log_space::log_singletone::add_logger(LOGGER_CONSOLE, NULL, NULL); + LOG_PRINT_L0("Starting..."); + LOG_PRINT_L0("Setting log level = " << log_level); + + bool opt_testnet = command_line::get_arg(vm, arg_testnet_on); + + std::string m_config_folder; + + auto data_dir_arg = opt_testnet ? command_line::arg_testnet_data_dir : command_line::arg_data_dir; + m_config_folder = command_line::get_arg(vm, data_dir_arg); + boost::filesystem::path output_dir {m_config_folder}; + output_dir /= "export"; + std::cout << "config directory: " << m_config_folder << ENDL; + + // If we wanted to use the memory pool, we would set up a fake_core. + +#if SOURCE_DB == DB_MEMORY + // blockchain_storage* core_storage = NULL; + // tx_memory_pool m_mempool(*core_storage); // is this fake anyway? just passing in NULL! so m_mempool can't be used anyway, right? + // core_storage = new blockchain_storage(&m_mempool); + + blockchain_storage* core_storage = new blockchain_storage(NULL); +#else + BlockchainDB* core_storage = new BlockchainLMDB(); +#endif + // tx_memory_pool m_mempool(core_storage); + + LOG_PRINT_L0("Initializing source blockchain storage..."); + r = core_storage->init(m_config_folder, opt_testnet); + CHECK_AND_ASSERT_MES(r, false, "Failed to initialize source blockchain storage"); + LOG_PRINT_L0("Source blockchain storage initialized OK"); + LOG_PRINT_L0("Exporting blockchain raw data..."); + + BlockchainExport be; + r = be.store_blockchain_raw(core_storage, NULL, output_dir, block_height); + CHECK_AND_ASSERT_MES(r, false, "Failed to export blockchain raw data"); + LOG_PRINT_L0("Blockchain raw data exported OK"); +} diff --git a/src/blockchain_converter/blockchain_export.h b/src/blockchain_converter/blockchain_export.h new file mode 100644 index 000000000..cd79d1c47 --- /dev/null +++ b/src/blockchain_converter/blockchain_export.h @@ -0,0 +1,74 @@ +// Copyright (c) 2014-2015, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include +#include +#include +#include +#include "cryptonote_core/cryptonote_basic.h" +#include "cryptonote_core/blockchain_storage.h" +#include "cryptonote_core/blockchain.h" +// TODO: +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/lmdb/db_lmdb.h" + +using namespace cryptonote; + +class BlockchainExport +{ +public: + bool store_blockchain_raw(cryptonote::blockchain_storage* cs, cryptonote::tx_memory_pool* txp, + boost::filesystem::path& output_dir, uint64_t use_block_height=0); + // + // BlockchainDB: + // bool store_blockchain_raw(cryptonote::BlockchainDB* cs, boost::filesystem::path& output_dir, uint64_t use_block_height=0); + +protected: + // blockchain_storage: + blockchain_storage* m_blockchain_storage; + // + // BlockchainDB: + // BlockchainDB* m_blockchain_storage; + tx_memory_pool* m_tx_pool; + typedef std::vector buffer_type; + std::ofstream * m_raw_data_file; + boost::archive::binary_oarchive * m_raw_archive; + buffer_type m_buffer; + boost::iostreams::stream>* m_output_stream; + + // open export file for write + bool open(const boost::filesystem::path& dir_path); + bool close(); + void write_block(block& block); + void serialize_block_to_text_buffer(const block& block); + void buffer_serialize_tx(const transaction& tx); + void buffer_write_num_txs(const std::list txs); + void flush_chunk(); +}; -- cgit v1.2.3 From edef0bb77175d225f93ac68fdc9f2ba37971ed30 Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Mon, 9 Mar 2015 16:21:44 -0400 Subject: Initial commit of BDB BlockchainDB implementation Basically verbatim copy of LMDB implementation, but with the guts ripped out and includes changed, etc. --- src/blockchain_db/berkeleydb/db_bdb.cpp | 465 ++++++++++++++++++++++++++++++++ src/blockchain_db/berkeleydb/db_bdb.h | 213 +++++++++++++++ 2 files changed, 678 insertions(+) create mode 100644 src/blockchain_db/berkeleydb/db_bdb.cpp create mode 100644 src/blockchain_db/berkeleydb/db_bdb.h (limited to 'src') diff --git a/src/blockchain_db/berkeleydb/db_bdb.cpp b/src/blockchain_db/berkeleydb/db_bdb.cpp new file mode 100644 index 000000000..9f79c4846 --- /dev/null +++ b/src/blockchain_db/berkeleydb/db_bdb.cpp @@ -0,0 +1,465 @@ +// Copyright (c) 2014, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "db_bdb.h" + +#include +#include // std::unique_ptr +#include // memcpy + +#include "cryptonote_core/cryptonote_format_utils.h" +#include "crypto/crypto.h" +#include "profile_tools.h" + +using epee::string_tools::pod_to_hex; + +namespace +{ + +template +inline void throw0(const T &e) +{ + LOG_PRINT_L0(e.what()); + throw e; +} + +template +inline void throw1(const T &e) +{ + LOG_PRINT_L1(e.what()); + throw e; +} + +} // anonymous namespace + +namespace cryptonote +{ + +void BlockchainBDB::add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const crypto::hash& blk_hash + ) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +void BlockchainBDB::remove_block() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +void BlockchainBDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +void BlockchainBDB::remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +void BlockchainBDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +void BlockchainBDB::remove_tx_outputs(const crypto::hash& tx_hash, const transaction& tx) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + +} + +// TODO: probably remove this function +void BlockchainBDB::remove_output(const tx_out& tx_output) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__ << " (unused version - does nothing)"); + return; +} + +void BlockchainBDB::remove_output(const uint64_t& out_index, const uint64_t amount) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +void BlockchainBDB::remove_amount_output_index(const uint64_t amount, const uint64_t global_output_index) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +void BlockchainBDB::add_spent_key(const crypto::key_image& k_image) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +void BlockchainBDB::remove_spent_key(const crypto::key_image& k_image) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +blobdata BlockchainBDB::output_to_blob(const tx_out& output) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); +} + +tx_out BlockchainBDB::output_from_blob(const blobdata& blob) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); +} + +uint64_t BlockchainBDB::get_output_global_index(const uint64_t& amount, const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +void BlockchainBDB::check_open() const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + if (!m_open) + throw0(DB_ERROR("DB operation attempted on a not-open DB instance")); +} + +BlockchainBDB::~BlockchainBDB() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + +} + +BlockchainBDB::BlockchainBDB(bool batch_transactions) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); +} + +void BlockchainBDB::open(const std::string& filename) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + +} + +// unused for now, create will happen on open if doesn't exist +void BlockchainBDB::create(const std::string& filename) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + throw DB_CREATE_FAILURE("create() is not implemented for this BlockchainDB, open() will create files if needed."); +} + +void BlockchainBDB::close() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); +} + +void BlockchainBDB::sync() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +void BlockchainBDB::reset() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + // TODO: this +} + +std::vector BlockchainBDB::get_filenames() const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); +} + +// TODO: this? +bool BlockchainBDB::lock() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + return false; +} + +// TODO: this? +void BlockchainBDB::unlock() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +bool BlockchainBDB::block_exists(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +block BlockchainBDB::get_block(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +uint64_t BlockchainBDB::get_block_height(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +block_header BlockchainBDB::get_block_header(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +block BlockchainBDB::get_block_from_height(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +uint64_t BlockchainBDB::get_block_timestamp(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +uint64_t BlockchainBDB::get_top_block_timestamp() const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +size_t BlockchainBDB::get_block_size(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +difficulty_type BlockchainBDB::get_block_cumulative_difficulty(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__ << " height: " << height); + check_open(); +} + +difficulty_type BlockchainBDB::get_block_difficulty(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +uint64_t BlockchainBDB::get_block_already_generated_coins(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +crypto::hash BlockchainBDB::get_block_hash_from_height(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +std::vector BlockchainBDB::get_blocks_range(const uint64_t& h1, const uint64_t& h2) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +std::vector BlockchainBDB::get_hashes_range(const uint64_t& h1, const uint64_t& h2) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +crypto::hash BlockchainBDB::top_block_hash() const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +block BlockchainBDB::get_top_block() const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +uint64_t BlockchainBDB::height() const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + return m_height; +} + +bool BlockchainBDB::tx_exists(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +uint64_t BlockchainBDB::get_tx_unlock_time(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +transaction BlockchainBDB::get_tx(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +uint64_t BlockchainBDB::get_tx_count() const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +std::vector BlockchainBDB::get_tx_list(const std::vector& hlist) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +uint64_t BlockchainBDB::get_tx_block_height(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +//FIXME: make sure the random method used here is appropriate +uint64_t BlockchainBDB::get_random_output(const uint64_t& amount) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +uint64_t BlockchainBDB::get_num_outputs(const uint64_t& amount) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +crypto::public_key BlockchainBDB::get_output_key(const uint64_t& amount, const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +tx_out BlockchainBDB::get_output(const crypto::hash& h, const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +// As this is not used, its return is now a blank output. +// This will save on space in the db. +tx_out BlockchainBDB::get_output(const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + return tx_out(); +} + +tx_out_index BlockchainBDB::get_output_tx_and_index_from_global(const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +tx_out_index BlockchainBDB::get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +std::vector BlockchainBDB::get_tx_output_indices(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +std::vector BlockchainBDB::get_tx_amount_output_indices(const crypto::hash& h) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + + + +bool BlockchainBDB::has_key_image(const crypto::key_image& img) const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +void BlockchainBDB::batch_start() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); +} + +void BlockchainBDB::batch_commit() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); +} + +void BlockchainBDB::batch_stop() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); +} + +void BlockchainBDB::batch_abort() +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); +} + +void BlockchainBDB::set_batch_transactions(bool batch_transactions) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); +} + +uint64_t BlockchainBDB::add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const std::vector& txs + ) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +void BlockchainBDB::pop_block(block& blk, std::vector& txs) +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); +} + +} // namespace cryptonote diff --git a/src/blockchain_db/berkeleydb/db_bdb.h b/src/blockchain_db/berkeleydb/db_bdb.h new file mode 100644 index 000000000..32686e6ba --- /dev/null +++ b/src/blockchain_db/berkeleydb/db_bdb.h @@ -0,0 +1,213 @@ +// Copyright (c) 2014, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "blockchain_db/blockchain_db.h" +#include "cryptonote_protocol/blobdatatype.h" // for type blobdata + +namespace cryptonote +{ + +class BlockchainBDB : public BlockchainDB +{ +public: + BlockchainBDB(bool batch_transactions=false); + ~BlockchainBDB(); + + virtual void open(const std::string& filename); + + virtual void create(const std::string& filename); + + virtual void close(); + + virtual void sync(); + + virtual void reset(); + + virtual std::vector get_filenames() const; + + virtual bool lock(); + + virtual void unlock(); + + virtual bool block_exists(const crypto::hash& h) const; + + virtual block get_block(const crypto::hash& h) const; + + virtual uint64_t get_block_height(const crypto::hash& h) const; + + virtual block_header get_block_header(const crypto::hash& h) const; + + virtual block get_block_from_height(const uint64_t& height) const; + + virtual uint64_t get_block_timestamp(const uint64_t& height) const; + + virtual uint64_t get_top_block_timestamp() const; + + virtual size_t get_block_size(const uint64_t& height) const; + + virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const; + + virtual difficulty_type get_block_difficulty(const uint64_t& height) const; + + virtual uint64_t get_block_already_generated_coins(const uint64_t& height) const; + + virtual crypto::hash get_block_hash_from_height(const uint64_t& height) const; + + virtual std::vector get_blocks_range(const uint64_t& h1, const uint64_t& h2) const; + + virtual std::vector get_hashes_range(const uint64_t& h1, const uint64_t& h2) const; + + virtual crypto::hash top_block_hash() const; + + virtual block get_top_block() const; + + virtual uint64_t height() const; + + virtual bool tx_exists(const crypto::hash& h) const; + + virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const; + + virtual transaction get_tx(const crypto::hash& h) const; + + virtual uint64_t get_tx_count() const; + + virtual std::vector get_tx_list(const std::vector& hlist) const; + + virtual uint64_t get_tx_block_height(const crypto::hash& h) const; + + virtual uint64_t get_random_output(const uint64_t& amount) const; + + virtual uint64_t get_num_outputs(const uint64_t& amount) const; + + virtual crypto::public_key get_output_key(const uint64_t& amount, const uint64_t& index) const; + + virtual tx_out get_output(const crypto::hash& h, const uint64_t& index) const; + + /** + * @brief get an output from its global index + * + * @param index global index of the output desired + * + * @return the output associated with the index. + * Will throw OUTPUT_DNE if not output has that global index. + * Will throw DB_ERROR if there is a non-specific LMDB error in fetching + */ + tx_out get_output(const uint64_t& index) const; + + virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const; + + virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const; + + virtual std::vector get_tx_output_indices(const crypto::hash& h) const; + virtual std::vector get_tx_amount_output_indices(const crypto::hash& h) const; + + virtual bool has_key_image(const crypto::key_image& img) const; + + virtual uint64_t add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const std::vector& txs + ); + + virtual void set_batch_transactions(bool batch_transactions); + virtual void batch_start(); + virtual void batch_commit(); + virtual void batch_stop(); + virtual void batch_abort(); + + virtual void pop_block(block& blk, std::vector& txs); + +private: + virtual void add_block( const block& blk + , const size_t& block_size + , const difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , const crypto::hash& block_hash + ); + + virtual void remove_block(); + + virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash); + + virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx); + + virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index); + + virtual void remove_output(const tx_out& tx_output); + + void remove_tx_outputs(const crypto::hash& tx_hash, const transaction& tx); + + void remove_output(const uint64_t& out_index, const uint64_t amount); + void remove_amount_output_index(const uint64_t amount, const uint64_t global_output_index); + + virtual void add_spent_key(const crypto::key_image& k_image); + + virtual void remove_spent_key(const crypto::key_image& k_image); + + /** + * @brief convert a tx output to a blob for storage + * + * @param output the output to convert + * + * @return the resultant blob + */ + blobdata output_to_blob(const tx_out& output); + + /** + * @brief convert a tx output blob to a tx output + * + * @param blob the blob to convert + * + * @return the resultant tx output + */ + tx_out output_from_blob(const blobdata& blob) const; + + /** + * @brief get the global index of the index-th output of the given amount + * + * @param amount the output amount + * @param index the index into the set of outputs of that amount + * + * @return the global index of the desired output + */ + uint64_t get_output_global_index(const uint64_t& amount, const uint64_t& index) const; + + void check_open() const; + + bool m_open; + uint64_t m_height; + uint64_t m_num_outputs; + std::string m_folder; + txn_safe* m_write_txn; // may point to either a short-lived txn or a batch txn + txn_safe m_write_batch_txn; // persist batch txn outside of BlockchainBDB + + bool m_batch_transactions; // support for batch transactions + bool m_batch_active; // whether batch transaction is in progress +}; + +} // namespace cryptonote -- cgit v1.2.3 From 5112dc37d75cd7c08661591aade3572128db3f9f Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Sat, 14 Mar 2015 17:24:51 -0400 Subject: Try to not pollute cryptonote namespace --- src/blockchain_db/lmdb/db_lmdb.cpp | 72 +++++++++++++++++++------------------- src/blockchain_db/lmdb/db_lmdb.h | 22 ++++++------ 2 files changed, 47 insertions(+), 47 deletions(-) (limited to 'src') diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index ee49e1827..1fd039905 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -553,7 +553,7 @@ uint64_t BlockchainLMDB::get_output_global_index(const uint64_t& amount, const u LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; + mdb_txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) throw0(DB_ERROR("Failed to create a transaction for the db")); @@ -665,7 +665,7 @@ void BlockchainLMDB::open(const std::string& filename) throw0(DB_ERROR(std::string("Failed to open lmdb environment: ").append(mdb_strerror(result)).c_str())); // get a read/write MDB_txn - txn_safe txn; + mdb_txn_safe txn; if (mdb_txn_begin(m_env, NULL, 0, txn)) throw0(DB_ERROR("Failed to create a transaction for the db")); @@ -802,7 +802,7 @@ bool BlockchainLMDB::block_exists(const crypto::hash& h) const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; + mdb_txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) throw0(DB_ERROR("Failed to create a transaction for the db")); @@ -835,7 +835,7 @@ uint64_t BlockchainLMDB::get_block_height(const crypto::hash& h) const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; + mdb_txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) throw0(DB_ERROR("Failed to create a transaction for the db")); @@ -865,7 +865,7 @@ block BlockchainLMDB::get_block_from_height(const uint64_t& height) const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; + mdb_txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) throw0(DB_ERROR("Failed to create a transaction for the db")); @@ -896,8 +896,8 @@ uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; - txn_safe* txn_ptr = &txn; + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; if (m_batch_active) txn_ptr = m_write_txn; else @@ -939,8 +939,8 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; - txn_safe* txn_ptr = &txn; + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; if (m_batch_active) txn_ptr = m_write_txn; else @@ -969,8 +969,8 @@ difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " height: " << height); check_open(); - txn_safe txn; - txn_safe* txn_ptr = &txn; + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; if (m_batch_active) txn_ptr = m_write_txn; else @@ -1015,8 +1015,8 @@ uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& heigh LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; - txn_safe* txn_ptr = &txn; + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; if (m_batch_active) txn_ptr = m_write_txn; else @@ -1045,8 +1045,8 @@ crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height) LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; - txn_safe* txn_ptr = &txn; + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; if (m_batch_active) txn_ptr = m_write_txn; else @@ -1138,8 +1138,8 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; - txn_safe* txn_ptr = &txn; + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; if (m_batch_active) txn_ptr = m_write_txn; else @@ -1173,7 +1173,7 @@ uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; + mdb_txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) throw0(DB_ERROR("Failed to create a transaction for the db")); @@ -1193,8 +1193,8 @@ transaction BlockchainLMDB::get_tx(const crypto::hash& h) const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; - txn_safe* txn_ptr = &txn; + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; if (m_batch_active) txn_ptr = m_write_txn; else @@ -1228,7 +1228,7 @@ uint64_t BlockchainLMDB::get_tx_count() const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; + mdb_txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) throw0(DB_ERROR("Failed to create a transaction for the db")); @@ -1268,8 +1268,8 @@ uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const // read transaction here, as validation is done prior to the write for block // and tx data. - txn_safe txn; - txn_safe* txn_ptr = &txn; + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; if (m_batch_active) txn_ptr = m_write_txn; else @@ -1312,7 +1312,7 @@ uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; + mdb_txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) throw0(DB_ERROR("Failed to create a transaction for the db")); @@ -1343,7 +1343,7 @@ crypto::public_key BlockchainLMDB::get_output_key(const uint64_t& amount, const uint64_t glob_index = get_output_global_index(amount, index); - txn_safe txn; + mdb_txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) throw0(DB_ERROR("Failed to create a transaction for the db")); @@ -1363,7 +1363,7 @@ tx_out BlockchainLMDB::get_output(const crypto::hash& h, const uint64_t& index) LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; + mdb_txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) throw0(DB_ERROR("Failed to create a transaction for the db")); @@ -1408,7 +1408,7 @@ tx_out BlockchainLMDB::get_output(const uint64_t& index) const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; + mdb_txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) throw0(DB_ERROR("Failed to create a transaction for the db")); @@ -1432,8 +1432,8 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index_from_global(const uint64_t& LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; - txn_safe* txn_ptr = &txn; + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; if (m_batch_active) txn_ptr = m_write_txn; else @@ -1468,8 +1468,8 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; - txn_safe* txn_ptr = &txn; + mdb_txn_safe txn; + mdb_txn_safe* txn_ptr = &txn; if (m_batch_active) txn_ptr = m_write_txn; else @@ -1518,7 +1518,7 @@ std::vector BlockchainLMDB::get_tx_output_indices(const crypto::hash& check_open(); std::vector index_vec; - txn_safe txn; + mdb_txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) throw0(DB_ERROR("Failed to create a transaction for the db")); @@ -1563,7 +1563,7 @@ std::vector BlockchainLMDB::get_tx_amount_output_indices(const crypto: transaction tx = get_tx(h); - txn_safe txn; + mdb_txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) throw0(DB_ERROR("Failed to create a transaction for the db")); @@ -1634,7 +1634,7 @@ bool BlockchainLMDB::has_key_image(const crypto::key_image& img) const LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; + mdb_txn_safe txn; if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn)) throw0(DB_ERROR("Failed to create a transaction for the db")); @@ -1745,7 +1745,7 @@ uint64_t BlockchainLMDB::add_block( const block& blk LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; + mdb_txn_safe txn; if (! m_batch_active) { if (mdb_txn_begin(m_env, NULL, 0, txn)) @@ -1783,7 +1783,7 @@ void BlockchainLMDB::pop_block(block& blk, std::vector& txs) LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - txn_safe txn; + mdb_txn_safe txn; if (! m_batch_active) { if (mdb_txn_begin(m_env, NULL, 0, txn)) diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 1a684548e..a04d0f6b4 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -33,17 +33,17 @@ namespace cryptonote { -struct txn_safe +struct mdb_txn_safe { - txn_safe() : m_txn(NULL) { } - ~txn_safe() + mdb_txn_safe() : m_txn(NULL) { } + ~mdb_txn_safe() { - LOG_PRINT_L3("txn_safe: destructor"); + LOG_PRINT_L3("mdb_txn_safe: destructor"); if (m_txn != NULL) { if (m_batch_txn) // this is a batch txn and should have been handled before this point for safety { - LOG_PRINT_L0("WARNING: txn_safe: m_txn is a batch txn and it's not NULL in destructor - calling mdb_txn_abort()"); + LOG_PRINT_L0("WARNING: mdb_txn_safe: m_txn is a batch txn and it's not NULL in destructor - calling mdb_txn_abort()"); } else { @@ -53,7 +53,7 @@ struct txn_safe // // NOTE: not sure if this is ever reached for a non-batch write // transaction, but it's probably not ideal if it did. - LOG_PRINT_L3("txn_safe: m_txn not NULL in destructor - calling mdb_txn_abort()"); + LOG_PRINT_L3("mdb_txn_safe: m_txn not NULL in destructor - calling mdb_txn_abort()"); } mdb_txn_abort(m_txn); } @@ -77,11 +77,11 @@ struct txn_safe // This should only be needed for batch transaction which must be ensured to // be aborted before mdb_env_close, not after. So we can't rely on - // BlockchainLMDB destructor to call txn_safe destructor, as that's too late + // BlockchainLMDB destructor to call mdb_txn_safe destructor, as that's too late // to properly abort, since mdb_env_close would have been called earlier. void abort() { - LOG_PRINT_L3("txn_safe: abort()"); + LOG_PRINT_L3("mdb_txn_safe: abort()"); if(m_txn != NULL) { mdb_txn_abort(m_txn); @@ -89,7 +89,7 @@ struct txn_safe } else { - LOG_PRINT_L0("WARNING: txn_safe: abort() called, but m_txn is NULL"); + LOG_PRINT_L0("WARNING: mdb_txn_safe: abort() called, but m_txn is NULL"); } } @@ -306,8 +306,8 @@ private: uint64_t m_height; uint64_t m_num_outputs; std::string m_folder; - txn_safe* m_write_txn; // may point to either a short-lived txn or a batch txn - txn_safe m_write_batch_txn; // persist batch txn outside of BlockchainLMDB + mdb_txn_safe* m_write_txn; // may point to either a short-lived txn or a batch txn + mdb_txn_safe m_write_batch_txn; // persist batch txn outside of BlockchainLMDB bool m_batch_transactions; // support for batch transactions bool m_batch_active; // whether batch transaction is in progress -- cgit v1.2.3 From 1bc89398b4cea4406418f7c385025a1d3c062054 Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Sat, 14 Mar 2015 19:28:34 -0400 Subject: BerkeleyDB BlockchainDB impl copy/paste/modify LMDB implementation code copy/paste/modified into the Berkeley DB implementation. Need to test if it builds, then if it works, and so on, but the code is all there. --- src/blockchain_db/berkeleydb/db_bdb.cpp | 1233 ++++++++++++++++++++++++++++++- src/blockchain_db/berkeleydb/db_bdb.h | 80 +- 2 files changed, 1304 insertions(+), 9 deletions(-) (limited to 'src') diff --git a/src/blockchain_db/berkeleydb/db_bdb.cpp b/src/blockchain_db/berkeleydb/db_bdb.cpp index 9f79c4846..4b8065a0a 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.cpp +++ b/src/blockchain_db/berkeleydb/db_bdb.cpp @@ -54,6 +54,113 @@ inline void throw1(const T &e) throw e; } +// cursor needs to be closed when it goes out of scope, +// this helps if the function using it throws +struct bdb_cur +{ + bdb_cur(DbTxn* txn, Db* dbi) + { + if (dbi->cursor(txn, &m_cur, 0)) + throw0(cryptonote::DB_ERROR("Error opening db cursor")); + done = false; + } + + ~bdb_cur() { close(); } + + operator Dbc*() { return m_cur; } + operator Dbc**() { return &m_cur; } + + void close() + { + if (!done) + { + m_cur->close(); + done = true; + } + } + +private: + Dbc* m_cur; + bool done; +}; + +const char* const BDB_BLOCKS = "blocks"; +const char* const BDB_BLOCK_TIMESTAMPS = "block_timestamps"; +const char* const BDB_BLOCK_HEIGHTS = "block_heights"; +const char* const BDB_BLOCK_HASHES = "block_hashes"; +const char* const BDB_BLOCK_SIZES = "block_sizes"; +const char* const BDB_BLOCK_DIFFS = "block_diffs"; +const char* const BDB_BLOCK_COINS = "block_coins"; + +const char* const BDB_TXS = "txs"; +const char* const BDB_TX_UNLOCKS = "tx_unlocks"; +const char* const BDB_TX_HEIGHTS = "tx_heights"; +const char* const BDB_TX_OUTPUTS = "tx_outputs"; + +const char* const BDB_OUTPUT_TXS = "output_txs"; +const char* const BDB_OUTPUT_INDICES = "output_indices"; +const char* const BDB_OUTPUT_AMOUNTS = "output_amounts"; +const char* const BDB_OUTPUT_KEYS = "output_keys"; + +const char* const BDB_SPENT_KEYS = "spent_keys"; + +template +struct Dbt_copy: public Dbt +{ + Dbt_copy(const T &t): t_copy(t) + { + init(); + } + + Dbt_copy() + { + init(); + } + + void init() + { + set_data(&t_copy); + set_size(sizeof(T)); + set_flags(DB_DBT_USERMEM); + } + + operator T() + { + return t_copy; + } +private: + T t_copy; +}; + +template<> +struct Dbt_copy: public Dbt +{ + Dbt_copy(const cryptonote::blobdata &bd) : m_data(new char[bd.size()]) + { + memcpy(m_data.get(), bd.data(), bd.size()); + set_data(&m_data); + set_size(bd.size()); + } +private: + std::unique_ptr m_data; +}; + +struct Dbt_safe : public Dbt +{ + Dbt_safe() + { + set_data(NULL); + } + ~Dbt_safe() + { + void* buf = get_data(); + if (buf != NULL) + { + free(buf); + } + } +}; + } // anonymous namespace namespace cryptonote @@ -68,36 +175,203 @@ void BlockchainBDB::add_block( const block& blk { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + Dbt_copy val_h(blk_hash); + if (m_block_heights->exists(m_write_txn, &val_h, 0) == 0) + throw1(BLOCK_EXISTS("Attempting to add block that's already in the db")); + + if (m_height > 0) + { + Dbt_copy parent_key(blk.prev_id); + Dbt_copy parent_h; + if (m_block_heights->get(m_write_txn, &parent_key, &parent_h, 0)) + { + LOG_PRINT_L3("m_height: " << m_height); + LOG_PRINT_L3("parent_key: " << blk.prev_id); + throw0(DB_ERROR("Failed to get top block hash to check for new block's parent")); + } + uint64_t parent_height = parent_h; + if (parent_height != m_height - 1) + throw0(BLOCK_PARENT_DNE("Top block is not new block's parent")); + } + + Dbt_copy key(m_height); + + Dbt_copy blob(block_to_blob(blk)); + auto res = m_blocks->put(m_write_txn, &key, &blob, DB_APPEND); + if (res) + throw0(DB_ERROR(std::string("Failed to add block blob to db transaction: "))); + + Dbt_copy sz(block_size); + if (mdb_put(*m_write_txn, m_block_sizes, &key, &sz, DB_APPEND)) + throw0(DB_ERROR("Failed to add block size to db transaction")); + + Dbt_copy ts(blk.timestamp); + if (m_block_timestamps->put(m_write_txn, &key, &ts, DB_APPEND)) + throw0(DB_ERROR("Failed to add block timestamp to db transaction")); + + Dbt_copy diff(cumulative_difficulty); + if (m_block_diffs->put(m_write_txn, &key, &diff, DB_APPEND)) + throw0(DB_ERROR("Failed to add block cumulative difficulty to db transaction")); + + Dbt_copy coinsgen(coins_generated); + if (m_block_coins->put(m_write_txn, &key, &coinsgen, DB_APPEND)) + throw0(DB_ERROR("Failed to add block total generated coins to db transaction")); + + if (m_block_heights->put(m_write_txn, &val_h, &key, DB_APPEND)) + throw0(DB_ERROR("Failed to add block height by hash to db transaction")); + + if (m_block_hashes->put(m_write_txn, &key, &val_h, DB_APPEND)) + throw0(DB_ERROR("Failed to add block hash to db transaction")); } void BlockchainBDB::remove_block() { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + if (m_height == 0) + throw0(BLOCK_DNE ("Attempting to remove block from an empty blockchain")); + + Dbt_copy k(m_height - 1); + Dbt_copy h; + if (m_block_hashes->get(m_write_txn, &k, &h, 0)) + throw1(BLOCK_DNE("Attempting to remove block that's not in the db")); + + if (m_blocks->del(m_write_txn, &k, 0)) + throw1(DB_ERROR("Failed to add removal of block to db transaction")); + + if (m_block_sizes->del(m_write_txn, &k, 0)) + throw1(DB_ERROR("Failed to add removal of block size to db transaction")); + + if (m_block_diffs->del(m_write_txn, &k, 0)) + throw1(DB_ERROR("Failed to add removal of block cumulative difficulty to db transaction")); + + if (m_block_coins->del(m_write_txn, &k, 0)) + throw1(DB_ERROR("Failed to add removal of block total generated coins to db transaction")); + + if (m_block_timestamps->del(m_write_txn, &k, 0)) + throw1(DB_ERROR("Failed to add removal of block timestamp to db transaction")); + + if (m_block_heights->del(m_write_txn, &h, 0)) + throw1(DB_ERROR("Failed to add removal of block height by hash to db transaction")); + + if (m_block_hashes->del(m_write_txn, &k, 0)) + throw1(DB_ERROR("Failed to add removal of block hash to db transaction")); } void BlockchainBDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + Dbt_copy val_h(tx_hash); + + if (m_txs->exists(m_write_txn, &val_h, 0) == 0) + throw1(TX_EXISTS("Attempting to add transaction that's already in the db")); + + Dbt_copy blob(tx_to_blob(tx)); + if (m_txs->put(m_write_txn, &val_h, &blob, DB_APPEND)) + throw0(DB_ERROR("Failed to add tx blob to db transaction")); + + Dbt_copy height(m_height); + if (m_tx_heights->put(m_write_txn, &val_h, &height, DB_APPEND)) + throw0(DB_ERROR("Failed to add tx block height to db transaction")); + + Dbt_copy unlock_time(tx.unlock_time); + if (m_tx_unlocks->put(m_write_txn, &val_h, &unlock_time, DB_APPEND)) + throw0(DB_ERROR("Failed to add tx unlock time to db transaction")); } void BlockchainBDB::remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + Dbt_copy val_h(tx_hash); + if (m_txs->exists(m_write_txn, &val_h, 0)) + throw1(TX_DNE("Attempting to remove transaction that isn't in the db")); + + if (m_txs->del(m_write_txn, &val_h, 0)) + throw1(DB_ERROR("Failed to add removal of tx to db transaction")); + if (m_tx_unlocks->del(m_write_txn, &val_h, 0)) + throw1(DB_ERROR("Failed to add removal of tx unlock time to db transaction")); + if (m_tx_heights->del(m_write_txn, &val_h, 0)) + throw1(DB_ERROR("Failed to add removal of tx block height to db transaction")); + + remove_tx_outputs(tx_hash, tx); + + if (m_tx_outputs->del(m_write_txn, &val_h, 0)) + throw1(DB_ERROR("Failed to add removal of tx outputs to db transaction")); + } void BlockchainBDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index) { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + Dbt_copy k(m_num_outputs); + Dbt_copy v(tx_hash); + + if (m_output_txs->put(m_write_txn, &k, &v, DB_APPEND)) + throw0(DB_ERROR("Failed to add output tx hash to db transaction")); + if (m_tx_outputs->put(m_write_txn, &v, &k, DB_APPEND)) + throw0(DB_ERROR("Failed to add tx output index to db transaction")); + + Dbt_copy val_local_index(local_index); + if (m_output_indices->put(m_write_txn, &k, &val_local_index, DB_APPEND)) + throw0(DB_ERROR("Failed to add tx output index to db transaction")); + + Dbt_copy val_amount(tx_output.amount); + if (auto result = m_output_amounts->put(m_write_txn, &val_amount, &k, DB_APPEND)) + throw0(DB_ERROR(std::string("Failed to add output amount to db transaction"))); + + if (tx_output.target.type() == typeid(txout_to_key)) + { + Dbt_copy val_pubkey(boost::get(tx_output.target).key); + if (m_output_keys->put(m_write_txn, &k, &val_pubkey, DB_APPEND)) + throw0(DB_ERROR("Failed to add output pubkey to db transaction")); + } + + m_num_outputs++; } void BlockchainBDB::remove_tx_outputs(const crypto::hash& tx_hash, const transaction& tx) { LOG_PRINT_L3("BlockchainBDB::" << __func__); + bdb_cur cur(m_write_txn, m_tx_outputs); + + Dbt_copy k(tx_hash); + Dbt_copy v; + + auto result = (Dbc*)cur->get(&k, &v, DB_SET); + if (result == DB_NOTFOUND) + { + throw0(DB_ERROR("Attempting to remove a tx's outputs, but none found.")); + } + else if (result) + { + throw0(DB_ERROR("DB error attempting to get an output")); + } + else + { + db_recno_t num_elems = 0; + (Dbc*)cur->count(&num_elems, 0); + + for (uint64_t i = 0; i < num_elems; ++i) + { + const tx_out tx_output = tx.vout[i]; + remove_output(v, tx_output.amount); + if (i < num_elems - 1) + { + (Dbc*)cur->get(&k, &v, DB_NEXT_DUP); + } + } + } + + cur.close(); } // TODO: probably remove this function @@ -111,40 +385,179 @@ void BlockchainBDB::remove_output(const uint64_t& out_index, const uint64_t amou { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + Dbt_copy k(out_index); + + auto result = m_output_indices->del(m_write_txn, &k, 0); + if (result == DB_NOTFOUND) + { + LOG_PRINT_L0("Unexpected: global output index not found in m_output_indices"); + } + else if (result) + { + throw1(DB_ERROR("Error adding removal of output tx index to db transaction")); + } + + result = m_output_txs->del(m_write_txn, &k, 0); + // if (result != 0 && result != DB_NOTFOUND) + // throw1(DB_ERROR("Error adding removal of output tx hash to db transaction")); + if (result == DB_NOTFOUND) + { + LOG_PRINT_L0("Unexpected: global output index not found in m_output_txs"); + } + else if (result) + { + throw1(DB_ERROR("Error adding removal of output tx hash to db transaction")); + } + + result = m_output_keys->del(m_write_txn, &k, 0); + if (result == DB_NOTFOUND) + { + LOG_PRINT_L0("Unexpected: global output index not found in m_output_keys"); + } + else if (result) + throw1(DB_ERROR("Error adding removal of output pubkey to db transaction")); + + remove_amount_output_index(amount, out_index); + + m_num_outputs--; } void BlockchainBDB::remove_amount_output_index(const uint64_t amount, const uint64_t global_output_index) { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + bdb_cur cur(m_write_txn, m_output_amounts); + + Dbt_copy k(amount); + Dbt_copy v; + + auto result = (Dbc*)cur->get(&k, &v, DB_SET); + if (result == DB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + db_recno_t num_elems = 0; + (Dbc*)cur->count(&num_elems, 0); + + uint64_t amount_output_index = 0; + uint64_t goi = 0; + bool found_index = false; + for (uint64_t i = 0; i < num_elems; ++i) + { + goi = v; + if (goi == global_output_index) + { + amount_output_index = i; + found_index = true; + break; + } + (Dbc*)cur->get(&k, &v, DB_NEXT_DUP); + } + if (found_index) + { + // found the amount output index + // now delete it + result = (Dbc*)cur->del(0); + if (result) + throw0(DB_ERROR(std::string("Error deleting amount output index ").append(boost::lexical_cast(amount_output_index)).c_str())); + } + else + { + // not found + throw1(OUTPUT_DNE("Failed to find amount output index")); + } + cur.close(); } void BlockchainBDB::add_spent_key(const crypto::key_image& k_image) { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + Dbt_copy val_key(k_image); + if (m_spent_keys->exists(m_write_txn, &val_key, 0) == 0) + throw1(KEY_IMAGE_EXISTS("Attempting to add spent key image that's already in the db")); + + Dbt_copy val('\0'); + if (auto result = m_spent_keys->put(m_write_txn, &val_key, &val, DB_APPEND)) + throw1(DB_ERROR(std::string("Error adding spent key image to db transaction: "))); } void BlockchainBDB::remove_spent_key(const crypto::key_image& k_image) { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + Dbt_copy k(k_image); + auto result = m_spent_keys->del(m_write_txn, &k, 0); + if (result != 0 && result != DB_NOTFOUND) + throw1(DB_ERROR("Error adding removal of key image to db transaction")); } blobdata BlockchainBDB::output_to_blob(const tx_out& output) { LOG_PRINT_L3("BlockchainBDB::" << __func__); + blobdata b; + if (!t_serializable_object_to_blob(output, b)) + throw1(DB_ERROR("Error serializing output to blob")); + return b; } tx_out BlockchainBDB::output_from_blob(const blobdata& blob) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); + std::stringstream ss; + ss << blob; + binary_archive ba(ss); + tx_out o; + + if (!(::serialization::serialize(ba, o))) + throw1(DB_ERROR("Error deserializing tx output blob")); + + return o; } uint64_t BlockchainBDB::get_output_global_index(const uint64_t& amount, const uint64_t& index) const { - LOG_PRINT_L3("BlockchainBDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + bdb_cur cur(txn, m_output_amounts); + + Dbt_copy k(amount); + Dbt_copy v; + + auto result = (Dbc*)cur->get(&k, &v, DB_SET); + if (result == MDB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + db_recno_t num_elems; + (Dbc*)cur->count(&num_elems, 0); + + if (num_elems <= index) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found")); + + for (uint64_t i = 0; i < index; ++i) + { + (Dbc*)cur->get(&k, &v, DB_NEXT_DUP); + } + + uint64_t glob_index = v; + + cur.close(); + + txn.commit(); + + return glob_index; } void BlockchainBDB::check_open() const @@ -158,17 +571,160 @@ BlockchainBDB::~BlockchainBDB() { LOG_PRINT_L3("BlockchainBDB::" << __func__); + if (m_open) + { + close(); + } } BlockchainBDB::BlockchainBDB(bool batch_transactions) { LOG_PRINT_L3("BlockchainBDB::" << __func__); + // initialize folder to something "safe" just in case + // someone accidentally misuses this class... + m_folder = "thishsouldnotexistbecauseitisgibberish"; + m_open = false; + + m_batch_transactions = batch_transactions; + m_write_txn = nullptr; + m_height = 0; } void BlockchainBDB::open(const std::string& filename) { LOG_PRINT_L3("BlockchainBDB::" << __func__); + if (m_open) + throw0(DB_OPEN_FAILURE("Attempted to open db, but it's already open")); + + boost::filesystem::path direc(filename); + if (boost::filesystem::exists(direc)) + { + if (!boost::filesystem::is_directory(direc)) + throw0(DB_OPEN_FAILURE("DB needs a directory path, but a file was passed")); + } + else + { + if (!boost::filesystem::create_directory(direc)) + throw0(DB_OPEN_FAILURE(std::string("Failed to create directory ").append(filename).c_str())); + } + + m_folder = filename; + + try + { + + //Create BerkeleyDB environment + m_env = new DbEnv(0); // no flags needed for DbEnv + + uint32_t db_env_open_flags = DB_CREATE | DB_INIT_MPOOL | DB_INIT_LOCK + | DB_INIT_LOG | DB_INIT_TXN | DB_RECOVER + | DB_THREAD; + + // last parameter left 0, files will be created with default rw access + m_env->open(filename.c_str(), db_env_open_flags, 0); + + // begin transaction to init dbs + bdb_txn_safe txn; + m_env->txn_begin(NULL, txn, 0); + + // create Dbs in the environment + m_blocks = new Db(m_env); + m_block_heights = new Db(m_env); + m_block_hashes = new Db(m_env); + m_block_timestamps = new Db(m_env); + m_block_sizes = new Db(m_env); + m_block_diffs = new Db(m_env); + m_block_coins = new Db(m_env); + + m_txs = new Db(m_env); + m_tx_unlocks = new Db(m_env); + m_tx_heights = new Db(m_env); + m_tx_outputs = new Db(m_env); + + m_output_txs = new Db(m_env); + m_output_indices = new Db(m_env); + m_output_amounts = new Db(m_env); + m_output_keys = new Db(m_env); + + m_spent_keys = new Db(m_env); + + // Tell DB about Dbs that need duplicate support + // Note: no need to tell about sorting, + // as the default is insertion order, which we want + m_tx_outputs->set_flags(DB_DUP); + m_output_amounts->set_flags(DB_DUP); + + // Tell DB about fixed-size values. + m_block_heights->set_re_len(sizeof(uint64_t)); + m_block_hashes->set_re_len(sizeof(crypto::hash)); + m_block_timestamps->set_re_len(sizeof(uint64_t)); + m_block_sizes->set_re_len(sizeof(size_t)); // should really store block size as uint64_t... + m_block_diffs->set_re_len(sizeof(difficulty_type)); + m_block_coins->set_re_len(sizeof(uint64_t)); + + m_tx_unlocks->set_re_len(sizeof(uint64_t)); + m_tx_heights->set_re_len(sizeof(uint64_t)); + m_tx_outputs->set_re_len(sizeof(uint64_t)); + + m_output_txs->set_re_len(sizeof(uint64_t)); + m_output_indices->set_re_len(sizeof(uint64_t)); + m_output_amounts->set_re_len(sizeof(uint64_t)); + m_output_keys->set_re_len(sizeof(crypto::public_key)); + + m_spent_keys->set_re_len(sizeof(crypto::key_image)); + + //TODO: Find out if we need to do Db::set_flags(DB_RENUMBER) + // for the RECNO databases. We shouldn't as we're only + // inserting/removing from the end, but we'll see. + + // open Dbs in the environment + // m_tx_outputs and m_output_amounts must be DB_HASH or DB_BTREE + // because they need duplicate entry support. The rest are DB_RECNO, + // as it seems that will be the most performant choice. + m_blocks->open(txn, BDB_BLOCKS, NULL, DB_RECNO, DB_CREATE, 0); + + m_block_timestamps->open(txn, BDB_BLOCK_TIMESTAMPS, NULL, DB_RECNO, DB_CREATE, 0); + m_block_heights->open(txn, BDB_BLOCK_HEIGHTS, NULL, DB_RECNO, DB_CREATE, 0); + m_block_hashes->open(txn, BDB_BLOCK_HASHES, NULL, DB_RECNO, DB_CREATE, 0); + m_block_sizes->open(txn, BDB_BLOCK_SIZES, NULL, DB_RECNO, DB_CREATE, 0); + m_block_diffs->open(txn, BDB_BLOCK_DIFFS, NULL, DB_RECNO, DB_CREATE, 0); + m_block_coins->open(txn, BDB_BLOCK_COINS, NULL, DB_RECNO, DB_CREATE, 0); + + m_txs->open(txn, BDB_TXS, NULL, DB_RECNO, DB_CREATE, 0); + m_tx_unlocks->open(txn, BDB_TX_UNLOCKS, NULL, DB_RECNO, DB_CREATE, 0); + m_tx_heights->open(txn, BDB_TX_HEIGHTS, NULL, DB_RECNO, DB_CREATE, 0); + m_tx_outputs->open(txn, BDB_TX_OUTPUTS, NULL, DB_HASH, DB_CREATE, 0); + + m_output_txs->open(txn, BDB_OUTPUT_TXS, NULL, DB_RECNO, DB_CREATE, 0); + m_output_indices->open(txn, BDB_OUTPUT_INDICES, NULL, DB_RECNO, DB_CREATE, 0); + m_output_amounts->open(txn, BDB_OUTPUT_AMOUNTS, NULL, DB_HASH, DB_CREATE, 0); + m_output_keys->open(txn, BDB_OUTPUT_KEYS, NULL, DB_RECNO, DB_CREATE, 0); + + m_spent_keys->open(txn, BDB_SPENT_KEYS, NULL, DB_RECNO, DB_CREATE, 0); + + DB_BTREE_STAT* stats; + + // DB_FAST_STAT can apparently cause an incorrect number of records + // to be returned. The flag should be set to 0 instead if this proves + // to be the case. + m_blocks->stat(txn, &stats, DB_FAST_STAT); + m_height = stats->bt_nkeys; + delete stats; + + // see above comment about DB_FAST_STAT + m_output_indices->stat(txn, &stats, DB_FAST_STAT); + m_num_outputs = stats->bt_nkeys; + delete stats; + + txn->commit(); + } + catch (const std::exception& e) + { + throw0(DB_OPEN_FAILURE(e.what())); + } + + m_open = true; } // unused for now, create will happen on open if doesn't exist @@ -180,24 +736,117 @@ void BlockchainBDB::create(const std::string& filename) void BlockchainBDB::close() { - LOG_PRINT_L3("BlockchainBDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + this->sync(); + + // FIXME: not yet thread safe!!! Use with care. + m_env->close(DB_FORCESYNC); } void BlockchainBDB::sync() { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + try + { + m_blocks->sync(); + m_block_heights->sync(); + m_block_hashes->sync(); + m_block_timestamps->sync(); + m_block_sizes->sync(); + m_block_diffs->sync(); + m_block_coins->sync(); + + m_txs->sync(); + m_tx_unlocks->sync(); + m_tx_heights->sync(); + m_tx_outputs->sync(); + + m_output_txs->sync(); + m_output_indices->sync(); + m_output_amounts->sync(); + m_output_keys->sync(); + + m_spent_keys->sync(); + } + catch (const std::exception& e) + { + throw0(DB_ERROR(std::string("Failed to sync database: ").append(e.what()))); + } } void BlockchainBDB::reset() { - LOG_PRINT_L3("BlockchainBDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); // TODO: this } std::vector BlockchainBDB::get_filenames() const { LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + std::vector filenames; + + char *fname, *dbname; + + m_blocks->get_dbname(&fname, &dbname); + filenames.push_back(fname); + + m_block_heights->get_dbname(&fname, &dbname); + filenames.push_back(fname); + + m_block_hashes->get_dbname(&fname, &dbname); + filenames.push_back(fname); + + m_block_timestamps->get_dbname(&fname, &dbname); + filenames.push_back(fname); + + m_block_sizes->get_dbname(&fname, &dbname); + filenames.push_back(fname); + + m_block_diffs->get_dbname(&fname, &dbname); + filenames.push_back(fname); + + m_block_coins->get_dbname(&fname, &dbname); + filenames.push_back(fname); + + m_txs->get_dbname(&fname, &dbname); + filenames.push_back(fname); + + m_tx_unlocks->get_dbname(&fname, &dbname); + filenames.push_back(fname); + + m_tx_heights->get_dbname(&fname, &dbname); + filenames.push_back(fname); + + m_tx_outputs->get_dbname(&fname, &dbname); + filenames.push_back(fname); + + m_output_txs->get_dbname(&fname, &dbname); + filenames.push_back(fname); + + m_output_indices->get_dbname(&fname, &dbname); + filenames.push_back(fname); + + m_output_amounts->get_dbname(&fname, &dbname); + filenames.push_back(fname); + + m_output_keys->get_dbname(&fname, &dbname); + filenames.push_back(fname); + + m_spent_keys->get_dbname(&fname, &dbname); + filenames.push_back(fname); + + return filenames; +} + +std::string BlockchainBDB::get_db_name() const +{ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + + return std::string("BerkeleyDB"); } // TODO: this? @@ -219,96 +868,296 @@ bool BlockchainBDB::block_exists(const crypto::hash& h) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy key(h); + + auto get_result = m_block_heights->exists(txn, &key, 0); + if (get_result == DB_NOTFOUND) + { + txn.commit(); + LOG_PRINT_L3("Block with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); + return false; + } + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch block index from hash")); + + txn.commit(); + return true; } block BlockchainBDB::get_block(const crypto::hash& h) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + return get_block_from_height(get_block_height(h)); } uint64_t BlockchainBDB::get_block_height(const crypto::hash& h) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy key(h); + Dbt_copy result; + + auto get_result = m_block_heights->get(txn, &key, &result, 0); + if (get_result == MDB_NOTFOUND) + throw1(BLOCK_DNE("Attempted to retrieve non-existent block height")); + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a block height from the db")); + + txn.commit(); + return result; } block_header BlockchainBDB::get_block_header(const crypto::hash& h) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + // block_header object is automatically cast from block object + return get_block(h); } block BlockchainBDB::get_block_from_height(const uint64_t& height) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy key(height); + Dbt_safe result; + auto get_result = mdb_get(txn, m_blocks, &key, &result); + if (get_result == MDB_NOTFOUND) + { + throw0(DB_ERROR(std::string("Attempt to get block from height ").append(boost::lexical_cast(height)).append(" failed -- block not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a block from the db")); + + txn.commit(); + + blobdata bd; + bd.assign(reinterpret_cast(result.get_data()), result.get_size()); + + block b; + if (!parse_and_validate_block_from_blob(bd, b)) + throw0(DB_ERROR("Failed to parse block from blob retrieved from the db")); + + return b; } uint64_t BlockchainBDB::get_block_timestamp(const uint64_t& height) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy key(height); + Dbt_copy result; + auto get_result = m_block_timestamps->get(txn, &key, &result, 0); + if (get_result == DB_NOTFOUND) + { + throw0(DB_ERROR(std::string("Attempt to get timestamp from height ").append(boost::lexical_cast(height)).append(" failed -- timestamp not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a timestamp from the db")); + + txn.commit(); + return result; } uint64_t BlockchainBDB::get_top_block_timestamp() const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + // if no blocks, return 0 + if (m_height == 0) + { + return 0; + } + + return get_block_timestamp(m_height - 1); } size_t BlockchainBDB::get_block_size(const uint64_t& height) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy key(height); + Dbt_copy result; + auto get_result = m_block_sizes->get(txn, &key, &result, 0); + if (get_result == DB_NOTFOUND) + { + throw0(DB_ERROR(std::string("Attempt to get block size from height ").append(boost::lexical_cast(height)).append(" failed -- block size not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a block size from the db")); + + txn.commit(); + return result; } difficulty_type BlockchainBDB::get_block_cumulative_difficulty(const uint64_t& height) const { LOG_PRINT_L3("BlockchainBDB::" << __func__ << " height: " << height); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy key(height); + Dbt_copy result; + auto get_result = m_block_diffs->get(txn, &key, &result, 0); + if (get_result == DB_NOTFOUND) + { + throw0(DB_ERROR(std::string("Attempt to get cumulative difficulty from height ").append(boost::lexical_cast(height)).append(" failed -- difficulty not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a cumulative difficulty from the db")); + + txn.commit(); + return result; } difficulty_type BlockchainBDB::get_block_difficulty(const uint64_t& height) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + difficulty_type diff1 = 0; + difficulty_type diff2 = 0; + + diff1 = get_block_cumulative_difficulty(height); + if (height != 0) + { + diff2 = get_block_cumulative_difficulty(height - 1); + } + + return diff1 - diff2; } uint64_t BlockchainBDB::get_block_already_generated_coins(const uint64_t& height) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy key(height); + Dbt_copy result; + auto get_result = m_block_coins->get(txn, &key, &result, 0); + if (get_result == DB_NOTFOUND) + { + throw0(DB_ERROR(std::string("Attempt to get generated coins from height ").append(boost::lexical_cast(height)).append(" failed -- block size not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a total generated coins from the db")); + + txn.commit(); + return result; } crypto::hash BlockchainBDB::get_block_hash_from_height(const uint64_t& height) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy key(height); + Dbt_copy result; + auto get_result = m_block_hashes->get(txn, &key, &result, 0); + if (get_result == DB_NOTFOUND) + { + throw0(BLOCK_DNE(std::string("Attempt to get hash from height ").append(boost::lexical_cast(height)).append(" failed -- hash not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR(std::string("Error attempting to retrieve a block hash from the db: "). + append(mdb_strerror(get_result)).c_str())); + + txn.commit(); + return result; } std::vector BlockchainBDB::get_blocks_range(const uint64_t& h1, const uint64_t& h2) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + std::vector v; + + for (uint64_t height = h1; height <= h2; ++height) + { + v.push_back(get_block_from_height(height)); + } + + return v; } std::vector BlockchainBDB::get_hashes_range(const uint64_t& h1, const uint64_t& h2) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + std::vector v; + + for (uint64_t height = h1; height <= h2; ++height) + { + v.push_back(get_block_hash_from_height(height)); + } + + return v; } crypto::hash BlockchainBDB::top_block_hash() const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + if (m_height != 0) + { + return get_block_hash_from_height(m_height - 1); + } + + return null_hash; } block BlockchainBDB::get_top_block() const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + if (m_height != 0) + { + return get_block_from_height(m_height - 1); + } + + block b; + return b; } uint64_t BlockchainBDB::height() const @@ -323,36 +1172,137 @@ bool BlockchainBDB::tx_exists(const crypto::hash& h) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy key(h); + + TIME_MEASURE_START(time1); + auto get_result = m_txs->exists(txn, &key, 0); + TIME_MEASURE_FINISH(time1); + time_tx_exists += time1; + if (get_result == DB_NOTFOUND) + { + txn.commit(); + LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); + return false; + } + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch transaction from hash")); + + return true; } uint64_t BlockchainBDB::get_tx_unlock_time(const crypto::hash& h) const { - LOG_PRINT_L3("BlockchainBDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy key(h); + Dbt_copy result; + auto get_result = m_tx_unlocks->get(txn, &key, &result, 0); + if (get_result == DB_NOTFOUND) + throw1(TX_DNE(std::string("tx unlock time with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch tx unlock time from hash")); + + return result; } transaction BlockchainBDB::get_tx(const crypto::hash& h) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy key(h); + Dbt_safe result; + auto get_result = m_txs->get(txn, &key, &result, 0); + if (get_result == DB_NOTFOUND) + throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch tx from hash")); + + blobdata bd; + bd.assign(reinterpret_cast(result.get_data()), result.get_size()); + + transaction tx; + if (!parse_and_validate_tx_from_blob(bd, tx)) + throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); + + txn.commit(); + + return tx; } uint64_t BlockchainBDB::get_tx_count() const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + DB_BTREE_STAT* stats; + + // DB_FAST_STAT can apparently cause an incorrect number of records + // to be returned. The flag should be set to 0 instead if this proves + // to be the case. + m_txs->stat(txn, &stats, DB_FAST_STAT); + auto num_txs = stats->bt_nkeys; + delete stats; + + txn.commit(); + + return num_txs; } std::vector BlockchainBDB::get_tx_list(const std::vector& hlist) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + std::vector v; + + for (auto& h : hlist) + { + v.push_back(get_tx(h)); + } + + return v; } uint64_t BlockchainBDB::get_tx_block_height(const crypto::hash& h) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy key(h); + Dbt_copy result; + auto get_result = m_tx_heights->get(txn, &key, &result, 0); + if (get_result == DB_NOTFOUND) + { + throw1(TX_DNE(std::string("tx height with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch tx height from hash")); + + txn.commit(); + + return result; } //FIXME: make sure the random method used here is appropriate @@ -360,24 +1310,71 @@ uint64_t BlockchainBDB::get_random_output(const uint64_t& amount) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + uint64_t num_outputs = get_num_outputs(amount); + if (num_outputs == 0) + throw1(OUTPUT_DNE("Attempting to get a random output for an amount, but none exist")); + + return crypto::rand() % num_outputs; } uint64_t BlockchainBDB::get_num_outputs(const uint64_t& amount) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + bdb_cur cur(txn, m_output_amounts); + + Dbt_copy k(amount); + Dbt_copy v; + auto result = (Dbc*)cur->get(&k, &v, DB_SET); + if (result == DB_NOTFOUND) + { + return 0; + } + else if (result) + throw0(DB_ERROR("DB error attempting to get number of outputs of an amount")); + + size_t num_elems = 0; + (Dbc*)cur->count(&num_elems, 0); + + txn.commit(); + + return num_elems; } crypto::public_key BlockchainBDB::get_output_key(const uint64_t& amount, const uint64_t& index) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + uint64_t glob_index = get_output_global_index(amount, index); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy k(glob_index); + Dbt_copy v; + auto get_result = m_output_keys->get(txn, &k, &v, 0); + if (get_result == DB_NOTFOUND) + throw0(DB_ERROR("Attempting to get output pubkey by global index, but key does not exist")); + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve an output pubkey from the db")); + + return v; } +// As this is not used, its return is now a blank output. +// This will save on space in the db. tx_out BlockchainBDB::get_output(const crypto::hash& h, const uint64_t& index) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); + return tx_out(); } // As this is not used, its return is now a blank output. @@ -392,24 +1389,181 @@ tx_out_index BlockchainBDB::get_output_tx_and_index_from_global(const uint64_t& { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy k(index); + Dbt_copy v; + + auto get_result = m_output_txs->get(txn, &k, &v, 0); + if (get_result == DB_NOTFOUND) + throw1(OUTPUT_DNE("output with given index not in db")); + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch output tx hash")); + + crypto::hash tx_hash = v; + + Dbt_copy result; + get_result = m_output_indices->get(txn, &k, &result, 0); + if (get_result == DB_NOTFOUND) + throw1(OUTPUT_DNE("output with given index not in db")); + else if (get_result) + throw0(DB_ERROR("DB error attempting to fetch output tx index")); + + txn.commit(); + + return tx_out_index(tx_hash, result); } tx_out_index BlockchainBDB::get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + bdb_cur cur(txn, m_output_amounts); + + Dbt_copy k(amount); + Dbt_copy v; + + auto result = (Dbc*)cur->get(&k, &v, DB_SET); + if (result == DB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + size_t num_elems = 0; + (Dbc*)cur->count(&num_elems, 0); + + if (num_elems <= index) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found")); + + for (uint64_t i = 0; i < index; ++i) + { + (Dbc*)cur->get(&k, &v, DB_NEXT_DUP); + } + + uint64_t glob_index = v; + + cur.close(); + + txn.commit(); + + return get_output_tx_and_index_from_global(glob_index); } std::vector BlockchainBDB::get_tx_output_indices(const crypto::hash& h) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + std::vector index_vec; + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + bdb_cur cur(txn, m_tx_outputs); + + Dbt_copy k(h); + Dbt_copy v; + auto result = (Dbc*)cur->get(&k, &v, DB_SET); + if (result == DB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output by tx hash and tx index, but output not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + size_t num_elems = 0; + (Dbc*)cur->count(&num_elems, 0); + + for (uint64_t i = 0; i < num_elems; ++i) + { + index_vec.push_back(v); + (Dbc*)cur->get(&k, &v, DB_NEXT_DUP); + } + + cur.close(); + txn.commit(); + + return index_vec; } std::vector BlockchainBDB::get_tx_amount_output_indices(const crypto::hash& h) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + std::vector index_vec; + std::vector index_vec2; + + // get the transaction's global output indices first + index_vec = get_tx_output_indices(h); + // these are next used to obtain the amount output indices + + transaction tx = get_tx(h); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + uint64_t i = 0; + uint64_t global_index; + for (const auto& vout : tx.vout) + { + uint64_t amount = vout.amount; + + global_index = index_vec[i]; + + bdb_cur cur(txn, m_output_amounts); + + Dbt_copy k(amount); + Dbt_copy v; + + auto result = (Dbc*)cur->get(&k, &v, DB_SET); + if (result == DB_NOTFOUND) + throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); + else if (result) + throw0(DB_ERROR("DB error attempting to get an output")); + + size_t num_elems = 0; + (Dbc*)cur->count(&num_elems, 0); + + uint64_t amount_output_index = 0; + uint64_t output_index = 0; + bool found_index = false; + for (uint64_t j = 0; j < num_elems; ++j) + { + output_index = v; + if (output_index == global_index) + { + amount_output_index = j; + found_index = true; + break; + } + (Dbc*)cur->get(&k, &v, DB_NEXT_DUP); + } + if (found_index) + { + index_vec2.push_back(amount_output_index); + } + else + { + // not found + cur.close(); + txn.commit(); + throw1(OUTPUT_DNE("specified output not found in db")); + } + + cur.close(); + ++i; + } + + txn.commit(); + + return index_vec2; } @@ -418,8 +1572,25 @@ bool BlockchainBDB::has_key_image(const crypto::key_image& img) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + + Dbt_copy val_key(img); + if (m_spent_keys->exists(txn, &val_key, 0) == 0) + { + txn.commit(); + return true; + } + + txn.commit(); + return false; } +// Ostensibly BerkeleyDB has batch transaction support built-in, +// so the following few functions will be NOP. + void BlockchainBDB::batch_start() { LOG_PRINT_L3("BlockchainBDB::" << __func__); @@ -442,7 +1613,9 @@ void BlockchainBDB::batch_abort() void BlockchainBDB::set_batch_transactions(bool batch_transactions) { - LOG_PRINT_L3("BlockchainBDB::" << __func__); + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + m_batch_transactions = batch_transactions; + LOG_PRINT_L3("batch transactions " << (m_batch_transactions ? "enabled" : "disabled")); } uint64_t BlockchainBDB::add_block( const block& blk @@ -454,12 +1627,60 @@ uint64_t BlockchainBDB::add_block( const block& blk { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + m_write_txn = &txn; + + uint64_t num_outputs = m_num_outputs; + try + { + BlockchainDB::add_block(blk, block_size, cumulative_difficulty, coins_generated, txs); + m_write_txn = NULL; + + TIME_MEASURE_START(time1); + txn.commit(); + TIME_MEASURE_FINISH(time1); + time_commit1 += time1; + } + } + catch (...) + { + m_num_outputs = num_outputs; + m_write_txn = NULL; + throw; + } + + return ++m_height; } void BlockchainBDB::pop_block(block& blk, std::vector& txs) { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); + + bdb_txn_safe txn; + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + m_write_txn = &txn; + + uint64_t num_outputs = m_num_outputs; + try + { + BlockchainDB::pop_block(blk, txs); + + m_write_txn = NULL; + txn.commit(); + } + catch (...) + { + m_num_outputs = num_outputs; + m_write_txn = NULL; + throw; + } + + --m_height; } } // namespace cryptonote diff --git a/src/blockchain_db/berkeleydb/db_bdb.h b/src/blockchain_db/berkeleydb/db_bdb.h index 32686e6ba..b28afbf20 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.h +++ b/src/blockchain_db/berkeleydb/db_bdb.h @@ -25,12 +25,66 @@ // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#include + #include "blockchain_db/blockchain_db.h" #include "cryptonote_protocol/blobdatatype.h" // for type blobdata namespace cryptonote { +struct bdb_txn_safe +{ + bdb_txn_safe() : m_txn(NULL) { } + ~bdb_txn_safe() + { + LOG_PRINT_L3("bdb_txn_safe: destructor"); + abort(); + } + + void commit(std::string message = "") + { + if (message.size() == 0) + { + message = "Failed to commit a transaction to the db"; + } + + if (txn->commit()) + { + m_txn = NULL; + LOG_PRINT_L0(message); + throw DB_ERROR(message.c_str()); + } + m_txn = NULL; + } + + void abort() + { + LOG_PRINT_L3("bdb_txn_safe: abort()"); + if(m_txn != NULL) + { + m_txn->abort(); + m_txn = NULL; + } + else + { + LOG_PRINT_L0("WARNING: bdb_txn_safe: abort() called, but m_txn is NULL"); + } + } + + operator DbTxn*() + { + return m_txn; + } + + operator DbTxn**() + { + return &m_txn; + } + + DbTxn* m_txn; +}; + class BlockchainBDB : public BlockchainDB { public: @@ -199,15 +253,35 @@ private: void check_open() const; + DbEnv* m_env; + + Db* m_blocks; + Db* m_block_heights; + Db* m_block_hashes; + Db* m_block_timestamps; + Db* m_block_sizes; + Db* m_block_diffs; + Db* m_block_coins; + + Db* m_txs; + Db* m_tx_unlocks; + Db* m_tx_heights; + Db* m_tx_outputs; + + Db* m_output_txs; + Db* m_output_indices; + Db* m_output_amounts; + Db* m_output_keys; + + Db* m_spent_keys; + bool m_open; uint64_t m_height; uint64_t m_num_outputs; std::string m_folder; - txn_safe* m_write_txn; // may point to either a short-lived txn or a batch txn - txn_safe m_write_batch_txn; // persist batch txn outside of BlockchainBDB + txn_safe m_write_txn; // may point to either a short-lived txn or a batch txn bool m_batch_transactions; // support for batch transactions - bool m_batch_active; // whether batch transaction is in progress }; } // namespace cryptonote -- cgit v1.2.3 From cade0da8f1e932a17886aa9893a580fa3e3289c7 Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Mon, 16 Mar 2015 03:12:54 -0400 Subject: CMake wiring, minor cleanup, minor test addition Make Cmake things aware of BerkeleyDB and BlockchainBDB Make the BlockchainDB unit tests aware of BlockchainBDB --- src/blockchain_db/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src') diff --git a/src/blockchain_db/CMakeLists.txt b/src/blockchain_db/CMakeLists.txt index 84b2d6a74..62b2eabe2 100644 --- a/src/blockchain_db/CMakeLists.txt +++ b/src/blockchain_db/CMakeLists.txt @@ -29,6 +29,7 @@ set(blockchain_db_sources blockchain_db.cpp lmdb/db_lmdb.cpp + berkeleydb/db_bdb.cpp ) set(blockchain_db_headers) @@ -36,6 +37,7 @@ set(blockchain_db_headers) set(blockchain_db_private_headers blockchain_db.h lmdb/db_lmdb.h + berkeleydb/db_bdb.h ) bitmonero_private_headers(blockchain_db @@ -57,4 +59,5 @@ target_link_libraries(blockchain_db ${Boost_SYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY} ${LMDB_LIBRARY} + ${BDB_LIBRARY} ${EXTRA_LIBRARIES}) -- cgit v1.2.3 From 43477b7dac861a83ade364ebb5a5c3da8228b3e4 Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Mon, 16 Mar 2015 09:14:51 -0400 Subject: BerkeleyDB Blockchain building, not working yet Everything except actually *using* BlockchainBDB is wired up, but the db itself is not yet working. Some error about user mem not large enough. I think I know what this error means, but I can't determine the cause. Notes: BerkeleyDB does not allow 0-indexing in its recno type databases, so block numbers *in the database* will be 1-indexed. Modifications to indexing have been made as needed. --- src/blockchain_db/CMakeLists.txt | 4 +- src/blockchain_db/berkeleydb/db_bdb.cpp | 379 ++++++++++++++++---------------- src/blockchain_db/berkeleydb/db_bdb.h | 6 +- 3 files changed, 200 insertions(+), 189 deletions(-) (limited to 'src') diff --git a/src/blockchain_db/CMakeLists.txt b/src/blockchain_db/CMakeLists.txt index 62b2eabe2..70b4c876c 100644 --- a/src/blockchain_db/CMakeLists.txt +++ b/src/blockchain_db/CMakeLists.txt @@ -54,10 +54,10 @@ target_link_libraries(blockchain_db ${Boost_DATE_TIME_LIBRARY} ${Boost_PROGRAM_OPTIONS_LIBRARY} ${Boost_SERIALIZATION_LIBRARY} + ${LMDB_LIBRARY} + ${BDB_LIBRARY} LINK_PRIVATE ${Boost_FILESYSTEM_LIBRARY} ${Boost_SYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY} - ${LMDB_LIBRARY} - ${BDB_LIBRARY} ${EXTRA_LIBRARIES}) diff --git a/src/blockchain_db/berkeleydb/db_bdb.cpp b/src/blockchain_db/berkeleydb/db_bdb.cpp index 4b8065a0a..6b510f51a 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.cpp +++ b/src/blockchain_db/berkeleydb/db_bdb.cpp @@ -69,6 +69,7 @@ struct bdb_cur operator Dbc*() { return m_cur; } operator Dbc**() { return &m_cur; } + Dbc* operator->() { return m_cur; } void close() { @@ -138,8 +139,9 @@ struct Dbt_copy: public Dbt Dbt_copy(const cryptonote::blobdata &bd) : m_data(new char[bd.size()]) { memcpy(m_data.get(), bd.data(), bd.size()); - set_data(&m_data); + set_data(m_data.get()); set_size(bd.size()); + set_flags(DB_DBT_USERMEM); } private: std::unique_ptr m_data; @@ -177,52 +179,52 @@ void BlockchainBDB::add_block( const block& blk check_open(); Dbt_copy val_h(blk_hash); - if (m_block_heights->exists(m_write_txn, &val_h, 0) == 0) + if (m_block_heights->exists(*m_write_txn, &val_h, 0) == 0) throw1(BLOCK_EXISTS("Attempting to add block that's already in the db")); - if (m_height > 0) + if (m_height > 1) { Dbt_copy parent_key(blk.prev_id); - Dbt_copy parent_h; - if (m_block_heights->get(m_write_txn, &parent_key, &parent_h, 0)) + Dbt_copy parent_h; + if (m_block_heights->get(*m_write_txn, &parent_key, &parent_h, 0)) { LOG_PRINT_L3("m_height: " << m_height); LOG_PRINT_L3("parent_key: " << blk.prev_id); throw0(DB_ERROR("Failed to get top block hash to check for new block's parent")); } - uint64_t parent_height = parent_h; + uint32_t parent_height = parent_h; if (parent_height != m_height - 1) throw0(BLOCK_PARENT_DNE("Top block is not new block's parent")); } - Dbt_copy key(m_height); + Dbt_copy key(m_height); Dbt_copy blob(block_to_blob(blk)); - auto res = m_blocks->put(m_write_txn, &key, &blob, DB_APPEND); + auto res = m_blocks->put(*m_write_txn, &key, &blob, 0); if (res) - throw0(DB_ERROR(std::string("Failed to add block blob to db transaction: "))); + throw0(DB_ERROR("Failed to add block blob to db transaction.")); Dbt_copy sz(block_size); - if (mdb_put(*m_write_txn, m_block_sizes, &key, &sz, DB_APPEND)) - throw0(DB_ERROR("Failed to add block size to db transaction")); + if (m_block_sizes->put(*m_write_txn, &key, &sz, 0)) + throw0(DB_ERROR("Failed to add block size to db transaction.")); Dbt_copy ts(blk.timestamp); - if (m_block_timestamps->put(m_write_txn, &key, &ts, DB_APPEND)) - throw0(DB_ERROR("Failed to add block timestamp to db transaction")); + if (m_block_timestamps->put(*m_write_txn, &key, &ts, 0)) + throw0(DB_ERROR("Failed to add block timestamp to db transaction.")); Dbt_copy diff(cumulative_difficulty); - if (m_block_diffs->put(m_write_txn, &key, &diff, DB_APPEND)) - throw0(DB_ERROR("Failed to add block cumulative difficulty to db transaction")); + if (m_block_diffs->put(*m_write_txn, &key, &diff, 0)) + throw0(DB_ERROR("Failed to add block cumulative difficulty to db transaction.")); Dbt_copy coinsgen(coins_generated); - if (m_block_coins->put(m_write_txn, &key, &coinsgen, DB_APPEND)) - throw0(DB_ERROR("Failed to add block total generated coins to db transaction")); + if (m_block_coins->put(*m_write_txn, &key, &coinsgen, 0)) + throw0(DB_ERROR("Failed to add block total generated coins to db transaction.")); - if (m_block_heights->put(m_write_txn, &val_h, &key, DB_APPEND)) - throw0(DB_ERROR("Failed to add block height by hash to db transaction")); + if (m_block_heights->put(*m_write_txn, &val_h, &key, 0)) + throw0(DB_ERROR("Failed to add block height by hash to db transaction.")); - if (m_block_hashes->put(m_write_txn, &key, &val_h, DB_APPEND)) - throw0(DB_ERROR("Failed to add block hash to db transaction")); + if (m_block_hashes->put(*m_write_txn, &key, &val_h, 0)) + throw0(DB_ERROR("Failed to add block hash to db transaction.")); } void BlockchainBDB::remove_block() @@ -230,33 +232,33 @@ void BlockchainBDB::remove_block() LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); - if (m_height == 0) + if (m_height <= 1) throw0(BLOCK_DNE ("Attempting to remove block from an empty blockchain")); - Dbt_copy k(m_height - 1); - Dbt_copy h; - if (m_block_hashes->get(m_write_txn, &k, &h, 0)) + Dbt_copy k(m_height - 1); + Dbt_copy h; + if (m_block_hashes->get(*m_write_txn, &k, &h, 0)) throw1(BLOCK_DNE("Attempting to remove block that's not in the db")); - if (m_blocks->del(m_write_txn, &k, 0)) + if (m_blocks->del(*m_write_txn, &k, 0)) throw1(DB_ERROR("Failed to add removal of block to db transaction")); - if (m_block_sizes->del(m_write_txn, &k, 0)) + if (m_block_sizes->del(*m_write_txn, &k, 0)) throw1(DB_ERROR("Failed to add removal of block size to db transaction")); - if (m_block_diffs->del(m_write_txn, &k, 0)) + if (m_block_diffs->del(*m_write_txn, &k, 0)) throw1(DB_ERROR("Failed to add removal of block cumulative difficulty to db transaction")); - if (m_block_coins->del(m_write_txn, &k, 0)) + if (m_block_coins->del(*m_write_txn, &k, 0)) throw1(DB_ERROR("Failed to add removal of block total generated coins to db transaction")); - if (m_block_timestamps->del(m_write_txn, &k, 0)) + if (m_block_timestamps->del(*m_write_txn, &k, 0)) throw1(DB_ERROR("Failed to add removal of block timestamp to db transaction")); - if (m_block_heights->del(m_write_txn, &h, 0)) + if (m_block_heights->del(*m_write_txn, &h, 0)) throw1(DB_ERROR("Failed to add removal of block height by hash to db transaction")); - if (m_block_hashes->del(m_write_txn, &k, 0)) + if (m_block_hashes->del(*m_write_txn, &k, 0)) throw1(DB_ERROR("Failed to add removal of block hash to db transaction")); } @@ -267,19 +269,19 @@ void BlockchainBDB::add_transaction_data(const crypto::hash& blk_hash, const tra Dbt_copy val_h(tx_hash); - if (m_txs->exists(m_write_txn, &val_h, 0) == 0) + if (m_txs->exists(*m_write_txn, &val_h, 0) == 0) throw1(TX_EXISTS("Attempting to add transaction that's already in the db")); Dbt_copy blob(tx_to_blob(tx)); - if (m_txs->put(m_write_txn, &val_h, &blob, DB_APPEND)) + if (m_txs->put(*m_write_txn, &val_h, &blob, 0)) throw0(DB_ERROR("Failed to add tx blob to db transaction")); - Dbt_copy height(m_height); - if (m_tx_heights->put(m_write_txn, &val_h, &height, DB_APPEND)) + Dbt_copy height(m_height); + if (m_tx_heights->put(*m_write_txn, &val_h, &height, 0)) throw0(DB_ERROR("Failed to add tx block height to db transaction")); Dbt_copy unlock_time(tx.unlock_time); - if (m_tx_unlocks->put(m_write_txn, &val_h, &unlock_time, DB_APPEND)) + if (m_tx_unlocks->put(*m_write_txn, &val_h, &unlock_time, 0)) throw0(DB_ERROR("Failed to add tx unlock time to db transaction")); } @@ -289,19 +291,19 @@ void BlockchainBDB::remove_transaction_data(const crypto::hash& tx_hash, const t check_open(); Dbt_copy val_h(tx_hash); - if (m_txs->exists(m_write_txn, &val_h, 0)) + if (m_txs->exists(*m_write_txn, &val_h, 0)) throw1(TX_DNE("Attempting to remove transaction that isn't in the db")); - if (m_txs->del(m_write_txn, &val_h, 0)) + if (m_txs->del(*m_write_txn, &val_h, 0)) throw1(DB_ERROR("Failed to add removal of tx to db transaction")); - if (m_tx_unlocks->del(m_write_txn, &val_h, 0)) + if (m_tx_unlocks->del(*m_write_txn, &val_h, 0)) throw1(DB_ERROR("Failed to add removal of tx unlock time to db transaction")); - if (m_tx_heights->del(m_write_txn, &val_h, 0)) + if (m_tx_heights->del(*m_write_txn, &val_h, 0)) throw1(DB_ERROR("Failed to add removal of tx block height to db transaction")); remove_tx_outputs(tx_hash, tx); - if (m_tx_outputs->del(m_write_txn, &val_h, 0)) + if (m_tx_outputs->del(*m_write_txn, &val_h, 0)) throw1(DB_ERROR("Failed to add removal of tx outputs to db transaction")); } @@ -311,26 +313,26 @@ void BlockchainBDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_out LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); - Dbt_copy k(m_num_outputs); + Dbt_copy k(m_num_outputs); Dbt_copy v(tx_hash); - if (m_output_txs->put(m_write_txn, &k, &v, DB_APPEND)) + if (m_output_txs->put(*m_write_txn, &k, &v, 0)) throw0(DB_ERROR("Failed to add output tx hash to db transaction")); - if (m_tx_outputs->put(m_write_txn, &v, &k, DB_APPEND)) + if (m_tx_outputs->put(*m_write_txn, &v, &k, 0)) throw0(DB_ERROR("Failed to add tx output index to db transaction")); Dbt_copy val_local_index(local_index); - if (m_output_indices->put(m_write_txn, &k, &val_local_index, DB_APPEND)) + if (m_output_indices->put(*m_write_txn, &k, &val_local_index, 0)) throw0(DB_ERROR("Failed to add tx output index to db transaction")); Dbt_copy val_amount(tx_output.amount); - if (auto result = m_output_amounts->put(m_write_txn, &val_amount, &k, DB_APPEND)) - throw0(DB_ERROR(std::string("Failed to add output amount to db transaction"))); + if (m_output_amounts->put(*m_write_txn, &val_amount, &k, 0)) + throw0(DB_ERROR("Failed to add output amount to db transaction.")); if (tx_output.target.type() == typeid(txout_to_key)) { Dbt_copy val_pubkey(boost::get(tx_output.target).key); - if (m_output_keys->put(m_write_txn, &k, &val_pubkey, DB_APPEND)) + if (m_output_keys->put(*m_write_txn, &k, &val_pubkey, 0)) throw0(DB_ERROR("Failed to add output pubkey to db transaction")); } @@ -341,12 +343,12 @@ void BlockchainBDB::remove_tx_outputs(const crypto::hash& tx_hash, const transac { LOG_PRINT_L3("BlockchainBDB::" << __func__); - bdb_cur cur(m_write_txn, m_tx_outputs); + bdb_cur cur(*m_write_txn, m_tx_outputs); Dbt_copy k(tx_hash); - Dbt_copy v; + Dbt_copy v; - auto result = (Dbc*)cur->get(&k, &v, DB_SET); + auto result = cur->get(&k, &v, DB_SET); if (result == DB_NOTFOUND) { throw0(DB_ERROR("Attempting to remove a tx's outputs, but none found.")); @@ -358,7 +360,7 @@ void BlockchainBDB::remove_tx_outputs(const crypto::hash& tx_hash, const transac else { db_recno_t num_elems = 0; - (Dbc*)cur->count(&num_elems, 0); + cur->count(&num_elems, 0); for (uint64_t i = 0; i < num_elems; ++i) { @@ -366,7 +368,7 @@ void BlockchainBDB::remove_tx_outputs(const crypto::hash& tx_hash, const transac remove_output(v, tx_output.amount); if (i < num_elems - 1) { - (Dbc*)cur->get(&k, &v, DB_NEXT_DUP); + cur->get(&k, &v, DB_NEXT_DUP); } } } @@ -386,9 +388,9 @@ void BlockchainBDB::remove_output(const uint64_t& out_index, const uint64_t amou LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); - Dbt_copy k(out_index); + Dbt_copy k(out_index); - auto result = m_output_indices->del(m_write_txn, &k, 0); + auto result = m_output_indices->del(*m_write_txn, &k, 0); if (result == DB_NOTFOUND) { LOG_PRINT_L0("Unexpected: global output index not found in m_output_indices"); @@ -398,7 +400,7 @@ void BlockchainBDB::remove_output(const uint64_t& out_index, const uint64_t amou throw1(DB_ERROR("Error adding removal of output tx index to db transaction")); } - result = m_output_txs->del(m_write_txn, &k, 0); + result = m_output_txs->del(*m_write_txn, &k, 0); // if (result != 0 && result != DB_NOTFOUND) // throw1(DB_ERROR("Error adding removal of output tx hash to db transaction")); if (result == DB_NOTFOUND) @@ -410,7 +412,7 @@ void BlockchainBDB::remove_output(const uint64_t& out_index, const uint64_t amou throw1(DB_ERROR("Error adding removal of output tx hash to db transaction")); } - result = m_output_keys->del(m_write_txn, &k, 0); + result = m_output_keys->del(*m_write_txn, &k, 0); if (result == DB_NOTFOUND) { LOG_PRINT_L0("Unexpected: global output index not found in m_output_keys"); @@ -428,19 +430,19 @@ void BlockchainBDB::remove_amount_output_index(const uint64_t amount, const uint LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); - bdb_cur cur(m_write_txn, m_output_amounts); + bdb_cur cur(*m_write_txn, m_output_amounts); Dbt_copy k(amount); - Dbt_copy v; + Dbt_copy v; - auto result = (Dbc*)cur->get(&k, &v, DB_SET); + auto result = cur->get(&k, &v, DB_SET); if (result == DB_NOTFOUND) throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); else if (result) throw0(DB_ERROR("DB error attempting to get an output")); db_recno_t num_elems = 0; - (Dbc*)cur->count(&num_elems, 0); + cur->count(&num_elems, 0); uint64_t amount_output_index = 0; uint64_t goi = 0; @@ -454,13 +456,13 @@ void BlockchainBDB::remove_amount_output_index(const uint64_t amount, const uint found_index = true; break; } - (Dbc*)cur->get(&k, &v, DB_NEXT_DUP); + cur->get(&k, &v, DB_NEXT_DUP); } if (found_index) { // found the amount output index // now delete it - result = (Dbc*)cur->del(0); + result = cur->del(0); if (result) throw0(DB_ERROR(std::string("Error deleting amount output index ").append(boost::lexical_cast(amount_output_index)).c_str())); } @@ -478,12 +480,12 @@ void BlockchainBDB::add_spent_key(const crypto::key_image& k_image) check_open(); Dbt_copy val_key(k_image); - if (m_spent_keys->exists(m_write_txn, &val_key, 0) == 0) + if (m_spent_keys->exists(*m_write_txn, &val_key, 0) == 0) throw1(KEY_IMAGE_EXISTS("Attempting to add spent key image that's already in the db")); Dbt_copy val('\0'); - if (auto result = m_spent_keys->put(m_write_txn, &val_key, &val, DB_APPEND)) - throw1(DB_ERROR(std::string("Error adding spent key image to db transaction: "))); + if (m_spent_keys->put(*m_write_txn, &val_key, &val, 0)) + throw1(DB_ERROR("Error adding spent key image to db transaction.")); } void BlockchainBDB::remove_spent_key(const crypto::key_image& k_image) @@ -492,7 +494,7 @@ void BlockchainBDB::remove_spent_key(const crypto::key_image& k_image) check_open(); Dbt_copy k(k_image); - auto result = m_spent_keys->del(m_write_txn, &k, 0); + auto result = m_spent_keys->del(*m_write_txn, &k, 0); if (result != 0 && result != DB_NOTFOUND) throw1(DB_ERROR("Error adding removal of key image to db transaction")); } @@ -532,23 +534,23 @@ uint64_t BlockchainBDB::get_output_global_index(const uint64_t& amount, const ui bdb_cur cur(txn, m_output_amounts); Dbt_copy k(amount); - Dbt_copy v; + Dbt_copy v; - auto result = (Dbc*)cur->get(&k, &v, DB_SET); - if (result == MDB_NOTFOUND) + auto result = cur->get(&k, &v, DB_SET); + if (result == DB_NOTFOUND) throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); else if (result) throw0(DB_ERROR("DB error attempting to get an output")); db_recno_t num_elems; - (Dbc*)cur->count(&num_elems, 0); + cur->count(&num_elems, 0); if (num_elems <= index) throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found")); for (uint64_t i = 0; i < index; ++i) { - (Dbc*)cur->get(&k, &v, DB_NEXT_DUP); + cur->get(&k, &v, DB_NEXT_DUP); } uint64_t glob_index = v; @@ -587,7 +589,7 @@ BlockchainBDB::BlockchainBDB(bool batch_transactions) m_batch_transactions = batch_transactions; m_write_txn = nullptr; - m_height = 0; + m_height = 1; } void BlockchainBDB::open(const std::string& filename) @@ -629,25 +631,25 @@ void BlockchainBDB::open(const std::string& filename) m_env->txn_begin(NULL, txn, 0); // create Dbs in the environment - m_blocks = new Db(m_env); - m_block_heights = new Db(m_env); - m_block_hashes = new Db(m_env); - m_block_timestamps = new Db(m_env); - m_block_sizes = new Db(m_env); - m_block_diffs = new Db(m_env); - m_block_coins = new Db(m_env); - - m_txs = new Db(m_env); - m_tx_unlocks = new Db(m_env); - m_tx_heights = new Db(m_env); - m_tx_outputs = new Db(m_env); - - m_output_txs = new Db(m_env); - m_output_indices = new Db(m_env); - m_output_amounts = new Db(m_env); - m_output_keys = new Db(m_env); - - m_spent_keys = new Db(m_env); + m_blocks = new Db(m_env, 0); + m_block_heights = new Db(m_env, 0); + m_block_hashes = new Db(m_env, 0); + m_block_timestamps = new Db(m_env, 0); + m_block_sizes = new Db(m_env, 0); + m_block_diffs = new Db(m_env, 0); + m_block_coins = new Db(m_env, 0); + + m_txs = new Db(m_env, 0); + m_tx_unlocks = new Db(m_env, 0); + m_tx_heights = new Db(m_env, 0); + m_tx_outputs = new Db(m_env, 0); + + m_output_txs = new Db(m_env, 0); + m_output_indices = new Db(m_env, 0); + m_output_amounts = new Db(m_env, 0); + m_output_keys = new Db(m_env, 0); + + m_spent_keys = new Db(m_env, 0); // Tell DB about Dbs that need duplicate support // Note: no need to tell about sorting, @@ -656,24 +658,16 @@ void BlockchainBDB::open(const std::string& filename) m_output_amounts->set_flags(DB_DUP); // Tell DB about fixed-size values. - m_block_heights->set_re_len(sizeof(uint64_t)); m_block_hashes->set_re_len(sizeof(crypto::hash)); m_block_timestamps->set_re_len(sizeof(uint64_t)); m_block_sizes->set_re_len(sizeof(size_t)); // should really store block size as uint64_t... m_block_diffs->set_re_len(sizeof(difficulty_type)); m_block_coins->set_re_len(sizeof(uint64_t)); - m_tx_unlocks->set_re_len(sizeof(uint64_t)); - m_tx_heights->set_re_len(sizeof(uint64_t)); - m_tx_outputs->set_re_len(sizeof(uint64_t)); - - m_output_txs->set_re_len(sizeof(uint64_t)); + m_output_txs->set_re_len(sizeof(crypto::hash)); m_output_indices->set_re_len(sizeof(uint64_t)); - m_output_amounts->set_re_len(sizeof(uint64_t)); m_output_keys->set_re_len(sizeof(crypto::public_key)); - m_spent_keys->set_re_len(sizeof(crypto::key_image)); - //TODO: Find out if we need to do Db::set_flags(DB_RENUMBER) // for the RECNO databases. We shouldn't as we're only // inserting/removing from the end, but we'll see. @@ -685,15 +679,15 @@ void BlockchainBDB::open(const std::string& filename) m_blocks->open(txn, BDB_BLOCKS, NULL, DB_RECNO, DB_CREATE, 0); m_block_timestamps->open(txn, BDB_BLOCK_TIMESTAMPS, NULL, DB_RECNO, DB_CREATE, 0); - m_block_heights->open(txn, BDB_BLOCK_HEIGHTS, NULL, DB_RECNO, DB_CREATE, 0); + m_block_heights->open(txn, BDB_BLOCK_HEIGHTS, NULL, DB_HASH, DB_CREATE, 0); m_block_hashes->open(txn, BDB_BLOCK_HASHES, NULL, DB_RECNO, DB_CREATE, 0); m_block_sizes->open(txn, BDB_BLOCK_SIZES, NULL, DB_RECNO, DB_CREATE, 0); m_block_diffs->open(txn, BDB_BLOCK_DIFFS, NULL, DB_RECNO, DB_CREATE, 0); m_block_coins->open(txn, BDB_BLOCK_COINS, NULL, DB_RECNO, DB_CREATE, 0); - m_txs->open(txn, BDB_TXS, NULL, DB_RECNO, DB_CREATE, 0); - m_tx_unlocks->open(txn, BDB_TX_UNLOCKS, NULL, DB_RECNO, DB_CREATE, 0); - m_tx_heights->open(txn, BDB_TX_HEIGHTS, NULL, DB_RECNO, DB_CREATE, 0); + m_txs->open(txn, BDB_TXS, NULL, DB_HASH, DB_CREATE, 0); + m_tx_unlocks->open(txn, BDB_TX_UNLOCKS, NULL, DB_HASH, DB_CREATE, 0); + m_tx_heights->open(txn, BDB_TX_HEIGHTS, NULL, DB_HASH, DB_CREATE, 0); m_tx_outputs->open(txn, BDB_TX_OUTPUTS, NULL, DB_HASH, DB_CREATE, 0); m_output_txs->open(txn, BDB_OUTPUT_TXS, NULL, DB_RECNO, DB_CREATE, 0); @@ -701,7 +695,7 @@ void BlockchainBDB::open(const std::string& filename) m_output_amounts->open(txn, BDB_OUTPUT_AMOUNTS, NULL, DB_HASH, DB_CREATE, 0); m_output_keys->open(txn, BDB_OUTPUT_KEYS, NULL, DB_RECNO, DB_CREATE, 0); - m_spent_keys->open(txn, BDB_SPENT_KEYS, NULL, DB_RECNO, DB_CREATE, 0); + m_spent_keys->open(txn, BDB_SPENT_KEYS, NULL, DB_HASH, DB_CREATE, 0); DB_BTREE_STAT* stats; @@ -709,15 +703,15 @@ void BlockchainBDB::open(const std::string& filename) // to be returned. The flag should be set to 0 instead if this proves // to be the case. m_blocks->stat(txn, &stats, DB_FAST_STAT); - m_height = stats->bt_nkeys; + m_height = stats->bt_nkeys + 1; delete stats; // see above comment about DB_FAST_STAT m_output_indices->stat(txn, &stats, DB_FAST_STAT); - m_num_outputs = stats->bt_nkeys; + m_num_outputs = stats->bt_nkeys + 1; delete stats; - txn->commit(); + txn.commit(); } catch (const std::exception& e) { @@ -736,10 +730,11 @@ void BlockchainBDB::create(const std::string& filename) void BlockchainBDB::close() { - LOG_PRINT_L3("BlockchainLMDB::" << __func__); + LOG_PRINT_L3("BlockchainBDB::" << __func__); this->sync(); // FIXME: not yet thread safe!!! Use with care. + m_open = false; m_env->close(DB_FORCESYNC); } @@ -750,29 +745,29 @@ void BlockchainBDB::sync() try { - m_blocks->sync(); - m_block_heights->sync(); - m_block_hashes->sync(); - m_block_timestamps->sync(); - m_block_sizes->sync(); - m_block_diffs->sync(); - m_block_coins->sync(); - - m_txs->sync(); - m_tx_unlocks->sync(); - m_tx_heights->sync(); - m_tx_outputs->sync(); - - m_output_txs->sync(); - m_output_indices->sync(); - m_output_amounts->sync(); - m_output_keys->sync(); - - m_spent_keys->sync(); + m_blocks->sync(0); + m_block_heights->sync(0); + m_block_hashes->sync(0); + m_block_timestamps->sync(0); + m_block_sizes->sync(0); + m_block_diffs->sync(0); + m_block_coins->sync(0); + + m_txs->sync(0); + m_tx_unlocks->sync(0); + m_tx_heights->sync(0); + m_tx_outputs->sync(0); + + m_output_txs->sync(0); + m_output_indices->sync(0); + m_output_amounts->sync(0); + m_output_keys->sync(0); + + m_spent_keys->sync(0); } catch (const std::exception& e) { - throw0(DB_ERROR(std::string("Failed to sync database: ").append(e.what()))); + throw0(DB_ERROR(std::string("Failed to sync database: ").append(e.what()).c_str())); } } @@ -790,56 +785,69 @@ std::vector BlockchainBDB::get_filenames() const std::vector filenames; char *fname, *dbname; + const char **pfname, **pdbname; + + pfname = (const char **)&fname; + pdbname = (const char **)&dbname; - m_blocks->get_dbname(&fname, &dbname); + m_blocks->get_dbname(pfname, pdbname); filenames.push_back(fname); - m_block_heights->get_dbname(&fname, &dbname); + m_block_heights->get_dbname(pfname, pdbname); filenames.push_back(fname); - m_block_hashes->get_dbname(&fname, &dbname); + m_block_hashes->get_dbname(pfname, pdbname); filenames.push_back(fname); - m_block_timestamps->get_dbname(&fname, &dbname); + m_block_timestamps->get_dbname(pfname, pdbname); filenames.push_back(fname); - m_block_sizes->get_dbname(&fname, &dbname); + m_block_sizes->get_dbname(pfname, pdbname); filenames.push_back(fname); - m_block_diffs->get_dbname(&fname, &dbname); + m_block_diffs->get_dbname(pfname, pdbname); filenames.push_back(fname); - m_block_coins->get_dbname(&fname, &dbname); + m_block_coins->get_dbname(pfname, pdbname); filenames.push_back(fname); - m_txs->get_dbname(&fname, &dbname); + m_txs->get_dbname(pfname, pdbname); filenames.push_back(fname); - m_tx_unlocks->get_dbname(&fname, &dbname); + m_tx_unlocks->get_dbname(pfname, pdbname); filenames.push_back(fname); - m_tx_heights->get_dbname(&fname, &dbname); + m_tx_heights->get_dbname(pfname, pdbname); filenames.push_back(fname); - m_tx_outputs->get_dbname(&fname, &dbname); + m_tx_outputs->get_dbname(pfname, pdbname); filenames.push_back(fname); - m_output_txs->get_dbname(&fname, &dbname); + m_output_txs->get_dbname(pfname, pdbname); filenames.push_back(fname); - m_output_indices->get_dbname(&fname, &dbname); + m_output_indices->get_dbname(pfname, pdbname); filenames.push_back(fname); - m_output_amounts->get_dbname(&fname, &dbname); + m_output_amounts->get_dbname(pfname, pdbname); filenames.push_back(fname); - m_output_keys->get_dbname(&fname, &dbname); + m_output_keys->get_dbname(pfname, pdbname); filenames.push_back(fname); - m_spent_keys->get_dbname(&fname, &dbname); + m_spent_keys->get_dbname(pfname, pdbname); filenames.push_back(fname); - return filenames; + std::vector full_paths; + + for (auto& filename : filenames) + { + boost::filesystem::path p(m_folder); + p /= filename; + full_paths.push_back(p.string()); + } + + return full_paths; } std::string BlockchainBDB::get_db_name() const @@ -910,13 +918,14 @@ uint64_t BlockchainBDB::get_block_height(const crypto::hash& h) const Dbt_copy result; auto get_result = m_block_heights->get(txn, &key, &result, 0); - if (get_result == MDB_NOTFOUND) + if (get_result == DB_NOTFOUND) throw1(BLOCK_DNE("Attempted to retrieve non-existent block height")); else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a block height from the db")); txn.commit(); - return result; + + return (uint64_t)result - 1; } block_header BlockchainBDB::get_block_header(const crypto::hash& h) const @@ -937,10 +946,10 @@ block BlockchainBDB::get_block_from_height(const uint64_t& height) const if (m_env->txn_begin(NULL, txn, 0)) throw0(DB_ERROR("Failed to create a transaction for the db")); - Dbt_copy key(height); + Dbt_copy key(height + 1); Dbt_safe result; - auto get_result = mdb_get(txn, m_blocks, &key, &result); - if (get_result == MDB_NOTFOUND) + auto get_result = m_blocks->get(txn, &key, &result, 0); + if (get_result == DB_NOTFOUND) { throw0(DB_ERROR(std::string("Attempt to get block from height ").append(boost::lexical_cast(height)).append(" failed -- block not in db").c_str())); } @@ -968,7 +977,7 @@ uint64_t BlockchainBDB::get_block_timestamp(const uint64_t& height) const if (m_env->txn_begin(NULL, txn, 0)) throw0(DB_ERROR("Failed to create a transaction for the db")); - Dbt_copy key(height); + Dbt_copy key(height + 1); Dbt_copy result; auto get_result = m_block_timestamps->get(txn, &key, &result, 0); if (get_result == DB_NOTFOUND) @@ -988,7 +997,7 @@ uint64_t BlockchainBDB::get_top_block_timestamp() const check_open(); // if no blocks, return 0 - if (m_height == 0) + if (m_height <= 0) { return 0; } @@ -1005,7 +1014,7 @@ size_t BlockchainBDB::get_block_size(const uint64_t& height) const if (m_env->txn_begin(NULL, txn, 0)) throw0(DB_ERROR("Failed to create a transaction for the db")); - Dbt_copy key(height); + Dbt_copy key(height + 1); Dbt_copy result; auto get_result = m_block_sizes->get(txn, &key, &result, 0); if (get_result == DB_NOTFOUND) @@ -1028,7 +1037,7 @@ difficulty_type BlockchainBDB::get_block_cumulative_difficulty(const uint64_t& h if (m_env->txn_begin(NULL, txn, 0)) throw0(DB_ERROR("Failed to create a transaction for the db")); - Dbt_copy key(height); + Dbt_copy key(height + 1); Dbt_copy result; auto get_result = m_block_diffs->get(txn, &key, &result, 0); if (get_result == DB_NOTFOUND) @@ -1068,7 +1077,7 @@ uint64_t BlockchainBDB::get_block_already_generated_coins(const uint64_t& height if (m_env->txn_begin(NULL, txn, 0)) throw0(DB_ERROR("Failed to create a transaction for the db")); - Dbt_copy key(height); + Dbt_copy key(height + 1); Dbt_copy result; auto get_result = m_block_coins->get(txn, &key, &result, 0); if (get_result == DB_NOTFOUND) @@ -1091,7 +1100,7 @@ crypto::hash BlockchainBDB::get_block_hash_from_height(const uint64_t& height) c if (m_env->txn_begin(NULL, txn, 0)) throw0(DB_ERROR("Failed to create a transaction for the db")); - Dbt_copy key(height); + Dbt_copy key(height + 1); Dbt_copy result; auto get_result = m_block_hashes->get(txn, &key, &result, 0); if (get_result == DB_NOTFOUND) @@ -1099,8 +1108,7 @@ crypto::hash BlockchainBDB::get_block_hash_from_height(const uint64_t& height) c throw0(BLOCK_DNE(std::string("Attempt to get hash from height ").append(boost::lexical_cast(height)).append(" failed -- hash not in db").c_str())); } else if (get_result) - throw0(DB_ERROR(std::string("Error attempting to retrieve a block hash from the db: "). - append(mdb_strerror(get_result)).c_str())); + throw0(DB_ERROR("Error attempting to retrieve a block hash from the db.")); txn.commit(); return result; @@ -1165,7 +1173,7 @@ uint64_t BlockchainBDB::height() const LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); - return m_height; + return m_height - 1; } bool BlockchainBDB::tx_exists(const crypto::hash& h) const @@ -1302,7 +1310,7 @@ uint64_t BlockchainBDB::get_tx_block_height(const crypto::hash& h) const txn.commit(); - return result; + return (uint64_t)result - 1; } //FIXME: make sure the random method used here is appropriate @@ -1331,7 +1339,7 @@ uint64_t BlockchainBDB::get_num_outputs(const uint64_t& amount) const Dbt_copy k(amount); Dbt_copy v; - auto result = (Dbc*)cur->get(&k, &v, DB_SET); + auto result = cur->get(&k, &v, DB_SET); if (result == DB_NOTFOUND) { return 0; @@ -1339,8 +1347,8 @@ uint64_t BlockchainBDB::get_num_outputs(const uint64_t& amount) const else if (result) throw0(DB_ERROR("DB error attempting to get number of outputs of an amount")); - size_t num_elems = 0; - (Dbc*)cur->count(&num_elems, 0); + db_recno_t num_elems = 0; + cur->count(&num_elems, 0); txn.commit(); @@ -1358,7 +1366,7 @@ crypto::public_key BlockchainBDB::get_output_key(const uint64_t& amount, const u if (m_env->txn_begin(NULL, txn, 0)) throw0(DB_ERROR("Failed to create a transaction for the db")); - Dbt_copy k(glob_index); + Dbt_copy k(glob_index); Dbt_copy v; auto get_result = m_output_keys->get(txn, &k, &v, 0); if (get_result == DB_NOTFOUND) @@ -1394,7 +1402,7 @@ tx_out_index BlockchainBDB::get_output_tx_and_index_from_global(const uint64_t& if (m_env->txn_begin(NULL, txn, 0)) throw0(DB_ERROR("Failed to create a transaction for the db")); - Dbt_copy k(index); + Dbt_copy k(index); Dbt_copy v; auto get_result = m_output_txs->get(txn, &k, &v, 0); @@ -1405,7 +1413,7 @@ tx_out_index BlockchainBDB::get_output_tx_and_index_from_global(const uint64_t& crypto::hash tx_hash = v; - Dbt_copy result; + Dbt_copy result; get_result = m_output_indices->get(txn, &k, &result, 0); if (get_result == DB_NOTFOUND) throw1(OUTPUT_DNE("output with given index not in db")); @@ -1431,21 +1439,21 @@ tx_out_index BlockchainBDB::get_output_tx_and_index(const uint64_t& amount, cons Dbt_copy k(amount); Dbt_copy v; - auto result = (Dbc*)cur->get(&k, &v, DB_SET); + auto result = cur->get(&k, &v, DB_SET); if (result == DB_NOTFOUND) throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); else if (result) throw0(DB_ERROR("DB error attempting to get an output")); - size_t num_elems = 0; - (Dbc*)cur->count(&num_elems, 0); + db_recno_t num_elems = 0; + cur->count(&num_elems, 0); if (num_elems <= index) throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found")); for (uint64_t i = 0; i < index; ++i) { - (Dbc*)cur->get(&k, &v, DB_NEXT_DUP); + cur->get(&k, &v, DB_NEXT_DUP); } uint64_t glob_index = v; @@ -1471,19 +1479,19 @@ std::vector BlockchainBDB::get_tx_output_indices(const crypto::hash& h Dbt_copy k(h); Dbt_copy v; - auto result = (Dbc*)cur->get(&k, &v, DB_SET); + auto result = cur->get(&k, &v, DB_SET); if (result == DB_NOTFOUND) throw1(OUTPUT_DNE("Attempting to get an output by tx hash and tx index, but output not found")); else if (result) throw0(DB_ERROR("DB error attempting to get an output")); - size_t num_elems = 0; - (Dbc*)cur->count(&num_elems, 0); + db_recno_t num_elems = 0; + cur->count(&num_elems, 0); for (uint64_t i = 0; i < num_elems; ++i) { index_vec.push_back(v); - (Dbc*)cur->get(&k, &v, DB_NEXT_DUP); + cur->get(&k, &v, DB_NEXT_DUP); } cur.close(); @@ -1522,14 +1530,14 @@ std::vector BlockchainBDB::get_tx_amount_output_indices(const crypto:: Dbt_copy k(amount); Dbt_copy v; - auto result = (Dbc*)cur->get(&k, &v, DB_SET); + auto result = cur->get(&k, &v, DB_SET); if (result == DB_NOTFOUND) throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); else if (result) throw0(DB_ERROR("DB error attempting to get an output")); - size_t num_elems = 0; - (Dbc*)cur->count(&num_elems, 0); + db_recno_t num_elems = 0; + cur->count(&num_elems, 0); uint64_t amount_output_index = 0; uint64_t output_index = 0; @@ -1543,7 +1551,7 @@ std::vector BlockchainBDB::get_tx_amount_output_indices(const crypto:: found_index = true; break; } - (Dbc*)cur->get(&k, &v, DB_NEXT_DUP); + cur->get(&k, &v, DB_NEXT_DUP); } if (found_index) { @@ -1643,16 +1651,17 @@ uint64_t BlockchainBDB::add_block( const block& blk txn.commit(); TIME_MEASURE_FINISH(time1); time_commit1 += time1; - } } - catch (...) + catch (const std::exception& e) { m_num_outputs = num_outputs; m_write_txn = NULL; - throw; + throw0(DB_ERROR(std::string("Error adding block: ").append(e.what()).c_str())); } - return ++m_height; + m_height++; + + return m_height - 1; } void BlockchainBDB::pop_block(block& blk, std::vector& txs) diff --git a/src/blockchain_db/berkeleydb/db_bdb.h b/src/blockchain_db/berkeleydb/db_bdb.h index b28afbf20..f804db515 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.h +++ b/src/blockchain_db/berkeleydb/db_bdb.h @@ -49,7 +49,7 @@ struct bdb_txn_safe message = "Failed to commit a transaction to the db"; } - if (txn->commit()) + if (m_txn->commit(0)) { m_txn = NULL; LOG_PRINT_L0(message); @@ -103,6 +103,8 @@ public: virtual std::vector get_filenames() const; + virtual std::string get_db_name() const; + virtual bool lock(); virtual void unlock(); @@ -279,7 +281,7 @@ private: uint64_t m_height; uint64_t m_num_outputs; std::string m_folder; - txn_safe m_write_txn; // may point to either a short-lived txn or a batch txn + bdb_txn_safe *m_write_txn; bool m_batch_transactions; // support for batch transactions }; -- cgit v1.2.3 From ffadb6571a3c3b1d5365d682304a98cee79c68c8 Mon Sep 17 00:00:00 2001 From: warptangent Date: Mon, 16 Mar 2015 11:57:26 -0700 Subject: blockchain_export: Add compile-time support for BlockchainDB This allows an LMDB database to be used as the blockchain to export. Adjust SOURCE_DB in src/blockchain_converter/blockchain_export.h depending on needs. Defaults to DB_MEMORY. DB_MEMORY is a sensible default for users migrating to LMDB, as it allows the exporter to use the in-memory blockchain while the other binaries work with LMDB, without recompiling anything. --- src/blockchain_converter/blockchain_export.cpp | 63 +++++++++++++++++--------- src/blockchain_converter/blockchain_export.h | 31 +++++++++---- 2 files changed, 63 insertions(+), 31 deletions(-) (limited to 'src') diff --git a/src/blockchain_converter/blockchain_export.cpp b/src/blockchain_converter/blockchain_export.cpp index 33944dba6..2467951de 100644 --- a/src/blockchain_converter/blockchain_export.cpp +++ b/src/blockchain_converter/blockchain_export.cpp @@ -45,12 +45,6 @@ #include "cryptonote_core/cryptonote_boost_serialization.h" #include "import.h" -// #include "fake_core.h" - -#define SOURCE_DB DB_MEMORY -// currently not supported: -// to use what was set at compile time (DB_MEMORY or DB_LMDB) -// #define SOURCE_DB BLOCKCHAIN_DB static int max_chunk = 0; static size_t height; @@ -151,21 +145,21 @@ void BlockchainExport::write_block(block& block) // put coinbase transaction first transaction coinbase_tx = block.miner_tx; crypto::hash coinbase_tx_hash = get_transaction_hash(coinbase_tx); - // blockchain_storage: +#if SOURCE_DB == DB_MEMORY const transaction* cb_tx_full = m_blockchain_storage->get_tx(coinbase_tx_hash); - // - // BlockchainDB: - // transaction cb_tx_full = m_blockchain_storage->get_tx(coinbase_tx_hash); - +#else + transaction cb_tx_full = m_blockchain_storage->get_db()->get_tx(coinbase_tx_hash); +#endif - // blockchain_storage: +#if SOURCE_DB == DB_MEMORY if (cb_tx_full != NULL) { txs.push_back(*cb_tx_full); } - // - // BlockchainDB: - // txs.push_back(cb_tx_full); +#else + // TODO: should check and abort if cb_tx_full equals null_hash? + txs.push_back(cb_tx_full); +#endif // now add all regular transactions BOOST_FOREACH(const auto& tx_id, block.tx_hashes) @@ -173,7 +167,7 @@ void BlockchainExport::write_block(block& block) #if SOURCE_DB == DB_MEMORY const transaction* tx = m_blockchain_storage->get_tx(tx_id); #else - transaction tx = m_blockchain_storage->get_tx(tx_id); + transaction tx = m_blockchain_storage->get_db()->get_tx(tx_id); #endif #if SOURCE_DB == DB_MEMORY @@ -208,9 +202,22 @@ void BlockchainExport::write_block(block& block) bool include_extra_block_data = true; if (include_extra_block_data) { +#if SOURCE_DB == DB_MEMORY size_t block_size = m_blockchain_storage->get_block_size(block_height); +#else + size_t block_size = m_blockchain_storage->get_db()->get_block_size(block_height); +#endif +#if SOURCE_DB == DB_MEMORY difficulty_type cumulative_difficulty = m_blockchain_storage->get_block_cumulative_difficulty(block_height); +#else + difficulty_type cumulative_difficulty = m_blockchain_storage->get_db()->get_block_cumulative_difficulty(block_height); +#endif +#if SOURCE_DB == DB_MEMORY uint64_t coins_generated = m_blockchain_storage->get_block_coins_generated(block_height); +#else + // TODO TEST to verify that this is the equivalent. make sure no off-by-one error with block height vs block number + uint64_t coins_generated = m_blockchain_storage->get_db()->get_block_already_generated_coins(block_height); +#endif *m_raw_archive << block_size; *m_raw_archive << cumulative_difficulty; @@ -234,7 +241,7 @@ bool BlockchainExport::BlockchainExport::close() #if SOURCE_DB == DB_MEMORY bool BlockchainExport::store_blockchain_raw(blockchain_storage* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_dir, uint64_t use_block_height) #else -bool BlockchainExport::store_blockchain_raw(BlockchainDB* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_dir, uint64_t use_block_height) +bool BlockchainExport::store_blockchain_raw(Blockchain* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_dir, uint64_t use_block_height) #endif { uint64_t use_block_height2 = 0; @@ -339,7 +346,7 @@ int main(int argc, char* argv[]) m_config_folder = command_line::get_arg(vm, data_dir_arg); boost::filesystem::path output_dir {m_config_folder}; output_dir /= "export"; - std::cout << "config directory: " << m_config_folder << ENDL; + LOG_PRINT_L0("Export directory: " << output_dir.string()); // If we wanted to use the memory pool, we would set up a fake_core. @@ -349,13 +356,25 @@ int main(int argc, char* argv[]) // core_storage = new blockchain_storage(&m_mempool); blockchain_storage* core_storage = new blockchain_storage(NULL); + LOG_PRINT_L0("Initializing source blockchain (in-memory database)"); + r = core_storage->init(m_config_folder, opt_testnet); #else - BlockchainDB* core_storage = new BlockchainLMDB(); + // Use Blockchain instead of lower-level BlockchainDB for two reasons: + // 1. Blockchain has the init() method for easy setup + // 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash() + // + // cannot match blockchain_storage setup above with just one line, + // e.g. + // Blockchain* core_storage = new Blockchain(NULL); + // because unlike blockchain_storage constructor, which takes a pointer to + // tx_memory_pool, Blockchain's constructor takes tx_memory_pool object. + LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); + Blockchain* core_storage = NULL; + tx_memory_pool m_mempool(*core_storage); + core_storage = new Blockchain(m_mempool); + r = core_storage->init(m_config_folder, opt_testnet); #endif - // tx_memory_pool m_mempool(core_storage); - LOG_PRINT_L0("Initializing source blockchain storage..."); - r = core_storage->init(m_config_folder, opt_testnet); CHECK_AND_ASSERT_MES(r, false, "Failed to initialize source blockchain storage"); LOG_PRINT_L0("Source blockchain storage initialized OK"); LOG_PRINT_L0("Exporting blockchain raw data..."); diff --git a/src/blockchain_converter/blockchain_export.h b/src/blockchain_converter/blockchain_export.h index cd79d1c47..43e25c039 100644 --- a/src/blockchain_converter/blockchain_export.h +++ b/src/blockchain_converter/blockchain_export.h @@ -35,27 +35,40 @@ #include "cryptonote_core/cryptonote_basic.h" #include "cryptonote_core/blockchain_storage.h" #include "cryptonote_core/blockchain.h" -// TODO: #include "blockchain_db/blockchain_db.h" #include "blockchain_db/lmdb/db_lmdb.h" +// CONFIG: choose one of the three #define's +// +// DB_MEMORY is a sensible default for users migrating to LMDB, as it allows +// the exporter to use the in-memory blockchain while the other binaries +// work with LMDB, without recompiling anything. +// +#define SOURCE_DB DB_MEMORY +// #define SOURCE_DB DB_LMDB +// to use global compile-time setting (DB_MEMORY or DB_LMDB): +// #define SOURCE_DB BLOCKCHAIN_DB + using namespace cryptonote; class BlockchainExport { public: - bool store_blockchain_raw(cryptonote::blockchain_storage* cs, cryptonote::tx_memory_pool* txp, +#if SOURCE_DB == DB_MEMORY + bool store_blockchain_raw(cryptonote::blockchain_storage* cs, cryptonote::tx_memory_pool* txp, boost::filesystem::path& output_dir, uint64_t use_block_height=0); - // - // BlockchainDB: - // bool store_blockchain_raw(cryptonote::BlockchainDB* cs, boost::filesystem::path& output_dir, uint64_t use_block_height=0); +#else + bool store_blockchain_raw(cryptonote::Blockchain* cs, cryptonote::tx_memory_pool* txp, + boost::filesystem::path& output_dir, uint64_t use_block_height=0); +#endif protected: - // blockchain_storage: +#if SOURCE_DB == DB_MEMORY blockchain_storage* m_blockchain_storage; - // - // BlockchainDB: - // BlockchainDB* m_blockchain_storage; +#else + Blockchain* m_blockchain_storage; +#endif + tx_memory_pool* m_tx_pool; typedef std::vector buffer_type; std::ofstream * m_raw_data_file; -- cgit v1.2.3 From e146027acd3887f0cad9dea547cc74c608d5ec0b Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Tue, 17 Mar 2015 17:18:45 -0400 Subject: BlockchainBDB passes unit tests --- src/blockchain_db/berkeleydb/db_bdb.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/blockchain_db/berkeleydb/db_bdb.cpp b/src/blockchain_db/berkeleydb/db_bdb.cpp index 6b510f51a..17208a787 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.cpp +++ b/src/blockchain_db/berkeleydb/db_bdb.cpp @@ -122,6 +122,7 @@ struct Dbt_copy: public Dbt { set_data(&t_copy); set_size(sizeof(T)); + set_ulen(sizeof(T)); set_flags(DB_DBT_USERMEM); } @@ -141,6 +142,7 @@ struct Dbt_copy: public Dbt memcpy(m_data.get(), bd.data(), bd.size()); set_data(m_data.get()); set_size(bd.size()); + set_ulen(bd.size()); set_flags(DB_DBT_USERMEM); } private: @@ -152,6 +154,7 @@ struct Dbt_safe : public Dbt Dbt_safe() { set_data(NULL); + set_flags(DB_DBT_MALLOC); } ~Dbt_safe() { @@ -1656,7 +1659,7 @@ uint64_t BlockchainBDB::add_block( const block& blk { m_num_outputs = num_outputs; m_write_txn = NULL; - throw0(DB_ERROR(std::string("Error adding block: ").append(e.what()).c_str())); + throw; } m_height++; -- cgit v1.2.3 From ead7fad5526dad53c7885895bd5e52c890a49191 Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Tue, 17 Mar 2015 22:12:09 -0400 Subject: BerkeleyDB implementation of BlockchainDB seems to be working! --- src/blockchain_db/berkeleydb/db_bdb.cpp | 34 ++++++++++++++++----------------- src/blockchain_db/berkeleydb/db_bdb.h | 6 ++++-- 2 files changed, 20 insertions(+), 20 deletions(-) (limited to 'src') diff --git a/src/blockchain_db/berkeleydb/db_bdb.cpp b/src/blockchain_db/berkeleydb/db_bdb.cpp index 17208a787..786c13b0b 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.cpp +++ b/src/blockchain_db/berkeleydb/db_bdb.cpp @@ -185,7 +185,7 @@ void BlockchainBDB::add_block( const block& blk if (m_block_heights->exists(*m_write_txn, &val_h, 0) == 0) throw1(BLOCK_EXISTS("Attempting to add block that's already in the db")); - if (m_height > 1) + if (m_height > 0) { Dbt_copy parent_key(blk.prev_id); Dbt_copy parent_h; @@ -196,11 +196,11 @@ void BlockchainBDB::add_block( const block& blk throw0(DB_ERROR("Failed to get top block hash to check for new block's parent")); } uint32_t parent_height = parent_h; - if (parent_height != m_height - 1) + if (parent_height != m_height) throw0(BLOCK_PARENT_DNE("Top block is not new block's parent")); } - Dbt_copy key(m_height); + Dbt_copy key(m_height + 1); Dbt_copy blob(block_to_blob(blk)); auto res = m_blocks->put(*m_write_txn, &key, &blob, 0); @@ -235,10 +235,10 @@ void BlockchainBDB::remove_block() LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); - if (m_height <= 1) + if (m_height == 0) throw0(BLOCK_DNE ("Attempting to remove block from an empty blockchain")); - Dbt_copy k(m_height - 1); + Dbt_copy k(m_height); Dbt_copy h; if (m_block_hashes->get(*m_write_txn, &k, &h, 0)) throw1(BLOCK_DNE("Attempting to remove block that's not in the db")); @@ -279,7 +279,7 @@ void BlockchainBDB::add_transaction_data(const crypto::hash& blk_hash, const tra if (m_txs->put(*m_write_txn, &val_h, &blob, 0)) throw0(DB_ERROR("Failed to add tx blob to db transaction")); - Dbt_copy height(m_height); + Dbt_copy height(m_height + 1); if (m_tx_heights->put(*m_write_txn, &val_h, &height, 0)) throw0(DB_ERROR("Failed to add tx block height to db transaction")); @@ -316,7 +316,7 @@ void BlockchainBDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_out LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); - Dbt_copy k(m_num_outputs); + Dbt_copy k(m_num_outputs + 1); Dbt_copy v(tx_hash); if (m_output_txs->put(*m_write_txn, &k, &v, 0)) @@ -592,10 +592,10 @@ BlockchainBDB::BlockchainBDB(bool batch_transactions) m_batch_transactions = batch_transactions; m_write_txn = nullptr; - m_height = 1; + m_height = 0; } -void BlockchainBDB::open(const std::string& filename) +void BlockchainBDB::open(const std::string& filename, const int db_flags) { LOG_PRINT_L3("BlockchainBDB::" << __func__); @@ -706,12 +706,12 @@ void BlockchainBDB::open(const std::string& filename) // to be returned. The flag should be set to 0 instead if this proves // to be the case. m_blocks->stat(txn, &stats, DB_FAST_STAT); - m_height = stats->bt_nkeys + 1; + m_height = stats->bt_nkeys; delete stats; // see above comment about DB_FAST_STAT m_output_indices->stat(txn, &stats, DB_FAST_STAT); - m_num_outputs = stats->bt_nkeys + 1; + m_num_outputs = stats->bt_nkeys; delete stats; txn.commit(); @@ -1000,7 +1000,7 @@ uint64_t BlockchainBDB::get_top_block_timestamp() const check_open(); // if no blocks, return 0 - if (m_height <= 0) + if (m_height == 0) { return 0; } @@ -1149,7 +1149,7 @@ crypto::hash BlockchainBDB::top_block_hash() const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); - if (m_height != 0) + if (m_height > 0) { return get_block_hash_from_height(m_height - 1); } @@ -1162,7 +1162,7 @@ block BlockchainBDB::get_top_block() const LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); - if (m_height != 0) + if (m_height > 0) { return get_block_from_height(m_height - 1); } @@ -1176,7 +1176,7 @@ uint64_t BlockchainBDB::height() const LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); - return m_height - 1; + return m_height; } bool BlockchainBDB::tx_exists(const crypto::hash& h) const @@ -1662,9 +1662,7 @@ uint64_t BlockchainBDB::add_block( const block& blk throw; } - m_height++; - - return m_height - 1; + return ++m_height; } void BlockchainBDB::pop_block(block& blk, std::vector& txs) diff --git a/src/blockchain_db/berkeleydb/db_bdb.h b/src/blockchain_db/berkeleydb/db_bdb.h index f804db515..b68ed287f 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.h +++ b/src/blockchain_db/berkeleydb/db_bdb.h @@ -39,7 +39,9 @@ struct bdb_txn_safe ~bdb_txn_safe() { LOG_PRINT_L3("bdb_txn_safe: destructor"); - abort(); + + if (m_txn != NULL) + abort(); } void commit(std::string message = "") @@ -91,7 +93,7 @@ public: BlockchainBDB(bool batch_transactions=false); ~BlockchainBDB(); - virtual void open(const std::string& filename); + virtual void open(const std::string& filename, const int db_flags); virtual void create(const std::string& filename); -- cgit v1.2.3 From 0386e9925bb7f42b364dc1284d8f9fd6fdfa2e6c Mon Sep 17 00:00:00 2001 From: warptangent Date: Sun, 22 Mar 2015 10:57:14 -0700 Subject: Add README for blockchain converter, importer, and exporter utilities --- src/blockchain_converter/README.md | 54 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 src/blockchain_converter/README.md (limited to 'src') diff --git a/src/blockchain_converter/README.md b/src/blockchain_converter/README.md new file mode 100644 index 000000000..00160c6b9 --- /dev/null +++ b/src/blockchain_converter/README.md @@ -0,0 +1,54 @@ + +For importing into the LMDB database, compile with `DATABASE=lmdb` + +e.g. + +`DATABASE=lmdb make release` + +This is also the default compile setting on the blockchain branch. + +By default, the exporter will use the original in-memory database (blockchain.bin) as its source. +This default is to make migrating to an LMDB database easy, without having to recompile anything. +To change the source, adjust `SOURCE_DB` in `src/blockchain_converter/blockchain_export.h` according to the comments. + +# Usage: + +See also each utility's "--help" option. + +## Export an existing in-memory database + +`$ blockchain_export` + +This loads the existing blockchain, for whichever database type it was compiled for, and exports it to `$MONERO_DATA_DIR/export/blockchain.raw` + +## Import the exported file + +`$ blockchain_import` + +This imports blocks from `$MONERO_DATA_DIR/export/blockchain.raw` into the current database. + +Defaults: `--batch on`, `--batch size 20000`, `--verify on` + +Batch size refers to number of blocks and can be adjusted for performance based on available RAM. + +Verification should only be turned off if importing from a trusted blockchain. + +```bash +# use default settings to import blockchain.raw into database +$ blockchain_import + +# fast import with large batch size, verification off +$ blockchain_import --batch-size 100000 --verify off + +# LMDB flags can be set by appending them to the database type: +# flags: nosync, nometasync, writemap, mapasync +$ blockchain_import --database lmdb#nosync +$ blockchain_import --database lmdb#nosync,nometasync +``` + +## Blockchain converter with batching +`blockchain_converter` has also been updated and includes batching for faster writes. However, on lower RAM systems, this will be slower than using the exporter and importer utilities. The converter needs to keep the blockchain in memory for the duration of the conversion, like the original bitmonerod, thus leaving less memory available to the destination database to operate. + +```bash +$ blockchain_converter --batch on --batch-size 20000 +``` -- cgit v1.2.3 From 4bedd68d2c058184333101e4d3a31856f6226cd4 Mon Sep 17 00:00:00 2001 From: warptangent Date: Sun, 22 Mar 2015 10:57:18 -0700 Subject: Update Blockchain::get_db() to return reference instead of pointer Where this method is used, a BlockchainDB object is always expected, so a pointer is unnecessary and less safe. --- src/blockchain_converter/blockchain_export.cpp | 10 +++++----- src/blockchain_converter/blockchain_import.cpp | 6 +++--- src/blockchain_converter/fake_core.h | 8 ++++---- src/cryptonote_core/blockchain.h | 4 ++-- 4 files changed, 14 insertions(+), 14 deletions(-) (limited to 'src') diff --git a/src/blockchain_converter/blockchain_export.cpp b/src/blockchain_converter/blockchain_export.cpp index 2467951de..f0fd44a4f 100644 --- a/src/blockchain_converter/blockchain_export.cpp +++ b/src/blockchain_converter/blockchain_export.cpp @@ -148,7 +148,7 @@ void BlockchainExport::write_block(block& block) #if SOURCE_DB == DB_MEMORY const transaction* cb_tx_full = m_blockchain_storage->get_tx(coinbase_tx_hash); #else - transaction cb_tx_full = m_blockchain_storage->get_db()->get_tx(coinbase_tx_hash); + transaction cb_tx_full = m_blockchain_storage->get_db().get_tx(coinbase_tx_hash); #endif #if SOURCE_DB == DB_MEMORY @@ -167,7 +167,7 @@ void BlockchainExport::write_block(block& block) #if SOURCE_DB == DB_MEMORY const transaction* tx = m_blockchain_storage->get_tx(tx_id); #else - transaction tx = m_blockchain_storage->get_db()->get_tx(tx_id); + transaction tx = m_blockchain_storage->get_db().get_tx(tx_id); #endif #if SOURCE_DB == DB_MEMORY @@ -205,18 +205,18 @@ void BlockchainExport::write_block(block& block) #if SOURCE_DB == DB_MEMORY size_t block_size = m_blockchain_storage->get_block_size(block_height); #else - size_t block_size = m_blockchain_storage->get_db()->get_block_size(block_height); + size_t block_size = m_blockchain_storage->get_db().get_block_size(block_height); #endif #if SOURCE_DB == DB_MEMORY difficulty_type cumulative_difficulty = m_blockchain_storage->get_block_cumulative_difficulty(block_height); #else - difficulty_type cumulative_difficulty = m_blockchain_storage->get_db()->get_block_cumulative_difficulty(block_height); + difficulty_type cumulative_difficulty = m_blockchain_storage->get_db().get_block_cumulative_difficulty(block_height); #endif #if SOURCE_DB == DB_MEMORY uint64_t coins_generated = m_blockchain_storage->get_block_coins_generated(block_height); #else // TODO TEST to verify that this is the equivalent. make sure no off-by-one error with block height vs block number - uint64_t coins_generated = m_blockchain_storage->get_db()->get_block_already_generated_coins(block_height); + uint64_t coins_generated = m_blockchain_storage->get_db().get_block_already_generated_coins(block_height); #endif *m_raw_archive << block_size; diff --git a/src/blockchain_converter/blockchain_import.cpp b/src/blockchain_converter/blockchain_import.cpp index 060fe8481..841624e43 100644 --- a/src/blockchain_converter/blockchain_import.cpp +++ b/src/blockchain_converter/blockchain_import.cpp @@ -214,7 +214,7 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path) // Reset stats, in case we're using newly created db, accumulating stats // from addition of genesis block. // This aligns internal db counts with importer counts. - simple_core.m_storage.get_db()->reset_stats(); + simple_core.m_storage.get_db().reset_stats(); } #endif boost::filesystem::path raw_file_path(import_file_path); @@ -497,7 +497,7 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path) simple_core.batch_start(); std::cout << ENDL; #if !defined(BLOCKCHAIN_DB) || (BLOCKCHAIN_DB == DB_LMDB) - simple_core.m_storage.get_db()->show_stats(); + simple_core.m_storage.get_db().show_stats(); #endif } } @@ -525,7 +525,7 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path) simple_core.batch_stop(); } #if !defined(BLOCKCHAIN_DB) || (BLOCKCHAIN_DB == DB_LMDB) - simple_core.m_storage.get_db()->show_stats(); + simple_core.m_storage.get_db().show_stats(); #endif if (h > 0) LOG_PRINT_L0("Finished at height: " << h << " block: " << h-1); diff --git a/src/blockchain_converter/fake_core.h b/src/blockchain_converter/fake_core.h index aff249cd9..175cb4660 100644 --- a/src/blockchain_converter/fake_core.h +++ b/src/blockchain_converter/fake_core.h @@ -55,7 +55,7 @@ struct fake_core_lmdb m_pool.init(path.string()); m_storage.init(path.string(), use_testnet, mdb_flags); if (do_batch) - m_storage.get_db()->set_batch_transactions(do_batch); + m_storage.get_db().set_batch_transactions(do_batch); support_batch = true; support_add_block = true; } @@ -71,17 +71,17 @@ struct fake_core_lmdb , const std::vector& txs ) { - return m_storage.get_db()->add_block(blk, block_size, cumulative_difficulty, coins_generated, txs); + return m_storage.get_db().add_block(blk, block_size, cumulative_difficulty, coins_generated, txs); } void batch_start() { - m_storage.get_db()->batch_start(); + m_storage.get_db().batch_start(); } void batch_stop() { - m_storage.get_db()->batch_stop(); + m_storage.get_db().batch_stop(); } }; diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index da4da075b..dc98a56a4 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -145,9 +145,9 @@ namespace cryptonote void set_enforce_dns_checkpoints(bool enforce); bool update_checkpoints(const std::string& file_path, bool check_dns); - BlockchainDB* get_db() + BlockchainDB& get_db() { - return m_db; + return *m_db; } private: -- cgit v1.2.3 From 7476d2e253b50ade052cc62bdac78cd190b11190 Mon Sep 17 00:00:00 2001 From: warptangent Date: Sun, 22 Mar 2015 10:57:21 -0700 Subject: blockchain_export: show progress during export --- src/blockchain_converter/blockchain_export.cpp | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) (limited to 'src') diff --git a/src/blockchain_converter/blockchain_export.cpp b/src/blockchain_converter/blockchain_export.cpp index f0fd44a4f..4587ce925 100644 --- a/src/blockchain_converter/blockchain_export.cpp +++ b/src/blockchain_converter/blockchain_export.cpp @@ -239,14 +239,16 @@ bool BlockchainExport::BlockchainExport::close() #if SOURCE_DB == DB_MEMORY -bool BlockchainExport::store_blockchain_raw(blockchain_storage* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_dir, uint64_t use_block_height) +bool BlockchainExport::store_blockchain_raw(blockchain_storage* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_dir, uint64_t requested_block_height) #else -bool BlockchainExport::store_blockchain_raw(Blockchain* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_dir, uint64_t use_block_height) +bool BlockchainExport::store_blockchain_raw(Blockchain* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_dir, uint64_t requested_block_height) #endif { - uint64_t use_block_height2 = 0; + uint64_t block_height = 0; m_blockchain_storage = _blockchain_storage; m_tx_pool = _tx_pool; + uint64_t progress_interval = 100; + std::string refresh_string = "\r \r"; LOG_PRINT_L0("Storing blocks raw data..."); if (!BlockchainExport::open(output_dir)) { @@ -255,15 +257,15 @@ bool BlockchainExport::store_blockchain_raw(Blockchain* _blockchain_storage, tx_ } block b; LOG_PRINT_L0("source blockchain height: " << m_blockchain_storage->get_current_blockchain_height()); - LOG_PRINT_L0("requested block height: " << use_block_height); - if ((use_block_height > 0) && (use_block_height < m_blockchain_storage->get_current_blockchain_height())) - use_block_height2 = use_block_height; + LOG_PRINT_L0("requested block height: " << requested_block_height); + if ((requested_block_height > 0) && (requested_block_height < m_blockchain_storage->get_current_blockchain_height())) + block_height = requested_block_height; else { - use_block_height2 = m_blockchain_storage->get_current_blockchain_height(); - LOG_PRINT_L0("using block height: " << use_block_height2); + block_height = m_blockchain_storage->get_current_blockchain_height(); + LOG_PRINT_L0("Using block height of source blockchain: " << block_height); } - for (height=0; height < use_block_height2; ++height) + for (height=0; height < block_height; ++height) { crypto::hash hash = m_blockchain_storage->get_block_id_by_height(height); m_blockchain_storage->get_block_by_hash(hash, b); @@ -271,11 +273,17 @@ bool BlockchainExport::store_blockchain_raw(Blockchain* _blockchain_storage, tx_ if (height % NUM_BLOCKS_PER_CHUNK == 0) { flush_chunk(); } + if (height % progress_interval == 0) { + std::cout << refresh_string; + std::cout << "height " << height << "/" << block_height << std::flush; + } } if (height % NUM_BLOCKS_PER_CHUNK != 0) { flush_chunk(); } + std::cout << refresh_string; + std::cout << "height " << height << "/" << block_height << ENDL; LOG_PRINT_L0("longest chunk was " << max_chunk << " bytes"); return BlockchainExport::close(); -- cgit v1.2.3 From 488080326c676acfcfd4edf7a6c9add71c63599a Mon Sep 17 00:00:00 2001 From: warptangent Date: Sun, 22 Mar 2015 10:57:24 -0700 Subject: blockchain_import: lengthen string for line clear --- src/blockchain_converter/blockchain_import.cpp | 35 ++++++++++++++------------ 1 file changed, 19 insertions(+), 16 deletions(-) (limited to 'src') diff --git a/src/blockchain_converter/blockchain_import.cpp b/src/blockchain_converter/blockchain_import.cpp index 841624e43..46d1ac189 100644 --- a/src/blockchain_converter/blockchain_import.cpp +++ b/src/blockchain_converter/blockchain_import.cpp @@ -57,6 +57,8 @@ static bool opt_testnet = true; // adjustable through command-line argument according to available RAM static uint64_t db_batch_size = 20000; +static std::string refresh_string = "\r \r"; + namespace po = boost::program_options; @@ -146,7 +148,7 @@ int count_blocks(std::string& import_file_path) int chunk_size; import_file.read(buffer1, STR_LENGTH_OF_INT); if (!import_file) { - std::cout << "\r \r"; + std::cout << refresh_string; LOG_PRINT_L1("End of import file reached"); quit = true; break; @@ -154,7 +156,7 @@ int count_blocks(std::string& import_file_path) h += NUM_BLOCKS_PER_CHUNK; if (h % progress_interval == 0) { - std::cout << "\r \r" << "block height: " << h << + std::cout << refresh_string << "block height: " << h << std::flush; } bytes_read += STR_LENGTH_OF_INT; @@ -162,32 +164,33 @@ int count_blocks(std::string& import_file_path) chunk_size = atoi(buffer1); if (chunk_size > BUFFER_SIZE) { - std::cout << "\r \r"; + std::cout << refresh_string; LOG_PRINT_L0("WARNING: chunk_size " << chunk_size << " > BUFFER_SIZE " << BUFFER_SIZE << " height: " << h); throw std::runtime_error("Aborting: chunk size exceeds buffer size"); } if (chunk_size > 100000) { - std::cout << "\r \r"; + std::cout << refresh_string; LOG_PRINT_L0("WARNING: chunk_size " << chunk_size << " > 100000" << " height: " << h); } else if (chunk_size <= 0) { - std::cout << "\r \r"; + std::cout << refresh_string; LOG_PRINT_L0("ERROR: chunk_size " << chunk_size << " <= 0" << " height: " << h); throw std::runtime_error("Aborting"); } // skip to next expected block size value import_file.seekg(chunk_size, std::ios_base::cur); if (! import_file) { - std::cout << "\r \r"; + std::cout << refresh_string; LOG_PRINT_L0("ERROR: unexpected end of import file: bytes read before error: " << import_file.gcount() << " of chunk_size " << chunk_size); throw std::runtime_error("Aborting"); } bytes_read += chunk_size; - std::cout << "\r \r"; + std::cout << refresh_string; + LOG_PRINT_L3("Total bytes scanned: " << bytes_read); } @@ -285,7 +288,7 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path) int chunk_size; import_file.read(buffer1, STR_LENGTH_OF_INT); if (! import_file) { - std::cout << "\r \r"; + std::cout << refresh_string; LOG_PRINT_L0("End of import file reached"); quit = 1; break; @@ -340,7 +343,7 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path) h++; if (h % display_interval == 0) { - std::cout << "\r \r"; + std::cout << refresh_string; LOG_PRINT_L0("loading block height " << h); } else @@ -352,7 +355,7 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path) } catch (const std::exception& e) { - std::cout << "\r \r"; + std::cout << refresh_string; LOG_PRINT_RED_L0("exception while de-archiving block, height=" << h); quit = 1; break; @@ -361,7 +364,7 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path) if (h % progress_interval == 0) { - std::cout << "\r \r" << "block " << h-1 + std::cout << refresh_string << "block " << h-1 << std::flush; } @@ -374,7 +377,7 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path) } catch (const std::exception& e) { - std::cout << "\r \r"; + std::cout << refresh_string; LOG_PRINT_RED_L0("exception while de-archiving tx-num, height=" << h); quit = 1; break; @@ -469,7 +472,7 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path) a >> cumulative_difficulty; a >> coins_generated; - std::cout << "\r \r"; + std::cout << refresh_string; LOG_PRINT_L2("block_size: " << block_size); LOG_PRINT_L2("cumulative_difficulty: " << cumulative_difficulty); LOG_PRINT_L2("coins_generated: " << coins_generated); @@ -480,7 +483,7 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path) } catch (const std::exception& e) { - std::cout << "\r \r"; + std::cout << refresh_string; LOG_PRINT_RED_L0("Error adding block to blockchain: " << e.what()); quit = 2; // make sure we don't commit partial block data break; @@ -491,7 +494,7 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path) { if (h % db_batch_size == 0) { - std::cout << "\r \r"; + std::cout << refresh_string; std::cout << ENDL << "[- batch commit at height " << h << " -]" << ENDL; simple_core.batch_stop(); simple_core.batch_start(); @@ -505,7 +508,7 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path) } catch (const std::exception& e) { - std::cout << "\r \r"; + std::cout << refresh_string; LOG_PRINT_RED_L0("exception while reading from import file, height=" << h); return 2; } -- cgit v1.2.3 From dbdcf11778ac9500ce55affae8db89ac868c42da Mon Sep 17 00:00:00 2001 From: warptangent Date: Sun, 22 Mar 2015 12:33:50 -0700 Subject: blockchain_converter: Add support for resume from last block Add option "--resume " where default is on. --- src/blockchain_converter/blockchain_converter.cpp | 47 ++++++++++++++++++----- 1 file changed, 37 insertions(+), 10 deletions(-) (limited to 'src') diff --git a/src/blockchain_converter/blockchain_converter.cpp b/src/blockchain_converter/blockchain_converter.cpp index eaf799a36..03ab5e434 100644 --- a/src/blockchain_converter/blockchain_converter.cpp +++ b/src/blockchain_converter/blockchain_converter.cpp @@ -44,14 +44,19 @@ #include "version.h" #include +namespace +{ + // CONFIG -static bool opt_batch = true; -static bool opt_testnet = false; +bool opt_batch = true; +bool opt_resume = true; +bool opt_testnet = false; // number of blocks per batch transaction // adjustable through command-line argument according to available RAM -static uint64_t db_batch_size = 20000; +uint64_t db_batch_size = 20000; +} namespace po = boost::program_options; @@ -113,11 +118,14 @@ int main(int argc, char* argv[]) const command_line::arg_descriptor arg_batch = {"batch", "Batch transactions for faster import", true}; + const command_line::arg_descriptor arg_resume = {"resume", + "Resume from current height if output database already exists", true}; // call add_options() directly for these arguments since command_line helpers // support only boolean switch, not boolean argument desc_cmd_sett.add_options() - (arg_batch.name, make_semantic(arg_batch), arg_batch.description) + (arg_batch.name, make_semantic(arg_batch), arg_batch.description) + (arg_resume.name, make_semantic(arg_resume), arg_resume.description) ; po::options_description desc_options("Allowed options"); @@ -137,6 +145,7 @@ int main(int argc, char* argv[]) int log_level = command_line::get_arg(vm, arg_log_level); opt_batch = command_line::get_arg(vm, arg_batch); + opt_resume = command_line::get_arg(vm, arg_resume); db_batch_size = command_line::get_arg(vm, arg_batch_size); if (command_line::get_arg(vm, command_line::arg_help)) @@ -178,23 +187,41 @@ int main(int argc, char* argv[]) { LOG_PRINT_L0("batch: " << std::boolalpha << opt_batch << std::noboolalpha); } + LOG_PRINT_L0("resume: " << std::boolalpha << opt_resume << std::noboolalpha); LOG_PRINT_L0("testnet: " << std::boolalpha << opt_testnet << std::noboolalpha); fake_core c(src_folder, opt_testnet); height = c.m_storage.get_current_blockchain_height(); - if (! num_blocks || num_blocks > height) - end_block = height - 1; - else - end_block = start_block + num_blocks - 1; BlockchainDB *blockchain; blockchain = new BlockchainLMDB(opt_batch); dest_folder /= blockchain->get_db_name(); LOG_PRINT_L0("Source blockchain: " << src_folder); LOG_PRINT_L0("Dest blockchain: " << dest_folder.string()); - LOG_PRINT_L0("Opening LMDB: " << dest_folder.string()); + LOG_PRINT_L0("Opening dest blockchain (BlockchainDB " << blockchain->get_db_name() << ")"); blockchain->open(dest_folder.string()); + LOG_PRINT_L0("Source blockchain height: " << height); + LOG_PRINT_L0("Dest blockchain height: " << blockchain->height()); + + if (opt_resume) + // next block number to add is same as current height + start_block = blockchain->height(); + + if (! num_blocks || (start_block + num_blocks > height)) + end_block = height - 1; + else + end_block = start_block + num_blocks - 1; + + LOG_PRINT_L0("start height: " << start_block+1 << " stop height: " << + end_block+1); + + if (start_block > end_block) + { + LOG_PRINT_L0("Finished: no blocks to add"); + delete blockchain; + return 0; + } if (opt_batch) blockchain->batch_start(); @@ -247,7 +274,7 @@ int main(int argc, char* argv[]) catch (const std::exception& e) { std::cout << ENDL; - std::cerr << "Error adding block to new blockchain: " << e.what() << ENDL; + std::cerr << "Error adding block " << i << " to new blockchain: " << e.what() << ENDL; delete blockchain; return 2; } -- cgit v1.2.3 From 7b14d4a17f739c383322312f1a597f264c074c6e Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Wed, 25 Mar 2015 11:41:30 -0400 Subject: Steps toward multiple dbs available -- working There will need to be some more refactoring for these changes to be considered complete/correct, but for now it's working. new daemon cli argument "--db-type", works for LMDB and BerkeleyDB. A good deal of refactoring is also present in this commit, namely Blockchain no longer instantiates BlockchainDB, but rather is passed a pointer to an already-instantiated BlockchainDB on init(). --- src/blockchain_converter/fake_core.h | 26 +++++++++++++++++++-- src/blockchain_db/berkeleydb/db_bdb.cpp | 7 ------ src/blockchain_db/berkeleydb/db_bdb.h | 3 --- src/blockchain_db/blockchain_db.cpp | 5 ++++ src/blockchain_db/blockchain_db.h | 6 +++-- src/blockchain_db/db_types.h | 40 ++++++++++++++++++++++++++++++++ src/blockchain_db/lmdb/db_lmdb.cpp | 7 ------ src/blockchain_db/lmdb/db_lmdb.h | 4 +--- src/cryptonote_core/blockchain.cpp | 41 ++++++++------------------------- src/cryptonote_core/blockchain.h | 4 +--- src/cryptonote_core/cryptonote_core.cpp | 41 +++++++++++++++++++++++++++++++++ src/daemon/command_line_args.h | 5 ++++ src/daemon/main.cpp | 15 ++++++++++++ 13 files changed, 146 insertions(+), 58 deletions(-) create mode 100644 src/blockchain_db/db_types.h (limited to 'src') diff --git a/src/blockchain_converter/fake_core.h b/src/blockchain_converter/fake_core.h index 175cb4660..f82b05d04 100644 --- a/src/blockchain_converter/fake_core.h +++ b/src/blockchain_converter/fake_core.h @@ -29,8 +29,10 @@ #pragma once #include -#include "cryptonote_core/blockchain.h" // BlockchainDB and LMDB +#include "cryptonote_core/blockchain.h" // BlockchainDB #include "cryptonote_core/blockchain_storage.h" // in-memory DB +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/lmdb/db_lmdb.h" using namespace cryptonote; @@ -53,7 +55,27 @@ struct fake_core_lmdb #endif { m_pool.init(path.string()); - m_storage.init(path.string(), use_testnet, mdb_flags); + + BlockchainDB* db = new BlockchainLMDB(); + + boost::filesystem::path folder(path); + + folder /= db->get_db_name(); + + LOG_PRINT_L0("Loading blockchain from folder " << folder.c_str() << " ..."); + + const std::string filename = folder.string(); + try + { + db->open(filename, mdb_flags); + } + catch (const std::exception& e) + { + LOG_PRINT_L0("Error opening database: " << e.what()); + throw; + } + + m_storage.init(db, use_testnet); if (do_batch) m_storage.get_db().set_batch_transactions(do_batch); support_batch = true; diff --git a/src/blockchain_db/berkeleydb/db_bdb.cpp b/src/blockchain_db/berkeleydb/db_bdb.cpp index 786c13b0b..4b254500b 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.cpp +++ b/src/blockchain_db/berkeleydb/db_bdb.cpp @@ -724,13 +724,6 @@ void BlockchainBDB::open(const std::string& filename, const int db_flags) m_open = true; } -// unused for now, create will happen on open if doesn't exist -void BlockchainBDB::create(const std::string& filename) -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - throw DB_CREATE_FAILURE("create() is not implemented for this BlockchainDB, open() will create files if needed."); -} - void BlockchainBDB::close() { LOG_PRINT_L3("BlockchainBDB::" << __func__); diff --git a/src/blockchain_db/berkeleydb/db_bdb.h b/src/blockchain_db/berkeleydb/db_bdb.h index b68ed287f..d4eb5434c 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.h +++ b/src/blockchain_db/berkeleydb/db_bdb.h @@ -95,8 +95,6 @@ public: virtual void open(const std::string& filename, const int db_flags); - virtual void create(const std::string& filename); - virtual void close(); virtual void sync(); @@ -279,7 +277,6 @@ private: Db* m_spent_keys; - bool m_open; uint64_t m_height; uint64_t m_num_outputs; std::string m_folder; diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp index 8b08d3805..d648be44e 100644 --- a/src/blockchain_db/blockchain_db.cpp +++ b/src/blockchain_db/blockchain_db.cpp @@ -129,6 +129,11 @@ void BlockchainDB::pop_block(block& blk, std::vector& txs) } } +bool BlockchainDB::is_open() +{ + return m_open; +} + void BlockchainDB::remove_transaction(const crypto::hash& tx_hash) { transaction tx = get_tx(tx_hash); diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 7b6b55a40..04d9c5384 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -62,6 +62,7 @@ * * General: * open() + * is_open() * close() * sync() * reset() @@ -328,8 +329,8 @@ public: // open the db at location , or create it if there isn't one. virtual void open(const std::string& filename, const int db_flags = 0) = 0; - // make sure implementation has a create function as well - virtual void create(const std::string& filename) = 0; + // returns true of the db is open/ready, else false + bool is_open(); // close and sync the db virtual void close() = 0; @@ -482,6 +483,7 @@ public: // returns true if key image is present in spent key images storage virtual bool has_key_image(const crypto::key_image& img) const = 0; + bool m_open; }; // class BlockchainDB diff --git a/src/blockchain_db/db_types.h b/src/blockchain_db/db_types.h new file mode 100644 index 000000000..b13007df4 --- /dev/null +++ b/src/blockchain_db/db_types.h @@ -0,0 +1,40 @@ +// Copyright (c) 2015, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers +#pragma once + +namespace cryptonote +{ + + const std::unordered_set blockchain_db_types = + { "lmdb" + , "berkeley" + }; + +} // namespace cryptonote diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 524a1c269..8e09dfab2 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -733,13 +733,6 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) // from here, init should be finished } -// unused for now, create will happen on open if doesn't exist -void BlockchainLMDB::create(const std::string& filename) -{ - LOG_PRINT_L3("BlockchainLMDB::" << __func__); - throw DB_CREATE_FAILURE("create() is not implemented for this BlockchainDB, open() will create files if needed."); -} - void BlockchainLMDB::close() { LOG_PRINT_L3("BlockchainLMDB::" << __func__); diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index c3c7ce439..8f1e07e0d 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -24,6 +24,7 @@ // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#pragma once #include "blockchain_db/blockchain_db.h" #include "cryptonote_protocol/blobdatatype.h" // for type blobdata @@ -116,8 +117,6 @@ public: virtual void open(const std::string& filename, const int mdb_flags=0); - virtual void create(const std::string& filename); - virtual void close(); virtual void sync(); @@ -302,7 +301,6 @@ private: MDB_dbi m_spent_keys; - bool m_open; uint64_t m_height; uint64_t m_num_outputs; std::string m_folder; diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 57934b3fc..0261a5614 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -39,7 +39,6 @@ #include "tx_pool.h" #include "blockchain.h" #include "blockchain_db/blockchain_db.h" -#include "blockchain_db/lmdb/db_lmdb.h" #include "cryptonote_format_utils.h" #include "cryptonote_boost_serialization.h" #include "cryptonote_config.h" @@ -65,8 +64,7 @@ using epee::string_tools::pod_to_hex; DISABLE_VS_WARNINGS(4267) //------------------------------------------------------------------ -// TODO: initialize m_db with a concrete implementation of BlockchainDB -Blockchain::Blockchain(tx_memory_pool& tx_pool):m_db(), m_tx_pool(tx_pool), m_current_block_cumul_sz_limit(0), m_is_in_checkpoint_zone(false), m_is_blockchain_storing(false), m_testnet(false), m_enforce_dns_checkpoints(false) +Blockchain::Blockchain(tx_memory_pool& tx_pool):m_db(), m_tx_pool(tx_pool), m_current_block_cumul_sz_limit(0), m_is_in_checkpoint_zone(false), m_is_blockchain_storing(false), m_enforce_dns_checkpoints(false) { LOG_PRINT_L3("Blockchain::" << __func__); } @@ -226,43 +224,24 @@ uint64_t Blockchain::get_current_blockchain_height() const //------------------------------------------------------------------ //FIXME: possibly move this into the constructor, to avoid accidentally // dereferencing a null BlockchainDB pointer -bool Blockchain::init(const std::string& config_folder, const bool testnet, const int db_flags) +bool Blockchain::init(BlockchainDB* db, const bool testnet) { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); - // TODO: make this configurable - m_db = new BlockchainLMDB(); - - m_config_folder = config_folder; - m_testnet = testnet; - - boost::filesystem::path folder(m_config_folder); - - folder /= m_db->get_db_name(); - - LOG_PRINT_L0("Loading blockchain from folder " << folder.c_str() << " ..."); - - const std::string filename = folder.string(); - try + if (db == nullptr) { - m_db->open(filename, db_flags); + LOG_ERROR("Attempted to init Blockchain with null DB"); + return false; } - catch (const DB_OPEN_FAILURE& e) + if (!db->is_open()) { - LOG_PRINT_L0("No blockchain file found, attempting to create one."); - try - { - m_db->create(filename); - } - catch (const DB_CREATE_FAILURE& db_create_error) - { - LOG_PRINT_L0("Unable to create BlockchainDB! This is not good..."); - //TODO: make sure whatever calls this handles the return value properly - return false; - } + LOG_ERROR("Attempted to init Blockchain with unopened DB"); + return false; } + m_db = db; + // if the blockchain is new, add the genesis block // this feels kinda kludgy to do it this way, but can be looked at later. // TODO: add function to create and store genesis block, diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index dc98a56a4..bc13901d2 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -81,7 +81,7 @@ namespace cryptonote Blockchain(tx_memory_pool& tx_pool); - bool init(const std::string& config_folder, const bool testnet = false, const int db_flags = 0); + bool init(BlockchainDB* db, const bool testnet = false); bool deinit(); void set_checkpoints(checkpoints&& chk_pts) { m_checkpoints = chk_pts; } @@ -180,11 +180,9 @@ namespace cryptonote outputs_container m_outputs; - std::string m_config_folder; checkpoints m_checkpoints; std::atomic m_is_in_checkpoint_zone; std::atomic m_is_blockchain_storing; - bool m_testnet; bool m_enforce_dns_checkpoints; bool switch_to_alternative_blockchain(std::list& alt_chain, bool discard_disconnected_chain); diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index f9b2b19ff..7864b55c8 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -44,6 +44,9 @@ using namespace epee; #include #include "daemon/command_line_args.h" #include "cryptonote_core/checkpoints_create.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/lmdb/db_lmdb.h" +#include "blockchain_db/berkeleydb/db_bdb.h" DISABLE_VS_WARNINGS(4355) @@ -194,7 +197,45 @@ namespace cryptonote r = m_mempool.init(m_config_folder); CHECK_AND_ASSERT_MES(r, false, "Failed to initialize memory pool"); +#if BLOCKCHAIN_DB == DB_LMDB + std::string db_type = command_line::get_arg(vm, daemon_args::arg_db_type); + + BlockchainDB* db = nullptr; + if (db_type == "lmdb") + { + db = new BlockchainLMDB(); + } + else if (db_type == "berkeley") + { + db = new BlockchainBDB(); + } + else + { + LOG_ERROR("Attempted to use non-existant database type"); + return false; + } + + boost::filesystem::path folder(m_config_folder); + + folder /= db->get_db_name(); + + LOG_PRINT_L0("Loading blockchain from folder " << folder.c_str() << " ..."); + + const std::string filename = folder.string(); + try + { + db->open(filename); + } + catch (const DB_ERROR& e) + { + LOG_PRINT_L0("Error opening database: " << e.what()); + return false; + } + + r = m_blockchain_storage.init(db, m_testnet); +#else r = m_blockchain_storage.init(m_config_folder, m_testnet); +#endif CHECK_AND_ASSERT_MES(r, false, "Failed to initialize blockchain storage"); // load json & DNS checkpoints, and verify them diff --git a/src/daemon/command_line_args.h b/src/daemon/command_line_args.h index bcf599128..2bd918478 100644 --- a/src/daemon/command_line_args.h +++ b/src/daemon/command_line_args.h @@ -70,6 +70,11 @@ namespace daemon_args , "checkpoints from DNS server will be enforced" , false }; + const command_line::arg_descriptor arg_db_type = { + "db-type" + , "Specify database type" + , "lmdb" + }; } // namespace daemon_args diff --git a/src/daemon/main.cpp b/src/daemon/main.cpp index 5d8baf497..3bad70378 100644 --- a/src/daemon/main.cpp +++ b/src/daemon/main.cpp @@ -42,6 +42,7 @@ #include "rpc/core_rpc_server.h" #include #include "daemon/command_line_args.h" +#include "blockchain_db/db_types.h" namespace po = boost::program_options; namespace bf = boost::filesystem; @@ -78,6 +79,7 @@ int main(int argc, char const * argv[]) command_line::add_arg(core_settings, daemon_args::arg_log_level); command_line::add_arg(core_settings, daemon_args::arg_testnet_on); command_line::add_arg(core_settings, daemon_args::arg_dns_checkpoints); + command_line::add_arg(core_settings, daemon_args::arg_db_type); daemonizer::init_options(hidden_options, visible_options); daemonize::t_executor::init_options(core_settings); @@ -128,6 +130,19 @@ int main(int argc, char const * argv[]) return 0; } + std::string db_type = command_line::get_arg(vm, daemon_args::arg_db_type); + + // verify that blockchaindb type is valid + if(cryptonote::blockchain_db_types.count(db_type) == 0) + { + std::cout << "Invalid database type (" << db_type << "), available types are:" << std::endl; + for (const auto& type : cryptonote::blockchain_db_types) + { + std::cout << "\t" << type << std::endl; + } + return 0; + } + bool testnet_mode = command_line::get_arg(vm, daemon_args::arg_testnet_on); auto data_dir_arg = testnet_mode ? command_line::arg_testnet_data_dir : command_line::arg_data_dir; -- cgit v1.2.3 From 9519526224e02618313f5dff23de62923de55984 Mon Sep 17 00:00:00 2001 From: Thomas Winget Date: Tue, 7 Apr 2015 14:27:37 -0400 Subject: Only compile BerkeleyDB as an option in non-static --- src/blockchain_db/CMakeLists.txt | 13 +++++++++++++ src/cryptonote_core/cryptonote_core.cpp | 7 +++++++ 2 files changed, 20 insertions(+) (limited to 'src') diff --git a/src/blockchain_db/CMakeLists.txt b/src/blockchain_db/CMakeLists.txt index 70b4c876c..adbe804aa 100644 --- a/src/blockchain_db/CMakeLists.txt +++ b/src/blockchain_db/CMakeLists.txt @@ -29,16 +29,29 @@ set(blockchain_db_sources blockchain_db.cpp lmdb/db_lmdb.cpp + ) + +if (NOT STATIC) + set(blockchain_db_sources + ${blockchain_db_sources} berkeleydb/db_bdb.cpp ) +endif() + set(blockchain_db_headers) set(blockchain_db_private_headers blockchain_db.h lmdb/db_lmdb.h + ) + +if (NOT STATIC) + set(blockchain_db_private_headers + ${blockchain_db_private_headers} berkeleydb/db_bdb.h ) +endif() bitmonero_private_headers(blockchain_db ${crypto_private_headers}) diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 7864b55c8..38c009ca8 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -46,7 +46,9 @@ using namespace epee; #include "cryptonote_core/checkpoints_create.h" #include "blockchain_db/blockchain_db.h" #include "blockchain_db/lmdb/db_lmdb.h" +#ifndef STATICLIB #include "blockchain_db/berkeleydb/db_bdb.h" +#endif DISABLE_VS_WARNINGS(4355) @@ -207,7 +209,12 @@ namespace cryptonote } else if (db_type == "berkeley") { +#ifndef STATICLIB db = new BlockchainBDB(); +#else + LOG_ERROR("BlockchainBDB not supported on STATIC builds"); + return false; +#endif } else { -- cgit v1.2.3