diff options
| -rw-r--r-- | src/blockchain_db/berkeleydb/db_bdb.cpp | 4 | ||||
| -rw-r--r-- | src/blockchain_db/berkeleydb/db_bdb.h | 4 | ||||
| -rw-r--r-- | src/blockchain_db/blockchain_db.cpp | 13 | ||||
| -rw-r--r-- | src/blockchain_db/blockchain_db.h | 36 | ||||
| -rw-r--r-- | src/blockchain_db/lmdb/db_lmdb.cpp | 348 | ||||
| -rw-r--r-- | src/blockchain_db/lmdb/db_lmdb.h | 21 | ||||
| -rw-r--r-- | src/blockchain_utilities/blockchain_usage.cpp | 2 | ||||
| -rw-r--r-- | src/cryptonote_basic/cryptonote_format_utils.cpp | 64 | ||||
| -rw-r--r-- | src/cryptonote_basic/cryptonote_format_utils.h | 4 | ||||
| -rw-r--r-- | src/cryptonote_core/blockchain.cpp | 15 | ||||
| -rw-r--r-- | src/cryptonote_core/blockchain.h | 9 | ||||
| -rw-r--r-- | src/cryptonote_core/cryptonote_core.cpp | 4 | ||||
| -rw-r--r-- | src/cryptonote_core/cryptonote_core.h | 2 | ||||
| -rw-r--r-- | src/rpc/core_rpc_server.cpp | 19 | ||||
| -rw-r--r-- | src/rpc/daemon_handler.cpp | 2 | ||||
| -rw-r--r-- | tests/unit_tests/hardfork.cpp | 6 |
16 files changed, 461 insertions, 92 deletions
diff --git a/src/blockchain_db/berkeleydb/db_bdb.cpp b/src/blockchain_db/berkeleydb/db_bdb.cpp index 3a66ecb93..e1b76ec1e 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.cpp +++ b/src/blockchain_db/berkeleydb/db_bdb.cpp @@ -313,7 +313,7 @@ void BlockchainBDB::remove_block() throw1(DB_ERROR("Failed to add removal of block hash to db transaction")); } -void BlockchainBDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) +void BlockchainBDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); @@ -655,7 +655,7 @@ bool BlockchainBDB::for_all_blocks(std::function<bool(uint64_t, const crypto::ha return ret; } -bool BlockchainBDB::for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)> f) const +bool BlockchainBDB::for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)> f, bool pruned) const { LOG_PRINT_L3("BlockchainBDB::" << __func__); check_open(); diff --git a/src/blockchain_db/berkeleydb/db_bdb.h b/src/blockchain_db/berkeleydb/db_bdb.h index 238a90686..cecbba28f 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.h +++ b/src/blockchain_db/berkeleydb/db_bdb.h @@ -360,7 +360,7 @@ private: virtual void remove_block(); - virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash); + virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash); virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx); @@ -381,7 +381,7 @@ private: virtual bool for_all_key_images(std::function<bool(const crypto::key_image&)>) const; virtual bool for_all_blocks(std::function<bool(uint64_t, const crypto::hash&, const cryptonote::block&)>) const; - virtual bool for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)>) const; + virtual bool for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)>, bool pruned) const; virtual bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, size_t tx_idx)> f) const; // Hard fork related storage diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp index 9f760dc0d..88ac34255 100644 --- a/src/blockchain_db/blockchain_db.cpp +++ b/src/blockchain_db/blockchain_db.cpp @@ -121,10 +121,10 @@ void BlockchainDB::pop_block() pop_block(blk, txs); } -void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr) +void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr, const crypto::hash* tx_prunable_hash_ptr) { bool miner_tx = false; - crypto::hash tx_hash; + crypto::hash tx_hash, tx_prunable_hash; if (!tx_hash_ptr) { // should only need to compute hash for miner transactions @@ -135,6 +135,13 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti { tx_hash = *tx_hash_ptr; } + if (tx.version >= 2) + { + if (!tx_prunable_hash_ptr) + tx_prunable_hash = get_transaction_prunable_hash(tx); + else + tx_prunable_hash = *tx_prunable_hash_ptr; + } for (const txin_v& tx_input : tx.vin) { @@ -161,7 +168,7 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti } } - uint64_t tx_id = add_transaction_data(blk_hash, tx, tx_hash); + uint64_t tx_id = add_transaction_data(blk_hash, tx, tx_hash, tx_prunable_hash); std::vector<uint64_t> amount_output_indices; diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 1ed715315..19ba32340 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -398,9 +398,10 @@ private: * @param blk_hash the hash of the block containing the transaction * @param tx the transaction to be added * @param tx_hash the hash of the transaction + * @param tx_prunable_hash the hash of the prunable part of the transaction * @return the transaction ID */ - virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) = 0; + virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) = 0; /** * @brief remove data about a transaction @@ -526,8 +527,9 @@ protected: * @param blk_hash hash of the block which has the transaction * @param tx the transaction to add * @param tx_hash_ptr the hash of the transaction, if already calculated + * @param tx_prunable_hash_ptr the hash of the prunable part of the transaction, if already calculated */ - void add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr = NULL); + void add_transaction(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash* tx_hash_ptr = NULL, const crypto::hash* tx_prunable_hash_ptr = NULL); mutable uint64_t time_tx_exists = 0; //!< a performance metric uint64_t time_commit1 = 0; //!< a performance metric @@ -1120,6 +1122,33 @@ public: virtual bool get_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const = 0; /** + * @brief fetches the pruned transaction blob with the given hash + * + * The subclass should return the pruned transaction stored which has the given + * hash. + * + * If the transaction does not exist, the subclass should return false. + * + * @param h the hash to look for + * + * @return true iff the transaction was found + */ + virtual bool get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const = 0; + + /** + * @brief fetches the prunable transaction hash + * + * The subclass should return the hash of the prunable transaction data. + * + * If the transaction hash does not exist, the subclass should return false. + * + * @param h the tx hash to look for + * + * @return true iff the transaction was found + */ + virtual bool get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const = 0; + + /** * @brief fetches the total number of transactions ever * * The subclass should return a count of all the transactions from @@ -1426,10 +1455,11 @@ public: * not found. Current implementations simply return false. * * @param std::function fn the function to run + * @param bool pruned whether to only get pruned tx data, or the whole * * @return false if the function returns false for any transaction, otherwise true */ - virtual bool for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)>) const = 0; + virtual bool for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)>, bool pruned) const = 0; /** * @brief runs a function over all outputs stored diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 5c20c8de4..65b7f39d5 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -52,9 +52,8 @@ using epee::string_tools::pod_to_hex; using namespace crypto; -// Increase when the DB changes in a non backward compatible way, and there -// is no automatic conversion, so that a full resync is needed. -#define VERSION 1 +// Increase when the DB structure changes +#define VERSION 2 namespace { @@ -164,7 +163,9 @@ int compare_string(const MDB_val *a, const MDB_val *b) * block_heights block hash block height * block_info block ID {block metadata} * - * txs txn ID txn blob + * txs_pruned txn ID pruned txn blob + * txs_prunable txn ID prunable txn blob + * txs_prunable_hash txn ID prunable txn hash * tx_indices txn hash {txn ID, metadata} * tx_outputs txn ID [txn amount output indices] * @@ -189,6 +190,9 @@ const char* const LMDB_BLOCK_HEIGHTS = "block_heights"; const char* const LMDB_BLOCK_INFO = "block_info"; const char* const LMDB_TXS = "txs"; +const char* const LMDB_TXS_PRUNED = "txs_pruned"; +const char* const LMDB_TXS_PRUNABLE = "txs_prunable"; +const char* const LMDB_TXS_PRUNABLE_HASH = "txs_prunable_hash"; const char* const LMDB_TX_INDICES = "tx_indices"; const char* const LMDB_TX_OUTPUTS = "tx_outputs"; @@ -764,7 +768,7 @@ void BlockchainLMDB::remove_block() throw1(DB_ERROR(lmdb_error("Failed to add removal of block info to db transaction: ", result).c_str())); } -uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) +uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); @@ -774,7 +778,9 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons int result; uint64_t tx_id = get_tx_count(); - CURSOR(txs) + CURSOR(txs_pruned) + CURSOR(txs_prunable) + CURSOR(txs_prunable_hash) CURSOR(tx_indices) MDB_val_set(val_tx_id, tx_id); @@ -800,10 +806,35 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons if (result) throw0(DB_ERROR(lmdb_error("Failed to add tx data to db transaction: ", result).c_str())); - MDB_val_copy<blobdata> blob(tx_to_blob(tx)); - result = mdb_cursor_put(m_cur_txs, &val_tx_id, &blob, MDB_APPEND); + cryptonote::blobdata blob = tx_to_blob(tx); + MDB_val_copy<blobdata> blobval(blob); + + std::stringstream ss; + binary_archive<true> ba(ss); + bool r = const_cast<cryptonote::transaction&>(tx).serialize_base(ba); + if (!r) + throw0(DB_ERROR("Failed to serialize pruned tx")); + std::string pruned = ss.str(); + MDB_val_copy<blobdata> pruned_blob(pruned); + result = mdb_cursor_put(m_cur_txs_pruned, &val_tx_id, &pruned_blob, MDB_APPEND); if (result) - throw0(DB_ERROR(lmdb_error("Failed to add tx blob to db transaction: ", result).c_str())); + throw0(DB_ERROR(lmdb_error("Failed to add pruned tx blob to db transaction: ", result).c_str())); + + if (pruned.size() > blob.size()) + throw0(DB_ERROR("pruned tx size is larger than tx size")); + cryptonote::blobdata prunable(blob.data() + pruned.size(), blob.size() - pruned.size()); + MDB_val_copy<blobdata> prunable_blob(prunable); + result = mdb_cursor_put(m_cur_txs_prunable, &val_tx_id, &prunable_blob, MDB_APPEND); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to add prunable tx blob to db transaction: ", result).c_str())); + + if (tx.version > 1) + { + MDB_val_set(val_prunable_hash, tx_prunable_hash); + result = mdb_cursor_put(m_cur_txs_prunable_hash, &val_tx_id, &val_prunable_hash, MDB_APPEND); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to add prunable tx prunable hash to db transaction: ", result).c_str())); + } return tx_id; } @@ -819,7 +850,9 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const mdb_txn_cursors *m_cursors = &m_wcursors; CURSOR(tx_indices) - CURSOR(txs) + CURSOR(txs_pruned) + CURSOR(txs_prunable) + CURSOR(txs_prunable_hash) CURSOR(tx_outputs) MDB_val_set(val_h, tx_hash); @@ -829,11 +862,26 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const txindex *tip = (txindex *)val_h.mv_data; MDB_val_set(val_tx_id, tip->data.tx_id); - if ((result = mdb_cursor_get(m_cur_txs, &val_tx_id, NULL, MDB_SET))) - throw1(DB_ERROR(lmdb_error("Failed to locate tx for removal: ", result).c_str())); - result = mdb_cursor_del(m_cur_txs, 0); + if ((result = mdb_cursor_get(m_cur_txs_pruned, &val_tx_id, NULL, MDB_SET))) + throw1(DB_ERROR(lmdb_error("Failed to locate pruned tx for removal: ", result).c_str())); + result = mdb_cursor_del(m_cur_txs_pruned, 0); + if (result) + throw1(DB_ERROR(lmdb_error("Failed to add removal of pruned tx to db transaction: ", result).c_str())); + + if ((result = mdb_cursor_get(m_cur_txs_prunable, &val_tx_id, NULL, MDB_SET))) + throw1(DB_ERROR(lmdb_error("Failed to locate prunable tx for removal: ", result).c_str())); + result = mdb_cursor_del(m_cur_txs_prunable, 0); if (result) - throw1(DB_ERROR(lmdb_error("Failed to add removal of tx to db transaction: ", result).c_str())); + throw1(DB_ERROR(lmdb_error("Failed to add removal of prunable tx to db transaction: ", result).c_str())); + + if (tx.version > 1) + { + if ((result = mdb_cursor_get(m_cur_txs_prunable_hash, &val_tx_id, NULL, MDB_SET))) + throw1(DB_ERROR(lmdb_error("Failed to locate prunable hash tx for removal: ", result).c_str())); + result = mdb_cursor_del(m_cur_txs_prunable_hash, 0); + if (result) + throw1(DB_ERROR(lmdb_error("Failed to add removal of prunable hash tx to db transaction: ", result).c_str())); + } remove_tx_outputs(tip->data.tx_id, tx); @@ -1199,6 +1247,9 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) lmdb_db_open(txn, LMDB_BLOCK_HEIGHTS, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_heights, "Failed to open db handle for m_block_heights"); lmdb_db_open(txn, LMDB_TXS, MDB_INTEGERKEY | MDB_CREATE, m_txs, "Failed to open db handle for m_txs"); + lmdb_db_open(txn, LMDB_TXS_PRUNED, MDB_INTEGERKEY | MDB_CREATE, m_txs_pruned, "Failed to open db handle for m_txs_pruned"); + lmdb_db_open(txn, LMDB_TXS_PRUNABLE, MDB_INTEGERKEY | MDB_CREATE, m_txs_prunable, "Failed to open db handle for m_txs_prunable"); + lmdb_db_open(txn, LMDB_TXS_PRUNABLE_HASH, MDB_INTEGERKEY | MDB_CREATE, m_txs_prunable_hash, "Failed to open db handle for m_txs_prunable_hash"); lmdb_db_open(txn, LMDB_TX_INDICES, MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_tx_indices, "Failed to open db handle for m_tx_indices"); lmdb_db_open(txn, LMDB_TX_OUTPUTS, MDB_INTEGERKEY | MDB_CREATE, m_tx_outputs, "Failed to open db handle for m_tx_outputs"); @@ -1364,8 +1415,12 @@ void BlockchainLMDB::reset() throw0(DB_ERROR(lmdb_error("Failed to drop m_block_info: ", result).c_str())); if (auto result = mdb_drop(txn, m_block_heights, 0)) throw0(DB_ERROR(lmdb_error("Failed to drop m_block_heights: ", result).c_str())); - if (auto result = mdb_drop(txn, m_txs, 0)) - throw0(DB_ERROR(lmdb_error("Failed to drop m_txs: ", result).c_str())); + if (auto result = mdb_drop(txn, m_txs_pruned, 0)) + throw0(DB_ERROR(lmdb_error("Failed to drop m_txs_pruned: ", result).c_str())); + if (auto result = mdb_drop(txn, m_txs_prunable, 0)) + throw0(DB_ERROR(lmdb_error("Failed to drop m_txs_prunable: ", result).c_str())); + if (auto result = mdb_drop(txn, m_txs_prunable_hash, 0)) + throw0(DB_ERROR(lmdb_error("Failed to drop m_txs_prunable_hash: ", result).c_str())); if (auto result = mdb_drop(txn, m_tx_indices, 0)) throw0(DB_ERROR(lmdb_error("Failed to drop m_tx_indices: ", result).c_str())); if (auto result = mdb_drop(txn, m_tx_outputs, 0)) @@ -2063,7 +2118,6 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) const TXN_PREFIX_RDONLY(); RCURSOR(tx_indices); - RCURSOR(txs); MDB_val_set(key, h); bool tx_found = false; @@ -2075,8 +2129,6 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) const else if (get_result != MDB_NOTFOUND) throw0(DB_ERROR(lmdb_error(std::string("DB error attempting to fetch transaction index from hash ") + epee::string_tools::pod_to_hex(h) + ": ", get_result).c_str())); - // This isn't needed as part of the check. we're not checking consistency of db. - // get_result = mdb_cursor_get(m_cur_txs, &val_tx_index, &result, MDB_SET); TIME_MEASURE_FINISH(time1); time_tx_exists += time1; @@ -2088,11 +2140,6 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) const return false; } - // Below not needed due to above comment. - // if (get_result == MDB_NOTFOUND) - // throw0(DB_ERROR(std::string("transaction with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found at index").c_str())); - // else if (get_result) - // throw0(DB_ERROR(lmdb_error(std::string("DB error attempting to fetch transaction ") + epee::string_tools::pod_to_hex(h) + " at index: ", get_result).c_str())); return true; } @@ -2158,7 +2205,43 @@ bool BlockchainLMDB::get_tx_blob(const crypto::hash& h, cryptonote::blobdata &bd TXN_PREFIX_RDONLY(); RCURSOR(tx_indices); - RCURSOR(txs); + RCURSOR(txs_pruned); + RCURSOR(txs_prunable); + + MDB_val_set(v, h); + MDB_val result0, result1; + auto get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); + if (get_result == 0) + { + txindex *tip = (txindex *)v.mv_data; + MDB_val_set(val_tx_id, tip->data.tx_id); + get_result = mdb_cursor_get(m_cur_txs_pruned, &val_tx_id, &result0, MDB_SET); + if (get_result == 0) + { + get_result = mdb_cursor_get(m_cur_txs_prunable, &val_tx_id, &result1, MDB_SET); + } + } + if (get_result == MDB_NOTFOUND) + return false; + else if (get_result) + throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx from hash", get_result).c_str())); + + bd.assign(reinterpret_cast<char*>(result0.mv_data), result0.mv_size); + bd.append(reinterpret_cast<char*>(result1.mv_data), result1.mv_size); + + TXN_POSTFIX_RDONLY(); + + return true; +} + +bool BlockchainLMDB::get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobdata &bd) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + RCURSOR(tx_indices); + RCURSOR(txs_pruned); MDB_val_set(v, h); MDB_val result; @@ -2167,7 +2250,7 @@ bool BlockchainLMDB::get_tx_blob(const crypto::hash& h, cryptonote::blobdata &bd { txindex *tip = (txindex *)v.mv_data; MDB_val_set(val_tx_id, tip->data.tx_id); - get_result = mdb_cursor_get(m_cur_txs, &val_tx_id, &result, MDB_SET); + get_result = mdb_cursor_get(m_cur_txs_pruned, &val_tx_id, &result, MDB_SET); } if (get_result == MDB_NOTFOUND) return false; @@ -2181,6 +2264,36 @@ bool BlockchainLMDB::get_tx_blob(const crypto::hash& h, cryptonote::blobdata &bd return true; } +bool BlockchainLMDB::get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + RCURSOR(tx_indices); + RCURSOR(txs_prunable_hash); + + MDB_val_set(v, tx_hash); + MDB_val result, val_tx_prunable_hash; + auto get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); + if (get_result == 0) + { + txindex *tip = (txindex *)v.mv_data; + MDB_val_set(val_tx_id, tip->data.tx_id); + get_result = mdb_cursor_get(m_cur_txs_prunable_hash, &val_tx_id, &result, MDB_SET); + } + if (get_result == MDB_NOTFOUND) + return false; + else if (get_result) + throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx prunable hash from tx hash", get_result).c_str())); + + prunable_hash = *(const crypto::hash*)result.mv_data; + + TXN_POSTFIX_RDONLY(); + + return true; +} + uint64_t BlockchainLMDB::get_tx_count() const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -2190,8 +2303,8 @@ uint64_t BlockchainLMDB::get_tx_count() const int result; MDB_stat db_stats; - if ((result = mdb_stat(m_txn, m_txs, &db_stats))) - throw0(DB_ERROR(lmdb_error("Failed to query m_txs: ", result).c_str())); + if ((result = mdb_stat(m_txn, m_txs_pruned, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_txs_pruned: ", result).c_str())); TXN_POSTFIX_RDONLY(); @@ -2267,7 +2380,6 @@ output_data_t BlockchainLMDB::get_output_key(const uint64_t &global_index) const TXN_PREFIX_RDONLY(); RCURSOR(output_txs); RCURSOR(tx_indices); - RCURSOR(txs); output_data_t od; MDB_val_set(v, global_index); @@ -2287,7 +2399,7 @@ output_data_t BlockchainLMDB::get_output_key(const uint64_t &global_index) const txindex *tip = (txindex *)val_h.mv_data; MDB_val_set(val_tx_id, tip->data.tx_id); MDB_val result; - get_result = mdb_cursor_get(m_cur_txs, &val_tx_id, &result, MDB_SET); + get_result = mdb_cursor_get(m_cur_txs_pruned, &val_tx_id, &result, MDB_SET); if (get_result == MDB_NOTFOUND) throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(ot->tx_hash)).append(" not found in db").c_str())); else if (get_result) @@ -2297,7 +2409,7 @@ output_data_t BlockchainLMDB::get_output_key(const uint64_t &global_index) const bd.assign(reinterpret_cast<char*>(result.mv_data), result.mv_size); transaction tx; - if (!parse_and_validate_tx_from_blob(bd, tx)) + if (!parse_and_validate_tx_base_from_blob(bd, tx)) throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); const tx_out tx_output = tx.vout[ot->local_index]; @@ -2516,13 +2628,14 @@ bool BlockchainLMDB::for_blocks_range(const uint64_t& h1, const uint64_t& h2, st return fret; } -bool BlockchainLMDB::for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)> f) const +bool BlockchainLMDB::for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)> f, bool pruned) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); TXN_PREFIX_RDONLY(); - RCURSOR(txs); + RCURSOR(txs_pruned); + RCURSOR(txs_prunable); RCURSOR(tx_indices); MDB_val k; @@ -2543,16 +2656,29 @@ bool BlockchainLMDB::for_all_transactions(std::function<bool(const crypto::hash& const crypto::hash hash = ti->key; k.mv_data = (void *)&ti->data.tx_id; k.mv_size = sizeof(ti->data.tx_id); - ret = mdb_cursor_get(m_cur_txs, &k, &v, MDB_SET); + + ret = mdb_cursor_get(m_cur_txs_pruned, &k, &v, MDB_SET); if (ret == MDB_NOTFOUND) break; if (ret) throw0(DB_ERROR(lmdb_error("Failed to enumerate transactions: ", ret).c_str())); + transaction tx; blobdata bd; bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size); - transaction tx; - if (!parse_and_validate_tx_from_blob(bd, tx)) - throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); + if (pruned) + { + if (!parse_and_validate_tx_base_from_blob(bd, tx)) + throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); + } + else + { + ret = mdb_cursor_get(m_cur_txs_prunable, &k, &v, MDB_SET); + if (ret) + throw0(DB_ERROR(lmdb_error("Failed to get prunable tx data the db: ", ret).c_str())); + bd.append(reinterpret_cast<char*>(v.mv_data), v.mv_size); + if (!parse_and_validate_tx_from_blob(bd, tx)) + throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); + } if (!f(hash, tx)) { fret = false; break; @@ -3311,7 +3437,7 @@ void BlockchainLMDB::fixup() ptr = (char *)k.mv_data; \ ptr[sizeof(name)-2] = 's' -#define LOGIF(y) if (ELPP->vRegistry()->allowed(y, MONERO_DEFAULT_LOG_CATEGORY)) +#define LOGIF(y) if (ELPP->vRegistry()->allowed(y, "global")) void BlockchainLMDB::migrate_0_1() { @@ -3322,7 +3448,7 @@ void BlockchainLMDB::migrate_0_1() MDB_val k, v; char *ptr; - MLOG_YELLOW(el::Level::Info, "Migrating blockchain from DB version 0 to 1 - this may take a while:"); + MGINFO_YELLOW("Migrating blockchain from DB version 0 to 1 - this may take a while:"); MINFO("updating blocks, hf_versions, outputs, txs, and spent_keys tables..."); do { @@ -3847,11 +3973,155 @@ void BlockchainLMDB::migrate_0_1() txn.commit(); } +void BlockchainLMDB::migrate_1_2() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + uint64_t i, z; + int result; + mdb_txn_safe txn(false); + MDB_val k, v; + char *ptr; + + MGINFO_YELLOW("Migrating blockchain from DB version 1 to 2 - this may take a while:"); + MINFO("updating txs_pruned and txs_prunable tables..."); + + do { + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + + MDB_stat db_stats_txs; + MDB_stat db_stats_txs_pruned; + MDB_stat db_stats_txs_prunable; + MDB_stat db_stats_txs_prunable_hash; + if ((result = mdb_stat(txn, m_txs, &db_stats_txs))) + throw0(DB_ERROR(lmdb_error("Failed to query m_txs: ", result).c_str())); + if ((result = mdb_stat(txn, m_txs_pruned, &db_stats_txs_pruned))) + throw0(DB_ERROR(lmdb_error("Failed to query m_txs_pruned: ", result).c_str())); + if ((result = mdb_stat(txn, m_txs_prunable, &db_stats_txs_prunable))) + throw0(DB_ERROR(lmdb_error("Failed to query m_txs_prunable: ", result).c_str())); + if ((result = mdb_stat(txn, m_txs_prunable_hash, &db_stats_txs_prunable_hash))) + throw0(DB_ERROR(lmdb_error("Failed to query m_txs_prunable_hash: ", result).c_str())); + if (db_stats_txs_pruned.ms_entries != db_stats_txs_prunable.ms_entries) + throw0(DB_ERROR("Mismatched sizes for txs_pruned and txs_prunable")); + if (db_stats_txs_pruned.ms_entries == db_stats_txs.ms_entries) + { + txn.commit(); + MINFO("txs already migrated"); + break; + } + + MINFO("updating txs tables:"); + + MDB_cursor *c_old, *c_cur0, *c_cur1, *c_cur2; + i = 0; + + while(1) { + if (!(i % 1000)) { + if (i) { + result = mdb_stat(txn, m_txs, &db_stats_txs); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to query m_txs: ", result).c_str())); + LOGIF(el::Level::Info) { + std::cout << i << " / " << (i + db_stats_txs.ms_entries) << " \r" << std::flush; + } + txn.commit(); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + } + result = mdb_cursor_open(txn, m_txs_pruned, &c_cur0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_pruned: ", result).c_str())); + result = mdb_cursor_open(txn, m_txs_prunable, &c_cur1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_prunable: ", result).c_str())); + result = mdb_cursor_open(txn, m_txs_prunable_hash, &c_cur2); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_prunable_hash: ", result).c_str())); + result = mdb_cursor_open(txn, m_txs, &c_old); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs: ", result).c_str())); + if (!i) { + i = db_stats_txs_pruned.ms_entries; + } + } + MDB_val_set(k, i); + result = mdb_cursor_get(c_old, &k, &v, MDB_SET); + if (result == MDB_NOTFOUND) { + txn.commit(); + break; + } + else if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from txs: ", result).c_str())); + + cryptonote::blobdata bd; + bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size); + transaction tx; + if (!parse_and_validate_tx_from_blob(bd, tx)) + throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); + std::stringstream ss; + binary_archive<true> ba(ss); + bool r = tx.serialize_base(ba); + if (!r) + throw0(DB_ERROR("Failed to serialize pruned tx")); + std::string pruned = ss.str(); + + if (pruned.size() > bd.size()) + throw0(DB_ERROR("Pruned tx is larger than raw tx")); + if (memcmp(pruned.data(), bd.data(), pruned.size())) + throw0(DB_ERROR("Pruned tx is not a prefix of the raw tx")); + + MDB_val nv; + nv.mv_data = (void*)pruned.data(); + nv.mv_size = pruned.size(); + result = mdb_cursor_put(c_cur0, (MDB_val *)&k, &nv, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to put a record into txs_pruned: ", result).c_str())); + + nv.mv_data = (void*)(bd.data() + pruned.size()); + nv.mv_size = bd.size() - pruned.size(); + result = mdb_cursor_put(c_cur1, (MDB_val *)&k, &nv, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to put a record into txs_prunable: ", result).c_str())); + + if (tx.version > 1) + { + crypto::hash prunable_hash = get_transaction_prunable_hash(tx); + MDB_val_set(val_prunable_hash, prunable_hash); + result = mdb_cursor_put(c_cur2, (MDB_val *)&k, &val_prunable_hash, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to put a record into txs_prunable_hash: ", result).c_str())); + } + + result = mdb_cursor_del(c_old, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from txs: ", result).c_str())); + + i++; + } + } while(0); + + uint32_t version = 2; + v.mv_data = (void *)&version; + v.mv_size = sizeof(version); + MDB_val_copy<const char *> vk("version"); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + result = mdb_put(txn, m_properties, &vk, &v, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to update version for the db: ", result).c_str())); + txn.commit(); +} + void BlockchainLMDB::migrate(const uint32_t oldversion) { switch(oldversion) { case 0: migrate_0_1(); /* FALLTHRU */ + case 1: + migrate_1_2(); /* FALLTHRU */ default: ; } diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index f1773bac8..cc1b06ca0 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -50,6 +50,9 @@ typedef struct mdb_txn_cursors MDB_cursor *m_txc_output_amounts; MDB_cursor *m_txc_txs; + MDB_cursor *m_txc_txs_pruned; + MDB_cursor *m_txc_txs_prunable; + MDB_cursor *m_txc_txs_prunable_hash; MDB_cursor *m_txc_tx_indices; MDB_cursor *m_txc_tx_outputs; @@ -67,6 +70,9 @@ typedef struct mdb_txn_cursors #define m_cur_output_txs m_cursors->m_txc_output_txs #define m_cur_output_amounts m_cursors->m_txc_output_amounts #define m_cur_txs m_cursors->m_txc_txs +#define m_cur_txs_pruned m_cursors->m_txc_txs_pruned +#define m_cur_txs_prunable m_cursors->m_txc_txs_prunable +#define m_cur_txs_prunable_hash m_cursors->m_txc_txs_prunable_hash #define m_cur_tx_indices m_cursors->m_txc_tx_indices #define m_cur_tx_outputs m_cursors->m_txc_tx_outputs #define m_cur_spent_keys m_cursors->m_txc_spent_keys @@ -83,6 +89,9 @@ typedef struct mdb_rflags bool m_rf_output_txs; bool m_rf_output_amounts; bool m_rf_txs; + bool m_rf_txs_pruned; + bool m_rf_txs_prunable; + bool m_rf_txs_prunable_hash; bool m_rf_tx_indices; bool m_rf_tx_outputs; bool m_rf_spent_keys; @@ -218,6 +227,8 @@ public: virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const; virtual bool get_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const; + virtual bool get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const; + virtual bool get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const; virtual uint64_t get_tx_count() const; @@ -254,7 +265,7 @@ public: virtual bool for_all_key_images(std::function<bool(const crypto::key_image&)>) const; virtual bool for_blocks_range(const uint64_t& h1, const uint64_t& h2, std::function<bool(uint64_t, const crypto::hash&, const cryptonote::block&)>) const; - virtual bool for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)>) const; + virtual bool for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)>, bool pruned) const; virtual bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, uint64_t height, size_t tx_idx)> f) const; virtual bool for_all_outputs(uint64_t amount, const std::function<bool(uint64_t height)> &f) const; @@ -311,7 +322,7 @@ private: virtual void remove_block(); - virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash); + virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash); virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx); @@ -373,6 +384,9 @@ private: // migrate from DB version 0 to 1 void migrate_0_1(); + // migrate from DB version 1 to 2 + void migrate_1_2(); + void cleanup_batch(); private: @@ -383,6 +397,9 @@ private: MDB_dbi m_block_info; MDB_dbi m_txs; + MDB_dbi m_txs_pruned; + MDB_dbi m_txs_prunable; + MDB_dbi m_txs_prunable_hash; MDB_dbi m_tx_indices; MDB_dbi m_tx_outputs; diff --git a/src/blockchain_utilities/blockchain_usage.cpp b/src/blockchain_utilities/blockchain_usage.cpp index 7c3c83167..b78f3591b 100644 --- a/src/blockchain_utilities/blockchain_usage.cpp +++ b/src/blockchain_utilities/blockchain_usage.cpp @@ -234,7 +234,7 @@ int main(int argc, char* argv[]) } } return true; - }); + }, true); std::unordered_map<uint64_t, uint64_t> counts; size_t total = 0; diff --git a/src/cryptonote_basic/cryptonote_format_utils.cpp b/src/cryptonote_basic/cryptonote_format_utils.cpp index 3c6885896..428be1c9c 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.cpp +++ b/src/cryptonote_basic/cryptonote_format_utils.cpp @@ -779,6 +779,61 @@ namespace cryptonote return get_transaction_hash(t, res, NULL); } //--------------------------------------------------------------- + bool calculate_transaction_prunable_hash(const transaction& t, crypto::hash& res) + { + if (t.version == 1) + return false; + transaction &tt = const_cast<transaction&>(t); + std::stringstream ss; + binary_archive<true> ba(ss); + const size_t inputs = t.vin.size(); + const size_t outputs = t.vout.size(); + const size_t mixin = t.vin.empty() ? 0 : t.vin[0].type() == typeid(txin_to_key) ? boost::get<txin_to_key>(t.vin[0]).key_offsets.size() - 1 : 0; + bool r = tt.rct_signatures.p.serialize_rctsig_prunable(ba, t.rct_signatures.type, inputs, outputs, mixin); + CHECK_AND_ASSERT_MES(r, false, "Failed to serialize rct signatures prunable"); + cryptonote::get_blob_hash(ss.str(), res); + return true; + } + //--------------------------------------------------------------- + crypto::hash get_transaction_prunable_hash(const transaction& t) + { + crypto::hash res; + CHECK_AND_ASSERT_THROW_MES(calculate_transaction_prunable_hash(t, res), "Failed to calculate tx prunable hash"); + return res; + } + //--------------------------------------------------------------- + crypto::hash get_pruned_transaction_hash(const transaction& t, const crypto::hash &pruned_data_hash) + { + // v1 transactions hash the entire blob + CHECK_AND_ASSERT_THROW_MES(t.version > 1, "Hash for pruned v1 tx cannot be calculated"); + + // v2 transactions hash different parts together, than hash the set of those hashes + crypto::hash hashes[3]; + + // prefix + get_transaction_prefix_hash(t, hashes[0]); + + transaction &tt = const_cast<transaction&>(t); + + // base rct + { + std::stringstream ss; + binary_archive<true> ba(ss); + const size_t inputs = t.vin.size(); + const size_t outputs = t.vout.size(); + bool r = tt.rct_signatures.serialize_rctsig_base(ba, inputs, outputs); + CHECK_AND_ASSERT_THROW_MES(r, "Failed to serialize rct signatures base"); + cryptonote::get_blob_hash(ss.str(), hashes[1]); + } + + // prunable rct + hashes[2] = pruned_data_hash; + + // the tx hash is the hash of the 3 hashes + crypto::hash res = cn_fast_hash(hashes, sizeof(hashes)); + return res; + } + //--------------------------------------------------------------- bool calculate_transaction_hash(const transaction& t, crypto::hash& res, size_t* blob_size) { // v1 transactions hash the entire blob @@ -814,14 +869,7 @@ namespace cryptonote } else { - std::stringstream ss; - binary_archive<true> ba(ss); - const size_t inputs = t.vin.size(); - const size_t outputs = t.vout.size(); - const size_t mixin = t.vin.empty() ? 0 : t.vin[0].type() == typeid(txin_to_key) ? boost::get<txin_to_key>(t.vin[0]).key_offsets.size() - 1 : 0; - bool r = tt.rct_signatures.p.serialize_rctsig_prunable(ba, t.rct_signatures.type, inputs, outputs, mixin); - CHECK_AND_ASSERT_MES(r, false, "Failed to serialize rct signatures prunable"); - cryptonote::get_blob_hash(ss.str(), hashes[2]); + CHECK_AND_ASSERT_MES(calculate_transaction_prunable_hash(t, hashes[2]), false, "Failed to get tx prunable hash"); } // the tx hash is the hash of the 3 hashes diff --git a/src/cryptonote_basic/cryptonote_format_utils.h b/src/cryptonote_basic/cryptonote_format_utils.h index 79466e9c4..8a5296d5b 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.h +++ b/src/cryptonote_basic/cryptonote_format_utils.h @@ -100,7 +100,11 @@ namespace cryptonote bool get_transaction_hash(const transaction& t, crypto::hash& res); bool get_transaction_hash(const transaction& t, crypto::hash& res, size_t& blob_size); bool get_transaction_hash(const transaction& t, crypto::hash& res, size_t* blob_size); + bool calculate_transaction_prunable_hash(const transaction& t, crypto::hash& res); + crypto::hash get_transaction_prunable_hash(const transaction& t); bool calculate_transaction_hash(const transaction& t, crypto::hash& res, size_t* blob_size); + crypto::hash get_pruned_transaction_hash(const transaction& t, const crypto::hash &pruned_data_hash); + blobdata get_block_hashing_blob(const block& b); bool calculate_block_hash(const block& b, crypto::hash& res); bool get_block_hash(const block& b, crypto::hash& res); diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index d1ef7d6fe..339876a93 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -2092,7 +2092,7 @@ bool Blockchain::get_blocks(const t_ids_container& block_ids, t_blocks_container //TODO: return type should be void, throw on exception // alternatively, return true only if no transactions missed template<class t_ids_container, class t_tx_container, class t_missed_container> -bool Blockchain::get_transactions_blobs(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const +bool Blockchain::get_transactions_blobs(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs, bool pruned) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -2102,7 +2102,9 @@ bool Blockchain::get_transactions_blobs(const t_ids_container& txs_ids, t_tx_con try { cryptonote::blobdata tx; - if (m_db->get_tx_blob(tx_hash, tx)) + if (pruned && m_db->get_pruned_tx_blob(tx_hash, tx)) + txs.push_back(std::move(tx)); + else if (!pruned && m_db->get_tx_blob(tx_hash, tx)) txs.push_back(std::move(tx)); else missed_txs.push_back(tx_hash); @@ -2187,7 +2189,7 @@ bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qbloc // find split point between ours and foreign blockchain (or start at // blockchain height <req_start_block>), and return up to max_count FULL // blocks by reference. -bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::list<std::pair<cryptonote::blobdata, std::list<cryptonote::blobdata> > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count) const +bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::list<std::pair<cryptonote::blobdata, std::list<cryptonote::blobdata> > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, size_t max_count) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -2220,7 +2222,7 @@ bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, cons block b; CHECK_AND_ASSERT_MES(parse_and_validate_block_from_blob(blocks.back().first, b), false, "internal error, invalid block"); std::list<crypto::hash> mis; - get_transactions_blobs(b.tx_hashes, blocks.back().second, mis); + get_transactions_blobs(b.tx_hashes, blocks.back().second, mis, pruned); CHECK_AND_ASSERT_MES(!mis.size(), false, "internal error, transaction from block not found"); size += blocks.back().first.size(); for (const auto &t: blocks.back().second) @@ -4514,9 +4516,9 @@ bool Blockchain::for_blocks_range(const uint64_t& h1, const uint64_t& h2, std::f return m_db->for_blocks_range(h1, h2, f); } -bool Blockchain::for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)> f) const +bool Blockchain::for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)> f, bool pruned) const { - return m_db->for_all_transactions(f); + return m_db->for_all_transactions(f, pruned); } bool Blockchain::for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, uint64_t height, size_t tx_idx)> f) const @@ -4531,4 +4533,5 @@ bool Blockchain::for_all_outputs(uint64_t amount, std::function<bool(uint64_t he namespace cryptonote { template bool Blockchain::get_transactions(const std::vector<crypto::hash>&, std::list<transaction>&, std::list<crypto::hash>&) const; +template bool Blockchain::get_transactions_blobs(const std::vector<crypto::hash>&, std::list<cryptonote::blobdata>&, std::list<crypto::hash>&, bool) const; } diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index f58885812..cb15925ff 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -415,11 +415,12 @@ namespace cryptonote * @param blocks return-by-reference the blocks and their transactions * @param total_height return-by-reference our current blockchain height * @param start_height return-by-reference the height of the first block returned + * @param pruned whether to return full or pruned tx blobs * @param max_count the max number of blocks to get * * @return true if a block found in common or req_start_block specified, else false */ - bool find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::list<std::pair<cryptonote::blobdata, std::list<cryptonote::blobdata> > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count) const; + bool find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::list<std::pair<cryptonote::blobdata, std::list<cryptonote::blobdata> > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, size_t max_count) const; /** * @brief retrieves a set of blocks and their transactions, and possibly other transactions @@ -679,11 +680,12 @@ namespace cryptonote * @param txs_ids a container of hashes for which to get the corresponding transactions * @param txs return-by-reference a container to store result transactions in * @param missed_txs return-by-reference a container to store missed transactions in + * @param pruned whether to return full or pruned blobs * * @return false if an unexpected exception occurs, else true */ template<class t_ids_container, class t_tx_container, class t_missed_container> - bool get_transactions_blobs(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const; + bool get_transactions_blobs(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs, bool pruned = false) const; template<class t_ids_container, class t_tx_container, class t_missed_container> bool get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const; @@ -858,10 +860,11 @@ namespace cryptonote * @brief perform a check on all transactions in the blockchain * * @param std::function the check to perform, pass/fail + * @param bool pruned whether to return pruned txes only * * @return false if any transaction fails the check, otherwise true */ - bool for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)>) const; + bool for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)>, bool pruned) const; /** * @brief perform a check on all outputs in the blockchain diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 01cd56a11..c0d82d556 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -1053,9 +1053,9 @@ namespace cryptonote return m_blockchain_storage.find_blockchain_supplement(qblock_ids, resp); } //----------------------------------------------------------------------------------------------- - bool core::find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::list<std::pair<cryptonote::blobdata, std::list<cryptonote::blobdata> > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count) const + bool core::find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::list<std::pair<cryptonote::blobdata, std::list<cryptonote::blobdata> > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, size_t max_count) const { - return m_blockchain_storage.find_blockchain_supplement(req_start_block, qblock_ids, blocks, total_height, start_height, max_count); + return m_blockchain_storage.find_blockchain_supplement(req_start_block, qblock_ids, blocks, total_height, start_height, pruned, max_count); } //----------------------------------------------------------------------------------------------- bool core::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) const diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index f3b9dddc0..17b5680e5 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -517,7 +517,7 @@ namespace cryptonote * * @note see Blockchain::find_blockchain_supplement(const uint64_t, const std::list<crypto::hash>&, std::list<std::pair<cryptonote::blobdata, std::list<transaction> > >&, uint64_t&, uint64_t&, size_t) const */ - bool find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::list<std::pair<cryptonote::blobdata, std::list<cryptonote::blobdata> > >& blocks, uint64_t& total_height, uint64_t& start_height, size_t max_count) const; + bool find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::list<std::pair<cryptonote::blobdata, std::list<cryptonote::blobdata> > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, size_t max_count) const; /** * @brief gets some stats about the daemon diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index e28914d96..b5ef2557a 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -219,18 +219,6 @@ namespace cryptonote return ss.str(); } //------------------------------------------------------------------------------------------------------------------------------ - static cryptonote::blobdata get_pruned_tx_blob(const cryptonote::blobdata &blobdata) - { - cryptonote::transaction tx; - - if (!cryptonote::parse_and_validate_tx_from_blob(blobdata, tx)) - { - MERROR("Failed to parse and validate tx from blob"); - return cryptonote::blobdata(); - } - return get_pruned_tx_blob(tx); - } - //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_get_blocks(const COMMAND_RPC_GET_BLOCKS_FAST::request& req, COMMAND_RPC_GET_BLOCKS_FAST::response& res) { PERF_TIMER(on_get_blocks); @@ -240,7 +228,7 @@ namespace cryptonote std::list<std::pair<cryptonote::blobdata, std::list<cryptonote::blobdata> > > bs; - if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, bs, res.current_height, res.start_height, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT)) + if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, bs, res.current_height, res.start_height, req.prune, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT)) { res.status = "Failed"; return false; @@ -272,10 +260,7 @@ namespace cryptonote for (std::list<cryptonote::blobdata>::iterator i = bd.second.begin(); i != bd.second.end(); ++i) { unpruned_size += i->size(); - if (req.prune) - res.blocks.back().txs.push_back(get_pruned_tx_blob(std::move(*i))); - else - res.blocks.back().txs.push_back(std::move(*i)); + res.blocks.back().txs.push_back(std::move(*i)); i->clear(); i->shrink_to_fit(); pruned_size += res.blocks.back().txs.back().size(); diff --git a/src/rpc/daemon_handler.cpp b/src/rpc/daemon_handler.cpp index 29020aa57..39f169cdf 100644 --- a/src/rpc/daemon_handler.cpp +++ b/src/rpc/daemon_handler.cpp @@ -52,7 +52,7 @@ namespace rpc { std::list<std::pair<blobdata, std::list<blobdata> > > blocks; - if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, blocks, res.current_height, res.start_height, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT)) + if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, blocks, res.current_height, res.start_height, req.prune, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT)) { res.status = Message::STATUS_FAILED; res.error_details = "core::find_blockchain_supplement() returned false"; diff --git a/tests/unit_tests/hardfork.cpp b/tests/unit_tests/hardfork.cpp index c42180c47..cdbf47296 100644 --- a/tests/unit_tests/hardfork.cpp +++ b/tests/unit_tests/hardfork.cpp @@ -64,6 +64,8 @@ public: virtual blobdata get_block_blob_from_height(const uint64_t& height) const { return cryptonote::t_serializable_object_to_blob(get_block_from_height(height)); } virtual blobdata get_block_blob(const crypto::hash& h) const { return blobdata(); } virtual bool get_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const { return false; } + virtual bool get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const { return false; } + virtual bool get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const { return false; } virtual uint64_t get_block_height(const crypto::hash& h) const { return 0; } virtual block_header get_block_header(const crypto::hash& h) const { return block_header(); } virtual uint64_t get_block_timestamp(const uint64_t& height) const { return 0; } @@ -99,7 +101,7 @@ public: virtual std::vector<uint64_t> get_tx_amount_output_indices(const uint64_t tx_index) const { return std::vector<uint64_t>(); } virtual bool has_key_image(const crypto::key_image& img) const { return false; } virtual void remove_block() { blocks.pop_back(); } - virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash) {return 0;} + virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) {return 0;} virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) {} virtual uint64_t add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time, const rct::key *commitment) {return 0;} virtual void add_tx_amount_output_indices(const uint64_t tx_index, const std::vector<uint64_t>& amount_output_indices) {} @@ -108,7 +110,7 @@ public: virtual bool for_all_key_images(std::function<bool(const crypto::key_image&)>) const { return true; } virtual bool for_blocks_range(const uint64_t&, const uint64_t&, std::function<bool(uint64_t, const crypto::hash&, const cryptonote::block&)>) const { return true; } - virtual bool for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)>) const { return true; } + virtual bool for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)>, bool pruned) const { return true; } virtual bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, uint64_t height, size_t tx_idx)> f) const { return true; } virtual bool for_all_outputs(uint64_t amount, const std::function<bool(uint64_t height)> &f) const { return true; } virtual bool is_read_only() const { return false; } |