diff options
Diffstat (limited to 'src/cryptonote_core/tx_pool.cpp')
| -rw-r--r-- | src/cryptonote_core/tx_pool.cpp | 376 |
1 files changed, 330 insertions, 46 deletions
diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index a68da0e62..d86a9f5f9 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2014-2022, The Monero Project +// Copyright (c) 2014-2023, The Monero Project // // All rights reserved. // @@ -40,7 +40,6 @@ #include "blockchain.h" #include "blockchain_db/locked_txn.h" #include "blockchain_db/blockchain_db.h" -#include "common/boost_serialization_helper.h" #include "int-util.h" #include "misc_language.h" #include "warnings.h" @@ -133,6 +132,12 @@ namespace cryptonote // class code expects unsigned values throughout if (m_next_check < time_t(0)) throw std::runtime_error{"Unexpected time_t (system clock) value"}; + + m_added_txs_start_time = (time_t)0; + m_removed_txs_start_time = (time_t)0; + // We don't set these to "now" already here as we don't know how long it takes from construction + // of the pool until it "goes to work". It's safer to set when the first actual txs enter the + // corresponding lists. } //--------------------------------------------------------------------------------- bool tx_memory_pool::add_tx(transaction &tx, /*const crypto::hash& tx_prefix_hash,*/ const crypto::hash &id, const cryptonote::blobdata &blob, size_t tx_weight, tx_verification_context& tvc, relay_method tx_relay, bool relayed, uint8_t version) @@ -207,6 +212,7 @@ namespace cryptonote { tvc.m_verifivation_failed = true; tvc.m_fee_too_low = true; + tvc.m_no_drop_offense = true; return false; } @@ -219,6 +225,16 @@ namespace cryptonote return false; } + size_t tx_extra_size = tx.extra.size(); + if (!kept_by_block && tx_extra_size > MAX_TX_EXTRA_SIZE) + { + LOG_PRINT_L1("transaction tx-extra is too big: " << tx_extra_size << " bytes, the limit is: " << MAX_TX_EXTRA_SIZE); + tvc.m_verifivation_failed = true; + tvc.m_tx_extra_too_big = true; + tvc.m_no_drop_offense = true; + return false; + } + // if the transaction came from a block popped from the chain, // don't check if we have its key images as spent. // TODO: Investigate why not? @@ -281,7 +297,7 @@ namespace cryptonote return false; m_blockchain.add_txpool_tx(id, blob, meta); - m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)(tx_weight ? tx_weight : 1), receive_time), id); + add_tx_to_transient_lists(id, fee / (double)(tx_weight ? tx_weight : 1), receive_time); lock.commit(); } catch (const std::exception &e) @@ -352,7 +368,7 @@ namespace cryptonote m_blockchain.remove_txpool_tx(id); m_blockchain.add_txpool_tx(id, blob, meta); - m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)(tx_weight ? tx_weight : 1), receive_time), id); + add_tx_to_transient_lists(id, meta.fee / (double)(tx_weight ? tx_weight : 1), receive_time); } lock.commit(); } @@ -373,7 +389,7 @@ namespace cryptonote ++m_cookie; - MINFO("Transaction added to pool: txid " << id << " weight: " << tx_weight << " fee/byte: " << (fee / (double)(tx_weight ? tx_weight : 1))); + MINFO("Transaction added to pool: txid " << id << " weight: " << tx_weight << " fee/byte: " << (fee / (double)(tx_weight ? tx_weight : 1)) << ", count: " << m_added_txs_by_id.size()); prune(m_txpool_max_weight); @@ -402,6 +418,19 @@ namespace cryptonote m_txpool_max_weight = bytes; } //--------------------------------------------------------------------------------- + void tx_memory_pool::reduce_txpool_weight(size_t weight) + { + if (weight > m_txpool_weight) + { + MERROR("Underflow in txpool weight"); + m_txpool_weight = 0; + } + else + { + m_txpool_weight -= weight; + } + } + //--------------------------------------------------------------------------------- void tx_memory_pool::prune(size_t bytes) { CRITICAL_REGION_LOCAL(m_transactions_lock); @@ -423,8 +452,14 @@ namespace cryptonote txpool_tx_meta_t meta; if (!m_blockchain.get_txpool_tx_meta(txid, meta)) { - MERROR("Failed to find tx_meta in txpool"); - return; + static bool warned = false; + if (!warned) + { + MERROR("Failed to find tx_meta in txpool (will only print once)"); + warned = true; + } + --it; + continue; } // don't prune the kept_by_block ones, they're likely added because we're adding a block with those if (meta.kept_by_block) @@ -442,10 +477,11 @@ namespace cryptonote // remove first, in case this throws, so key images aren't removed MINFO("Pruning tx " << txid << " from txpool: weight: " << meta.weight << ", fee/byte: " << it->first.first); m_blockchain.remove_txpool_tx(txid); - m_txpool_weight -= meta.weight; + reduce_txpool_weight(meta.weight); remove_transaction_keyimages(tx, txid); MINFO("Pruned tx " << txid << " from txpool: weight: " << meta.weight << ", fee/byte: " << it->first.first); - m_txs_by_fee_and_receive_time.erase(it--); + remove_tx_from_transient_lists(it, txid, !meta.matches(relay_category::broadcasted)); + it--; changed = true; } catch (const std::exception &e) @@ -527,8 +563,7 @@ namespace cryptonote CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); - auto sorted_it = find_tx_in_sorted_container(id); - + bool sensitive = false; try { LockedTXN lock(m_blockchain.get_db()); @@ -559,10 +594,11 @@ namespace cryptonote do_not_relay = meta.do_not_relay; double_spend_seen = meta.double_spend_seen; pruned = meta.pruned; + sensitive = !meta.matches(relay_category::broadcasted); // remove first, in case this throws, so key images aren't removed m_blockchain.remove_txpool_tx(id); - m_txpool_weight -= tx_weight; + reduce_txpool_weight(tx_weight); remove_transaction_keyimages(tx, id); lock.commit(); } @@ -572,13 +608,12 @@ namespace cryptonote return false; } - if (sorted_it != m_txs_by_fee_and_receive_time.end()) - m_txs_by_fee_and_receive_time.erase(sorted_it); + remove_tx_from_transient_lists(find_tx_in_sorted_container(id), id, sensitive); ++m_cookie; return true; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::get_transaction_info(const crypto::hash &txid, tx_details &td) const + bool tx_memory_pool::get_transaction_info(const crypto::hash &txid, tx_details &td, bool include_sensitive_data, bool include_blob) const { PERF_TIMER(get_transaction_info); CRITICAL_REGION_LOCAL(m_transactions_lock); @@ -590,7 +625,12 @@ namespace cryptonote txpool_tx_meta_t meta; if (!m_blockchain.get_txpool_tx_meta(txid, meta)) { - MERROR("Failed to find tx in txpool"); + LOG_PRINT_L2("Failed to find tx in txpool: " << txid); + return false; + } + if (!include_sensitive_data && !meta.matches(relay_category::broadcasted)) + { + // We don't want sensitive data && the tx is sensitive, so no need to return it return false; } cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(txid, relay_category::all); @@ -616,11 +656,13 @@ namespace cryptonote td.kept_by_block = meta.kept_by_block; td.last_failed_height = meta.last_failed_height; td.last_failed_id = meta.last_failed_id; - td.receive_time = meta.receive_time; - td.last_relayed_time = meta.dandelionpp_stem ? 0 : meta.last_relayed_time; + td.receive_time = include_sensitive_data ? meta.receive_time : 0; + td.last_relayed_time = (include_sensitive_data && !meta.dandelionpp_stem) ? meta.last_relayed_time : 0; td.relayed = meta.relayed; td.do_not_relay = meta.do_not_relay; td.double_spend_seen = meta.double_spend_seen; + if (include_blob) + td.tx_blob = std::move(txblob); } catch (const std::exception &e) { @@ -630,6 +672,25 @@ namespace cryptonote return true; } + //------------------------------------------------------------------ + bool tx_memory_pool::get_transactions_info(const std::vector<crypto::hash>& txids, std::vector<std::pair<crypto::hash, tx_details>>& txs, bool include_sensitive) const + { + CRITICAL_REGION_LOCAL(m_transactions_lock); + CRITICAL_REGION_LOCAL1(m_blockchain); + + txs.clear(); + + for (const auto &it: txids) + { + tx_details details; + bool success = get_transaction_info(it, details, include_sensitive, true/*include_blob*/); + if (success) + { + txs.push_back(std::make_pair(it, std::move(details))); + } + } + return true; + } //--------------------------------------------------------------------------------- bool tx_memory_pool::get_complement(const std::vector<crypto::hash> &hashes, std::vector<cryptonote::blobdata> &txes) const { @@ -691,15 +752,7 @@ namespace cryptonote (tx_age > CRYPTONOTE_MEMPOOL_TX_FROM_ALT_BLOCK_LIVETIME && meta.kept_by_block) ) { LOG_PRINT_L1("Tx " << txid << " removed from tx pool due to outdated, age: " << tx_age ); - auto sorted_it = find_tx_in_sorted_container(txid); - if (sorted_it == m_txs_by_fee_and_receive_time.end()) - { - LOG_PRINT_L1("Removing tx " << txid << " from tx pool, but it was not found in the sorted txs container!"); - } - else - { - m_txs_by_fee_and_receive_time.erase(sorted_it); - } + remove_tx_from_transient_lists(find_tx_in_sorted_container(txid), txid, !meta.matches(relay_category::broadcasted)); m_timed_out_transactions.insert(txid); remove.push_back(std::make_pair(txid, meta.weight)); } @@ -725,7 +778,7 @@ namespace cryptonote { // remove first, so we only remove key images if the tx removal succeeds m_blockchain.remove_txpool_tx(txid); - m_txpool_weight -= entry.second; + reduce_txpool_weight(entry.second); remove_transaction_keyimages(tx, txid); } } @@ -820,8 +873,10 @@ namespace cryptonote return true; } //--------------------------------------------------------------------------------- - void tx_memory_pool::set_relayed(const epee::span<const crypto::hash> hashes, const relay_method method) + void tx_memory_pool::set_relayed(const epee::span<const crypto::hash> hashes, const relay_method method, std::vector<bool> &just_broadcasted) { + just_broadcasted.clear(); + crypto::random_poisson_seconds embargo_duration{dandelionpp_embargo_average}; const auto now = std::chrono::system_clock::now(); uint64_t next_relay = uint64_t{std::numeric_limits<time_t>::max()}; @@ -831,12 +886,14 @@ namespace cryptonote LockedTXN lock(m_blockchain.get_db()); for (const auto& hash : hashes) { + bool was_just_broadcasted = false; try { txpool_tx_meta_t meta; if (m_blockchain.get_txpool_tx_meta(hash, meta)) { // txes can be received as "stem" or "fluff" in either order + const bool already_broadcasted = meta.matches(relay_category::broadcasted); meta.upgrade_relay_method(method); meta.relayed = true; @@ -849,6 +906,12 @@ namespace cryptonote meta.last_relayed_time = std::chrono::system_clock::to_time_t(now); m_blockchain.update_txpool_tx(hash, meta); + // wait until db update succeeds to ensure tx is visible in the pool + was_just_broadcasted = !already_broadcasted && meta.matches(relay_category::broadcasted); + + if (was_just_broadcasted) + // Make sure the tx gets re-added with an updated time + add_tx_to_transient_lists(hash, meta.fee / (double)meta.weight, std::chrono::system_clock::to_time_t(now)); } } catch (const std::exception &e) @@ -856,6 +919,7 @@ namespace cryptonote MERROR("Failed to update txpool transaction metadata: " << e.what()); // continue } + just_broadcasted.emplace_back(was_just_broadcasted); } lock.commit(); set_if_less(m_next_check, time_t(next_relay)); @@ -900,6 +964,81 @@ namespace cryptonote }, false, category); } //------------------------------------------------------------------ + bool tx_memory_pool::get_pool_info(time_t start_time, bool include_sensitive, size_t max_tx_count, std::vector<std::pair<crypto::hash, tx_details>>& added_txs, std::vector<crypto::hash>& remaining_added_txids, std::vector<crypto::hash>& removed_txs, bool& incremental) const + { + CRITICAL_REGION_LOCAL(m_transactions_lock); + CRITICAL_REGION_LOCAL1(m_blockchain); + + incremental = true; + if (start_time == (time_t)0) + { + // Giving no start time means give back whole pool + incremental = false; + } + else if ((m_added_txs_start_time != (time_t)0) && (m_removed_txs_start_time != (time_t)0)) + { + if ((start_time <= m_added_txs_start_time) || (start_time <= m_removed_txs_start_time)) + { + // If either of the two lists do not go back far enough it's not possible to + // deliver incremental pool info + incremental = false; + } + // The check uses "<=": We cannot be sure to have ALL txs exactly at start_time, only AFTER that time + } + else + { + // Some incremental info still missing completely + incremental = false; + } + + added_txs.clear(); + remaining_added_txids.clear(); + removed_txs.clear(); + + std::vector<crypto::hash> txids; + if (!incremental) + { + LOG_PRINT_L2("Giving back the whole pool"); + // Give back the whole pool in 'added_txs'; because calling 'get_transaction_info' right inside the + // anonymous method somehow results in an LMDB error with transactions we have to build a list of + // ids first and get the full info afterwards + get_transaction_hashes(txids, include_sensitive); + if (txids.size() > max_tx_count) + { + remaining_added_txids = std::vector<crypto::hash>(txids.begin() + max_tx_count, txids.end()); + txids.erase(txids.begin() + max_tx_count, txids.end()); + } + get_transactions_info(txids, added_txs, include_sensitive); + return true; + } + + // Give back incrementally, based on time of entry into the map + for (const auto &pit : m_added_txs_by_id) + { + if (pit.second >= start_time) + txids.push_back(pit.first); + } + get_transactions_info(txids, added_txs, include_sensitive); + if (added_txs.size() > max_tx_count) + { + remaining_added_txids.reserve(added_txs.size() - max_tx_count); + for (size_t i = max_tx_count; i < added_txs.size(); ++i) + remaining_added_txids.push_back(added_txs[i].first); + added_txs.erase(added_txs.begin() + max_tx_count, added_txs.end()); + } + + std::multimap<time_t, removed_tx_info>::const_iterator rit = m_removed_txs_by_time.lower_bound(start_time); + while (rit != m_removed_txs_by_time.end()) + { + if (include_sensitive || !rit->second.sensitive) + { + removed_txs.push_back(rit->second.txid); + } + ++rit; + } + return true; + } + //------------------------------------------------------------------ void tx_memory_pool::get_transaction_backlog(std::vector<tx_backlog_entry>& backlog, bool include_sensitive) const { CRITICAL_REGION_LOCAL(m_transactions_lock); @@ -917,26 +1056,61 @@ namespace cryptonote { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); - const relay_category category = include_sensitive ? relay_category::all : relay_category::broadcasted; - backlog.reserve(m_blockchain.get_txpool_tx_count(include_sensitive)); - txpool_tx_meta_t tmp_meta; - m_blockchain.for_all_txpool_txes([this, &backlog, &tmp_meta](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata_ref *bd){ - transaction tx; - if (!(meta.pruned ? parse_and_validate_tx_base_from_blob(*bd, tx) : parse_and_validate_tx_from_blob(*bd, tx))) + + std::vector<tx_block_template_backlog_entry> tmp; + uint64_t total_weight = 0; + + // First get everything from the mempool, filter it later + m_blockchain.for_all_txpool_txes([&tmp, &total_weight](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata_ref*){ + tmp.emplace_back(tx_block_template_backlog_entry{txid, meta.weight, meta.fee}); + total_weight += meta.weight; + return true; + }, false, include_sensitive ? relay_category::all : relay_category::broadcasted); + + // Limit backlog to 112.5% of current median weight. This is enough to mine a full block with the optimal block reward + const uint64_t median_weight = m_blockchain.get_current_cumulative_block_weight_median(); + const uint64_t max_backlog_weight = median_weight + (median_weight / 8); + + // If the total weight is too high, choose the best paying transactions + if (total_weight > max_backlog_weight) + std::sort(tmp.begin(), tmp.end(), [](const auto& a, const auto& b){ return a.fee * b.weight > b.fee * a.weight; }); + + backlog.clear(); + uint64_t w = 0; + + std::unordered_set<crypto::key_image> k_images; + + for (const tx_block_template_backlog_entry& e : tmp) + { + try { - MERROR("Failed to parse tx from txpool"); - // continue - return true; - } - tx.set_hash(txid); + txpool_tx_meta_t meta; + if (!m_blockchain.get_txpool_tx_meta(e.id, meta)) + continue; - tmp_meta = meta; + cryptonote::blobdata txblob; + if (!m_blockchain.get_txpool_tx_blob(e.id, txblob, relay_category::all)) + continue; - if (is_transaction_ready_to_go(tmp_meta, txid, *bd, tx)) - backlog.push_back({txid, meta.weight, meta.fee}); + cryptonote::transaction tx; + if (is_transaction_ready_to_go(meta, e.id, txblob, tx)) + { + if (have_key_images(k_images, tx)) + continue; + append_key_images(k_images, tx); - return true; - }, true, category); + backlog.push_back(e); + w += e.weight; + if (w > max_backlog_weight) + break; + } + } + catch (const std::exception &e) + { + MERROR("Failed to check transaction readiness: " << e.what()); + // continue, not fatal + } + } } //------------------------------------------------------------------ void tx_memory_pool::get_transaction_stats(struct txpool_stats& stats, bool include_sensitive) const @@ -1569,6 +1743,12 @@ namespace cryptonote CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); + // Simply throw away incremental info, too difficult to update + m_added_txs_by_id.clear(); + m_added_txs_start_time = (time_t)0; + m_removed_txs_by_time.clear(); + m_removed_txs_start_time = (time_t)0; + MINFO("Validating txpool contents for v" << (unsigned)version); LockedTXN lock(m_blockchain.get_db()); @@ -1626,6 +1806,106 @@ namespace cryptonote return n_removed; } //--------------------------------------------------------------------------------- + void tx_memory_pool::add_tx_to_transient_lists(const crypto::hash& txid, double fee, time_t receive_time) + { + + time_t now = time(NULL); + const std::unordered_map<crypto::hash, time_t>::iterator it = m_added_txs_by_id.find(txid); + if (it == m_added_txs_by_id.end()) + { + m_added_txs_by_id.insert(std::make_pair(txid, now)); + } + else + { + // This tx was already added to the map earlier, probably because then it was in the "stem" + // phase of Dandelion++ and now is in the "fluff" phase i.e. got broadcasted: We have to set + // a new time for clients that are not allowed to see sensitive txs to make sure they will + // see it now if they query incrementally + it->second = now; + + auto sorted_it = find_tx_in_sorted_container(txid); + if (sorted_it == m_txs_by_fee_and_receive_time.end()) + { + MERROR("Re-adding tx " << txid << " to tx pool, but it was not found in the sorted txs container"); + } + else + { + m_txs_by_fee_and_receive_time.erase(sorted_it); + } + } + m_txs_by_fee_and_receive_time.emplace(std::pair<double, time_t>(fee, receive_time), txid); + + // Don't check for "resurrected" txs in case of reorgs i.e. don't check in 'm_removed_txs_by_time' + // whether we have that txid there and if yes remove it; this results in possible duplicates + // where we return certain txids as deleted AND in the pool at the same time which requires + // clients to process deleted ones BEFORE processing pool txs + if (m_added_txs_start_time == (time_t)0) + { + m_added_txs_start_time = now; + } + } + //--------------------------------------------------------------------------------- + void tx_memory_pool::remove_tx_from_transient_lists(const cryptonote::sorted_tx_container::iterator& sorted_it, const crypto::hash& txid, bool sensitive) + { + if (sorted_it == m_txs_by_fee_and_receive_time.end()) + { + LOG_PRINT_L1("Removing tx " << txid << " from tx pool, but it was not found in the sorted txs container!"); + } + else + { + m_txs_by_fee_and_receive_time.erase(sorted_it); + } + + const std::unordered_map<crypto::hash, time_t>::iterator it = m_added_txs_by_id.find(txid); + if (it != m_added_txs_by_id.end()) + { + m_added_txs_by_id.erase(it); + } + else + { + MDEBUG("Removing tx " << txid << " from tx pool, but it was not found in the map of added txs"); + } + track_removed_tx(txid, sensitive); + } + //--------------------------------------------------------------------------------- + void tx_memory_pool::track_removed_tx(const crypto::hash& txid, bool sensitive) + { + time_t now = time(NULL); + m_removed_txs_by_time.insert(std::make_pair(now, removed_tx_info{txid, sensitive})); + MDEBUG("Transaction removed from pool: txid " << txid << ", total entries in removed list now " << m_removed_txs_by_time.size()); + if (m_removed_txs_start_time == (time_t)0) + { + m_removed_txs_start_time = now; + } + + // Simple system to make sure the list of removed ids does not swell to an unmanageable size: Set + // an absolute size limit plus delete entries that are x minutes old (which is ok because clients + // will sync with sensible time intervalls and should not ask for incremental info e.g. 1 hour back) + const int MAX_REMOVED = 20000; + if (m_removed_txs_by_time.size() > MAX_REMOVED) + { + auto erase_it = m_removed_txs_by_time.begin(); + std::advance(erase_it, MAX_REMOVED / 4 + 1); + m_removed_txs_by_time.erase(m_removed_txs_by_time.begin(), erase_it); + m_removed_txs_start_time = m_removed_txs_by_time.begin()->first; + MDEBUG("Erased old transactions from big removed list, leaving " << m_removed_txs_by_time.size()); + } + else + { + time_t earliest = now - (30 * 60); // 30 minutes + std::map<time_t, removed_tx_info>::iterator from, to; + from = m_removed_txs_by_time.begin(); + to = m_removed_txs_by_time.lower_bound(earliest); + int distance = std::distance(from, to); + if (distance > 0) + { + m_removed_txs_by_time.erase(from, to); + m_removed_txs_start_time = earliest; + MDEBUG("Erased " << distance << " old transactions from removed list, leaving " << m_removed_txs_by_time.size()); + } + } + } + //--------------------------------------------------------------------------------- bool tx_memory_pool::init(size_t max_txpool_weight, bool mine_stem_txes) { CRITICAL_REGION_LOCAL(m_transactions_lock); @@ -1633,6 +1913,10 @@ namespace cryptonote m_txpool_max_weight = max_txpool_weight ? max_txpool_weight : DEFAULT_TXPOOL_MAX_WEIGHT; m_txs_by_fee_and_receive_time.clear(); + m_added_txs_by_id.clear(); + m_added_txs_start_time = (time_t)0; + m_removed_txs_by_time.clear(); + m_removed_txs_start_time = (time_t)0; m_spent_key_images.clear(); m_txpool_weight = 0; std::vector<crypto::hash> remove; @@ -1657,7 +1941,7 @@ namespace cryptonote MFATAL("Failed to insert key images from txpool tx"); return false; } - m_txs_by_fee_and_receive_time.emplace(std::pair<double, time_t>(meta.fee / (double)meta.weight, meta.receive_time), txid); + add_tx_to_transient_lists(txid, meta.fee / (double)meta.weight, meta.receive_time); m_txpool_weight += meta.weight; return true; }, true, relay_category::all); |