diff options
Diffstat (limited to 'src')
43 files changed, 715 insertions, 108 deletions
diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 1d7b10648..de2700ad0 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -467,7 +467,12 @@ void mdb_txn_safe::allow_new_txns() creation_gate.clear(); } -void lmdb_resized(MDB_env *env) +void mdb_txn_safe::increment_txns(int i) +{ + num_active_txns += i; +} + +void lmdb_resized(MDB_env *env, int isactive) { mdb_txn_safe::prevent_new_txns(); @@ -478,7 +483,11 @@ void lmdb_resized(MDB_env *env) mdb_env_info(env, &mei); uint64_t old = mei.me_mapsize; + if (isactive) + mdb_txn_safe::increment_txns(-1); mdb_txn_safe::wait_no_active_txns(); + if (isactive) + mdb_txn_safe::increment_txns(1); int result = mdb_env_set_mapsize(env, 0); if (result) @@ -496,7 +505,7 @@ inline int lmdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB { int res = mdb_txn_begin(env, parent, flags, txn); if (res == MDB_MAP_RESIZED) { - lmdb_resized(env); + lmdb_resized(env, 1); res = mdb_txn_begin(env, parent, flags, txn); } return res; @@ -506,7 +515,7 @@ inline int lmdb_txn_renew(MDB_txn *txn) { int res = mdb_txn_renew(txn); if (res == MDB_MAP_RESIZED) { - lmdb_resized(mdb_txn_env(txn)); + lmdb_resized(mdb_txn_env(txn), 0); res = mdb_txn_renew(txn); } return res; @@ -1267,11 +1276,11 @@ BlockchainLMDB::~BlockchainLMDB() // batch transaction shouldn't be active at this point. If it is, consider it aborted. if (m_batch_active) { - try { batch_abort(); } + try { BlockchainLMDB::batch_abort(); } catch (...) { /* ignore */ } } if (m_open) - close(); + BlockchainLMDB::close(); } BlockchainLMDB::BlockchainLMDB(bool batch_transactions): BlockchainDB() @@ -1569,9 +1578,9 @@ void BlockchainLMDB::close() if (m_batch_active) { LOG_PRINT_L3("close() first calling batch_abort() due to active batch transaction"); - batch_abort(); + BlockchainLMDB::batch_abort(); } - this->sync(); + BlockchainLMDB::sync(); m_tinfo.reset(); // FIXME: not yet thread safe!!! Use with care. @@ -1584,7 +1593,7 @@ void BlockchainLMDB::sync() LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - if (is_read_only()) + if (BlockchainLMDB::is_read_only()) return; // Does nothing unless LMDB environment was opened with MDB_NOSYNC or in part diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 0e6d70039..d87bc6e49 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -154,6 +154,7 @@ struct mdb_txn_safe static void prevent_new_txns(); static void wait_no_active_txns(); static void allow_new_txns(); + static void increment_txns(int); mdb_threadinfo* m_tinfo; MDB_txn* m_txn; diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 8e427b6b8..99d9bd8bf 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -99,7 +99,7 @@ monero_add_library(common target_link_libraries(common PUBLIC cncrypto - ${UNBOUND_LIBRARY} + ${UNBOUND_LIBRARIES} ${LIBUNWIND_LIBRARIES} ${Boost_DATE_TIME_LIBRARY} ${Boost_FILESYSTEM_LIBRARY} diff --git a/src/cryptonote_basic/cryptonote_basic.h b/src/cryptonote_basic/cryptonote_basic.h index 6394a7071..d111f6f32 100644 --- a/src/cryptonote_basic/cryptonote_basic.h +++ b/src/cryptonote_basic/cryptonote_basic.h @@ -49,6 +49,7 @@ #include "misc_language.h" #include "ringct/rctTypes.h" #include "device/device.hpp" +#include "cryptonote_basic/fwd.h" namespace cryptonote { diff --git a/src/cryptonote_basic/events.h b/src/cryptonote_basic/events.h index 6c6742215..3417ece8c 100644 --- a/src/cryptonote_basic/events.h +++ b/src/cryptonote_basic/events.h @@ -41,6 +41,8 @@ namespace cryptonote { cryptonote::transaction tx; crypto::hash hash; + uint64_t blob_size; + uint64_t weight; bool res; //!< Listeners must ignore `tx` when this is false. }; } diff --git a/src/cryptonote_basic/fwd.h b/src/cryptonote_basic/fwd.h index d54223461..901ad151b 100644 --- a/src/cryptonote_basic/fwd.h +++ b/src/cryptonote_basic/fwd.h @@ -33,4 +33,5 @@ namespace cryptonote struct block; class transaction; struct txpool_event; + struct tx_block_template_backlog_entry; } diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index f0e6794b9..33407bf95 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -1238,6 +1238,12 @@ bool Blockchain::switch_to_alternative_blockchain(std::list<block_extended_info> reorg_notify->notify("%s", std::to_string(split_height).c_str(), "%h", std::to_string(m_db->height()).c_str(), "%n", std::to_string(m_db->height() - split_height).c_str(), "%d", std::to_string(discarded_blocks).c_str(), NULL); + crypto::hash prev_id; + if (!get_block_hash(alt_chain.back().bl, prev_id)) + MERROR("Failed to get block hash of an alternative chain's tip"); + else + send_miner_notifications(prev_id, alt_chain.back().already_generated_coins); + for (const auto& notifier : m_block_notifiers) { std::size_t notify_height = split_height; @@ -1784,6 +1790,30 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m return create_block_template(b, NULL, miner_address, diffic, height, expected_reward, ex_nonce, seed_height, seed_hash); } //------------------------------------------------------------------ +bool Blockchain::get_miner_data(uint8_t& major_version, uint64_t& height, crypto::hash& prev_id, crypto::hash& seed_hash, difficulty_type& difficulty, uint64_t& median_weight, uint64_t& already_generated_coins, std::vector<tx_block_template_backlog_entry>& tx_backlog) +{ + prev_id = m_db->top_block_hash(&height); + ++height; + + major_version = m_hardfork->get_ideal_version(height); + + seed_hash = crypto::null_hash; + if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION) + { + uint64_t seed_height, next_height; + crypto::rx_seedheights(height, &seed_height, &next_height); + seed_hash = get_block_id_by_height(seed_height); + } + + difficulty = get_difficulty_for_next_block(); + median_weight = m_current_block_cumul_weight_median; + already_generated_coins = m_db->get_block_already_generated_coins(height - 1); + + m_tx_pool.get_block_template_backlog(tx_backlog); + + return true; +} +//------------------------------------------------------------------ // for an alternate chain, get the timestamps from the main chain to complete // the needed number of timestamps for the BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW. bool Blockchain::complete_timestamps_vector(uint64_t start_top_height, std::vector<uint64_t>& timestamps) const @@ -4362,6 +4392,7 @@ leave: get_difficulty_for_next_block(); // just to cache it invalidate_block_template_cache(); + send_miner_notifications(id, already_generated_coins); for (const auto& notifier: m_block_notifiers) notifier(new_height - 1, {std::addressof(bl), 1}); @@ -4990,6 +5021,8 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete unsigned nblocks = batches; if (i < extra) ++nblocks; + if (nblocks == 0) + break; tpool.submit(&waiter, boost::bind(&Blockchain::block_longhash_worker, this, thread_height, epee::span<const block>(&blocks[thread_height - height], nblocks), std::ref(maps[i])), true); thread_height += nblocks; } @@ -5270,7 +5303,7 @@ void Blockchain::set_user_options(uint64_t maxthreads, bool sync_on_blocks, uint m_max_prepare_blocks_threads = maxthreads; } -void Blockchain::add_block_notify(boost::function<void(std::uint64_t, epee::span<const block>)>&& notify) +void Blockchain::add_block_notify(BlockNotifyCallback&& notify) { if (notify) { @@ -5279,6 +5312,15 @@ void Blockchain::add_block_notify(boost::function<void(std::uint64_t, epee::span } } +void Blockchain::add_miner_notify(MinerNotifyCallback&& notify) +{ + if (notify) + { + CRITICAL_REGION_LOCAL(m_blockchain_lock); + m_miner_notifiers.push_back(std::move(notify)); + } +} + void Blockchain::safesyncmode(const bool onoff) { /* all of this is no-op'd if the user set a specific @@ -5531,6 +5573,33 @@ void Blockchain::cache_block_template(const block &b, const cryptonote::account_ m_btc_valid = true; } +void Blockchain::send_miner_notifications(const crypto::hash &prev_id, uint64_t already_generated_coins) +{ + if (m_miner_notifiers.empty()) + return; + + const uint64_t height = m_db->height(); + const uint8_t major_version = m_hardfork->get_ideal_version(height); + const difficulty_type diff = get_difficulty_for_next_block(); + const uint64_t median_weight = m_current_block_cumul_weight_median; + + crypto::hash seed_hash = crypto::null_hash; + if (m_hardfork->get_current_version() >= RX_BLOCK_VERSION) + { + uint64_t seed_height, next_height; + crypto::rx_seedheights(height, &seed_height, &next_height); + seed_hash = get_block_id_by_height(seed_height); + } + + std::vector<tx_block_template_backlog_entry> tx_backlog; + m_tx_pool.get_block_template_backlog(tx_backlog); + + for (const auto& notifier : m_miner_notifiers) + { + notifier(major_version, height, prev_id, seed_hash, diff, median_weight, already_generated_coins, tx_backlog); + } +} + namespace cryptonote { template bool Blockchain::get_transactions(const std::vector<crypto::hash>&, std::vector<transaction>&, std::vector<crypto::hash>&, bool) const; template bool Blockchain::get_split_transactions_blobs(const std::vector<crypto::hash>&, std::vector<std::tuple<crypto::hash, cryptonote::blobdata, crypto::hash, cryptonote::blobdata>>&, std::vector<crypto::hash>&) const; diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index a0e7967de..9afbfbc2d 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -90,6 +90,9 @@ namespace cryptonote */ typedef std::function<const epee::span<const unsigned char>(cryptonote::network_type network)> GetCheckpointsCallback; + typedef boost::function<void(uint64_t /* height */, epee::span<const block> /* blocks */)> BlockNotifyCallback; + typedef boost::function<void(uint8_t /* major_version */, uint64_t /* height */, const crypto::hash& /* prev_id */, const crypto::hash& /* seed_hash */, difficulty_type /* diff */, uint64_t /* median_weight */, uint64_t /* already_generated_coins */, const std::vector<tx_block_template_backlog_entry>& /* tx_backlog */)> MinerNotifyCallback; + /************************************************************************/ /* */ /************************************************************************/ @@ -371,6 +374,22 @@ namespace cryptonote bool create_block_template(block& b, const crypto::hash *from_block, const account_public_address& miner_address, difficulty_type& di, uint64_t& height, uint64_t& expected_reward, const blobdata& ex_nonce, uint64_t &seed_height, crypto::hash &seed_hash); /** + * @brief gets data required to create a block template and start mining on it + * + * @param major_version current hardfork version + * @param height current blockchain height + * @param prev_id hash of the top block + * @param seed_hash seed hash used for RandomX initialization + * @param difficulty current mining difficulty + * @param median_weight current median block weight + * @param already_generated_coins current emission + * @param tx_backlog transactions in mempool ready to be mined + * + * @return true if block template filled in successfully, else false + */ + bool get_miner_data(uint8_t& major_version, uint64_t& height, crypto::hash& prev_id, crypto::hash& seed_hash, difficulty_type& difficulty, uint64_t& median_weight, uint64_t& already_generated_coins, std::vector<tx_block_template_backlog_entry>& tx_backlog); + + /** * @brief checks if a block is known about with a given hash * * This function checks the main chain, alternate chains, and invalid blocks @@ -775,7 +794,14 @@ namespace cryptonote * * @param notify the notify object to call at every new block */ - void add_block_notify(boost::function<void(std::uint64_t, epee::span<const block>)> &¬ify); + void add_block_notify(BlockNotifyCallback&& notify); + + /** + * @brief sets a miner notify object to call for every new block + * + * @param notify the notify object to call at every new block + */ + void add_miner_notify(MinerNotifyCallback&& notify); /** * @brief sets a reorg notify object to call for every reorg @@ -1157,7 +1183,8 @@ namespace cryptonote the callable object has a single `std::shared_ptr` or `std::weap_ptr` internally. Whereas, the libstdc++ `std::function` will allocate. */ - std::vector<boost::function<void(std::uint64_t, epee::span<const block>)>> m_block_notifiers; + std::vector<BlockNotifyCallback> m_block_notifiers; + std::vector<MinerNotifyCallback> m_miner_notifiers; std::shared_ptr<tools::Notify> m_reorg_notify; // for prepare_handle_incoming_blocks @@ -1537,5 +1564,13 @@ namespace cryptonote * At some point, may be used to push an update to miners */ void cache_block_template(const block &b, const cryptonote::account_public_address &address, const blobdata &nonce, const difficulty_type &diff, uint64_t height, uint64_t expected_reward, uint64_t seed_height, const crypto::hash &seed_hash, uint64_t pool_cookie); + + /** + * @brief sends new block notifications to ZMQ `miner_data` subscribers + * + * @param prev_id hash of new blockchain tip + * @param already_generated_coins total coins mined by the network so far + */ + void send_miner_notifications(const crypto::hash &prev_id, uint64_t already_generated_coins); }; } // namespace cryptonote diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 1da14221a..4c6536318 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -388,6 +388,7 @@ namespace cryptonote m_fluffy_blocks_enabled = !get_arg(vm, arg_no_fluffy_blocks); m_offline = get_arg(vm, arg_offline); m_disable_dns_checkpoints = get_arg(vm, arg_disable_dns_checkpoints); + if (!command_line::is_arg_defaulted(vm, arg_fluffy_blocks)) MWARNING(arg_fluffy_blocks.name << " is obsolete, it is now default"); @@ -460,7 +461,7 @@ namespace cryptonote return m_blockchain_storage.get_alternative_blocks_count(); } //----------------------------------------------------------------------------------------------- - bool core::init(const boost::program_options::variables_map& vm, const cryptonote::test_options *test_options, const GetCheckpointsCallback& get_checkpoints/* = nullptr */) + bool core::init(const boost::program_options::variables_map& vm, const cryptonote::test_options *test_options, const GetCheckpointsCallback& get_checkpoints/* = nullptr */, bool allow_dns) { start_time = std::time(nullptr); @@ -471,6 +472,7 @@ namespace cryptonote } bool r = handle_command_line(vm); CHECK_AND_ASSERT_MES(r, false, "Failed to handle command line"); + m_disable_dns_checkpoints |= not allow_dns; std::string db_sync_mode = command_line::get_arg(vm, cryptonote::arg_db_sync_mode); bool db_salvage = command_line::get_arg(vm, cryptonote::arg_db_salvage) != 0; @@ -697,7 +699,7 @@ namespace cryptonote CHECK_AND_ASSERT_MES(update_checkpoints(skip_dns_checkpoints), false, "One or more checkpoints loaded from json or dns conflicted with existing checkpoints."); // DNS versions checking - if (check_updates_string == "disabled") + if (check_updates_string == "disabled" || not allow_dns) check_updates_level = UPDATES_DISABLED; else if (check_updates_string == "notify") check_updates_level = UPDATES_NOTIFY; @@ -1063,8 +1065,9 @@ namespace cryptonote if (already_have[i]) continue; - const uint64_t weight = results[i].tx.pruned ? get_pruned_transaction_weight(results[i].tx) : get_transaction_weight(results[i].tx, it->blob.size()); - ok &= add_new_tx(results[i].tx, results[i].hash, tx_blobs[i].blob, weight, tvc[i], tx_relay, relayed); + results[i].blob_size = it->blob.size(); + results[i].weight = results[i].tx.pruned ? get_pruned_transaction_weight(results[i].tx) : get_transaction_weight(results[i].tx, it->blob.size()); + ok &= add_new_tx(results[i].tx, results[i].hash, tx_blobs[i].blob, results[i].weight, tvc[i], tx_relay, relayed); if(tvc[i].m_verifivation_failed) {MERROR_VER("Transaction verification failed: " << results[i].hash);} @@ -1403,6 +1406,11 @@ namespace cryptonote return m_blockchain_storage.create_block_template(b, prev_block, adr, diffic, height, expected_reward, ex_nonce, seed_height, seed_hash); } //----------------------------------------------------------------------------------------------- + bool core::get_miner_data(uint8_t& major_version, uint64_t& height, crypto::hash& prev_id, crypto::hash& seed_hash, difficulty_type& difficulty, uint64_t& median_weight, uint64_t& already_generated_coins, std::vector<tx_block_template_backlog_entry>& tx_backlog) + { + return m_blockchain_storage.get_miner_data(major_version, height, prev_id, seed_hash, difficulty, median_weight, already_generated_coins, tx_backlog); + } + //----------------------------------------------------------------------------------------------- bool core::find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, bool clip_pruned, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const { return m_blockchain_storage.find_blockchain_supplement(qblock_ids, clip_pruned, resp); diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 8478049f9..d2bffdaee 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -237,6 +237,13 @@ namespace cryptonote virtual bool get_block_template(block& b, const crypto::hash *prev_block, const account_public_address& adr, difficulty_type& diffic, uint64_t& height, uint64_t& expected_reward, const blobdata& ex_nonce, uint64_t &seed_height, crypto::hash &seed_hash); /** + * @copydoc Blockchain::get_miner_data + * + * @note see Blockchain::get_miner_data + */ + bool get_miner_data(uint8_t& major_version, uint64_t& height, crypto::hash& prev_id, crypto::hash& seed_hash, difficulty_type& difficulty, uint64_t& median_weight, uint64_t& already_generated_coins, std::vector<tx_block_template_backlog_entry>& tx_backlog); + + /** * @brief called when a transaction is relayed. * @note Should only be invoked from `levin_notify`. */ @@ -276,10 +283,11 @@ namespace cryptonote * @param vm command line parameters * @param test_options configuration options for testing * @param get_checkpoints if set, will be called to get checkpoints data, must return checkpoints data pointer and size or nullptr if there ain't any checkpoints for specific network type + * @param allow_dns whether or not to allow DNS requests * * @return false if one of the init steps fails, otherwise true */ - bool init(const boost::program_options::variables_map& vm, const test_options *test_options = NULL, const GetCheckpointsCallback& get_checkpoints = nullptr); + bool init(const boost::program_options::variables_map& vm, const test_options *test_options = NULL, const GetCheckpointsCallback& get_checkpoints = nullptr, bool allow_dns = true); /** * @copydoc Blockchain::reset_and_set_genesis_block diff --git a/src/cryptonote_core/cryptonote_tx_utils.h b/src/cryptonote_core/cryptonote_tx_utils.h index 73cdd31cd..06412d6bf 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.h +++ b/src/cryptonote_core/cryptonote_tx_utils.h @@ -108,6 +108,15 @@ namespace cryptonote }; //--------------------------------------------------------------- + + struct tx_block_template_backlog_entry + { + crypto::hash id; + uint64_t weight; + uint64_t fee; + }; + + //--------------------------------------------------------------- crypto::public_key get_destination_view_key_pub(const std::vector<tx_destination_entry> &destinations, const boost::optional<cryptonote::account_public_address>& change_addr); bool construct_tx(const account_keys& sender_account_keys, std::vector<tx_source_entry> &sources, const std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time); bool construct_tx_with_tx_key(const account_keys& sender_account_keys, const std::unordered_map<crypto::public_key, subaddress_index>& subaddresses, std::vector<tx_source_entry>& sources, std::vector<tx_destination_entry>& destinations, const boost::optional<cryptonote::account_public_address>& change_addr, const std::vector<uint8_t> &extra, transaction& tx, uint64_t unlock_time, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys, bool rct = false, const rct::RCTConfig &rct_config = { rct::RangeProofBorromean, 0 }, rct::multisig_out *msout = NULL, bool shuffle_outs = true); diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index a7e96e23a..84605d6f5 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -913,6 +913,32 @@ namespace cryptonote }, false, category); } //------------------------------------------------------------------ + void tx_memory_pool::get_block_template_backlog(std::vector<tx_block_template_backlog_entry>& backlog, bool include_sensitive) const + { + CRITICAL_REGION_LOCAL(m_transactions_lock); + CRITICAL_REGION_LOCAL1(m_blockchain); + const relay_category category = include_sensitive ? relay_category::all : relay_category::broadcasted; + backlog.reserve(m_blockchain.get_txpool_tx_count(include_sensitive)); + txpool_tx_meta_t tmp_meta; + m_blockchain.for_all_txpool_txes([this, &backlog, &tmp_meta](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata_ref *bd){ + transaction tx; + if (!(meta.pruned ? parse_and_validate_tx_base_from_blob(*bd, tx) : parse_and_validate_tx_from_blob(*bd, tx))) + { + MERROR("Failed to parse tx from txpool"); + // continue + return true; + } + tx.set_hash(txid); + + tmp_meta = meta; + + if (is_transaction_ready_to_go(tmp_meta, txid, *bd, tx)) + backlog.push_back({txid, meta.weight, meta.fee}); + + return true; + }, true, category); + } + //------------------------------------------------------------------ void tx_memory_pool::get_transaction_stats(struct txpool_stats& stats, bool include_sensitive) const { CRITICAL_REGION_LOCAL(m_transactions_lock); @@ -1222,11 +1248,11 @@ namespace cryptonote return ret; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::is_transaction_ready_to_go(txpool_tx_meta_t& txd, const crypto::hash &txid, const cryptonote::blobdata &txblob, transaction &tx) const + bool tx_memory_pool::is_transaction_ready_to_go(txpool_tx_meta_t& txd, const crypto::hash &txid, const cryptonote::blobdata_ref& txblob, transaction &tx) const { - struct transction_parser + struct transaction_parser { - transction_parser(const cryptonote::blobdata &txblob, const crypto::hash &txid, transaction &tx): txblob(txblob), txid(txid), tx(tx), parsed(false) {} + transaction_parser(const cryptonote::blobdata_ref &txblob, const crypto::hash &txid, transaction &tx): txblob(txblob), txid(txid), tx(tx), parsed(false) {} cryptonote::transaction &operator()() { if (!parsed) @@ -1238,7 +1264,7 @@ namespace cryptonote } return tx; } - const cryptonote::blobdata &txblob; + const cryptonote::blobdata_ref &txblob; const crypto::hash &txid; transaction &tx; bool parsed; @@ -1289,6 +1315,11 @@ namespace cryptonote return true; } //--------------------------------------------------------------------------------- + bool tx_memory_pool::is_transaction_ready_to_go(txpool_tx_meta_t& txd, const crypto::hash &txid, const cryptonote::blobdata& txblob, transaction &tx) const + { + return is_transaction_ready_to_go(txd, txid, cryptonote::blobdata_ref{txblob.data(), txblob.size()}, tx); + } + //--------------------------------------------------------------------------------- bool tx_memory_pool::have_key_images(const std::unordered_set<crypto::key_image>& k_images, const transaction_prefix& tx) { for(size_t i = 0; i!= tx.vin.size(); i++) diff --git a/src/cryptonote_core/tx_pool.h b/src/cryptonote_core/tx_pool.h index ab2a57ea2..80b38c51d 100644 --- a/src/cryptonote_core/tx_pool.h +++ b/src/cryptonote_core/tx_pool.h @@ -266,6 +266,15 @@ namespace cryptonote void get_transaction_backlog(std::vector<tx_backlog_entry>& backlog, bool include_sensitive = false) const; /** + * @brief get (hash, weight, fee) for all transactions in the pool - the minimum required information to create a block template + * + * @param backlog return-by-reference that data + * @param include_sensitive return stempool, anonymity-pool, and unrelayed txes + * + */ + void get_block_template_backlog(std::vector<tx_block_template_backlog_entry>& backlog, bool include_sensitive = false) const; + + /** * @brief get a summary statistics of all transaction hashes in the pool * * @param stats return-by-reference the pool statistics @@ -540,6 +549,7 @@ namespace cryptonote * * @return true if the transaction is good to go, otherwise false */ + bool is_transaction_ready_to_go(txpool_tx_meta_t& txd, const crypto::hash &txid, const cryptonote::blobdata_ref &txblob, transaction&tx) const; bool is_transaction_ready_to_go(txpool_tx_meta_t& txd, const crypto::hash &txid, const cryptonote::blobdata &txblob, transaction&tx) const; /** diff --git a/src/cryptonote_protocol/levin_notify.cpp b/src/cryptonote_protocol/levin_notify.cpp index 0b065c3c3..53de407b6 100644 --- a/src/cryptonote_protocol/levin_notify.cpp +++ b/src/cryptonote_protocol/levin_notify.cpp @@ -287,6 +287,12 @@ namespace levin boost::asio::steady_timer next_epoch; boost::asio::steady_timer flush_txs; boost::asio::io_service::strand strand; + struct context_t { + std::vector<cryptonote::blobdata> fluff_txs; + std::chrono::steady_clock::time_point flush_time; + bool m_is_income; + }; + boost::unordered_map<boost::uuids::uuid, context_t> contexts; net::dandelionpp::connection_map map;//!< Tracks outgoing uuid's for noise channels or Dandelion++ stems std::deque<noise_channel> channels; //!< Never touch after init; only update elements on `noise_channel.strand` std::atomic<std::size_t> connection_count; //!< Only update in strand, can be read at any time @@ -363,14 +369,16 @@ namespace levin const auto now = std::chrono::steady_clock::now(); auto next_flush = std::chrono::steady_clock::time_point::max(); std::vector<std::pair<std::vector<blobdata>, boost::uuids::uuid>> connections{}; - zone_->p2p->foreach_connection([timer_error, now, &next_flush, &connections] (detail::p2p_context& context) + for (auto &e: zone_->contexts) { + auto &id = e.first; + auto &context = e.second; if (!context.fluff_txs.empty()) { if (context.flush_time <= now || timer_error) // flush on canceled timer { context.flush_time = std::chrono::steady_clock::time_point::max(); - connections.emplace_back(std::move(context.fluff_txs), context.m_connection_id); + connections.emplace_back(std::move(context.fluff_txs), id); context.fluff_txs.clear(); } else // not flushing yet @@ -378,8 +386,7 @@ namespace levin } else // nothing to flush context.flush_time = std::chrono::steady_clock::time_point::max(); - return true; - }); + } /* Always send with `fluff` flag, even over i2p/tor. The hidden service will disable the forwarding delay and immediately fluff. The i2p/tor @@ -427,22 +434,21 @@ namespace levin MDEBUG("Queueing " << txs.size() << " transaction(s) for Dandelion++ fluffing"); - - zone->p2p->foreach_connection([txs, now, &zone, &source, &in_duration, &out_duration, &next_flush] (detail::p2p_context& context) + for (auto &e: zone->contexts) { + auto &id = e.first; + auto &context = e.second; // When i2p/tor, only fluff to outbound connections - if (context.handshake_complete() && source != context.m_connection_id && (zone->nzone == epee::net_utils::zone::public_ || !context.m_is_income)) + if (source != id && (zone->nzone == epee::net_utils::zone::public_ || !context.m_is_income)) { if (context.fluff_txs.empty()) context.flush_time = now + (context.m_is_income ? in_duration() : out_duration()); next_flush = std::min(next_flush, context.flush_time); context.fluff_txs.reserve(context.fluff_txs.size() + txs.size()); - for (const blobdata& tx : txs) - context.fluff_txs.push_back(tx); // must copy instead of move (multiple conns) + context.fluff_txs.insert(context.fluff_txs.end(), txs.begin(), txs.end()); } - return true; - }); + } if (next_flush == std::chrono::steady_clock::time_point::max()) MWARNING("Unable to send transaction(s), no available connections"); @@ -749,6 +755,32 @@ namespace levin ); } + void notify::on_handshake_complete(const boost::uuids::uuid &id, bool is_income) + { + if (!zone_) + return; + + auto& zone = zone_; + zone_->strand.dispatch([zone, id, is_income]{ + zone->contexts[id] = { + .fluff_txs = {}, + .flush_time = std::chrono::steady_clock::time_point::max(), + .m_is_income = is_income, + }; + }); + } + + void notify::on_connection_close(const boost::uuids::uuid &id) + { + if (!zone_) + return; + + auto& zone = zone_; + zone_->strand.dispatch([zone, id]{ + zone->contexts.erase(id); + }); + } + void notify::run_epoch() { if (!zone_) diff --git a/src/cryptonote_protocol/levin_notify.h b/src/cryptonote_protocol/levin_notify.h index abbf9d461..12704746a 100644 --- a/src/cryptonote_protocol/levin_notify.h +++ b/src/cryptonote_protocol/levin_notify.h @@ -101,6 +101,9 @@ namespace levin //! Probe for new outbound connection - skips if not needed. void new_out_connection(); + void on_handshake_complete(const boost::uuids::uuid &id, bool is_income); + void on_connection_close(const boost::uuids::uuid &id); + //! Run the logic for the next epoch immediately. Only use in testing. void run_epoch(); diff --git a/src/daemon/command_line_args.h b/src/daemon/command_line_args.h index 6c3e163e6..a988fe25f 100644 --- a/src/daemon/command_line_args.h +++ b/src/daemon/command_line_args.h @@ -96,6 +96,16 @@ namespace daemon_args , 0 }; + const command_line::arg_descriptor<std::string> arg_proxy = { + "proxy", + "Network communication through proxy: <socks-ip:port> i.e. \"127.0.0.1:9050\"", + "", + }; + const command_line::arg_descriptor<bool> arg_proxy_allow_dns_leaks = { + "proxy-allow-dns-leaks", + "Allow DNS leaks outside of proxy", + false, + }; const command_line::arg_descriptor<bool> arg_public_node = { "public-node" , "Allow other users to use the node as a remote (restricted RPC mode, view-only commands) and advertise it over P2P" diff --git a/src/daemon/core.h b/src/daemon/core.h index 804d7474d..0811cf420 100644 --- a/src/daemon/core.h +++ b/src/daemon/core.h @@ -32,6 +32,7 @@ #include "cryptonote_core/cryptonote_core.h" #include "cryptonote_protocol/cryptonote_protocol_handler.h" #include "misc_log_ex.h" +#include "daemon/command_line_args.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "daemon" @@ -66,7 +67,14 @@ public: #else const cryptonote::GetCheckpointsCallback& get_checkpoints = nullptr; #endif - if (!m_core.init(m_vm_HACK, nullptr, get_checkpoints)) + + if (command_line::is_arg_defaulted(vm, daemon_args::arg_proxy) && command_line::get_arg(vm, daemon_args::arg_proxy_allow_dns_leaks)) { + MLOG_RED(el::Level::Warning, "--" << daemon_args::arg_proxy_allow_dns_leaks.name << " is enabled, but --" + << daemon_args::arg_proxy.name << " is not specified."); + } + + const bool allow_dns = command_line::is_arg_defaulted(vm, daemon_args::arg_proxy) || command_line::get_arg(vm, daemon_args::arg_proxy_allow_dns_leaks); + if (!m_core.init(m_vm_HACK, nullptr, get_checkpoints, allow_dns)) { throw std::runtime_error("Failed to initialize core"); } diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 99430b2b0..3f1885423 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -120,6 +120,7 @@ public: if (shared) { core.get().get_blockchain_storage().add_block_notify(cryptonote::listener::zmq_pub::chain_main{shared}); + core.get().get_blockchain_storage().add_miner_notify(cryptonote::listener::zmq_pub::miner_data{shared}); core.get().set_txpool_listener(cryptonote::listener::zmq_pub::txpool_add{shared}); } } diff --git a/src/daemon/main.cpp b/src/daemon/main.cpp index d413906df..70aec5538 100644 --- a/src/daemon/main.cpp +++ b/src/daemon/main.cpp @@ -152,6 +152,8 @@ int main(int argc, char const * argv[]) command_line::add_arg(core_settings, daemon_args::arg_max_log_file_size); command_line::add_arg(core_settings, daemon_args::arg_max_log_files); command_line::add_arg(core_settings, daemon_args::arg_max_concurrency); + command_line::add_arg(core_settings, daemon_args::arg_proxy); + command_line::add_arg(core_settings, daemon_args::arg_proxy_allow_dns_leaks); command_line::add_arg(core_settings, daemon_args::arg_public_node); command_line::add_arg(core_settings, daemon_args::arg_zmq_rpc_bind_ip); command_line::add_arg(core_settings, daemon_args::arg_zmq_rpc_bind_port); diff --git a/src/daemon/p2p.h b/src/daemon/p2p.h index f68efccc2..38862c017 100644 --- a/src/daemon/p2p.h +++ b/src/daemon/p2p.h @@ -33,6 +33,7 @@ #include "cryptonote_protocol/cryptonote_protocol_handler.h" #include "p2p/net_node.h" #include "daemon/protocol.h" +#include "daemon/command_line_args.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "daemon" @@ -61,7 +62,7 @@ public: { //initialize objects MGINFO("Initializing p2p server..."); - if (!m_server.init(vm)) + if (!m_server.init(vm, command_line::get_arg(vm, daemon_args::arg_proxy), command_line::get_arg(vm, daemon_args::arg_proxy_allow_dns_leaks))) { throw std::runtime_error("Failed to initialize p2p server."); } diff --git a/src/daemon/rpc.h b/src/daemon/rpc.h index af48bcc45..bff7dc449 100644 --- a/src/daemon/rpc.h +++ b/src/daemon/rpc.h @@ -62,7 +62,7 @@ public: { MGINFO("Initializing " << m_description << " RPC server..."); - if (!m_server.init(vm, restricted, port, allow_rpc_payment)) + if (!m_server.init(vm, restricted, port, allow_rpc_payment, command_line::get_arg(vm, daemon_args::arg_proxy))) { throw std::runtime_error("Failed to initialize " + m_description + " RPC server."); } diff --git a/src/device/device.hpp b/src/device/device.hpp index 582eb2242..6005e157d 100644 --- a/src/device/device.hpp +++ b/src/device/device.hpp @@ -91,7 +91,6 @@ namespace hw { public: device(): mode(NONE) {} - device(const device &hwdev) {} virtual ~device() {} explicit virtual operator bool() const = 0; diff --git a/src/device/device_cold.hpp b/src/device/device_cold.hpp index d435b448c..07009b9d2 100644 --- a/src/device/device_cold.hpp +++ b/src/device/device_cold.hpp @@ -162,6 +162,26 @@ namespace hw { * Live refresh process termination */ virtual void live_refresh_finish() =0; + + /** + * Requests public address, uses empty passphrase if asked for. + */ + virtual bool get_public_address_with_no_passphrase(cryptonote::account_public_address &pubkey) =0; + + /** + * Reset session ID, restart with a new session. + */ + virtual void reset_session() =0; + + /** + * Returns true if device already asked for passphrase entry before (i.e., obviously supports passphrase entry) + */ + virtual bool seen_passphrase_entry_prompt() =0; + + /** + * Uses empty passphrase for all passphrase queries. + */ + virtual void set_use_empty_passphrase(bool always_use_empty_passphrase) =0; }; } diff --git a/src/device_trezor/device_trezor.cpp b/src/device_trezor/device_trezor.cpp index c2070b0d1..03e8bbba4 100644 --- a/src/device_trezor/device_trezor.cpp +++ b/src/device_trezor/device_trezor.cpp @@ -178,6 +178,15 @@ namespace trezor { } } + bool device_trezor::get_public_address_with_no_passphrase(cryptonote::account_public_address &pubkey) { + m_reply_with_empty_passphrase = true; + const auto empty_passphrase_reverter = epee::misc_utils::create_scope_leave_handler([&]() { + m_reply_with_empty_passphrase = false; + }); + + return get_public_address(pubkey); + } + bool device_trezor::get_secret_keys(crypto::secret_key &viewkey , crypto::secret_key &spendkey) { try { MDEBUG("Loading view-only key from the Trezor. Please check the Trezor for a confirmation."); @@ -206,6 +215,18 @@ namespace trezor { get_address(index, payment_id, true); } + void device_trezor::reset_session() { + m_device_session_id.clear(); + } + + bool device_trezor::seen_passphrase_entry_prompt() { + return m_seen_passphrase_entry_message; + } + + void device_trezor::set_use_empty_passphrase(bool always_use_empty_passphrase) { + m_always_use_empty_passphrase = always_use_empty_passphrase; + } + /* ======================================================================= */ /* Helpers */ /* ======================================================================= */ diff --git a/src/device_trezor/device_trezor.hpp b/src/device_trezor/device_trezor.hpp index d91d1de3f..15337d2b4 100644 --- a/src/device_trezor/device_trezor.hpp +++ b/src/device_trezor/device_trezor.hpp @@ -205,6 +205,26 @@ namespace trezor { const ::tools::wallet2::unsigned_tx_set & unsigned_tx, ::tools::wallet2::signed_tx_set & signed_tx, hw::tx_aux_data & aux_data) override; + + /** + * Requests public address, uses empty passphrase if asked for. + */ + bool get_public_address_with_no_passphrase(cryptonote::account_public_address &pubkey) override; + + /** + * Reset session ID, restart with a new session. + */ + virtual void reset_session() override; + + /** + * Returns true if device already asked for passphrase entry before (i.e., obviously supports passphrase entry) + */ + bool seen_passphrase_entry_prompt() override; + + /** + * Uses empty passphrase for all passphrase queries. + */ + void set_use_empty_passphrase(bool use_always_empty_passphrase) override; }; #endif diff --git a/src/device_trezor/device_trezor_base.cpp b/src/device_trezor/device_trezor_base.cpp index b0b4342f5..016eb2816 100644 --- a/src/device_trezor/device_trezor_base.cpp +++ b/src/device_trezor/device_trezor_base.cpp @@ -45,7 +45,10 @@ namespace trezor { const uint32_t device_trezor_base::DEFAULT_BIP44_PATH[] = {0x8000002c, 0x80000080}; - device_trezor_base::device_trezor_base(): m_callback(nullptr), m_last_msg_type(messages::MessageType_Success) { + device_trezor_base::device_trezor_base(): m_callback(nullptr), m_last_msg_type(messages::MessageType_Success), + m_reply_with_empty_passphrase(false), + m_always_use_empty_passphrase(false), + m_seen_passphrase_entry_message(false) { #ifdef WITH_TREZOR_DEBUGGING m_debug = false; #endif @@ -155,6 +158,9 @@ namespace trezor { TREZOR_AUTO_LOCK_DEVICE(); m_device_session_id.clear(); m_features.reset(); + m_seen_passphrase_entry_message = false; + m_reply_with_empty_passphrase = false; + m_always_use_empty_passphrase = false; if (m_transport){ try { @@ -476,6 +482,7 @@ namespace trezor { return; } + m_seen_passphrase_entry_message = true; bool on_device = true; if (msg->has__on_device() && !msg->_on_device()){ on_device = false; // do not enter on device, old devices. @@ -491,19 +498,21 @@ namespace trezor { } boost::optional<epee::wipeable_string> passphrase; - TREZOR_CALLBACK_GET(passphrase, on_passphrase_request, on_device); + if (m_reply_with_empty_passphrase || m_always_use_empty_passphrase) { + MDEBUG("Answering passphrase prompt with an empty passphrase, always use empty: " << m_always_use_empty_passphrase); + on_device = false; + passphrase = epee::wipeable_string(""); + } else if (m_passphrase){ + MWARNING("Answering passphrase prompt with a stored passphrase (do not use; passphrase can be seen by a potential malware / attacker)"); + on_device = false; + passphrase = epee::wipeable_string(m_passphrase.get()); + } else { + TREZOR_CALLBACK_GET(passphrase, on_passphrase_request, on_device); + } messages::common::PassphraseAck m; m.set_on_device(on_device); if (!on_device) { - if (!passphrase && m_passphrase) { - passphrase = m_passphrase; - } - - if (m_passphrase) { - m_passphrase = boost::none; - } - if (passphrase) { m.set_allocated_passphrase(new std::string(passphrase->data(), passphrase->size())); } diff --git a/src/device_trezor/device_trezor_base.hpp b/src/device_trezor/device_trezor_base.hpp index 0162b23df..de49397d5 100644 --- a/src/device_trezor/device_trezor_base.hpp +++ b/src/device_trezor/device_trezor_base.hpp @@ -101,6 +101,9 @@ namespace trezor { messages::MessageType m_last_msg_type; cryptonote::network_type network_type; + bool m_reply_with_empty_passphrase; + bool m_always_use_empty_passphrase; + bool m_seen_passphrase_entry_message; #ifdef WITH_TREZOR_DEBUGGING std::shared_ptr<trezor_debug_callback> m_debug_callback; diff --git a/src/net/socks.h b/src/net/socks.h index 739c972ab..506b53195 100644 --- a/src/net/socks.h +++ b/src/net/socks.h @@ -201,6 +201,13 @@ namespace socks std::shared_ptr<client> self_; void operator()(boost::system::error_code error = boost::system::error_code{}); }; + + //! Calls `async_close` on `self` at destruction. NOP if `nullptr`. + struct close_on_exit + { + std::shared_ptr<client> self; + ~close_on_exit() { async_close{std::move(self)}(); } + }; }; template<typename Handler> diff --git a/src/p2p/net_node.cpp b/src/p2p/net_node.cpp index 84cc1581e..d9050200a 100644 --- a/src/p2p/net_node.cpp +++ b/src/p2p/net_node.cpp @@ -94,6 +94,9 @@ namespace case net::i2p_address::get_type_id(): set = client->set_connect_command(remote.as<net::i2p_address>()); break; + case epee::net_utils::ipv4_network_address::get_type_id(): + set = client->set_connect_command(remote.as<epee::net_utils::ipv4_network_address>()); + break; default: MERROR("Unsupported network address in socks_connect"); return false; @@ -339,6 +342,7 @@ namespace nodetool } }; + net::socks::client::close_on_exit close_client{}; boost::unique_future<client_result> socks_result{}; { boost::promise<client_result> socks_promise{}; @@ -347,6 +351,7 @@ namespace nodetool auto client = net::socks::make_connect_client( boost::asio::ip::tcp::socket{service}, net::socks::version::v4a, notify{std::move(socks_promise)} ); + close_client.self = client; if (!start_socks(std::move(client), proxy, remote)) return boost::none; } @@ -368,7 +373,10 @@ namespace nodetool { auto result = socks_result.get(); if (!result.first) + { + close_client.self.reset(); return {std::move(result.second)}; + } MERROR("Failed to make socks connection to " << remote.str() << " (via " << proxy << "): " << result.first.message()); } diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index f2888674b..ac815a100 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -111,15 +111,11 @@ namespace nodetool struct p2p_connection_context_t: base_type //t_payload_net_handler::connection_context //public net_utils::connection_context_base { p2p_connection_context_t() - : fluff_txs(), - flush_time(std::chrono::steady_clock::time_point::max()), - peer_id(0), + : peer_id(0), support_flags(0), m_in_timedsync(false) {} - std::vector<cryptonote::blobdata> fluff_txs; - std::chrono::steady_clock::time_point flush_time; peerid_type peer_id; uint32_t support_flags; bool m_in_timedsync; @@ -259,6 +255,7 @@ namespace nodetool m_offline(false), is_closing(false), m_network_id(), + m_enable_dns_seed_nodes(true), max_connections(1) {} virtual ~node_server(); @@ -267,7 +264,7 @@ namespace nodetool bool run(); network_zone& add_zone(epee::net_utils::zone zone); - bool init(const boost::program_options::variables_map& vm); + bool init(const boost::program_options::variables_map& vm, const std::string& proxy = {}, bool proxy_dns_leaks_allowed = {}); bool deinit(); bool send_stop_signal(); uint32_t get_this_peer_port(){return m_listening_port;} @@ -516,6 +513,7 @@ namespace nodetool epee::net_utils::ssl_support_t m_ssl_support; + bool m_enable_dns_seed_nodes; bool m_enable_dns_blocklist; uint32_t max_connections; diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index cfeac3d37..d4b39869c 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -741,6 +741,12 @@ namespace nodetool { return get_ip_seed_nodes(); } + if (!m_enable_dns_seed_nodes) + { + // TODO: a domain can be set through socks, so that the remote side does the lookup for the DNS seed nodes. + m_fallback_seed_nodes_added.test_and_set(); + return get_ip_seed_nodes(); + } std::set<std::string> full_addrs; @@ -880,10 +886,21 @@ namespace nodetool } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::init(const boost::program_options::variables_map& vm) + bool node_server<t_payload_net_handler>::init(const boost::program_options::variables_map& vm, const std::string& proxy, bool proxy_dns_leaks_allowed) { bool res = handle_command_line(vm); CHECK_AND_ASSERT_MES(res, false, "Failed to handle command line"); + if (proxy.size()) + { + const auto endpoint = net::get_tcp_endpoint(proxy); + CHECK_AND_ASSERT_MES(endpoint, false, "Failed to parse proxy: " << proxy << " - " << endpoint.error()); + network_zone& public_zone = m_network_zones[epee::net_utils::zone::public_]; + public_zone.m_connect = &socks_connect; + public_zone.m_proxy_address = *endpoint; + public_zone.m_can_pingback = false; + m_enable_dns_seed_nodes &= proxy_dns_leaks_allowed; + m_enable_dns_blocklist &= proxy_dns_leaks_allowed; + } if (m_nettype == cryptonote::TESTNET) { @@ -1412,6 +1429,7 @@ namespace nodetool ape.first_seen = first_seen_stamp ? first_seen_stamp : time(nullptr); zone.m_peerlist.append_with_peer_anchor(ape); + zone.m_notifier.on_handshake_complete(con->m_connection_id, con->m_is_income); zone.m_notifier.new_out_connection(); LOG_DEBUG_CC(*con, "CONNECTION HANDSHAKED OK."); @@ -2526,6 +2544,8 @@ namespace nodetool return 1; } + zone.m_notifier.on_handshake_complete(context.m_connection_id, context.m_is_income); + if(has_too_many_connections(context.m_remote_address)) { LOG_PRINT_CCONTEXT_L1("CONNECTION FROM " << context.m_remote_address.host_str() << " REFUSED, too many connections from the same address"); @@ -2652,6 +2672,9 @@ namespace nodetool zone.m_peerlist.remove_from_peer_anchor(na); } + if (!zone.m_net_server.is_stop_signal_sent()) { + zone.m_notifier.on_connection_close(context.m_connection_id); + } m_payload_handler.on_connection_close(context); MINFO("["<< epee::net_utils::print_connection_context(context) << "] CLOSE CONNECTION"); diff --git a/src/ringct/multiexp.cc b/src/ringct/multiexp.cc index 620f7d0dd..784c90a4e 100644 --- a/src/ringct/multiexp.cc +++ b/src/ringct/multiexp.cc @@ -449,7 +449,6 @@ rct::key straus(const std::vector<MultiexpData> &data, const std::shared_ptr<str STEP = STEP ? STEP : 192; MULTIEXP_PERF(PERF_TIMER_START_UNIT(setup, 1000000)); - static constexpr unsigned int mask = (1<<STRAUS_C)-1; std::shared_ptr<straus_cached_data> local_cache = cache == NULL ? straus_init_cache(data) : cache; ge_cached cached; ge_p1p1 p1; @@ -483,6 +482,7 @@ rct::key straus(const std::vector<MultiexpData> &data, const std::shared_ptr<str memcpy(bytes33, data[j].scalar.bytes, 32); bytes33[32] = 0; bytes = bytes33; + static constexpr unsigned int mask = (1<<STRAUS_C)-1; for (size_t i = 0; i < 256; ++i) digits[j*256+i] = ((bytes[i>>3] | (bytes[(i>>3)+1]<<8)) >> (i&7)) & mask; #else diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 8d8a68efb..da36f3c64 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -242,11 +242,11 @@ namespace cryptonote auto get_nodes = [this]() { return get_public_nodes(credits_per_hash_threshold); }; - m_bootstrap_daemon.reset(new bootstrap_daemon(std::move(get_nodes), rpc_payment_enabled, proxy)); + m_bootstrap_daemon.reset(new bootstrap_daemon(std::move(get_nodes), rpc_payment_enabled, m_bootstrap_daemon_proxy.empty() ? proxy : m_bootstrap_daemon_proxy)); } else { - m_bootstrap_daemon.reset(new bootstrap_daemon(address, credentials, rpc_payment_enabled, proxy)); + m_bootstrap_daemon.reset(new bootstrap_daemon(address, credentials, rpc_payment_enabled, m_bootstrap_daemon_proxy.empty() ? proxy : m_bootstrap_daemon_proxy)); } m_should_use_bootstrap_daemon = m_bootstrap_daemon.get() != nullptr; @@ -264,8 +264,10 @@ namespace cryptonote , const bool restricted , const std::string& port , bool allow_rpc_payment + , const std::string& proxy ) { + m_bootstrap_daemon_proxy = proxy; m_restricted = restricted; m_net_server.set_threads_prefix("RPC"); m_net_server.set_connection_filter(&m_p2p); @@ -970,14 +972,26 @@ namespace cryptonote LOG_PRINT_L2("Found " << found_in_pool << "/" << vh.size() << " transactions in the pool"); } - std::vector<std::string>::const_iterator txhi = req.txs_hashes.begin(); - std::vector<crypto::hash>::const_iterator vhi = vh.begin(); + CHECK_AND_ASSERT_MES(txs.size() + missed_txs.size() == vh.size(), false, "mismatched number of txs"); + + auto txhi = req.txs_hashes.cbegin(); + auto vhi = vh.cbegin(); + auto missedi = missed_txs.cbegin(); + for(auto& tx: txs) { res.txs.push_back(COMMAND_RPC_GET_TRANSACTIONS::entry()); COMMAND_RPC_GET_TRANSACTIONS::entry &e = res.txs.back(); + while (missedi != missed_txs.end() && *missedi == *vhi) + { + ++vhi; + ++txhi; + ++missedi; + } + crypto::hash tx_hash = *vhi++; + CHECK_AND_ASSERT_MES(tx_hash == std::get<0>(tx), false, "mismatched tx hash"); e.tx_hash = *txhi++; e.prunable_hash = epee::string_tools::pod_to_hex(std::get<2>(tx)); if (req.split || req.prune || std::get<3>(tx).empty()) @@ -1861,6 +1875,43 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_getminerdata(const COMMAND_RPC_GETMINERDATA::request& req, COMMAND_RPC_GETMINERDATA::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) + { + if(!check_core_ready()) + { + error_resp.code = CORE_RPC_ERROR_CODE_CORE_BUSY; + error_resp.message = "Core is busy"; + return false; + } + + crypto::hash prev_id, seed_hash; + difficulty_type difficulty; + + std::vector<tx_block_template_backlog_entry> tx_backlog; + if (!m_core.get_miner_data(res.major_version, res.height, prev_id, seed_hash, difficulty, res.median_weight, res.already_generated_coins, tx_backlog)) + { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = "Internal error: failed to get miner data"; + LOG_ERROR("Failed to get miner data"); + return false; + } + + res.tx_backlog.clear(); + res.tx_backlog.reserve(tx_backlog.size()); + + for (const auto& entry : tx_backlog) + { + res.tx_backlog.emplace_back(COMMAND_RPC_GETMINERDATA::response::tx_backlog_entry{string_tools::pod_to_hex(entry.id), entry.weight, entry.fee}); + } + + res.prev_id = string_tools::pod_to_hex(prev_id); + res.seed_hash = string_tools::pod_to_hex(seed_hash); + res.difficulty = cryptonote::hex(difficulty); + + res.status = CORE_RPC_STATUS_OK; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_add_aux_pow(const COMMAND_RPC_ADD_AUX_POW::request& req, COMMAND_RPC_ADD_AUX_POW::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { RPC_TRACKER(add_aux_pow); diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index b21e43ab0..84b14383a 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -91,7 +91,8 @@ namespace cryptonote const boost::program_options::variables_map& vm, const bool restricted, const std::string& port, - bool allow_rpc_payment + bool allow_rpc_payment, + const std::string& proxy = {} ); network_type nettype() const { return m_core.get_nettype(); } @@ -147,6 +148,7 @@ namespace cryptonote MAP_JON_RPC_WE("on_getblockhash", on_getblockhash, COMMAND_RPC_GETBLOCKHASH) MAP_JON_RPC_WE("get_block_template", on_getblocktemplate, COMMAND_RPC_GETBLOCKTEMPLATE) MAP_JON_RPC_WE("getblocktemplate", on_getblocktemplate, COMMAND_RPC_GETBLOCKTEMPLATE) + MAP_JON_RPC_WE("get_miner_data", on_getminerdata, COMMAND_RPC_GETMINERDATA) MAP_JON_RPC_WE("add_aux_pow", on_add_aux_pow, COMMAND_RPC_ADD_AUX_POW) MAP_JON_RPC_WE("submit_block", on_submitblock, COMMAND_RPC_SUBMITBLOCK) MAP_JON_RPC_WE("submitblock", on_submitblock, COMMAND_RPC_SUBMITBLOCK) @@ -228,6 +230,7 @@ namespace cryptonote bool on_getblockcount(const COMMAND_RPC_GETBLOCKCOUNT::request& req, COMMAND_RPC_GETBLOCKCOUNT::response& res, const connection_context *ctx = NULL); bool on_getblockhash(const COMMAND_RPC_GETBLOCKHASH::request& req, COMMAND_RPC_GETBLOCKHASH::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_getblocktemplate(const COMMAND_RPC_GETBLOCKTEMPLATE::request& req, COMMAND_RPC_GETBLOCKTEMPLATE::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_getminerdata(const COMMAND_RPC_GETMINERDATA::request& req, COMMAND_RPC_GETMINERDATA::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_add_aux_pow(const COMMAND_RPC_ADD_AUX_POW::request& req, COMMAND_RPC_ADD_AUX_POW::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_submitblock(const COMMAND_RPC_SUBMITBLOCK::request& req, COMMAND_RPC_SUBMITBLOCK::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_generateblocks(const COMMAND_RPC_GENERATEBLOCKS::request& req, COMMAND_RPC_GENERATEBLOCKS::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); @@ -289,6 +292,7 @@ private: nodetool::node_server<cryptonote::t_cryptonote_protocol_handler<cryptonote::core> >& m_p2p; boost::shared_mutex m_bootstrap_daemon_mutex; std::unique_ptr<bootstrap_daemon> m_bootstrap_daemon; + std::string m_bootstrap_daemon_proxy; bool m_should_use_bootstrap_daemon; std::chrono::system_clock::time_point m_bootstrap_height_check_time; bool m_was_bootstrap_ever_used; diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index ff8c98b98..1dbfc83a7 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -88,7 +88,7 @@ namespace cryptonote // advance which version they will stop working with // Don't go over 32767 for any of these #define CORE_RPC_VERSION_MAJOR 3 -#define CORE_RPC_VERSION_MINOR 7 +#define CORE_RPC_VERSION_MINOR 8 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR) @@ -940,6 +940,56 @@ namespace cryptonote typedef epee::misc_utils::struct_init<response_t> response; }; + struct COMMAND_RPC_GETMINERDATA + { + struct request_t: public rpc_request_base + { + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_PARENT(rpc_request_base) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<request_t> request; + + struct response_t: public rpc_response_base + { + uint8_t major_version; + uint64_t height; + std::string prev_id; + std::string seed_hash; + std::string difficulty; + uint64_t median_weight; + uint64_t already_generated_coins; + + struct tx_backlog_entry + { + std::string id; + uint64_t weight; + uint64_t fee; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(id) + KV_SERIALIZE(weight) + KV_SERIALIZE(fee) + END_KV_SERIALIZE_MAP() + }; + + std::vector<tx_backlog_entry> tx_backlog; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_PARENT(rpc_response_base) + KV_SERIALIZE(major_version) + KV_SERIALIZE(height) + KV_SERIALIZE(prev_id) + KV_SERIALIZE(seed_hash) + KV_SERIALIZE(difficulty) + KV_SERIALIZE(median_weight) + KV_SERIALIZE(already_generated_coins) + KV_SERIALIZE(tx_backlog) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<response_t> response; + }; + struct COMMAND_RPC_ADD_AUX_POW { struct aux_pow_t diff --git a/src/rpc/zmq_pub.cpp b/src/rpc/zmq_pub.cpp index eac530968..074b55207 100644 --- a/src/rpc/zmq_pub.cpp +++ b/src/rpc/zmq_pub.cpp @@ -48,6 +48,8 @@ #include "cryptonote_basic/events.h" #include "misc_log_ex.h" #include "serialization/json_object.h" +#include "ringct/rctTypes.h" +#include "cryptonote_core/cryptonote_tx_utils.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net.zmq" @@ -57,6 +59,7 @@ namespace constexpr const char txpool_signal[] = "tx_signal"; using chain_writer = void(epee::byte_stream&, std::uint64_t, epee::span<const cryptonote::block>); + using miner_writer = void(epee::byte_stream&, uint8_t, uint64_t, const crypto::hash&, const crypto::hash&, cryptonote::difficulty_type, uint64_t, uint64_t, const std::vector<cryptonote::tx_block_template_backlog_entry>&); using txpool_writer = void(epee::byte_stream&, epee::span<const cryptonote::txpool_event>); template<typename F> @@ -116,13 +119,30 @@ namespace const epee::span<const cryptonote::block> blocks; }; + //! Object for miner data serialization + struct miner_data + { + uint8_t major_version; + uint64_t height; + const crypto::hash& prev_id; + const crypto::hash& seed_hash; + cryptonote::difficulty_type diff; + uint64_t median_weight; + uint64_t already_generated_coins; + const std::vector<cryptonote::tx_block_template_backlog_entry>& tx_backlog; + }; + //! Object for "minimal" tx serialization struct minimal_txpool { const cryptonote::transaction& tx; + crypto::hash hash; + uint64_t blob_size; + uint64_t weight; + uint64_t fee; }; - void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const minimal_chain self) + void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const minimal_chain& self) { namespace adapt = boost::adaptors; @@ -143,19 +163,27 @@ namespace dest.EndObject(); } - void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const minimal_txpool self) + void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const miner_data& self) { - crypto::hash id{}; - std::size_t blob_size = 0; - if (!get_transaction_hash(self.tx, id, blob_size)) - { - MERROR("ZMQ/Pub failure: get_transaction_hash"); - return; - } + dest.StartObject(); + INSERT_INTO_JSON_OBJECT(dest, major_version, self.major_version); + INSERT_INTO_JSON_OBJECT(dest, height, self.height); + INSERT_INTO_JSON_OBJECT(dest, prev_id, self.prev_id); + INSERT_INTO_JSON_OBJECT(dest, seed_hash, self.seed_hash); + INSERT_INTO_JSON_OBJECT(dest, difficulty, cryptonote::hex(self.diff)); + INSERT_INTO_JSON_OBJECT(dest, median_weight, self.median_weight); + INSERT_INTO_JSON_OBJECT(dest, already_generated_coins, self.already_generated_coins); + INSERT_INTO_JSON_OBJECT(dest, tx_backlog, self.tx_backlog); + dest.EndObject(); + } + void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const minimal_txpool& self) + { dest.StartObject(); - INSERT_INTO_JSON_OBJECT(dest, id, id); - INSERT_INTO_JSON_OBJECT(dest, blob_size, blob_size); + INSERT_INTO_JSON_OBJECT(dest, id, self.hash); + INSERT_INTO_JSON_OBJECT(dest, blob_size, self.blob_size); + INSERT_INTO_JSON_OBJECT(dest, weight, self.weight); + INSERT_INTO_JSON_OBJECT(dest, fee, self.fee); dest.EndObject(); } @@ -169,6 +197,11 @@ namespace json_pub(buf, minimal_chain{height, blocks}); } + void json_miner_data(epee::byte_stream& buf, uint8_t major_version, uint64_t height, const crypto::hash& prev_id, const crypto::hash& seed_hash, cryptonote::difficulty_type diff, uint64_t median_weight, uint64_t already_generated_coins, const std::vector<cryptonote::tx_block_template_backlog_entry>& tx_backlog) + { + json_pub(buf, miner_data{major_version, height, prev_id, seed_hash, diff, median_weight, already_generated_coins, tx_backlog}); + } + // boost::adaptors are in place "views" - no copy/move takes place // moving transactions (via sort, etc.), is expensive! @@ -187,7 +220,7 @@ namespace namespace adapt = boost::adaptors; const auto to_minimal_tx = [](const cryptonote::txpool_event& event) { - return minimal_txpool{event.tx}; + return minimal_txpool{event.tx, event.hash, event.blob_size, event.weight, cryptonote::get_tx_fee(event.tx)}; }; json_pub(buf, (txes | adapt::filtered(is_valid{}) | adapt::transformed(to_minimal_tx))); } @@ -198,6 +231,11 @@ namespace {u8"json-minimal-chain_main", json_minimal_chain} }}; + constexpr const std::array<context<miner_writer>, 1> miner_contexts = + {{ + {u8"json-full-miner_data", json_miner_data}, + }}; + constexpr const std::array<context<txpool_writer>, 2> txpool_contexts = {{ {u8"json-full-txpool_add", json_full_txpool}, @@ -321,6 +359,7 @@ namespace cryptonote { namespace listener zmq_pub::zmq_pub(void* context) : relay_(), chain_subs_{{0}}, + miner_subs_{{0}}, txpool_subs_{{0}}, sync_() { @@ -328,6 +367,7 @@ zmq_pub::zmq_pub(void* context) throw std::logic_error{"ZMQ context cannot be NULL"}; verify_sorted(chain_contexts, "chain_contexts"); + verify_sorted(miner_contexts, "miner_contexts"); verify_sorted(txpool_contexts, "txpool_contexts"); relay_.reset(zmq_socket(context, ZMQ_PAIR)); @@ -348,22 +388,25 @@ bool zmq_pub::sub_request(boost::string_ref message) message.remove_prefix(1); const auto chain_range = get_range(chain_contexts, message); + const auto miner_range = get_range(miner_contexts, message); const auto txpool_range = get_range(txpool_contexts, message); - if (!chain_range.empty() || !txpool_range.empty()) + if (!chain_range.empty() || !miner_range.empty() || !txpool_range.empty()) { MDEBUG("Client " << (tag ? "subscribed" : "unsubscribed") << " to " << - chain_range.size() << " chain topic(s) and " << txpool_range.size() << " txpool topic(s)"); + chain_range.size() << " chain topic(s), " << miner_range.size() << " miner topic(s) and " << txpool_range.size() << " txpool topic(s)"); const boost::lock_guard<boost::mutex> lock{sync_}; switch (tag) { case 0: remove_subscriptions(chain_subs_, chain_range, chain_contexts.begin()); + remove_subscriptions(miner_subs_, miner_range, miner_contexts.begin()); remove_subscriptions(txpool_subs_, txpool_range, txpool_contexts.begin()); return true; case 1: add_subscriptions(chain_subs_, chain_range, chain_contexts.begin()); + add_subscriptions(miner_subs_, miner_range, miner_contexts.begin()); add_subscriptions(txpool_subs_, txpool_range, txpool_contexts.begin()); return true; default: @@ -436,6 +479,25 @@ std::size_t zmq_pub::send_chain_main(const std::uint64_t height, const epee::spa return 0; } +std::size_t zmq_pub::send_miner_data(uint8_t major_version, uint64_t height, const crypto::hash& prev_id, const crypto::hash& seed_hash, difficulty_type diff, uint64_t median_weight, uint64_t already_generated_coins, const std::vector<tx_block_template_backlog_entry>& tx_backlog) +{ + boost::unique_lock<boost::mutex> guard{sync_}; + + const auto subs_copy = miner_subs_; + guard.unlock(); + + for (const std::size_t sub : subs_copy) + { + if (sub) + { + auto messages = make_pubs(subs_copy, miner_contexts, major_version, height, prev_id, seed_hash, diff, median_weight, already_generated_coins, tx_backlog); + guard.lock(); + return send_messages(relay_.get(), messages); + } + } + return 0; +} + std::size_t zmq_pub::send_txpool_add(std::vector<txpool_event> txes) { if (txes.empty()) @@ -466,6 +528,15 @@ void zmq_pub::chain_main::operator()(const std::uint64_t height, epee::span<cons MERROR("Unable to send ZMQ/Pub - ZMQ server destroyed"); } +void zmq_pub::miner_data::operator()(uint8_t major_version, uint64_t height, const crypto::hash& prev_id, const crypto::hash& seed_hash, difficulty_type diff, uint64_t median_weight, uint64_t already_generated_coins, const std::vector<tx_block_template_backlog_entry>& tx_backlog) const +{ + const std::shared_ptr<zmq_pub> self = self_.lock(); + if (self) + self->send_miner_data(major_version, height, prev_id, seed_hash, diff, median_weight, already_generated_coins, tx_backlog); + else + MERROR("Unable to send ZMQ/Pub - ZMQ server destroyed"); +} + void zmq_pub::txpool_add::operator()(std::vector<cryptonote::txpool_event> txes) const { const std::shared_ptr<zmq_pub> self = self_.lock(); diff --git a/src/rpc/zmq_pub.h b/src/rpc/zmq_pub.h index 02e6b8103..c636e1d7b 100644 --- a/src/rpc/zmq_pub.h +++ b/src/rpc/zmq_pub.h @@ -39,6 +39,7 @@ #include "cryptonote_basic/fwd.h" #include "net/zmq.h" #include "span.h" +#include "cryptonote_basic/difficulty.h" namespace cryptonote { namespace listener { @@ -59,6 +60,7 @@ class zmq_pub net::zmq::socket relay_; std::deque<std::vector<txpool_event>> txes_; std::array<std::size_t, 2> chain_subs_; + std::array<std::size_t, 1> miner_subs_; std::array<std::size_t, 2> txpool_subs_; boost::mutex sync_; //!< Synchronizes counts in `*_subs_` arrays. @@ -88,6 +90,11 @@ class zmq_pub \return Number of ZMQ messages sent to relay. */ std::size_t send_chain_main(std::uint64_t height, epee::span<const cryptonote::block> blocks); + /*! Send a `ZMQ_PUB` notification for a new miner data. + Thread-safe. + \return Number of ZMQ messages sent to relay. */ + std::size_t send_miner_data(uint8_t major_version, uint64_t height, const crypto::hash& prev_id, const crypto::hash& seed_hash, difficulty_type diff, uint64_t median_weight, uint64_t already_generated_coins, const std::vector<tx_block_template_backlog_entry>& tx_backlog); + /*! Send a `ZMQ_PUB` notification for new tx(es) being added to the local pool. Thread-safe. \return Number of ZMQ messages sent to relay. */ @@ -100,6 +107,13 @@ class zmq_pub void operator()(std::uint64_t height, epee::span<const cryptonote::block> blocks) const; }; + //! Callable for `send_miner_data` with weak ownership to `zmq_pub` object. + struct miner_data + { + std::weak_ptr<zmq_pub> self_; + void operator()(uint8_t major_version, uint64_t height, const crypto::hash& prev_id, const crypto::hash& seed_hash, difficulty_type diff, uint64_t median_weight, uint64_t already_generated_coins, const std::vector<tx_block_template_backlog_entry>& tx_backlog) const; + }; + //! Callable for `send_txpool_add` with weak ownership to `zmq_pub` object. struct txpool_add { diff --git a/src/serialization/json_object.cpp b/src/serialization/json_object.cpp index 28e207ff2..b03da1edc 100644 --- a/src/serialization/json_object.cpp +++ b/src/serialization/json_object.cpp @@ -34,6 +34,7 @@ #include <type_traits> #include "cryptonote_basic/cryptonote_basic_impl.h" +#include "cryptonote_core/cryptonote_tx_utils.h" // drop macro from windows.h #ifdef GetObject @@ -1411,6 +1412,27 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::output_distribu GET_FROM_JSON_OBJECT(val, dist.data.base, base); } +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::tx_block_template_backlog_entry& entry) +{ + dest.StartObject(); + INSERT_INTO_JSON_OBJECT(dest, id, entry.id); + INSERT_INTO_JSON_OBJECT(dest, weight, entry.weight); + INSERT_INTO_JSON_OBJECT(dest, fee, entry.fee); + dest.EndObject(); +} + +void fromJsonValue(const rapidjson::Value& val, cryptonote::tx_block_template_backlog_entry& entry) +{ + if (!val.IsObject()) + { + throw WRONG_TYPE("json object"); + } + + GET_FROM_JSON_OBJECT(val, entry.id, id); + GET_FROM_JSON_OBJECT(val, entry.weight, weight); + GET_FROM_JSON_OBJECT(val, entry.fee, fee); +} + } // namespace json } // namespace cryptonote diff --git a/src/serialization/json_object.h b/src/serialization/json_object.h index 35ea990b3..c858faf5a 100644 --- a/src/serialization/json_object.h +++ b/src/serialization/json_object.h @@ -304,6 +304,9 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::DaemonInfo& inf void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::rpc::output_distribution& dist); void fromJsonValue(const rapidjson::Value& val, cryptonote::rpc::output_distribution& dist); +void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const cryptonote::tx_block_template_backlog_entry& entry); +void fromJsonValue(const rapidjson::Value& val, cryptonote::tx_block_template_backlog_entry& entry); + template <typename Map> typename std::enable_if<sfinae::is_map_like<Map>::value, void>::type toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const Map& map); diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index c56a21346..22f71c49f 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -49,6 +49,7 @@ using namespace epee; #include "cryptonote_core/tx_sanity_check.h" #include "wallet_rpc_helpers.h" #include "wallet2.h" +#include "wallet_args.h" #include "cryptonote_basic/cryptonote_format_utils.h" #include "net/parse.h" #include "rpc/core_rpc_server_commands_defs.h" @@ -276,7 +277,7 @@ struct options { const command_line::arg_descriptor<bool> trusted_daemon = {"trusted-daemon", tools::wallet2::tr("Enable commands which rely on a trusted daemon"), false}; const command_line::arg_descriptor<bool> untrusted_daemon = {"untrusted-daemon", tools::wallet2::tr("Disable commands which rely on a trusted daemon"), false}; const command_line::arg_descriptor<std::string> password = {"password", tools::wallet2::tr("Wallet password (escape/quote as needed)"), "", true}; - const command_line::arg_descriptor<std::string> password_file = {"password-file", tools::wallet2::tr("Wallet password file"), "", true}; + const command_line::arg_descriptor<std::string> password_file = wallet_args::arg_password_file(); const command_line::arg_descriptor<int> daemon_port = {"daemon-port", tools::wallet2::tr("Use daemon instance at port <arg> instead of 18081"), 0}; const command_line::arg_descriptor<std::string> daemon_login = {"daemon-login", tools::wallet2::tr("Specify username[:password] for daemon RPC client"), "", true}; const command_line::arg_descriptor<std::string> daemon_ssl = {"daemon-ssl", tools::wallet2::tr("Enable SSL on daemon RPC connections: enabled|disabled|autodetect"), "autodetect"}; @@ -532,7 +533,7 @@ std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variabl boost::optional<tools::password_container> get_password(const boost::program_options::variables_map& vm, const options& opts, const std::function<boost::optional<tools::password_container>(const char*, bool)> &password_prompter, const bool verify) { - if (command_line::has_arg(vm, opts.password) && command_line::has_arg(vm, opts.password_file)) + if (command_line::has_arg(vm, opts.password) && !command_line::is_arg_defaulted(vm, opts.password_file)) { THROW_WALLET_EXCEPTION(tools::error::wallet_internal_error, tools::wallet2::tr("can't specify more than one of --password and --password-file")); } @@ -542,10 +543,11 @@ boost::optional<tools::password_container> get_password(const boost::program_opt return tools::password_container{command_line::get_arg(vm, opts.password)}; } - if (command_line::has_arg(vm, opts.password_file)) + if (!command_line::is_arg_defaulted(vm, opts.password_file)) { std::string password; - bool r = epee::file_io_utils::load_file_to_string(command_line::get_arg(vm, opts.password_file), + const auto password_file = command_line::get_arg(vm, opts.password_file); + bool r = epee::file_io_utils::load_file_to_string(password_file, password); THROW_WALLET_EXCEPTION_IF(!r, tools::error::wallet_internal_error, tools::wallet2::tr("the password file specified could not be read")); @@ -1022,13 +1024,7 @@ gamma_picker::gamma_picker(const std::vector<uint64_t> &rct_offsets, double shap end = rct_offsets.data() + rct_offsets.size() - CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE; num_rct_outputs = *(end - 1); THROW_WALLET_EXCEPTION_IF(num_rct_outputs == 0, error::wallet_internal_error, "No rct outputs"); - THROW_WALLET_EXCEPTION_IF(outputs_to_consider == 0, error::wallet_internal_error, "No rct outputs to consider"); - average_output_time = DIFFICULTY_TARGET_V2 * blocks_to_consider / outputs_to_consider; // this assumes constant target over the whole rct range - if (average_output_time == 0) { - // TODO: apply this to all cases; do so alongside a hard fork, where all clients will update at the same time, preventing anonymity puddle formation - average_output_time = DIFFICULTY_TARGET_V2 * blocks_to_consider / static_cast<double>(outputs_to_consider); - } - THROW_WALLET_EXCEPTION_IF(average_output_time == 0, error::wallet_internal_error, "Average seconds per output cannot be 0."); + average_output_time = DIFFICULTY_TARGET_V2 * blocks_to_consider / static_cast<double>(outputs_to_consider); // this assumes constant target over the whole rct range }; gamma_picker::gamma_picker(const std::vector<uint64_t> &rct_offsets): gamma_picker(rct_offsets, GAMMA_SHAPE, GAMMA_SCALE) {} @@ -2860,7 +2856,7 @@ void wallet2::process_parsed_blocks(uint64_t start_height, const std::vector<cry THROW_WALLET_EXCEPTION_IF(txidx >= tx_cache_data.size(), error::wallet_internal_error, "txidx out of range"); const cryptonote::transaction& tx = parsed_blocks[i].block.miner_tx; const size_t n_vouts = (m_refresh_type == RefreshType::RefreshOptimizeCoinbase && tx.version < 2) ? 1 : tx.vout.size(); - tpool.submit(&waiter, [&, i, n_vouts, txidx](){ geniod(tx, n_vouts, txidx); }, true); + tpool.submit(&waiter, [&, n_vouts, txidx](){ geniod(tx, n_vouts, txidx); }, true); } ++txidx; for (size_t j = 0; j < parsed_blocks[i].txes.size(); ++j) @@ -4465,7 +4461,26 @@ bool wallet2::load_keys_buf(const std::string& keys_buf, const epee::wipeable_st m_account.set_device(hwdev); account_public_address device_account_public_address; - THROW_WALLET_EXCEPTION_IF(!hwdev.get_public_address(device_account_public_address), error::wallet_internal_error, "Cannot get a device address"); + bool fetch_device_address = true; + + ::hw::device_cold* dev_cold = nullptr; + if (m_key_device_type == hw::device::device_type::TREZOR && (dev_cold = dynamic_cast<::hw::device_cold*>(&hwdev)) != nullptr) { + THROW_WALLET_EXCEPTION_IF(!dev_cold->get_public_address_with_no_passphrase(device_account_public_address), error::wallet_internal_error, "Cannot get a device address"); + if (device_account_public_address == m_account.get_keys().m_account_address) { + LOG_PRINT_L0("Wallet opened with an empty passphrase"); + fetch_device_address = false; + dev_cold->set_use_empty_passphrase(true); + } else { + fetch_device_address = true; + LOG_PRINT_L0("Wallet opening with an empty passphrase failed. Retry again: " << fetch_device_address); + dev_cold->reset_session(); + } + } + + if (fetch_device_address) { + THROW_WALLET_EXCEPTION_IF(!hwdev.get_public_address(device_account_public_address), error::wallet_internal_error, "Cannot get a device address"); + } + THROW_WALLET_EXCEPTION_IF(device_account_public_address != m_account.get_keys().m_account_address, error::wallet_internal_error, "Device wallet does not match wallet address. If the device uses the passphrase feature, please check whether the passphrase was entered correctly (it may have been misspelled - different passphrases generate different wallets, passphrase is case-sensitive). " "Device address: " + cryptonote::get_account_address_as_str(m_nettype, false, device_account_public_address) + ", wallet address: " + m_account.get_public_address_str(m_nettype)); @@ -10261,6 +10276,38 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp const size_t num_outputs = get_num_outputs(tx.dsts, m_transfers, tx.selected_transfers); needed_fee = estimate_fee(use_per_byte_fee, use_rct ,tx.selected_transfers.size(), fake_outs_count, num_outputs, extra.size(), bulletproof, clsag, base_fee, fee_multiplier, fee_quantization_mask); + auto try_carving_from_partial_payment = [&](uint64_t needed_fee, uint64_t available_for_fee) + { + // The check against original_output_index is to ensure the last entry in tx.dsts is really + // a partial payment. Otherwise multiple requested outputs to the same address could + // fool this logic into thinking there is a partial payment. + if (needed_fee > available_for_fee && !dsts.empty() && dsts[0].amount > 0 && tx.dsts.size() > original_output_index) + { + // we don't have enough for the fee, but we've only partially paid the current address, + // so we can take the fee from the paid amount, since we'll have to make another tx anyway + LOG_PRINT_L2("Attempting to carve tx fee " << print_money(needed_fee) << " from partial payment (first pass)"); + std::vector<cryptonote::tx_destination_entry>::iterator i; + i = std::find_if(tx.dsts.begin(), tx.dsts.end(), + [&](const cryptonote::tx_destination_entry &d) { return !memcmp (&d.addr, &dsts[0].addr, sizeof(dsts[0].addr)); }); + THROW_WALLET_EXCEPTION_IF(i == tx.dsts.end(), error::wallet_internal_error, "paid address not found in outputs"); + if (i->amount > needed_fee) + { + uint64_t new_paid_amount = i->amount /*+ test_ptx.fee*/ - needed_fee; + LOG_PRINT_L2("Adjusting amount paid to " << get_account_address_as_str(m_nettype, i->is_subaddress, i->addr) << " from " << + print_money(i->amount) << " to " << print_money(new_paid_amount) << " to accommodate " << + print_money(needed_fee) << " fee"); + dsts[0].amount += i->amount - new_paid_amount; + i->amount = new_paid_amount; + test_ptx.fee = needed_fee; + available_for_fee = needed_fee; + } + } + return available_for_fee; + }; + + // Try to carve the estimated fee from the partial payment (if there is one) + available_for_fee = try_carving_from_partial_payment(needed_fee, available_for_fee); + uint64_t inputs = 0, outputs = needed_fee; for (size_t idx: tx.selected_transfers) inputs += m_transfers[idx].amount(); for (const auto &o: tx.dsts) outputs += o.amount; @@ -10286,26 +10333,8 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp LOG_PRINT_L2("Made a " << get_weight_string(test_ptx.tx, txBlob.size()) << " tx, with " << print_money(available_for_fee) << " available for fee (" << print_money(needed_fee) << " needed)"); - if (needed_fee > available_for_fee && !dsts.empty() && dsts[0].amount > 0) - { - // we don't have enough for the fee, but we've only partially paid the current address, - // so we can take the fee from the paid amount, since we'll have to make another tx anyway - std::vector<cryptonote::tx_destination_entry>::iterator i; - i = std::find_if(tx.dsts.begin(), tx.dsts.end(), - [&](const cryptonote::tx_destination_entry &d) { return !memcmp (&d.addr, &dsts[0].addr, sizeof(dsts[0].addr)); }); - THROW_WALLET_EXCEPTION_IF(i == tx.dsts.end(), error::wallet_internal_error, "paid address not found in outputs"); - if (i->amount > needed_fee) - { - uint64_t new_paid_amount = i->amount /*+ test_ptx.fee*/ - needed_fee; - LOG_PRINT_L2("Adjusting amount paid to " << get_account_address_as_str(m_nettype, i->is_subaddress, i->addr) << " from " << - print_money(i->amount) << " to " << print_money(new_paid_amount) << " to accommodate " << - print_money(needed_fee) << " fee"); - dsts[0].amount += i->amount - new_paid_amount; - i->amount = new_paid_amount; - test_ptx.fee = needed_fee; - available_for_fee = needed_fee; - } - } + // Try to carve the fee from the partial payment again after updating from estimate to actual + available_for_fee = try_carving_from_partial_payment(needed_fee, available_for_fee); if (needed_fee > available_for_fee) { diff --git a/src/wallet/wallet_args.cpp b/src/wallet/wallet_args.cpp index 55058bf4e..066e98e52 100644 --- a/src/wallet/wallet_args.cpp +++ b/src/wallet/wallet_args.cpp @@ -80,6 +80,10 @@ namespace wallet_args { return {"rpc-client-secret-key", wallet_args::tr("Set RPC client secret key for RPC payments"), ""}; } + command_line::arg_descriptor<std::string> arg_password_file() + { + return {"password-file", wallet_args::tr("Wallet password file"), ""}; + } const char* tr(const char* str) { diff --git a/src/wallet/wallet_args.h b/src/wallet/wallet_args.h index 4af1b58fe..21e5f187c 100644 --- a/src/wallet/wallet_args.h +++ b/src/wallet/wallet_args.h @@ -37,6 +37,7 @@ namespace wallet_args command_line::arg_descriptor<std::string> arg_generate_from_json(); command_line::arg_descriptor<std::string> arg_wallet_file(); command_line::arg_descriptor<std::string> arg_rpc_client_secret_key(); + command_line::arg_descriptor<std::string> arg_password_file(); const char* tr(const char* str); diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index e1a06886b..4655e24cd 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -41,6 +41,7 @@ using namespace epee; #include "wallet/wallet_args.h" #include "common/command_line.h" #include "common/i18n.h" +#include "common/scoped_message_writer.h" #include "cryptonote_config.h" #include "cryptonote_basic/cryptonote_format_utils.h" #include "cryptonote_basic/account.h" @@ -4525,10 +4526,12 @@ public: const auto arg_wallet_file = wallet_args::arg_wallet_file(); const auto arg_from_json = wallet_args::arg_generate_from_json(); const auto arg_rpc_client_secret_key = wallet_args::arg_rpc_client_secret_key(); + const auto arg_password_file = wallet_args::arg_password_file(); const auto wallet_file = command_line::get_arg(vm, arg_wallet_file); const auto from_json = command_line::get_arg(vm, arg_from_json); const auto wallet_dir = command_line::get_arg(vm, arg_wallet_dir); + const auto password_file = command_line::get_arg(vm, arg_password_file); const auto prompt_for_password = command_line::get_arg(vm, arg_prompt_for_password); const auto password_prompt = prompt_for_password ? password_prompter : nullptr; @@ -4538,6 +4541,12 @@ public: return false; } + if(!wallet_dir.empty() && !password_file.empty()) + { + LOG_ERROR(tools::wallet_rpc_server::tr("--password-file is not allowed in combination with --wallet-dir")); + return false; + } + if (!wallet_dir.empty()) { wal = NULL; @@ -4716,7 +4725,7 @@ int main(int argc, char** argv) { tools::wallet_rpc_server::tr("This is the RPC monero wallet. It needs to connect to a monero\ndaemon to work correctly."), desc_params, po::positional_options_description(), - [](const std::string &s, bool emphasis){ epee::set_console_color(emphasis ? epee::console_color_white : epee::console_color_default, true); std::cout << s << std::endl; if (emphasis) epee::reset_console_color(); }, + [](const std::string &s, bool emphasis){ tools::scoped_message_writer(emphasis ? epee::console_color_white : epee::console_color_default, true) << s; }, "monero-wallet-rpc.log", true ); |