diff options
Diffstat (limited to 'src')
69 files changed, 1280 insertions, 375 deletions
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index da6d76d97..8a21763c8 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -133,7 +133,7 @@ if(NOT IOS) add_subdirectory(blockchain_utilities) endif() -if(CMAKE_BUILD_TYPE STREQUAL Debug) +if(BUILD_DEBUG_UTILITIES) add_subdirectory(debug_utilities) endif() diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 567be6a65..b6b8c6c3e 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -754,6 +754,21 @@ public: virtual void batch_stop() = 0; /** + * @brief aborts a batch transaction + * + * If the subclass implements batching, this function should abort the + * batch it is currently on. + * + * If no batch is in-progress, this function should throw a DB_ERROR. + * This exception may change in the future if it is deemed necessary to + * have a more granular exception type for this scenario. + * + * If any of this cannot be done, the subclass should throw the corresponding + * subclass of DB_EXCEPTION + */ + virtual void batch_abort() = 0; + + /** * @brief sets whether or not to batch transactions * * If the subclass implements batching, this function tells it to begin diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 3278c7387..784fabf4a 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -770,8 +770,8 @@ void BlockchainLMDB::add_block(const block& blk, size_t block_weight, uint64_t l bi.bi_timestamp = blk.timestamp; bi.bi_coins = coins_generated; bi.bi_weight = block_weight; - bi.bi_diff_hi = (cumulative_difficulty >> 64).convert_to<uint64_t>(); - bi.bi_diff_lo = (cumulative_difficulty << 64 >> 64).convert_to<uint64_t>(); + bi.bi_diff_hi = ((cumulative_difficulty >> 64) & 0xffffffffffffffff).convert_to<uint64_t>(); + bi.bi_diff_lo = (cumulative_difficulty & 0xffffffffffffffff).convert_to<uint64_t>(); bi.bi_hash = blk_hash; bi.bi_cum_rct = num_rct_outs; if (blk.major_version >= 4) @@ -1077,11 +1077,11 @@ void BlockchainLMDB::add_tx_amount_output_indices(const uint64_t tx_id, int result = 0; - int num_outputs = amount_output_indices.size(); + size_t num_outputs = amount_output_indices.size(); MDB_val_set(k_tx_id, tx_id); MDB_val v; - v.mv_data = (void *)amount_output_indices.data(); + v.mv_data = num_outputs ? (void *)amount_output_indices.data() : (void*)""; v.mv_size = sizeof(uint64_t) * num_outputs; // LOG_PRINT_L1("tx_outputs[tx_hash] size: " << v.mv_size); @@ -1953,7 +1953,7 @@ bool BlockchainLMDB::prune_worker(int mode, uint32_t pruning_seed) TIME_MEASURE_START(t); - size_t n_total_records = 0, n_prunable_records = 0, n_pruned_records = 0; + size_t n_total_records = 0, n_prunable_records = 0, n_pruned_records = 0, commit_counter = 0; uint64_t n_bytes = 0; mdb_txn_safe txn; @@ -2056,6 +2056,7 @@ bool BlockchainLMDB::prune_worker(int mode, uint32_t pruning_seed) { MDEBUG("Pruning at height " << block_height << "/" << blockchain_height); ++n_pruned_records; + ++commit_counter; n_bytes += k.mv_size + v.mv_size; result = mdb_cursor_del(c_txs_prunable, 0); if (result) @@ -2065,6 +2066,25 @@ bool BlockchainLMDB::prune_worker(int mode, uint32_t pruning_seed) result = mdb_cursor_del(c_txs_prunable_tip, 0); if (result) throw0(DB_ERROR(lmdb_error("Failed to delete transaction tip data: ", result).c_str())); + + if (mode != prune_mode_check && commit_counter >= 4096) + { + MDEBUG("Committing txn at checkpoint..."); + txn.commit(); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + result = mdb_cursor_open(txn, m_txs_pruned, &c_txs_pruned); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_pruned: ", result).c_str())); + result = mdb_cursor_open(txn, m_txs_prunable, &c_txs_prunable); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_prunable: ", result).c_str())); + result = mdb_cursor_open(txn, m_txs_prunable_tip, &c_txs_prunable_tip); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_prunable_tip: ", result).c_str())); + commit_counter = 0; + } } } } @@ -2134,6 +2154,7 @@ bool BlockchainLMDB::prune_worker(int mode, uint32_t pruning_seed) result = mdb_cursor_del(c_txs_prunable, 0); if (result) throw0(DB_ERROR(lmdb_error("Failed to delete transaction prunable data: ", result).c_str())); + ++commit_counter; } } } @@ -2150,6 +2171,34 @@ bool BlockchainLMDB::prune_worker(int mode, uint32_t pruning_seed) ", seed " << epee::string_tools::to_string_hex(pruning_seed)); } } + + if (mode != prune_mode_check && commit_counter >= 4096) + { + MDEBUG("Committing txn at checkpoint..."); + txn.commit(); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + result = mdb_cursor_open(txn, m_txs_pruned, &c_txs_pruned); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_pruned: ", result).c_str())); + result = mdb_cursor_open(txn, m_txs_prunable, &c_txs_prunable); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_prunable: ", result).c_str())); + result = mdb_cursor_open(txn, m_txs_prunable_tip, &c_txs_prunable_tip); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for txs_prunable_tip: ", result).c_str())); + result = mdb_cursor_open(txn, m_tx_indices, &c_tx_indices); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for tx_indices: ", result).c_str())); + MDB_val val; + val.mv_size = sizeof(ti); + val.mv_data = (void *)&ti; + result = mdb_cursor_get(c_tx_indices, (MDB_val*)&zerokval, &val, MDB_GET_BOTH); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to restore cursor for tx_indices: ", result).c_str())); + commit_counter = 0; + } } mdb_cursor_close(c_tx_indices); } diff --git a/src/blockchain_db/testdb.h b/src/blockchain_db/testdb.h index 6c97713d5..34e635899 100644 --- a/src/blockchain_db/testdb.h +++ b/src/blockchain_db/testdb.h @@ -54,6 +54,7 @@ public: virtual void unlock() override { } virtual bool batch_start(uint64_t batch_num_blocks=0, uint64_t batch_bytes=0) override { return true; } virtual void batch_stop() override {} + virtual void batch_abort() override {} virtual void set_batch_transactions(bool) override {} virtual void block_wtxn_start() override {} virtual void block_wtxn_stop() override {} diff --git a/src/blockchain_utilities/README.md b/src/blockchain_utilities/README.md index ad5963f27..1462e3186 100644 --- a/src/blockchain_utilities/README.md +++ b/src/blockchain_utilities/README.md @@ -79,7 +79,7 @@ LMDB flags (more than one may be specified): ## Examples: -``` +```bash $ monero-blockchain-import --database lmdb#fastest $ monero-blockchain-import --database lmdb#nosync diff --git a/src/blocks/checkpoints.dat b/src/blocks/checkpoints.dat Binary files differindex adc433522..a7d309753 100644 --- a/src/blocks/checkpoints.dat +++ b/src/blocks/checkpoints.dat diff --git a/src/checkpoints/checkpoints.cpp b/src/checkpoints/checkpoints.cpp index e31b96646..11bbe2e24 100644 --- a/src/checkpoints/checkpoints.cpp +++ b/src/checkpoints/checkpoints.cpp @@ -209,6 +209,7 @@ namespace cryptonote ADD_CHECKPOINT(1579000, "7d0d7a2346373afd41ed1e744a939fc5d474a7dbaa257be5c6fff4009e789241"); ADD_CHECKPOINT(1668900, "ac2dcaf3d2f58ffcf8391639f0f1ebafcb8eac43c49479c7c37f611868d07568"); ADD_CHECKPOINT(1775600, "1c6e01c661dc22cab939e79ec6a5272190624ce8356d2f7b958e4f9a57fdb05e"); + ADD_CHECKPOINT(1856000, "9b57f17f29c71a3acd8a7904b93c41fa6eb8d2b7c73936ce4f1702d14880ba29"); return true; } diff --git a/src/common/dns_utils.cpp b/src/common/dns_utils.cpp index 5e03bf897..dc1f335a7 100644 --- a/src/common/dns_utils.cpp +++ b/src/common/dns_utils.cpp @@ -48,7 +48,6 @@ static const char *DEFAULT_DNS_PUBLIC_ADDR[] = "80.67.169.40", // FDN (France) "89.233.43.71", // http://censurfridns.dk (Denmark) "109.69.8.51", // punCAT (Spain) - "77.109.148.137", // Xiala.net (Switzerland) "193.58.251.251", // SkyDNS (Russia) }; diff --git a/src/common/password.cpp b/src/common/password.cpp index 03d13db42..33e1f48fd 100644 --- a/src/common/password.cpp +++ b/src/common/password.cpp @@ -57,7 +57,7 @@ namespace DWORD mode_old; ::GetConsoleMode(h_cin, &mode_old); - DWORD mode_new = mode_old & ~(hide_input ? ENABLE_ECHO_INPUT : 0); + DWORD mode_new = mode_old & ~((hide_input ? ENABLE_ECHO_INPUT : 0) | ENABLE_LINE_INPUT); ::SetConsoleMode(h_cin, mode_new); bool r = true; @@ -77,10 +77,6 @@ namespace } else if (ucs2_ch == L'\r') { - continue; - } - else if (ucs2_ch == L'\n') - { std::cout << std::endl; break; } diff --git a/src/common/rpc_client.h b/src/common/rpc_client.h index cb5f79da8..dab3e562d 100644 --- a/src/common/rpc_client.h +++ b/src/common/rpc_client.h @@ -36,6 +36,7 @@ #include "storages/http_abstract_invoke.h" #include "net/http_auth.h" #include "net/http_client.h" +#include "net/net_ssl.h" #include "string_tools.h" namespace tools @@ -49,11 +50,12 @@ namespace tools uint32_t ip , uint16_t port , boost::optional<epee::net_utils::http::login> user + , epee::net_utils::ssl_options_t ssl_options ) : m_http_client{} { m_http_client.set_server( - epee::string_tools::get_ip_string_from_int32(ip), std::to_string(port), std::move(user) + epee::string_tools::get_ip_string_from_int32(ip), std::to_string(port), std::move(user), std::move(ssl_options) ); } diff --git a/src/crypto/keccak.c b/src/crypto/keccak.c index 170911262..851c70a25 100644 --- a/src/crypto/keccak.c +++ b/src/crypto/keccak.c @@ -105,9 +105,12 @@ void keccak(const uint8_t *in, size_t inlen, uint8_t *md, int mdlen) memset(st, 0, sizeof(st)); for ( ; inlen >= rsiz; inlen -= rsiz, in += rsiz) { - for (i = 0; i < rsizw; i++) - st[i] ^= swap64le(((uint64_t *) in)[i]); - keccakf(st, KECCAK_ROUNDS); + for (i = 0; i < rsizw; i++) { + uint64_t ina; + memcpy(&ina, in + i * 8, 8); + st[i] ^= swap64le(ina); + } + keccakf(st, KECCAK_ROUNDS); } // last block and padding @@ -116,7 +119,8 @@ void keccak(const uint8_t *in, size_t inlen, uint8_t *md, int mdlen) local_abort("Bad keccak use"); } - memcpy(temp, in, inlen); + if (inlen > 0) + memcpy(temp, in, inlen); temp[inlen++] = 1; memset(temp + inlen, 0, rsiz - inlen); temp[rsiz - 1] |= 0x80; diff --git a/src/crypto/tree-hash.c b/src/crypto/tree-hash.c index 7802fb67f..0a5860f3b 100644 --- a/src/crypto/tree-hash.c +++ b/src/crypto/tree-hash.c @@ -30,6 +30,7 @@ #include <assert.h> #include <stddef.h> +#include <stdlib.h> #include <string.h> #include "hash-ops.h" @@ -82,23 +83,24 @@ void tree_hash(const char (*hashes)[HASH_SIZE], size_t count, char *root_hash) { size_t cnt = tree_hash_cnt( count ); - char ints[cnt][HASH_SIZE]; - memset(ints, 0 , sizeof(ints)); // zero out as extra protection for using uninitialized mem + char *ints = calloc(cnt, HASH_SIZE); // zero out as extra protection for using uninitialized mem + assert(ints); memcpy(ints, hashes, (2 * cnt - count) * HASH_SIZE); for (i = 2 * cnt - count, j = 2 * cnt - count; j < cnt; i += 2, ++j) { - cn_fast_hash(hashes[i], 64, ints[j]); + cn_fast_hash(hashes[i], 64, ints + j * HASH_SIZE); } assert(i == count); while (cnt > 2) { cnt >>= 1; for (i = 0, j = 0; j < cnt; i += 2, ++j) { - cn_fast_hash(ints[i], 64, ints[j]); + cn_fast_hash(ints + i * HASH_SIZE, 64, ints + j * HASH_SIZE); } } - cn_fast_hash(ints[0], 64, root_hash); + cn_fast_hash(ints, 64, root_hash); + free(ints); } } diff --git a/src/cryptonote_basic/cryptonote_basic.h b/src/cryptonote_basic/cryptonote_basic.h index 20d92bdf1..055c4a22b 100644 --- a/src/cryptonote_basic/cryptonote_basic.h +++ b/src/cryptonote_basic/cryptonote_basic.h @@ -320,7 +320,7 @@ namespace cryptonote } if (!typename Archive<W>::is_saving()) pruned = true; - return true; + return ar.stream().good(); } private: diff --git a/src/cryptonote_basic/cryptonote_boost_serialization.h b/src/cryptonote_basic/cryptonote_boost_serialization.h index 3dd98f0c6..79ce610a9 100644 --- a/src/cryptonote_basic/cryptonote_boost_serialization.h +++ b/src/cryptonote_basic/cryptonote_boost_serialization.h @@ -365,11 +365,11 @@ namespace boost else { // store high part - cryptonote::difficulty_type x_ = x >> 64; + cryptonote::difficulty_type x_ = (x >> 64) & 0xffffffffffffffff; uint64_t v = x_.convert_to<uint64_t>(); a & v; // store low part - x_ = x << 64 >> 64; + x_ = x & 0xffffffffffffffff; v = x_.convert_to<uint64_t>(); a & v; } diff --git a/src/cryptonote_basic/cryptonote_format_utils.cpp b/src/cryptonote_basic/cryptonote_format_utils.cpp index 566622c1a..7d7de416d 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.cpp +++ b/src/cryptonote_basic/cryptonote_format_utils.cpp @@ -221,8 +221,7 @@ namespace cryptonote tx.invalidate_hashes(); //TODO: validate tx - get_transaction_hash(tx, tx_hash); - return true; + return get_transaction_hash(tx, tx_hash); } //--------------------------------------------------------------- bool parse_and_validate_tx_from_blob(const blobdata& tx_blob, transaction& tx, crypto::hash& tx_hash, crypto::hash& tx_prefix_hash) @@ -975,6 +974,7 @@ namespace cryptonote { crypto::hash h = null_hash; get_transaction_hash(t, h, NULL); + CHECK_AND_ASSERT_THROW_MES(get_transaction_hash(t, h, NULL), "Failed to calculate transaction hash"); return h; } //--------------------------------------------------------------- @@ -1327,7 +1327,7 @@ namespace cryptonote txs_ids.reserve(1 + b.tx_hashes.size()); crypto::hash h = null_hash; size_t bl_sz = 0; - get_transaction_hash(b.miner_tx, h, bl_sz); + CHECK_AND_ASSERT_THROW_MES(get_transaction_hash(b.miner_tx, h, bl_sz), "Failed to calculate transaction hash"); txs_ids.push_back(h); for(auto& th: b.tx_hashes) txs_ids.push_back(th); diff --git a/src/cryptonote_basic/miner.cpp b/src/cryptonote_basic/miner.cpp index 173679e21..e594eb049 100644 --- a/src/cryptonote_basic/miner.cpp +++ b/src/cryptonote_basic/miner.cpp @@ -123,7 +123,7 @@ namespace cryptonote m_miner_extra_sleep(BACKGROUND_MINING_DEFAULT_MINER_EXTRA_SLEEP_MILLIS), m_block_reward(0) { - + m_attrs.set_stack_size(THREAD_STACK_SIZE); } //----------------------------------------------------------------------------------------------------- miner::~miner() @@ -360,7 +360,7 @@ namespace cryptonote return m_threads_total; } //----------------------------------------------------------------------------------------------------- - bool miner::start(const account_public_address& adr, size_t threads_count, const boost::thread::attributes& attrs, bool do_background, bool ignore_battery) + bool miner::start(const account_public_address& adr, size_t threads_count, bool do_background, bool ignore_battery) { m_block_reward = 0; m_mine_address = adr; @@ -371,7 +371,6 @@ namespace cryptonote m_threads_autodetect.push_back({epee::misc_utils::get_ns_count(), m_total_hashes}); m_threads_total = 1; } - m_attrs = attrs; m_starter_nonce = crypto::rand<uint32_t>(); CRITICAL_REGION_LOCAL(m_threads_lock); if(is_mining()) @@ -395,7 +394,7 @@ namespace cryptonote for(size_t i = 0; i != m_threads_total; i++) { - m_threads.push_back(boost::thread(attrs, boost::bind(&miner::worker_thread, this))); + m_threads.push_back(boost::thread(m_attrs, boost::bind(&miner::worker_thread, this))); } if (threads_count == 0) @@ -405,7 +404,7 @@ namespace cryptonote if( get_is_background_mining_enabled() ) { - m_background_mining_thread = boost::thread(attrs, boost::bind(&miner::background_worker_thread, this)); + m_background_mining_thread = boost::thread(m_attrs, boost::bind(&miner::background_worker_thread, this)); LOG_PRINT_L0("Background mining controller thread started" ); } @@ -487,10 +486,7 @@ namespace cryptonote { if(m_do_mining) { - boost::thread::attributes attrs; - attrs.set_stack_size(THREAD_STACK_SIZE); - - start(m_mine_address, m_threads_total, attrs, get_is_background_mining_enabled(), get_ignore_battery()); + start(m_mine_address, m_threads_total, get_is_background_mining_enabled(), get_ignore_battery()); } } //----------------------------------------------------------------------------------------------------- diff --git a/src/cryptonote_basic/miner.h b/src/cryptonote_basic/miner.h index 285075f51..ac7a0381c 100644 --- a/src/cryptonote_basic/miner.h +++ b/src/cryptonote_basic/miner.h @@ -64,7 +64,7 @@ namespace cryptonote static void init_options(boost::program_options::options_description& desc); bool set_block_template(const block& bl, const difficulty_type& diffic, uint64_t height, uint64_t block_reward); bool on_block_chain_update(); - bool start(const account_public_address& adr, size_t threads_count, const boost::thread::attributes& attrs, bool do_background = false, bool ignore_battery = false); + bool start(const account_public_address& adr, size_t threads_count, bool do_background = false, bool ignore_battery = false); uint64_t get_speed() const; uint32_t get_threads_count() const; void send_stop_signal(); diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 1b51c2e06..e542b5e7f 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -179,9 +179,11 @@ Blockchain::Blockchain(tx_memory_pool& tx_pool) : m_long_term_block_weights_window(CRYPTONOTE_LONG_TERM_BLOCK_WEIGHT_WINDOW_SIZE), m_long_term_effective_median_block_weight(0), m_long_term_block_weights_cache_tip_hash(crypto::null_hash), + m_long_term_block_weights_cache_rolling_median(CRYPTONOTE_LONG_TERM_BLOCK_WEIGHT_WINDOW_SIZE), m_difficulty_for_next_block_top_hash(crypto::null_hash), m_difficulty_for_next_block(1), - m_btc_valid(false) + m_btc_valid(false), + m_batch_success(true) { LOG_PRINT_L3("Blockchain::" << __func__); } @@ -519,7 +521,10 @@ bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline } if (test_options && test_options->long_term_block_weight_window) + { m_long_term_block_weights_window = test_options->long_term_block_weight_window; + m_long_term_block_weights_cache_rolling_median = epee::misc_utils::rolling_median_t<uint64_t>(m_long_term_block_weights_window); + } { db_txn_guard txn_guard(m_db, m_db->is_read_only()); @@ -615,17 +620,13 @@ void Blockchain::pop_blocks(uint64_t nblocks) CRITICAL_REGION_LOCAL(m_tx_pool); CRITICAL_REGION_LOCAL1(m_blockchain_lock); - while (!m_db->batch_start()) - { - m_blockchain_lock.unlock(); - m_tx_pool.unlock(); - epee::misc_utils::sleep_no_w(1000); - m_tx_pool.lock(); - m_blockchain_lock.lock(); - } + bool stop_batch = m_db->batch_start(); try { + const uint64_t blockchain_height = m_db->height(); + if (blockchain_height > 0) + nblocks = std::min(nblocks, blockchain_height - 1); for (i=0; i < nblocks; ++i) { pop_block_from_blockchain(); @@ -633,10 +634,14 @@ void Blockchain::pop_blocks(uint64_t nblocks) } catch (const std::exception& e) { - LOG_ERROR("Error when popping blocks, only " << i << " blocks are popped: " << e.what()); + LOG_ERROR("Error when popping blocks after processing " << i << " blocks: " << e.what()); + if (stop_batch) + m_db->batch_abort(); + return; } - m_db->batch_stop(); + if (stop_batch) + m_db->batch_stop(); } //------------------------------------------------------------------ // This function tells BlockchainDB to remove the top block from the @@ -1283,21 +1288,20 @@ void Blockchain::get_last_n_blocks_weights(std::vector<uint64_t>& weights, size_ weights = m_db->get_block_weights(start_offset, count); } //------------------------------------------------------------------ -void Blockchain::get_long_term_block_weights(std::vector<uint64_t>& weights, uint64_t start_height, size_t count) const +uint64_t Blockchain::get_long_term_block_weight_median(uint64_t start_height, size_t count) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); PERF_TIMER(get_long_term_block_weights); - if (count == 0) - return; + CHECK_AND_ASSERT_THROW_MES(count > 0, "count == 0"); bool cached = false; uint64_t blockchain_height = m_db->height(); uint64_t tip_height = start_height + count - 1; crypto::hash tip_hash = crypto::null_hash; - if (tip_height < blockchain_height && count == m_long_term_block_weights_cache.size()) + if (tip_height < blockchain_height && count == (size_t)m_long_term_block_weights_cache_rolling_median.size()) { tip_hash = m_db->get_block_hash_from_height(tip_height); cached = tip_hash == m_long_term_block_weights_cache_tip_hash; @@ -1306,32 +1310,30 @@ void Blockchain::get_long_term_block_weights(std::vector<uint64_t>& weights, uin if (cached) { MTRACE("requesting " << count << " from " << start_height << ", cached"); - weights = m_long_term_block_weights_cache; - return; + return m_long_term_block_weights_cache_rolling_median.median(); } // in the vast majority of uncached cases, most is still cached, // as we just move the window one block up: - if (tip_height > 0 && count == m_long_term_block_weights_cache.size() && tip_height < blockchain_height) + if (tip_height > 0 && count == (size_t)m_long_term_block_weights_cache_rolling_median.size() && tip_height < blockchain_height) { crypto::hash old_tip_hash = m_db->get_block_hash_from_height(tip_height - 1); if (old_tip_hash == m_long_term_block_weights_cache_tip_hash) { - weights = m_long_term_block_weights_cache; - for (size_t i = 1; i < weights.size(); ++i) - weights[i - 1] = weights[i]; MTRACE("requesting " << count << " from " << start_height << ", incremental"); - weights.back() = m_db->get_block_long_term_weight(tip_height); - m_long_term_block_weights_cache = weights; m_long_term_block_weights_cache_tip_hash = tip_hash; - return; + m_long_term_block_weights_cache_rolling_median.insert(m_db->get_block_long_term_weight(tip_height)); + return m_long_term_block_weights_cache_rolling_median.median(); } } MTRACE("requesting " << count << " from " << start_height << ", uncached"); - weights = m_db->get_long_term_block_weights(start_height, count); - m_long_term_block_weights_cache = weights; + std::vector<uint64_t> weights = m_db->get_long_term_block_weights(start_height, count); m_long_term_block_weights_cache_tip_hash = tip_hash; + m_long_term_block_weights_cache_rolling_median.clear(); + for (uint64_t w: weights) + m_long_term_block_weights_cache_rolling_median.insert(w); + return m_long_term_block_weights_cache_rolling_median.median(); } //------------------------------------------------------------------ uint64_t Blockchain::get_current_cumulative_block_weight_limit() const @@ -1372,7 +1374,8 @@ bool Blockchain::create_block_template(block& b, const crypto::hash *from_block, // just as we compare it, we'll just use a slightly old template, but // this would be the case anyway if we'd lock, and the change happened // just after the block template was created - if (!memcmp(&miner_address, &m_btc_address, sizeof(cryptonote::account_public_address)) && m_btc_nonce == ex_nonce && m_btc_pool_cookie == m_tx_pool.cookie()) { + if (!memcmp(&miner_address, &m_btc_address, sizeof(cryptonote::account_public_address)) && m_btc_nonce == ex_nonce + && m_btc_pool_cookie == m_tx_pool.cookie() && m_btc.prev_id == get_tail_id()) { MDEBUG("Using cached template"); m_btc.timestamp = time(NULL); // update timestamp unconditionally b = m_btc; @@ -1890,10 +1893,14 @@ bool Blockchain::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NO if (missed_tx_ids.size() != 0) { - LOG_ERROR("Error retrieving blocks, missed " << missed_tx_ids.size() - << " transactions for block with hash: " << get_block_hash(bl.second) - << std::endl - ); + // do not display an error if the peer asked for an unpruned block which we are not meant to have + if (tools::has_unpruned_block(get_block_height(bl.second), get_current_blockchain_height(), get_blockchain_pruning_seed())) + { + LOG_ERROR("Error retrieving blocks, missed " << missed_tx_ids.size() + << " transactions for block with hash: " << get_block_hash(bl.second) + << std::endl + ); + } // append missed transaction hashes to response missed_ids field, // as done below if any standalone transactions were requested @@ -2323,8 +2330,8 @@ bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qbloc if (result) { cryptonote::difficulty_type wide_cumulative_difficulty = m_db->get_block_cumulative_difficulty(resp.total_height - 1); - resp.cumulative_difficulty = (wide_cumulative_difficulty << 64 >> 64).convert_to<uint64_t>(); - resp.cumulative_difficulty_top64 = (wide_cumulative_difficulty >> 64).convert_to<uint64_t>(); + resp.cumulative_difficulty = (wide_cumulative_difficulty & 0xffffffffffffffff).convert_to<uint64_t>(); + resp.cumulative_difficulty_top64 = ((wide_cumulative_difficulty >> 64) & 0xffffffffffffffff).convert_to<uint64_t>(); } return result; @@ -3841,6 +3848,7 @@ leave: catch (const KEY_IMAGE_EXISTS& e) { LOG_ERROR("Error adding block with hash: " << id << " to blockchain, what = " << e.what()); + m_batch_success = false; bvc.m_verifivation_failed = true; return_tx_to_pool(txs); return false; @@ -3849,6 +3857,7 @@ leave: { //TODO: figure out the best way to deal with this failure LOG_ERROR("Error adding block with hash: " << id << " to blockchain, what = " << e.what()); + m_batch_success = false; bvc.m_verifivation_failed = true; return_tx_to_pool(txs); return false; @@ -3932,9 +3941,7 @@ uint64_t Blockchain::get_next_long_term_block_weight(uint64_t block_weight) cons if (hf_version < HF_VERSION_LONG_TERM_BLOCK_WEIGHT) return block_weight; - std::vector<uint64_t> weights; - get_long_term_block_weights(weights, db_height - nblocks, nblocks); - uint64_t long_term_median = epee::misc_utils::median(weights); + uint64_t long_term_median = get_long_term_block_weight_median(db_height - nblocks, nblocks); uint64_t long_term_effective_median_block_weight = std::max<uint64_t>(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, long_term_median); uint64_t short_term_constraint = long_term_effective_median_block_weight + long_term_effective_median_block_weight * 2 / 5; @@ -3964,7 +3971,6 @@ bool Blockchain::update_next_cumulative_weight_limit(uint64_t *long_term_effecti { const uint64_t block_weight = m_db->get_block_weight(db_height - 1); - std::vector<uint64_t> weights, new_weights; uint64_t long_term_median; if (db_height == 1) { @@ -3975,9 +3981,7 @@ bool Blockchain::update_next_cumulative_weight_limit(uint64_t *long_term_effecti uint64_t nblocks = std::min<uint64_t>(m_long_term_block_weights_window, db_height); if (nblocks == db_height) --nblocks; - get_long_term_block_weights(weights, db_height - nblocks - 1, nblocks); - new_weights = weights; - long_term_median = epee::misc_utils::median(weights); + long_term_median = get_long_term_block_weight_median(db_height - nblocks - 1, nblocks); } m_long_term_effective_median_block_weight = std::max<uint64_t>(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, long_term_median); @@ -3985,13 +3989,19 @@ bool Blockchain::update_next_cumulative_weight_limit(uint64_t *long_term_effecti uint64_t short_term_constraint = m_long_term_effective_median_block_weight + m_long_term_effective_median_block_weight * 2 / 5; uint64_t long_term_block_weight = std::min<uint64_t>(block_weight, short_term_constraint); - if (new_weights.empty()) - new_weights.resize(1); - new_weights[0] = long_term_block_weight; - long_term_median = epee::misc_utils::median(new_weights); + if (db_height == 1) + { + long_term_median = long_term_block_weight; + } + else + { + m_long_term_block_weights_cache_tip_hash = m_db->get_block_hash_from_height(db_height - 1); + m_long_term_block_weights_cache_rolling_median.insert(long_term_block_weight); + long_term_median = m_long_term_block_weights_cache_rolling_median.median(); + } m_long_term_effective_median_block_weight = std::max<uint64_t>(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, long_term_median); - weights.clear(); + std::vector<uint64_t> weights; get_last_n_blocks_weights(weights, CRYPTONOTE_REWARD_BLOCKS_WINDOW); uint64_t short_term_median = epee::misc_utils::median(weights); @@ -4155,7 +4165,10 @@ bool Blockchain::cleanup_handle_incoming_blocks(bool force_sync) try { - m_db->batch_stop(); + if (m_batch_success) + m_db->batch_stop(); + else + m_db->batch_abort(); success = true; } catch (const std::exception &e) @@ -4379,6 +4392,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete m_tx_pool.lock(); m_blockchain_lock.lock(); } + m_batch_success = true; const uint64_t height = m_db->height(); if ((height + blocks_entry.size()) < m_blocks_hash_check.size()) @@ -4831,7 +4845,7 @@ void Blockchain::cancel() } #if defined(PER_BLOCK_CHECKPOINT) -static const char expected_block_hashes_hash[] = "570ce2357b08fadac6058e34f95c5e08323f9325de260d07b091a281a948a7b0"; +static const char expected_block_hashes_hash[] = "7dafb40b414a0e59bfced6682ef519f0b416bc914dd3d622b72e0dd1a47117c2"; void Blockchain::load_compiled_in_block_hashes(const GetCheckpointsCallback& get_checkpoints) { if (get_checkpoints == nullptr || !m_fast_sync) diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index 244e2a89a..32ed96b5b 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -37,7 +37,6 @@ #include <boost/multi_index/global_fun.hpp> #include <boost/multi_index/hashed_index.hpp> #include <boost/multi_index/member.hpp> -#include <boost/circular_buffer.hpp> #include <atomic> #include <functional> #include <unordered_map> @@ -46,6 +45,7 @@ #include "span.h" #include "syncobj.h" #include "string_tools.h" +#include "rolling_median.h" #include "cryptonote_basic/cryptonote_basic.h" #include "common/util.h" #include "cryptonote_protocol/cryptonote_protocol_defs.h" @@ -1064,7 +1064,7 @@ namespace cryptonote uint64_t m_long_term_block_weights_window; uint64_t m_long_term_effective_median_block_weight; mutable crypto::hash m_long_term_block_weights_cache_tip_hash; - mutable std::vector<uint64_t> m_long_term_block_weights_cache; + mutable epee::misc_utils::rolling_median_t<uint64_t> m_long_term_block_weights_cache_rolling_median; epee::critical_section m_difficulty_lock; crypto::hash m_difficulty_for_next_block_top_hash; @@ -1102,6 +1102,9 @@ namespace cryptonote uint64_t m_btc_expected_reward; bool m_btc_valid; + + bool m_batch_success; + std::shared_ptr<tools::Notify> m_block_notify; std::shared_ptr<tools::Notify> m_reorg_notify; @@ -1314,15 +1317,16 @@ namespace cryptonote void get_last_n_blocks_weights(std::vector<uint64_t>& weights, size_t count) const; /** - * @brief gets recent block long term weights for median calculation + * @brief gets block long term weight median * - * get the block long term weights of the last <count> blocks, and return by reference <weights>. + * get the block long term weight median of <count> blocks starting at <start_height> * - * @param weights return-by-reference the list of weights * @param start_height the block height of the first block to query * @param count the number of blocks to get weights for + * + * @return the long term median block weight */ - void get_long_term_block_weights(std::vector<uint64_t>& weights, uint64_t start_height, size_t count) const; + uint64_t get_long_term_block_weight_median(uint64_t start_height, size_t count) const; /** * @brief checks if a transaction is unlocked (its outputs spendable) diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index beafa5e3b..83bc5640c 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -676,9 +676,15 @@ namespace cryptonote if (prune_blockchain) { // display a message if the blockchain is not pruned yet - if (m_blockchain_storage.get_current_blockchain_height() > 1 && !m_blockchain_storage.get_blockchain_pruning_seed()) + if (!m_blockchain_storage.get_blockchain_pruning_seed()) + { MGINFO("Pruning blockchain..."); - CHECK_AND_ASSERT_MES(m_blockchain_storage.prune_blockchain(), false, "Failed to prune blockchain"); + CHECK_AND_ASSERT_MES(m_blockchain_storage.prune_blockchain(), false, "Failed to prune blockchain"); + } + else + { + CHECK_AND_ASSERT_MES(m_blockchain_storage.update_blockchain_pruning(), false, "Failed to update blockchain pruning"); + } } return load_state_data(); diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index c1cbe2acd..49d5a8ccc 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -95,13 +95,17 @@ namespace cryptonote // the whole prepare/handle/cleanup incoming block sequence. class LockedTXN { public: - LockedTXN(Blockchain &b): m_blockchain(b), m_batch(false) { + LockedTXN(Blockchain &b): m_blockchain(b), m_batch(false), m_active(false) { m_batch = m_blockchain.get_db().batch_start(); + m_active = true; } - ~LockedTXN() { try { if (m_batch) { m_blockchain.get_db().batch_stop(); } } catch (const std::exception &e) { MWARNING("LockedTXN dtor filtering exception: " << e.what()); } } + void commit() { try { if (m_batch && m_active) { m_blockchain.get_db().batch_stop(); m_active = false; } } catch (const std::exception &e) { MWARNING("LockedTXN::commit filtering exception: " << e.what()); } } + void abort() { try { if (m_batch && m_active) { m_blockchain.get_db().batch_abort(); m_active = false; } } catch (const std::exception &e) { MWARNING("LockedTXN::abort filtering exception: " << e.what()); } } + ~LockedTXN() { abort(); } private: Blockchain &m_blockchain; bool m_batch; + bool m_active; }; } //--------------------------------------------------------------------------------- @@ -255,6 +259,7 @@ namespace cryptonote if (!insert_key_images(tx, id, kept_by_block)) return false; m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)tx_weight, receive_time), id); + lock.commit(); } catch (const std::exception &e) { @@ -299,6 +304,7 @@ namespace cryptonote if (!insert_key_images(tx, id, kept_by_block)) return false; m_txs_by_fee_and_receive_time.emplace(std::pair<double, std::time_t>(fee / (double)tx_weight, receive_time), id); + lock.commit(); } catch (const std::exception &e) { @@ -398,6 +404,7 @@ namespace cryptonote return; } } + lock.commit(); if (changed) ++m_cookie; if (m_txpool_weight > bytes) @@ -494,6 +501,7 @@ namespace cryptonote m_blockchain.remove_txpool_tx(id); m_txpool_weight -= tx_weight; remove_transaction_keyimages(tx, id); + lock.commit(); } catch (const std::exception &e) { @@ -578,6 +586,7 @@ namespace cryptonote // ignore error } } + lock.commit(); ++m_cookie; } return true; @@ -641,6 +650,7 @@ namespace cryptonote // continue } } + lock.commit(); } //--------------------------------------------------------------------------------- size_t tx_memory_pool::get_transactions_count(bool include_unrelayed_txes) const @@ -1119,6 +1129,7 @@ namespace cryptonote } } } + lock.commit(); if (changed) ++m_cookie; } @@ -1271,6 +1282,7 @@ namespace cryptonote append_key_images(k_images, tx); LOG_PRINT_L2(" added, new block weight " << total_weight << "/" << max_total_weight << ", coinbase " << print_money(best_coinbase)); } + lock.commit(); expected_reward = best_coinbase; LOG_PRINT_L2("Block template filled with " << bl.tx_hashes.size() << " txes, weight " @@ -1336,6 +1348,7 @@ namespace cryptonote // continue } } + lock.commit(); } if (n_removed > 0) ++m_cookie; @@ -1395,6 +1408,7 @@ namespace cryptonote // ignore error } } + lock.commit(); } m_cookie = 0; diff --git a/src/cryptonote_core/tx_sanity_check.cpp b/src/cryptonote_core/tx_sanity_check.cpp index 10198a3d3..36be91f2c 100644 --- a/src/cryptonote_core/tx_sanity_check.cpp +++ b/src/cryptonote_core/tx_sanity_check.cpp @@ -88,7 +88,7 @@ bool tx_sanity_check(Blockchain &blockchain, const cryptonote::blobdata &tx_blob std::vector<uint64_t> offsets(rct_indices.begin(), rct_indices.end()); uint64_t median = epee::misc_utils::median(offsets); - if (median < n_available * 9 / 10) + if (median < n_available * 6 / 10) { MERROR("median is " << median << "/" << n_available); return false; diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.h b/src/cryptonote_protocol/cryptonote_protocol_handler.h index 0927b5d7f..dcc5ec6ed 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.h @@ -52,6 +52,7 @@ PUSH_WARNINGS DISABLE_VS_WARNINGS(4355) #define LOCALHOST_INT 2130706433 +#define CURRENCY_PROTOCOL_MAX_OBJECT_REQUEST_COUNT 500 namespace cryptonote { diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index 7adca3158..03d04a074 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -341,6 +341,11 @@ namespace cryptonote if(m_core.have_block(hshd.top_id)) { + if (target > hshd.current_height) + { + MINFO(context << "peer is not ahead of us and we're syncing, disconnecting"); + return false; + } context.m_state = cryptonote_connection_context::state_normal; if(is_inital && target == m_core.get_current_blockchain_height()) on_connection_synchronized(); @@ -410,8 +415,8 @@ namespace cryptonote m_core.get_blockchain_top(hshd.current_height, hshd.top_id); hshd.top_version = m_core.get_ideal_hard_fork_version(hshd.current_height); difficulty_type wide_cumulative_difficulty = m_core.get_block_cumulative_difficulty(hshd.current_height); - hshd.cumulative_difficulty = (wide_cumulative_difficulty << 64 >> 64).convert_to<uint64_t>(); - hshd.cumulative_difficulty_top64 = (wide_cumulative_difficulty >> 64).convert_to<uint64_t>(); + hshd.cumulative_difficulty = (wide_cumulative_difficulty & 0xffffffffffffffff).convert_to<uint64_t>(); + hshd.cumulative_difficulty_top64 = ((wide_cumulative_difficulty >> 64) & 0xffffffffffffffff).convert_to<uint64_t>(); hshd.current_height +=1; hshd.pruning_seed = m_core.get_blockchain_pruning_seed(); return true; @@ -809,12 +814,27 @@ namespace cryptonote NOTIFY_NEW_FLUFFY_BLOCK::request fluffy_response; fluffy_response.b.block = t_serializable_object_to_blob(b); fluffy_response.current_blockchain_height = arg.current_blockchain_height; + std::vector<bool> seen(b.tx_hashes.size(), false); for(auto& tx_idx: arg.missing_tx_indices) { if(tx_idx < b.tx_hashes.size()) { MDEBUG(" tx " << b.tx_hashes[tx_idx]); + if (seen[tx_idx]) + { + LOG_ERROR_CCONTEXT + ( + "Failed to handle request NOTIFY_REQUEST_FLUFFY_MISSING_TX" + << ", request is asking for duplicate tx " + << ", tx index = " << tx_idx << ", block tx count " << b.tx_hashes.size() + << ", block_height = " << arg.current_blockchain_height + << ", dropping connection" + ); + drop_connection(context, true, false); + return 1; + } txids.push_back(b.tx_hashes[tx_idx]); + seen[tx_idx] = true; } else { @@ -914,6 +934,17 @@ namespace cryptonote int t_cryptonote_protocol_handler<t_core>::handle_request_get_objects(int command, NOTIFY_REQUEST_GET_OBJECTS::request& arg, cryptonote_connection_context& context) { MLOG_P2P_MESSAGE("Received NOTIFY_REQUEST_GET_OBJECTS (" << arg.blocks.size() << " blocks, " << arg.txs.size() << " txes)"); + + if (arg.blocks.size() + arg.txs.size() > CURRENCY_PROTOCOL_MAX_OBJECT_REQUEST_COUNT) + { + LOG_ERROR_CCONTEXT( + "Requested objects count is too big (" + << arg.blocks.size() + arg.txs.size() << ") expected not more then " + << CURRENCY_PROTOCOL_MAX_OBJECT_REQUEST_COUNT); + drop_connection(context, false, false); + return 1; + } + NOTIFY_RESPONSE_GET_OBJECTS::request rsp; if(!m_core.handle_get_objects(arg, rsp, context)) { diff --git a/src/daemon/command_parser_executor.cpp b/src/daemon/command_parser_executor.cpp index 17b945c9a..ed799096b 100644 --- a/src/daemon/command_parser_executor.cpp +++ b/src/daemon/command_parser_executor.cpp @@ -40,10 +40,11 @@ t_command_parser_executor::t_command_parser_executor( uint32_t ip , uint16_t port , const boost::optional<tools::login>& login + , const epee::net_utils::ssl_options_t& ssl_options , bool is_rpc , cryptonote::core_rpc_server* rpc_server ) - : m_executor(ip, port, login, is_rpc, rpc_server) + : m_executor(ip, port, login, ssl_options, is_rpc, rpc_server) {} bool t_command_parser_executor::print_peer_list(const std::vector<std::string>& args) @@ -595,6 +596,13 @@ bool t_command_parser_executor::unban(const std::vector<std::string>& args) return m_executor.unban(ip); } +bool t_command_parser_executor::banned(const std::vector<std::string>& args) +{ + if (args.size() != 1) return false; + std::string address = args[0]; + return m_executor.banned(address); +} + bool t_command_parser_executor::flush_txpool(const std::vector<std::string>& args) { if (args.size() > 1) return false; diff --git a/src/daemon/command_parser_executor.h b/src/daemon/command_parser_executor.h index 098018642..aeaf4d254 100644 --- a/src/daemon/command_parser_executor.h +++ b/src/daemon/command_parser_executor.h @@ -40,6 +40,7 @@ #include "daemon/rpc_command_executor.h" #include "common/common_fwd.h" +#include "net/net_fwd.h" #include "rpc/core_rpc_server.h" namespace daemonize { @@ -53,6 +54,7 @@ public: uint32_t ip , uint16_t port , const boost::optional<tools::login>& login + , const epee::net_utils::ssl_options_t& ssl_options , bool is_rpc , cryptonote::core_rpc_server* rpc_server = NULL ); @@ -125,6 +127,8 @@ public: bool unban(const std::vector<std::string>& args); + bool banned(const std::vector<std::string>& args); + bool flush_txpool(const std::vector<std::string>& args); bool output_histogram(const std::vector<std::string>& args); diff --git a/src/daemon/command_server.cpp b/src/daemon/command_server.cpp index 69ad6ff10..f0b4b4ba0 100644 --- a/src/daemon/command_server.cpp +++ b/src/daemon/command_server.cpp @@ -43,10 +43,11 @@ t_command_server::t_command_server( uint32_t ip , uint16_t port , const boost::optional<tools::login>& login + , const epee::net_utils::ssl_options_t& ssl_options , bool is_rpc , cryptonote::core_rpc_server* rpc_server ) - : m_parser(ip, port, login, is_rpc, rpc_server) + : m_parser(ip, port, login, ssl_options, is_rpc, rpc_server) , m_command_lookup() , m_is_rpc(is_rpc) { @@ -242,10 +243,16 @@ t_command_server::t_command_server( m_command_lookup.set_handler( "unban" , std::bind(&t_command_parser_executor::unban, &m_parser, p::_1) - , "unban <IP>" + , "unban <address>" , "Unban a given <IP>." ); m_command_lookup.set_handler( + "banned" + , std::bind(&t_command_parser_executor::banned, &m_parser, p::_1) + , "banned <address>" + , "Check whether an <address> is banned." + ); + m_command_lookup.set_handler( "flush_txpool" , std::bind(&t_command_parser_executor::flush_txpool, &m_parser, p::_1) , "flush_txpool [<txid>]" diff --git a/src/daemon/command_server.h b/src/daemon/command_server.h index c8e77f551..da532223e 100644 --- a/src/daemon/command_server.h +++ b/src/daemon/command_server.h @@ -43,6 +43,7 @@ Passing RPC commands: #include "common/common_fwd.h" #include "console_handler.h" #include "daemon/command_parser_executor.h" +#include "net/net_fwd.h" namespace daemonize { @@ -57,6 +58,7 @@ public: uint32_t ip , uint16_t port , const boost::optional<tools::login>& login + , const epee::net_utils::ssl_options_t& ssl_options , bool is_rpc = true , cryptonote::core_rpc_server* rpc_server = NULL ); diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 531c080de..5084b6283 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -45,6 +45,7 @@ #include "daemon/command_server.h" #include "daemon/command_server.h" #include "daemon/command_line_args.h" +#include "net/net_ssl.h" #include "version.h" using namespace epee; @@ -163,7 +164,7 @@ bool t_daemon::run(bool interactive) if (interactive && mp_internals->rpcs.size()) { // The first three variables are not used when the fourth is false - rpc_commands.reset(new daemonize::t_command_server(0, 0, boost::none, false, mp_internals->rpcs.front()->get_server())); + rpc_commands.reset(new daemonize::t_command_server(0, 0, boost::none, epee::net_utils::ssl_support_t::e_ssl_support_disabled, false, mp_internals->rpcs.front()->get_server())); rpc_commands->start_handling(std::bind(&daemonize::t_daemon::stop_p2p, this)); } diff --git a/src/daemon/main.cpp b/src/daemon/main.cpp index dbbb2308c..690d4d60e 100644 --- a/src/daemon/main.cpp +++ b/src/daemon/main.cpp @@ -324,7 +324,11 @@ int main(int argc, char const * argv[]) } } - daemonize::t_command_server rpc_commands{rpc_ip, rpc_port, std::move(login)}; + auto ssl_options = cryptonote::rpc_args::process_ssl(vm, true); + if (!ssl_options) + return 1; + + daemonize::t_command_server rpc_commands{rpc_ip, rpc_port, std::move(login), std::move(*ssl_options)}; if (rpc_commands.process_command_vec(command)) { return 0; diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp index 186296dc9..bf4b3b09e 100644 --- a/src/daemon/rpc_command_executor.cpp +++ b/src/daemon/rpc_command_executor.cpp @@ -127,6 +127,7 @@ t_rpc_command_executor::t_rpc_command_executor( uint32_t ip , uint16_t port , const boost::optional<tools::login>& login + , const epee::net_utils::ssl_options_t& ssl_options , bool is_rpc , cryptonote::core_rpc_server* rpc_server ) @@ -137,7 +138,7 @@ t_rpc_command_executor::t_rpc_command_executor( boost::optional<epee::net_utils::http::login> http_login{}; if (login) http_login.emplace(login->username, login->password.password()); - m_rpc_client = new tools::t_rpc_client(ip, port, std::move(http_login)); + m_rpc_client = new tools::t_rpc_client(ip, port, std::move(http_login), ssl_options); } else { @@ -722,7 +723,7 @@ bool t_rpc_command_executor::print_blockchain_info(uint64_t start_block_index, u tools::msg_writer() << "" << std::endl; tools::msg_writer() << "height: " << header.height << ", timestamp: " << header.timestamp << " (" << tools::get_human_readable_timestamp(header.timestamp) << ")" - << ", size: " << header.block_size << ", weight: " << header.block_weight << ", transactions: " << header.num_txes << std::endl + << ", size: " << header.block_size << ", weight: " << header.block_weight << " (long term " << header.long_term_weight << "), transactions: " << header.num_txes << std::endl << "major version: " << (unsigned)header.major_version << ", minor version: " << (unsigned)header.minor_version << std::endl << "block id: " << header.hash << ", previous block id: " << header.prev_hash << std::endl << "difficulty: " << header.difficulty << ", nonce " << header.nonce << ", reward " << cryptonote::print_money(header.reward) << std::endl; @@ -1640,14 +1641,14 @@ bool t_rpc_command_executor::print_bans() for (auto i = res.bans.begin(); i != res.bans.end(); ++i) { - tools::msg_writer() << epee::string_tools::get_ip_string_from_int32(i->ip) << " banned for " << i->seconds << " seconds"; + tools::msg_writer() << i->host << " banned for " << i->seconds << " seconds"; } return true; } -bool t_rpc_command_executor::ban(const std::string &ip, time_t seconds) +bool t_rpc_command_executor::ban(const std::string &address, time_t seconds) { cryptonote::COMMAND_RPC_SETBANS::request req; cryptonote::COMMAND_RPC_SETBANS::response res; @@ -1655,11 +1656,8 @@ bool t_rpc_command_executor::ban(const std::string &ip, time_t seconds) epee::json_rpc::error error_resp; cryptonote::COMMAND_RPC_SETBANS::ban ban; - if (!epee::string_tools::get_ip_int32_from_string(ban.ip, ip)) - { - tools::fail_msg_writer() << "Invalid IP"; - return true; - } + ban.host = address; + ban.ip = 0; ban.ban = true; ban.seconds = seconds; req.bans.push_back(ban); @@ -1683,7 +1681,7 @@ bool t_rpc_command_executor::ban(const std::string &ip, time_t seconds) return true; } -bool t_rpc_command_executor::unban(const std::string &ip) +bool t_rpc_command_executor::unban(const std::string &address) { cryptonote::COMMAND_RPC_SETBANS::request req; cryptonote::COMMAND_RPC_SETBANS::response res; @@ -1691,11 +1689,8 @@ bool t_rpc_command_executor::unban(const std::string &ip) epee::json_rpc::error error_resp; cryptonote::COMMAND_RPC_SETBANS::ban ban; - if (!epee::string_tools::get_ip_int32_from_string(ban.ip, ip)) - { - tools::fail_msg_writer() << "Invalid IP"; - return true; - } + ban.host = address; + ban.ip = 0; ban.ban = false; ban.seconds = 0; req.bans.push_back(ban); @@ -1719,6 +1714,39 @@ bool t_rpc_command_executor::unban(const std::string &ip) return true; } +bool t_rpc_command_executor::banned(const std::string &address) +{ + cryptonote::COMMAND_RPC_BANNED::request req; + cryptonote::COMMAND_RPC_BANNED::response res; + std::string fail_message = "Unsuccessful"; + epee::json_rpc::error error_resp; + + req.address = address; + + if (m_is_rpc) + { + if (!m_rpc_client->json_rpc_request(req, res, "banned", fail_message.c_str())) + { + return true; + } + } + else + { + if (!m_rpc_server->on_banned(req, res, error_resp) || res.status != CORE_RPC_STATUS_OK) + { + tools::fail_msg_writer() << make_error(fail_message, res.status); + return true; + } + } + + if (res.banned) + tools::msg_writer() << address << " is banned for " << res.seconds << " seconds"; + else + tools::msg_writer() << address << " is not banned"; + + return true; +} + bool t_rpc_command_executor::flush_txpool(const std::string &txid) { cryptonote::COMMAND_RPC_FLUSH_TRANSACTION_POOL::request req; diff --git a/src/daemon/rpc_command_executor.h b/src/daemon/rpc_command_executor.h index 3c2686b3f..61c25736d 100644 --- a/src/daemon/rpc_command_executor.h +++ b/src/daemon/rpc_command_executor.h @@ -43,6 +43,7 @@ #include "common/common_fwd.h" #include "common/rpc_client.h" #include "cryptonote_basic/cryptonote_basic.h" +#include "net/net_fwd.h" #include "rpc/core_rpc_server.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -61,6 +62,7 @@ public: uint32_t ip , uint16_t port , const boost::optional<tools::login>& user + , const epee::net_utils::ssl_options_t& ssl_options , bool is_rpc = true , cryptonote::core_rpc_server* rpc_server = NULL ); @@ -135,9 +137,11 @@ public: bool print_bans(); - bool ban(const std::string &ip, time_t seconds); + bool ban(const std::string &address, time_t seconds); - bool unban(const std::string &ip); + bool unban(const std::string &address); + + bool banned(const std::string &address); bool flush_txpool(const std::string &txid); diff --git a/src/debug_utilities/CMakeLists.txt b/src/debug_utilities/CMakeLists.txt index 7bc2c324f..03c2b3e20 100644 --- a/src/debug_utilities/CMakeLists.txt +++ b/src/debug_utilities/CMakeLists.txt @@ -69,3 +69,25 @@ set_property(TARGET object_sizes PROPERTY OUTPUT_NAME "monero-utils-object-sizes") + +set(dns_checks_sources + dns_checks.cpp + ) + +monero_add_executable(dns_checks + ${dns_checks_sources} + ${dns_checks_private_headers}) + +target_link_libraries(dns_checks + LINK_PRIVATE + common + epee + version + ${Boost_PROGRAM_OPTIONS_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT}) + +set_property(TARGET dns_checks + PROPERTY + OUTPUT_NAME "monero-utils-dns-checks") + diff --git a/src/debug_utilities/dns_checks.cpp b/src/debug_utilities/dns_checks.cpp new file mode 100644 index 000000000..3c9daa769 --- /dev/null +++ b/src/debug_utilities/dns_checks.cpp @@ -0,0 +1,149 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <string> +#include <vector> +#include <map> +#include <algorithm> +#include <boost/program_options.hpp> +#include "misc_log_ex.h" +#include "common/util.h" +#include "common/command_line.h" +#include "common/dns_utils.h" +#include "version.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "debugtools.dnschecks" + +namespace po = boost::program_options; + +enum lookup_t { LOOKUP_A, LOOKUP_TXT }; + +static std::vector<std::string> lookup(lookup_t type, const char *hostname) +{ + bool dnssec_available = false, dnssec_valid = false; + std::vector<std::string> res; + switch (type) + { + case LOOKUP_A: res = tools::DNSResolver::instance().get_ipv4(hostname, dnssec_available, dnssec_valid); break; + case LOOKUP_TXT: res = tools::DNSResolver::instance().get_txt_record(hostname, dnssec_available, dnssec_valid); break; + default: MERROR("Invalid lookup type: " << (int)type); return {}; + } + if (!dnssec_available) + { + MWARNING("No DNSSEC for " << hostname); + return {}; + } + if (!dnssec_valid) + { + MWARNING("Invalid DNSSEC check for " << hostname); + return {}; + } + MINFO(res.size() << " valid signed result(s) for " << hostname); + return res; +} + +static void lookup(lookup_t type, const std::vector<std::string> hostnames) +{ + std::vector<std::vector<std::string>> results; + for (const std::string &hostname: hostnames) + { + auto res = lookup(type, hostname.c_str()); + if (!res.empty()) + { + std::sort(res.begin(), res.end()); + results.push_back(res); + } + } + std::map<std::vector<std::string>, size_t> counter; + for (const auto &e: results) + counter[e]++; + size_t count = 0; + for (const auto &e: counter) + count = std::max(count, e.second); + if (results.size() > 1) + { + if (count < results.size()) + MERROR("Only " << count << "/" << results.size() << " records match"); + else + MINFO(count << "/" << results.size() << " records match"); + } +} + +int main(int argc, char* argv[]) +{ + TRY_ENTRY(); + + tools::on_startup(); + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + po::store(po::parse_command_line(argc, argv, desc_options), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + mlog_configure("", true); + mlog_set_categories("+" MONERO_DEFAULT_LOG_CATEGORY ":INFO"); + + lookup(LOOKUP_A, {"seeds.moneroseeds.se", "seeds.moneroseeds.ae.org", "seeds.moneroseeds.ch", "seeds.moneroseeds.li"}); + + lookup(LOOKUP_TXT, {"updates.moneropulse.org", "updates.moneropulse.net", "updates.moneropulse.co", "updates.moneropulse.se"}); + + lookup(LOOKUP_TXT, {"checkpoints.moneropulse.org", "checkpoints.moneropulse.net", "checkpoints.moneropulse.co", "checkpoints.moneropulse.se"}); + + // those are in the code, but don't seem to actually exist +#if 0 + lookup(LOOKUP_TXT, {"testpoints.moneropulse.org", "testpoints.moneropulse.net", "testpoints.moneropulse.co", "testpoints.moneropulse.se"); + + lookup(LOOKUP_TXT, {"stagenetpoints.moneropulse.org", "stagenetpoints.moneropulse.net", "stagenetpoints.moneropulse.co", "stagenetpoints.moneropulse.se"}); +#endif + + lookup(LOOKUP_TXT, {"segheights.moneropulse.org", "segheights.moneropulse.net", "segheights.moneropulse.co", "segheights.moneropulse.se"}); + + return 0; + CATCH_ENTRY_L0("main", 1); +} diff --git a/src/device/device_ledger.cpp b/src/device/device_ledger.cpp index 200370564..eba633da8 100644 --- a/src/device/device_ledger.cpp +++ b/src/device/device_ledger.cpp @@ -90,6 +90,20 @@ namespace hw { AKout = keys.AKout; } + ABPkeys &ABPkeys::operator=(const ABPkeys& keys) { + if (&keys == this) + return *this; + Aout = keys.Aout; + Bout = keys.Bout; + is_subaddress = keys.is_subaddress; + is_change_address = keys.is_change_address; + additional_key = keys.additional_key; + index = keys.index; + Pout = keys.Pout; + AKout = keys.AKout; + return *this; + } + bool Keymap::find(const rct::key& P, ABPkeys& keys) const { size_t sz = ABP.size(); for (size_t i=0; i<sz; i++) { diff --git a/src/device/device_ledger.hpp b/src/device/device_ledger.hpp index d4d98ce4a..fe9028733 100644 --- a/src/device/device_ledger.hpp +++ b/src/device/device_ledger.hpp @@ -77,6 +77,7 @@ namespace hw { ABPkeys(const rct::key& A, const rct::key& B, const bool is_subaddr, bool is_subaddress, bool is_change_address, size_t index, const rct::key& P,const rct::key& AK); ABPkeys(const ABPkeys& keys) ; ABPkeys() {index=0;is_subaddress=false;is_subaddress=false;is_change_address=false;} + ABPkeys &operator=(const ABPkeys &keys); }; class Keymap { diff --git a/src/device_trezor/device_trezor_base.cpp b/src/device_trezor/device_trezor_base.cpp index 5adadbfc4..b7adf433d 100644 --- a/src/device_trezor/device_trezor_base.cpp +++ b/src/device_trezor/device_trezor_base.cpp @@ -115,10 +115,14 @@ namespace trezor { MDEBUG("Enumerating Trezor devices..."); enumerate(trans); + sort_transports_by_env(trans); - MDEBUG("Enumeration yielded " << trans.size() << " devices"); + MDEBUG("Enumeration yielded " << trans.size() << " Trezor devices"); for (auto &cur : trans) { MDEBUG(" device: " << *(cur.get())); + } + + for (auto &cur : trans) { std::string cur_path = cur->get_path(); if (boost::starts_with(cur_path, this->name)) { MDEBUG("Device Match: " << cur_path); diff --git a/src/device_trezor/trezor/transport.cpp b/src/device_trezor/trezor/transport.cpp index dd9b0b52f..59b281f13 100644 --- a/src/device_trezor/trezor/transport.cpp +++ b/src/device_trezor/trezor/transport.cpp @@ -31,11 +31,13 @@ #include <libusb.h> #endif +#include <algorithm> #include <boost/endian/conversion.hpp> #include <boost/asio/io_service.hpp> #include <boost/asio/ip/udp.hpp> #include <boost/date_time/posix_time/posix_time_types.hpp> #include <boost/format.hpp> +#include "common/apply_permutation.h" #include "transport.hpp" #include "messages/messages-common.pb.h" @@ -95,6 +97,47 @@ namespace trezor{ return patch | (((uint64_t)minor) << bits_2) | (((uint64_t)major) << (bits_1 + bits_2)); } + typedef struct { + uint16_t trezor_type; + uint16_t id_vendor; + uint16_t id_product; + } trezor_usb_desc_t; + + static trezor_usb_desc_t TREZOR_DESC_T1 = {1, 0x534C, 0x0001}; + static trezor_usb_desc_t TREZOR_DESC_T2 = {2, 0x1209, 0x53C1}; + static trezor_usb_desc_t TREZOR_DESC_T2_BL = {3, 0x1209, 0x53C0}; + + static trezor_usb_desc_t TREZOR_DESCS[] = { + TREZOR_DESC_T1, + TREZOR_DESC_T2, + TREZOR_DESC_T2_BL, + }; + + static size_t TREZOR_DESCS_LEN = sizeof(TREZOR_DESCS)/sizeof(TREZOR_DESCS[0]); + + static ssize_t get_device_idx(uint16_t id_vendor, uint16_t id_product){ + for(size_t i = 0; i < TREZOR_DESCS_LEN; ++i){ + if (TREZOR_DESCS[i].id_vendor == id_vendor && TREZOR_DESCS[i].id_product == id_product){ + return i; + } + } + + return -1; + } + + static bool is_device_supported(ssize_t device_idx){ + CHECK_AND_ASSERT_THROW_MES(device_idx < (ssize_t)TREZOR_DESCS_LEN, "Device desc idx too big"); + if (device_idx < 0){ + return false; + } + +#ifdef TREZOR_1_SUPPORTED + return true; +#else + return TREZOR_DESCS[device_idx].trezor_type != 1; +#endif + } + // // Helpers // @@ -312,6 +355,24 @@ namespace trezor{ for(rapidjson::Value::ConstValueIterator itr = bridge_res.Begin(); itr != bridge_res.End(); ++itr){ auto element = itr->GetObject(); auto t = std::make_shared<BridgeTransport>(boost::make_optional(json_get_string(element["path"]))); + + auto itr_vendor = element.FindMember("vendor"); + auto itr_product = element.FindMember("product"); + if (itr_vendor != element.MemberEnd() && itr_product != element.MemberEnd() + && itr_vendor->value.IsNumber() && itr_product->value.IsNumber()){ + try { + const auto id_vendor = (uint16_t) itr_vendor->value.GetUint64(); + const auto id_product = (uint16_t) itr_product->value.GetUint64(); + const auto device_idx = get_device_idx(id_vendor, id_product); + if (!is_device_supported(device_idx)){ + MDEBUG("Device with idx " << device_idx << " is not supported. Vendor: " << id_vendor << ", product: " << id_product); + continue; + } + } catch(const std::exception &e){ + MERROR("Could not detect vendor & product: " << e.what()); + } + } + t->m_device_info.emplace(); t->m_device_info->CopyFrom(*itr, t->m_device_info->GetAllocator()); res.push_back(t); @@ -710,24 +771,20 @@ namespace trezor{ #ifdef WITH_DEVICE_TREZOR_WEBUSB static bool is_trezor1(libusb_device_descriptor * info){ - return info->idVendor == 0x534C && info->idProduct == 0x0001; + return info->idVendor == TREZOR_DESC_T1.id_vendor && info->idProduct == TREZOR_DESC_T1.id_product; } static bool is_trezor2(libusb_device_descriptor * info){ - return info->idVendor == 0x1209 && info->idProduct == 0x53C1; + return info->idVendor == TREZOR_DESC_T2.id_vendor && info->idProduct == TREZOR_DESC_T2.id_product; } static bool is_trezor2_bl(libusb_device_descriptor * info){ - return info->idVendor == 0x1209 && info->idProduct == 0x53C0; + return info->idVendor == TREZOR_DESC_T2_BL.id_vendor && info->idProduct == TREZOR_DESC_T2_BL.id_product; } - static uint8_t get_trezor_dev_mask(libusb_device_descriptor * info){ - uint8_t mask = 0; + static ssize_t get_trezor_dev_id(libusb_device_descriptor *info){ CHECK_AND_ASSERT_THROW_MES(info, "Empty device descriptor"); - mask |= is_trezor1(info) ? 1 : 0; - mask |= is_trezor2(info) ? 2 : 0; - mask |= is_trezor2_bl(info) ? 4 : 0; - return mask; + return get_device_idx(info->idVendor, info->idProduct); } static void set_libusb_log(libusb_context *ctx){ @@ -844,12 +901,12 @@ namespace trezor{ continue; } - const auto trezor_mask = get_trezor_dev_mask(&desc); - if (!trezor_mask){ + const auto trezor_dev_idx = get_trezor_dev_id(&desc); + if (!is_device_supported(trezor_dev_idx)){ continue; } - MTRACE("Found Trezor device: " << desc.idVendor << ":" << desc.idProduct << " mask " << (int)trezor_mask); + MTRACE("Found Trezor device: " << desc.idVendor << ":" << desc.idProduct << " dev_idx " << (int)trezor_dev_idx); auto t = std::make_shared<WebUsbTransport>(boost::make_optional(&desc)); t->m_bus_id = libusb_get_bus_number(devs[i]); @@ -909,8 +966,8 @@ namespace trezor{ continue; } - const auto trezor_mask = get_trezor_dev_mask(&desc); - if (!trezor_mask) { + const auto trezor_dev_idx = get_trezor_dev_id(&desc); + if (!is_device_supported(trezor_dev_idx)){ continue; } @@ -921,7 +978,7 @@ namespace trezor{ get_libusb_ports(devs[i], path); MTRACE("Found Trezor device: " << desc.idVendor << ":" << desc.idProduct - << ", mask: " << (int)trezor_mask + << ", dev_idx: " << (int)trezor_dev_idx << ". path: " << get_usb_path(bus_id, path)); if (bus_id == m_bus_id && path == m_port_numbers) { @@ -1110,6 +1167,39 @@ namespace trezor{ #endif } + void sort_transports_by_env(t_transport_vect & res){ + const char *env_trezor_path = getenv("TREZOR_PATH"); + if (!env_trezor_path){ + return; + } + + // Sort transports by the longest matching prefix with TREZOR_PATH + std::string trezor_path(env_trezor_path); + std::vector<size_t> match_idx(res.size()); + std::vector<size_t> path_permutation(res.size()); + + for(size_t i = 0; i < res.size(); ++i){ + auto cpath = res[i]->get_path(); + std::string * s1 = &trezor_path; + std::string * s2 = &cpath; + + // first has to be shorter in std::mismatch(). Returns first non-matching iterators. + if (s1->size() >= s2->size()){ + std::swap(s1, s2); + } + + const auto mism = std::mismatch(s1->begin(), s1->end(), s2->begin()); + match_idx[i] = mism.first - s1->begin(); + path_permutation[i] = i; + } + + std::sort(path_permutation.begin(), path_permutation.end(), [&](const size_t i0, const size_t i1) { + return match_idx[i0] > match_idx[i1]; + }); + + tools::apply_permutation(path_permutation, res); + } + std::shared_ptr<Transport> transport(const std::string & path){ if (boost::starts_with(path, BridgeTransport::PATH_PREFIX)){ return std::make_shared<BridgeTransport>(path.substr(strlen(BridgeTransport::PATH_PREFIX))); diff --git a/src/device_trezor/trezor/transport.hpp b/src/device_trezor/trezor/transport.hpp index cde862547..affd91553 100644 --- a/src/device_trezor/trezor/transport.hpp +++ b/src/device_trezor/trezor/transport.hpp @@ -303,6 +303,11 @@ namespace trezor { void enumerate(t_transport_vect & res); /** + * Sorts found transports by TREZOR_PATH environment variable. + */ + void sort_transports_by_env(t_transport_vect & res); + + /** * Transforms path to the transport */ std::shared_ptr<Transport> transport(const std::string & path); diff --git a/src/lmdb/error.cpp b/src/lmdb/error.cpp index 359677064..91479521e 100644 --- a/src/lmdb/error.cpp +++ b/src/lmdb/error.cpp @@ -55,7 +55,7 @@ namespace { break; // map to nothing generic case MDB_PAGE_NOTFOUND: case MDB_CORRUPTED: - return std::errc::state_not_recoverable; + return std::errc::bad_address; case MDB_PANIC: case MDB_VERSION_MISMATCH: case MDB_INVALID: diff --git a/src/mnemonics/language_base.h b/src/mnemonics/language_base.h index 653314b04..7d2599e9a 100644 --- a/src/mnemonics/language_base.h +++ b/src/mnemonics/language_base.h @@ -129,7 +129,7 @@ namespace Language case 1: *wptr++ = cp; break;
case 2: *wptr++ = 0xc0 | (cp >> 6); *wptr++ = 0x80 | (cp & 0x3f); break;
case 3: *wptr++ = 0xe0 | (cp >> 12); *wptr++ = 0x80 | ((cp >> 6) & 0x3f); *wptr++ = 0x80 | (cp & 0x3f); break;
- case 4: *wptr++ = 0xf0 | (cp >> 18); *wptr += 0x80 | ((cp >> 12) & 0x3f); *wptr++ = 0x80 | ((cp >> 6) & 0x3f); *wptr++ = 0x80 | (cp & 0x3f); break;
+ case 4: *wptr++ = 0xf0 | (cp >> 18); *wptr++ = 0x80 | ((cp >> 12) & 0x3f); *wptr++ = 0x80 | ((cp >> 6) & 0x3f); *wptr++ = 0x80 | (cp & 0x3f); break;
default: throw std::runtime_error("Invalid UTF-8");
}
*wptr = 0;
diff --git a/src/net/error.h b/src/net/error.h index c8338f7e2..7c852dd20 100644 --- a/src/net/error.h +++ b/src/net/error.h @@ -42,7 +42,8 @@ namespace net invalid_i2p_address, invalid_port, //!< Outside of 0-65535 range invalid_tor_address,//!< Invalid base32 or length - unsupported_address //!< Type not supported by `get_network_address` + unsupported_address,//!< Type not supported by `get_network_address` + invalid_mask, //!< Outside of 0-32 range }; //! \return `std::error_category` for `net` namespace. diff --git a/src/net/parse.cpp b/src/net/parse.cpp index eaaadb67e..d93d7d352 100644 --- a/src/net/parse.cpp +++ b/src/net/parse.cpp @@ -58,4 +58,27 @@ namespace net return {epee::net_utils::ipv4_network_address{ip, port}}; return make_error_code(net::error::unsupported_address); } + + expect<epee::net_utils::ipv4_network_subnet> + get_ipv4_subnet_address(const boost::string_ref address, bool allow_implicit_32) + { + uint32_t mask = 32; + const boost::string_ref::size_type slash = address.find_first_of('/'); + if (slash != boost::string_ref::npos) + { + if (!epee::string_tools::get_xtype_from_string(mask, std::string{address.substr(slash + 1)})) + return make_error_code(net::error::invalid_mask); + if (mask > 32) + return make_error_code(net::error::invalid_mask); + } + else if (!allow_implicit_32) + return make_error_code(net::error::invalid_mask); + + std::uint32_t ip = 0; + boost::string_ref S(address.data(), slash != boost::string_ref::npos ? slash : address.size()); + if (!epee::string_tools::get_ip_int32_from_string(ip, std::string(S))) + return make_error_code(net::error::invalid_host); + + return {epee::net_utils::ipv4_network_subnet{ip, (uint8_t)mask}}; + } } diff --git a/src/net/parse.h b/src/net/parse.h index 5804c4128..9f0d66ea6 100644 --- a/src/net/parse.h +++ b/src/net/parse.h @@ -50,5 +50,18 @@ namespace net */ expect<epee::net_utils::network_address> get_network_address(boost::string_ref address, std::uint16_t default_port); + + /*! + Identifies an IPv4 subnet in CIDR notatioa and returns it as a generic + `network_address`. If the type is unsupported, it might be a hostname, + and `error() == net::error::kUnsupportedAddress` is returned. + + \param address An ipv4 address. + \param allow_implicit_32 whether to accept "raw" IPv4 addresses, with CIDR notation + + \return A tor or IPv4 address, else error. + */ + expect<epee::net_utils::ipv4_network_subnet> + get_ipv4_subnet_address(boost::string_ref address, bool allow_implicit_32 = false); } diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index 42bb3b061..9ee5ce0de 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -248,7 +248,11 @@ namespace nodetool void change_max_in_public_peers(size_t count); virtual bool block_host(const epee::net_utils::network_address &adress, time_t seconds = P2P_IP_BLOCKTIME); virtual bool unblock_host(const epee::net_utils::network_address &address); - virtual std::map<std::string, time_t> get_blocked_hosts() { CRITICAL_REGION_LOCAL(m_blocked_hosts_lock); return m_blocked_hosts; } + virtual bool block_subnet(const epee::net_utils::ipv4_network_subnet &subnet, time_t seconds = P2P_IP_BLOCKTIME); + virtual bool unblock_subnet(const epee::net_utils::ipv4_network_subnet &subnet); + virtual bool is_host_blocked(const epee::net_utils::network_address &address, time_t *seconds) { CRITICAL_REGION_LOCAL(m_blocked_hosts_lock); return !is_remote_host_allowed(address, seconds); } + virtual std::map<epee::net_utils::network_address, time_t> get_blocked_hosts() { CRITICAL_REGION_LOCAL(m_blocked_hosts_lock); return m_blocked_hosts; } + virtual std::map<epee::net_utils::ipv4_network_subnet, time_t> get_blocked_subnets() { CRITICAL_REGION_LOCAL(m_blocked_hosts_lock); return m_blocked_subnets; } virtual void add_used_stripe_peer(const typename t_payload_net_handler::connection_context &context); virtual void remove_used_stripe_peer(const typename t_payload_net_handler::connection_context &context); @@ -319,7 +323,7 @@ namespace nodetool virtual bool for_connection(const boost::uuids::uuid&, std::function<bool(typename t_payload_net_handler::connection_context&, peerid_type, uint32_t)> f); virtual bool add_host_fail(const epee::net_utils::network_address &address); //----------------- i_connection_filter -------------------------------------------------------- - virtual bool is_remote_host_allowed(const epee::net_utils::network_address &address); + virtual bool is_remote_host_allowed(const epee::net_utils::network_address &address, time_t *t = NULL); //----------------------------------------------------------------------------------------------- bool parse_peer_from_string(epee::net_utils::network_address& pe, const std::string& node_addr, uint16_t default_port = 0); bool handle_command_line( @@ -461,8 +465,9 @@ namespace nodetool std::map<epee::net_utils::network_address, time_t> m_conn_fails_cache; epee::critical_section m_conn_fails_cache_lock; - epee::critical_section m_blocked_hosts_lock; - std::map<std::string, time_t> m_blocked_hosts; + epee::critical_section m_blocked_hosts_lock; // for both hosts and subnets + std::map<epee::net_utils::network_address, time_t> m_blocked_hosts; + std::map<epee::net_utils::ipv4_network_subnet, time_t> m_blocked_subnets; epee::critical_section m_host_fails_score_lock; std::map<std::string, uint64_t> m_host_fails_score; diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index be97edbe5..ae51a63b2 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -155,19 +155,55 @@ namespace nodetool } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::is_remote_host_allowed(const epee::net_utils::network_address &address) + bool node_server<t_payload_net_handler>::is_remote_host_allowed(const epee::net_utils::network_address &address, time_t *t) { CRITICAL_REGION_LOCAL(m_blocked_hosts_lock); - auto it = m_blocked_hosts.find(address.host_str()); - if(it == m_blocked_hosts.end()) - return true; - if(time(nullptr) >= it->second) + + const time_t now = time(nullptr); + + // look in the hosts list + auto it = m_blocked_hosts.find(address); + if (it != m_blocked_hosts.end()) { - m_blocked_hosts.erase(it); - MCLOG_CYAN(el::Level::Info, "global", "Host " << address.host_str() << " unblocked."); - return true; + if (now >= it->second) + { + m_blocked_hosts.erase(it); + MCLOG_CYAN(el::Level::Info, "global", "Host " << address.host_str() << " unblocked."); + it = m_blocked_hosts.end(); + } + else + { + if (t) + *t = it->second - now; + return false; + } } - return false; + + // manually loop in subnets + if (address.get_type_id() == epee::net_utils::address_type::ipv4) + { + auto ipv4_address = address.template as<epee::net_utils::ipv4_network_address>(); + std::map<epee::net_utils::ipv4_network_subnet, time_t>::iterator it; + for (it = m_blocked_subnets.begin(); it != m_blocked_subnets.end(); ) + { + if (now >= it->second) + { + it = m_blocked_subnets.erase(it); + MCLOG_CYAN(el::Level::Info, "global", "Subnet " << it->first.host_str() << " unblocked."); + continue; + } + if (it->first.matches(ipv4_address)) + { + if (t) + *t = it->second - now; + return false; + } + ++it; + } + } + + // not found in hosts or subnets, allowed + return true; } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> @@ -184,7 +220,7 @@ namespace nodetool limit = std::numeric_limits<time_t>::max(); else limit = now + seconds; - m_blocked_hosts[addr.host_str()] = limit; + m_blocked_hosts[addr] = limit; // drop any connection to that address. This should only have to look into // the zone related to the connection, but really make sure everything is @@ -214,7 +250,7 @@ namespace nodetool bool node_server<t_payload_net_handler>::unblock_host(const epee::net_utils::network_address &address) { CRITICAL_REGION_LOCAL(m_blocked_hosts_lock); - auto i = m_blocked_hosts.find(address.host_str()); + auto i = m_blocked_hosts.find(address); if (i == m_blocked_hosts.end()) return false; m_blocked_hosts.erase(i); @@ -223,6 +259,58 @@ namespace nodetool } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> + bool node_server<t_payload_net_handler>::block_subnet(const epee::net_utils::ipv4_network_subnet &subnet, time_t seconds) + { + const time_t now = time(nullptr); + + CRITICAL_REGION_LOCAL(m_blocked_hosts_lock); + time_t limit; + if (now > std::numeric_limits<time_t>::max() - seconds) + limit = std::numeric_limits<time_t>::max(); + else + limit = now + seconds; + m_blocked_subnets[subnet] = limit; + + // drop any connection to that subnet. This should only have to look into + // the zone related to the connection, but really make sure everything is + // swept ... + std::vector<boost::uuids::uuid> conns; + for(auto& zone : m_network_zones) + { + zone.second.m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) + { + if (cntxt.m_remote_address.get_type_id() != epee::net_utils::ipv4_network_address::get_type_id()) + return true; + auto ipv4_address = cntxt.m_remote_address.template as<epee::net_utils::ipv4_network_address>(); + if (subnet.matches(ipv4_address)) + { + conns.push_back(cntxt.m_connection_id); + } + return true; + }); + for (const auto &c: conns) + zone.second.m_net_server.get_config_object().close(c); + + conns.clear(); + } + + MCLOG_CYAN(el::Level::Info, "global", "Subnet " << subnet.host_str() << " blocked."); + return true; + } + //----------------------------------------------------------------------------------- + template<class t_payload_net_handler> + bool node_server<t_payload_net_handler>::unblock_subnet(const epee::net_utils::ipv4_network_subnet &subnet) + { + CRITICAL_REGION_LOCAL(m_blocked_hosts_lock); + auto i = m_blocked_subnets.find(subnet); + if (i == m_blocked_subnets.end()) + return false; + m_blocked_subnets.erase(i); + MCLOG_CYAN(el::Level::Info, "global", "Subnet " << subnet.host_str() << " unblocked."); + return true; + } + //----------------------------------------------------------------------------------- + template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::add_host_fail(const epee::net_utils::network_address &address) { if(!address.is_blockable()) @@ -944,7 +1032,10 @@ namespace nodetool } if(!context.m_is_income) m_network_zones.at(context.m_remote_address.get_zone()).m_peerlist.set_peer_just_seen(context.peer_id, context.m_remote_address, context.m_pruning_seed, context.m_rpc_port); - m_payload_handler.process_payload_sync_data(rsp.payload_data, context, false); + if (!m_payload_handler.process_payload_sync_data(rsp.payload_data, context, false)) + { + m_network_zones.at(context.m_remote_address.get_zone()).m_net_server.get_config_object().close(context.m_connection_id ); + } }); if(!r) @@ -1090,6 +1181,7 @@ namespace nodetool LOG_PRINT_CC_PRIORITY_NODE(is_priority, *con, "Failed to HANDSHAKE with peer " << na.str() /*<< ", try " << try_count*/); + zone.m_net_server.get_config_object().close(con->m_connection_id); return false; } @@ -1149,7 +1241,7 @@ namespace nodetool bool is_priority = is_priority_node(na); LOG_PRINT_CC_PRIORITY_NODE(is_priority, *con, "Failed to HANDSHAKE with peer " << na.str()); - + zone.m_net_server.get_config_object().close(con->m_connection_id); return false; } @@ -1226,19 +1318,53 @@ namespace nodetool size_t random_index; const uint32_t next_needed_pruning_stripe = m_payload_handler.get_next_needed_pruning_stripe().second; + // build a set of all the /16 we're connected to, and prefer a peer that's not in that set + std::set<uint32_t> classB; + if (&zone == &m_network_zones.at(epee::net_utils::zone::public_)) // at returns reference, not copy + { + zone.m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) + { + if (cntxt.m_remote_address.get_type_id() == epee::net_utils::ipv4_network_address::get_type_id()) + { + + const epee::net_utils::network_address na = cntxt.m_remote_address; + const uint32_t actual_ip = na.as<const epee::net_utils::ipv4_network_address>().ip(); + classB.insert(actual_ip & 0x0000ffff); + } + return true; + }); + } + std::deque<size_t> filtered; const size_t limit = use_white_list ? 20 : std::numeric_limits<size_t>::max(); - size_t idx = 0; - zone.m_peerlist.foreach (use_white_list, [&filtered, &idx, limit, next_needed_pruning_stripe](const peerlist_entry &pe){ - if (filtered.size() >= limit) - return false; - if (next_needed_pruning_stripe == 0 || pe.pruning_seed == 0) - filtered.push_back(idx); - else if (next_needed_pruning_stripe == tools::get_pruning_stripe(pe.pruning_seed)) - filtered.push_front(idx); - ++idx; - return true; - }); + size_t idx = 0, skipped = 0; + for (int step = 0; step < 2; ++step) + { + bool skip_duplicate_class_B = step == 0; + zone.m_peerlist.foreach (use_white_list, [&classB, &filtered, &idx, &skipped, skip_duplicate_class_B, limit, next_needed_pruning_stripe](const peerlist_entry &pe){ + if (filtered.size() >= limit) + return false; + bool skip = false; + if (skip_duplicate_class_B && pe.adr.get_type_id() == epee::net_utils::ipv4_network_address::get_type_id()) + { + const epee::net_utils::network_address na = pe.adr; + uint32_t actual_ip = na.as<const epee::net_utils::ipv4_network_address>().ip(); + skip = classB.find(actual_ip & 0x0000ffff) != classB.end(); + } + if (skip) + ++skipped; + else if (next_needed_pruning_stripe == 0 || pe.pruning_seed == 0) + filtered.push_back(idx); + else if (next_needed_pruning_stripe == tools::get_pruning_stripe(pe.pruning_seed)) + filtered.push_front(idx); + ++idx; + return true; + }); + if (skipped == 0 || !filtered.empty()) + break; + if (skipped) + MGINFO("Skipping " << skipped << " possible peers as they share a class B with existing peers"); + } if (filtered.empty()) { MDEBUG("No available peer in " << (use_white_list ? "white" : "gray") << " list filtered by " << next_needed_pruning_stripe); @@ -1955,7 +2081,7 @@ namespace nodetool const epee::net_utils::zone zone_type = context.m_remote_address.get_zone(); network_zone& zone = m_network_zones.at(zone_type); - zone.m_peerlist.get_peerlist_head(rsp.local_peerlist_new); + zone.m_peerlist.get_peerlist_head(rsp.local_peerlist_new, true); m_payload_handler.get_payload_sync_data(rsp.payload_data); /* Tor/I2P nodes receiving connections via forwarding (from tor/i2p daemon) @@ -2058,7 +2184,7 @@ namespace nodetool }); //fill response - zone.m_peerlist.get_peerlist_head(rsp.local_peerlist_new); + zone.m_peerlist.get_peerlist_head(rsp.local_peerlist_new, true); get_local_node_data(rsp.node_data, zone); m_payload_handler.get_payload_sync_data(rsp.payload_data); LOG_DEBUG_CC(context, "COMMAND_HANDSHAKE"); diff --git a/src/p2p/net_node_common.h b/src/p2p/net_node_common.h index 26451b333..34d151f5f 100644 --- a/src/p2p/net_node_common.h +++ b/src/p2p/net_node_common.h @@ -56,7 +56,8 @@ namespace nodetool virtual bool for_connection(const boost::uuids::uuid&, std::function<bool(t_connection_context&, peerid_type, uint32_t)> f)=0; virtual bool block_host(const epee::net_utils::network_address &address, time_t seconds = 0)=0; virtual bool unblock_host(const epee::net_utils::network_address &address)=0; - virtual std::map<std::string, time_t> get_blocked_hosts()=0; + virtual std::map<epee::net_utils::network_address, time_t> get_blocked_hosts()=0; + virtual std::map<epee::net_utils::ipv4_network_subnet, time_t> get_blocked_subnets()=0; virtual bool add_host_fail(const epee::net_utils::network_address &address)=0; virtual void add_used_stripe_peer(const t_connection_context &context)=0; virtual void remove_used_stripe_peer(const t_connection_context &context)=0; @@ -112,9 +113,13 @@ namespace nodetool { return true; } - virtual std::map<std::string, time_t> get_blocked_hosts() + virtual std::map<epee::net_utils::network_address, time_t> get_blocked_hosts() { - return std::map<std::string, time_t>(); + return std::map<epee::net_utils::network_address, time_t>(); + } + virtual std::map<epee::net_utils::ipv4_network_subnet, time_t> get_blocked_subnets() + { + return std::map<epee::net_utils::ipv4_network_subnet, time_t>(); } virtual bool add_host_fail(const epee::net_utils::network_address &address) { diff --git a/src/p2p/net_peerlist.h b/src/p2p/net_peerlist.h index 52814af94..883997fd6 100644 --- a/src/p2p/net_peerlist.h +++ b/src/p2p/net_peerlist.h @@ -102,7 +102,7 @@ namespace nodetool size_t get_white_peers_count(){CRITICAL_REGION_LOCAL(m_peerlist_lock); return m_peers_white.size();} size_t get_gray_peers_count(){CRITICAL_REGION_LOCAL(m_peerlist_lock); return m_peers_gray.size();} bool merge_peerlist(const std::vector<peerlist_entry>& outer_bs); - bool get_peerlist_head(std::vector<peerlist_entry>& bs_head, uint32_t depth = P2P_DEFAULT_PEERS_IN_HANDSHAKE); + bool get_peerlist_head(std::vector<peerlist_entry>& bs_head, bool anonymize, uint32_t depth = P2P_DEFAULT_PEERS_IN_HANDSHAKE); void get_peerlist(std::vector<peerlist_entry>& pl_gray, std::vector<peerlist_entry>& pl_white); void get_peerlist(peerlist_types& peers); bool get_white_peer_by_index(peerlist_entry& p, size_t i); @@ -263,23 +263,40 @@ namespace nodetool } //-------------------------------------------------------------------------------------------------- inline - bool peerlist_manager::get_peerlist_head(std::vector<peerlist_entry>& bs_head, uint32_t depth) + bool peerlist_manager::get_peerlist_head(std::vector<peerlist_entry>& bs_head, bool anonymize, uint32_t depth) { - CRITICAL_REGION_LOCAL(m_peerlist_lock); peers_indexed::index<by_time>::type& by_time_index=m_peers_white.get<by_time>(); uint32_t cnt = 0; - bs_head.reserve(depth); + + // picks a random set of peers within the first 120%, rather than a set of the first 100%. + // The intent is that if someone asks twice, they can't easily tell: + // - this address was not in the first list, but is in the second, so the only way this can be + // is if its last_seen was recently reset, so this means the target node recently had a new + // connection to that address + // - this address was in the first list, and not in the second, which means either the address + // was moved to the gray list (if it's not accessibe, which the attacker can check if + // the address accepts incoming connections) or it was the oldest to still fit in the 250 items, + // so its last_seen is old. + const uint32_t pick_depth = anonymize ? depth + depth / 5 : depth; + bs_head.reserve(pick_depth); for(const peers_indexed::value_type& vl: boost::adaptors::reverse(by_time_index)) { - if(!vl.last_seen) - continue; - - if(cnt++ >= depth) + if(cnt++ >= pick_depth) break; bs_head.push_back(vl); } + + if (anonymize) + { + std::random_shuffle(bs_head.begin(), bs_head.end()); + if (bs_head.size() > depth) + bs_head.resize(depth); + for (auto &e: bs_head) + e.last_seen = 0; + } + return true; } //-------------------------------------------------------------------------------------------------- @@ -327,8 +344,14 @@ namespace nodetool trim_white_peerlist(); }else { - //update record in white list - m_peers_white.replace(by_addr_it_wt, ple); + //update record in white list + peerlist_entry new_ple = ple; + if (by_addr_it_wt->pruning_seed && ple.pruning_seed == 0) // guard against older nodes not passing pruning info around + new_ple.pruning_seed = by_addr_it_wt->pruning_seed; + if (by_addr_it_wt->rpc_port && ple.rpc_port == 0) // guard against older nodes not passing RPC port around + new_ple.rpc_port = by_addr_it_wt->rpc_port; + new_ple.last_seen = by_addr_it_wt->last_seen; // do not overwrite the last seen timestamp, incoming peer list are untrusted + m_peers_white.replace(by_addr_it_wt, new_ple); } //remove from gray list, if need auto by_addr_it_gr = m_peers_gray.get<by_addr>().find(ple.adr); @@ -362,8 +385,14 @@ namespace nodetool trim_gray_peerlist(); }else { - //update record in white list - m_peers_gray.replace(by_addr_it_gr, ple); + //update record in gray list + peerlist_entry new_ple = ple; + if (by_addr_it_gr->pruning_seed && ple.pruning_seed == 0) // guard against older nodes not passing pruning info around + new_ple.pruning_seed = by_addr_it_gr->pruning_seed; + if (by_addr_it_gr->rpc_port && ple.rpc_port == 0) // guard against older nodes not passing RPC port around + new_ple.rpc_port = by_addr_it_gr->rpc_port; + new_ple.last_seen = by_addr_it_gr->last_seen; // do not overwrite the last seen timestamp, incoming peer list are untrusted + m_peers_gray.replace(by_addr_it_gr, new_ple); } return true; CATCH_ENTRY_L0("peerlist_manager::append_with_peer_gray()", false); diff --git a/src/p2p/net_peerlist_boost_serialization.h b/src/p2p/net_peerlist_boost_serialization.h index 40ef2ebcd..32f30adca 100644 --- a/src/p2p/net_peerlist_boost_serialization.h +++ b/src/p2p/net_peerlist_boost_serialization.h @@ -134,10 +134,11 @@ namespace boost a & port; a & length; - if (length > net::tor_address::buffer_size()) + const size_t buffer_size = net::tor_address::buffer_size(); + if (length > buffer_size) MONERO_THROW(net::error::invalid_tor_address, "Tor address too long"); - char host[net::tor_address::buffer_size()] = {0}; + char host[buffer_size] = {0}; a.load_binary(host, length); host[sizeof(host) - 1] = 0; @@ -155,10 +156,11 @@ namespace boost a & port; a & length; - if (length > net::i2p_address::buffer_size()) + const size_t buffer_size = net::i2p_address::buffer_size(); + if (length > buffer_size) MONERO_THROW(net::error::invalid_i2p_address, "i2p address too long"); - char host[net::i2p_address::buffer_size()] = {0}; + char host[buffer_size] = {0}; a.load_binary(host, length); host[sizeof(host) - 1] = 0; diff --git a/src/p2p/p2p_protocol_defs.h b/src/p2p/p2p_protocol_defs.h index 59c6099d5..85774fcd5 100644 --- a/src/p2p/p2p_protocol_defs.h +++ b/src/p2p/p2p_protocol_defs.h @@ -81,7 +81,8 @@ namespace nodetool BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(adr) KV_SERIALIZE(id) - KV_SERIALIZE(last_seen) + if (!is_store || this_ref.last_seen != 0) + KV_SERIALIZE_OPT(last_seen, (int64_t)0) KV_SERIALIZE_OPT(pruning_seed, (uint32_t)0) KV_SERIALIZE_OPT(rpc_port, (uint16_t)0) END_KV_SERIALIZE_MAP() @@ -132,7 +133,7 @@ namespace nodetool ss << pe.id << "\t" << pe.adr.str() << " \trpc port " << (pe.rpc_port > 0 ? std::to_string(pe.rpc_port) : "-") << " \tpruning seed " << pe.pruning_seed - << " \tlast_seen: " << epee::misc_utils::get_time_interval_string(now_time - pe.last_seen) + << " \tlast_seen: " << (pe.last_seen == 0 ? std::string("never") : epee::misc_utils::get_time_interval_string(now_time - pe.last_seen)) << std::endl; } return ss.str(); diff --git a/src/ringct/rctTypes.h b/src/ringct/rctTypes.h index e5413f1dc..f8729b872 100644 --- a/src/ringct/rctTypes.h +++ b/src/ringct/rctTypes.h @@ -252,7 +252,7 @@ namespace rct { { FIELD(type) if (type == RCTTypeNull) - return true; + return ar.stream().good(); if (type != RCTTypeFull && type != RCTTypeSimple && type != RCTTypeBulletproof && type != RCTTypeBulletproof2) return false; VARINT_FIELD(txnFee) @@ -312,7 +312,7 @@ namespace rct { ar.delimit_array(); } ar.end_array(); - return true; + return ar.stream().good(); } }; struct rctSigPrunable { @@ -325,7 +325,7 @@ namespace rct { bool serialize_rctsig_prunable(Archive<W> &ar, uint8_t type, size_t inputs, size_t outputs, size_t mixin) { if (type == RCTTypeNull) - return true; + return ar.stream().good(); if (type != RCTTypeFull && type != RCTTypeSimple && type != RCTTypeBulletproof && type != RCTTypeBulletproof2) return false; if (type == RCTTypeBulletproof || type == RCTTypeBulletproof2) @@ -429,7 +429,7 @@ namespace rct { } ar.end_array(); } - return true; + return ar.stream().good(); } }; diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 4c73a8bb3..82da06bc5 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -28,6 +28,7 @@ // // Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers +#include <boost/preprocessor/stringize.hpp> #include "include_base_utils.h" #include "string_tools.h" using namespace epee; @@ -58,6 +59,8 @@ using namespace epee; #define MAX_RESTRICTED_FAKE_OUTS_COUNT 40 #define MAX_RESTRICTED_GLOBAL_FAKE_OUTS_COUNT 5000 +#define OUTPUT_HISTOGRAM_RECENT_CUTOFF_RESTRICTION (3 * 86400) // 3 days max, the wallet requests 1.8 days + namespace { void add_reason(std::string &reasons, const char *reason) @@ -74,9 +77,9 @@ namespace void store_difficulty(cryptonote::difficulty_type difficulty, uint64_t &sdiff, std::string &swdiff, uint64_t &stop64) { - sdiff = (difficulty << 64 >> 64).convert_to<uint64_t>(); + sdiff = (difficulty & 0xffffffffffffffff).convert_to<uint64_t>(); swdiff = cryptonote::hex(difficulty); - stop64 = (difficulty >> 64).convert_to<uint64_t>(); + stop64 = ((difficulty >> 64) & 0xffffffffffffffff).convert_to<uint64_t>(); } } @@ -89,15 +92,9 @@ namespace cryptonote command_line::add_arg(desc, arg_rpc_bind_port); command_line::add_arg(desc, arg_rpc_restricted_bind_port); command_line::add_arg(desc, arg_restricted_rpc); - command_line::add_arg(desc, arg_rpc_ssl); - command_line::add_arg(desc, arg_rpc_ssl_private_key); - command_line::add_arg(desc, arg_rpc_ssl_certificate); - command_line::add_arg(desc, arg_rpc_ssl_ca_certificates); - command_line::add_arg(desc, arg_rpc_ssl_allowed_fingerprints); - command_line::add_arg(desc, arg_rpc_ssl_allow_any_cert); command_line::add_arg(desc, arg_bootstrap_daemon_address); command_line::add_arg(desc, arg_bootstrap_daemon_login); - cryptonote::rpc_args::init_options(desc); + cryptonote::rpc_args::init_options(desc, true); } //------------------------------------------------------------------------------------------------------------------------------ core_rpc_server::core_rpc_server( @@ -117,7 +114,7 @@ namespace cryptonote m_restricted = restricted; m_net_server.set_threads_prefix("RPC"); - auto rpc_config = cryptonote::rpc_args::process(vm); + auto rpc_config = cryptonote::rpc_args::process(vm, true); if (!rpc_config) return false; @@ -150,38 +147,9 @@ namespace cryptonote if (rpc_config->login) http_login.emplace(std::move(rpc_config->login->username), std::move(rpc_config->login->password).password()); - epee::net_utils::ssl_options_t ssl_options = epee::net_utils::ssl_support_t::e_ssl_support_autodetect; - if (command_line::get_arg(vm, arg_rpc_ssl_allow_any_cert)) - ssl_options.verification = epee::net_utils::ssl_verification_t::none; - else - { - std::string ssl_ca_path = command_line::get_arg(vm, arg_rpc_ssl_ca_certificates); - const std::vector<std::string> ssl_allowed_fingerprint_strings = command_line::get_arg(vm, arg_rpc_ssl_allowed_fingerprints); - std::vector<std::vector<uint8_t>> ssl_allowed_fingerprints{ ssl_allowed_fingerprint_strings.size() }; - std::transform(ssl_allowed_fingerprint_strings.begin(), ssl_allowed_fingerprint_strings.end(), ssl_allowed_fingerprints.begin(), epee::from_hex::vector); - - if (!ssl_ca_path.empty() || !ssl_allowed_fingerprints.empty()) - ssl_options = epee::net_utils::ssl_options_t{std::move(ssl_allowed_fingerprints), std::move(ssl_ca_path)}; - } - - ssl_options.auth = epee::net_utils::ssl_authentication_t{ - command_line::get_arg(vm, arg_rpc_ssl_private_key), command_line::get_arg(vm, arg_rpc_ssl_certificate) - }; - - // user specified CA file or fingeprints implies enabled SSL by default - if (ssl_options.verification != epee::net_utils::ssl_verification_t::user_certificates || !command_line::is_arg_defaulted(vm, arg_rpc_ssl)) - { - const std::string ssl = command_line::get_arg(vm, arg_rpc_ssl); - if (!epee::net_utils::ssl_support_from_string(ssl_options.support, ssl)) - { - MFATAL("Invalid RPC SSL support: " << ssl); - return false; - } - } - auto rng = [](size_t len, uint8_t *ptr){ return crypto::rand(len, ptr); }; return epee::http_server_impl_base<core_rpc_server, connection_context>::init( - rng, std::move(port), std::move(rpc_config->bind_ip), std::move(rpc_config->access_control_origins), std::move(http_login), std::move(ssl_options) + rng, std::move(port), std::move(rpc_config->bind_ip), std::move(rpc_config->access_control_origins), std::move(http_login), std::move(rpc_config->ssl_options) ); } //------------------------------------------------------------------------------------------------------------------------------ @@ -852,6 +820,7 @@ namespace cryptonote res.sanity_check_failed = true; return true; } + res.sanity_check_failed = false; cryptonote_connection_context fake_context = AUTO_VAL_INIT(fake_context); tx_verification_context tvc = AUTO_VAL_INIT(tvc); @@ -939,16 +908,13 @@ namespace cryptonote return true; } - boost::thread::attributes attrs; - attrs.set_stack_size(THREAD_STACK_SIZE); - cryptonote::miner &miner= m_core.get_miner(); if (miner.is_mining()) { res.status = "Already mining"; return true; } - if(!miner.start(info.address, static_cast<size_t>(req.threads_count), attrs, req.do_background_mining, req.ignore_battery)) + if(!miner.start(info.address, static_cast<size_t>(req.threads_count), req.do_background_mining, req.ignore_battery)) { res.status = "Failed, mining not started"; LOG_PRINT_L0(res.status); @@ -1806,20 +1772,60 @@ namespace cryptonote PERF_TIMER(on_get_bans); auto now = time(nullptr); - std::map<std::string, time_t> blocked_hosts = m_p2p.get_blocked_hosts(); - for (std::map<std::string, time_t>::const_iterator i = blocked_hosts.begin(); i != blocked_hosts.end(); ++i) + std::map<epee::net_utils::network_address, time_t> blocked_hosts = m_p2p.get_blocked_hosts(); + for (std::map<epee::net_utils::network_address, time_t>::const_iterator i = blocked_hosts.begin(); i != blocked_hosts.end(); ++i) { if (i->second > now) { COMMAND_RPC_GETBANS::ban b; - b.host = i->first; + b.host = i->first.host_str(); b.ip = 0; uint32_t ip; - if (epee::string_tools::get_ip_int32_from_string(ip, i->first)) + if (epee::string_tools::get_ip_int32_from_string(ip, b.host)) b.ip = ip; b.seconds = i->second - now; res.bans.push_back(b); } } + std::map<epee::net_utils::ipv4_network_subnet, time_t> blocked_subnets = m_p2p.get_blocked_subnets(); + for (std::map<epee::net_utils::ipv4_network_subnet, time_t>::const_iterator i = blocked_subnets.begin(); i != blocked_subnets.end(); ++i) + { + if (i->second > now) { + COMMAND_RPC_GETBANS::ban b; + b.host = i->first.host_str(); + b.ip = 0; + b.seconds = i->second - now; + res.bans.push_back(b); + } + } + + res.status = CORE_RPC_STATUS_OK; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_banned(const COMMAND_RPC_BANNED::request& req, COMMAND_RPC_BANNED::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) + { + PERF_TIMER(on_banned); + + auto na_parsed = net::get_network_address(req.address, 0); + if (!na_parsed) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_PARAM; + error_resp.message = "Unsupported host type"; + return false; + } + epee::net_utils::network_address na = std::move(*na_parsed); + + time_t seconds; + if (m_p2p.is_host_blocked(na, &seconds)) + { + res.banned = true; + res.seconds = seconds; + } + else + { + res.banned = false; + res.seconds = 0; + } res.status = CORE_RPC_STATUS_OK; return true; @@ -1832,13 +1838,29 @@ namespace cryptonote for (auto i = req.bans.begin(); i != req.bans.end(); ++i) { epee::net_utils::network_address na; + + // try subnet first + if (!i->host.empty()) + { + auto ns_parsed = net::get_ipv4_subnet_address(i->host); + if (ns_parsed) + { + if (i->ban) + m_p2p.block_subnet(*ns_parsed, i->seconds); + else + m_p2p.unblock_subnet(*ns_parsed); + continue; + } + } + + // then host if (!i->host.empty()) { auto na_parsed = net::get_network_address(i->host, 0); if (!na_parsed) { error_resp.code = CORE_RPC_ERROR_CODE_WRONG_PARAM; - error_resp.message = "Unsupported host type"; + error_resp.message = "Unsupported host/subnet type"; return false; } na = std::move(*na_parsed); @@ -1919,6 +1941,13 @@ namespace cryptonote if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_OUTPUT_HISTOGRAM>(invoke_http_mode::JON_RPC, "get_output_histogram", req, res, r)) return r; + const bool restricted = m_restricted && ctx; + if (restricted && req.recent_cutoff > 0 && req.recent_cutoff < (uint64_t)time(NULL) - OUTPUT_HISTOGRAM_RECENT_CUTOFF_RESTRICTION) + { + res.status = "Recent cutoff is too old"; + return true; + } + std::map<uint64_t, std::tuple<uint64_t, uint64_t, uint64_t>> histogram; try { @@ -2098,6 +2127,13 @@ namespace cryptonote bool core_rpc_server::on_update(const COMMAND_RPC_UPDATE::request& req, COMMAND_RPC_UPDATE::response& res, const connection_context *ctx) { PERF_TIMER(on_update); + + if (m_core.offline()) + { + res.status = "Daemon is running offline"; + return true; + } + static const char software[] = "monero"; #ifdef BUILD_TAG static const char buildtag[] = BOOST_PP_STRINGIZE(BUILD_TAG); @@ -2308,7 +2344,7 @@ namespace cryptonote const uint64_t req_to_height = req.to_height ? req.to_height : (m_core.get_current_blockchain_height() - 1); for (uint64_t amount: req.amounts) { - auto data = rpc::RpcHandler::get_output_distribution([this](uint64_t amount, uint64_t from, uint64_t to, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) { return m_core.get_output_distribution(amount, from, to, start_height, distribution, base); }, amount, req.from_height, req_to_height, req.cumulative); + auto data = rpc::RpcHandler::get_output_distribution([this](uint64_t amount, uint64_t from, uint64_t to, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) { return m_core.get_output_distribution(amount, from, to, start_height, distribution, base); }, amount, req.from_height, req_to_height, [this](uint64_t height) { return m_core.get_blockchain_storage().get_db().get_block_hash_from_height(height); }, req.cumulative, m_core.get_current_blockchain_height()); if (!data) { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; @@ -2351,7 +2387,7 @@ namespace cryptonote const uint64_t req_to_height = req.to_height ? req.to_height : (m_core.get_current_blockchain_height() - 1); for (uint64_t amount: req.amounts) { - auto data = rpc::RpcHandler::get_output_distribution([this](uint64_t amount, uint64_t from, uint64_t to, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) { return m_core.get_output_distribution(amount, from, to, start_height, distribution, base); }, amount, req.from_height, req_to_height, req.cumulative); + auto data = rpc::RpcHandler::get_output_distribution([this](uint64_t amount, uint64_t from, uint64_t to, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) { return m_core.get_output_distribution(amount, from, to, start_height, distribution, base); }, amount, req.from_height, req_to_height, [this](uint64_t height) { return m_core.get_blockchain_storage().get_db().get_block_hash_from_height(height); }, req.cumulative, m_core.get_current_blockchain_height()); if (!data) { res.status = "Failed to get output distribution"; @@ -2423,40 +2459,6 @@ namespace cryptonote , false }; - const command_line::arg_descriptor<std::string> core_rpc_server::arg_rpc_ssl = { - "rpc-ssl" - , "Enable SSL on RPC connections: enabled|disabled|autodetect" - , "autodetect" - }; - - const command_line::arg_descriptor<std::string> core_rpc_server::arg_rpc_ssl_private_key = { - "rpc-ssl-private-key" - , "Path to a PEM format private key" - , "" - }; - - const command_line::arg_descriptor<std::string> core_rpc_server::arg_rpc_ssl_certificate = { - "rpc-ssl-certificate" - , "Path to a PEM format certificate" - , "" - }; - - const command_line::arg_descriptor<std::string> core_rpc_server::arg_rpc_ssl_ca_certificates = { - "rpc-ssl-ca-certificates" - , "Path to file containing concatenated PEM format certificate(s) to replace system CA(s)." - }; - - const command_line::arg_descriptor<std::vector<std::string>> core_rpc_server::arg_rpc_ssl_allowed_fingerprints = { - "rpc-ssl-allowed-fingerprints" - , "List of certificate fingerprints to allow" - }; - - const command_line::arg_descriptor<bool> core_rpc_server::arg_rpc_ssl_allow_any_cert = { - "rpc-ssl-allow-any-cert" - , "Allow any peer certificate" - , false - }; - const command_line::arg_descriptor<std::string> core_rpc_server::arg_bootstrap_daemon_address = { "bootstrap-daemon-address" , "URL of a 'bootstrap' remote daemon that the connected wallets can use while this daemon is still not fully synced" diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index e4683bbe2..266661fb0 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -154,6 +154,7 @@ namespace cryptonote MAP_JON_RPC_WE("hard_fork_info", on_hard_fork_info, COMMAND_RPC_HARD_FORK_INFO) MAP_JON_RPC_WE_IF("set_bans", on_set_bans, COMMAND_RPC_SETBANS, !m_restricted) MAP_JON_RPC_WE_IF("get_bans", on_get_bans, COMMAND_RPC_GETBANS, !m_restricted) + MAP_JON_RPC_WE_IF("banned", on_banned, COMMAND_RPC_BANNED, !m_restricted) MAP_JON_RPC_WE_IF("flush_txpool", on_flush_txpool, COMMAND_RPC_FLUSH_TRANSACTION_POOL, !m_restricted) MAP_JON_RPC_WE("get_output_histogram", on_get_output_histogram, COMMAND_RPC_GET_OUTPUT_HISTOGRAM) MAP_JON_RPC_WE("get_version", on_get_version, COMMAND_RPC_GET_VERSION) @@ -220,6 +221,7 @@ namespace cryptonote bool on_hard_fork_info(const COMMAND_RPC_HARD_FORK_INFO::request& req, COMMAND_RPC_HARD_FORK_INFO::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_set_bans(const COMMAND_RPC_SETBANS::request& req, COMMAND_RPC_SETBANS::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_get_bans(const COMMAND_RPC_GETBANS::request& req, COMMAND_RPC_GETBANS::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_banned(const COMMAND_RPC_BANNED::request& req, COMMAND_RPC_BANNED::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_flush_txpool(const COMMAND_RPC_FLUSH_TRANSACTION_POOL::request& req, COMMAND_RPC_FLUSH_TRANSACTION_POOL::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_get_output_histogram(const COMMAND_RPC_GET_OUTPUT_HISTOGRAM::request& req, COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_get_version(const COMMAND_RPC_GET_VERSION::request& req, COMMAND_RPC_GET_VERSION::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index cfe4bbf23..a78faf5aa 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -84,7 +84,7 @@ namespace cryptonote // advance which version they will stop working with // Don't go over 32767 for any of these #define CORE_RPC_VERSION_MAJOR 2 -#define CORE_RPC_VERSION_MINOR 6 +#define CORE_RPC_VERSION_MINOR 7 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR) @@ -1876,6 +1876,33 @@ namespace cryptonote typedef epee::misc_utils::struct_init<response_t> response; }; + struct COMMAND_RPC_BANNED + { + struct request_t + { + std::string address; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(address) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<request_t> request; + + struct response_t + { + std::string status; + bool banned; + uint32_t seconds; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + KV_SERIALIZE(banned) + KV_SERIALIZE(seconds) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<response_t> response; + }; + struct COMMAND_RPC_FLUSH_TRANSACTION_POOL { struct request_t diff --git a/src/rpc/daemon_handler.cpp b/src/rpc/daemon_handler.cpp index 7c8953930..612b2cab6 100644 --- a/src/rpc/daemon_handler.cpp +++ b/src/rpc/daemon_handler.cpp @@ -408,10 +408,7 @@ namespace rpc return; } - boost::thread::attributes attrs; - attrs.set_stack_size(THREAD_STACK_SIZE); - - if(!m_core.get_miner().start(info.address, static_cast<size_t>(req.threads_count), attrs, req.do_background_mining, req.ignore_battery)) + if(!m_core.get_miner().start(info.address, static_cast<size_t>(req.threads_count), req.do_background_mining, req.ignore_battery)) { res.error_details = "Failed, mining not started"; LOG_PRINT_L0(res.error_details); @@ -437,7 +434,7 @@ namespace rpc auto& chain = m_core.get_blockchain_storage(); res.info.wide_difficulty = chain.get_difficulty_for_next_block(); - res.info.difficulty = (res.info.wide_difficulty << 64 >> 64).convert_to<uint64_t>(); + res.info.difficulty = (res.info.wide_difficulty & 0xffffffffffffffff).convert_to<uint64_t>(); res.info.target = chain.get_difficulty_target(); @@ -459,7 +456,7 @@ namespace rpc res.info.testnet = m_core.get_nettype() == TESTNET; res.info.stagenet = m_core.get_nettype() == STAGENET; res.info.wide_cumulative_difficulty = m_core.get_blockchain_storage().get_db().get_block_cumulative_difficulty(res.info.height - 1); - res.info.cumulative_difficulty = (res.info.wide_cumulative_difficulty << 64 >> 64).convert_to<uint64_t>(); + res.info.cumulative_difficulty = (res.info.wide_cumulative_difficulty & 0xffffffffffffffff).convert_to<uint64_t>(); res.info.block_size_limit = res.info.block_weight_limit = m_core.get_blockchain_storage().get_current_cumulative_block_weight_limit(); res.info.block_size_median = res.info.block_weight_median = m_core.get_blockchain_storage().get_current_cumulative_block_weight_median(); res.info.start_time = (uint64_t)m_core.get_start_time(); @@ -778,7 +775,7 @@ namespace rpc const uint64_t req_to_height = req.to_height ? req.to_height : (m_core.get_current_blockchain_height() - 1); for (std::uint64_t amount : req.amounts) { - auto data = rpc::RpcHandler::get_output_distribution([this](uint64_t amount, uint64_t from, uint64_t to, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) { return m_core.get_output_distribution(amount, from, to, start_height, distribution, base); }, amount, req.from_height, req_to_height, req.cumulative); + auto data = rpc::RpcHandler::get_output_distribution([this](uint64_t amount, uint64_t from, uint64_t to, uint64_t &start_height, std::vector<uint64_t> &distribution, uint64_t &base) { return m_core.get_output_distribution(amount, from, to, start_height, distribution, base); }, amount, req.from_height, req_to_height, [this](uint64_t height) { return m_core.get_blockchain_storage().get_db().get_block_hash_from_height(height); }, req.cumulative, m_core.get_current_blockchain_height()); if (!data) { res.distributions.clear(); @@ -829,7 +826,7 @@ namespace rpc } header.wide_difficulty = m_core.get_blockchain_storage().block_difficulty(header.height); - header.difficulty = (header.wide_difficulty << 64 >> 64).convert_to<uint64_t>(); + header.difficulty = (header.wide_difficulty & 0xffffffffffffffff).convert_to<uint64_t>(); return true; } diff --git a/src/rpc/rpc_args.cpp b/src/rpc/rpc_args.cpp index f2be94f51..4479bd1f1 100644 --- a/src/rpc/rpc_args.cpp +++ b/src/rpc/rpc_args.cpp @@ -33,28 +33,95 @@ #include <boost/bind.hpp> #include "common/command_line.h" #include "common/i18n.h" +#include "hex.h" namespace cryptonote { + namespace + { + boost::optional<epee::net_utils::ssl_options_t> do_process_ssl(const boost::program_options::variables_map& vm, const rpc_args::descriptors& arg, const bool any_cert_option) + { + bool ssl_required = false; + epee::net_utils::ssl_options_t ssl_options = epee::net_utils::ssl_support_t::e_ssl_support_enabled; + if (any_cert_option && command_line::get_arg(vm, arg.rpc_ssl_allow_any_cert)) + ssl_options.verification = epee::net_utils::ssl_verification_t::none; + else + { + std::string ssl_ca_file = command_line::get_arg(vm, arg.rpc_ssl_ca_certificates); + const std::vector<std::string> ssl_allowed_fingerprints = command_line::get_arg(vm, arg.rpc_ssl_allowed_fingerprints); + + std::vector<std::vector<uint8_t>> allowed_fingerprints{ ssl_allowed_fingerprints.size() }; + std::transform(ssl_allowed_fingerprints.begin(), ssl_allowed_fingerprints.end(), allowed_fingerprints.begin(), epee::from_hex::vector); + for (const auto &fpr: allowed_fingerprints) + { + if (fpr.size() != SSL_FINGERPRINT_SIZE) + { + MERROR("SHA-256 fingerprint should be " BOOST_PP_STRINGIZE(SSL_FINGERPRINT_SIZE) " bytes long."); + return boost::none; + } + } + + if (!allowed_fingerprints.empty() || !ssl_ca_file.empty()) + { + ssl_required = true; + ssl_options = epee::net_utils::ssl_options_t{ + std::move(allowed_fingerprints), std::move(ssl_ca_file) + }; + + if (command_line::get_arg(vm, arg.rpc_ssl_allow_chained)) + ssl_options.verification = epee::net_utils::ssl_verification_t::user_ca; + } + } + + // user specified CA file or fingeprints implies enabled SSL by default + if (!ssl_required && !epee::net_utils::ssl_support_from_string(ssl_options.support, command_line::get_arg(vm, arg.rpc_ssl))) + { + MERROR("Invalid argument for " << std::string(arg.rpc_ssl.name)); + return boost::none; + } + + ssl_options.auth = epee::net_utils::ssl_authentication_t{ + command_line::get_arg(vm, arg.rpc_ssl_private_key), command_line::get_arg(vm, arg.rpc_ssl_certificate) + }; + + return {std::move(ssl_options)}; + } + } // anonymous + rpc_args::descriptors::descriptors() : rpc_bind_ip({"rpc-bind-ip", rpc_args::tr("Specify IP to bind RPC server"), "127.0.0.1"}) , rpc_login({"rpc-login", rpc_args::tr("Specify username[:password] required for RPC server"), "", true}) , confirm_external_bind({"confirm-external-bind", rpc_args::tr("Confirm rpc-bind-ip value is NOT a loopback (local) IP")}) , rpc_access_control_origins({"rpc-access-control-origins", rpc_args::tr("Specify a comma separated list of origins to allow cross origin resource sharing"), ""}) + , rpc_ssl({"rpc-ssl", rpc_args::tr("Enable SSL on RPC connections: enabled|disabled|autodetect"), "autodetect"}) + , rpc_ssl_private_key({"rpc-ssl-private-key", rpc_args::tr("Path to a PEM format private key"), ""}) + , rpc_ssl_certificate({"rpc-ssl-certificate", rpc_args::tr("Path to a PEM format certificate"), ""}) + , rpc_ssl_ca_certificates({"rpc-ssl-ca-certificates", rpc_args::tr("Path to file containing concatenated PEM format certificate(s) to replace system CA(s)."), ""}) + , rpc_ssl_allowed_fingerprints({"rpc-ssl-allowed-fingerprints", rpc_args::tr("List of certificate fingerprints to allow")}) + , rpc_ssl_allow_chained({"rpc-ssl-allow-chained", rpc_args::tr("Allow user (via --rpc-ssl-certificates) chain certificates"), false}) + , rpc_ssl_allow_any_cert({"rpc-ssl-allow-any-cert", rpc_args::tr("Allow any peer certificate"), false}) {} const char* rpc_args::tr(const char* str) { return i18n_translate(str, "cryptonote::rpc_args"); } - void rpc_args::init_options(boost::program_options::options_description& desc) + void rpc_args::init_options(boost::program_options::options_description& desc, const bool any_cert_option) { const descriptors arg{}; command_line::add_arg(desc, arg.rpc_bind_ip); command_line::add_arg(desc, arg.rpc_login); command_line::add_arg(desc, arg.confirm_external_bind); command_line::add_arg(desc, arg.rpc_access_control_origins); + command_line::add_arg(desc, arg.rpc_ssl); + command_line::add_arg(desc, arg.rpc_ssl_private_key); + command_line::add_arg(desc, arg.rpc_ssl_certificate); + command_line::add_arg(desc, arg.rpc_ssl_ca_certificates); + command_line::add_arg(desc, arg.rpc_ssl_allowed_fingerprints); + command_line::add_arg(desc, arg.rpc_ssl_allow_chained); + if (any_cert_option) + command_line::add_arg(desc, arg.rpc_ssl_allow_any_cert); } - boost::optional<rpc_args> rpc_args::process(const boost::program_options::variables_map& vm) + boost::optional<rpc_args> rpc_args::process(const boost::program_options::variables_map& vm, const bool any_cert_option) { const descriptors arg{}; rpc_args config{}; @@ -118,6 +185,17 @@ namespace cryptonote config.access_control_origins = std::move(access_control_origins); } + auto ssl_options = do_process_ssl(vm, arg, any_cert_option); + if (!ssl_options) + return boost::none; + config.ssl_options = std::move(*ssl_options); + return {std::move(config)}; } + + boost::optional<epee::net_utils::ssl_options_t> rpc_args::process_ssl(const boost::program_options::variables_map& vm, const bool any_cert_option) + { + const descriptors arg{}; + return do_process_ssl(vm, arg, any_cert_option); + } } diff --git a/src/rpc/rpc_args.h b/src/rpc/rpc_args.h index 216ba3712..619f02b42 100644 --- a/src/rpc/rpc_args.h +++ b/src/rpc/rpc_args.h @@ -35,6 +35,7 @@ #include "common/command_line.h" #include "common/password.h" +#include "net/net_ssl.h" namespace cryptonote { @@ -54,16 +55,29 @@ namespace cryptonote const command_line::arg_descriptor<std::string> rpc_login; const command_line::arg_descriptor<bool> confirm_external_bind; const command_line::arg_descriptor<std::string> rpc_access_control_origins; + const command_line::arg_descriptor<std::string> rpc_ssl; + const command_line::arg_descriptor<std::string> rpc_ssl_private_key; + const command_line::arg_descriptor<std::string> rpc_ssl_certificate; + const command_line::arg_descriptor<std::string> rpc_ssl_ca_certificates; + const command_line::arg_descriptor<std::vector<std::string>> rpc_ssl_allowed_fingerprints; + const command_line::arg_descriptor<bool> rpc_ssl_allow_chained; + const command_line::arg_descriptor<bool> rpc_ssl_allow_any_cert; }; + // `allow_any_cert` bool toggles `--rpc-ssl-allow-any-cert` configuration + static const char* tr(const char* str); - static void init_options(boost::program_options::options_description& desc); + static void init_options(boost::program_options::options_description& desc, const bool any_cert_option = false); //! \return Arguments specified by user, or `boost::none` if error - static boost::optional<rpc_args> process(const boost::program_options::variables_map& vm); + static boost::optional<rpc_args> process(const boost::program_options::variables_map& vm, const bool any_cert_option = false); + + //! \return SSL arguments specified by user, or `boost::none` if error + static boost::optional<epee::net_utils::ssl_options_t> process_ssl(const boost::program_options::variables_map& vm, const bool any_cert_option = false); std::string bind_ip; std::vector<std::string> access_control_origins; boost::optional<tools::login> login; // currently `boost::none` if unspecified by user + epee::net_utils::ssl_options_t ssl_options = epee::net_utils::ssl_support_t::e_ssl_support_enabled; }; } diff --git a/src/rpc/rpc_handler.cpp b/src/rpc/rpc_handler.cpp index e0a81c70f..af5cb98a3 100644 --- a/src/rpc/rpc_handler.cpp +++ b/src/rpc/rpc_handler.cpp @@ -26,26 +26,49 @@ namespace rpc } boost::optional<output_distribution_data> - RpcHandler::get_output_distribution(const std::function<bool(uint64_t, uint64_t, uint64_t, uint64_t&, std::vector<uint64_t>&, uint64_t&)> &f, uint64_t amount, uint64_t from_height, uint64_t to_height, bool cumulative) + RpcHandler::get_output_distribution(const std::function<bool(uint64_t, uint64_t, uint64_t, uint64_t&, std::vector<uint64_t>&, uint64_t&)> &f, uint64_t amount, uint64_t from_height, uint64_t to_height, const std::function<crypto::hash(uint64_t)> &get_hash, bool cumulative, uint64_t blockchain_height) { static struct D { boost::mutex mutex; std::vector<std::uint64_t> cached_distribution; std::uint64_t cached_from, cached_to, cached_start_height, cached_base; + crypto::hash cached_m10_hash; + crypto::hash cached_top_hash; bool cached; - D(): cached_from(0), cached_to(0), cached_start_height(0), cached_base(0), cached(false) {} + D(): cached_from(0), cached_to(0), cached_start_height(0), cached_base(0), cached_m10_hash(crypto::null_hash), cached_top_hash(crypto::null_hash), cached(false) {} } d; const boost::unique_lock<boost::mutex> lock(d.mutex); - if (d.cached && amount == 0 && d.cached_from == from_height && d.cached_to == to_height) + crypto::hash top_hash = crypto::null_hash; + if (d.cached_to < blockchain_height) + top_hash = get_hash(d.cached_to); + if (d.cached && amount == 0 && d.cached_from == from_height && d.cached_to == to_height && d.cached_top_hash == top_hash) return process_distribution(cumulative, d.cached_start_height, d.cached_distribution, d.cached_base); std::vector<std::uint64_t> distribution; std::uint64_t start_height, base; // see if we can extend the cache - a common case - if (d.cached && amount == 0 && d.cached_from == from_height && to_height > d.cached_to) + bool can_extend = d.cached && amount == 0 && d.cached_from == from_height && to_height > d.cached_to && top_hash == d.cached_top_hash; + if (!can_extend) + { + // we kept track of the hash 10 blocks below, if it exists, so if it matches, + // we can still pop the last 10 cached slots and try again + if (d.cached && amount == 0 && d.cached_from == from_height && d.cached_to - d.cached_from >= 10 && to_height > d.cached_to - 10) + { + crypto::hash hash10 = get_hash(d.cached_to - 10); + if (hash10 == d.cached_m10_hash) + { + d.cached_to -= 10; + d.cached_top_hash = hash10; + d.cached_m10_hash = crypto::null_hash; + d.cached_distribution.resize(d.cached_distribution.size() - 10); + can_extend = true; + } + } + } + if (can_extend) { std::vector<std::uint64_t> new_distribution; if (!f(amount, d.cached_to + 1, to_height, start_height, new_distribution, base)) @@ -74,6 +97,8 @@ namespace rpc { d.cached_from = from_height; d.cached_to = to_height; + d.cached_top_hash = get_hash(d.cached_to); + d.cached_m10_hash = d.cached_to >= 10 ? get_hash(d.cached_to - 10) : crypto::null_hash; d.cached_distribution = distribution; d.cached_start_height = start_height; d.cached_base = base; diff --git a/src/rpc/rpc_handler.h b/src/rpc/rpc_handler.h index 2439eaa58..b81983d28 100644 --- a/src/rpc/rpc_handler.h +++ b/src/rpc/rpc_handler.h @@ -32,6 +32,7 @@ #include <cstdint> #include <string> #include <vector> +#include "crypto/hash.h" namespace cryptonote { @@ -56,7 +57,7 @@ class RpcHandler virtual std::string handle(const std::string& request) = 0; static boost::optional<output_distribution_data> - get_output_distribution(const std::function<bool(uint64_t, uint64_t, uint64_t, uint64_t&, std::vector<uint64_t>&, uint64_t&)> &f, uint64_t amount, uint64_t from_height, uint64_t to_height, bool cumulative); + get_output_distribution(const std::function<bool(uint64_t, uint64_t, uint64_t, uint64_t&, std::vector<uint64_t>&, uint64_t&)> &f, uint64_t amount, uint64_t from_height, uint64_t to_height, const std::function<crypto::hash(uint64_t)> &get_hash, bool cumulative, uint64_t blockchain_height); }; diff --git a/src/serialization/binary_archive.h b/src/serialization/binary_archive.h index a0e4eff9d..9f60cf311 100644 --- a/src/serialization/binary_archive.h +++ b/src/serialization/binary_archive.h @@ -146,7 +146,8 @@ struct binary_archive<false> : public binary_archive_base<std::istream, false> void serialize_uvarint(T &v) { typedef std::istreambuf_iterator<char> it; - tools::read_varint(it(stream_), it(), v); // XXX handle failure + if (tools::read_varint(it(stream_), it(), v) < 0) + stream_.setstate(std::ios_base::failbit); } void begin_array(size_t &s) diff --git a/src/serialization/difficulty_type.h b/src/serialization/difficulty_type.h index e32e24b78..c551095f9 100644 --- a/src/serialization/difficulty_type.h +++ b/src/serialization/difficulty_type.h @@ -54,8 +54,8 @@ inline bool do_serialize(Archive<true>& ar, cryptonote::difficulty_type &diff) { if (!ar.stream().good()) return false; - const uint64_t hi = (diff >> 64).convert_to<uint64_t>(); - const uint64_t lo = (diff << 64 >> 64).convert_to<uint64_t>(); + const uint64_t hi = ((diff >> 64) & 0xffffffffffffffff).convert_to<uint64_t>(); + const uint64_t lo = (diff & 0xffffffffffffffff).convert_to<uint64_t>(); ar.serialize_varint(hi); ar.serialize_varint(lo); if (!ar.stream().good()) diff --git a/src/serialization/serialization.h b/src/serialization/serialization.h index 007bf265f..553e9951f 100644 --- a/src/serialization/serialization.h +++ b/src/serialization/serialization.h @@ -212,7 +212,7 @@ inline bool do_serialize(Archive &ar, bool &v) * \brief self-explanatory */ #define END_SERIALIZE() \ - return true; \ + return ar.stream().good(); \ } /*! \macro VALUE(f) diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index 1640fa990..e8b203d9d 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -92,6 +92,8 @@ typedef cryptonote::simple_wallet sw; #define MIN_RING_SIZE 11 // Used to inform user about min ring size -- does not track actual protocol +#define OLD_AGE_WARN_THRESHOLD (30 * 86400 / DIFFICULTY_TARGET_V2) // 30 days + #define LOCK_IDLE_SCOPE() \ bool auto_refresh_enabled = m_auto_refresh_enabled.load(std::memory_order_relaxed); \ m_auto_refresh_enabled.store(false, std::memory_order_relaxed); \ @@ -4934,7 +4936,7 @@ void simple_wallet::on_new_block(uint64_t height, const cryptonote::block& block m_refresh_progress_reporter.update(height, false); } //---------------------------------------------------------------------------------------------------- -void simple_wallet::on_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index) +void simple_wallet::on_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index, uint64_t unlock_time) { message_writer(console_color_green, false) << "\r" << tr("Height ") << height << ", " << @@ -4956,6 +4958,8 @@ void simple_wallet::on_money_received(uint64_t height, const crypto::hash &txid, (m_long_payment_id_support ? tr("WARNING: this transaction uses an unencrypted payment ID: consider using subaddresses instead.") : tr("WARNING: this transaction uses an unencrypted payment ID: these are obsolete. Support will be withdrawn in the future. Use subaddresses instead.")); } } + if (unlock_time) + message_writer() << tr("NOTE: This transaction is locked, see details with: show_transfer ") + epee::string_tools::pod_to_hex(txid); if (m_auto_refresh_refreshing) m_cmd_binder.print_prompt(); else @@ -5606,6 +5610,43 @@ bool simple_wallet::print_ring_members(const std::vector<tools::wallet2::pending return true; } //---------------------------------------------------------------------------------------------------- +bool simple_wallet::prompt_if_old(const std::vector<tools::wallet2::pending_tx> &ptx_vector) +{ + // count the number of old outputs + std::string err; + uint64_t bc_height = get_daemon_blockchain_height(err); + if (!err.empty()) + return true; + + int max_n_old = 0; + for (const auto &ptx: ptx_vector) + { + int n_old = 0; + for (const auto i: ptx.selected_transfers) + { + const tools::wallet2::transfer_details &td = m_wallet->get_transfer_details(i); + uint64_t age = bc_height - td.m_block_height; + if (age > OLD_AGE_WARN_THRESHOLD) + ++n_old; + } + max_n_old = std::max(max_n_old, n_old); + } + if (max_n_old > 1) + { + std::stringstream prompt; + prompt << tr("Transaction spends more than one very old output. Privacy would be better if they were sent separately."); + prompt << ENDL << tr("Spend them now anyway?"); + std::string accepted = input_line(prompt.str(), true); + if (std::cin.eof()) + return false; + if (!command_line::is_yes(accepted)) + { + return false; + } + } + return true; +} +//---------------------------------------------------------------------------------------------------- bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::string> &args_, bool called_by_mms) { // "transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> <amount> [<payment_id>]" @@ -5907,6 +5948,12 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri } } + if (!prompt_if_old(ptx_vector)) + { + fail_msg_writer() << tr("transaction cancelled."); + return false; + } + // if more than one tx necessary, prompt user to confirm if (m_wallet->always_confirm_transfers() || ptx_vector.size() > 1) { @@ -6090,7 +6137,8 @@ bool simple_wallet::locked_transfer(const std::vector<std::string> &args_) //---------------------------------------------------------------------------------------------------- bool simple_wallet::locked_sweep_all(const std::vector<std::string> &args_) { - return sweep_main(0, true, args_); + sweep_main(0, true, args_); + return true; } //---------------------------------------------------------------------------------------------------- @@ -6410,6 +6458,12 @@ bool simple_wallet::sweep_main(uint64_t below, bool locked, const std::vector<st return true; } + if (!prompt_if_old(ptx_vector)) + { + fail_msg_writer() << tr("transaction cancelled."); + return false; + } + // give user total and fee, and prompt to confirm uint64_t total_fee = 0, total_sent = 0; for (size_t n = 0; n < ptx_vector.size(); ++n) @@ -6756,7 +6810,8 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) //---------------------------------------------------------------------------------------------------- bool simple_wallet::sweep_all(const std::vector<std::string> &args_) { - return sweep_main(0, false, args_); + sweep_main(0, false, args_); + return true; } //---------------------------------------------------------------------------------------------------- bool simple_wallet::sweep_below(const std::vector<std::string> &args_) @@ -6772,7 +6827,8 @@ bool simple_wallet::sweep_below(const std::vector<std::string> &args_) fail_msg_writer() << tr("invalid amount threshold"); return true; } - return sweep_main(below, false, std::vector<std::string>(++args_.begin(), args_.end())); + sweep_main(below, false, std::vector<std::string>(++args_.begin(), args_.end())); + return true; } //---------------------------------------------------------------------------------------------------- bool simple_wallet::donate(const std::vector<std::string> &args_) @@ -7683,6 +7739,8 @@ bool simple_wallet::get_transfers(std::vector<std::string>& local_args, std::vec local_args.erase(local_args.begin()); } + const uint64_t last_block_height = m_wallet->get_blockchain_current_height(); + if (in || coinbase) { std::list<std::pair<crypto::hash, tools::wallet2::payment_details>> payments; m_wallet->get_payments(payments, min_height, max_height, m_current_subaddress_account, subaddr_indices); @@ -7697,6 +7755,25 @@ bool simple_wallet::get_transfers(std::vector<std::string>& local_args, std::vec std::string destination = m_wallet->get_subaddress_as_str({m_current_subaddress_account, pd.m_subaddr_index.minor}); const std::string type = pd.m_coinbase ? tr("block") : tr("in"); const bool unlocked = m_wallet->is_transfer_unlocked(pd.m_unlock_time, pd.m_block_height); + std::string locked_msg = "unlocked"; + if (!unlocked) + { + locked_msg = "locked"; + const uint64_t unlock_time = pd.m_unlock_time; + if (pd.m_unlock_time < CRYPTONOTE_MAX_BLOCK_NUMBER) + { + uint64_t bh = std::max(pd.m_unlock_time, pd.m_block_height + CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE); + if (bh >= last_block_height) + locked_msg = std::to_string(bh - last_block_height) + " blks"; + } + else + { + uint64_t current_time = static_cast<uint64_t>(time(NULL)); + uint64_t threshold = current_time + (m_wallet->use_fork_rules(2, 0) ? CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_SECONDS_V2 : CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_SECONDS_V1); + if (threshold < pd.m_unlock_time) + locked_msg = get_human_readable_timespan(std::chrono::seconds(pd.m_unlock_time - threshold)); + } + } transfers.push_back({ type, pd.m_block_height, @@ -7710,7 +7787,7 @@ bool simple_wallet::get_transfers(std::vector<std::string>& local_args, std::vec {{destination, pd.m_amount}}, {pd.m_subaddr_index.minor}, note, - (unlocked) ? "unlocked" : "locked" + locked_msg }); } } diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h index 76d446ba5..4bf7fa334 100644 --- a/src/simplewallet/simplewallet.h +++ b/src/simplewallet/simplewallet.h @@ -261,6 +261,7 @@ namespace cryptonote void on_refresh_finished(uint64_t start_height, uint64_t fetched_blocks, bool is_init, bool received_money); std::pair<std::string, std::string> show_outputs_line(const std::vector<uint64_t> &heights, uint64_t blockchain_height, uint64_t highlight_height = std::numeric_limits<uint64_t>::max()) const; bool freeze_thaw(const std::vector<std::string>& args, bool freeze); + bool prompt_if_old(const std::vector<tools::wallet2::pending_tx> &ptx_vector); struct transfer_view { @@ -310,7 +311,7 @@ namespace cryptonote //----------------- i_wallet2_callback --------------------- virtual void on_new_block(uint64_t height, const cryptonote::block& block); - virtual void on_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index); + virtual void on_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index, uint64_t unlock_time); virtual void on_unconfirmed_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index); virtual void on_money_spent(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& in_tx, uint64_t amount, const cryptonote::transaction& spend_tx, const cryptonote::subaddress_index& subaddr_index); virtual void on_skip_transaction(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx); diff --git a/src/version.cpp.in b/src/version.cpp.in index 8aaa41b19..28ce38df7 100644 --- a/src/version.cpp.in +++ b/src/version.cpp.in @@ -1,5 +1,5 @@ #define DEF_MONERO_VERSION_TAG "@VERSIONTAG@" -#define DEF_MONERO_VERSION "0.14.1.0" +#define DEF_MONERO_VERSION "0.14.1.2" #define DEF_MONERO_RELEASE_NAME "Boron Butterfly" #define DEF_MONERO_VERSION_FULL DEF_MONERO_VERSION "-" DEF_MONERO_VERSION_TAG diff --git a/src/wallet/api/wallet.cpp b/src/wallet/api/wallet.cpp index 032b873d6..1711db482 100644 --- a/src/wallet/api/wallet.cpp +++ b/src/wallet/api/wallet.cpp @@ -157,7 +157,7 @@ struct Wallet2CallbackImpl : public tools::i_wallet2_callback } } - virtual void on_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index) + virtual void on_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index, uint64_t unlock_time) { std::string tx_hash = epee::string_tools::pod_to_hex(txid); diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index c222c3115..d14cec4cc 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -39,6 +39,7 @@ #include <boost/algorithm/string/join.hpp> #include <boost/asio/ip/address.hpp> #include <boost/range/adaptor/transformed.hpp> +#include <boost/preprocessor/stringize.hpp> #include "include_base_utils.h" using namespace epee; @@ -131,6 +132,9 @@ using namespace cryptonote; #define GAMMA_SHAPE 19.28 #define GAMMA_SCALE (1/1.61) +#define DEFAULT_MIN_OUTPUT_COUNT 5 +#define DEFAULT_MIN_OUTPUT_VALUE (2*COIN) + static const std::string MULTISIG_SIGNATURE_MAGIC = "SigMultisigPkV1"; static const std::string MULTISIG_EXTRA_INFO_MAGIC = "MultisigxV1"; @@ -340,6 +344,11 @@ std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variabl { std::vector<std::vector<uint8_t>> ssl_allowed_fingerprints{ daemon_ssl_allowed_fingerprints.size() }; std::transform(daemon_ssl_allowed_fingerprints.begin(), daemon_ssl_allowed_fingerprints.end(), ssl_allowed_fingerprints.begin(), epee::from_hex::vector); + for (const auto &fpr: ssl_allowed_fingerprints) + { + THROW_WALLET_EXCEPTION_IF(fpr.size() != SSL_FINGERPRINT_SIZE, tools::error::wallet_internal_error, + "SHA-256 fingerprint should be " BOOST_PP_STRINGIZE(SSL_FINGERPRINT_SIZE) " bytes long."); + } ssl_options = epee::net_utils::ssl_options_t{ std::move(ssl_allowed_fingerprints), std::move(daemon_ssl_ca_file) @@ -390,8 +399,11 @@ std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variabl { const boost::string_ref real_daemon = boost::string_ref{daemon_address}.substr(0, daemon_address.rfind(':')); + /* If SSL or proxy is enabled, then a specific cert, CA or fingerprint must + be specified. This is specific to the wallet. */ const bool verification_required = - ssl_options.support == epee::net_utils::ssl_support_t::e_ssl_support_enabled || use_proxy; + ssl_options.verification != epee::net_utils::ssl_verification_t::none && + (ssl_options.support == epee::net_utils::ssl_support_t::e_ssl_support_enabled || use_proxy); THROW_WALLET_EXCEPTION_IF( verification_required && !ssl_options.has_strong_verification(real_daemon), @@ -994,7 +1006,7 @@ uint64_t gamma_picker::pick() const uint64_t n_rct = rct_offsets[index] - first_rct; if (n_rct == 0) return std::numeric_limits<uint64_t>::max(); // bad pick - MDEBUG("Picking 1/" << n_rct << " in block " << index); + MTRACE("Picking 1/" << n_rct << " in block " << index); return first_rct + crypto::rand_idx(n_rct); }; @@ -2007,7 +2019,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote } LOG_PRINT_L0("Received money: " << print_money(td.amount()) << ", with tx: " << txid); if (0 != m_callback) - m_callback->on_money_received(height, txid, tx, td.m_amount, td.m_subaddr_index); + m_callback->on_money_received(height, txid, tx, td.m_amount, td.m_subaddr_index, td.m_tx.unlock_time); } total_received_1 += amount; notify = true; @@ -2077,7 +2089,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote LOG_PRINT_L0("Received money: " << print_money(td.amount()) << ", with tx: " << txid); if (0 != m_callback) - m_callback->on_money_received(height, txid, tx, td.m_amount, td.m_subaddr_index); + m_callback->on_money_received(height, txid, tx, td.m_amount, td.m_subaddr_index, td.m_tx.unlock_time); } total_received_1 += extra_amount; notify = true; @@ -7750,7 +7762,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> } } - if (num_outs <= requested_outputs_count && !existing_ring_found) + if (num_outs <= requested_outputs_count) { for (uint64_t i = 0; i < num_outs; i++) req.outputs.push_back({amount, i}); @@ -7776,6 +7788,8 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> // while we still need more mixins uint64_t num_usable_outs = num_outs; bool allow_blackballed = false; + MDEBUG("Starting gamma picking with " << num_outs << ", num_usable_outs " << num_usable_outs + << ", requested_outputs_count " << requested_outputs_count); while (num_found < requested_outputs_count) { // if we've gone through every possible output, we've gotten all we can @@ -7875,6 +7889,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> picks[type].insert(i); req.outputs.push_back({amount, i}); ++num_found; + MDEBUG("picked " << i << ", " << num_found << " now picked"); } for (const auto &pick: picks) @@ -9372,9 +9387,16 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp idx = pop_best_value(indices, tx.selected_transfers, true); // we might not want to add it if it's a large output and we don't have many left - if (m_transfers[idx].amount() >= m_min_output_value) { - if (get_count_above(m_transfers, *unused_transfers_indices, m_min_output_value) < m_min_output_count) { - LOG_PRINT_L2("Second output was not strictly needed, and we're running out of outputs above " << print_money(m_min_output_value) << ", not adding"); + uint64_t min_output_value = m_min_output_value; + uint32_t min_output_count = m_min_output_count; + if (min_output_value == 0 && min_output_count == 0) + { + min_output_value = DEFAULT_MIN_OUTPUT_VALUE; + min_output_count = DEFAULT_MIN_OUTPUT_COUNT; + } + if (m_transfers[idx].amount() >= min_output_value) { + if (get_count_above(m_transfers, *unused_transfers_indices, min_output_value) < min_output_count) { + LOG_PRINT_L2("Second output was not strictly needed, and we're running out of outputs above " << print_money(min_output_value) << ", not adding"); break; } } @@ -12607,8 +12629,7 @@ std::string wallet2::make_uri(const std::string &address, const std::string &pay if (!payment_id.empty()) { crypto::hash pid32; - crypto::hash8 pid8; - if (!wallet2::parse_long_payment_id(payment_id, pid32) && !wallet2::parse_short_payment_id(payment_id, pid8)) + if (!wallet2::parse_long_payment_id(payment_id, pid32)) { error = "Invalid payment id"; return std::string(); @@ -12702,8 +12723,7 @@ bool wallet2::parse_uri(const std::string &uri, std::string &address, std::strin return false; } crypto::hash hash; - crypto::hash8 hash8; - if (!wallet2::parse_long_payment_id(kv[1], hash) && !wallet2::parse_short_payment_id(kv[1], hash8)) + if (!wallet2::parse_long_payment_id(kv[1], hash)) { error = "Invalid payment id: " + kv[1]; return false; diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index d101e87f5..921c150cb 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -118,7 +118,7 @@ private: public: // Full wallet callbacks virtual void on_new_block(uint64_t height, const cryptonote::block& block) {} - virtual void on_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index) {} + virtual void on_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index, uint64_t unlock_time) {} virtual void on_unconfirmed_money_received(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t amount, const cryptonote::subaddress_index& subaddr_index) {} virtual void on_money_spent(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& in_tx, uint64_t amount, const cryptonote::transaction& spend_tx, const cryptonote::subaddress_index& subaddr_index) {} virtual void on_skip_transaction(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx) {} diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index 4076ae957..47235dc44 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -31,6 +31,7 @@ #include <boost/asio/ip/address.hpp> #include <boost/filesystem/operations.hpp> #include <boost/algorithm/string.hpp> +#include <boost/preprocessor/stringize.hpp> #include <cstdint> #include "include_base_utils.h" using namespace epee; @@ -65,11 +66,6 @@ namespace const command_line::arg_descriptor<bool> arg_restricted = {"restricted-rpc", "Restricts to view-only commands", false}; const command_line::arg_descriptor<std::string> arg_wallet_dir = {"wallet-dir", "Directory for newly created wallets"}; const command_line::arg_descriptor<bool> arg_prompt_for_password = {"prompt-for-password", "Prompts for password when not provided", false}; - const command_line::arg_descriptor<std::string> arg_rpc_ssl = {"rpc-ssl", tools::wallet2::tr("Enable SSL on wallet RPC connections: enabled|disabled|autodetect"), "autodetect"}; - const command_line::arg_descriptor<std::string> arg_rpc_ssl_private_key = {"rpc-ssl-private-key", tools::wallet2::tr("Path to a PEM format private key"), ""}; - const command_line::arg_descriptor<std::string> arg_rpc_ssl_certificate = {"rpc-ssl-certificate", tools::wallet2::tr("Path to a PEM format certificate"), ""}; - const command_line::arg_descriptor<std::string> arg_rpc_ssl_ca_certificates = {"rpc-ssl-ca-certificates", tools::wallet2::tr("Path to file containing concatenated PEM format certificate(s) to replace system CA(s).")}; - const command_line::arg_descriptor<std::vector<std::string>> arg_rpc_ssl_allowed_fingerprints = {"rpc-ssl-allowed-fingerprints", tools::wallet2::tr("List of certificate fingerprints to allow")}; constexpr const char default_rpc_username[] = "monero"; @@ -243,37 +239,6 @@ namespace tools assert(bool(http_login)); } // end auth enabled - auto rpc_ssl_private_key = command_line::get_arg(vm, arg_rpc_ssl_private_key); - auto rpc_ssl_certificate = command_line::get_arg(vm, arg_rpc_ssl_certificate); - auto rpc_ssl_ca_file = command_line::get_arg(vm, arg_rpc_ssl_ca_certificates); - auto rpc_ssl_allowed_fingerprints = command_line::get_arg(vm, arg_rpc_ssl_allowed_fingerprints); - auto rpc_ssl = command_line::get_arg(vm, arg_rpc_ssl); - epee::net_utils::ssl_options_t rpc_ssl_options = epee::net_utils::ssl_support_t::e_ssl_support_enabled; - - if (!rpc_ssl_ca_file.empty() || !rpc_ssl_allowed_fingerprints.empty()) - { - std::vector<std::vector<uint8_t>> allowed_fingerprints{ rpc_ssl_allowed_fingerprints.size() }; - std::transform(rpc_ssl_allowed_fingerprints.begin(), rpc_ssl_allowed_fingerprints.end(), allowed_fingerprints.begin(), epee::from_hex::vector); - - rpc_ssl_options = epee::net_utils::ssl_options_t{ - std::move(allowed_fingerprints), std::move(rpc_ssl_ca_file) - }; - } - - // user specified CA file or fingeprints implies enabled SSL by default - if (rpc_ssl_options.verification != epee::net_utils::ssl_verification_t::user_certificates || !command_line::is_arg_defaulted(vm, arg_rpc_ssl)) - { - if (!epee::net_utils::ssl_support_from_string(rpc_ssl_options.support, rpc_ssl)) - { - MERROR("Invalid argument for " << std::string(arg_rpc_ssl.name)); - return false; - } - } - - rpc_ssl_options.auth = epee::net_utils::ssl_authentication_t{ - std::move(rpc_ssl_private_key), std::move(rpc_ssl_certificate) - }; - m_auto_refresh_period = DEFAULT_AUTO_REFRESH_PERIOD; m_last_auto_refresh_time = boost::posix_time::min_date_time; @@ -283,7 +248,7 @@ namespace tools auto rng = [](size_t len, uint8_t *ptr) { return crypto::rand(len, ptr); }; return epee::http_server_impl_base<wallet_rpc_server, connection_context>::init( rng, std::move(bind_port), std::move(rpc_config->bind_ip), std::move(rpc_config->access_control_origins), std::move(http_login), - std::move(rpc_ssl_options) + std::move(rpc_config->ssl_options) ); } //------------------------------------------------------------------------------------------------------------------------------ @@ -385,7 +350,7 @@ namespace tools entry.destinations.push_back(wallet_rpc::transfer_destination()); wallet_rpc::transfer_destination &td = entry.destinations.back(); td.amount = d.amount; - td.address = get_account_address_as_str(m_wallet->nettype(), d.is_subaddress, d.addr); + td.address = d.original.empty() ? get_account_address_as_str(m_wallet->nettype(), d.is_subaddress, d.addr) : d.original; } entry.type = "out"; @@ -410,6 +375,14 @@ namespace tools entry.amount = pd.m_amount_in - pd.m_change - entry.fee; entry.unlock_time = pd.m_tx.unlock_time; entry.note = m_wallet->get_tx_note(txid); + + for (const auto &d: pd.m_dests) { + entry.destinations.push_back(wallet_rpc::transfer_destination()); + wallet_rpc::transfer_destination &td = entry.destinations.back(); + td.amount = d.amount; + td.address = d.original.empty() ? get_account_address_as_str(m_wallet->nettype(), d.is_subaddress, d.addr) : d.original; + } + entry.type = is_failed ? "failed" : "pending"; entry.subaddr_index = { pd.m_subaddr_account, 0 }; for (uint32_t i: pd.m_subaddr_indices) @@ -1772,6 +1745,11 @@ namespace tools else if (payment_id_str.size() == 2 * sizeof(payment_id8)) { r = epee::string_tools::hex_to_pod(payment_id_str, payment_id8); + if (r) + { + memcpy(payment_id.data, payment_id8.data, 8); + memset(payment_id.data + 8, 0, 24); + } } else { @@ -1833,14 +1811,12 @@ namespace tools wallet2::transfer_container transfers; m_wallet->get_transfers(transfers); - bool transfers_found = false; for (const auto& td : transfers) { if (!filter || available != td.m_spent) { if (req.account_index != td.m_subaddr_index.major || (!req.subaddr_indices.empty() && req.subaddr_indices.count(td.m_subaddr_index.minor) == 0)) continue; - transfers_found = true; wallet_rpc::transfer_details rpc_transfers; rpc_transfers.amount = td.amount(); rpc_transfers.spent = td.m_spent; @@ -2805,20 +2781,20 @@ namespace tools } crypto::hash long_payment_id; - crypto::hash8 short_payment_id; if (!wallet2::parse_long_payment_id(req.payment_id, payment_id)) { if (!wallet2::parse_short_payment_id(req.payment_id, info.payment_id)) { er.code = WALLET_RPC_ERROR_CODE_WRONG_PAYMENT_ID; - er.message = "Payment id has invalid format: \"" + req.payment_id + "\", expected 16 or 64 character string"; + er.message = "Payment id has invalid format: \"" + req.payment_id + "\", expected 64 character string"; return false; } else { - memcpy(payment_id.data, info.payment_id.data, 8); - memset(payment_id.data + 8, 0, 24); + er.code = WALLET_RPC_ERROR_CODE_WRONG_PAYMENT_ID; + er.message = "Payment id has invalid format: standalone short payment IDs are forbidden, they must be part of an integrated address"; + return false; } } } @@ -4069,9 +4045,10 @@ namespace tools { cryptonote::TESTNET, "testnet" }, { cryptonote::STAGENET, "stagenet" }, }; + if (!req.any_net_type && !m_wallet) return not_open(er); for (const auto &net_type: net_types) { - if (!req.any_net_type && net_type.type != m_wallet->nettype()) + if (!req.any_net_type && (!m_wallet || net_type.type != m_wallet->nettype())) continue; if (req.allow_openalias) { @@ -4149,10 +4126,15 @@ namespace tools std::move(req.ssl_private_key_path), std::move(req.ssl_certificate_path) }; - if (ssl_options.support == epee::net_utils::ssl_support_t::e_ssl_support_enabled && !ssl_options.has_strong_verification(boost::string_ref{})) + const bool verification_required = + ssl_options.verification != epee::net_utils::ssl_verification_t::none && + ssl_options.support == epee::net_utils::ssl_support_t::e_ssl_support_enabled; + + if (verification_required && !ssl_options.has_strong_verification(boost::string_ref{})) { er.code = WALLET_RPC_ERROR_CODE_NO_DAEMON_CONNECTION; er.message = "SSL is enabled but no user certificate or fingerprints were provided"; + return false; } if (!m_wallet->set_daemon(req.address, boost::none, req.trusted, std::move(ssl_options))) @@ -4177,7 +4159,7 @@ namespace tools { er.code = WALLET_RPC_ERROR_CODE_INVALID_LOG_LEVEL; er.message = "Error: log level not valid"; - return true; + return false; } mlog_set_log_level(req.level); return true; @@ -4393,11 +4375,6 @@ int main(int argc, char** argv) { command_line::add_arg(desc_params, arg_from_json); command_line::add_arg(desc_params, arg_wallet_dir); command_line::add_arg(desc_params, arg_prompt_for_password); - command_line::add_arg(desc_params, arg_rpc_ssl); - command_line::add_arg(desc_params, arg_rpc_ssl_private_key); - command_line::add_arg(desc_params, arg_rpc_ssl_certificate); - command_line::add_arg(desc_params, arg_rpc_ssl_ca_certificates); - command_line::add_arg(desc_params, arg_rpc_ssl_allowed_fingerprints); daemonizer::init_options(hidden_options, desc_params); desc_params.add(hidden_options); |