diff options
42 files changed, 584 insertions, 287 deletions
diff --git a/contrib/epee/include/net/http_client.h b/contrib/epee/include/net/http_client.h index 67e63f7bf..8e099e2bc 100644 --- a/contrib/epee/include/net/http_client.h +++ b/contrib/epee/include/net/http_client.h @@ -293,6 +293,9 @@ using namespace std; , m_lock() {} + const std::string &get_host() const { return m_host_buff; }; + const std::string &get_port() const { return m_port; }; + bool set_server(const std::string& address, boost::optional<login> user) { http::url_content parsed{}; diff --git a/contrib/epee/include/string_tools.h b/contrib/epee/include/string_tools.h index 258caa49e..ce7b2fb87 100644 --- a/contrib/epee/include/string_tools.h +++ b/contrib/epee/include/string_tools.h @@ -314,6 +314,18 @@ POP_WARNINGS return str; } //---------------------------------------------------------------------------- + inline std::string pad_string(std::string s, size_t n, char c = ' ', bool prepend = false) + { + if (s.size() < n) + { + if (prepend) + s = std::string(n - s.size(), c) + s; + else + s.append(n - s.size(), c); + } + return s; + } + //---------------------------------------------------------------------------- template<class t_pod_type> std::string pod_to_hex(const t_pod_type& s) { diff --git a/contrib/rlwrap/monerocommands_bitmonerod.txt b/contrib/rlwrap/monerocommands_bitmonerod.txt index 9572e40fe..c4f77b37d 100644 --- a/contrib/rlwrap/monerocommands_bitmonerod.txt +++ b/contrib/rlwrap/monerocommands_bitmonerod.txt @@ -32,4 +32,5 @@ status stop_daemon stop_mining stop_save_graph +sync_info unban diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp index a6774a25c..01a59e079 100644 --- a/src/blockchain_db/blockchain_db.cpp +++ b/src/blockchain_db/blockchain_db.cpp @@ -33,6 +33,19 @@ #include "profile_tools.h" #include "ringct/rctOps.h" +#include "lmdb/db_lmdb.h" +#ifdef BERKELEY_DB +#include "berkeleydb/db_bdb.h" +#endif + +static const char *db_types[] = { + "lmdb", +#ifdef BERKELEY_DB + "berkeley", +#endif + NULL +}; + #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "blockchain.db" @@ -41,6 +54,41 @@ using epee::string_tools::pod_to_hex; namespace cryptonote { +bool blockchain_valid_db_type(const std::string& db_type) +{ + int i; + for (i=0; db_types[i]; i++) + { + if (db_types[i] == db_type) + return true; + } + return false; +} + +std::string blockchain_db_types(const std::string& sep) +{ + int i; + std::string ret = ""; + for (i=0; db_types[i]; i++) + { + if (i) + ret += sep; + ret += db_types[i]; + } + return ret; +} + +BlockchainDB *new_db(const std::string& db_type) +{ + if (db_type == "lmdb") + return new BlockchainLMDB(); +#if defined(BERKELEY_DB) + if (db_type == "berkeley") + return new BlockchainBDB(); +#endif + return NULL; +} + void BlockchainDB::pop_block() { block blk; diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 27e63801d..ad246d85e 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -145,6 +145,12 @@ struct txpool_tx_meta_t uint8_t padding[77]; // till 192 bytes }; +#define DBF_SAFE 1 +#define DBF_FAST 2 +#define DBF_FASTEST 4 +#define DBF_RDONLY 8 +#define DBF_SALVAGE 0x10 + /*********************************** * Exception Definitions ***********************************/ @@ -600,6 +606,13 @@ public: virtual void sync() = 0; /** + * @brief toggle safe syncs for the DB + * + * Used to switch DBF_SAFE on or off after starting up with DBF_FAST. + */ + virtual void safesyncmode(const bool onoff) = 0; + + /** * @brief Remove everything from the BlockchainDB * * This function should completely remove all data from a BlockchainDB. @@ -1491,6 +1504,7 @@ public: }; // class BlockchainDB +BlockchainDB *new_db(const std::string& db_type); } // namespace cryptonote diff --git a/src/blockchain_db/db_types.h b/src/blockchain_db/db_types.h index 6c21b029e..8e2f58a61 100644 --- a/src/blockchain_db/db_types.h +++ b/src/blockchain_db/db_types.h @@ -31,9 +31,6 @@ namespace cryptonote { - - const std::unordered_set<std::string> blockchain_db_types = - { "lmdb" - }; - + bool blockchain_valid_db_type(const std::string& db_type); + std::string blockchain_db_types(const std::string& sep); } // namespace cryptonote diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 4441c7578..4100d9cca 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -1083,9 +1083,10 @@ BlockchainLMDB::BlockchainLMDB(bool batch_transactions) m_hardfork = nullptr; } -void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) +void BlockchainLMDB::open(const std::string& filename, const int db_flags) { int result; + int mdb_flags = MDB_NORDAHEAD; LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -1124,6 +1125,15 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) size_t mapsize = DEFAULT_MAPSIZE; + if (db_flags & DBF_FAST) + mdb_flags |= MDB_NOSYNC; + if (db_flags & DBF_FASTEST) + mdb_flags |= MDB_NOSYNC | MDB_WRITEMAP | MDB_MAPASYNC; + if (db_flags & DBF_RDONLY) + mdb_flags = MDB_RDONLY; + if (db_flags & DBF_SALVAGE) + mdb_flags |= MDB_PREVSNAPSHOT; + if (auto result = mdb_env_open(m_env, filename.c_str(), mdb_flags, 0644)) throw0(DB_ERROR(lmdb_error("Failed to open lmdb environment: ", result).c_str())); @@ -1308,6 +1318,11 @@ void BlockchainLMDB::sync() } } +void BlockchainLMDB::safesyncmode(const bool onoff) +{ + mdb_env_set_flags(m_env, MDB_NOSYNC|MDB_MAPASYNC, !onoff); +} + void BlockchainLMDB::reset() { LOG_PRINT_L3("BlockchainLMDB::" << __func__); diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 14e5d34e2..3a11ddf0d 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -165,6 +165,8 @@ public: virtual void sync(); + virtual void safesyncmode(const bool onoff); + virtual void reset(); virtual std::vector<std::string> get_filenames() const; diff --git a/src/blockchain_utilities/blockchain_export.cpp b/src/blockchain_utilities/blockchain_export.cpp index f145bc107..20eca09f2 100644 --- a/src/blockchain_utilities/blockchain_export.cpp +++ b/src/blockchain_utilities/blockchain_export.cpp @@ -31,10 +31,6 @@ #include "common/command_line.h" #include "cryptonote_core/tx_pool.h" #include "blockchain_db/blockchain_db.h" -#include "blockchain_db/lmdb/db_lmdb.h" -#if defined(BERKELEY_DB) -#include "blockchain_db/berkeleydb/db_bdb.h" -#endif #include "blockchain_db/db_types.h" #include "version.h" @@ -44,17 +40,6 @@ namespace po = boost::program_options; using namespace epee; -std::string join_set_strings(const std::unordered_set<std::string>& db_types_all, const char* delim) -{ - std::string result; - std::ostringstream s; - std::copy(db_types_all.begin(), db_types_all.end(), std::ostream_iterator<std::string>(s, delim)); - result = s.str(); - if (result.length() > 0) - result.erase(result.end()-strlen(delim), result.end()); - return result; -} - int main(int argc, char* argv[]) { TRY_ENTRY(); @@ -63,10 +48,7 @@ int main(int argc, char* argv[]) std::string default_db_type = "lmdb"; - std::unordered_set<std::string> db_types_all = cryptonote::blockchain_db_types; - db_types_all.insert("memory"); - - std::string available_dbs = join_set_strings(db_types_all, ", "); + std::string available_dbs = cryptonote::blockchain_db_types(", "); available_dbs = "available: " + available_dbs; uint32_t log_level = 0; @@ -144,18 +126,11 @@ int main(int argc, char* argv[]) m_config_folder = command_line::get_arg(vm, data_dir_arg); std::string db_type = command_line::get_arg(vm, arg_database); - if (db_types_all.count(db_type) == 0) + if (!cryptonote::blockchain_valid_db_type(db_type)) { std::cerr << "Invalid database type: " << db_type << std::endl; return 1; } -#if !defined(BERKELEY_DB) - if (db_type == "berkeley") - { - LOG_ERROR("BerkeleyDB support disabled."); - return false; - } -#endif if (command_line::has_arg(vm, arg_output_file)) output_file_path = boost::filesystem::path(command_line::get_arg(vm, arg_output_file)); @@ -179,19 +154,8 @@ int main(int argc, char* argv[]) tx_memory_pool m_mempool(*core_storage); core_storage = new Blockchain(m_mempool); - int db_flags = 0; - - BlockchainDB* db = nullptr; - if (db_type == "lmdb") - { - db_flags |= MDB_RDONLY; - db = new BlockchainLMDB(); - } -#if defined(BERKELEY_DB) - else if (db_type == "berkeley") - db = new BlockchainBDB(); -#endif - else + BlockchainDB* db = new_db(db_type); + if (db == NULL) { LOG_ERROR("Attempted to use non-existent database type: " << db_type); throw std::runtime_error("Attempting to use non-existent database type"); @@ -205,7 +169,7 @@ int main(int argc, char* argv[]) LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); try { - db->open(filename, db_flags); + db->open(filename, DBF_RDONLY); } catch (const std::exception& e) { diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp index 6f908c799..ded854ca4 100644 --- a/src/blockchain_utilities/blockchain_import.cpp +++ b/src/blockchain_utilities/blockchain_import.cpp @@ -42,8 +42,6 @@ #include "blockchain_db/db_types.h" #include "cryptonote_core/cryptonote_core.h" -#include <lmdb.h> // for db flag arguments - #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "bcutil" @@ -78,40 +76,16 @@ namespace po = boost::program_options; using namespace cryptonote; using namespace epee; - -std::string join_set_strings(const std::unordered_set<std::string>& db_types_all, const char* delim) -{ - std::string result; - std::ostringstream s; - std::copy(db_types_all.begin(), db_types_all.end(), std::ostream_iterator<std::string>(s, delim)); - result = s.str(); - if (result.length() > 0) - result.erase(result.end()-strlen(delim), result.end()); - return result; -} - -// db_type: lmdb, berkeley // db_mode: safe, fast, fastest -int get_db_flags_from_mode(const std::string& db_type, const std::string& db_mode) +int get_db_flags_from_mode(const std::string& db_mode) { - uint64_t BDB_FAST_MODE = 0; - uint64_t BDB_FASTEST_MODE = 0; - uint64_t BDB_SAFE_MODE = 0; - -#if defined(BERKELEY_DB) - BDB_FAST_MODE = DB_TXN_WRITE_NOSYNC; - BDB_FASTEST_MODE = DB_TXN_NOSYNC; - BDB_SAFE_MODE = DB_TXN_SYNC; -#endif - int db_flags = 0; - bool islmdb = db_type == "lmdb"; if (db_mode == "safe") - db_flags = islmdb ? MDB_NORDAHEAD : BDB_SAFE_MODE; + db_flags = DBF_SAFE; else if (db_mode == "fast") - db_flags = islmdb ? MDB_NOMETASYNC | MDB_NOSYNC | MDB_NORDAHEAD : BDB_FAST_MODE; + db_flags = DBF_FAST; else if (db_mode == "fastest") - db_flags = islmdb ? MDB_WRITEMAP | MDB_MAPASYNC | MDB_NORDAHEAD | MDB_NOMETASYNC | MDB_NOSYNC : BDB_FASTEST_MODE; + db_flags = DBF_FASTEST; return db_flags; } @@ -132,14 +106,6 @@ int parse_db_arguments(const std::string& db_arg_str, std::string& db_type, int& return 1; } -#if !defined(BERKELEY_DB) - if (db_type == "berkeley") - { - MFATAL("BerkeleyDB support disabled."); - return false; - } -#endif - std::string db_arg_str2 = db_args[1]; boost::split(db_args, db_arg_str2, boost::is_any_of(",")); @@ -155,51 +121,7 @@ int parse_db_arguments(const std::string& db_arg_str, std::string& db_type, int& } if (! db_mode.empty()) { - db_flags = get_db_flags_from_mode(db_type, db_mode); - } - else - { - for (auto& it : db_args) - { - boost::algorithm::trim(it); - if (it.empty()) - continue; - if (db_type == "lmdb") - { - MINFO("LMDB flag: " << it); - if (it == "nosync") - db_flags |= MDB_NOSYNC; - else if (it == "nometasync") - db_flags |= MDB_NOMETASYNC; - else if (it == "writemap") - db_flags |= MDB_WRITEMAP; - else if (it == "mapasync") - db_flags |= MDB_MAPASYNC; - else if (it == "nordahead") - db_flags |= MDB_NORDAHEAD; - else - { - std::cerr << "unrecognized database flag: " << it << ENDL; - return 1; - } - } -#if defined(BERKELEY_DB) - else if (db_type == "berkeley") - { - if (it == "txn_write_nosync") - db_flags = DB_TXN_WRITE_NOSYNC; - else if (it == "txn_nosync") - db_flags = DB_TXN_NOSYNC; - else if (it == "txn_sync") - db_flags = DB_TXN_SYNC; - else - { - std::cerr << "unrecognized database flag: " << it << ENDL; - return 1; - } - } -#endif - } + db_flags = get_db_flags_from_mode(db_mode); } return 0; } @@ -580,12 +502,8 @@ int main(int argc, char* argv[]) epee::string_tools::set_module_name_and_folder(argv[0]); std::string default_db_type = "lmdb"; - std::string default_db_engine_compiled = "blockchain_db"; - - std::unordered_set<std::string> db_types_all = cryptonote::blockchain_db_types; - db_types_all.insert("memory"); - std::string available_dbs = join_set_strings(db_types_all, ", "); + std::string available_dbs = cryptonote::blockchain_db_types(", "); available_dbs = "available: " + available_dbs; uint32_t log_level = 0; @@ -731,7 +649,6 @@ int main(int argc, char* argv[]) std::string db_type; - std::string db_engine_compiled; int db_flags = 0; int res = 0; res = parse_db_arguments(db_arg_str, db_type, db_flags); @@ -741,25 +658,12 @@ int main(int argc, char* argv[]) return 1; } - if (db_types_all.count(db_type) == 0) + if (!cryptonote::blockchain_valid_db_type(db_type)) { std::cerr << "Invalid database type: " << db_type << std::endl; return 1; } - if ((db_type == "lmdb") -#if defined(BERKELEY_DB) - || (db_type == "berkeley") -#endif - ) - { - db_engine_compiled = "blockchain_db"; - } - else - { - db_engine_compiled = "memory"; - } - MINFO("database: " << db_type); MINFO("database flags: " << db_flags); MINFO("verify: " << std::boolalpha << opt_verify << std::noboolalpha); @@ -788,16 +692,6 @@ int main(int argc, char* argv[]) // properties to do so. Both ways work, but fake core isn't necessary in that // circumstance. - if (db_type != "lmdb" -#if defined(BERKELEY_DB) - && db_type != "berkeley" -#endif - ) - { - std::cerr << "database type unrecognized" << ENDL; - return 1; - } - cryptonote::cryptonote_protocol_stub pr; //TODO: stub only for this kind of test, make real validation of relayed objects cryptonote::core core(&pr); core.disable_dns_checkpoints(true); diff --git a/src/blockchain_utilities/blocksdat_file.h b/src/blockchain_utilities/blocksdat_file.h index 4e9d8173b..0a5913058 100644 --- a/src/blockchain_utilities/blocksdat_file.h +++ b/src/blockchain_utilities/blocksdat_file.h @@ -38,7 +38,6 @@ #include "cryptonote_basic/cryptonote_boost_serialization.h" #include "cryptonote_core/blockchain.h" #include "blockchain_db/blockchain_db.h" -#include "blockchain_db/lmdb/db_lmdb.h" #include <algorithm> #include <cstdio> diff --git a/src/common/command_line.cpp b/src/common/command_line.cpp index 925d8ff3b..ad66b2009 100644 --- a/src/common/command_line.cpp +++ b/src/common/command_line.cpp @@ -96,7 +96,7 @@ namespace command_line , "checkpoints from DNS server will be enforced" , false }; - std::string arg_db_type_description = "Specify database type, available: " + boost::algorithm::join(cryptonote::blockchain_db_types, ", "); + std::string arg_db_type_description = "Specify database type, available: " + cryptonote::blockchain_db_types(", "); const command_line::arg_descriptor<std::string> arg_db_type = { "db-type" , arg_db_type_description.c_str() diff --git a/src/common/rpc_client.h b/src/common/rpc_client.h index 8494b4a60..297020ef2 100644 --- a/src/common/rpc_client.h +++ b/src/common/rpc_client.h @@ -69,7 +69,7 @@ namespace tools bool ok = connection.is_open(); if (!ok) { - fail_msg_writer() << "Couldn't connect to daemon"; + fail_msg_writer() << "Couldn't connect to daemon: " << m_http_client.get_host() << ":" << m_http_client.get_port(); return false; } ok = ok && epee::net_utils::invoke_http_json_rpc("/json_rpc", method_name, req, res, m_http_client, t_http_connection::TIMEOUT()); @@ -98,7 +98,7 @@ namespace tools ok = ok && epee::net_utils::invoke_http_json_rpc("/json_rpc", method_name, req, res, m_http_client, t_http_connection::TIMEOUT()); if (!ok) { - fail_msg_writer() << "Couldn't connect to daemon"; + fail_msg_writer() << "Couldn't connect to daemon: " << m_http_client.get_host() << ":" << m_http_client.get_port(); return false; } else if (res.status != CORE_RPC_STATUS_OK) // TODO - handle CORE_RPC_STATUS_BUSY ? @@ -126,7 +126,7 @@ namespace tools ok = ok && epee::net_utils::invoke_http_json(relative_url, req, res, m_http_client, t_http_connection::TIMEOUT()); if (!ok) { - fail_msg_writer() << "Couldn't connect to daemon"; + fail_msg_writer() << "Couldn't connect to daemon: " << m_http_client.get_host() << ":" << m_http_client.get_port(); return false; } else if (res.status != CORE_RPC_STATUS_OK) // TODO - handle CORE_RPC_STATUS_BUSY ? diff --git a/src/common/thread_group.cpp b/src/common/thread_group.cpp index 860d0b732..691a27a25 100644 --- a/src/common/thread_group.cpp +++ b/src/common/thread_group.cpp @@ -32,6 +32,7 @@ #include <limits> #include <stdexcept> +#include "cryptonote_config.h" #include "common/util.h" namespace tools @@ -63,8 +64,10 @@ thread_group::data::data(std::size_t count) , has_work() , stop(false) { threads.reserve(count); + boost::thread::attributes attrs; + attrs.set_stack_size(THREAD_STACK_SIZE); while (count--) { - threads.push_back(boost::thread(&thread_group::data::run, this)); + threads.push_back(boost::thread(attrs, boost::bind(&thread_group::data::run, this))); } } diff --git a/src/crypto/crypto.cpp b/src/crypto/crypto.cpp index 1c7adff3b..5fb670f87 100644 --- a/src/crypto/crypto.cpp +++ b/src/crypto/crypto.cpp @@ -36,18 +36,13 @@ #include <memory> #include <boost/thread/mutex.hpp> #include <boost/thread/lock_guard.hpp> +#include <boost/shared_ptr.hpp> #include "common/varint.h" #include "warnings.h" #include "crypto.h" #include "hash.h" -#if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__DragonFly__) - #include <alloca.h> -#else - #include <stdlib.h> -#endif - namespace crypto { using std::abort; @@ -411,7 +406,9 @@ POP_WARNINGS ge_p3 image_unp; ge_dsmp image_pre; ec_scalar sum, k, h; - rs_comm *const buf = reinterpret_cast<rs_comm *>(alloca(rs_comm_size(pubs_count))); + boost::shared_ptr<rs_comm> buf(reinterpret_cast<rs_comm *>(malloc(rs_comm_size(pubs_count))), free); + if (!buf) + abort(); assert(sec_index < pubs_count); #if !defined(NDEBUG) { @@ -459,7 +456,7 @@ POP_WARNINGS sc_add(&sum, &sum, &sig[i].c); } } - hash_to_scalar(buf, rs_comm_size(pubs_count), h); + hash_to_scalar(buf.get(), rs_comm_size(pubs_count), h); sc_sub(&sig[sec_index].c, &h, &sum); sc_mulsub(&sig[sec_index].r, &sig[sec_index].c, &sec, &k); } @@ -471,7 +468,9 @@ POP_WARNINGS ge_p3 image_unp; ge_dsmp image_pre; ec_scalar sum, h; - rs_comm *const buf = reinterpret_cast<rs_comm *>(alloca(rs_comm_size(pubs_count))); + boost::shared_ptr<rs_comm> buf(reinterpret_cast<rs_comm *>(malloc(rs_comm_size(pubs_count))), free); + if (!buf) + return false; #if !defined(NDEBUG) for (i = 0; i < pubs_count; i++) { assert(check_key(*pubs[i])); @@ -499,7 +498,7 @@ POP_WARNINGS ge_tobytes(&buf->ab[i].b, &tmp2); sc_add(&sum, &sum, &sig[i].c); } - hash_to_scalar(buf, rs_comm_size(pubs_count), h); + hash_to_scalar(buf.get(), rs_comm_size(pubs_count), h); sc_sub(&h, &h, &sum); return sc_isnonzero(&h) == 0; } diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 14a990131..c1faa703f 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -59,6 +59,8 @@ #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "blockchain" +#define FIND_BLOCKCHAIN_SUPPLEMENT_MAX_SIZE (100*1024*1024) // 100 MB + //#include "serialization/json_archive.h" /* TODO: @@ -99,6 +101,9 @@ static const struct { // version 5 starts from block 1288616, which is on or around the 15th of April, 2017. Fork time finalised on 2017-03-14. { 5, 1288616, 0, 1489520158 }, + + // version 6 starts from block 1400000, which is on or around the 16th of September, 2017. Fork time finalised on 2017-08-18. + { 6, 1400000, 0, 1503046577 }, }; static const uint64_t mainnet_hard_fork_version_1_till = 1009826; @@ -126,7 +131,7 @@ static const uint64_t testnet_hard_fork_version_1_till = 624633; //------------------------------------------------------------------ Blockchain::Blockchain(tx_memory_pool& tx_pool) : m_db(), m_tx_pool(tx_pool), m_hardfork(NULL), m_timestamps_and_difficulties_height(0), m_current_block_cumul_sz_limit(0), - m_enforce_dns_checkpoints(false), m_max_prepare_blocks_threads(4), m_db_blocks_per_sync(1), m_db_sync_mode(db_async), m_fast_sync(true), m_show_time_stats(false), m_sync_counter(0), m_cancel(false) + m_enforce_dns_checkpoints(false), m_max_prepare_blocks_threads(4), m_db_blocks_per_sync(1), m_db_sync_mode(db_async), m_db_default_sync(false), m_fast_sync(true), m_show_time_stats(false), m_sync_counter(0), m_cancel(false) { LOG_PRINT_L3("Blockchain::" << __func__); } @@ -2072,8 +2077,8 @@ bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, cons m_db->block_txn_start(true); total_height = get_current_blockchain_height(); - size_t count = 0; - for(size_t i = start_height; i < total_height && count < max_count; i++, count++) + size_t count = 0, size = 0; + for(size_t i = start_height; i < total_height && count < max_count && (size < FIND_BLOCKCHAIN_SUPPLEMENT_MAX_SIZE || count < 3); i++, count++) { blocks.resize(blocks.size()+1); blocks.back().first = m_db->get_block_blob_from_height(i); @@ -2082,6 +2087,9 @@ bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, cons std::list<crypto::hash> mis; get_transactions_blobs(b.tx_hashes, blocks.back().second, mis); CHECK_AND_ASSERT_MES(!mis.size(), false, "internal error, transaction from block not found"); + size += blocks.back().first.size(); + for (const auto &t: blocks.back().second) + size += t.size(); } m_db->block_txn_stop(); return true; @@ -4034,12 +4042,29 @@ bool Blockchain::for_all_txpool_txes(std::function<bool(const crypto::hash&, con void Blockchain::set_user_options(uint64_t maxthreads, uint64_t blocks_per_sync, blockchain_db_sync_mode sync_mode, bool fast_sync) { + if (sync_mode == db_defaultsync) + { + m_db_default_sync = true; + sync_mode = db_async; + } m_db_sync_mode = sync_mode; m_fast_sync = fast_sync; m_db_blocks_per_sync = blocks_per_sync; m_max_prepare_blocks_threads = maxthreads; } +void Blockchain::safesyncmode(const bool onoff) +{ + /* all of this is no-op'd if the user set a specific + * --db-sync-mode at startup. + */ + if (m_db_default_sync) + { + m_db->safesyncmode(onoff); + m_db_sync_mode = onoff ? db_nosync : db_async; + } +} + HardFork::State Blockchain::get_hard_fork_state() const { return m_hardfork->get_state(); diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index 7fa78584b..b8ea657b4 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -65,6 +65,7 @@ namespace cryptonote */ enum blockchain_db_sync_mode { + db_defaultsync, //!< user didn't specify, use db_async db_sync, //!< handle syncing calls instead of the backing db, synchronously db_async, //!< handle syncing calls instead of the backing db, asynchronously db_nosync //!< Leave syncing up to the backing db (safest, but slowest because of disk I/O) @@ -701,6 +702,11 @@ namespace cryptonote blockchain_db_sync_mode sync_mode, bool fast_sync); /** + * @brief Put DB in safe sync mode + */ + void safesyncmode(const bool onoff); + + /** * @brief set whether or not to show/print time statistics * * @param stats the new time stats setting @@ -932,6 +938,7 @@ namespace cryptonote blockchain_db_sync_mode m_db_sync_mode; bool m_fast_sync; bool m_show_time_stats; + bool m_db_default_sync; uint64_t m_db_blocks_per_sync; uint64_t m_max_prepare_blocks_threads; uint64_t m_fake_pow_calc_time; diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 19f4aaca9..c406dd0b4 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -48,10 +48,6 @@ using namespace epee; #include "cryptonote_basic/checkpoints.h" #include "ringct/rctTypes.h" #include "blockchain_db/blockchain_db.h" -#include "blockchain_db/lmdb/db_lmdb.h" -#if defined(BERKELEY_DB) -#include "blockchain_db/berkeleydb/db_bdb.h" -#endif #include "ringct/rctSigs.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -246,6 +242,12 @@ namespace cryptonote return m_blockchain_storage.get_transactions_blobs(txs_ids, txs, missed_txs); } //----------------------------------------------------------------------------------------------- + bool core::get_txpool_backlog(std::vector<tx_backlog_entry>& backlog) const + { + m_mempool.get_transaction_backlog(backlog); + return true; + } + //----------------------------------------------------------------------------------------------- bool core::get_transactions(const std::vector<crypto::hash>& txs_ids, std::list<transaction>& txs, std::list<crypto::hash>& missed_txs) const { return m_blockchain_storage.get_transactions(txs_ids, txs, missed_txs); @@ -308,20 +310,8 @@ namespace cryptonote // folder might not be a directory, etc, etc catch (...) { } - BlockchainDB* db = nullptr; - uint64_t DBS_FAST_MODE = 0; - uint64_t DBS_FASTEST_MODE = 0; - uint64_t DBS_SAFE_MODE = 0; - uint64_t DBS_SALVAGE = 0; - if (db_type == "lmdb") - { - db = new BlockchainLMDB(); - DBS_SAFE_MODE = MDB_NORDAHEAD; - DBS_FAST_MODE = MDB_NORDAHEAD | MDB_NOSYNC; - DBS_FASTEST_MODE = MDB_NORDAHEAD | MDB_NOSYNC | MDB_WRITEMAP | MDB_MAPASYNC; - DBS_SALVAGE = MDB_PREVSNAPSHOT; - } - else + BlockchainDB* db = new_db(db_type); + if (db == NULL) { LOG_ERROR("Attempted to use non-existent database type"); return false; @@ -332,7 +322,7 @@ namespace cryptonote const std::string filename = folder.string(); // default to fast:async:1 - blockchain_db_sync_mode sync_mode = db_async; + blockchain_db_sync_mode sync_mode = db_defaultsync; uint64_t blocks_per_sync = 1; try @@ -347,7 +337,7 @@ namespace cryptonote MDEBUG("option: " << option); // default to fast:async:1 - uint64_t DEFAULT_FLAGS = DBS_FAST_MODE; + uint64_t DEFAULT_FLAGS = DBF_FAST; if(options.size() == 0) { @@ -361,15 +351,19 @@ namespace cryptonote if(options[0] == "safe") { safemode = true; - db_flags = DBS_SAFE_MODE; + db_flags = DBF_SAFE; sync_mode = db_nosync; } else if(options[0] == "fast") - db_flags = DBS_FAST_MODE; + { + db_flags = DBF_FAST; + sync_mode = db_async; + } else if(options[0] == "fastest") { - db_flags = DBS_FASTEST_MODE; + db_flags = DBF_FASTEST; blocks_per_sync = 1000; // default to fastest:async:1000 + sync_mode = db_async; } else db_flags = DEFAULT_FLAGS; @@ -392,7 +386,7 @@ namespace cryptonote } if (db_salvage) - db_flags |= DBS_SALVAGE; + db_flags |= DBF_SALVAGE; db->open(filename, db_flags); if(!db->m_open) @@ -1049,6 +1043,11 @@ namespace cryptonote m_miner.on_synchronized(); } //----------------------------------------------------------------------------------------------- + void core::safesyncmode(const bool onoff) + { + m_blockchain_storage.safesyncmode(onoff); + } + //----------------------------------------------------------------------------------------------- bool core::add_new_block(const block& b, block_verification_context& bvc) { return m_blockchain_storage.add_new_block(b, bvc); diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index f17a6dfe6..f565afd87 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -428,6 +428,13 @@ namespace cryptonote bool get_pool_transactions(std::list<transaction>& txs) const; /** + * @copydoc tx_memory_pool::get_txpool_backlog + * + * @note see tx_memory_pool::get_txpool_backlog + */ + bool get_txpool_backlog(std::vector<tx_backlog_entry>& backlog) const; + + /** * @copydoc tx_memory_pool::get_transactions * * @note see tx_memory_pool::get_transactions @@ -614,6 +621,13 @@ namespace cryptonote void on_synchronized(); /** + * @copydoc Blockchain::safesyncmode + * + * 2note see Blockchain::safesyncmode + */ + void safesyncmode(const bool onoff); + + /** * @brief sets the target blockchain height * * @param target_blockchain_height the height to set diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index 33b1d4101..7b98acb63 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -553,6 +553,17 @@ namespace cryptonote }); } //------------------------------------------------------------------ + void tx_memory_pool::get_transaction_backlog(std::vector<tx_backlog_entry>& backlog) const + { + CRITICAL_REGION_LOCAL(m_transactions_lock); + CRITICAL_REGION_LOCAL1(m_blockchain); + const uint64_t now = time(NULL); + m_blockchain.for_all_txpool_txes([&backlog, now](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ + backlog.push_back({meta.blob_size, meta.fee, meta.receive_time - now}); + return true; + }); + } + //------------------------------------------------------------------ void tx_memory_pool::get_transaction_stats(struct txpool_stats& stats) const { CRITICAL_REGION_LOCAL(m_transactions_lock); @@ -575,7 +586,7 @@ namespace cryptonote stats.num_10m++; if (meta.last_failed_height) stats.num_failing++; - uint64_t age = now - meta.receive_time; + uint64_t age = now - meta.receive_time + (now == meta.receive_time); agebytes[age].txs++; agebytes[age].bytes += meta.blob_size; return true; diff --git a/src/cryptonote_core/tx_pool.h b/src/cryptonote_core/tx_pool.h index 47a41d070..6414620c9 100644 --- a/src/cryptonote_core/tx_pool.h +++ b/src/cryptonote_core/tx_pool.h @@ -243,6 +243,13 @@ namespace cryptonote void get_transaction_hashes(std::vector<crypto::hash>& txs) const; /** + * @brief get (size, fee, receive time) for all transaction in the pool + * + * @param txs return-by-reference that data + */ + void get_transaction_backlog(std::vector<tx_backlog_entry>& backlog) const; + + /** * @brief get a summary statistics of all transaction hashes in the pool * * @param stats return-by-reference the pool statistics diff --git a/src/cryptonote_protocol/block_queue.cpp b/src/cryptonote_protocol/block_queue.cpp index 94d31404e..02a8e3ec2 100644 --- a/src/cryptonote_protocol/block_queue.cpp +++ b/src/cryptonote_protocol/block_queue.cpp @@ -52,8 +52,11 @@ namespace cryptonote void block_queue::add_blocks(uint64_t height, std::list<cryptonote::block_complete_entry> bcel, const boost::uuids::uuid &connection_id, float rate, size_t size) { boost::unique_lock<boost::recursive_mutex> lock(mutex); - remove_span(height); + std::list<crypto::hash> hashes; + bool has_hashes = remove_span(height, &hashes); blocks.insert(span(height, std::move(bcel), connection_id, rate, size)); + if (has_hashes) + set_span_hashes(height, connection_id, hashes); } void block_queue::add_blocks(uint64_t height, uint64_t nblocks, const boost::uuids::uuid &connection_id, boost::posix_time::ptime time) @@ -92,17 +95,20 @@ void block_queue::flush_stale_spans(const std::set<boost::uuids::uuid> &live_con } } -void block_queue::remove_span(uint64_t start_block_height) +bool block_queue::remove_span(uint64_t start_block_height, std::list<crypto::hash> *hashes) { boost::unique_lock<boost::recursive_mutex> lock(mutex); for (block_map::iterator i = blocks.begin(); i != blocks.end(); ++i) { if (i->start_block_height == start_block_height) { + if (hashes) + *hashes = std::move(i->hashes); blocks.erase(i); - return; + return true; } } + return false; } void block_queue::remove_spans(const boost::uuids::uuid &connection_id, uint64_t start_block_height) @@ -278,6 +284,22 @@ bool block_queue::get_next_span(uint64_t &height, std::list<cryptonote::block_co return false; } +bool block_queue::has_next_span(const boost::uuids::uuid &connection_id, bool &filled) const +{ + boost::unique_lock<boost::recursive_mutex> lock(mutex); + if (blocks.empty()) + return false; + block_map::const_iterator i = blocks.begin(); + if (is_blockchain_placeholder(*i)) + ++i; + if (i == blocks.end()) + return false; + if (i->connection_id != connection_id) + return false; + filled = !i->blocks.empty(); + return true; +} + size_t block_queue::get_data_size() const { boost::unique_lock<boost::recursive_mutex> lock(mutex); diff --git a/src/cryptonote_protocol/block_queue.h b/src/cryptonote_protocol/block_queue.h index fa1a0f217..13d4619bf 100644 --- a/src/cryptonote_protocol/block_queue.h +++ b/src/cryptonote_protocol/block_queue.h @@ -71,7 +71,7 @@ namespace cryptonote void add_blocks(uint64_t height, uint64_t nblocks, const boost::uuids::uuid &connection_id, boost::posix_time::ptime time = boost::date_time::min_date_time); void flush_spans(const boost::uuids::uuid &connection_id, bool all = false); void flush_stale_spans(const std::set<boost::uuids::uuid> &live_connections); - void remove_span(uint64_t start_block_height); + bool remove_span(uint64_t start_block_height, std::list<crypto::hash> *hashes = NULL); void remove_spans(const boost::uuids::uuid &connection_id, uint64_t start_block_height); uint64_t get_max_block_height() const; void print() const; @@ -82,6 +82,7 @@ namespace cryptonote std::pair<uint64_t, uint64_t> get_next_span_if_scheduled(std::list<crypto::hash> &hashes, boost::uuids::uuid &connection_id, boost::posix_time::ptime &time) const; void set_span_hashes(uint64_t start_height, const boost::uuids::uuid &connection_id, std::list<crypto::hash> hashes); bool get_next_span(uint64_t &height, std::list<cryptonote::block_complete_entry> &bcel, boost::uuids::uuid &connection_id, bool filled = true) const; + bool has_next_span(const boost::uuids::uuid &connection_id, bool &filled) const; size_t get_data_size() const; size_t get_num_filled_spans_prefix() const; size_t get_num_filled_spans() const; diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.h b/src/cryptonote_protocol/cryptonote_protocol_handler.h index d94747769..d54687e6a 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.h @@ -111,6 +111,7 @@ namespace cryptonote std::list<connection_info> get_connections(); const block_queue &get_block_queue() const { return m_block_queue; } void stop(); + void on_connection_close(cryptonote_connection_context &context); private: //----------------- commands handlers ---------------------------------------------- int handle_notify_new_block(int command, NOTIFY_NEW_BLOCK::request& arg, cryptonote_connection_context& context); @@ -133,6 +134,7 @@ namespace cryptonote bool should_download_next_span(cryptonote_connection_context& context) const; void drop_connection(cryptonote_connection_context &context, bool add_fail, bool flush_all_spans); bool kick_idle_peers(); + int try_add_next_blocks(cryptonote_connection_context &context); t_core& m_core; diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index daefe88b7..e762cf9c8 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -106,6 +106,11 @@ namespace cryptonote LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); post_notify<NOTIFY_REQUEST_CHAIN>(r, context); } + else if(context.m_state == cryptonote_connection_context::state_standby) + { + context.m_state = cryptonote_connection_context::state_synchronizing; + try_add_next_blocks(context); + } return true; } @@ -263,7 +268,9 @@ namespace cryptonote const uint8_t version = m_core.get_ideal_hard_fork_version(hshd.current_height - 1); if (version >= 6 && version != hshd.top_version) { - LOG_DEBUG_CC(context, "Ignoring due to wrong top version " << (unsigned)hshd.top_version << ", expected " << (unsigned)version); + if (version < hshd.top_version) + MCLOG_RED(el::Level::Warning, "global", context << " peer claims higher version that we think - we may be forked from the network and a software upgrade may be needed"); + LOG_DEBUG_CC(context, "Ignoring due to wrong top version for block " << (hshd.current_height - 1) << ": " << (unsigned)hshd.top_version << ", expected " << (unsigned)version); return false; } @@ -286,15 +293,17 @@ namespace cryptonote /* As I don't know if accessing hshd from core could be a good practice, I prefer pushing target height to the core at the same time it is pushed to the user. Nz. */ - m_core.set_target_blockchain_height(static_cast<int64_t>(hshd.current_height)); + m_core.set_target_blockchain_height((hshd.current_height)); int64_t diff = static_cast<int64_t>(hshd.current_height) - static_cast<int64_t>(m_core.get_current_blockchain_height()); - int64_t max_block_height = max(static_cast<int64_t>(hshd.current_height),static_cast<int64_t>(m_core.get_current_blockchain_height())); - int64_t last_block_v1 = m_core.get_testnet() ? 624633 : 1009826; - int64_t diff_v2 = max_block_height > last_block_v1 ? min(abs(diff), max_block_height - last_block_v1) : 0; + uint64_t abs_diff = std::abs(diff); + uint64_t max_block_height = max(hshd.current_height,m_core.get_current_blockchain_height()); + uint64_t last_block_v1 = m_core.get_testnet() ? 624633 : 1009826; + uint64_t diff_v2 = max_block_height > last_block_v1 ? min(abs_diff, max_block_height - last_block_v1) : 0; MCLOG(is_inital ? el::Level::Info : el::Level::Debug, "global", context << "Sync data returned a new top block candidate: " << m_core.get_current_blockchain_height() << " -> " << hshd.current_height - << " [Your node is " << std::abs(diff) << " blocks (" << ((abs(diff) - diff_v2) / (24 * 60 * 60 / DIFFICULTY_TARGET_V1)) + (diff_v2 / (24 * 60 * 60 / DIFFICULTY_TARGET_V2)) << " days) " + << " [Your node is " << abs_diff << " blocks (" << ((abs_diff - diff_v2) / (24 * 60 * 60 / DIFFICULTY_TARGET_V1)) + (diff_v2 / (24 * 60 * 60 / DIFFICULTY_TARGET_V2)) << " days) " << (0 <= diff ? std::string("behind") : std::string("ahead")) << "] " << ENDL << "SYNCHRONIZATION started"); + m_core.safesyncmode(false); } LOG_PRINT_L1("Remote blockchain height: " << hshd.current_height << ", id: " << hshd.top_id); context.m_state = cryptonote_connection_context::state_synchronizing; @@ -309,7 +318,7 @@ namespace cryptonote bool t_cryptonote_protocol_handler<t_core>::get_payload_sync_data(CORE_SYNC_DATA& hshd) { m_core.get_blockchain_top(hshd.current_height, hshd.top_id); - hshd.top_version = m_core.get_hard_fork_version(hshd.current_height); + hshd.top_version = m_core.get_ideal_hard_fork_version(hshd.current_height); hshd.cumulative_difficulty = m_core.get_block_cumulative_difficulty(hshd.current_height); hshd.current_height +=1; return true; @@ -819,8 +828,6 @@ namespace cryptonote { MLOG_P2P_MESSAGE("Received NOTIFY_RESPONSE_GET_OBJECTS (" << arg.blocks.size() << " blocks, " << arg.txs.size() << " txes)"); - bool force_next_span = false; - // calculate size of request size_t size = 0; for (const auto &element : arg.txs) size += element.size(); @@ -938,19 +945,34 @@ namespace cryptonote context.m_last_known_hash = cryptonote::get_blob_hash(arg.blocks.back().block); - if (m_core.get_test_drop_download() && m_core.get_test_drop_download_height()) { // DISCARD BLOCKS for testing + if (!m_core.get_test_drop_download() || !m_core.get_test_drop_download_height()) { // DISCARD BLOCKS for testing + return 1; + } + } - // We try to lock the sync lock. If we can, it means no other thread is - // currently adding blocks, so we do that for as long as we can from the - // block queue. Then, we go back to download. - const boost::unique_lock<boost::mutex> sync{m_sync_lock, boost::try_to_lock}; - if (!sync.owns_lock()) - { - MINFO("Failed to lock m_sync_lock, going back to download"); - goto skip; - } - MDEBUG(context << " lock m_sync_lock, adding blocks to chain..."); +skip: + try_add_next_blocks(context); + return 1; + } + template<class t_core> + int t_cryptonote_protocol_handler<t_core>::try_add_next_blocks(cryptonote_connection_context& context) + { + bool force_next_span = false; + + { + // We try to lock the sync lock. If we can, it means no other thread is + // currently adding blocks, so we do that for as long as we can from the + // block queue. Then, we go back to download. + const boost::unique_lock<boost::mutex> sync{m_sync_lock, boost::try_to_lock}; + if (!sync.owns_lock()) + { + MINFO("Failed to lock m_sync_lock, going back to download"); + goto skip; + } + MDEBUG(context << " lock m_sync_lock, adding blocks to chain..."); + + { m_core.pause_mine(); epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler( boost::bind(&t_core::resume_mine, &m_core)); @@ -984,21 +1006,15 @@ namespace cryptonote // - later in an alt chain // - orphan // if it was requested, then it'll be resolved later, otherwise it's an orphan - bool parent_requested = false; - m_p2p->for_each_connection([&](cryptonote_connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags)->bool{ - if (context.m_requested_objects.find(new_block.prev_id) != context.m_requested_objects.end()) - { - parent_requested = true; - return false; - } - return true; - }); + bool parent_requested = m_block_queue.requested(new_block.prev_id); if (!parent_requested) { - LOG_ERROR_CCONTEXT("Got block with unknown parent which was not requested - dropping connection"); - // in case the peer had dropped beforehand, remove the span anyway so other threads can wake up and get it - m_block_queue.remove_spans(span_connection_id, start_height); - return 1; + // this can happen if a connection was sicced onto a late span, if it did not have those blocks, + // since we don't know that at the sic time + LOG_ERROR_CCONTEXT("Got block with unknown parent which was not requested - querying block hashes"); + context.m_needed_objects.clear(); + context.m_last_response_height = 0; + goto skip; } // parent was requested, so we wait for it to be retrieved @@ -1007,6 +1023,7 @@ namespace cryptonote } const boost::posix_time::ptime start = boost::posix_time::microsec_clock::universal_time(); + context.m_last_request_time = start; m_core.prepare_handle_incoming_blocks(blocks); @@ -1108,7 +1125,7 @@ namespace cryptonote << timing_message); } } - } // if not DISCARD BLOCK + } if (should_download_next_span(context)) { @@ -1179,9 +1196,17 @@ skip: std::list<crypto::hash> hashes; boost::uuids::uuid span_connection_id; boost::posix_time::ptime request_time; - std::pair<uint64_t, uint64_t> span = m_block_queue.get_next_span_if_scheduled(hashes, span_connection_id, request_time); + std::pair<uint64_t, uint64_t> span; + + span = m_block_queue.get_start_gap_span(); + if (span.second > 0) + { + MDEBUG(context << " we should download it as there is a gap"); + return true; + } // if the next span is not scheduled (or there is none) + span = m_block_queue.get_next_span_if_scheduled(hashes, span_connection_id, request_time); if (span.second == 0) { // we might be in a weird case where there is a filled next span, @@ -1270,6 +1295,17 @@ skip: first = false; context.m_state = cryptonote_connection_context::state_standby; } + + // this needs doing after we went to standby, so the callback knows what to do + bool filled; + if (m_block_queue.has_next_span(context.m_connection_id, filled) && !filled) + { + MDEBUG(context << " we have the next span, and it is scheduled, resuming"); + ++context.m_callback_request_count; + m_p2p->request_callback(context); + return 1; + } + for (size_t n = 0; n < 50; ++n) { if (m_stopping) @@ -1289,9 +1325,8 @@ skip: size_t count = 0; const size_t count_limit = m_core.get_block_sync_size(m_core.get_current_blockchain_height()); std::pair<uint64_t, uint64_t> span = std::make_pair(0, 0); - if (force_next_span) { - MDEBUG(context << " force_next_span is true, trying next span"); + MDEBUG(context << " checking for gap"); span = m_block_queue.get_start_gap_span(); if (span.second > 0) { @@ -1311,6 +1346,9 @@ skip: } MDEBUG(context << " we have the hashes for this gap"); } + } + if (force_next_span) + { if (span.second == 0) { std::list<crypto::hash> hashes; @@ -1360,7 +1398,12 @@ skip: for (const auto &hash: hashes) { req.blocks.push_back(hash); + ++count; context.m_requested_objects.insert(hash); + // that's atrocious O(n) wise, but this is rare + auto i = std::find(context.m_needed_objects.begin(), context.m_needed_objects.end(), hash); + if (i != context.m_needed_objects.end()) + context.m_needed_objects.erase(i); } } } @@ -1384,14 +1427,12 @@ skip: return false; } - std::list<crypto::hash> hashes; auto it = context.m_needed_objects.begin(); for (size_t n = 0; n < span.second; ++n) { req.blocks.push_back(*it); ++count; context.m_requested_objects.insert(*it); - hashes.push_back(*it); auto j = it++; context.m_needed_objects.erase(j); } @@ -1399,7 +1440,7 @@ skip: context.m_last_request_time = boost::posix_time::microsec_clock::universal_time(); LOG_PRINT_CCONTEXT_L1("-->>NOTIFY_REQUEST_GET_OBJECTS: blocks.size()=" << req.blocks.size() << ", txs.size()=" << req.txs.size() - << "requested blocks count=" << count << " / " << count_limit << " from " << span.first); + << "requested blocks count=" << count << " / " << count_limit << " from " << span.first << ", first hash " << req.blocks.front()); //epee::net_utils::network_throttle_manager::get_global_throttle_inreq().logger_handle_net("log/dr-monero/net/req-all.data", sec, get_avg_block_size()); post_notify<NOTIFY_REQUEST_GET_OBJECTS>(req, context); @@ -1473,6 +1514,7 @@ skip: << "**********************************************************************"); m_core.on_synchronized(); } + m_core.safesyncmode(true); return true; } //------------------------------------------------------------------------------------------------------------------------ @@ -1523,6 +1565,10 @@ skip: drop_connection(context, false, false); return 1; } + + if (arg.total_height > m_core.get_target_blockchain_height()) + m_core.set_target_blockchain_height(arg.total_height); + return 1; } //------------------------------------------------------------------------------------------------------------------------ @@ -1582,8 +1628,15 @@ skip: { if (add_fail) m_p2p->add_host_fail(context.m_remote_address); + m_p2p->drop_connection(context); + m_block_queue.flush_spans(context.m_connection_id, flush_all_spans); + } + //------------------------------------------------------------------------------------------------------------------------ + template<class t_core> + void t_cryptonote_protocol_handler<t_core>::on_connection_close(cryptonote_connection_context &context) + { uint64_t target = 0; m_p2p->for_each_connection([&](const connection_context& cntxt, nodetool::peerid_type peer_id, uint32_t support_flags) { if (cntxt.m_state >= cryptonote_connection_context::state_synchronizing && cntxt.m_connection_id != context.m_connection_id) @@ -1597,7 +1650,7 @@ skip: m_core.set_target_blockchain_height(target); } - m_block_queue.flush_spans(context.m_connection_id, flush_all_spans); + m_block_queue.flush_spans(context.m_connection_id, false); } //------------------------------------------------------------------------------------------------------------------------ diff --git a/src/daemon/main.cpp b/src/daemon/main.cpp index 19dd02171..456eeee64 100644 --- a/src/daemon/main.cpp +++ b/src/daemon/main.cpp @@ -145,13 +145,10 @@ int main(int argc, char const * argv[]) std::string db_type = command_line::get_arg(vm, command_line::arg_db_type); // verify that blockchaindb type is valid - if(cryptonote::blockchain_db_types.count(db_type) == 0) + if(!cryptonote::blockchain_valid_db_type(db_type)) { - std::cout << "Invalid database type (" << db_type << "), available types are:" << std::endl; - for (const auto& type : cryptonote::blockchain_db_types) - { - std::cout << "\t" << type << std::endl; - } + std::cout << "Invalid database type (" << db_type << "), available types are: " << + cryptonote::blockchain_db_types(", ") << std::endl; return 0; } diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp index bd2142ab1..e35756a22 100644 --- a/src/daemon/rpc_command_executor.cpp +++ b/src/daemon/rpc_command_executor.cpp @@ -113,18 +113,6 @@ namespace { return base; return base + " -- " + status; } - - std::string pad(std::string s, size_t n, char c = ' ', bool prepend = false) - { - if (s.size() < n) - { - if (prepend) - s = std::string(n - s.size(), c) + s; - else - s.append(n - s.size(), c); - } - return s; - } } t_rpc_command_executor::t_rpc_command_executor( @@ -497,7 +485,7 @@ bool t_rpc_command_executor::print_connections() { tools::msg_writer() //<< std::setw(30) << std::left << in_out << std::setw(30) << std::left << address - << std::setw(20) << pad(info.peer_id, 16, '0', true) + << std::setw(20) << epee::string_tools::pad_string(info.peer_id, 16, '0', true) << std::setw(20) << info.support_flags << std::setw(30) << std::to_string(info.recv_count) + "(" + std::to_string(info.recv_idle_time) + ")/" + std::to_string(info.send_count) + "(" + std::to_string(info.send_idle_time) + ")" << std::setw(25) << info.state @@ -939,6 +927,8 @@ bool t_rpc_command_executor::print_transaction_pool_short() { bool t_rpc_command_executor::print_transaction_pool_stats() { cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL_STATS::request req; cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL_STATS::response res; + cryptonote::COMMAND_RPC_GET_INFO::request ireq; + cryptonote::COMMAND_RPC_GET_INFO::response ires; std::string fail_message = "Problem fetching transaction pool stats"; @@ -948,6 +938,10 @@ bool t_rpc_command_executor::print_transaction_pool_stats() { { return true; } + if (!m_rpc_client->rpc_request(ireq, ires, "/getinfo", fail_message.c_str())) + { + return true; + } } else { @@ -957,15 +951,32 @@ bool t_rpc_command_executor::print_transaction_pool_stats() { tools::fail_msg_writer() << make_error(fail_message, res.status); return true; } + if (!m_rpc_server->on_get_info(ireq, ires) || ires.status != CORE_RPC_STATUS_OK) + { + tools::fail_msg_writer() << make_error(fail_message, ires.status); + return true; + } } size_t n_transactions = res.pool_stats.txs_total; const uint64_t now = time(NULL); size_t avg_bytes = n_transactions ? res.pool_stats.bytes_total / n_transactions : 0; + std::string backlog_message; + const uint64_t full_reward_zone = ires.block_size_limit / 2; + if (res.pool_stats.bytes_total <= full_reward_zone) + { + backlog_message = "no backlog"; + } + else + { + uint64_t backlog = (res.pool_stats.bytes_total + full_reward_zone - 1) / full_reward_zone; + backlog_message = (boost::format("estimated %u block (%u minutes) backlog") % backlog % (backlog * DIFFICULTY_TARGET_V2 / 60)).str(); + } + tools::msg_writer() << n_transactions << " tx(es), " << res.pool_stats.bytes_total << " bytes total (min " << res.pool_stats.bytes_min << ", max " << res.pool_stats.bytes_max << ", avg " << avg_bytes << ")" << std::endl - << "fees " << cryptonote::print_money(res.pool_stats.fee_total) << " (avg " << cryptonote::print_money(n_transactions ? res.pool_stats.fee_total / n_transactions : 0) << " per tx" << ", " << cryptonote::print_money(res.pool_stats.bytes_total ? res.pool_stats.fee_total / res.pool_stats.bytes_total : 0) << " per byte )" << std::endl - << res.pool_stats.num_not_relayed << " not relayed, " << res.pool_stats.num_failing << " failing, " << res.pool_stats.num_10m << " older than 10 minutes (oldest " << (res.pool_stats.oldest == 0 ? "-" : get_human_time_ago(res.pool_stats.oldest, now)) << ")"; + << "fees " << cryptonote::print_money(res.pool_stats.fee_total) << " (avg " << cryptonote::print_money(n_transactions ? res.pool_stats.fee_total / n_transactions : 0) << " per tx" << ", " << cryptonote::print_money(res.pool_stats.bytes_total ? res.pool_stats.fee_total / res.pool_stats.bytes_total : 0) << " per byte)" << std::endl + << res.pool_stats.num_not_relayed << " not relayed, " << res.pool_stats.num_failing << " failing, " << res.pool_stats.num_10m << " older than 10 minutes (oldest " << (res.pool_stats.oldest == 0 ? "-" : get_human_time_ago(res.pool_stats.oldest, now)) << "), " << backlog_message; if (n_transactions > 1 && res.pool_stats.histo.size()) { @@ -1744,12 +1755,12 @@ bool t_rpc_command_executor::sync_info() tools::success_msg_writer() << std::to_string(res.peers.size()) << " peers"; for (const auto &p: res.peers) { - std::string address = pad(p.info.address, 24); + std::string address = epee::string_tools::pad_string(p.info.address, 24); uint64_t nblocks = 0, size = 0; for (const auto &s: res.spans) if (s.rate > 0.0f && s.connection_id == p.info.connection_id) nblocks += s.nblocks, size += s.size; - tools::success_msg_writer() << address << " " << pad(p.info.peer_id, 16, '0', true) << " " << p.info.height << " " << p.info.current_download << " kB/s, " << nblocks << " blocks / " << size/1e6 << " MB queued"; + tools::success_msg_writer() << address << " " << epee::string_tools::pad_string(p.info.peer_id, 16, '0', true) << " " << p.info.height << " " << p.info.current_download << " kB/s, " << nblocks << " blocks / " << size/1e6 << " MB queued"; } uint64_t total_size = 0; @@ -1758,7 +1769,7 @@ bool t_rpc_command_executor::sync_info() tools::success_msg_writer() << std::to_string(res.spans.size()) << " spans, " << total_size/1e6 << " MB"; for (const auto &s: res.spans) { - std::string address = pad(s.remote_address, 24); + std::string address = epee::string_tools::pad_string(s.remote_address, 24); if (s.size == 0) { tools::success_msg_writer() << address << " " << s.nblocks << " (" << s.start_block_height << " - " << (s.start_block_height + s.nblocks - 1) << ") -"; diff --git a/src/debug_utilities/CMakeLists.txt b/src/debug_utilities/CMakeLists.txt index 99198dc57..e6d49fd61 100644 --- a/src/debug_utilities/CMakeLists.txt +++ b/src/debug_utilities/CMakeLists.txt @@ -37,6 +37,7 @@ monero_add_executable(cn_deserialize target_link_libraries(cn_deserialize LINK_PRIVATE cryptonote_core + common blockchain_db p2p epee diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index e179fc14f..889cfaf9d 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -1077,7 +1077,7 @@ namespace nodetool bool node_server<t_payload_net_handler>::make_new_connection_from_anchor_peerlist(const std::vector<anchor_peerlist_entry>& anchor_peerlist) { for (const auto& pe: anchor_peerlist) { - _note("Considering connecting (out) to peer: " << pe.id << " " << pe.adr.str()); + _note("Considering connecting (out) to peer: " << peerid_type(pe.id) << " " << pe.adr.str()); if(is_peer_used(pe)) { _note("Peer is used"); @@ -1092,7 +1092,7 @@ namespace nodetool continue; } - MDEBUG("Selected peer: " << pe.id << " " << pe.adr.str() + MDEBUG("Selected peer: " << peerid_to_string(pe.id) << " " << pe.adr.str() << "[peer_type=" << anchor << "] first_seen: " << epee::misc_utils::get_time_interval_string(time(NULL) - pe.first_seen)); @@ -1145,7 +1145,7 @@ namespace nodetool ++try_count; - _note("Considering connecting (out) to peer: " << pe.id << " " << pe.adr.str()); + _note("Considering connecting (out) to peer: " << peerid_to_string(pe.id) << " " << pe.adr.str()); if(is_peer_used(pe)) { _note("Peer is used"); @@ -1158,7 +1158,7 @@ namespace nodetool if(is_addr_recently_failed(pe.adr)) continue; - MDEBUG("Selected peer: " << pe.id << " " << pe.adr.str() + MDEBUG("Selected peer: " << peerid_to_string(pe.id) << " " << pe.adr.str() << "[peer_list=" << (use_white_list ? white : gray) << "] last_seen: " << (pe.last_seen ? epee::misc_utils::get_time_interval_string(time(NULL) - pe.last_seen) : "never")); @@ -1795,6 +1795,8 @@ namespace nodetool m_peerlist.remove_from_peer_anchor(na); } + m_payload_handler.on_connection_close(context); + MINFO("["<< epee::net_utils::print_connection_context(context) << "] CLOSE CONNECTION"); } @@ -1960,14 +1962,14 @@ namespace nodetool if (!success) { m_peerlist.remove_from_peer_gray(pe); - LOG_PRINT_L2("PEER EVICTED FROM GRAY PEER LIST IP address: " << pe.adr.host_str() << " Peer ID: " << std::hex << pe.id); + LOG_PRINT_L2("PEER EVICTED FROM GRAY PEER LIST IP address: " << pe.adr.host_str() << " Peer ID: " << peerid_type(pe.id)); return true; } m_peerlist.set_peer_just_seen(pe.id, pe.adr); - LOG_PRINT_L2("PEER PROMOTED TO WHITE PEER LIST IP address: " << pe.adr.host_str() << " Peer ID: " << std::hex << pe.id); + LOG_PRINT_L2("PEER PROMOTED TO WHITE PEER LIST IP address: " << pe.adr.host_str() << " Peer ID: " << peerid_type(pe.id)); return true; } diff --git a/src/p2p/p2p_protocol_defs.h b/src/p2p/p2p_protocol_defs.h index f38615def..f2b2cd1da 100644 --- a/src/p2p/p2p_protocol_defs.h +++ b/src/p2p/p2p_protocol_defs.h @@ -44,6 +44,13 @@ namespace nodetool typedef boost::uuids::uuid uuid; typedef uint64_t peerid_type; + static inline std::string peerid_to_string(peerid_type peer_id) + { + std::ostringstream s; + s << std::hex << peer_id; + return epee::string_tools::pad_string(s.str(), 16, '0', true); + } + #pragma pack (push, 1) struct network_address_old diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index a5de36118..a6a61705b 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -205,14 +205,17 @@ namespace cryptonote } size_t txidx = 0; ntxes += bd.second.size(); - for(const auto& t: bd.second) + for (std::list<cryptonote::blobdata>::iterator i = bd.second.begin(); i != bd.second.end(); ++i) { + unpruned_size += i->size(); if (req.prune) - res.blocks.back().txs.push_back(get_pruned_tx_blob(t)); + res.blocks.back().txs.push_back(get_pruned_tx_blob(std::move(*i))); else - res.blocks.back().txs.push_back(t); + res.blocks.back().txs.push_back(std::move(*i)); + i->clear(); + i->shrink_to_fit(); pruned_size += res.blocks.back().txs.back().size(); - unpruned_size += t.size(); + res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices()); bool r = m_core.get_tx_outputs_gindexs(b.tx_hashes[txidx++], res.output_indices.back().indices.back().indices); if (!r) @@ -1725,6 +1728,26 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_get_txpool_backlog(const COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::response& res, epee::json_rpc::error& error_resp) + { + if(!check_core_busy()) + { + error_resp.code = CORE_RPC_ERROR_CODE_CORE_BUSY; + error_resp.message = "Core is busy."; + return false; + } + + if (!m_core.get_txpool_backlog(res.backlog)) + { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = "Failed to get txpool backlog"; + return false; + } + + res.status = CORE_RPC_STATUS_OK; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ const command_line::arg_descriptor<std::string> core_rpc_server::arg_rpc_bind_port = { "rpc-bind-port" diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index 1d1d9da66..b526277a8 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -125,6 +125,7 @@ namespace cryptonote MAP_JON_RPC_WE_IF("get_alternate_chains",on_get_alternate_chains, COMMAND_RPC_GET_ALTERNATE_CHAINS, !m_restricted) MAP_JON_RPC_WE_IF("relay_tx", on_relay_tx, COMMAND_RPC_RELAY_TX, !m_restricted) MAP_JON_RPC_WE_IF("sync_info", on_sync_info, COMMAND_RPC_SYNC_INFO, !m_restricted) + MAP_JON_RPC_WE_IF("get_txpool_backlog", on_get_txpool_backlog, COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG, !m_restricted) END_JSON_RPC_MAP() END_URI_MAP2() @@ -182,6 +183,7 @@ namespace cryptonote bool on_get_alternate_chains(const COMMAND_RPC_GET_ALTERNATE_CHAINS::request& req, COMMAND_RPC_GET_ALTERNATE_CHAINS::response& res, epee::json_rpc::error& error_resp); bool on_relay_tx(const COMMAND_RPC_RELAY_TX::request& req, COMMAND_RPC_RELAY_TX::response& res, epee::json_rpc::error& error_resp); bool on_sync_info(const COMMAND_RPC_SYNC_INFO::request& req, COMMAND_RPC_SYNC_INFO::response& res, epee::json_rpc::error& error_resp); + bool on_get_txpool_backlog(const COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::response& res, epee::json_rpc::error& error_resp); //----------------------- private: diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index c413f9af8..59508bcaa 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -1071,6 +1071,33 @@ namespace cryptonote }; }; + struct tx_backlog_entry + { + uint64_t blob_size; + uint64_t fee; + uint64_t time_in_pool; + }; + + struct COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG + { + struct request + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string status; + std::vector<tx_backlog_entry> backlog; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + KV_SERIALIZE_CONTAINER_POD_AS_BLOB(backlog) + END_KV_SERIALIZE_MAP() + }; + }; + struct txpool_histo { uint32_t txs; diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index 2ccb6c7e9..479adcafc 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -650,6 +650,17 @@ bool simple_wallet::set_merge_destinations(const std::vector<std::string> &args/ return true; } +bool simple_wallet::set_confirm_backlog(const std::vector<std::string> &args/* = std::vector<std::string>()*/) +{ + const auto pwd_container = get_and_verify_password(); + if (pwd_container) + { + m_wallet->confirm_backlog(is_it_true(args[1])); + m_wallet->rewrite(m_wallet_file, pwd_container->password()); + } + return true; +} + bool simple_wallet::help(const std::vector<std::string> &args/* = std::vector<std::string>()*/) { success_msg_writer() << get_commands_str(); @@ -690,7 +701,7 @@ simple_wallet::simple_wallet() m_cmd_binder.set_handler("viewkey", boost::bind(&simple_wallet::viewkey, this, _1), tr("Display private view key")); m_cmd_binder.set_handler("spendkey", boost::bind(&simple_wallet::spendkey, this, _1), tr("Display private spend key")); m_cmd_binder.set_handler("seed", boost::bind(&simple_wallet::seed, this, _1), tr("Display Electrum-style mnemonic seed")); - m_cmd_binder.set_handler("set", boost::bind(&simple_wallet::set_variable, this, _1), tr("Available options: seed language - set wallet seed language; always-confirm-transfers <1|0> - whether to confirm unsplit txes; print-ring-members <1|0> - whether to print detailed information about ring members during confirmation; store-tx-info <1|0> - whether to store outgoing tx info (destination address, payment ID, tx secret key) for future reference; default-ring-size <n> - set default ring size (default is 5); auto-refresh <1|0> - whether to automatically sync new blocks from the daemon; refresh-type <full|optimize-coinbase|no-coinbase|default> - set wallet refresh behaviour; priority [0|1|2|3|4] - default/unimportant/normal/elevated/priority fee; confirm-missing-payment-id <1|0>; ask-password <1|0>; unit <monero|millinero|micronero|nanonero|piconero> - set default monero (sub-)unit; min-outputs-count [n] - try to keep at least that many outputs of value at least min-outputs-value; min-outputs-value [n] - try to keep at least min-outputs-count outputs of at least that value; merge-destinations <1|0> - whether to merge multiple payments to the same destination address")); + m_cmd_binder.set_handler("set", boost::bind(&simple_wallet::set_variable, this, _1), tr("Available options: seed language - set wallet seed language; always-confirm-transfers <1|0> - whether to confirm unsplit txes; print-ring-members <1|0> - whether to print detailed information about ring members during confirmation; store-tx-info <1|0> - whether to store outgoing tx info (destination address, payment ID, tx secret key) for future reference; default-ring-size <n> - set default ring size (default is 5); auto-refresh <1|0> - whether to automatically sync new blocks from the daemon; refresh-type <full|optimize-coinbase|no-coinbase|default> - set wallet refresh behaviour; priority [0|1|2|3|4] - default/unimportant/normal/elevated/priority fee; confirm-missing-payment-id <1|0>; ask-password <1|0>; unit <monero|millinero|micronero|nanonero|piconero> - set default monero (sub-)unit; min-outputs-count [n] - try to keep at least that many outputs of value at least min-outputs-value; min-outputs-value [n] - try to keep at least min-outputs-count outputs of at least that value; merge-destinations <1|0> - whether to merge multiple payments to the same destination address; confirm-backlog <1|0> - whether to warn if there is transaction backlog")); m_cmd_binder.set_handler("rescan_spent", boost::bind(&simple_wallet::rescan_spent, this, _1), tr("Rescan blockchain for spent outputs")); m_cmd_binder.set_handler("get_tx_key", boost::bind(&simple_wallet::get_tx_key, this, _1), tr("Get transaction key (r) for a given <txid>")); m_cmd_binder.set_handler("check_tx_key", boost::bind(&simple_wallet::check_tx_key, this, _1), tr("Check amount going to <address> in <txid>")); @@ -732,6 +743,7 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args) success_msg_writer() << "min-outputs-count = " << m_wallet->get_min_output_count(); success_msg_writer() << "min-outputs-value = " << cryptonote::print_money(m_wallet->get_min_output_value()); success_msg_writer() << "merge-destinations = " << m_wallet->merge_destinations(); + success_msg_writer() << "confirm-backlog = " << m_wallet->confirm_backlog(); return true; } else @@ -777,6 +789,7 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args) CHECK_SIMPLE_VARIABLE("min-outputs-count", set_min_output_count, tr("unsigned integer")); CHECK_SIMPLE_VARIABLE("min-outputs-value", set_min_output_value, tr("amount")); CHECK_SIMPLE_VARIABLE("merge-destinations", set_merge_destinations, tr("0 or 1")); + CHECK_SIMPLE_VARIABLE("confirm-backlog", set_confirm_backlog, tr("0 or 1")); } fail_msg_writer() << tr("set: unrecognized argument(s)"); return true; @@ -2429,6 +2442,49 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri return true; } + // if we need to check for backlog, check the worst case tx + if (m_wallet->confirm_backlog()) + { + std::stringstream prompt; + double worst_fee_per_byte = std::numeric_limits<double>::max(); + uint64_t size = 0, fee = 0; + for (size_t n = 0; n < ptx_vector.size(); ++n) + { + const uint64_t blob_size = cryptonote::tx_to_blob(ptx_vector[n].tx).size(); + const double fee_per_byte = ptx_vector[n].fee / (double)blob_size; + if (fee_per_byte < worst_fee_per_byte) + { + worst_fee_per_byte = fee_per_byte; + fee = ptx_vector[n].fee; + } + size += blob_size; + } + try + { + uint64_t nblocks = m_wallet->estimate_backlog(size, fee); + if (nblocks > 0) + prompt << (boost::format(tr("There is currently a %u block backlog at that fee level. Is this okay? (Y/Yes/N/No)")) % nblocks).str(); + } + catch (const std::exception &e) + { + prompt << tr("Failed to check for backlog: ") << e.what() << ENDL << tr("Is this okay anyway? (Y/Yes/N/No): "); + } + + std::string prompt_str = prompt.str(); + if (!prompt_str.empty()) + { + std::string accepted = command_line::input_line(prompt_str); + if (std::cin.eof()) + return true; + if (!command_line::is_yes(accepted)) + { + fail_msg_writer() << tr("transaction cancelled."); + + return true; + } + } + } + // if more than one tx necessary, prompt user to confirm if (m_wallet->always_confirm_transfers() || ptx_vector.size() > 1) { diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h index 8022c9bb2..eac4cbc99 100644 --- a/src/simplewallet/simplewallet.h +++ b/src/simplewallet/simplewallet.h @@ -120,6 +120,7 @@ namespace cryptonote bool set_min_output_count(const std::vector<std::string> &args = std::vector<std::string>()); bool set_min_output_value(const std::vector<std::string> &args = std::vector<std::string>()); bool set_merge_destinations(const std::vector<std::string> &args = std::vector<std::string>()); + bool set_confirm_backlog(const std::vector<std::string> &args = std::vector<std::string>()); bool help(const std::vector<std::string> &args = std::vector<std::string>()); bool start_mining(const std::vector<std::string> &args); bool stop_mining(const std::vector<std::string> &args); diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index b63e07b2d..09ca8efe1 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -1963,6 +1963,9 @@ bool wallet2::store_keys(const std::string& keys_file_name, const std::string& p value2.SetInt(m_merge_destinations ? 1 :0); json.AddMember("merge_destinations", value2, json.GetAllocator()); + value2.SetInt(m_confirm_backlog ? 1 :0); + json.AddMember("confirm_backlog", value2, json.GetAllocator()); + value2.SetInt(m_testnet ? 1 :0); json.AddMember("testnet", value2, json.GetAllocator()); @@ -2037,6 +2040,7 @@ bool wallet2::load_keys(const std::string& keys_file_name, const std::string& pa m_min_output_count = 0; m_min_output_value = 0; m_merge_destinations = false; + m_confirm_backlog = true; } else { @@ -2107,6 +2111,8 @@ bool wallet2::load_keys(const std::string& keys_file_name, const std::string& pa m_min_output_value = field_min_output_value; GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, merge_destinations, int, Int, false, false); m_merge_destinations = field_merge_destinations; + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, confirm_backlog, int, Int, false, true); + m_confirm_backlog = field_confirm_backlog; GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, testnet, int, Int, false, m_testnet); // Wallet is being opened with testnet flag, but is saved as a mainnet wallet THROW_WALLET_EXCEPTION_IF(m_testnet && !field_testnet, error::wallet_internal_error, "Mainnet wallet can not be opened as testnet wallet"); @@ -5741,6 +5747,58 @@ bool wallet2::is_synced() const return get_blockchain_current_height() >= height; } //---------------------------------------------------------------------------------------------------- +uint64_t wallet2::estimate_backlog(uint64_t blob_size, uint64_t fee) +{ + THROW_WALLET_EXCEPTION_IF(blob_size == 0, error::wallet_internal_error, "Invalid 0 fee"); + THROW_WALLET_EXCEPTION_IF(fee == 0, error::wallet_internal_error, "Invalid 0 fee"); + + // get txpool backlog + epee::json_rpc::request<cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::request> req = AUTO_VAL_INIT(req); + epee::json_rpc::response<cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::response, std::string> res = AUTO_VAL_INIT(res); + m_daemon_rpc_mutex.lock(); + req.jsonrpc = "2.0"; + req.id = epee::serialization::storage_entry(0); + req.method = "get_txpool_backlog"; + bool r = net_utils::invoke_http_json("/json_rpc", req, res, m_http_client, rpc_timeout); + m_daemon_rpc_mutex.unlock(); + THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "Failed to connect to daemon"); + THROW_WALLET_EXCEPTION_IF(res.result.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_txpool_backlog"); + THROW_WALLET_EXCEPTION_IF(res.result.status != CORE_RPC_STATUS_OK, error::get_tx_pool_error); + + epee::json_rpc::request<cryptonote::COMMAND_RPC_GET_INFO::request> req_t = AUTO_VAL_INIT(req_t); + epee::json_rpc::response<cryptonote::COMMAND_RPC_GET_INFO::response, std::string> resp_t = AUTO_VAL_INIT(resp_t); + m_daemon_rpc_mutex.lock(); + req_t.jsonrpc = "2.0"; + req_t.id = epee::serialization::storage_entry(0); + req_t.method = "get_info"; + r = net_utils::invoke_http_json("/json_rpc", req_t, resp_t, m_http_client); + m_daemon_rpc_mutex.unlock(); + THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "get_info"); + THROW_WALLET_EXCEPTION_IF(resp_t.result.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_info"); + THROW_WALLET_EXCEPTION_IF(resp_t.result.status != CORE_RPC_STATUS_OK, error::get_tx_pool_error); + + double our_fee_byte = fee / (double)blob_size; + uint64_t priority_size = 0; + for (const auto &i: res.result.backlog) + { + if (i.blob_size == 0) + { + MWARNING("Got 0 sized blob from txpool, ignored"); + continue; + } + double this_fee_byte = i.fee / (double)i.blob_size; + if (this_fee_byte < our_fee_byte) + continue; + priority_size += i.blob_size; + } + + uint64_t full_reward_zone = resp_t.result.block_size_limit / 2; + uint64_t nblocks = (priority_size + full_reward_zone - 1) / full_reward_zone; + MDEBUG("estimate_backlog: priority_size " << priority_size << " for " << our_fee_byte << " (" << our_fee_byte << " piconero fee/byte), " + << nblocks << " blocks at block size " << full_reward_zone); + return nblocks; +} +//---------------------------------------------------------------------------------------------------- void wallet2::generate_genesis(cryptonote::block& b) { if (m_testnet) { diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index ba9abacf5..f30c97635 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -105,7 +105,7 @@ namespace tools }; private: - wallet2(const wallet2&) : m_run(true), m_callback(0), m_testnet(false), m_always_confirm_transfers(true), m_print_ring_members(false), m_store_tx_info(true), m_default_mixin(0), m_default_priority(0), m_refresh_type(RefreshOptimizeCoinbase), m_auto_refresh(true), m_refresh_from_block_height(0), m_confirm_missing_payment_id(true), m_ask_password(true), m_min_output_count(0), m_min_output_value(0), m_merge_destinations(false), m_is_initialized(false),m_node_rpc_proxy(m_http_client, m_daemon_rpc_mutex) {} + wallet2(const wallet2&) : m_run(true), m_callback(0), m_testnet(false), m_always_confirm_transfers(true), m_print_ring_members(false), m_store_tx_info(true), m_default_mixin(0), m_default_priority(0), m_refresh_type(RefreshOptimizeCoinbase), m_auto_refresh(true), m_refresh_from_block_height(0), m_confirm_missing_payment_id(true), m_ask_password(true), m_min_output_count(0), m_min_output_value(0), m_merge_destinations(false), m_confirm_backlog(true), m_is_initialized(false),m_node_rpc_proxy(m_http_client, m_daemon_rpc_mutex) {} public: static const char* tr(const char* str); @@ -131,7 +131,7 @@ namespace tools static bool verify_password(const std::string& keys_file_name, const std::string& password, bool watch_only); - wallet2(bool testnet = false, bool restricted = false) : m_run(true), m_callback(0), m_testnet(testnet), m_always_confirm_transfers(true), m_print_ring_members(false), m_store_tx_info(true), m_default_mixin(0), m_default_priority(0), m_refresh_type(RefreshOptimizeCoinbase), m_auto_refresh(true), m_refresh_from_block_height(0), m_confirm_missing_payment_id(true), m_ask_password(true), m_min_output_count(0), m_min_output_value(0), m_merge_destinations(false), m_is_initialized(false), m_restricted(restricted), is_old_file_format(false), m_node_rpc_proxy(m_http_client, m_daemon_rpc_mutex) {} + wallet2(bool testnet = false, bool restricted = false) : m_run(true), m_callback(0), m_testnet(testnet), m_always_confirm_transfers(true), m_print_ring_members(false), m_store_tx_info(true), m_default_mixin(0), m_default_priority(0), m_refresh_type(RefreshOptimizeCoinbase), m_auto_refresh(true), m_refresh_from_block_height(0), m_confirm_missing_payment_id(true), m_ask_password(true), m_min_output_count(0), m_min_output_value(0), m_merge_destinations(false), m_confirm_backlog(true), m_is_initialized(false), m_restricted(restricted), is_old_file_format(false), m_node_rpc_proxy(m_http_client, m_daemon_rpc_mutex) {} struct transfer_details { @@ -539,6 +539,8 @@ namespace tools uint64_t get_min_output_value() const { return m_min_output_value; } void merge_destinations(bool merge) { m_merge_destinations = merge; } bool merge_destinations() const { return m_merge_destinations; } + bool confirm_backlog() const { return m_confirm_backlog; } + void confirm_backlog(bool always) { m_confirm_backlog = always; } bool get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key) const; @@ -602,6 +604,8 @@ namespace tools bool is_synced() const; + uint64_t estimate_backlog(uint64_t blob_size, uint64_t fee); + private: /*! * \brief Stores wallet information to wallet file. @@ -700,6 +704,7 @@ namespace tools uint32_t m_min_output_count; uint64_t m_min_output_value; bool m_merge_destinations; + bool m_confirm_backlog; bool m_is_initialized; NodeRPCProxy m_node_rpc_proxy; std::unordered_set<crypto::hash> m_scanned_pool_txs[2]; diff --git a/tests/core_proxy/CMakeLists.txt b/tests/core_proxy/CMakeLists.txt index d22fecc9c..680e34911 100644 --- a/tests/core_proxy/CMakeLists.txt +++ b/tests/core_proxy/CMakeLists.txt @@ -39,6 +39,8 @@ target_link_libraries(core_proxy PRIVATE cryptonote_core cryptonote_protocol + common + blockchain_db p2p epee ${CMAKE_THREAD_LIBS_INIT} diff --git a/tests/core_proxy/core_proxy.h b/tests/core_proxy/core_proxy.h index 516d1f515..c4fb462e3 100644 --- a/tests/core_proxy/core_proxy.h +++ b/tests/core_proxy/core_proxy.h @@ -66,6 +66,7 @@ namespace tests public: void on_synchronized(){} + void safesyncmode(const bool){} uint64_t get_current_blockchain_height(){return 1;} void set_target_blockchain_height(uint64_t) {} bool init(const boost::program_options::variables_map& vm); diff --git a/tests/fuzz/CMakeLists.txt b/tests/fuzz/CMakeLists.txt index 853d46a12..cb77f8f56 100644 --- a/tests/fuzz/CMakeLists.txt +++ b/tests/fuzz/CMakeLists.txt @@ -30,6 +30,8 @@ add_executable(block_fuzz_tests block.cpp fuzzer.cpp) target_link_libraries(block_fuzz_tests PRIVATE cryptonote_core + common + blockchain_db p2p epee ${CMAKE_THREAD_LIBS_INIT} @@ -42,6 +44,8 @@ add_executable(transaction_fuzz_tests transaction.cpp fuzzer.cpp) target_link_libraries(transaction_fuzz_tests PRIVATE cryptonote_core + common + blockchain_db p2p epee ${CMAKE_THREAD_LIBS_INIT} @@ -55,6 +59,8 @@ target_link_libraries(signature_fuzz_tests PRIVATE wallet cryptonote_core + common + blockchain_db p2p epee ${CMAKE_THREAD_LIBS_INIT} @@ -68,6 +74,8 @@ target_link_libraries(cold-outputs_fuzz_tests PRIVATE wallet cryptonote_core + common + blockchain_db p2p epee ${CMAKE_THREAD_LIBS_INIT} @@ -81,6 +89,8 @@ target_link_libraries(cold-transaction_fuzz_tests PRIVATE wallet cryptonote_core + common + blockchain_db p2p epee ${CMAKE_THREAD_LIBS_INIT} diff --git a/tests/unit_tests/ban.cpp b/tests/unit_tests/ban.cpp index 627e85348..bcffc85c9 100644 --- a/tests/unit_tests/ban.cpp +++ b/tests/unit_tests/ban.cpp @@ -43,6 +43,7 @@ class test_core { public: void on_synchronized(){} + void safesyncmode(const bool){} uint64_t get_current_blockchain_height() const {return 1;} void set_target_blockchain_height(uint64_t) {} bool init(const boost::program_options::variables_map& vm) {return true ;} diff --git a/tests/unit_tests/hardfork.cpp b/tests/unit_tests/hardfork.cpp index febf8b2f7..c30feb461 100644 --- a/tests/unit_tests/hardfork.cpp +++ b/tests/unit_tests/hardfork.cpp @@ -31,7 +31,7 @@ #include <algorithm> #include "gtest/gtest.h" -#include "blockchain_db/lmdb/db_lmdb.h" +#include "blockchain_db/blockchain_db.h" #include "cryptonote_basic/cryptonote_format_utils.h" #include "cryptonote_basic/hardfork.h" @@ -47,6 +47,7 @@ public: virtual void open(const std::string& filename, const int db_flags = 0) { } virtual void close() {} virtual void sync() {} + virtual void safesyncmode(const bool onoff) {} virtual void reset() {} virtual std::vector<std::string> get_filenames() const { return std::vector<std::string>(); } virtual std::string get_db_name() const { return std::string(); } |