diff options
46 files changed, 811 insertions, 450 deletions
diff --git a/contrib/epee/include/console_handler.h b/contrib/epee/include/console_handler.h index a3b2d30eb..b8336b270 100644 --- a/contrib/epee/include/console_handler.h +++ b/contrib/epee/include/console_handler.h @@ -52,11 +52,10 @@ namespace epee , m_has_read_request(false) , m_read_status(state_init) { - m_reader_thread = boost::thread(std::bind(&async_stdin_reader::reader_thread_func, this)); #ifdef HAVE_READLINE m_readline_buffer.start(); - m_readline_thread = boost::thread(std::bind(&async_stdin_reader::readline_thread_func, this)); #endif + m_reader_thread = boost::thread(std::bind(&async_stdin_reader::reader_thread_func, this)); } ~async_stdin_reader() @@ -115,7 +114,6 @@ namespace epee m_reader_thread.join(); #ifdef HAVE_READLINE m_readline_buffer.stop(); - m_readline_thread.join(); #endif } } @@ -193,16 +191,6 @@ namespace epee return true; } -#ifdef HAVE_READLINE - void readline_thread_func() - { - while (m_run.load(std::memory_order_relaxed)) - { - m_readline_buffer.process(); - } - } -#endif - void reader_thread_func() { while (true) @@ -212,12 +200,20 @@ namespace epee std::string line; bool read_ok = true; +#ifdef HAVE_READLINE +reread: +#endif if (wait_stdin_data()) { if (m_run.load(std::memory_order_relaxed)) { #ifdef HAVE_READLINE - m_readline_buffer.get_line(line); + switch (m_readline_buffer.get_line(line)) + { + case rdln::empty: goto eof; + case rdln::partial: goto reread; + case rdln::full: break; + } #else std::getline(std::cin, line); #endif @@ -229,6 +225,9 @@ namespace epee read_ok = false; } if (std::cin.eof()) { +#ifdef HAVE_READLINE +eof: +#endif m_read_status = state_eos; m_response_cv.notify_one(); break; @@ -263,7 +262,6 @@ namespace epee boost::thread m_reader_thread; std::atomic<bool> m_run; #ifdef HAVE_READLINE - boost::thread m_readline_thread; rdln::readline_buffer m_readline_buffer; #endif diff --git a/contrib/epee/include/net/http_client.h b/contrib/epee/include/net/http_client.h index 67e63f7bf..8e099e2bc 100644 --- a/contrib/epee/include/net/http_client.h +++ b/contrib/epee/include/net/http_client.h @@ -293,6 +293,9 @@ using namespace std; , m_lock() {} + const std::string &get_host() const { return m_host_buff; }; + const std::string &get_port() const { return m_port; }; + bool set_server(const std::string& address, boost::optional<login> user) { http::url_content parsed{}; diff --git a/contrib/epee/include/readline_buffer.h b/contrib/epee/include/readline_buffer.h index 28a153414..cda7e34f9 100644 --- a/contrib/epee/include/readline_buffer.h +++ b/contrib/epee/include/readline_buffer.h @@ -8,25 +8,25 @@ namespace rdln { + typedef enum { empty, partial, full } linestatus; class readline_buffer : public std::stringbuf { public: readline_buffer(); void start(); void stop(); - int process(); bool is_running() const { return m_cout_buf != NULL; } - void get_line(std::string& line) const; + linestatus get_line(std::string& line) const; void set_prompt(const std::string& prompt); static void add_completion(const std::string& command); static const std::vector<std::string>& get_completions(); protected: virtual int sync(); - + private: std::streambuf* m_cout_buf; static std::vector<std::string>& completion_commands(); diff --git a/contrib/epee/include/string_tools.h b/contrib/epee/include/string_tools.h index 258caa49e..ce7b2fb87 100644 --- a/contrib/epee/include/string_tools.h +++ b/contrib/epee/include/string_tools.h @@ -314,6 +314,18 @@ POP_WARNINGS return str; } //---------------------------------------------------------------------------- + inline std::string pad_string(std::string s, size_t n, char c = ' ', bool prepend = false) + { + if (s.size() < n) + { + if (prepend) + s = std::string(n - s.size(), c) + s; + else + s.append(n - s.size(), c); + } + return s; + } + //---------------------------------------------------------------------------- template<class t_pod_type> std::string pod_to_hex(const t_pod_type& s) { diff --git a/contrib/epee/src/readline_buffer.cpp b/contrib/epee/src/readline_buffer.cpp index 42b474052..291bba94c 100644 --- a/contrib/epee/src/readline_buffer.cpp +++ b/contrib/epee/src/readline_buffer.cpp @@ -3,19 +3,15 @@ #include <readline/history.h> #include <sys/select.h> #include <unistd.h> -#include <mutex> -#include <condition_variable> #include <boost/thread.hpp> #include <boost/algorithm/string.hpp> -static int process_input(); static void install_line_handler(); static void remove_line_handler(); -static std::string last_line; -static std::string last_prompt; -static std::mutex line_mutex, sync_mutex, process_mutex; -static std::condition_variable have_line; +static boost::mutex sync_mutex; +static rdln::linestatus line_stat; +static char *the_line; namespace { @@ -55,7 +51,6 @@ rdln::readline_buffer::readline_buffer() void rdln::readline_buffer::start() { - std::unique_lock<std::mutex> lock(process_mutex); if(m_cout_buf != NULL) return; m_cout_buf = std::cout.rdbuf(); @@ -65,9 +60,6 @@ void rdln::readline_buffer::start() void rdln::readline_buffer::stop() { - std::unique_lock<std::mutex> lock_process(process_mutex); - std::unique_lock<std::mutex> lock_sync(sync_mutex); - have_line.notify_all(); if(m_cout_buf == NULL) return; std::cout.rdbuf(m_cout_buf); @@ -75,20 +67,26 @@ void rdln::readline_buffer::stop() remove_line_handler(); } -void rdln::readline_buffer::get_line(std::string& line) const +rdln::linestatus rdln::readline_buffer::get_line(std::string& line) const { - std::unique_lock<std::mutex> lock(line_mutex); - have_line.wait(lock); - line = last_line; + boost::lock_guard<boost::mutex> lock(sync_mutex); + line_stat = rdln::partial; + rl_callback_read_char(); + if (line_stat == rdln::full) + { + line = the_line; + free(the_line); + the_line = NULL; + } + return line_stat; } void rdln::readline_buffer::set_prompt(const std::string& prompt) { - last_prompt = prompt; if(m_cout_buf == NULL) return; - std::lock_guard<std::mutex> lock(sync_mutex); - rl_set_prompt(last_prompt.c_str()); + boost::lock_guard<boost::mutex> lock(sync_mutex); + rl_set_prompt(prompt.c_str()); rl_redisplay(); } @@ -104,126 +102,78 @@ const std::vector<std::string>& rdln::readline_buffer::get_completions() return completion_commands(); } -int rdln::readline_buffer::process() +int rdln::readline_buffer::sync() { - process_mutex.lock(); - if(m_cout_buf == NULL) + boost::lock_guard<boost::mutex> lock(sync_mutex); +#if RL_READLINE_VERSION < 0x0700 + char lbuf[2] = {0,0}; + char *line = NULL; + int end = 0, point = 0; +#endif + + if (rl_end || *rl_prompt) { - process_mutex.unlock(); - boost::this_thread::sleep_for(boost::chrono::milliseconds( 1 )); - return 0; +#if RL_READLINE_VERSION >= 0x0700 + rl_clear_visible_line(); +#else + line = rl_line_buffer; + end = rl_end; + point = rl_point; + rl_line_buffer = lbuf; + rl_end = 0; + rl_point = 0; + rl_save_prompt(); + rl_redisplay(); +#endif } - int count = process_input(); - process_mutex.unlock(); - boost::this_thread::sleep_for(boost::chrono::milliseconds( 1 )); - return count; -} -int rdln::readline_buffer::sync() -{ - std::lock_guard<std::mutex> lock(sync_mutex); - char* saved_line; - int saved_point; - - saved_point = rl_point; - saved_line = rl_copy_text(0, rl_end); - - rl_set_prompt(""); - rl_replace_line("", 0); - rl_redisplay(); - do { m_cout_buf->sputc( this->sgetc() ); } while ( this->snextc() != EOF ); - - rl_set_prompt(last_prompt.c_str()); - rl_replace_line(saved_line, 0); - rl_point = saved_point; - rl_redisplay(); - free(saved_line); - - return 0; -} -static int process_input() -{ - int count; - struct timeval t; - fd_set fds; - - t.tv_sec = 0; - t.tv_usec = 1000; - - FD_ZERO(&fds); - FD_SET(STDIN_FILENO, &fds); - count = select(STDIN_FILENO + 1, &fds, NULL, NULL, &t); - if (count < 1) +#if RL_READLINE_VERSION < 0x0700 + if (end || *rl_prompt) { - return count; + rl_restore_prompt(); + rl_line_buffer = line; + rl_end = end; + rl_point = point; } - rl_callback_read_char(); - return count; -} - -static void handle_line(char* line) -{ - free(line); - rl_done = 1; - return; -} - -static int handle_enter(int x, int y) -{ - std::lock_guard<std::mutex> lock(sync_mutex); - char* line = NULL; - - line = rl_copy_text(0, rl_end); - std::string test_line = line; - free(line); - boost::trim_right(test_line); - - rl_crlf(); +#endif rl_on_new_line(); - - if(test_line.empty()) - { - last_line = ""; - rl_set_prompt(last_prompt.c_str()); - rl_replace_line("", 1); - rl_redisplay(); - have_line.notify_one(); - return 0; - } - - rl_set_prompt(""); - rl_replace_line("", 1); rl_redisplay(); - - if (!test_line.empty()) - { - last_line = test_line; - add_history(test_line.c_str()); - history_set_pos(history_length); - } - if(last_line != "exit" && last_line != "q") - { - rl_set_prompt(last_prompt.c_str()); - rl_replace_line("", 1); - rl_redisplay(); - } - - have_line.notify_one(); return 0; } -static int startup_hook() +static void handle_line(char* line) { - rl_bind_key(RETURN, handle_enter); - rl_bind_key(NEWLINE, handle_enter); - return 0; + bool exit = false; + if (line) + { + line_stat = rdln::full; + the_line = line; + std::string test_line = line; + boost::trim_right(test_line); + if(!test_line.empty()) + { + add_history(test_line.c_str()); + history_set_pos(history_length); + if (test_line == "exit" || test_line == "q") + exit = true; + } + } else + /* EOF */ + { + line_stat = rdln::empty; + exit = true; + } + rl_done = 1; + if (exit) + rl_set_prompt(""); + return; } static char* completion_matches(const char* text, int state) @@ -258,7 +208,6 @@ static char** attempted_completion(const char* text, int start, int end) static void install_line_handler() { - rl_startup_hook = startup_hook; rl_attempted_completion_function = attempted_completion; rl_callback_handler_install("", handle_line); stifle_history(500); @@ -269,8 +218,6 @@ static void remove_line_handler() rl_replace_line("", 0); rl_set_prompt(""); rl_redisplay(); - rl_unbind_key(RETURN); - rl_unbind_key(NEWLINE); rl_callback_handler_remove(); } diff --git a/contrib/rlwrap/monerocommands_bitmonerod.txt b/contrib/rlwrap/monerocommands_bitmonerod.txt index 9572e40fe..c4f77b37d 100644 --- a/contrib/rlwrap/monerocommands_bitmonerod.txt +++ b/contrib/rlwrap/monerocommands_bitmonerod.txt @@ -32,4 +32,5 @@ status stop_daemon stop_mining stop_save_graph +sync_info unban diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp index a6774a25c..01a59e079 100644 --- a/src/blockchain_db/blockchain_db.cpp +++ b/src/blockchain_db/blockchain_db.cpp @@ -33,6 +33,19 @@ #include "profile_tools.h" #include "ringct/rctOps.h" +#include "lmdb/db_lmdb.h" +#ifdef BERKELEY_DB +#include "berkeleydb/db_bdb.h" +#endif + +static const char *db_types[] = { + "lmdb", +#ifdef BERKELEY_DB + "berkeley", +#endif + NULL +}; + #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "blockchain.db" @@ -41,6 +54,41 @@ using epee::string_tools::pod_to_hex; namespace cryptonote { +bool blockchain_valid_db_type(const std::string& db_type) +{ + int i; + for (i=0; db_types[i]; i++) + { + if (db_types[i] == db_type) + return true; + } + return false; +} + +std::string blockchain_db_types(const std::string& sep) +{ + int i; + std::string ret = ""; + for (i=0; db_types[i]; i++) + { + if (i) + ret += sep; + ret += db_types[i]; + } + return ret; +} + +BlockchainDB *new_db(const std::string& db_type) +{ + if (db_type == "lmdb") + return new BlockchainLMDB(); +#if defined(BERKELEY_DB) + if (db_type == "berkeley") + return new BlockchainBDB(); +#endif + return NULL; +} + void BlockchainDB::pop_block() { block blk; diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 27e63801d..ad246d85e 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -145,6 +145,12 @@ struct txpool_tx_meta_t uint8_t padding[77]; // till 192 bytes }; +#define DBF_SAFE 1 +#define DBF_FAST 2 +#define DBF_FASTEST 4 +#define DBF_RDONLY 8 +#define DBF_SALVAGE 0x10 + /*********************************** * Exception Definitions ***********************************/ @@ -600,6 +606,13 @@ public: virtual void sync() = 0; /** + * @brief toggle safe syncs for the DB + * + * Used to switch DBF_SAFE on or off after starting up with DBF_FAST. + */ + virtual void safesyncmode(const bool onoff) = 0; + + /** * @brief Remove everything from the BlockchainDB * * This function should completely remove all data from a BlockchainDB. @@ -1491,6 +1504,7 @@ public: }; // class BlockchainDB +BlockchainDB *new_db(const std::string& db_type); } // namespace cryptonote diff --git a/src/blockchain_db/db_types.h b/src/blockchain_db/db_types.h index 6c21b029e..8e2f58a61 100644 --- a/src/blockchain_db/db_types.h +++ b/src/blockchain_db/db_types.h @@ -31,9 +31,6 @@ namespace cryptonote { - - const std::unordered_set<std::string> blockchain_db_types = - { "lmdb" - }; - + bool blockchain_valid_db_type(const std::string& db_type); + std::string blockchain_db_types(const std::string& sep); } // namespace cryptonote diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 4441c7578..4100d9cca 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -1083,9 +1083,10 @@ BlockchainLMDB::BlockchainLMDB(bool batch_transactions) m_hardfork = nullptr; } -void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) +void BlockchainLMDB::open(const std::string& filename, const int db_flags) { int result; + int mdb_flags = MDB_NORDAHEAD; LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -1124,6 +1125,15 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) size_t mapsize = DEFAULT_MAPSIZE; + if (db_flags & DBF_FAST) + mdb_flags |= MDB_NOSYNC; + if (db_flags & DBF_FASTEST) + mdb_flags |= MDB_NOSYNC | MDB_WRITEMAP | MDB_MAPASYNC; + if (db_flags & DBF_RDONLY) + mdb_flags = MDB_RDONLY; + if (db_flags & DBF_SALVAGE) + mdb_flags |= MDB_PREVSNAPSHOT; + if (auto result = mdb_env_open(m_env, filename.c_str(), mdb_flags, 0644)) throw0(DB_ERROR(lmdb_error("Failed to open lmdb environment: ", result).c_str())); @@ -1308,6 +1318,11 @@ void BlockchainLMDB::sync() } } +void BlockchainLMDB::safesyncmode(const bool onoff) +{ + mdb_env_set_flags(m_env, MDB_NOSYNC|MDB_MAPASYNC, !onoff); +} + void BlockchainLMDB::reset() { LOG_PRINT_L3("BlockchainLMDB::" << __func__); diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 14e5d34e2..3a11ddf0d 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -165,6 +165,8 @@ public: virtual void sync(); + virtual void safesyncmode(const bool onoff); + virtual void reset(); virtual std::vector<std::string> get_filenames() const; diff --git a/src/blockchain_utilities/blockchain_export.cpp b/src/blockchain_utilities/blockchain_export.cpp index f145bc107..20eca09f2 100644 --- a/src/blockchain_utilities/blockchain_export.cpp +++ b/src/blockchain_utilities/blockchain_export.cpp @@ -31,10 +31,6 @@ #include "common/command_line.h" #include "cryptonote_core/tx_pool.h" #include "blockchain_db/blockchain_db.h" -#include "blockchain_db/lmdb/db_lmdb.h" -#if defined(BERKELEY_DB) -#include "blockchain_db/berkeleydb/db_bdb.h" -#endif #include "blockchain_db/db_types.h" #include "version.h" @@ -44,17 +40,6 @@ namespace po = boost::program_options; using namespace epee; -std::string join_set_strings(const std::unordered_set<std::string>& db_types_all, const char* delim) -{ - std::string result; - std::ostringstream s; - std::copy(db_types_all.begin(), db_types_all.end(), std::ostream_iterator<std::string>(s, delim)); - result = s.str(); - if (result.length() > 0) - result.erase(result.end()-strlen(delim), result.end()); - return result; -} - int main(int argc, char* argv[]) { TRY_ENTRY(); @@ -63,10 +48,7 @@ int main(int argc, char* argv[]) std::string default_db_type = "lmdb"; - std::unordered_set<std::string> db_types_all = cryptonote::blockchain_db_types; - db_types_all.insert("memory"); - - std::string available_dbs = join_set_strings(db_types_all, ", "); + std::string available_dbs = cryptonote::blockchain_db_types(", "); available_dbs = "available: " + available_dbs; uint32_t log_level = 0; @@ -144,18 +126,11 @@ int main(int argc, char* argv[]) m_config_folder = command_line::get_arg(vm, data_dir_arg); std::string db_type = command_line::get_arg(vm, arg_database); - if (db_types_all.count(db_type) == 0) + if (!cryptonote::blockchain_valid_db_type(db_type)) { std::cerr << "Invalid database type: " << db_type << std::endl; return 1; } -#if !defined(BERKELEY_DB) - if (db_type == "berkeley") - { - LOG_ERROR("BerkeleyDB support disabled."); - return false; - } -#endif if (command_line::has_arg(vm, arg_output_file)) output_file_path = boost::filesystem::path(command_line::get_arg(vm, arg_output_file)); @@ -179,19 +154,8 @@ int main(int argc, char* argv[]) tx_memory_pool m_mempool(*core_storage); core_storage = new Blockchain(m_mempool); - int db_flags = 0; - - BlockchainDB* db = nullptr; - if (db_type == "lmdb") - { - db_flags |= MDB_RDONLY; - db = new BlockchainLMDB(); - } -#if defined(BERKELEY_DB) - else if (db_type == "berkeley") - db = new BlockchainBDB(); -#endif - else + BlockchainDB* db = new_db(db_type); + if (db == NULL) { LOG_ERROR("Attempted to use non-existent database type: " << db_type); throw std::runtime_error("Attempting to use non-existent database type"); @@ -205,7 +169,7 @@ int main(int argc, char* argv[]) LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); try { - db->open(filename, db_flags); + db->open(filename, DBF_RDONLY); } catch (const std::exception& e) { diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp index 6f908c799..ded854ca4 100644 --- a/src/blockchain_utilities/blockchain_import.cpp +++ b/src/blockchain_utilities/blockchain_import.cpp @@ -42,8 +42,6 @@ #include "blockchain_db/db_types.h" #include "cryptonote_core/cryptonote_core.h" -#include <lmdb.h> // for db flag arguments - #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "bcutil" @@ -78,40 +76,16 @@ namespace po = boost::program_options; using namespace cryptonote; using namespace epee; - -std::string join_set_strings(const std::unordered_set<std::string>& db_types_all, const char* delim) -{ - std::string result; - std::ostringstream s; - std::copy(db_types_all.begin(), db_types_all.end(), std::ostream_iterator<std::string>(s, delim)); - result = s.str(); - if (result.length() > 0) - result.erase(result.end()-strlen(delim), result.end()); - return result; -} - -// db_type: lmdb, berkeley // db_mode: safe, fast, fastest -int get_db_flags_from_mode(const std::string& db_type, const std::string& db_mode) +int get_db_flags_from_mode(const std::string& db_mode) { - uint64_t BDB_FAST_MODE = 0; - uint64_t BDB_FASTEST_MODE = 0; - uint64_t BDB_SAFE_MODE = 0; - -#if defined(BERKELEY_DB) - BDB_FAST_MODE = DB_TXN_WRITE_NOSYNC; - BDB_FASTEST_MODE = DB_TXN_NOSYNC; - BDB_SAFE_MODE = DB_TXN_SYNC; -#endif - int db_flags = 0; - bool islmdb = db_type == "lmdb"; if (db_mode == "safe") - db_flags = islmdb ? MDB_NORDAHEAD : BDB_SAFE_MODE; + db_flags = DBF_SAFE; else if (db_mode == "fast") - db_flags = islmdb ? MDB_NOMETASYNC | MDB_NOSYNC | MDB_NORDAHEAD : BDB_FAST_MODE; + db_flags = DBF_FAST; else if (db_mode == "fastest") - db_flags = islmdb ? MDB_WRITEMAP | MDB_MAPASYNC | MDB_NORDAHEAD | MDB_NOMETASYNC | MDB_NOSYNC : BDB_FASTEST_MODE; + db_flags = DBF_FASTEST; return db_flags; } @@ -132,14 +106,6 @@ int parse_db_arguments(const std::string& db_arg_str, std::string& db_type, int& return 1; } -#if !defined(BERKELEY_DB) - if (db_type == "berkeley") - { - MFATAL("BerkeleyDB support disabled."); - return false; - } -#endif - std::string db_arg_str2 = db_args[1]; boost::split(db_args, db_arg_str2, boost::is_any_of(",")); @@ -155,51 +121,7 @@ int parse_db_arguments(const std::string& db_arg_str, std::string& db_type, int& } if (! db_mode.empty()) { - db_flags = get_db_flags_from_mode(db_type, db_mode); - } - else - { - for (auto& it : db_args) - { - boost::algorithm::trim(it); - if (it.empty()) - continue; - if (db_type == "lmdb") - { - MINFO("LMDB flag: " << it); - if (it == "nosync") - db_flags |= MDB_NOSYNC; - else if (it == "nometasync") - db_flags |= MDB_NOMETASYNC; - else if (it == "writemap") - db_flags |= MDB_WRITEMAP; - else if (it == "mapasync") - db_flags |= MDB_MAPASYNC; - else if (it == "nordahead") - db_flags |= MDB_NORDAHEAD; - else - { - std::cerr << "unrecognized database flag: " << it << ENDL; - return 1; - } - } -#if defined(BERKELEY_DB) - else if (db_type == "berkeley") - { - if (it == "txn_write_nosync") - db_flags = DB_TXN_WRITE_NOSYNC; - else if (it == "txn_nosync") - db_flags = DB_TXN_NOSYNC; - else if (it == "txn_sync") - db_flags = DB_TXN_SYNC; - else - { - std::cerr << "unrecognized database flag: " << it << ENDL; - return 1; - } - } -#endif - } + db_flags = get_db_flags_from_mode(db_mode); } return 0; } @@ -580,12 +502,8 @@ int main(int argc, char* argv[]) epee::string_tools::set_module_name_and_folder(argv[0]); std::string default_db_type = "lmdb"; - std::string default_db_engine_compiled = "blockchain_db"; - - std::unordered_set<std::string> db_types_all = cryptonote::blockchain_db_types; - db_types_all.insert("memory"); - std::string available_dbs = join_set_strings(db_types_all, ", "); + std::string available_dbs = cryptonote::blockchain_db_types(", "); available_dbs = "available: " + available_dbs; uint32_t log_level = 0; @@ -731,7 +649,6 @@ int main(int argc, char* argv[]) std::string db_type; - std::string db_engine_compiled; int db_flags = 0; int res = 0; res = parse_db_arguments(db_arg_str, db_type, db_flags); @@ -741,25 +658,12 @@ int main(int argc, char* argv[]) return 1; } - if (db_types_all.count(db_type) == 0) + if (!cryptonote::blockchain_valid_db_type(db_type)) { std::cerr << "Invalid database type: " << db_type << std::endl; return 1; } - if ((db_type == "lmdb") -#if defined(BERKELEY_DB) - || (db_type == "berkeley") -#endif - ) - { - db_engine_compiled = "blockchain_db"; - } - else - { - db_engine_compiled = "memory"; - } - MINFO("database: " << db_type); MINFO("database flags: " << db_flags); MINFO("verify: " << std::boolalpha << opt_verify << std::noboolalpha); @@ -788,16 +692,6 @@ int main(int argc, char* argv[]) // properties to do so. Both ways work, but fake core isn't necessary in that // circumstance. - if (db_type != "lmdb" -#if defined(BERKELEY_DB) - && db_type != "berkeley" -#endif - ) - { - std::cerr << "database type unrecognized" << ENDL; - return 1; - } - cryptonote::cryptonote_protocol_stub pr; //TODO: stub only for this kind of test, make real validation of relayed objects cryptonote::core core(&pr); core.disable_dns_checkpoints(true); diff --git a/src/blockchain_utilities/blocksdat_file.h b/src/blockchain_utilities/blocksdat_file.h index 4e9d8173b..0a5913058 100644 --- a/src/blockchain_utilities/blocksdat_file.h +++ b/src/blockchain_utilities/blocksdat_file.h @@ -38,7 +38,6 @@ #include "cryptonote_basic/cryptonote_boost_serialization.h" #include "cryptonote_core/blockchain.h" #include "blockchain_db/blockchain_db.h" -#include "blockchain_db/lmdb/db_lmdb.h" #include <algorithm> #include <cstdio> diff --git a/src/common/command_line.cpp b/src/common/command_line.cpp index 925d8ff3b..ad66b2009 100644 --- a/src/common/command_line.cpp +++ b/src/common/command_line.cpp @@ -96,7 +96,7 @@ namespace command_line , "checkpoints from DNS server will be enforced" , false }; - std::string arg_db_type_description = "Specify database type, available: " + boost::algorithm::join(cryptonote::blockchain_db_types, ", "); + std::string arg_db_type_description = "Specify database type, available: " + cryptonote::blockchain_db_types(", "); const command_line::arg_descriptor<std::string> arg_db_type = { "db-type" , arg_db_type_description.c_str() diff --git a/src/common/rpc_client.h b/src/common/rpc_client.h index 8494b4a60..297020ef2 100644 --- a/src/common/rpc_client.h +++ b/src/common/rpc_client.h @@ -69,7 +69,7 @@ namespace tools bool ok = connection.is_open(); if (!ok) { - fail_msg_writer() << "Couldn't connect to daemon"; + fail_msg_writer() << "Couldn't connect to daemon: " << m_http_client.get_host() << ":" << m_http_client.get_port(); return false; } ok = ok && epee::net_utils::invoke_http_json_rpc("/json_rpc", method_name, req, res, m_http_client, t_http_connection::TIMEOUT()); @@ -98,7 +98,7 @@ namespace tools ok = ok && epee::net_utils::invoke_http_json_rpc("/json_rpc", method_name, req, res, m_http_client, t_http_connection::TIMEOUT()); if (!ok) { - fail_msg_writer() << "Couldn't connect to daemon"; + fail_msg_writer() << "Couldn't connect to daemon: " << m_http_client.get_host() << ":" << m_http_client.get_port(); return false; } else if (res.status != CORE_RPC_STATUS_OK) // TODO - handle CORE_RPC_STATUS_BUSY ? @@ -126,7 +126,7 @@ namespace tools ok = ok && epee::net_utils::invoke_http_json(relative_url, req, res, m_http_client, t_http_connection::TIMEOUT()); if (!ok) { - fail_msg_writer() << "Couldn't connect to daemon"; + fail_msg_writer() << "Couldn't connect to daemon: " << m_http_client.get_host() << ":" << m_http_client.get_port(); return false; } else if (res.status != CORE_RPC_STATUS_OK) // TODO - handle CORE_RPC_STATUS_BUSY ? diff --git a/src/common/thread_group.cpp b/src/common/thread_group.cpp index 860d0b732..691a27a25 100644 --- a/src/common/thread_group.cpp +++ b/src/common/thread_group.cpp @@ -32,6 +32,7 @@ #include <limits> #include <stdexcept> +#include "cryptonote_config.h" #include "common/util.h" namespace tools @@ -63,8 +64,10 @@ thread_group::data::data(std::size_t count) , has_work() , stop(false) { threads.reserve(count); + boost::thread::attributes attrs; + attrs.set_stack_size(THREAD_STACK_SIZE); while (count--) { - threads.push_back(boost::thread(&thread_group::data::run, this)); + threads.push_back(boost::thread(attrs, boost::bind(&thread_group::data::run, this))); } } diff --git a/src/crypto/crypto.cpp b/src/crypto/crypto.cpp index 1c7adff3b..5fb670f87 100644 --- a/src/crypto/crypto.cpp +++ b/src/crypto/crypto.cpp @@ -36,18 +36,13 @@ #include <memory> #include <boost/thread/mutex.hpp> #include <boost/thread/lock_guard.hpp> +#include <boost/shared_ptr.hpp> #include "common/varint.h" #include "warnings.h" #include "crypto.h" #include "hash.h" -#if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__DragonFly__) - #include <alloca.h> -#else - #include <stdlib.h> -#endif - namespace crypto { using std::abort; @@ -411,7 +406,9 @@ POP_WARNINGS ge_p3 image_unp; ge_dsmp image_pre; ec_scalar sum, k, h; - rs_comm *const buf = reinterpret_cast<rs_comm *>(alloca(rs_comm_size(pubs_count))); + boost::shared_ptr<rs_comm> buf(reinterpret_cast<rs_comm *>(malloc(rs_comm_size(pubs_count))), free); + if (!buf) + abort(); assert(sec_index < pubs_count); #if !defined(NDEBUG) { @@ -459,7 +456,7 @@ POP_WARNINGS sc_add(&sum, &sum, &sig[i].c); } } - hash_to_scalar(buf, rs_comm_size(pubs_count), h); + hash_to_scalar(buf.get(), rs_comm_size(pubs_count), h); sc_sub(&sig[sec_index].c, &h, &sum); sc_mulsub(&sig[sec_index].r, &sig[sec_index].c, &sec, &k); } @@ -471,7 +468,9 @@ POP_WARNINGS ge_p3 image_unp; ge_dsmp image_pre; ec_scalar sum, h; - rs_comm *const buf = reinterpret_cast<rs_comm *>(alloca(rs_comm_size(pubs_count))); + boost::shared_ptr<rs_comm> buf(reinterpret_cast<rs_comm *>(malloc(rs_comm_size(pubs_count))), free); + if (!buf) + return false; #if !defined(NDEBUG) for (i = 0; i < pubs_count; i++) { assert(check_key(*pubs[i])); @@ -499,7 +498,7 @@ POP_WARNINGS ge_tobytes(&buf->ab[i].b, &tmp2); sc_add(&sum, &sum, &sig[i].c); } - hash_to_scalar(buf, rs_comm_size(pubs_count), h); + hash_to_scalar(buf.get(), rs_comm_size(pubs_count), h); sc_sub(&h, &h, &sum); return sc_isnonzero(&h) == 0; } diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 14a990131..c1faa703f 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -59,6 +59,8 @@ #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "blockchain" +#define FIND_BLOCKCHAIN_SUPPLEMENT_MAX_SIZE (100*1024*1024) // 100 MB + //#include "serialization/json_archive.h" /* TODO: @@ -99,6 +101,9 @@ static const struct { // version 5 starts from block 1288616, which is on or around the 15th of April, 2017. Fork time finalised on 2017-03-14. { 5, 1288616, 0, 1489520158 }, + + // version 6 starts from block 1400000, which is on or around the 16th of September, 2017. Fork time finalised on 2017-08-18. + { 6, 1400000, 0, 1503046577 }, }; static const uint64_t mainnet_hard_fork_version_1_till = 1009826; @@ -126,7 +131,7 @@ static const uint64_t testnet_hard_fork_version_1_till = 624633; //------------------------------------------------------------------ Blockchain::Blockchain(tx_memory_pool& tx_pool) : m_db(), m_tx_pool(tx_pool), m_hardfork(NULL), m_timestamps_and_difficulties_height(0), m_current_block_cumul_sz_limit(0), - m_enforce_dns_checkpoints(false), m_max_prepare_blocks_threads(4), m_db_blocks_per_sync(1), m_db_sync_mode(db_async), m_fast_sync(true), m_show_time_stats(false), m_sync_counter(0), m_cancel(false) + m_enforce_dns_checkpoints(false), m_max_prepare_blocks_threads(4), m_db_blocks_per_sync(1), m_db_sync_mode(db_async), m_db_default_sync(false), m_fast_sync(true), m_show_time_stats(false), m_sync_counter(0), m_cancel(false) { LOG_PRINT_L3("Blockchain::" << __func__); } @@ -2072,8 +2077,8 @@ bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, cons m_db->block_txn_start(true); total_height = get_current_blockchain_height(); - size_t count = 0; - for(size_t i = start_height; i < total_height && count < max_count; i++, count++) + size_t count = 0, size = 0; + for(size_t i = start_height; i < total_height && count < max_count && (size < FIND_BLOCKCHAIN_SUPPLEMENT_MAX_SIZE || count < 3); i++, count++) { blocks.resize(blocks.size()+1); blocks.back().first = m_db->get_block_blob_from_height(i); @@ -2082,6 +2087,9 @@ bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, cons std::list<crypto::hash> mis; get_transactions_blobs(b.tx_hashes, blocks.back().second, mis); CHECK_AND_ASSERT_MES(!mis.size(), false, "internal error, transaction from block not found"); + size += blocks.back().first.size(); + for (const auto &t: blocks.back().second) + size += t.size(); } m_db->block_txn_stop(); return true; @@ -4034,12 +4042,29 @@ bool Blockchain::for_all_txpool_txes(std::function<bool(const crypto::hash&, con void Blockchain::set_user_options(uint64_t maxthreads, uint64_t blocks_per_sync, blockchain_db_sync_mode sync_mode, bool fast_sync) { + if (sync_mode == db_defaultsync) + { + m_db_default_sync = true; + sync_mode = db_async; + } m_db_sync_mode = sync_mode; m_fast_sync = fast_sync; m_db_blocks_per_sync = blocks_per_sync; m_max_prepare_blocks_threads = maxthreads; } +void Blockchain::safesyncmode(const bool onoff) +{ + /* all of this is no-op'd if the user set a specific + * --db-sync-mode at startup. + */ + if (m_db_default_sync) + { + m_db->safesyncmode(onoff); + m_db_sync_mode = onoff ? db_nosync : db_async; + } +} + HardFork::State Blockchain::get_hard_fork_state() const { return m_hardfork->get_state(); diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index 7fa78584b..b8ea657b4 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -65,6 +65,7 @@ namespace cryptonote */ enum blockchain_db_sync_mode { + db_defaultsync, //!< user didn't specify, use db_async db_sync, //!< handle syncing calls instead of the backing db, synchronously db_async, //!< handle syncing calls instead of the backing db, asynchronously db_nosync //!< Leave syncing up to the backing db (safest, but slowest because of disk I/O) @@ -701,6 +702,11 @@ namespace cryptonote blockchain_db_sync_mode sync_mode, bool fast_sync); /** + * @brief Put DB in safe sync mode + */ + void safesyncmode(const bool onoff); + + /** * @brief set whether or not to show/print time statistics * * @param stats the new time stats setting @@ -932,6 +938,7 @@ namespace cryptonote blockchain_db_sync_mode m_db_sync_mode; bool m_fast_sync; bool m_show_time_stats; + bool m_db_default_sync; uint64_t m_db_blocks_per_sync; uint64_t m_max_prepare_blocks_threads; uint64_t m_fake_pow_calc_time; diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 19f4aaca9..c406dd0b4 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -48,10 +48,6 @@ using namespace epee; #include "cryptonote_basic/checkpoints.h" #include "ringct/rctTypes.h" #include "blockchain_db/blockchain_db.h" -#include "blockchain_db/lmdb/db_lmdb.h" -#if defined(BERKELEY_DB) -#include "blockchain_db/berkeleydb/db_bdb.h" -#endif #include "ringct/rctSigs.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -246,6 +242,12 @@ namespace cryptonote return m_blockchain_storage.get_transactions_blobs(txs_ids, txs, missed_txs); } //----------------------------------------------------------------------------------------------- + bool core::get_txpool_backlog(std::vector<tx_backlog_entry>& backlog) const + { + m_mempool.get_transaction_backlog(backlog); + return true; + } + //----------------------------------------------------------------------------------------------- bool core::get_transactions(const std::vector<crypto::hash>& txs_ids, std::list<transaction>& txs, std::list<crypto::hash>& missed_txs) const { return m_blockchain_storage.get_transactions(txs_ids, txs, missed_txs); @@ -308,20 +310,8 @@ namespace cryptonote // folder might not be a directory, etc, etc catch (...) { } - BlockchainDB* db = nullptr; - uint64_t DBS_FAST_MODE = 0; - uint64_t DBS_FASTEST_MODE = 0; - uint64_t DBS_SAFE_MODE = 0; - uint64_t DBS_SALVAGE = 0; - if (db_type == "lmdb") - { - db = new BlockchainLMDB(); - DBS_SAFE_MODE = MDB_NORDAHEAD; - DBS_FAST_MODE = MDB_NORDAHEAD | MDB_NOSYNC; - DBS_FASTEST_MODE = MDB_NORDAHEAD | MDB_NOSYNC | MDB_WRITEMAP | MDB_MAPASYNC; - DBS_SALVAGE = MDB_PREVSNAPSHOT; - } - else + BlockchainDB* db = new_db(db_type); + if (db == NULL) { LOG_ERROR("Attempted to use non-existent database type"); return false; @@ -332,7 +322,7 @@ namespace cryptonote const std::string filename = folder.string(); // default to fast:async:1 - blockchain_db_sync_mode sync_mode = db_async; + blockchain_db_sync_mode sync_mode = db_defaultsync; uint64_t blocks_per_sync = 1; try @@ -347,7 +337,7 @@ namespace cryptonote MDEBUG("option: " << option); // default to fast:async:1 - uint64_t DEFAULT_FLAGS = DBS_FAST_MODE; + uint64_t DEFAULT_FLAGS = DBF_FAST; if(options.size() == 0) { @@ -361,15 +351,19 @@ namespace cryptonote if(options[0] == "safe") { safemode = true; - db_flags = DBS_SAFE_MODE; + db_flags = DBF_SAFE; sync_mode = db_nosync; } else if(options[0] == "fast") - db_flags = DBS_FAST_MODE; + { + db_flags = DBF_FAST; + sync_mode = db_async; + } else if(options[0] == "fastest") { - db_flags = DBS_FASTEST_MODE; + db_flags = DBF_FASTEST; blocks_per_sync = 1000; // default to fastest:async:1000 + sync_mode = db_async; } else db_flags = DEFAULT_FLAGS; @@ -392,7 +386,7 @@ namespace cryptonote } if (db_salvage) - db_flags |= DBS_SALVAGE; + db_flags |= DBF_SALVAGE; db->open(filename, db_flags); if(!db->m_open) @@ -1049,6 +1043,11 @@ namespace cryptonote m_miner.on_synchronized(); } //----------------------------------------------------------------------------------------------- + void core::safesyncmode(const bool onoff) + { + m_blockchain_storage.safesyncmode(onoff); + } + //----------------------------------------------------------------------------------------------- bool core::add_new_block(const block& b, block_verification_context& bvc) { return m_blockchain_storage.add_new_block(b, bvc); diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index f17a6dfe6..f565afd87 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -428,6 +428,13 @@ namespace cryptonote bool get_pool_transactions(std::list<transaction>& txs) const; /** + * @copydoc tx_memory_pool::get_txpool_backlog + * + * @note see tx_memory_pool::get_txpool_backlog + */ + bool get_txpool_backlog(std::vector<tx_backlog_entry>& backlog) const; + + /** * @copydoc tx_memory_pool::get_transactions * * @note see tx_memory_pool::get_transactions @@ -614,6 +621,13 @@ namespace cryptonote void on_synchronized(); /** + * @copydoc Blockchain::safesyncmode + * + * 2note see Blockchain::safesyncmode + */ + void safesyncmode(const bool onoff); + + /** * @brief sets the target blockchain height * * @param target_blockchain_height the height to set diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index 33b1d4101..265a4ca3e 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -553,6 +553,17 @@ namespace cryptonote }); } //------------------------------------------------------------------ + void tx_memory_pool::get_transaction_backlog(std::vector<tx_backlog_entry>& backlog) const + { + CRITICAL_REGION_LOCAL(m_transactions_lock); + CRITICAL_REGION_LOCAL1(m_blockchain); + const uint64_t now = time(NULL); + m_blockchain.for_all_txpool_txes([&backlog, now](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ + backlog.push_back({meta.blob_size, meta.fee, meta.receive_time - now}); + return true; + }); + } + //------------------------------------------------------------------ void tx_memory_pool::get_transaction_stats(struct txpool_stats& stats) const { CRITICAL_REGION_LOCAL(m_transactions_lock); @@ -560,7 +571,10 @@ namespace cryptonote const uint64_t now = time(NULL); std::map<uint64_t, txpool_histo> agebytes; stats.txs_total = m_blockchain.get_txpool_tx_count(); - m_blockchain.for_all_txpool_txes([&stats, now, &agebytes](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ + std::vector<uint32_t> sizes; + sizes.reserve(stats.txs_total); + m_blockchain.for_all_txpool_txes([&stats, &sizes, now, &agebytes](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ + sizes.push_back(meta.blob_size); stats.bytes_total += meta.blob_size; if (!stats.bytes_min || meta.blob_size < stats.bytes_min) stats.bytes_min = meta.blob_size; @@ -575,11 +589,12 @@ namespace cryptonote stats.num_10m++; if (meta.last_failed_height) stats.num_failing++; - uint64_t age = now - meta.receive_time; + uint64_t age = now - meta.receive_time + (now == meta.receive_time); agebytes[age].txs++; agebytes[age].bytes += meta.blob_size; return true; }); + stats.bytes_med = epee::misc_utils::median(sizes); if (stats.txs_total > 1) { /* looking for 98th percentile */ @@ -905,7 +920,21 @@ namespace cryptonote // Skip transactions that are not ready to be // included into the blockchain or that are // missing key images - if (!is_transaction_ready_to_go(meta, tx)) + const cryptonote::txpool_tx_meta_t original_meta = meta; + bool ready = is_transaction_ready_to_go(meta, tx); + if (memcmp(&original_meta, &meta, sizeof(meta))) + { + try + { + m_blockchain.update_txpool_tx(sorted_it->second, meta); + } + catch (const std::exception &e) + { + MERROR("Failed to update tx meta: " << e.what()); + // continue, not fatal + } + } + if (!ready) { LOG_PRINT_L2(" not ready to go"); sorted_it++; diff --git a/src/cryptonote_core/tx_pool.h b/src/cryptonote_core/tx_pool.h index 47a41d070..6414620c9 100644 --- a/src/cryptonote_core/tx_pool.h +++ b/src/cryptonote_core/tx_pool.h @@ -243,6 +243,13 @@ namespace cryptonote void get_transaction_hashes(std::vector<crypto::hash>& txs) const; /** + * @brief get (size, fee, receive time) for all transaction in the pool + * + * @param txs return-by-reference that data + */ + void get_transaction_backlog(std::vector<tx_backlog_entry>& backlog) const; + + /** * @brief get a summary statistics of all transaction hashes in the pool * * @param stats return-by-reference the pool statistics diff --git a/src/cryptonote_protocol/block_queue.cpp b/src/cryptonote_protocol/block_queue.cpp index 94d31404e..02a8e3ec2 100644 --- a/src/cryptonote_protocol/block_queue.cpp +++ b/src/cryptonote_protocol/block_queue.cpp @@ -52,8 +52,11 @@ namespace cryptonote void block_queue::add_blocks(uint64_t height, std::list<cryptonote::block_complete_entry> bcel, const boost::uuids::uuid &connection_id, float rate, size_t size) { boost::unique_lock<boost::recursive_mutex> lock(mutex); - remove_span(height); + std::list<crypto::hash> hashes; + bool has_hashes = remove_span(height, &hashes); blocks.insert(span(height, std::move(bcel), connection_id, rate, size)); + if (has_hashes) + set_span_hashes(height, connection_id, hashes); } void block_queue::add_blocks(uint64_t height, uint64_t nblocks, const boost::uuids::uuid &connection_id, boost::posix_time::ptime time) @@ -92,17 +95,20 @@ void block_queue::flush_stale_spans(const std::set<boost::uuids::uuid> &live_con } } -void block_queue::remove_span(uint64_t start_block_height) +bool block_queue::remove_span(uint64_t start_block_height, std::list<crypto::hash> *hashes) { boost::unique_lock<boost::recursive_mutex> lock(mutex); for (block_map::iterator i = blocks.begin(); i != blocks.end(); ++i) { if (i->start_block_height == start_block_height) { + if (hashes) + *hashes = std::move(i->hashes); blocks.erase(i); - return; + return true; } } + return false; } void block_queue::remove_spans(const boost::uuids::uuid &connection_id, uint64_t start_block_height) @@ -278,6 +284,22 @@ bool block_queue::get_next_span(uint64_t &height, std::list<cryptonote::block_co return false; } +bool block_queue::has_next_span(const boost::uuids::uuid &connection_id, bool &filled) const +{ + boost::unique_lock<boost::recursive_mutex> lock(mutex); + if (blocks.empty()) + return false; + block_map::const_iterator i = blocks.begin(); + if (is_blockchain_placeholder(*i)) + ++i; + if (i == blocks.end()) + return false; + if (i->connection_id != connection_id) + return false; + filled = !i->blocks.empty(); + return true; +} + size_t block_queue::get_data_size() const { boost::unique_lock<boost::recursive_mutex> lock(mutex); diff --git a/src/cryptonote_protocol/block_queue.h b/src/cryptonote_protocol/block_queue.h index fa1a0f217..13d4619bf 100644 --- a/src/cryptonote_protocol/block_queue.h +++ b/src/cryptonote_protocol/block_queue.h @@ -71,7 +71,7 @@ namespace cryptonote void add_blocks(uint64_t height, uint64_t nblocks, const boost::uuids::uuid &connection_id, boost::posix_time::ptime time = boost::date_time::min_date_time); void flush_spans(const boost::uuids::uuid &connection_id, bool all = false); void flush_stale_spans(const std::set<boost::uuids::uuid> &live_connections); - void remove_span(uint64_t start_block_height); + bool remove_span(uint64_t start_block_height, std::list<crypto::hash> *hashes = NULL); void remove_spans(const boost::uuids::uuid &connection_id, uint64_t start_block_height); uint64_t get_max_block_height() const; void print() const; @@ -82,6 +82,7 @@ namespace cryptonote std::pair<uint64_t, uint64_t> get_next_span_if_scheduled(std::list<crypto::hash> &hashes, boost::uuids::uuid &connection_id, boost::posix_time::ptime &time) const; void set_span_hashes(uint64_t start_height, const boost::uuids::uuid &connection_id, std::list<crypto::hash> hashes); bool get_next_span(uint64_t &height, std::list<cryptonote::block_complete_entry> &bcel, boost::uuids::uuid &connection_id, bool filled = true) const; + bool has_next_span(const boost::uuids::uuid &connection_id, bool &filled) const; size_t get_data_size() const; size_t get_num_filled_spans_prefix() const; size_t get_num_filled_spans() const; diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.h b/src/cryptonote_protocol/cryptonote_protocol_handler.h index d94747769..d54687e6a 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.h @@ -111,6 +111,7 @@ namespace cryptonote std::list<connection_info> get_connections(); const block_queue &get_block_queue() const { return m_block_queue; } void stop(); + void on_connection_close(cryptonote_connection_context &context); private: //----------------- commands handlers ---------------------------------------------- int handle_notify_new_block(int command, NOTIFY_NEW_BLOCK::request& arg, cryptonote_connection_context& context); @@ -133,6 +134,7 @@ namespace cryptonote bool should_download_next_span(cryptonote_connection_context& context) const; void drop_connection(cryptonote_connection_context &context, bool add_fail, bool flush_all_spans); bool kick_idle_peers(); + int try_add_next_blocks(cryptonote_connection_context &context); t_core& m_core; diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index daefe88b7..e762cf9c8 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -106,6 +106,11 @@ namespace cryptonote LOG_PRINT_CCONTEXT_L2("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); post_notify<NOTIFY_REQUEST_CHAIN>(r, context); } + else if(context.m_state == cryptonote_connection_context::state_standby) + { + context.m_state = cryptonote_connection_context::state_synchronizing; + try_add_next_blocks(context); + } return true; } @@ -263,7 +268,9 @@ namespace cryptonote const uint8_t version = m_core.get_ideal_hard_fork_version(hshd.current_height - 1); if (version >= 6 && version != hshd.top_version) { - LOG_DEBUG_CC(context, "Ignoring due to wrong top version " << (unsigned)hshd.top_version << ", expected " << (unsigned)version); + if (version < hshd.top_version) + MCLOG_RED(el::Level::Warning, "global", context << " peer claims higher version that we think - we may be forked from the network and a software upgrade may be needed"); + LOG_DEBUG_CC(context, "Ignoring due to wrong top version for block " << (hshd.current_height - 1) << ": " << (unsigned)hshd.top_version << ", expected " << (unsigned)version); return false; } @@ -286,15 +293,17 @@ namespace cryptonote /* As I don't know if accessing hshd from core could be a good practice, I prefer pushing target height to the core at the same time it is pushed to the user. Nz. */ - m_core.set_target_blockchain_height(static_cast<int64_t>(hshd.current_height)); + m_core.set_target_blockchain_height((hshd.current_height)); int64_t diff = static_cast<int64_t>(hshd.current_height) - static_cast<int64_t>(m_core.get_current_blockchain_height()); - int64_t max_block_height = max(static_cast<int64_t>(hshd.current_height),static_cast<int64_t>(m_core.get_current_blockchain_height())); - int64_t last_block_v1 = m_core.get_testnet() ? 624633 : 1009826; - int64_t diff_v2 = max_block_height > last_block_v1 ? min(abs(diff), max_block_height - last_block_v1) : 0; + uint64_t abs_diff = std::abs(diff); + uint64_t max_block_height = max(hshd.current_height,m_core.get_current_blockchain_height()); + uint64_t last_block_v1 = m_core.get_testnet() ? 624633 : 1009826; + uint64_t diff_v2 = max_block_height > last_block_v1 ? min(abs_diff, max_block_height - last_block_v1) : 0; MCLOG(is_inital ? el::Level::Info : el::Level::Debug, "global", context << "Sync data returned a new top block candidate: " << m_core.get_current_blockchain_height() << " -> " << hshd.current_height - << " [Your node is " << std::abs(diff) << " blocks (" << ((abs(diff) - diff_v2) / (24 * 60 * 60 / DIFFICULTY_TARGET_V1)) + (diff_v2 / (24 * 60 * 60 / DIFFICULTY_TARGET_V2)) << " days) " + << " [Your node is " << abs_diff << " blocks (" << ((abs_diff - diff_v2) / (24 * 60 * 60 / DIFFICULTY_TARGET_V1)) + (diff_v2 / (24 * 60 * 60 / DIFFICULTY_TARGET_V2)) << " days) " << (0 <= diff ? std::string("behind") : std::string("ahead")) << "] " << ENDL << "SYNCHRONIZATION started"); + m_core.safesyncmode(false); } LOG_PRINT_L1("Remote blockchain height: " << hshd.current_height << ", id: " << hshd.top_id); context.m_state = cryptonote_connection_context::state_synchronizing; @@ -309,7 +318,7 @@ namespace cryptonote bool t_cryptonote_protocol_handler<t_core>::get_payload_sync_data(CORE_SYNC_DATA& hshd) { m_core.get_blockchain_top(hshd.current_height, hshd.top_id); - hshd.top_version = m_core.get_hard_fork_version(hshd.current_height); + hshd.top_version = m_core.get_ideal_hard_fork_version(hshd.current_height); hshd.cumulative_difficulty = m_core.get_block_cumulative_difficulty(hshd.current_height); hshd.current_height +=1; return true; @@ -819,8 +828,6 @@ namespace cryptonote { MLOG_P2P_MESSAGE("Received NOTIFY_RESPONSE_GET_OBJECTS (" << arg.blocks.size() << " blocks, " << arg.txs.size() << " txes)"); - bool force_next_span = false; - // calculate size of request size_t size = 0; for (const auto &element : arg.txs) size += element.size(); @@ -938,19 +945,34 @@ namespace cryptonote context.m_last_known_hash = cryptonote::get_blob_hash(arg.blocks.back().block); - if (m_core.get_test_drop_download() && m_core.get_test_drop_download_height()) { // DISCARD BLOCKS for testing + if (!m_core.get_test_drop_download() || !m_core.get_test_drop_download_height()) { // DISCARD BLOCKS for testing + return 1; + } + } - // We try to lock the sync lock. If we can, it means no other thread is - // currently adding blocks, so we do that for as long as we can from the - // block queue. Then, we go back to download. - const boost::unique_lock<boost::mutex> sync{m_sync_lock, boost::try_to_lock}; - if (!sync.owns_lock()) - { - MINFO("Failed to lock m_sync_lock, going back to download"); - goto skip; - } - MDEBUG(context << " lock m_sync_lock, adding blocks to chain..."); +skip: + try_add_next_blocks(context); + return 1; + } + template<class t_core> + int t_cryptonote_protocol_handler<t_core>::try_add_next_blocks(cryptonote_connection_context& context) + { + bool force_next_span = false; + + { + // We try to lock the sync lock. If we can, it means no other thread is + // currently adding blocks, so we do that for as long as we can from the + // block queue. Then, we go back to download. + const boost::unique_lock<boost::mutex> sync{m_sync_lock, boost::try_to_lock}; + if (!sync.owns_lock()) + { + MINFO("Failed to lock m_sync_lock, going back to download"); + goto skip; + } + MDEBUG(context << " lock m_sync_lock, adding blocks to chain..."); + + { m_core.pause_mine(); epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler( boost::bind(&t_core::resume_mine, &m_core)); @@ -984,21 +1006,15 @@ namespace cryptonote // - later in an alt chain // - orphan // if it was requested, then it'll be resolved later, otherwise it's an orphan - bool parent_requested = false; - m_p2p->for_each_connection([&](cryptonote_connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags)->bool{ - if (context.m_requested_objects.find(new_block.prev_id) != context.m_requested_objects.end()) - { - parent_requested = true; - return false; - } - return true; - }); + bool parent_requested = m_block_queue.requested(new_block.prev_id); if (!parent_requested) { - LOG_ERROR_CCONTEXT("Got block with unknown parent which was not requested - dropping connection"); - // in case the peer had dropped beforehand, remove the span anyway so other threads can wake up and get it - m_block_queue.remove_spans(span_connection_id, start_height); - return 1; + // this can happen if a connection was sicced onto a late span, if it did not have those blocks, + // since we don't know that at the sic time + LOG_ERROR_CCONTEXT("Got block with unknown parent which was not requested - querying block hashes"); + context.m_needed_objects.clear(); + context.m_last_response_height = 0; + goto skip; } // parent was requested, so we wait for it to be retrieved @@ -1007,6 +1023,7 @@ namespace cryptonote } const boost::posix_time::ptime start = boost::posix_time::microsec_clock::universal_time(); + context.m_last_request_time = start; m_core.prepare_handle_incoming_blocks(blocks); @@ -1108,7 +1125,7 @@ namespace cryptonote << timing_message); } } - } // if not DISCARD BLOCK + } if (should_download_next_span(context)) { @@ -1179,9 +1196,17 @@ skip: std::list<crypto::hash> hashes; boost::uuids::uuid span_connection_id; boost::posix_time::ptime request_time; - std::pair<uint64_t, uint64_t> span = m_block_queue.get_next_span_if_scheduled(hashes, span_connection_id, request_time); + std::pair<uint64_t, uint64_t> span; + + span = m_block_queue.get_start_gap_span(); + if (span.second > 0) + { + MDEBUG(context << " we should download it as there is a gap"); + return true; + } // if the next span is not scheduled (or there is none) + span = m_block_queue.get_next_span_if_scheduled(hashes, span_connection_id, request_time); if (span.second == 0) { // we might be in a weird case where there is a filled next span, @@ -1270,6 +1295,17 @@ skip: first = false; context.m_state = cryptonote_connection_context::state_standby; } + + // this needs doing after we went to standby, so the callback knows what to do + bool filled; + if (m_block_queue.has_next_span(context.m_connection_id, filled) && !filled) + { + MDEBUG(context << " we have the next span, and it is scheduled, resuming"); + ++context.m_callback_request_count; + m_p2p->request_callback(context); + return 1; + } + for (size_t n = 0; n < 50; ++n) { if (m_stopping) @@ -1289,9 +1325,8 @@ skip: size_t count = 0; const size_t count_limit = m_core.get_block_sync_size(m_core.get_current_blockchain_height()); std::pair<uint64_t, uint64_t> span = std::make_pair(0, 0); - if (force_next_span) { - MDEBUG(context << " force_next_span is true, trying next span"); + MDEBUG(context << " checking for gap"); span = m_block_queue.get_start_gap_span(); if (span.second > 0) { @@ -1311,6 +1346,9 @@ skip: } MDEBUG(context << " we have the hashes for this gap"); } + } + if (force_next_span) + { if (span.second == 0) { std::list<crypto::hash> hashes; @@ -1360,7 +1398,12 @@ skip: for (const auto &hash: hashes) { req.blocks.push_back(hash); + ++count; context.m_requested_objects.insert(hash); + // that's atrocious O(n) wise, but this is rare + auto i = std::find(context.m_needed_objects.begin(), context.m_needed_objects.end(), hash); + if (i != context.m_needed_objects.end()) + context.m_needed_objects.erase(i); } } } @@ -1384,14 +1427,12 @@ skip: return false; } - std::list<crypto::hash> hashes; auto it = context.m_needed_objects.begin(); for (size_t n = 0; n < span.second; ++n) { req.blocks.push_back(*it); ++count; context.m_requested_objects.insert(*it); - hashes.push_back(*it); auto j = it++; context.m_needed_objects.erase(j); } @@ -1399,7 +1440,7 @@ skip: context.m_last_request_time = boost::posix_time::microsec_clock::universal_time(); LOG_PRINT_CCONTEXT_L1("-->>NOTIFY_REQUEST_GET_OBJECTS: blocks.size()=" << req.blocks.size() << ", txs.size()=" << req.txs.size() - << "requested blocks count=" << count << " / " << count_limit << " from " << span.first); + << "requested blocks count=" << count << " / " << count_limit << " from " << span.first << ", first hash " << req.blocks.front()); //epee::net_utils::network_throttle_manager::get_global_throttle_inreq().logger_handle_net("log/dr-monero/net/req-all.data", sec, get_avg_block_size()); post_notify<NOTIFY_REQUEST_GET_OBJECTS>(req, context); @@ -1473,6 +1514,7 @@ skip: << "**********************************************************************"); m_core.on_synchronized(); } + m_core.safesyncmode(true); return true; } //------------------------------------------------------------------------------------------------------------------------ @@ -1523,6 +1565,10 @@ skip: drop_connection(context, false, false); return 1; } + + if (arg.total_height > m_core.get_target_blockchain_height()) + m_core.set_target_blockchain_height(arg.total_height); + return 1; } //------------------------------------------------------------------------------------------------------------------------ @@ -1582,8 +1628,15 @@ skip: { if (add_fail) m_p2p->add_host_fail(context.m_remote_address); + m_p2p->drop_connection(context); + m_block_queue.flush_spans(context.m_connection_id, flush_all_spans); + } + //------------------------------------------------------------------------------------------------------------------------ + template<class t_core> + void t_cryptonote_protocol_handler<t_core>::on_connection_close(cryptonote_connection_context &context) + { uint64_t target = 0; m_p2p->for_each_connection([&](const connection_context& cntxt, nodetool::peerid_type peer_id, uint32_t support_flags) { if (cntxt.m_state >= cryptonote_connection_context::state_synchronizing && cntxt.m_connection_id != context.m_connection_id) @@ -1597,7 +1650,7 @@ skip: m_core.set_target_blockchain_height(target); } - m_block_queue.flush_spans(context.m_connection_id, flush_all_spans); + m_block_queue.flush_spans(context.m_connection_id, false); } //------------------------------------------------------------------------------------------------------------------------ diff --git a/src/daemon/main.cpp b/src/daemon/main.cpp index 19dd02171..456eeee64 100644 --- a/src/daemon/main.cpp +++ b/src/daemon/main.cpp @@ -145,13 +145,10 @@ int main(int argc, char const * argv[]) std::string db_type = command_line::get_arg(vm, command_line::arg_db_type); // verify that blockchaindb type is valid - if(cryptonote::blockchain_db_types.count(db_type) == 0) + if(!cryptonote::blockchain_valid_db_type(db_type)) { - std::cout << "Invalid database type (" << db_type << "), available types are:" << std::endl; - for (const auto& type : cryptonote::blockchain_db_types) - { - std::cout << "\t" << type << std::endl; - } + std::cout << "Invalid database type (" << db_type << "), available types are: " << + cryptonote::blockchain_db_types(", ") << std::endl; return 0; } diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp index bd2142ab1..3d6a01cd1 100644 --- a/src/daemon/rpc_command_executor.cpp +++ b/src/daemon/rpc_command_executor.cpp @@ -113,18 +113,6 @@ namespace { return base; return base + " -- " + status; } - - std::string pad(std::string s, size_t n, char c = ' ', bool prepend = false) - { - if (s.size() < n) - { - if (prepend) - s = std::string(n - s.size(), c) + s; - else - s.append(n - s.size(), c); - } - return s; - } } t_rpc_command_executor::t_rpc_command_executor( @@ -497,7 +485,7 @@ bool t_rpc_command_executor::print_connections() { tools::msg_writer() //<< std::setw(30) << std::left << in_out << std::setw(30) << std::left << address - << std::setw(20) << pad(info.peer_id, 16, '0', true) + << std::setw(20) << epee::string_tools::pad_string(info.peer_id, 16, '0', true) << std::setw(20) << info.support_flags << std::setw(30) << std::to_string(info.recv_count) + "(" + std::to_string(info.recv_idle_time) + ")/" + std::to_string(info.send_count) + "(" + std::to_string(info.send_idle_time) + ")" << std::setw(25) << info.state @@ -939,6 +927,8 @@ bool t_rpc_command_executor::print_transaction_pool_short() { bool t_rpc_command_executor::print_transaction_pool_stats() { cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL_STATS::request req; cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL_STATS::response res; + cryptonote::COMMAND_RPC_GET_INFO::request ireq; + cryptonote::COMMAND_RPC_GET_INFO::response ires; std::string fail_message = "Problem fetching transaction pool stats"; @@ -948,6 +938,10 @@ bool t_rpc_command_executor::print_transaction_pool_stats() { { return true; } + if (!m_rpc_client->rpc_request(ireq, ires, "/getinfo", fail_message.c_str())) + { + return true; + } } else { @@ -957,15 +951,32 @@ bool t_rpc_command_executor::print_transaction_pool_stats() { tools::fail_msg_writer() << make_error(fail_message, res.status); return true; } + if (!m_rpc_server->on_get_info(ireq, ires) || ires.status != CORE_RPC_STATUS_OK) + { + tools::fail_msg_writer() << make_error(fail_message, ires.status); + return true; + } } size_t n_transactions = res.pool_stats.txs_total; const uint64_t now = time(NULL); size_t avg_bytes = n_transactions ? res.pool_stats.bytes_total / n_transactions : 0; - tools::msg_writer() << n_transactions << " tx(es), " << res.pool_stats.bytes_total << " bytes total (min " << res.pool_stats.bytes_min << ", max " << res.pool_stats.bytes_max << ", avg " << avg_bytes << ")" << std::endl - << "fees " << cryptonote::print_money(res.pool_stats.fee_total) << " (avg " << cryptonote::print_money(n_transactions ? res.pool_stats.fee_total / n_transactions : 0) << " per tx" << ", " << cryptonote::print_money(res.pool_stats.bytes_total ? res.pool_stats.fee_total / res.pool_stats.bytes_total : 0) << " per byte )" << std::endl - << res.pool_stats.num_not_relayed << " not relayed, " << res.pool_stats.num_failing << " failing, " << res.pool_stats.num_10m << " older than 10 minutes (oldest " << (res.pool_stats.oldest == 0 ? "-" : get_human_time_ago(res.pool_stats.oldest, now)) << ")"; + std::string backlog_message; + const uint64_t full_reward_zone = ires.block_size_limit / 2; + if (res.pool_stats.bytes_total <= full_reward_zone) + { + backlog_message = "no backlog"; + } + else + { + uint64_t backlog = (res.pool_stats.bytes_total + full_reward_zone - 1) / full_reward_zone; + backlog_message = (boost::format("estimated %u block (%u minutes) backlog") % backlog % (backlog * DIFFICULTY_TARGET_V2 / 60)).str(); + } + + tools::msg_writer() << n_transactions << " tx(es), " << res.pool_stats.bytes_total << " bytes total (min " << res.pool_stats.bytes_min << ", max " << res.pool_stats.bytes_max << ", avg " << avg_bytes << ", median " << res.pool_stats.bytes_med << ")" << std::endl + << "fees " << cryptonote::print_money(res.pool_stats.fee_total) << " (avg " << cryptonote::print_money(n_transactions ? res.pool_stats.fee_total / n_transactions : 0) << " per tx" << ", " << cryptonote::print_money(res.pool_stats.bytes_total ? res.pool_stats.fee_total / res.pool_stats.bytes_total : 0) << " per byte)" << std::endl + << res.pool_stats.num_not_relayed << " not relayed, " << res.pool_stats.num_failing << " failing, " << res.pool_stats.num_10m << " older than 10 minutes (oldest " << (res.pool_stats.oldest == 0 ? "-" : get_human_time_ago(res.pool_stats.oldest, now)) << "), " << backlog_message; if (n_transactions > 1 && res.pool_stats.histo.size()) { @@ -1744,12 +1755,12 @@ bool t_rpc_command_executor::sync_info() tools::success_msg_writer() << std::to_string(res.peers.size()) << " peers"; for (const auto &p: res.peers) { - std::string address = pad(p.info.address, 24); + std::string address = epee::string_tools::pad_string(p.info.address, 24); uint64_t nblocks = 0, size = 0; for (const auto &s: res.spans) if (s.rate > 0.0f && s.connection_id == p.info.connection_id) nblocks += s.nblocks, size += s.size; - tools::success_msg_writer() << address << " " << pad(p.info.peer_id, 16, '0', true) << " " << p.info.height << " " << p.info.current_download << " kB/s, " << nblocks << " blocks / " << size/1e6 << " MB queued"; + tools::success_msg_writer() << address << " " << epee::string_tools::pad_string(p.info.peer_id, 16, '0', true) << " " << p.info.height << " " << p.info.current_download << " kB/s, " << nblocks << " blocks / " << size/1e6 << " MB queued"; } uint64_t total_size = 0; @@ -1758,7 +1769,7 @@ bool t_rpc_command_executor::sync_info() tools::success_msg_writer() << std::to_string(res.spans.size()) << " spans, " << total_size/1e6 << " MB"; for (const auto &s: res.spans) { - std::string address = pad(s.remote_address, 24); + std::string address = epee::string_tools::pad_string(s.remote_address, 24); if (s.size == 0) { tools::success_msg_writer() << address << " " << s.nblocks << " (" << s.start_block_height << " - " << (s.start_block_height + s.nblocks - 1) << ") -"; diff --git a/src/debug_utilities/CMakeLists.txt b/src/debug_utilities/CMakeLists.txt index 99198dc57..e6d49fd61 100644 --- a/src/debug_utilities/CMakeLists.txt +++ b/src/debug_utilities/CMakeLists.txt @@ -37,6 +37,7 @@ monero_add_executable(cn_deserialize target_link_libraries(cn_deserialize LINK_PRIVATE cryptonote_core + common blockchain_db p2p epee diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index e179fc14f..889cfaf9d 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -1077,7 +1077,7 @@ namespace nodetool bool node_server<t_payload_net_handler>::make_new_connection_from_anchor_peerlist(const std::vector<anchor_peerlist_entry>& anchor_peerlist) { for (const auto& pe: anchor_peerlist) { - _note("Considering connecting (out) to peer: " << pe.id << " " << pe.adr.str()); + _note("Considering connecting (out) to peer: " << peerid_type(pe.id) << " " << pe.adr.str()); if(is_peer_used(pe)) { _note("Peer is used"); @@ -1092,7 +1092,7 @@ namespace nodetool continue; } - MDEBUG("Selected peer: " << pe.id << " " << pe.adr.str() + MDEBUG("Selected peer: " << peerid_to_string(pe.id) << " " << pe.adr.str() << "[peer_type=" << anchor << "] first_seen: " << epee::misc_utils::get_time_interval_string(time(NULL) - pe.first_seen)); @@ -1145,7 +1145,7 @@ namespace nodetool ++try_count; - _note("Considering connecting (out) to peer: " << pe.id << " " << pe.adr.str()); + _note("Considering connecting (out) to peer: " << peerid_to_string(pe.id) << " " << pe.adr.str()); if(is_peer_used(pe)) { _note("Peer is used"); @@ -1158,7 +1158,7 @@ namespace nodetool if(is_addr_recently_failed(pe.adr)) continue; - MDEBUG("Selected peer: " << pe.id << " " << pe.adr.str() + MDEBUG("Selected peer: " << peerid_to_string(pe.id) << " " << pe.adr.str() << "[peer_list=" << (use_white_list ? white : gray) << "] last_seen: " << (pe.last_seen ? epee::misc_utils::get_time_interval_string(time(NULL) - pe.last_seen) : "never")); @@ -1795,6 +1795,8 @@ namespace nodetool m_peerlist.remove_from_peer_anchor(na); } + m_payload_handler.on_connection_close(context); + MINFO("["<< epee::net_utils::print_connection_context(context) << "] CLOSE CONNECTION"); } @@ -1960,14 +1962,14 @@ namespace nodetool if (!success) { m_peerlist.remove_from_peer_gray(pe); - LOG_PRINT_L2("PEER EVICTED FROM GRAY PEER LIST IP address: " << pe.adr.host_str() << " Peer ID: " << std::hex << pe.id); + LOG_PRINT_L2("PEER EVICTED FROM GRAY PEER LIST IP address: " << pe.adr.host_str() << " Peer ID: " << peerid_type(pe.id)); return true; } m_peerlist.set_peer_just_seen(pe.id, pe.adr); - LOG_PRINT_L2("PEER PROMOTED TO WHITE PEER LIST IP address: " << pe.adr.host_str() << " Peer ID: " << std::hex << pe.id); + LOG_PRINT_L2("PEER PROMOTED TO WHITE PEER LIST IP address: " << pe.adr.host_str() << " Peer ID: " << peerid_type(pe.id)); return true; } diff --git a/src/p2p/p2p_protocol_defs.h b/src/p2p/p2p_protocol_defs.h index f38615def..f2b2cd1da 100644 --- a/src/p2p/p2p_protocol_defs.h +++ b/src/p2p/p2p_protocol_defs.h @@ -44,6 +44,13 @@ namespace nodetool typedef boost::uuids::uuid uuid; typedef uint64_t peerid_type; + static inline std::string peerid_to_string(peerid_type peer_id) + { + std::ostringstream s; + s << std::hex << peer_id; + return epee::string_tools::pad_string(s.str(), 16, '0', true); + } + #pragma pack (push, 1) struct network_address_old diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index a5de36118..a6a61705b 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -205,14 +205,17 @@ namespace cryptonote } size_t txidx = 0; ntxes += bd.second.size(); - for(const auto& t: bd.second) + for (std::list<cryptonote::blobdata>::iterator i = bd.second.begin(); i != bd.second.end(); ++i) { + unpruned_size += i->size(); if (req.prune) - res.blocks.back().txs.push_back(get_pruned_tx_blob(t)); + res.blocks.back().txs.push_back(get_pruned_tx_blob(std::move(*i))); else - res.blocks.back().txs.push_back(t); + res.blocks.back().txs.push_back(std::move(*i)); + i->clear(); + i->shrink_to_fit(); pruned_size += res.blocks.back().txs.back().size(); - unpruned_size += t.size(); + res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices()); bool r = m_core.get_tx_outputs_gindexs(b.tx_hashes[txidx++], res.output_indices.back().indices.back().indices); if (!r) @@ -1725,6 +1728,26 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_get_txpool_backlog(const COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::response& res, epee::json_rpc::error& error_resp) + { + if(!check_core_busy()) + { + error_resp.code = CORE_RPC_ERROR_CODE_CORE_BUSY; + error_resp.message = "Core is busy."; + return false; + } + + if (!m_core.get_txpool_backlog(res.backlog)) + { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = "Failed to get txpool backlog"; + return false; + } + + res.status = CORE_RPC_STATUS_OK; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ const command_line::arg_descriptor<std::string> core_rpc_server::arg_rpc_bind_port = { "rpc-bind-port" diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index 1d1d9da66..b526277a8 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -125,6 +125,7 @@ namespace cryptonote MAP_JON_RPC_WE_IF("get_alternate_chains",on_get_alternate_chains, COMMAND_RPC_GET_ALTERNATE_CHAINS, !m_restricted) MAP_JON_RPC_WE_IF("relay_tx", on_relay_tx, COMMAND_RPC_RELAY_TX, !m_restricted) MAP_JON_RPC_WE_IF("sync_info", on_sync_info, COMMAND_RPC_SYNC_INFO, !m_restricted) + MAP_JON_RPC_WE_IF("get_txpool_backlog", on_get_txpool_backlog, COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG, !m_restricted) END_JSON_RPC_MAP() END_URI_MAP2() @@ -182,6 +183,7 @@ namespace cryptonote bool on_get_alternate_chains(const COMMAND_RPC_GET_ALTERNATE_CHAINS::request& req, COMMAND_RPC_GET_ALTERNATE_CHAINS::response& res, epee::json_rpc::error& error_resp); bool on_relay_tx(const COMMAND_RPC_RELAY_TX::request& req, COMMAND_RPC_RELAY_TX::response& res, epee::json_rpc::error& error_resp); bool on_sync_info(const COMMAND_RPC_SYNC_INFO::request& req, COMMAND_RPC_SYNC_INFO::response& res, epee::json_rpc::error& error_resp); + bool on_get_txpool_backlog(const COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::request& req, COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::response& res, epee::json_rpc::error& error_resp); //----------------------- private: diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index c413f9af8..88327dd75 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -1071,6 +1071,33 @@ namespace cryptonote }; }; + struct tx_backlog_entry + { + uint64_t blob_size; + uint64_t fee; + uint64_t time_in_pool; + }; + + struct COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG + { + struct request + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string status; + std::vector<tx_backlog_entry> backlog; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(status) + KV_SERIALIZE_CONTAINER_POD_AS_BLOB(backlog) + END_KV_SERIALIZE_MAP() + }; + }; + struct txpool_histo { uint32_t txs; @@ -1087,6 +1114,7 @@ namespace cryptonote uint64_t bytes_total; uint32_t bytes_min; uint32_t bytes_max; + uint32_t bytes_med; uint64_t fee_total; uint64_t oldest; uint32_t txs_total; @@ -1100,6 +1128,7 @@ namespace cryptonote KV_SERIALIZE(bytes_total) KV_SERIALIZE(bytes_min) KV_SERIALIZE(bytes_max) + KV_SERIALIZE(bytes_med) KV_SERIALIZE(fee_total) KV_SERIALIZE(oldest) KV_SERIALIZE(txs_total) diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index 1811eeb3c..479adcafc 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -63,6 +63,10 @@ #include "wallet/wallet_args.h" #include <stdexcept> +#ifdef HAVE_READLINE +#include "readline_buffer.h" +#endif + using namespace std; using namespace epee; using namespace cryptonote; @@ -646,6 +650,17 @@ bool simple_wallet::set_merge_destinations(const std::vector<std::string> &args/ return true; } +bool simple_wallet::set_confirm_backlog(const std::vector<std::string> &args/* = std::vector<std::string>()*/) +{ + const auto pwd_container = get_and_verify_password(); + if (pwd_container) + { + m_wallet->confirm_backlog(is_it_true(args[1])); + m_wallet->rewrite(m_wallet_file, pwd_container->password()); + } + return true; +} + bool simple_wallet::help(const std::vector<std::string> &args/* = std::vector<std::string>()*/) { success_msg_writer() << get_commands_str(); @@ -686,7 +701,7 @@ simple_wallet::simple_wallet() m_cmd_binder.set_handler("viewkey", boost::bind(&simple_wallet::viewkey, this, _1), tr("Display private view key")); m_cmd_binder.set_handler("spendkey", boost::bind(&simple_wallet::spendkey, this, _1), tr("Display private spend key")); m_cmd_binder.set_handler("seed", boost::bind(&simple_wallet::seed, this, _1), tr("Display Electrum-style mnemonic seed")); - m_cmd_binder.set_handler("set", boost::bind(&simple_wallet::set_variable, this, _1), tr("Available options: seed language - set wallet seed language; always-confirm-transfers <1|0> - whether to confirm unsplit txes; print-ring-members <1|0> - whether to print detailed information about ring members during confirmation; store-tx-info <1|0> - whether to store outgoing tx info (destination address, payment ID, tx secret key) for future reference; default-ring-size <n> - set default ring size (default is 5); auto-refresh <1|0> - whether to automatically sync new blocks from the daemon; refresh-type <full|optimize-coinbase|no-coinbase|default> - set wallet refresh behaviour; priority [0|1|2|3|4] - default/unimportant/normal/elevated/priority fee; confirm-missing-payment-id <1|0>; ask-password <1|0>; unit <monero|millinero|micronero|nanonero|piconero> - set default monero (sub-)unit; min-outputs-count [n] - try to keep at least that many outputs of value at least min-outputs-value; min-outputs-value [n] - try to keep at least min-outputs-count outputs of at least that value; merge-destinations <1|0> - whether to merge multiple payments to the same destination address")); + m_cmd_binder.set_handler("set", boost::bind(&simple_wallet::set_variable, this, _1), tr("Available options: seed language - set wallet seed language; always-confirm-transfers <1|0> - whether to confirm unsplit txes; print-ring-members <1|0> - whether to print detailed information about ring members during confirmation; store-tx-info <1|0> - whether to store outgoing tx info (destination address, payment ID, tx secret key) for future reference; default-ring-size <n> - set default ring size (default is 5); auto-refresh <1|0> - whether to automatically sync new blocks from the daemon; refresh-type <full|optimize-coinbase|no-coinbase|default> - set wallet refresh behaviour; priority [0|1|2|3|4] - default/unimportant/normal/elevated/priority fee; confirm-missing-payment-id <1|0>; ask-password <1|0>; unit <monero|millinero|micronero|nanonero|piconero> - set default monero (sub-)unit; min-outputs-count [n] - try to keep at least that many outputs of value at least min-outputs-value; min-outputs-value [n] - try to keep at least min-outputs-count outputs of at least that value; merge-destinations <1|0> - whether to merge multiple payments to the same destination address; confirm-backlog <1|0> - whether to warn if there is transaction backlog")); m_cmd_binder.set_handler("rescan_spent", boost::bind(&simple_wallet::rescan_spent, this, _1), tr("Rescan blockchain for spent outputs")); m_cmd_binder.set_handler("get_tx_key", boost::bind(&simple_wallet::get_tx_key, this, _1), tr("Get transaction key (r) for a given <txid>")); m_cmd_binder.set_handler("check_tx_key", boost::bind(&simple_wallet::check_tx_key, this, _1), tr("Check amount going to <address> in <txid>")); @@ -728,6 +743,7 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args) success_msg_writer() << "min-outputs-count = " << m_wallet->get_min_output_count(); success_msg_writer() << "min-outputs-value = " << cryptonote::print_money(m_wallet->get_min_output_value()); success_msg_writer() << "merge-destinations = " << m_wallet->merge_destinations(); + success_msg_writer() << "confirm-backlog = " << m_wallet->confirm_backlog(); return true; } else @@ -773,6 +789,7 @@ bool simple_wallet::set_variable(const std::vector<std::string> &args) CHECK_SIMPLE_VARIABLE("min-outputs-count", set_min_output_count, tr("unsigned integer")); CHECK_SIMPLE_VARIABLE("min-outputs-value", set_min_output_value, tr("amount")); CHECK_SIMPLE_VARIABLE("merge-destinations", set_merge_destinations, tr("0 or 1")); + CHECK_SIMPLE_VARIABLE("confirm-backlog", set_confirm_backlog, tr("0 or 1")); } fail_msg_writer() << tr("set: unrecognized argument(s)"); return true; @@ -1835,6 +1852,10 @@ bool simple_wallet::refresh_main(uint64_t start_height, bool reset) if (reset) m_wallet->rescan_blockchain(false); +#ifdef HAVE_READLINE + rdln::suspend_readline pause_readline; +#endif + message_writer() << tr("Starting refresh..."); uint64_t fetched_blocks = 0; @@ -2421,6 +2442,49 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri return true; } + // if we need to check for backlog, check the worst case tx + if (m_wallet->confirm_backlog()) + { + std::stringstream prompt; + double worst_fee_per_byte = std::numeric_limits<double>::max(); + uint64_t size = 0, fee = 0; + for (size_t n = 0; n < ptx_vector.size(); ++n) + { + const uint64_t blob_size = cryptonote::tx_to_blob(ptx_vector[n].tx).size(); + const double fee_per_byte = ptx_vector[n].fee / (double)blob_size; + if (fee_per_byte < worst_fee_per_byte) + { + worst_fee_per_byte = fee_per_byte; + fee = ptx_vector[n].fee; + } + size += blob_size; + } + try + { + uint64_t nblocks = m_wallet->estimate_backlog(size, fee); + if (nblocks > 0) + prompt << (boost::format(tr("There is currently a %u block backlog at that fee level. Is this okay? (Y/Yes/N/No)")) % nblocks).str(); + } + catch (const std::exception &e) + { + prompt << tr("Failed to check for backlog: ") << e.what() << ENDL << tr("Is this okay anyway? (Y/Yes/N/No): "); + } + + std::string prompt_str = prompt.str(); + if (!prompt_str.empty()) + { + std::string accepted = command_line::input_line(prompt_str); + if (std::cin.eof()) + return true; + if (!command_line::is_yes(accepted)) + { + fail_msg_writer() << tr("transaction cancelled."); + + return true; + } + } + } + // if more than one tx necessary, prompt user to confirm if (m_wallet->always_confirm_transfers() || ptx_vector.size() > 1) { diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h index 8022c9bb2..eac4cbc99 100644 --- a/src/simplewallet/simplewallet.h +++ b/src/simplewallet/simplewallet.h @@ -120,6 +120,7 @@ namespace cryptonote bool set_min_output_count(const std::vector<std::string> &args = std::vector<std::string>()); bool set_min_output_value(const std::vector<std::string> &args = std::vector<std::string>()); bool set_merge_destinations(const std::vector<std::string> &args = std::vector<std::string>()); + bool set_confirm_backlog(const std::vector<std::string> &args = std::vector<std::string>()); bool help(const std::vector<std::string> &args = std::vector<std::string>()); bool start_mining(const std::vector<std::string> &args); bool stop_mining(const std::vector<std::string> &args); diff --git a/src/wallet/api/wallet.cpp b/src/wallet/api/wallet.cpp index c0974f880..7afc1f449 100644 --- a/src/wallet/api/wallet.cpp +++ b/src/wallet/api/wallet.cpp @@ -379,7 +379,32 @@ bool WalletImpl::createWatchOnly(const std::string &path, const std::string &pas const cryptonote::account_public_address address = m_wallet->get_account().get_keys().m_account_address; try { + // Generate view only wallet view_wallet->generate(path, password, address, viewkey); + + // Export/Import outputs + auto outputs = m_wallet->export_outputs(); + view_wallet->import_outputs(outputs); + + // Copy scanned blockchain + auto bc = m_wallet->export_blockchain(); + view_wallet->import_blockchain(bc); + + // copy payments + auto payments = m_wallet->export_payments(); + view_wallet->import_payments(payments); + + // copy confirmed outgoing payments + std::list<std::pair<crypto::hash, tools::wallet2::confirmed_transfer_details>> out_payments; + m_wallet->get_payments_out(out_payments, 0); + view_wallet->import_payments_out(out_payments); + + // Export/Import key images + // We already know the spent status from the outputs we exported, thus no need to check them again + auto key_images = m_wallet->export_key_images(); + uint64_t spent = 0; + uint64_t unspent = 0; + view_wallet->import_key_images(key_images,spent,unspent,false); m_status = Status_Ok; } catch (const std::exception &e) { LOG_ERROR("Error creating view only wallet: " << e.what()); @@ -387,6 +412,8 @@ bool WalletImpl::createWatchOnly(const std::string &path, const std::string &pas m_errorString = e.what(); return false; } + // Store wallet + view_wallet->store(); return true; } @@ -862,6 +889,11 @@ bool WalletImpl::exportKeyImages(const string &filename) bool WalletImpl::importKeyImages(const string &filename) { + if (!trustedDaemon()) { + m_status = Status_Error; + m_errorString = tr("Key images can only be imported with a trusted daemon"); + return false; + } try { uint64_t spent = 0, unspent = 0; diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index b63e07b2d..4f21e3e77 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -1963,6 +1963,9 @@ bool wallet2::store_keys(const std::string& keys_file_name, const std::string& p value2.SetInt(m_merge_destinations ? 1 :0); json.AddMember("merge_destinations", value2, json.GetAllocator()); + value2.SetInt(m_confirm_backlog ? 1 :0); + json.AddMember("confirm_backlog", value2, json.GetAllocator()); + value2.SetInt(m_testnet ? 1 :0); json.AddMember("testnet", value2, json.GetAllocator()); @@ -2037,6 +2040,7 @@ bool wallet2::load_keys(const std::string& keys_file_name, const std::string& pa m_min_output_count = 0; m_min_output_value = 0; m_merge_destinations = false; + m_confirm_backlog = true; } else { @@ -2107,6 +2111,8 @@ bool wallet2::load_keys(const std::string& keys_file_name, const std::string& pa m_min_output_value = field_min_output_value; GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, merge_destinations, int, Int, false, false); m_merge_destinations = field_merge_destinations; + GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, confirm_backlog, int, Int, false, true); + m_confirm_backlog = field_confirm_backlog; GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, testnet, int, Int, false, m_testnet); // Wallet is being opened with testnet flag, but is saved as a mainnet wallet THROW_WALLET_EXCEPTION_IF(m_testnet && !field_testnet, error::wallet_internal_error, "Mainnet wallet can not be opened as testnet wallet"); @@ -5324,7 +5330,7 @@ uint64_t wallet2::import_key_images(const std::string &filename, uint64_t &spent } //---------------------------------------------------------------------------------------------------- -uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_image, crypto::signature>> &signed_key_images, uint64_t &spent, uint64_t &unspent) +uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_image, crypto::signature>> &signed_key_images, uint64_t &spent, uint64_t &unspent, bool check_spent) { COMMAND_RPC_IS_KEY_IMAGE_SPENT::request req = AUTO_VAL_INIT(req); COMMAND_RPC_IS_KEY_IMAGE_SPENT::response daemon_resp = AUTO_VAL_INIT(daemon_resp); @@ -5373,34 +5379,88 @@ uint64_t wallet2::import_key_images(const std::vector<std::pair<crypto::key_imag m_transfers[n].m_key_image_known = true; } - m_daemon_rpc_mutex.lock(); - bool r = epee::net_utils::invoke_http_json("/is_key_image_spent", req, daemon_resp, m_http_client, rpc_timeout); - m_daemon_rpc_mutex.unlock(); - THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "is_key_image_spent"); - THROW_WALLET_EXCEPTION_IF(daemon_resp.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "is_key_image_spent"); - THROW_WALLET_EXCEPTION_IF(daemon_resp.status != CORE_RPC_STATUS_OK, error::is_key_image_spent_error, daemon_resp.status); - THROW_WALLET_EXCEPTION_IF(daemon_resp.spent_status.size() != signed_key_images.size(), error::wallet_internal_error, - "daemon returned wrong response for is_key_image_spent, wrong amounts count = " + - std::to_string(daemon_resp.spent_status.size()) + ", expected " + std::to_string(signed_key_images.size())); - + if(check_spent) + { + m_daemon_rpc_mutex.lock(); + bool r = epee::net_utils::invoke_http_json("/is_key_image_spent", req, daemon_resp, m_http_client, rpc_timeout); + m_daemon_rpc_mutex.unlock(); + THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "is_key_image_spent"); + THROW_WALLET_EXCEPTION_IF(daemon_resp.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "is_key_image_spent"); + THROW_WALLET_EXCEPTION_IF(daemon_resp.status != CORE_RPC_STATUS_OK, error::is_key_image_spent_error, daemon_resp.status); + THROW_WALLET_EXCEPTION_IF(daemon_resp.spent_status.size() != signed_key_images.size(), error::wallet_internal_error, + "daemon returned wrong response for is_key_image_spent, wrong amounts count = " + + std::to_string(daemon_resp.spent_status.size()) + ", expected " + std::to_string(signed_key_images.size())); + for (size_t n = 0; n < daemon_resp.spent_status.size(); ++n) + { + transfer_details &td = m_transfers[n]; + td.m_spent = daemon_resp.spent_status[n] != COMMAND_RPC_IS_KEY_IMAGE_SPENT::UNSPENT; + } + } spent = 0; unspent = 0; - for (size_t n = 0; n < daemon_resp.spent_status.size(); ++n) + for(size_t i = 0; i < m_transfers.size(); ++i) { - transfer_details &td = m_transfers[n]; + transfer_details &td = m_transfers[i]; uint64_t amount = td.amount(); - td.m_spent = daemon_resp.spent_status[n] != COMMAND_RPC_IS_KEY_IMAGE_SPENT::UNSPENT; if (td.m_spent) spent += amount; else unspent += amount; - LOG_PRINT_L2("Transfer " << n << ": " << print_money(amount) << " (" << td.m_global_output_index << "): " - << (td.m_spent ? "spent" : "unspent") << " (key image " << req.key_images[n] << ")"); + LOG_PRINT_L2("Transfer " << i << ": " << print_money(amount) << " (" << td.m_global_output_index << "): " + << (td.m_spent ? "spent" : "unspent") << " (key image " << req.key_images[i] << ")"); } - LOG_PRINT_L1("Total: " << print_money(spent) << " spent, " << print_money(unspent) << " unspent"); - + MDEBUG("Total: " << print_money(spent) << " spent, " << print_money(unspent) << " unspent"); return m_transfers[signed_key_images.size() - 1].m_block_height; } +wallet2::payment_container wallet2::export_payments() const +{ + payment_container payments; + for (auto const &p : m_payments) + { + payments.emplace(p); + } + return payments; +} +void wallet2::import_payments(const payment_container &payments) +{ + m_payments.clear(); + for (auto const &p : payments) + { + m_payments.emplace(p); + } +} +void wallet2::import_payments_out(const std::list<std::pair<crypto::hash,wallet2::confirmed_transfer_details>> &confirmed_payments) +{ + m_confirmed_txs.clear(); + for (auto const &p : confirmed_payments) + { + m_confirmed_txs.emplace(p); + } +} + +std::vector<crypto::hash> wallet2::export_blockchain() const +{ + std::vector<crypto::hash> bc; + for (auto const &b : m_blockchain) + { + bc.push_back(b); + } + return bc; +} + +void wallet2::import_blockchain(const std::vector<crypto::hash> &bc) +{ + m_blockchain.clear(); + for (auto const &b : bc) + { + m_blockchain.push_back(b); + } + cryptonote::block genesis; + generate_genesis(genesis); + crypto::hash genesis_hash = get_block_hash(genesis); + check_genesis(genesis_hash); + m_local_bc_height = m_blockchain.size(); +} //---------------------------------------------------------------------------------------------------- std::vector<tools::wallet2::transfer_details> wallet2::export_outputs() const { @@ -5741,6 +5801,58 @@ bool wallet2::is_synced() const return get_blockchain_current_height() >= height; } //---------------------------------------------------------------------------------------------------- +uint64_t wallet2::estimate_backlog(uint64_t blob_size, uint64_t fee) +{ + THROW_WALLET_EXCEPTION_IF(blob_size == 0, error::wallet_internal_error, "Invalid 0 fee"); + THROW_WALLET_EXCEPTION_IF(fee == 0, error::wallet_internal_error, "Invalid 0 fee"); + + // get txpool backlog + epee::json_rpc::request<cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::request> req = AUTO_VAL_INIT(req); + epee::json_rpc::response<cryptonote::COMMAND_RPC_GET_TRANSACTION_POOL_BACKLOG::response, std::string> res = AUTO_VAL_INIT(res); + m_daemon_rpc_mutex.lock(); + req.jsonrpc = "2.0"; + req.id = epee::serialization::storage_entry(0); + req.method = "get_txpool_backlog"; + bool r = net_utils::invoke_http_json("/json_rpc", req, res, m_http_client, rpc_timeout); + m_daemon_rpc_mutex.unlock(); + THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "Failed to connect to daemon"); + THROW_WALLET_EXCEPTION_IF(res.result.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_txpool_backlog"); + THROW_WALLET_EXCEPTION_IF(res.result.status != CORE_RPC_STATUS_OK, error::get_tx_pool_error); + + epee::json_rpc::request<cryptonote::COMMAND_RPC_GET_INFO::request> req_t = AUTO_VAL_INIT(req_t); + epee::json_rpc::response<cryptonote::COMMAND_RPC_GET_INFO::response, std::string> resp_t = AUTO_VAL_INIT(resp_t); + m_daemon_rpc_mutex.lock(); + req_t.jsonrpc = "2.0"; + req_t.id = epee::serialization::storage_entry(0); + req_t.method = "get_info"; + r = net_utils::invoke_http_json("/json_rpc", req_t, resp_t, m_http_client); + m_daemon_rpc_mutex.unlock(); + THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "get_info"); + THROW_WALLET_EXCEPTION_IF(resp_t.result.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "get_info"); + THROW_WALLET_EXCEPTION_IF(resp_t.result.status != CORE_RPC_STATUS_OK, error::get_tx_pool_error); + + double our_fee_byte = fee / (double)blob_size; + uint64_t priority_size = 0; + for (const auto &i: res.result.backlog) + { + if (i.blob_size == 0) + { + MWARNING("Got 0 sized blob from txpool, ignored"); + continue; + } + double this_fee_byte = i.fee / (double)i.blob_size; + if (this_fee_byte < our_fee_byte) + continue; + priority_size += i.blob_size; + } + + uint64_t full_reward_zone = resp_t.result.block_size_limit / 2; + uint64_t nblocks = (priority_size + full_reward_zone - 1) / full_reward_zone; + MDEBUG("estimate_backlog: priority_size " << priority_size << " for " << our_fee_byte << " (" << our_fee_byte << " piconero fee/byte), " + << nblocks << " blocks at block size " << full_reward_zone); + return nblocks; +} +//---------------------------------------------------------------------------------------------------- void wallet2::generate_genesis(cryptonote::block& b) { if (m_testnet) { diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index ba9abacf5..3d5033535 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -105,7 +105,7 @@ namespace tools }; private: - wallet2(const wallet2&) : m_run(true), m_callback(0), m_testnet(false), m_always_confirm_transfers(true), m_print_ring_members(false), m_store_tx_info(true), m_default_mixin(0), m_default_priority(0), m_refresh_type(RefreshOptimizeCoinbase), m_auto_refresh(true), m_refresh_from_block_height(0), m_confirm_missing_payment_id(true), m_ask_password(true), m_min_output_count(0), m_min_output_value(0), m_merge_destinations(false), m_is_initialized(false),m_node_rpc_proxy(m_http_client, m_daemon_rpc_mutex) {} + wallet2(const wallet2&) : m_run(true), m_callback(0), m_testnet(false), m_always_confirm_transfers(true), m_print_ring_members(false), m_store_tx_info(true), m_default_mixin(0), m_default_priority(0), m_refresh_type(RefreshOptimizeCoinbase), m_auto_refresh(true), m_refresh_from_block_height(0), m_confirm_missing_payment_id(true), m_ask_password(true), m_min_output_count(0), m_min_output_value(0), m_merge_destinations(false), m_confirm_backlog(true), m_is_initialized(false),m_node_rpc_proxy(m_http_client, m_daemon_rpc_mutex) {} public: static const char* tr(const char* str); @@ -131,7 +131,7 @@ namespace tools static bool verify_password(const std::string& keys_file_name, const std::string& password, bool watch_only); - wallet2(bool testnet = false, bool restricted = false) : m_run(true), m_callback(0), m_testnet(testnet), m_always_confirm_transfers(true), m_print_ring_members(false), m_store_tx_info(true), m_default_mixin(0), m_default_priority(0), m_refresh_type(RefreshOptimizeCoinbase), m_auto_refresh(true), m_refresh_from_block_height(0), m_confirm_missing_payment_id(true), m_ask_password(true), m_min_output_count(0), m_min_output_value(0), m_merge_destinations(false), m_is_initialized(false), m_restricted(restricted), is_old_file_format(false), m_node_rpc_proxy(m_http_client, m_daemon_rpc_mutex) {} + wallet2(bool testnet = false, bool restricted = false) : m_run(true), m_callback(0), m_testnet(testnet), m_always_confirm_transfers(true), m_print_ring_members(false), m_store_tx_info(true), m_default_mixin(0), m_default_priority(0), m_refresh_type(RefreshOptimizeCoinbase), m_auto_refresh(true), m_refresh_from_block_height(0), m_confirm_missing_payment_id(true), m_ask_password(true), m_min_output_count(0), m_min_output_value(0), m_merge_destinations(false), m_confirm_backlog(true), m_is_initialized(false), m_restricted(restricted), is_old_file_format(false), m_node_rpc_proxy(m_http_client, m_daemon_rpc_mutex) {} struct transfer_details { @@ -539,6 +539,8 @@ namespace tools uint64_t get_min_output_value() const { return m_min_output_value; } void merge_destinations(bool merge) { m_merge_destinations = merge; } bool merge_destinations() const { return m_merge_destinations; } + bool confirm_backlog() const { return m_confirm_backlog; } + void confirm_backlog(bool always) { m_confirm_backlog = always; } bool get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key) const; @@ -580,12 +582,17 @@ namespace tools std::string sign(const std::string &data) const; bool verify(const std::string &data, const cryptonote::account_public_address &address, const std::string &signature) const; + // Import/Export wallet data std::vector<tools::wallet2::transfer_details> export_outputs() const; size_t import_outputs(const std::vector<tools::wallet2::transfer_details> &outputs); - + payment_container export_payments() const; + void import_payments(const payment_container &payments); + void import_payments_out(const std::list<std::pair<crypto::hash,wallet2::confirmed_transfer_details>> &confirmed_payments); + std::vector<crypto::hash> export_blockchain() const; + void import_blockchain(const std::vector<crypto::hash> &bc); bool export_key_images(const std::string filename); std::vector<std::pair<crypto::key_image, crypto::signature>> export_key_images() const; - uint64_t import_key_images(const std::vector<std::pair<crypto::key_image, crypto::signature>> &signed_key_images, uint64_t &spent, uint64_t &unspent); + uint64_t import_key_images(const std::vector<std::pair<crypto::key_image, crypto::signature>> &signed_key_images, uint64_t &spent, uint64_t &unspent, bool check_spent = true); uint64_t import_key_images(const std::string &filename, uint64_t &spent, uint64_t &unspent); void update_pool_state(bool refreshed = false); @@ -602,6 +609,8 @@ namespace tools bool is_synced() const; + uint64_t estimate_backlog(uint64_t blob_size, uint64_t fee); + private: /*! * \brief Stores wallet information to wallet file. @@ -700,6 +709,7 @@ namespace tools uint32_t m_min_output_count; uint64_t m_min_output_value; bool m_merge_destinations; + bool m_confirm_backlog; bool m_is_initialized; NodeRPCProxy m_node_rpc_proxy; std::unordered_set<crypto::hash> m_scanned_pool_txs[2]; diff --git a/tests/core_proxy/CMakeLists.txt b/tests/core_proxy/CMakeLists.txt index d22fecc9c..680e34911 100644 --- a/tests/core_proxy/CMakeLists.txt +++ b/tests/core_proxy/CMakeLists.txt @@ -39,6 +39,8 @@ target_link_libraries(core_proxy PRIVATE cryptonote_core cryptonote_protocol + common + blockchain_db p2p epee ${CMAKE_THREAD_LIBS_INIT} diff --git a/tests/core_proxy/core_proxy.h b/tests/core_proxy/core_proxy.h index 516d1f515..c4fb462e3 100644 --- a/tests/core_proxy/core_proxy.h +++ b/tests/core_proxy/core_proxy.h @@ -66,6 +66,7 @@ namespace tests public: void on_synchronized(){} + void safesyncmode(const bool){} uint64_t get_current_blockchain_height(){return 1;} void set_target_blockchain_height(uint64_t) {} bool init(const boost::program_options::variables_map& vm); diff --git a/tests/fuzz/CMakeLists.txt b/tests/fuzz/CMakeLists.txt index 853d46a12..cb77f8f56 100644 --- a/tests/fuzz/CMakeLists.txt +++ b/tests/fuzz/CMakeLists.txt @@ -30,6 +30,8 @@ add_executable(block_fuzz_tests block.cpp fuzzer.cpp) target_link_libraries(block_fuzz_tests PRIVATE cryptonote_core + common + blockchain_db p2p epee ${CMAKE_THREAD_LIBS_INIT} @@ -42,6 +44,8 @@ add_executable(transaction_fuzz_tests transaction.cpp fuzzer.cpp) target_link_libraries(transaction_fuzz_tests PRIVATE cryptonote_core + common + blockchain_db p2p epee ${CMAKE_THREAD_LIBS_INIT} @@ -55,6 +59,8 @@ target_link_libraries(signature_fuzz_tests PRIVATE wallet cryptonote_core + common + blockchain_db p2p epee ${CMAKE_THREAD_LIBS_INIT} @@ -68,6 +74,8 @@ target_link_libraries(cold-outputs_fuzz_tests PRIVATE wallet cryptonote_core + common + blockchain_db p2p epee ${CMAKE_THREAD_LIBS_INIT} @@ -81,6 +89,8 @@ target_link_libraries(cold-transaction_fuzz_tests PRIVATE wallet cryptonote_core + common + blockchain_db p2p epee ${CMAKE_THREAD_LIBS_INIT} diff --git a/tests/unit_tests/ban.cpp b/tests/unit_tests/ban.cpp index 627e85348..bcffc85c9 100644 --- a/tests/unit_tests/ban.cpp +++ b/tests/unit_tests/ban.cpp @@ -43,6 +43,7 @@ class test_core { public: void on_synchronized(){} + void safesyncmode(const bool){} uint64_t get_current_blockchain_height() const {return 1;} void set_target_blockchain_height(uint64_t) {} bool init(const boost::program_options::variables_map& vm) {return true ;} diff --git a/tests/unit_tests/hardfork.cpp b/tests/unit_tests/hardfork.cpp index febf8b2f7..c30feb461 100644 --- a/tests/unit_tests/hardfork.cpp +++ b/tests/unit_tests/hardfork.cpp @@ -31,7 +31,7 @@ #include <algorithm> #include "gtest/gtest.h" -#include "blockchain_db/lmdb/db_lmdb.h" +#include "blockchain_db/blockchain_db.h" #include "cryptonote_basic/cryptonote_format_utils.h" #include "cryptonote_basic/hardfork.h" @@ -47,6 +47,7 @@ public: virtual void open(const std::string& filename, const int db_flags = 0) { } virtual void close() {} virtual void sync() {} + virtual void safesyncmode(const bool onoff) {} virtual void reset() {} virtual std::vector<std::string> get_filenames() const { return std::vector<std::string>(); } virtual std::string get_db_name() const { return std::string(); } |