diff options
68 files changed, 1075 insertions, 528 deletions
diff --git a/contrib/epee/include/console_handler.h b/contrib/epee/include/console_handler.h index 5c63556ae..95b986ff5 100644 --- a/contrib/epee/include/console_handler.h +++ b/contrib/epee/include/console_handler.h @@ -36,6 +36,7 @@ #ifdef __OpenBSD__ #include <stdio.h> #endif +#include <boost/thread.hpp> namespace epee { @@ -47,7 +48,7 @@ namespace epee , m_has_read_request(false) , m_read_status(state_init) { - m_reader_thread = std::thread(std::bind(&async_stdin_reader::reader_thread_func, this)); + m_reader_thread = boost::thread(std::bind(&async_stdin_reader::reader_thread_func, this)); } ~async_stdin_reader() @@ -64,7 +65,7 @@ namespace epee if (state_eos == m_read_status) return false; - std::unique_lock<std::mutex> lock(m_response_mutex); + boost::unique_lock<boost::mutex> lock(m_response_mutex); while (state_init == m_read_status) { m_response_cv.wait(lock); @@ -103,7 +104,7 @@ namespace epee private: bool start_read() { - std::unique_lock<std::mutex> lock(m_request_mutex); + boost::unique_lock<boost::mutex> lock(m_request_mutex); if (!m_run.load(std::memory_order_relaxed) || m_has_read_request) return false; @@ -114,7 +115,7 @@ namespace epee bool wait_read() { - std::unique_lock<std::mutex> lock(m_request_mutex); + boost::unique_lock<boost::mutex> lock(m_request_mutex); while (m_run.load(std::memory_order_relaxed) && !m_has_read_request) { m_request_cv.wait(lock); @@ -187,7 +188,7 @@ namespace epee } else { - std::unique_lock<std::mutex> lock(m_response_mutex); + boost::unique_lock<boost::mutex> lock(m_response_mutex); if (m_run.load(std::memory_order_relaxed)) { m_line = std::move(line); @@ -212,17 +213,17 @@ namespace epee }; private: - std::thread m_reader_thread; + boost::thread m_reader_thread; std::atomic<bool> m_run; std::string m_line; bool m_has_read_request; t_state m_read_status; - std::mutex m_request_mutex; - std::mutex m_response_mutex; - std::condition_variable m_request_cv; - std::condition_variable m_response_cv; + boost::mutex m_request_mutex; + boost::mutex m_response_mutex; + boost::condition_variable m_request_cv; + boost::condition_variable m_response_cv; }; diff --git a/contrib/epee/include/misc_os_dependent.h b/contrib/epee/include/misc_os_dependent.h index 4d9c991e4..2abca0446 100644 --- a/contrib/epee/include/misc_os_dependent.h +++ b/contrib/epee/include/misc_os_dependent.h @@ -52,6 +52,12 @@ namespace misc_utils { #if defined(_MSC_VER) return ::GetTickCount64(); +#elif defined(WIN32) +# if defined(WIN64) + return GetTickCount64(); +# else + return GetTickCount(); +# endif #elif defined(__MACH__) clock_serv_t cclock; mach_timespec_t mts; @@ -98,7 +104,7 @@ namespace misc_utils inline std::string get_thread_string_id() { -#if defined(_MSC_VER) +#if defined(_WIN32) return boost::lexical_cast<std::string>(GetCurrentThreadId()); #elif defined(__GNUC__) return boost::lexical_cast<std::string>(pthread_self()); diff --git a/contrib/epee/include/net/abstract_tcp_server2.h b/contrib/epee/include/net/abstract_tcp_server2.h index f3dd7291c..80f3f6db0 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.h +++ b/contrib/epee/include/net/abstract_tcp_server2.h @@ -153,8 +153,8 @@ namespace net_utils // for calculate speed (last 60 sec) network_throttle m_throttle_speed_in; network_throttle m_throttle_speed_out; - std::mutex m_throttle_speed_in_mutex; - std::mutex m_throttle_speed_out_mutex; + boost::mutex m_throttle_speed_in_mutex; + boost::mutex m_throttle_speed_out_mutex; public: void setRpcStation(); @@ -307,7 +307,7 @@ namespace net_utils /// The next connection to be accepted connection_ptr new_connection_; - std::mutex connections_mutex; + boost::mutex connections_mutex; std::deque<std::pair<boost::system_time, connection_ptr>> connections_; }; // class <>boosted_tcp_server diff --git a/contrib/epee/include/net/abstract_tcp_server2.inl b/contrib/epee/include/net/abstract_tcp_server2.inl index 4fb6fa75d..698e1947a 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.inl +++ b/contrib/epee/include/net/abstract_tcp_server2.inl @@ -41,7 +41,7 @@ #include <boost/utility/value_init.hpp> #include <boost/asio/deadline_timer.hpp> #include <boost/date_time/posix_time/posix_time.hpp> // TODO -#include <boost/thread/thread.hpp> // TODO +#include <boost/thread/v2/thread.hpp> // TODO #include "misc_language.h" #include "pragma_comp_defs.h" @@ -294,7 +294,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) if (delay > 0) { long int ms = (long int)(delay * 100); epee::net_utils::data_logger::get_instance().add_data("sleep_down", ms); - std::this_thread::sleep_for(std::chrono::milliseconds(ms)); + boost::this_thread::sleep_for(boost::chrono::milliseconds(ms)); } } while(delay > 0); } // any form of sleeping diff --git a/contrib/epee/include/net/levin_protocol_handler_async.h b/contrib/epee/include/net/levin_protocol_handler_async.h index a7fbffb4b..7331faa35 100644 --- a/contrib/epee/include/net/levin_protocol_handler_async.h +++ b/contrib/epee/include/net/levin_protocol_handler_async.h @@ -26,6 +26,7 @@ #pragma once #include <boost/uuid/uuid_generators.hpp> +#include <boost/unordered_map.hpp> #include <boost/interprocess/detail/atomic.hpp> #include <boost/smart_ptr/make_shared.hpp> @@ -52,7 +53,7 @@ class async_protocol_handler; template<class t_connection_context> class async_protocol_handler_config { - typedef std::map<boost::uuids::uuid, async_protocol_handler<t_connection_context>* > connections_map; + typedef boost::unordered_map<boost::uuids::uuid, async_protocol_handler<t_connection_context>* > connections_map; critical_section m_connects_lock; connections_map m_connects; diff --git a/contrib/epee/include/syncobj.h b/contrib/epee/include/syncobj.h index b81eb43a9..8912fc018 100644 --- a/contrib/epee/include/syncobj.h +++ b/contrib/epee/include/syncobj.h @@ -30,18 +30,23 @@ #ifndef __WINH_OBJ_H__ #define __WINH_OBJ_H__ -#include <condition_variable> -#include <mutex> +#include <boost/chrono/duration.hpp> #include <boost/thread/locks.hpp> #include <boost/thread/mutex.hpp> #include <boost/thread/recursive_mutex.hpp> -#include <thread> -#include <chrono> +#include <boost/thread/v2/thread.hpp> namespace epee { - extern unsigned int g_test_dbg_lock_sleep; + namespace debug + { + inline unsigned int &g_test_dbg_lock_sleep() + { + static unsigned int value = 0; + return value; + } + } struct simple_event { @@ -51,22 +56,22 @@ namespace epee void raise() { - std::unique_lock<std::mutex> lock(m_mx); + boost::unique_lock<boost::mutex> lock(m_mx); m_rised = true; m_cond_var.notify_one(); } void wait() { - std::unique_lock<std::mutex> lock(m_mx); + boost::unique_lock<boost::mutex> lock(m_mx); while (!m_rised) m_cond_var.wait(lock); m_rised = false; } private: - std::mutex m_mx; - std::condition_variable m_cond_var; + boost::mutex m_mx; + boost::condition_variable m_cond_var; bool m_rised; }; @@ -219,10 +224,10 @@ namespace epee #define SHARED_CRITICAL_REGION_BEGIN(x) { shared_guard critical_region_var(x) #define EXCLUSIVE_CRITICAL_REGION_BEGIN(x) { exclusive_guard critical_region_var(x) -#define CRITICAL_REGION_LOCAL(x) {std::this_thread::sleep_for(std::chrono::milliseconds(epee::g_test_dbg_lock_sleep));} epee::critical_region_t<decltype(x)> critical_region_var(x) -#define CRITICAL_REGION_BEGIN(x) { std::this_thread::sleep_for(std::chrono::milliseconds(epee::g_test_dbg_lock_sleep)); epee::critical_region_t<decltype(x)> critical_region_var(x) -#define CRITICAL_REGION_LOCAL1(x) {std::this_thread::sleep_for(std::chrono::milliseconds(epee::g_test_dbg_lock_sleep));} epee::critical_region_t<decltype(x)> critical_region_var1(x) -#define CRITICAL_REGION_BEGIN1(x) { std::this_thread::sleep_for(std::chrono::milliseconds(epee::g_test_dbg_lock_sleep)); epee::critical_region_t<decltype(x)> critical_region_var1(x) +#define CRITICAL_REGION_LOCAL(x) {boost::this_thread::sleep_for(boost::chrono::milliseconds(epee::debug::g_test_dbg_lock_sleep()));} epee::critical_region_t<decltype(x)> critical_region_var(x) +#define CRITICAL_REGION_BEGIN(x) { boost::this_thread::sleep_for(boost::chrono::milliseconds(epee::debug::g_test_dbg_lock_sleep())); epee::critical_region_t<decltype(x)> critical_region_var(x) +#define CRITICAL_REGION_LOCAL1(x) {boost::this_thread::sleep_for(boost::chrono::milliseconds(epee::debug::g_test_dbg_lock_sleep()));} epee::critical_region_t<decltype(x)> critical_region_var1(x) +#define CRITICAL_REGION_BEGIN1(x) { boost::this_thread::sleep_for(boost::chrono::milliseconds(epee::debug::g_test_dbg_lock_sleep())); epee::critical_region_t<decltype(x)> critical_region_var1(x) #define CRITICAL_REGION_END() } diff --git a/contrib/otshell_utils/lib_common1.hpp b/contrib/otshell_utils/lib_common1.hpp index 456e63fbb..2cb466beb 100644 --- a/contrib/otshell_utils/lib_common1.hpp +++ b/contrib/otshell_utils/lib_common1.hpp @@ -19,9 +19,11 @@ #include <functional> #include <memory> -#include <thread> #include <atomic> -#include <mutex> +#include <boost/thread.hpp> +#include <boost/thread/mutex.hpp> +#include <boost/thread/recursive_mutex.hpp> +#include <boost/thread/lock_guard.hpp> // list of thigs from libraries that we pull into namespace nOT::nNewcli @@ -45,8 +47,7 @@ using std::shared_ptr; \ using std::weak_ptr; \ using std::enable_shared_from_this; \ - using std::mutex; \ - using std::lock_guard; \ + using boost::lock_guard; \ #endif diff --git a/contrib/otshell_utils/utils.cpp b/contrib/otshell_utils/utils.cpp index e1d7d9a14..ef9b37f03 100644 --- a/contrib/otshell_utils/utils.cpp +++ b/contrib/otshell_utils/utils.cpp @@ -12,6 +12,7 @@ #include <fstream> #include <iostream> #include <iomanip> +#include <chrono> #include "utils.hpp" @@ -32,6 +33,7 @@ #if defined(OS_TYPE_WINDOWS) #include <windows.h> + #include <process.h> #elif defined(OS_TYPE_POSIX) #include <sys/types.h> #include <sys/stat.h> @@ -128,7 +130,7 @@ std::string get_current_time() { cNullstream g_nullstream; // extern a stream that does nothing (eats/discards data) -std::recursive_mutex gLoggerGuard; // extern +boost::recursive_mutex gLoggerGuard; // extern std::atomic<int> gLoggerGuardDepth; // extern std::atomic<int> & gLoggerGuardDepth_Get() { @@ -302,7 +304,7 @@ mPid2Number_Biggest(0) mIsBroken=false; // ok, constr. succeeded, so string is not broken now // this is here, because it could be using logging itself to log creation of first thread/PID etc - Thread2Number( std::this_thread::get_id() ); // convert current id to short number, useful to reserve a number so that main thread is usually called 1 + Thread2Number( boost::this_thread::get_id() ); // convert current id to short number, useful to reserve a number so that main thread is usually called 1 Pid2Number( getpid() ); // add this proces ID as first one } @@ -347,7 +349,7 @@ std::ostream & cLogger::write_stream(int level, const std::string & channel ) { output << windows_stream(level); #endif output << icon(level) << ' '; - std::thread::id this_id = std::this_thread::get_id(); + boost::thread::id this_id = boost::this_thread::get_id(); output << "{" << Thread2Number(this_id) << "}"; auto nicePid = Pid2Number(getpid()); if (nicePid>0) output << " {p" << nicePid << "}"; @@ -516,7 +518,7 @@ std::string cLogger::endline() const { #endif } -int cLogger::Thread2Number(const std::thread::id id) { +int cLogger::Thread2Number(const boost::thread::id id) { auto found = mThread2Number.find( id ); if (found == mThread2Number.end()) { // new one mThread2Number_Biggest++; diff --git a/contrib/otshell_utils/utils.hpp b/contrib/otshell_utils/utils.hpp index 297f13ac0..98508b565 100644 --- a/contrib/otshell_utils/utils.hpp +++ b/contrib/otshell_utils/utils.hpp @@ -29,7 +29,7 @@ // #define opt_debug_debug #ifdef opt_debug_debug - #define _dbg_dbg(X) do { std::cerr<<"_dbg_dbg: " << OT_CODE_STAMP << " {thread=" << std::this_thread::get_id()<<"} " \ + #define _dbg_dbg(X) do { std::cerr<<"_dbg_dbg: " << OT_CODE_STAMP << " {thread=" << boost::this_thread::get_id()<<"} " \ << " {pid="<<getpid()<<"} " << ": " << X << std::endl; } while(0) #else #define _dbg_dbg(X) do { } while(0) @@ -79,7 +79,7 @@ extern cNullstream g_nullstream; // a stream that does nothing (eats/discards da // TODO make _dbg_ignore thread-safe everywhere -extern std::recursive_mutex gLoggerGuard; // the mutex guarding logging/debugging code e.g. protecting streams, files, etc +extern boost::recursive_mutex gLoggerGuard; // the mutex guarding logging/debugging code e.g. protecting streams, files, etc std::atomic<int> & gLoggerGuardDepth_Get(); // getter for the global singleton of counter (it guarantees initializing it to 0). This counter shows the current recursion (re-entrant) level of debug macros. @@ -91,7 +91,7 @@ std::atomic<int> & gLoggerGuardDepth_Get(); // getter for the global singleton o _dbg_dbg("WRITE DEBUG: LEVEL="<<LEVEL<<" VAR: " << VAR ); \ auto level=LEVEL; short int part=0; \ try { \ - std::lock_guard<std::recursive_mutex> mutex_guard( nOT::nUtils::gLoggerGuard ); \ + boost::lock_guard<boost::recursive_mutex> mutex_guard( nOT::nUtils::gLoggerGuard ); \ part=1; \ try { \ ++nOT::nUtils::gLoggerGuardDepth_Get(); \ @@ -111,7 +111,7 @@ std::atomic<int> & gLoggerGuardDepth_Get(); // getter for the global singleton o _dbg_dbg("WRITE DEBUG: LEVEL="<<LEVEL<<" CHANNEL="<<CHANNEL<<" VAR: " << VAR ); \ auto level=LEVEL; short int part=0; \ try { \ - std::lock_guard<std::recursive_mutex> mutex_guard( nOT::nUtils::gLoggerGuard ); \ + boost::lock_guard<boost::recursive_mutex> mutex_guard( nOT::nUtils::gLoggerGuard ); \ part=1; \ try { \ ++nOT::nUtils::gLoggerGuardDepth_Get(); \ @@ -253,9 +253,9 @@ class cLogger { void OpenNewChannel_(const std::string & channel); ///< internal function, will throw in case of problems std::string GetLogBaseDir() const; - std::map< std::thread::id , int > mThread2Number; ///< change long thread IDs into a short nice number to show + std::map< boost::thread::id , int > mThread2Number; ///< change long thread IDs into a short nice number to show int mThread2Number_Biggest; ///< current biggest value held there (biggest key) - works as growing-only counter basically - int Thread2Number(const std::thread::id id); ///< convert the system's thread id into a nice short our id; make one if new thread + int Thread2Number(const boost::thread::id id); ///< convert the system's thread id into a nice short our id; make one if new thread std::map< t_anypid , int > mPid2Number; ///< change long proces PID into a short nice number to show int mPid2Number_Biggest; ///< current biggest value held there (biggest key) - works as growing-only counter basically diff --git a/external/db_drivers/liblmdb/midl.h b/external/db_drivers/liblmdb/midl.h index ed1d75e36..e16aa0c3c 100644 --- a/external/db_drivers/liblmdb/midl.h +++ b/external/db_drivers/liblmdb/midl.h @@ -61,7 +61,7 @@ typedef MDB_ID *MDB_IDL; * limiting factors: sizeof(ID), thread stack size */ #ifdef MDB_VL32 -#define MDB_IDL_LOGN 10 /* DB_SIZE is 2^10, UM_SIZE is 2^11 */ +#define MDB_IDL_LOGN 14 /* DB_SIZE is 2^14, UM_SIZE is 2^15 */ #else #define MDB_IDL_LOGN 16 /* DB_SIZE is 2^16, UM_SIZE is 2^17 */ #endif diff --git a/src/blockchain_db/CMakeLists.txt b/src/blockchain_db/CMakeLists.txt index 4953d92a6..39459d5c4 100644 --- a/src/blockchain_db/CMakeLists.txt +++ b/src/blockchain_db/CMakeLists.txt @@ -65,6 +65,7 @@ target_link_libraries(blockchain_db crypto cryptonote_core ${Boost_DATE_TIME_LIBRARY} + ${Boost_CHRONO_LIBRARY} ${Boost_PROGRAM_OPTIONS_LIBRARY} ${Boost_SERIALIZATION_LIBRARY} ${LMDB_LIBRARY} diff --git a/src/blockchain_db/berkeleydb/db_bdb.cpp b/src/blockchain_db/berkeleydb/db_bdb.cpp index f035bf4c4..1fef9e619 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.cpp +++ b/src/blockchain_db/berkeleydb/db_bdb.cpp @@ -135,7 +135,7 @@ const unsigned int DB_BUFFER_LENGTH = 32 * MB; const unsigned int DB_DEF_CACHESIZE = 256 * MB; #if defined(BDB_BULK_CAN_THREAD) -const unsigned int DB_BUFFER_COUNT = std::thread::hardware_concurrency(); +const unsigned int DB_BUFFER_COUNT = boost::thread::hardware_concurrency(); #else const unsigned int DB_BUFFER_COUNT = 1; #endif @@ -1073,8 +1073,10 @@ void BlockchainBDB::sync() m_spent_keys->sync(0); - m_hf_starting_heights->sync(0); - m_hf_versions->sync(0); + if (m_hf_starting_heights != nullptr) + m_hf_starting_heights->sync(0); + if (m_hf_versions != nullptr) + m_hf_versions->sync(0); m_properties->sync(0); } @@ -1835,7 +1837,7 @@ void BlockchainBDB::set_batch_transactions(bool batch_transactions) LOG_PRINT_L3("batch transactions " << (m_batch_transactions ? "enabled" : "disabled")); } -void BlockchainBDB::block_txn_start() +void BlockchainBDB::block_txn_start(bool readonly) { // TODO } @@ -2208,12 +2210,91 @@ uint64_t BlockchainBDB::get_hard_fork_starting_height(uint8_t version) const void BlockchainBDB::check_hard_fork_info() { - /* FIXME: Some other time */ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + if (m_hf_versions == nullptr) + { + LOG_PRINT_L0("hf versions DB not open, so not checking"); + return; + } + + DB_BTREE_STAT* db_stat1, * db_stat2; + + // DB_FAST_STAT can apparently cause an incorrect number of records + // to be returned. The flag should be set to 0 instead if this proves + // to be the case. + + // Set txn to NULL and DB_FAST_STAT to zero (0) for reliability. + m_blocks->stat(NULL, &db_stat1, 0); + m_hf_versions->stat(NULL, &db_stat2, 0); + if (db_stat1->bt_nkeys != db_stat2->bt_nkeys) + { + LOG_PRINT_L0("num blocks " << db_stat1->bt_nkeys << " != " << "num hf_versions " << db_stat2->bt_nkeys << " - will clear the two hard fork DBs"); + + bdb_txn_safe txn; + bdb_txn_safe* txn_ptr = &txn; + if (m_write_txn) + txn_ptr = m_write_txn; + else + { + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } + + try + { + uint32_t count; + m_hf_starting_heights->truncate(*txn_ptr, &count, 0); + LOG_PRINT_L0("hf_starting_heights count: " << count); + m_hf_versions->truncate(*txn_ptr, &count, 0); + LOG_PRINT_L0("hf_versions count: " << count); + + if (!m_write_txn) + txn.commit(); + } + catch (const std::exception& e) + { + throw0(DB_ERROR(std::string("Failed to clear two hard fork DBs: ").append(e.what()).c_str())); + } + } + delete db_stat1; + delete db_stat2; } void BlockchainBDB::drop_hard_fork_info() { - /* TODO */ + LOG_PRINT_L3("BlockchainBDB::" << __func__); + check_open(); + + bdb_txn_safe txn; + bdb_txn_safe* txn_ptr = &txn; + if (m_write_txn) + txn_ptr = m_write_txn; + else + { + if (m_env->txn_begin(NULL, txn, 0)) + throw0(DB_ERROR("Failed to create a transaction for the db")); + } + + try + { + m_hf_starting_heights->close(0); + m_hf_versions->close(0); + m_hf_starting_heights = nullptr; + m_hf_versions = nullptr; + if (m_env->dbremove(*txn_ptr, BDB_HF_STARTING_HEIGHTS, NULL, 0) != 0) + LOG_ERROR("Error removing hf_starting_heights"); + if (m_env->dbremove(*txn_ptr, BDB_HF_VERSIONS, NULL, 0) != 0) + LOG_ERROR("Error removing hf_versions"); + + if (!m_write_txn) + txn.commit(); + } + catch (const std::exception& e) + { + throw0(DB_ERROR(std::string("Failed to drop hard fork info: ").append(e.what()).c_str())); + } } void BlockchainBDB::set_hard_fork_version(uint64_t height, uint8_t version) diff --git a/src/blockchain_db/berkeleydb/db_bdb.h b/src/blockchain_db/berkeleydb/db_bdb.h index 42119da93..d7cbd24e7 100644 --- a/src/blockchain_db/berkeleydb/db_bdb.h +++ b/src/blockchain_db/berkeleydb/db_bdb.h @@ -31,6 +31,7 @@ #include "cryptonote_protocol/blobdatatype.h" // for type blobdata #include <unordered_map> +#include <condition_variable> // ND: Enables multi-threaded bulk reads for when getting indices. // TODO: Disabled for now, as it doesn't seem to provide noticeable improvements (??. Reason: TBD. @@ -129,7 +130,7 @@ public: T acquire_buffer() { - std::unique_lock<std::mutex> lock(m_lock); + boost::unique_lock<boost::mutex> lock(m_lock); m_cv.wait(lock, [&]{ return m_count > 0; }); --m_count; @@ -153,7 +154,7 @@ public: void release_buffer(T buffer) { - std::unique_lock<std::mutex> lock(m_lock); + boost::unique_lock<boost::mutex> lock(m_lock); assert(buffer != nullptr); auto it = m_buffer_map.find(buffer); @@ -195,10 +196,10 @@ private: std::vector<T> m_buffers; std::unordered_map<T, size_t> m_buffer_map; - std::condition_variable m_cv; + boost::condition_variable m_cv; std::vector<bool> m_open_slot; size_t m_count; - std::mutex m_lock; + boost::mutex m_lock; size_t m_buffer_count; }; @@ -328,7 +329,7 @@ public: virtual void batch_stop(); virtual void batch_abort(); - virtual void block_txn_start(); + virtual void block_txn_start(bool readonly); virtual void block_txn_stop(); virtual void block_txn_abort(); diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp index 68bef3892..a66f4a403 100644 --- a/src/blockchain_db/blockchain_db.cpp +++ b/src/blockchain_db/blockchain_db.cpp @@ -99,7 +99,7 @@ uint64_t BlockchainDB::add_block( const block& blk , const std::vector<transaction>& txs ) { - block_txn_start(); + block_txn_start(false); TIME_MEASURE_START(time1); crypto::hash blk_hash = get_block_hash(blk); @@ -227,6 +227,9 @@ void BlockchainDB::fixup() static const char * const mainnet_genesis_hex = "418015bb9ae982a1975da7d79277c2705727a56894ba0fb246adaabb1f4632e3"; crypto::hash mainnet_genesis_hash; epee::string_tools::hex_to_pod(mainnet_genesis_hex, mainnet_genesis_hash ); + set_batch_transactions(true); + batch_start(); + if (get_block_hash_from_height(0) == mainnet_genesis_hash) { // block 202612 (511 key images in 511 transactions) @@ -762,9 +765,6 @@ void BlockchainDB::fixup() "633cdedeb3b96ec4f234c670254c6f721e0b368d00b48c6b26759db7d62cf52d", }; - set_batch_transactions(true); - batch_start(); - if (height() > 202612) { for (const auto &kis: key_images_202612) @@ -791,9 +791,8 @@ void BlockchainDB::fixup() } } } - - batch_stop(); } + batch_stop(); } } // namespace cryptonote diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 2aa4506e6..3396b8c20 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -381,7 +381,7 @@ public: virtual void batch_stop() = 0; virtual void set_batch_transactions(bool) = 0; - virtual void block_txn_start() = 0; + virtual void block_txn_start(bool readonly=false) = 0; virtual void block_txn_stop() = 0; virtual void block_txn_abort() = 0; diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 5119be3f5..69673215c 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -38,6 +38,10 @@ #include "crypto/crypto.h" #include "profile_tools.h" +#if defined(__i386) || defined(__x86_64) +#define MISALIGNED_OK 1 +#endif + using epee::string_tools::pod_to_hex; // Increase when the DB changes in a non backward compatible way, and there @@ -61,45 +65,6 @@ inline void throw1(const T &e) throw e; } -// cursor needs to be closed when it goes out of scope, -// this helps if the function using it throws -struct lmdb_cur -{ - lmdb_cur(MDB_txn* txn, MDB_dbi dbi) - { - if (mdb_cursor_open(txn, dbi, &m_cur)) - throw0(cryptonote::DB_ERROR("Error opening db cursor")); - done = false; - } - - ~lmdb_cur() - { - close(); - } - - operator MDB_cursor*() - { - return m_cur; - } - operator MDB_cursor**() - { - return &m_cur; - } - - void close() - { - if (!done) - { - mdb_cursor_close(m_cur); - done = true; - } - } - -private: - MDB_cursor* m_cur; - bool done; -}; - template<typename T> struct MDB_val_copy: public MDB_val { @@ -143,22 +108,26 @@ private: std::unique_ptr<char[]> data; }; -auto compare_uint64 = [](const MDB_val *a, const MDB_val *b) +int compare_uint64(const MDB_val *a, const MDB_val *b) { +#ifdef MISALIGNED_OK const uint64_t va = *(const uint64_t*)a->mv_data; const uint64_t vb = *(const uint64_t*)b->mv_data; +#else + uint64_t va, vb; + memcpy(&va, a->mv_data, sizeof(uint64_t)); + memcpy(&vb, b->mv_data, sizeof(uint64_t)); +#endif if (va < vb) return -1; else if (va == vb) return 0; else return 1; }; -auto compare_uint8 = [](const MDB_val *a, const MDB_val *b) +int compare_uint8(const MDB_val *a, const MDB_val *b) { const uint8_t va = *(const uint8_t*)a->mv_data; const uint8_t vb = *(const uint8_t*)b->mv_data; - if (va < vb) return -1; - else if (va == vb) return 0; - else return 1; + return va - vb; }; int compare_hash32(const MDB_val *a, const MDB_val *b) @@ -206,11 +175,6 @@ const char* const LMDB_HF_VERSIONS = "hf_versions"; const char* const LMDB_PROPERTIES = "properties"; -inline void lmdb_db_open(MDB_txn* txn, const char* name, int flags, MDB_dbi& dbi, const std::string& error_string) -{ - if (mdb_dbi_open(txn, name, flags, &dbi)) - throw0(cryptonote::DB_OPEN_FAILURE(error_string.c_str())); -} const std::string lmdb_error(const std::string& error_string, int mdb_res) { @@ -218,13 +182,32 @@ const std::string lmdb_error(const std::string& error_string, int mdb_res) return full_string; } +inline void lmdb_db_open(MDB_txn* txn, const char* name, int flags, MDB_dbi& dbi, const std::string& error_string) +{ + if (auto res = mdb_dbi_open(txn, name, flags, &dbi)) + throw0(cryptonote::DB_OPEN_FAILURE(lmdb_error(error_string + " : ", res).c_str())); +} + + } // anonymous namespace #define CURSOR(name) \ if (!m_cur_ ## name) { \ int result = mdb_cursor_open(*m_write_txn, m_ ## name, &m_cur_ ## name); \ if (result) \ - throw0(DB_ERROR(std::string("Failed to open cursor: ").append(mdb_strerror(result)).c_str())); \ + throw0(DB_ERROR(lmdb_error("Failed to open cursor: ", result).c_str())); \ + } + +#define RCURSOR(name) \ + if (!m_cur_ ## name) { \ + int result = mdb_cursor_open(m_txn, m_ ## name, (MDB_cursor **)&m_cur_ ## name); \ + if (result) \ + throw0(DB_ERROR(lmdb_error("Failed to open cursor: ", result).c_str())); \ + if (!m_write_txn) \ + m_tinfo->m_ti_rflags.m_rf_ ## name = true; \ + } else if (!m_write_txn && !m_tinfo->m_ti_rflags.m_rf_ ## name) { \ + mdb_cursor_renew(m_txn, m_cur_ ## name); \ + m_tinfo->m_ti_rflags.m_rf_ ## name = true; \ } namespace cryptonote @@ -232,6 +215,17 @@ namespace cryptonote std::atomic<uint64_t> mdb_txn_safe::num_active_txns{0}; std::atomic_flag mdb_txn_safe::creation_gate = ATOMIC_FLAG_INIT; +mdb_threadinfo::~mdb_threadinfo() +{ + MDB_cursor **cur = &m_ti_rcursors.m_txc_blocks; + unsigned i; + for (i=0; i<sizeof(mdb_txn_cursors)/sizeof(MDB_cursor *); i++) + if (cur[i]) + mdb_cursor_close(cur[i]); + if (m_ti_rtxn) + mdb_txn_abort(m_ti_rtxn); +} + mdb_txn_safe::mdb_txn_safe() : m_txn(NULL) { while (creation_gate.test_and_set()); @@ -273,7 +267,7 @@ void mdb_txn_safe::commit(std::string message) if (auto result = mdb_txn_commit(m_txn)) { m_txn = nullptr; - throw0(DB_ERROR((message + ": ").append(mdb_strerror(result)).c_str())); + throw0(DB_ERROR(lmdb_error(message + ": ", result).c_str())); } m_txn = nullptr; } @@ -529,6 +523,7 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); + mdb_txn_cursors *m_cursors = &m_wcursors; CURSOR(block_heights) MDB_val_copy<crypto::hash> val_h(blk_hash); @@ -564,35 +559,35 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const MDB_val_copy<blobdata> blob(block_to_blob(blk)); result = mdb_cursor_put(m_cur_blocks, &key, &blob, MDB_APPEND); if (result) - throw0(DB_ERROR(std::string("Failed to add block blob to db transaction: ").append(mdb_strerror(result)).c_str())); + throw0(DB_ERROR(lmdb_error("Failed to add block blob to db transaction: ", result).c_str())); MDB_val_copy<size_t> sz(block_size); result = mdb_cursor_put(m_cur_block_sizes, &key, &sz, MDB_APPEND); if (result) - throw0(DB_ERROR(std::string("Failed to add block size to db transaction: ").append(mdb_strerror(result)).c_str())); + throw0(DB_ERROR(lmdb_error("Failed to add block size to db transaction: ", result).c_str())); MDB_val_copy<uint64_t> ts(blk.timestamp); result = mdb_cursor_put(m_cur_block_timestamps, &key, &ts, MDB_APPEND); if (result) - throw0(DB_ERROR(std::string("Failed to add block timestamp to db transaction: ").append(mdb_strerror(result)).c_str())); + throw0(DB_ERROR(lmdb_error("Failed to add block timestamp to db transaction: ", result).c_str())); MDB_val_copy<difficulty_type> diff(cumulative_difficulty); result = mdb_cursor_put(m_cur_block_diffs, &key, &diff, MDB_APPEND); if (result) - throw0(DB_ERROR(std::string("Failed to add block cumulative difficulty to db transaction: ").append(mdb_strerror(result)).c_str())); + throw0(DB_ERROR(lmdb_error("Failed to add block cumulative difficulty to db transaction: ", result).c_str())); MDB_val_copy<uint64_t> coinsgen(coins_generated); result = mdb_cursor_put(m_cur_block_coins, &key, &coinsgen, MDB_APPEND); if (result) - throw0(DB_ERROR(std::string("Failed to add block total generated coins to db transaction: ").append(mdb_strerror(result)).c_str())); + throw0(DB_ERROR(lmdb_error("Failed to add block total generated coins to db transaction: ", result).c_str())); result = mdb_cursor_put(m_cur_block_heights, &val_h, &key, 0); if (result) - throw0(DB_ERROR(std::string("Failed to add block height by hash to db transaction: ").append(mdb_strerror(result)).c_str())); + throw0(DB_ERROR(lmdb_error("Failed to add block height by hash to db transaction: ", result).c_str())); result = mdb_cursor_put(m_cur_block_hashes, &key, &val_h, MDB_APPEND); if (result) - throw0(DB_ERROR(std::string("Failed to add block hash to db transaction: ").append(mdb_strerror(result)).c_str())); + throw0(DB_ERROR(lmdb_error("Failed to add block hash to db transaction: ", result).c_str())); m_cum_size += block_size; m_cum_count++; @@ -637,6 +632,7 @@ void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const tr { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); + mdb_txn_cursors *m_cursors = &m_wcursors; int result = 0; @@ -652,17 +648,17 @@ void BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, const tr MDB_val_copy<blobdata> blob(tx_to_blob(tx)); result = mdb_cursor_put(m_cur_txs, &val_h, &blob, 0); if (result) - throw0(DB_ERROR(std::string("Failed to add tx blob to db transaction: ").append(mdb_strerror(result)).c_str())); + throw0(DB_ERROR(lmdb_error("Failed to add tx blob to db transaction: ", result).c_str())); MDB_val_copy<uint64_t> height(m_height); result = mdb_cursor_put(m_cur_tx_heights, &val_h, &height, 0); if (result) - throw0(DB_ERROR(std::string("Failed to add tx block height to db transaction: ").append(mdb_strerror(result)).c_str())); + throw0(DB_ERROR(lmdb_error("Failed to add tx block height to db transaction: ", result).c_str())); MDB_val_copy<uint64_t> unlock_time(tx.unlock_time); result = mdb_cursor_put(m_cur_tx_unlocks, &val_h, &unlock_time, 0); if (result) - throw0(DB_ERROR(std::string("Failed to add tx unlock time to db transaction: ").append(mdb_strerror(result)).c_str())); + throw0(DB_ERROR(lmdb_error("Failed to add tx unlock time to db transaction: ", result).c_str())); } void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) @@ -688,13 +684,14 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const if (result == MDB_NOTFOUND) LOG_PRINT_L1("tx has no outputs to remove: " << tx_hash); else if (result) - throw1(DB_ERROR(std::string("Failed to add removal of tx outputs to db transaction: ").append(mdb_strerror(result)).c_str())); + throw1(DB_ERROR(lmdb_error("Failed to add removal of tx outputs to db transaction: ", result).c_str())); } void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); + mdb_txn_cursors *m_cursors = &m_wcursors; int result = 0; @@ -709,20 +706,20 @@ void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_ou result = mdb_cursor_put(m_cur_output_txs, &k, &v, MDB_APPEND); if (result) - throw0(DB_ERROR(std::string("Failed to add output tx hash to db transaction: ").append(mdb_strerror(result)).c_str())); + throw0(DB_ERROR(lmdb_error("Failed to add output tx hash to db transaction: ", result).c_str())); result = mdb_cursor_put(m_cur_tx_outputs, &v, &k, 0); if (result) - throw0(DB_ERROR(std::string("Failed to add <tx hash, global output index> to db transaction: ").append(mdb_strerror(result)).c_str())); + throw0(DB_ERROR(lmdb_error("Failed to add <tx hash, global output index> to db transaction: ", result).c_str())); MDB_val_copy<uint64_t> val_local_index(local_index); result = mdb_cursor_put(m_cur_output_indices, &k, &val_local_index, MDB_APPEND); if (result) - throw0(DB_ERROR(std::string("Failed to add tx output index to db transaction: ").append(mdb_strerror(result)).c_str())); + throw0(DB_ERROR(lmdb_error("Failed to add tx output index to db transaction: ", result).c_str())); MDB_val_copy<uint64_t> val_amount(tx_output.amount); result = mdb_cursor_put(m_cur_output_amounts, &val_amount, &k, 0); if (result) - throw0(DB_ERROR(std::string("Failed to add output amount to db transaction: ").append(mdb_strerror(result)).c_str())); + throw0(DB_ERROR(lmdb_error("Failed to add output amount to db transaction: ", result).c_str())); if (tx_output.target.type() == typeid(txout_to_key)) { @@ -748,12 +745,12 @@ void BlockchainLMDB::remove_tx_outputs(const crypto::hash& tx_hash, const transa { LOG_PRINT_L3("BlockchainLMDB::" << __func__); - lmdb_cur cur(*m_write_txn, m_tx_outputs); - + mdb_txn_cursors *m_cursors = &m_wcursors; MDB_val_copy<crypto::hash> k(tx_hash); MDB_val v; + CURSOR(tx_outputs) - auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + auto result = mdb_cursor_get(m_cur_tx_outputs, &k, &v, MDB_SET); if (result == MDB_NOTFOUND) { LOG_PRINT_L2("tx has no outputs, so no global output indices"); @@ -764,10 +761,10 @@ void BlockchainLMDB::remove_tx_outputs(const crypto::hash& tx_hash, const transa } else { - size_t num_elems = 0; - mdb_cursor_count(cur, &num_elems); + mdb_size_t num_elems = 0; + mdb_cursor_count(m_cur_tx_outputs, &num_elems); - mdb_cursor_get(cur, &k, &v, MDB_LAST_DUP); + mdb_cursor_get(m_cur_tx_outputs, &k, &v, MDB_LAST_DUP); for (uint64_t i = num_elems; i > 0; --i) { @@ -775,12 +772,10 @@ void BlockchainLMDB::remove_tx_outputs(const crypto::hash& tx_hash, const transa remove_output(*(const uint64_t*)v.mv_data, tx_output.amount); if (i > 1) { - mdb_cursor_get(cur, &k, &v, MDB_PREV_DUP); + mdb_cursor_get(m_cur_tx_outputs, &k, &v, MDB_PREV_DUP); } } } - - cur.close(); } // TODO: probably remove this function @@ -804,7 +799,7 @@ void BlockchainLMDB::remove_output(const uint64_t& out_index, const uint64_t amo } else if (result) { - throw1(DB_ERROR("Error adding removal of output tx index to db transaction")); + throw1(DB_ERROR(lmdb_error("Error adding removal of output tx index to db transaction", result).c_str())); } result = mdb_del(*m_write_txn, m_output_txs, &k, NULL); @@ -816,7 +811,7 @@ void BlockchainLMDB::remove_output(const uint64_t& out_index, const uint64_t amo } else if (result) { - throw1(DB_ERROR("Error adding removal of output tx hash to db transaction")); + throw1(DB_ERROR(lmdb_error("Error adding removal of output tx hash to db transaction", result).c_str())); } result = mdb_del(*m_write_txn, m_output_keys, &k, NULL); @@ -825,7 +820,7 @@ void BlockchainLMDB::remove_output(const uint64_t& out_index, const uint64_t amo LOG_PRINT_L0("Unexpected: global output index not found in m_output_keys"); } else if (result) - throw1(DB_ERROR("Error adding removal of output pubkey to db transaction")); + throw1(DB_ERROR(lmdb_error("Error adding removal of output pubkey to db transaction", result).c_str())); remove_amount_output_index(amount, out_index); @@ -836,29 +831,29 @@ void BlockchainLMDB::remove_amount_output_index(const uint64_t amount, const uin { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - - lmdb_cur cur(*m_write_txn, m_output_amounts); + mdb_txn_cursors *m_cursors = &m_wcursors; + CURSOR(output_amounts); MDB_val_copy<uint64_t> k(amount); MDB_val v; - auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + auto result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_SET); if (result == MDB_NOTFOUND) throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); else if (result) - throw0(DB_ERROR("DB error attempting to get an output")); + throw0(DB_ERROR(lmdb_error("DB error attempting to get an output", result).c_str())); - size_t num_elems = 0; - mdb_cursor_count(cur, &num_elems); + mdb_size_t num_elems = 0; + mdb_cursor_count(m_cur_output_amounts, &num_elems); - mdb_cursor_get(cur, &k, &v, MDB_LAST_DUP); + mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_LAST_DUP); uint64_t amount_output_index = 0; uint64_t goi = 0; bool found_index = false; for (uint64_t i = num_elems; i > 0; --i) { - mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); + mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_CURRENT); goi = *(const uint64_t *)v.mv_data; if (goi == global_output_index) { @@ -867,29 +862,28 @@ void BlockchainLMDB::remove_amount_output_index(const uint64_t amount, const uin break; } if (i > 1) - mdb_cursor_get(cur, &k, &v, MDB_PREV_DUP); + mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_PREV_DUP); } if (found_index) { // found the amount output index // now delete it - result = mdb_cursor_del(cur, 0); + result = mdb_cursor_del(m_cur_output_amounts, 0); if (result) throw0(DB_ERROR(std::string("Error deleting amount output index ").append(boost::lexical_cast<std::string>(amount_output_index)).c_str())); } else { // not found - cur.close(); throw1(OUTPUT_DNE("Failed to find amount output index")); } - cur.close(); } void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); + mdb_txn_cursors *m_cursors = &m_wcursors; CURSOR(spent_keys) @@ -902,7 +896,7 @@ void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image) unused.mv_size = sizeof(char); unused.mv_data = &anything; if (auto result = mdb_cursor_put(m_cur_spent_keys, &val_key, &unused, 0)) - throw1(DB_ERROR(std::string("Error adding spent key image to db transaction: ").append(mdb_strerror(result)).c_str())); + throw1(DB_ERROR(lmdb_error("Error adding spent key image to db transaction: ", result).c_str())); } void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image) @@ -913,7 +907,7 @@ void BlockchainLMDB::remove_spent_key(const crypto::key_image& k_image) MDB_val_copy<crypto::key_image> k(k_image); auto result = mdb_del(*m_write_txn, m_spent_keys, &k, NULL); if (result != 0 && result != MDB_NOTFOUND) - throw1(DB_ERROR("Error adding removal of key image to db transaction")); + throw1(DB_ERROR(lmdb_error("Error adding removal of key image to db transaction", result).c_str())); } blobdata BlockchainLMDB::output_to_blob(const tx_out& output) const @@ -1028,7 +1022,7 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) size_t mapsize = DEFAULT_MAPSIZE; if (auto result = mdb_env_open(m_env, filename.c_str(), mdb_flags, 0644)) - throw0(DB_ERROR(std::string("Failed to open lmdb environment: ").append(mdb_strerror(result)).c_str())); + throw0(DB_ERROR(lmdb_error("Failed to open lmdb environment: ", result).c_str())); MDB_envinfo mei; mdb_env_info(m_env, &mei); @@ -1037,7 +1031,7 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags) if (cur_mapsize < mapsize) { if (auto result = mdb_env_set_mapsize(m_env, mapsize)) - throw0(DB_ERROR(std::string("Failed to set max memory map size: ").append(mdb_strerror(result)).c_str())); + throw0(DB_ERROR(lmdb_error("Failed to set max memory map size: ", result).c_str())); mdb_env_info(m_env, &mei); cur_mapsize = (double)mei.me_mapsize; LOG_PRINT_L1("LMDB memory map size: " << cur_mapsize); @@ -1201,6 +1195,7 @@ void BlockchainLMDB::close() batch_abort(); } this->sync(); + m_tinfo.reset(); // FIXME: not yet thread safe!!! Use with care. mdb_env_close(m_env); @@ -1216,7 +1211,7 @@ void BlockchainLMDB::sync() // MDB_NOMETASYNC. Force flush to be synchronous. if (auto result = mdb_env_sync(m_env, true)) { - throw0(DB_ERROR(std::string("Failed to sync database: ").append(mdb_strerror(result)).c_str())); + throw0(DB_ERROR(lmdb_error("Failed to sync database: ", result).c_str())); } } @@ -1303,7 +1298,11 @@ void BlockchainLMDB::unlock() throw0(DB_ERROR(lmdb_error(std::string("Failed to create a transaction for the db in ")+__FUNCTION__+": ", mdb_res).c_str())); \ } \ -#define TXN_PREFIX_RDONLY(); TXN_PREFIX(MDB_RDONLY); +#define TXN_PREFIX_RDONLY() \ + bool my_rtxn = block_rtxn_start(); \ + MDB_txn *m_txn = m_write_txn ? m_write_txn->m_txn : m_tinfo->m_ti_rtxn +#define TXN_POSTFIX_RDONLY() \ + if (my_rtxn) block_rtxn_stop() #define TXN_POSTFIX_SUCCESS() \ do { \ @@ -1343,20 +1342,21 @@ bool BlockchainLMDB::block_exists(const crypto::hash& h) const check_open(); TXN_PREFIX_RDONLY(); + const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; + RCURSOR(block_heights); MDB_val_copy<crypto::hash> key(h); - MDB_val result; - auto get_result = mdb_get(*txn_ptr, m_block_heights, &key, &result); + auto get_result = mdb_cursor_get(m_cur_block_heights, &key, NULL, MDB_SET); if (get_result == MDB_NOTFOUND) { - TXN_POSTFIX_SUCCESS(); + TXN_POSTFIX_RDONLY(); LOG_PRINT_L3("Block with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); return false; } else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch block index from hash")); + throw0(DB_ERROR(lmdb_error("DB error attempting to fetch block index from hash", get_result).c_str())); - TXN_POSTFIX_SUCCESS(); + TXN_POSTFIX_RDONLY(); return true; } @@ -1374,17 +1374,19 @@ uint64_t BlockchainLMDB::get_block_height(const crypto::hash& h) const check_open(); TXN_PREFIX_RDONLY(); + const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; + RCURSOR(block_heights); MDB_val_copy<crypto::hash> key(h); MDB_val result; - auto get_result = mdb_get(*txn_ptr, m_block_heights, &key, &result); + auto get_result = mdb_cursor_get(m_cur_block_heights, &key, &result, MDB_SET); if (get_result == MDB_NOTFOUND) throw1(BLOCK_DNE("Attempted to retrieve non-existent block height")); else if (get_result) throw0(DB_ERROR("Error attempting to retrieve a block height from the db")); uint64_t ret = *(const uint64_t *)result.mv_data; - TXN_POSTFIX_SUCCESS(); + TXN_POSTFIX_RDONLY(); return ret; } @@ -1403,10 +1405,12 @@ block BlockchainLMDB::get_block_from_height(const uint64_t& height) const check_open(); TXN_PREFIX_RDONLY(); + const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; + RCURSOR(blocks); MDB_val_copy<uint64_t> key(height); MDB_val result; - auto get_result = mdb_get(*txn_ptr, m_blocks, &key, &result); + auto get_result = mdb_cursor_get(m_cur_blocks, &key, &result, MDB_SET); if (get_result == MDB_NOTFOUND) { throw0(BLOCK_DNE(std::string("Attempt to get block from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- block not in db").c_str())); @@ -1421,7 +1425,7 @@ block BlockchainLMDB::get_block_from_height(const uint64_t& height) const if (!parse_and_validate_block_from_blob(bd, b)) throw0(DB_ERROR("Failed to parse block from blob retrieved from the db")); - TXN_POSTFIX_SUCCESS(); + TXN_POSTFIX_RDONLY(); return b; } @@ -1432,10 +1436,12 @@ uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const check_open(); TXN_PREFIX_RDONLY(); + const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; + RCURSOR(block_timestamps); MDB_val_copy<uint64_t> key(height); MDB_val result; - auto get_result = mdb_get(*txn_ptr, m_block_timestamps, &key, &result); + auto get_result = mdb_cursor_get(m_cur_block_timestamps, &key, &result, MDB_SET); if (get_result == MDB_NOTFOUND) { throw0(BLOCK_DNE(std::string("Attempt to get timestamp from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- timestamp not in db").c_str())); @@ -1444,7 +1450,7 @@ uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const throw0(DB_ERROR("Error attempting to retrieve a timestamp from the db")); uint64_t ret = *(const uint64_t *)result.mv_data; - TXN_POSTFIX_SUCCESS(); + TXN_POSTFIX_RDONLY(); return ret; } @@ -1468,10 +1474,12 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) const check_open(); TXN_PREFIX_RDONLY(); + const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; + RCURSOR(block_sizes); MDB_val_copy<uint64_t> key(height); MDB_val result; - auto get_result = mdb_get(*txn_ptr, m_block_sizes, &key, &result); + auto get_result = mdb_cursor_get(m_cur_block_sizes, &key, &result, MDB_SET); if (get_result == MDB_NOTFOUND) { throw0(BLOCK_DNE(std::string("Attempt to get block size from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- block size not in db").c_str())); @@ -1480,7 +1488,7 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) const throw0(DB_ERROR("Error attempting to retrieve a block size from the db")); size_t ret = *(const size_t *)result.mv_data; - TXN_POSTFIX_SUCCESS(); + TXN_POSTFIX_RDONLY(); return ret; } @@ -1490,10 +1498,12 @@ difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& check_open(); TXN_PREFIX_RDONLY(); + const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; + RCURSOR(block_diffs); MDB_val_copy<uint64_t> key(height); MDB_val result; - auto get_result = mdb_get(*txn_ptr, m_block_diffs, &key, &result); + auto get_result = mdb_cursor_get(m_cur_block_diffs, &key, &result, MDB_SET); if (get_result == MDB_NOTFOUND) { throw0(BLOCK_DNE(std::string("Attempt to get cumulative difficulty from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- difficulty not in db").c_str())); @@ -1502,7 +1512,7 @@ difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& throw0(DB_ERROR("Error attempting to retrieve a cumulative difficulty from the db")); difficulty_type ret = *(const difficulty_type*)result.mv_data; - TXN_POSTFIX_SUCCESS(); + TXN_POSTFIX_RDONLY(); return ret; } @@ -1529,10 +1539,12 @@ uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& heigh check_open(); TXN_PREFIX_RDONLY(); + const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; + RCURSOR(block_coins); MDB_val_copy<uint64_t> key(height); MDB_val result; - auto get_result = mdb_get(*txn_ptr, m_block_coins, &key, &result); + auto get_result = mdb_cursor_get(m_cur_block_coins, &key, &result, MDB_SET); if (get_result == MDB_NOTFOUND) { throw0(BLOCK_DNE(std::string("Attempt to get generated coins from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- block size not in db").c_str())); @@ -1541,7 +1553,7 @@ uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& heigh throw0(DB_ERROR("Error attempting to retrieve a total generated coins from the db")); uint64_t ret = *(const uint64_t*)result.mv_data; - TXN_POSTFIX_SUCCESS(); + TXN_POSTFIX_RDONLY(); return ret; } @@ -1551,20 +1563,21 @@ crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height) check_open(); TXN_PREFIX_RDONLY(); + const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; + RCURSOR(block_hashes); MDB_val_copy<uint64_t> key(height); MDB_val result; - auto get_result = mdb_get(*txn_ptr, m_block_hashes, &key, &result); + auto get_result = mdb_cursor_get(m_cur_block_hashes, &key, &result, MDB_SET); if (get_result == MDB_NOTFOUND) { throw0(BLOCK_DNE(std::string("Attempt to get hash from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- hash not in db").c_str())); } else if (get_result) - throw0(DB_ERROR(std::string("Error attempting to retrieve a block hash from the db: "). - append(mdb_strerror(get_result)).c_str())); + throw0(DB_ERROR(lmdb_error("Error attempting to retrieve a block hash from the db: ", get_result).c_str())); crypto::hash ret = *(const crypto::hash*)result.mv_data; - TXN_POSTFIX_SUCCESS(); + TXN_POSTFIX_RDONLY(); return ret; } @@ -1636,16 +1649,18 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) const check_open(); TXN_PREFIX_RDONLY(); + const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; + RCURSOR(txs); MDB_val_copy<crypto::hash> key(h); MDB_val result; TIME_MEASURE_START(time1); - auto get_result = mdb_get(*txn_ptr, m_txs, &key, &result); + auto get_result = mdb_cursor_get(m_cur_txs, &key, &result, MDB_SET); TIME_MEASURE_FINISH(time1); time_tx_exists += time1; - TXN_POSTFIX_SUCCESS(); + TXN_POSTFIX_RDONLY(); if (get_result == MDB_NOTFOUND) { @@ -1653,7 +1668,7 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) const return false; } else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch transaction from hash")); + throw0(DB_ERROR(lmdb_error("DB error attempting to fetch transaction from hash", get_result).c_str())); return true; } @@ -1664,17 +1679,19 @@ uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) const check_open(); TXN_PREFIX_RDONLY(); + const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; + RCURSOR(tx_unlocks); MDB_val_copy<crypto::hash> key(h); MDB_val result; - auto get_result = mdb_get(*txn_ptr, m_tx_unlocks, &key, &result); + auto get_result = mdb_cursor_get(m_cur_tx_unlocks, &key, &result, MDB_SET); if (get_result == MDB_NOTFOUND) throw1(TX_DNE(std::string("tx unlock time with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch tx unlock time from hash")); + throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx unlock time from hash", get_result).c_str())); uint64_t ret = *(const uint64_t*)result.mv_data; - TXN_POSTFIX_SUCCESS(); + TXN_POSTFIX_RDONLY(); return ret; } @@ -1684,14 +1701,16 @@ transaction BlockchainLMDB::get_tx(const crypto::hash& h) const check_open(); TXN_PREFIX_RDONLY(); + const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; + RCURSOR(txs); MDB_val_copy<crypto::hash> key(h); MDB_val result; - auto get_result = mdb_get(*txn_ptr, m_txs, &key, &result); + auto get_result = mdb_cursor_get(m_cur_txs, &key, &result, MDB_SET); if (get_result == MDB_NOTFOUND) throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch tx from hash")); + throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx from hash", get_result).c_str())); blobdata bd; bd.assign(reinterpret_cast<char*>(result.mv_data), result.mv_size); @@ -1700,7 +1719,7 @@ transaction BlockchainLMDB::get_tx(const crypto::hash& h) const if (!parse_and_validate_tx_from_blob(bd, tx)) throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); - TXN_POSTFIX_SUCCESS(); + TXN_POSTFIX_RDONLY(); return tx; } @@ -1713,10 +1732,10 @@ uint64_t BlockchainLMDB::get_tx_count() const TXN_PREFIX_RDONLY(); MDB_stat db_stats; - if (mdb_stat(*txn_ptr, m_txs, &db_stats)) + if (mdb_stat(m_txn, m_txs, &db_stats)) throw0(DB_ERROR("Failed to query m_txs")); - TXN_POSTFIX_SUCCESS(); + TXN_POSTFIX_RDONLY(); return db_stats.ms_entries; } @@ -1741,19 +1760,21 @@ uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const check_open(); TXN_PREFIX_RDONLY(); + const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; + RCURSOR(tx_heights); MDB_val_copy<crypto::hash> key(h); MDB_val result; - auto get_result = mdb_get(*txn_ptr, m_tx_heights, &key, &result); + auto get_result = mdb_cursor_get(m_cur_tx_heights, &key, &result, MDB_SET); if (get_result == MDB_NOTFOUND) { throw1(TX_DNE(std::string("tx height with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); } else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch tx height from hash")); + throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx height from hash", get_result).c_str())); uint64_t ret = *(const uint64_t*)result.mv_data; - TXN_POSTFIX_SUCCESS(); + TXN_POSTFIX_RDONLY(); return ret; } @@ -1763,24 +1784,24 @@ uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) const check_open(); TXN_PREFIX_RDONLY(); - - lmdb_cur cur(*txn_ptr, m_output_amounts); + const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; + RCURSOR(output_amounts); MDB_val_copy<uint64_t> k(amount); MDB_val v; - auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + auto result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_SET); if (result == MDB_NOTFOUND) { - TXN_POSTFIX_SUCCESS(); + TXN_POSTFIX_RDONLY(); return 0; } else if (result) throw0(DB_ERROR("DB error attempting to get number of outputs of an amount")); - size_t num_elems = 0; - mdb_cursor_count(cur, &num_elems); + mdb_size_t num_elems = 0; + mdb_cursor_count(m_cur_output_amounts, &num_elems); - TXN_POSTFIX_SUCCESS(); + TXN_POSTFIX_RDONLY(); return num_elems; } @@ -1791,16 +1812,18 @@ output_data_t BlockchainLMDB::get_output_key(const uint64_t &global_index) const check_open(); TXN_PREFIX_RDONLY(); + const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; + RCURSOR(output_keys); MDB_val_copy<uint64_t> k(global_index); MDB_val v; - auto get_result = mdb_get(*txn_ptr, m_output_keys, &k, &v); + auto get_result = mdb_cursor_get(m_cur_output_keys, &k, &v, MDB_SET); if (get_result == MDB_NOTFOUND) throw1(OUTPUT_DNE("Attempting to get output pubkey by global index, but key does not exist")); else if (get_result) throw0(DB_ERROR("Error attempting to retrieve an output pubkey from the db")); output_data_t ret = *(const output_data_t *) v.mv_data; - TXN_POSTFIX_SUCCESS(); + TXN_POSTFIX_RDONLY(); return ret; } @@ -1819,11 +1842,14 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index_from_global(const uint64_t& check_open(); TXN_PREFIX_RDONLY(); + const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; + RCURSOR(output_txs); + RCURSOR(output_indices); MDB_val_copy<uint64_t> k(index); MDB_val v; - auto get_result = mdb_get(*txn_ptr, m_output_txs, &k, &v); + auto get_result = mdb_cursor_get(m_cur_output_txs, &k, &v, MDB_SET); if (get_result == MDB_NOTFOUND) throw1(OUTPUT_DNE("output with given index not in db")); else if (get_result) @@ -1831,14 +1857,14 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index_from_global(const uint64_t& crypto::hash tx_hash = *(const crypto::hash*)v.mv_data; - get_result = mdb_get(*txn_ptr, m_output_indices, &k, &v); + get_result = mdb_cursor_get(m_cur_output_indices, &k, &v, MDB_SET); if (get_result == MDB_NOTFOUND) throw1(OUTPUT_DNE("output with given index not in db")); else if (get_result) throw0(DB_ERROR("DB error attempting to fetch output tx index")); tx_out_index ret = tx_out_index(tx_hash, *(const uint64_t *)v.mv_data); - TXN_POSTFIX_SUCCESS(); + TXN_POSTFIX_RDONLY(); return ret; } @@ -1862,31 +1888,30 @@ std::vector<uint64_t> BlockchainLMDB::get_tx_output_indices(const crypto::hash& std::vector<uint64_t> index_vec; TXN_PREFIX_RDONLY(); - - lmdb_cur cur(*txn_ptr, m_tx_outputs); + const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; + RCURSOR(tx_outputs); MDB_val_copy<crypto::hash> k(h); MDB_val v; - auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + auto result = mdb_cursor_get(m_cur_tx_outputs, &k, &v, MDB_SET); if (result == MDB_NOTFOUND) throw1(OUTPUT_DNE("Attempting to get an output by tx hash and tx index, but output not found")); else if (result) - throw0(DB_ERROR("DB error attempting to get an output")); + throw0(DB_ERROR(lmdb_error("DB error attempting to get an output", result).c_str())); - size_t num_elems = 0; - mdb_cursor_count(cur, &num_elems); + mdb_size_t num_elems = 0; + mdb_cursor_count(m_cur_tx_outputs, &num_elems); - mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); + mdb_cursor_get(m_cur_tx_outputs, &k, &v, MDB_FIRST_DUP); for (uint64_t i = 0; i < num_elems; ++i) { - mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); + mdb_cursor_get(m_cur_tx_outputs, &k, &v, MDB_GET_CURRENT); index_vec.push_back(*(const uint64_t *)v.mv_data); - mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); + mdb_cursor_get(m_cur_tx_outputs, &k, &v, MDB_NEXT_DUP); } - cur.close(); - TXN_POSTFIX_SUCCESS(); + TXN_POSTFIX_RDONLY(); return index_vec; } @@ -1905,6 +1930,8 @@ std::vector<uint64_t> BlockchainLMDB::get_tx_amount_output_indices(const crypto: transaction tx = get_tx(h); TXN_PREFIX_RDONLY(); + const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; + RCURSOR(output_amounts); uint64_t i = 0; uint64_t global_index; @@ -1914,28 +1941,26 @@ std::vector<uint64_t> BlockchainLMDB::get_tx_amount_output_indices(const crypto: global_index = index_vec[i]; - lmdb_cur cur(*txn_ptr, m_output_amounts); - MDB_val_copy<uint64_t> k(amount); MDB_val v; - auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + auto result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_SET); if (result == MDB_NOTFOUND) throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); else if (result) - throw0(DB_ERROR("DB error attempting to get an output")); + throw0(DB_ERROR(lmdb_error("DB error attempting to get an output", result).c_str())); - size_t num_elems = 0; - mdb_cursor_count(cur, &num_elems); + mdb_size_t num_elems = 0; + mdb_cursor_count(m_cur_output_amounts, &num_elems); - mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); + mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_FIRST_DUP); uint64_t amount_output_index = 0; uint64_t output_index = 0; bool found_index = false; for (uint64_t j = 0; j < num_elems; ++j) { - mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); + mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_CURRENT); output_index = *(const uint64_t *)v.mv_data; if (output_index == global_index) { @@ -1943,7 +1968,7 @@ std::vector<uint64_t> BlockchainLMDB::get_tx_amount_output_indices(const crypto: found_index = true; break; } - mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); + mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_NEXT_DUP); } if (found_index) { @@ -1952,16 +1977,14 @@ std::vector<uint64_t> BlockchainLMDB::get_tx_amount_output_indices(const crypto: else { // not found - cur.close(); - TXN_POSTFIX_SUCCESS(); + TXN_POSTFIX_RDONLY(); throw1(OUTPUT_DNE("specified output not found in db")); } - cur.close(); ++i; } - TXN_POSTFIX_SUCCESS(); + TXN_POSTFIX_RDONLY(); return index_vec2; } @@ -1974,16 +1997,17 @@ bool BlockchainLMDB::has_key_image(const crypto::key_image& img) const check_open(); TXN_PREFIX_RDONLY(); + const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; + RCURSOR(spent_keys); MDB_val_copy<crypto::key_image> val_key(img); - MDB_val unused; - if (mdb_get(*txn_ptr, m_spent_keys, &val_key, &unused) == 0) + if (mdb_cursor_get(m_cur_spent_keys, &val_key, NULL, MDB_SET) == 0) { - TXN_POSTFIX_SUCCESS(); + TXN_POSTFIX_RDONLY(); return true; } - TXN_POSTFIX_SUCCESS(); + TXN_POSTFIX_RDONLY(); return false; } @@ -1993,16 +2017,17 @@ bool BlockchainLMDB::for_all_key_images(std::function<bool(const crypto::key_ima check_open(); TXN_PREFIX_RDONLY(); + const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; + RCURSOR(spent_keys); MDB_val k; MDB_val v; bool ret = true; - lmdb_cur cur(*txn_ptr, m_spent_keys); MDB_cursor_op op = MDB_FIRST; while (1) { - int ret = mdb_cursor_get(cur, &k, &v, op); + int ret = mdb_cursor_get(m_cur_spent_keys, &k, &v, op); op = MDB_NEXT; if (ret == MDB_NOTFOUND) break; @@ -2015,8 +2040,7 @@ bool BlockchainLMDB::for_all_key_images(std::function<bool(const crypto::key_ima } } - cur.close(); - TXN_POSTFIX_SUCCESS(); + TXN_POSTFIX_RDONLY(); return ret; } @@ -2027,16 +2051,17 @@ bool BlockchainLMDB::for_all_blocks(std::function<bool(uint64_t, const crypto::h check_open(); TXN_PREFIX_RDONLY(); + const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; + RCURSOR(blocks); MDB_val k; MDB_val v; bool ret = true; - lmdb_cur cur(*txn_ptr, m_blocks); MDB_cursor_op op = MDB_FIRST; while (1) { - int ret = mdb_cursor_get(cur, &k, &v, op); + int ret = mdb_cursor_get(m_cur_blocks, &k, &v, op); op = MDB_NEXT; if (ret == MDB_NOTFOUND) break; @@ -2057,8 +2082,7 @@ bool BlockchainLMDB::for_all_blocks(std::function<bool(uint64_t, const crypto::h } } - cur.close(); - TXN_POSTFIX_SUCCESS(); + TXN_POSTFIX_RDONLY(); return ret; } @@ -2069,16 +2093,17 @@ bool BlockchainLMDB::for_all_transactions(std::function<bool(const crypto::hash& check_open(); TXN_PREFIX_RDONLY(); + const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; + RCURSOR(txs); MDB_val k; MDB_val v; bool ret = true; - lmdb_cur cur(*txn_ptr, m_txs); MDB_cursor_op op = MDB_FIRST; while (1) { - int ret = mdb_cursor_get(cur, &k, &v, op); + int ret = mdb_cursor_get(m_cur_txs, &k, &v, op); op = MDB_NEXT; if (ret == MDB_NOTFOUND) break; @@ -2096,8 +2121,7 @@ bool BlockchainLMDB::for_all_transactions(std::function<bool(const crypto::hash& } } - cur.close(); - TXN_POSTFIX_SUCCESS(); + TXN_POSTFIX_RDONLY(); return ret; } @@ -2108,16 +2132,17 @@ bool BlockchainLMDB::for_all_outputs(std::function<bool(uint64_t amount, const c check_open(); TXN_PREFIX_RDONLY(); + const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; + RCURSOR(output_amounts); MDB_val k; MDB_val v; bool ret = true; - lmdb_cur cur(*txn_ptr, m_output_amounts); MDB_cursor_op op = MDB_FIRST; while (1) { - int ret = mdb_cursor_get(cur, &k, &v, op); + int ret = mdb_cursor_get(m_cur_output_amounts, &k, &v, op); op = MDB_NEXT; if (ret == MDB_NOTFOUND) break; @@ -2132,8 +2157,7 @@ bool BlockchainLMDB::for_all_outputs(std::function<bool(uint64_t amount, const c } } - cur.close(); - TXN_POSTFIX_SUCCESS(); + TXN_POSTFIX_RDONLY(); return ret; } @@ -2167,8 +2191,9 @@ void BlockchainLMDB::batch_start(uint64_t batch_num_blocks) // active m_write_batch_txn->m_batch_txn = true; m_write_txn = m_write_batch_txn; + m_batch_active = true; - memset(&m_cursors, 0, sizeof(m_cursors)); + memset(&m_wcursors, 0, sizeof(m_wcursors)); LOG_PRINT_L3("batch transaction: begin"); } @@ -2193,7 +2218,7 @@ void BlockchainLMDB::batch_commit() m_write_txn = nullptr; delete m_write_batch_txn; - memset(&m_cursors, 0, sizeof(m_cursors)); + memset(&m_wcursors, 0, sizeof(m_wcursors)); } void BlockchainLMDB::batch_stop() @@ -2216,7 +2241,7 @@ void BlockchainLMDB::batch_stop() delete m_write_batch_txn; m_write_batch_txn = nullptr; m_batch_active = false; - memset(&m_cursors, 0, sizeof(m_cursors)); + memset(&m_wcursors, 0, sizeof(m_wcursors)); LOG_PRINT_L3("batch transaction: end"); } @@ -2234,7 +2259,7 @@ void BlockchainLMDB::batch_abort() m_write_batch_txn->abort(); m_batch_active = false; m_write_batch_txn = nullptr; - memset(&m_cursors, 0, sizeof(m_cursors)); + memset(&m_wcursors, 0, sizeof(m_wcursors)); LOG_PRINT_L3("batch transaction: aborted"); } @@ -2245,9 +2270,68 @@ void BlockchainLMDB::set_batch_transactions(bool batch_transactions) LOG_PRINT_L3("batch transactions " << (m_batch_transactions ? "enabled" : "disabled")); } -void BlockchainLMDB::block_txn_start() +// return true if we started the txn, false if already started +bool BlockchainLMDB::block_rtxn_start() const +{ + if (m_write_txn) + return false; + if (!m_tinfo.get()) + { + m_tinfo.reset(new mdb_threadinfo); + memset(&m_tinfo->m_ti_rcursors, 0, sizeof(m_tinfo->m_ti_rcursors)); + memset(&m_tinfo->m_ti_rflags, 0, sizeof(m_tinfo->m_ti_rflags)); + if (auto mdb_res = mdb_txn_begin(m_env, NULL, MDB_RDONLY, &m_tinfo->m_ti_rtxn)) + throw0(DB_ERROR_TXN_START(lmdb_error("Failed to create a read transaction for the db: ", mdb_res).c_str())); + } else if (!m_tinfo->m_ti_rflags.m_rf_txn) + { + if (auto mdb_res = mdb_txn_renew(m_tinfo->m_ti_rtxn)) + throw0(DB_ERROR_TXN_START(lmdb_error("Failed to renew a read transaction for the db: ", mdb_res).c_str())); + } else + { + return false; + } + m_tinfo->m_ti_rflags.m_rf_txn = true; + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + return true; +} + +void BlockchainLMDB::block_rtxn_stop() const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); + mdb_txn_reset(m_tinfo->m_ti_rtxn); + memset(&m_tinfo->m_ti_rflags, 0, sizeof(m_tinfo->m_ti_rflags)); +} + +void BlockchainLMDB::block_txn_start(bool readonly) +{ + if (readonly) + { + bool didit = false; + if (m_write_txn) + return; + if (!m_tinfo.get()) + { + m_tinfo.reset(new mdb_threadinfo); + memset(&m_tinfo->m_ti_rcursors, 0, sizeof(m_tinfo->m_ti_rcursors)); + memset(&m_tinfo->m_ti_rflags, 0, sizeof(m_tinfo->m_ti_rflags)); + if (auto mdb_res = mdb_txn_begin(m_env, NULL, MDB_RDONLY, &m_tinfo->m_ti_rtxn)) + throw0(DB_ERROR_TXN_START(lmdb_error("Failed to create a read transaction for the db: ", mdb_res).c_str())); + didit = true; + } else if (!m_tinfo->m_ti_rflags.m_rf_txn) + { + if (auto mdb_res = mdb_txn_renew(m_tinfo->m_ti_rtxn)) + throw0(DB_ERROR_TXN_START(lmdb_error("Failed to renew a read transaction for the db: ", mdb_res).c_str())); + didit = true; + } + if (didit) + { + m_tinfo->m_ti_rflags.m_rf_txn = true; + LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " RO"); + } + return; + } + + LOG_PRINT_L3("BlockchainLMDB::" << __func__); // Distinguish the exceptions here from exceptions that would be thrown while // using the txn and committing it. // @@ -2266,7 +2350,7 @@ void BlockchainLMDB::block_txn_start() m_write_txn = nullptr; throw0(DB_ERROR_TXN_START(lmdb_error("Failed to create a transaction for the db: ", mdb_res).c_str())); } - memset(&m_cursors, 0, sizeof(m_cursors)); + memset(&m_wcursors, 0, sizeof(m_wcursors)); } } @@ -2275,14 +2359,22 @@ void BlockchainLMDB::block_txn_stop() LOG_PRINT_L3("BlockchainLMDB::" << __func__); if (! m_batch_active) { - TIME_MEASURE_START(time1); - m_write_txn->commit(); - TIME_MEASURE_FINISH(time1); - time_commit1 += time1; + if (m_write_txn) + { + TIME_MEASURE_START(time1); + m_write_txn->commit(); + TIME_MEASURE_FINISH(time1); + time_commit1 += time1; - delete m_write_txn; - m_write_txn = nullptr; - memset(&m_cursors, 0, sizeof(m_cursors)); + delete m_write_txn; + m_write_txn = nullptr; + memset(&m_wcursors, 0, sizeof(m_wcursors)); + } + else if (m_tinfo->m_ti_rtxn) + { + mdb_txn_reset(m_tinfo->m_ti_rtxn); + memset(&m_tinfo->m_ti_rflags, 0, sizeof(m_tinfo->m_ti_rflags)); + } } } @@ -2295,8 +2387,13 @@ void BlockchainLMDB::block_txn_abort() { delete m_write_txn; m_write_txn = nullptr; - memset(&m_cursors, 0, sizeof(m_cursors)); + memset(&m_wcursors, 0, sizeof(m_wcursors)); } + else if (m_tinfo->m_ti_rtxn) + { + mdb_txn_reset(m_tinfo->m_ti_rtxn); + memset(&m_tinfo->m_ti_rflags, 0, sizeof(m_tinfo->m_ti_rflags)); + } else { // This would probably mean an earlier exception was caught, but then we @@ -2348,31 +2445,18 @@ void BlockchainLMDB::pop_block(block& blk, std::vector<transaction>& txs) LOG_PRINT_L3("BlockchainLMDB::" << __func__); check_open(); - mdb_txn_safe txn; - if (! m_batch_active) - { - if (auto mdb_res = mdb_txn_begin(m_env, NULL, 0, txn)) - throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", mdb_res).c_str())); - m_write_txn = &txn; - memset(&m_cursors, 0, sizeof(m_cursors)); - } + block_txn_start(false); uint64_t num_outputs = m_num_outputs; try { BlockchainDB::pop_block(blk, txs); - if (! m_batch_active) - { - m_write_txn = nullptr; - memset(&m_cursors, 0, sizeof(m_cursors)); - txn.commit(); - } + block_txn_stop(); } catch (...) { m_num_outputs = num_outputs; - m_write_txn = nullptr; - memset(&m_cursors, 0, sizeof(m_cursors)); + block_txn_abort(); throw; } @@ -2387,13 +2471,16 @@ void BlockchainLMDB::get_output_tx_and_index_from_global(const std::vector<uint6 tx_out_indices.clear(); TXN_PREFIX_RDONLY(); + const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; + RCURSOR(output_txs); + RCURSOR(output_indices); for (const uint64_t &index : global_indices) { MDB_val_copy<uint64_t> k(index); MDB_val v; - auto get_result = mdb_get(*txn_ptr, m_output_txs, &k, &v); + auto get_result = mdb_cursor_get(m_cur_output_txs, &k, &v, MDB_SET); if (get_result == MDB_NOTFOUND) throw1(OUTPUT_DNE("output with given index not in db")); else if (get_result) @@ -2401,7 +2488,7 @@ void BlockchainLMDB::get_output_tx_and_index_from_global(const std::vector<uint6 crypto::hash tx_hash = *(const crypto::hash*) v.mv_data; - get_result = mdb_get(*txn_ptr, m_output_indices, &k, &v); + get_result = mdb_cursor_get(m_cur_output_indices, &k, &v, MDB_SET); if (get_result == MDB_NOTFOUND) throw1(OUTPUT_DNE("output with given index not in db")); else if (get_result) @@ -2411,7 +2498,7 @@ void BlockchainLMDB::get_output_tx_and_index_from_global(const std::vector<uint6 tx_out_indices.push_back(result); } - TXN_POSTFIX_SUCCESS(); + TXN_POSTFIX_RDONLY(); } void BlockchainLMDB::get_output_global_indices(const uint64_t& amount, const std::vector<uint64_t> &offsets, @@ -2430,19 +2517,19 @@ void BlockchainLMDB::get_output_global_indices(const uint64_t& amount, const std } TXN_PREFIX_RDONLY(); - - lmdb_cur cur(*txn_ptr, m_output_amounts); + const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; + RCURSOR(output_amounts); MDB_val_copy<uint64_t> k(amount); MDB_val v; - auto result = mdb_cursor_get(cur, &k, &v, MDB_SET); + auto result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_SET); if (result == MDB_NOTFOUND) throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); else if (result) - throw0(DB_ERROR("DB error attempting to get an output")); + throw0(DB_ERROR(lmdb_error("DB error attempting to get an output", result).c_str())); - size_t num_elems = 0; - mdb_cursor_count(cur, &num_elems); + mdb_size_t num_elems = 0; + mdb_cursor_count(m_cur_output_amounts, &num_elems); if (max <= 1 && num_elems <= max) throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found")); @@ -2452,13 +2539,13 @@ void BlockchainLMDB::get_output_global_indices(const uint64_t& amount, const std { for (const uint64_t& index : offsets) { - mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP); + mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_FIRST_DUP); for (uint64_t i = 0; i < index; ++i) { - mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP); + mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_NEXT_DUP); } - mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT); + mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_CURRENT); uint64_t glob_index = *(const uint64_t*) v.mv_data; LOG_PRINT_L3("Amount: " << amount << " M0->v: " << glob_index); global_indices.push_back(glob_index); @@ -2475,22 +2562,51 @@ void BlockchainLMDB::get_output_global_indices(const uint64_t& amount, const std LOG_PRINT_L1("Index: " << index << " Elems: " << num_elems << " partial results found for get_output_tx_and_index"); break; } - while (index >= curcount) + if (!curcount && index > num_elems/2) { - TIME_MEASURE_START(db1); - if (mdb_cursor_get(cur, &k, &v, curcount == 0 ? MDB_GET_MULTIPLE : MDB_NEXT_MULTIPLE) != 0) + mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_LAST_DUP); + mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_PREV); /* kludge to unset C_EOF */ + mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_NEXT); + mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_GET_MULTIPLE); + + curcount = num_elems; + while(1) { - // allow partial results - result = false; - break; + TIME_MEASURE_START(db1); + int count = v.mv_size / sizeof(uint64_t); + curcount -= count; + if (curcount > index) + { + mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_PREV_MULTIPLE); + } else + { + blockstart = curcount; + curcount += count; + break; + } + TIME_MEASURE_FINISH(db1); + t_dbmul += db1; } - int count = v.mv_size / sizeof(uint64_t); - - blockstart = curcount; - curcount += count; - TIME_MEASURE_FINISH(db1); - t_dbmul += db1; + } else + { + while (index >= curcount) + { + TIME_MEASURE_START(db1); + if (mdb_cursor_get(m_cur_output_amounts, &k, &v, curcount == 0 ? MDB_GET_MULTIPLE : MDB_NEXT_MULTIPLE) != 0) + { + // allow partial results + result = false; + break; + } + + int count = v.mv_size / sizeof(uint64_t); + + blockstart = curcount; + curcount += count; + TIME_MEASURE_FINISH(db1); + t_dbmul += db1; + } } LOG_PRINT_L3("Records returned: " << curcount << " Index: " << index); @@ -2507,8 +2623,7 @@ void BlockchainLMDB::get_output_global_indices(const uint64_t& amount, const std } } - cur.close(); - TXN_POSTFIX_SUCCESS(); + TXN_POSTFIX_RDONLY(); TIME_MEASURE_FINISH(txx); LOG_PRINT_L3("txx: " << txx << " db1: " << t_dbmul << " db2: " << t_dbscan); @@ -2521,31 +2636,32 @@ void BlockchainLMDB::get_output_key(const uint64_t &amount, const std::vector<ui check_open(); outputs.clear(); + TXN_PREFIX_RDONLY(); + const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; std::vector <uint64_t> global_indices; get_output_global_indices(amount, offsets, global_indices); if (global_indices.size() > 0) { - TXN_PREFIX_RDONLY(); - lmdb_cur cur(*txn_ptr, m_output_keys); + RCURSOR(output_keys); for (const uint64_t &index : global_indices) { MDB_val_copy<uint64_t> k(index); MDB_val v; - auto get_result = mdb_cursor_get(cur, &k, &v, MDB_SET); + auto get_result = mdb_cursor_get(m_cur_output_keys, &k, &v, MDB_SET); if (get_result == MDB_NOTFOUND) throw1(OUTPUT_DNE("Attempting to get output pubkey by global index, but key does not exist")); else if (get_result) - throw0(DB_ERROR("Error attempting to retrieve an output pubkey from the db")); + throw0(DB_ERROR(lmdb_error("Error attempting to retrieve an output pubkey from the db", get_result).c_str())); output_data_t data = *(const output_data_t *) v.mv_data; outputs.push_back(data); } - TXN_POSTFIX_SUCCESS(); } + TXN_POSTFIX_RDONLY(); TIME_MEASURE_FINISH(db3); LOG_PRINT_L3("db3: " << db3); @@ -2619,7 +2735,7 @@ void BlockchainLMDB::set_hard_fork_starting_height(uint8_t version, uint64_t hei MDB_val_copy<uint8_t> val_key(version); MDB_val_copy<uint64_t> val_value(height); if (auto result = mdb_put(*txn_ptr, m_hf_starting_heights, &val_key, &val_value, MDB_APPEND)) - throw1(DB_ERROR(std::string("Error adding hard fork starting height to db transaction: ").append(mdb_strerror(result)).c_str())); + throw1(DB_ERROR(lmdb_error("Error adding hard fork starting height to db transaction: ", result).c_str())); TXN_BLOCK_POSTFIX_SUCCESS(); } @@ -2633,14 +2749,19 @@ uint64_t BlockchainLMDB::get_hard_fork_starting_height(uint8_t version) const MDB_val_copy<uint8_t> val_key(version); MDB_val val_ret; - auto result = mdb_get(*txn_ptr, m_hf_starting_heights, &val_key, &val_ret); + auto result = mdb_get(m_txn, m_hf_starting_heights, &val_key, &val_ret); if (result == MDB_NOTFOUND) return std::numeric_limits<uint64_t>::max(); if (result) throw0(DB_ERROR("Error attempting to retrieve a hard fork starting height from the db")); - uint64_t ret = *(const uint64_t*)val_ret.mv_data; - TXN_POSTFIX_SUCCESS(); + uint64_t ret; +#ifdef MISALIGNED_OK + ret = *(const uint64_t*)val_ret.mv_data; +#else + memcpy(&ret, val_ret.mv_data, sizeof(uint64_t)); +#endif + TXN_POSTFIX_RDONLY(); return ret; } @@ -2658,7 +2779,7 @@ void BlockchainLMDB::set_hard_fork_version(uint64_t height, uint8_t version) if (result == MDB_KEYEXIST) result = mdb_put(*txn_ptr, m_hf_versions, &val_key, &val_value, 0); if (result) - throw1(DB_ERROR(std::string("Error adding hard fork version to db transaction: ").append(mdb_strerror(result)).c_str())); + throw1(DB_ERROR(lmdb_error("Error adding hard fork version to db transaction: ", result).c_str())); TXN_BLOCK_POSTFIX_SUCCESS(); } @@ -2669,15 +2790,17 @@ uint8_t BlockchainLMDB::get_hard_fork_version(uint64_t height) const check_open(); TXN_PREFIX_RDONLY(); + const mdb_txn_cursors *m_cursors = m_write_txn ? &m_wcursors : &m_tinfo->m_ti_rcursors; + RCURSOR(hf_versions); MDB_val_copy<uint64_t> val_key(height); MDB_val val_ret; - auto result = mdb_get(*txn_ptr, m_hf_versions, &val_key, &val_ret); + auto result = mdb_cursor_get(m_cur_hf_versions, &val_key, &val_ret, MDB_SET); if (result == MDB_NOTFOUND || result) - throw0(DB_ERROR("Error attempting to retrieve a hard fork version from the db")); + throw0(DB_ERROR(lmdb_error("Error attempting to retrieve a hard fork version at height " + boost::lexical_cast<std::string>(height) + " from the db: ", result).c_str())); uint8_t ret = *(const uint8_t*)val_ret.mv_data; - TXN_POSTFIX_SUCCESS(); + TXN_POSTFIX_RDONLY(); return ret; } @@ -2686,7 +2809,7 @@ bool BlockchainLMDB::is_read_only() const unsigned int flags; auto result = mdb_env_get_flags(m_env, &flags); if (result) - throw0(DB_ERROR(std::string("Error getting database environment info: ").append(mdb_strerror(result)).c_str())); + throw0(DB_ERROR(lmdb_error("Error getting database environment info: ", result).c_str())); if (flags & MDB_RDONLY) return true; diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index 150f59475..e58643efa 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -30,6 +30,7 @@ #include "blockchain_db/blockchain_db.h" #include "cryptonote_protocol/blobdatatype.h" // for type blobdata +#include <boost/thread/tss.hpp> #include <lmdb.h> @@ -38,7 +39,7 @@ namespace cryptonote { -struct mdb_txn_cursors +typedef struct mdb_txn_cursors { MDB_cursor *m_txc_blocks; MDB_cursor *m_txc_block_heights; @@ -59,24 +60,58 @@ struct mdb_txn_cursors MDB_cursor *m_txc_tx_outputs; MDB_cursor *m_txc_spent_keys; -}; -#define m_cur_blocks m_cursors.m_txc_blocks -#define m_cur_block_heights m_cursors.m_txc_block_heights -#define m_cur_block_hashes m_cursors.m_txc_block_hashes -#define m_cur_block_timestamps m_cursors.m_txc_block_timestamps -#define m_cur_block_sizes m_cursors.m_txc_block_sizes -#define m_cur_block_diffs m_cursors.m_txc_block_diffs -#define m_cur_block_coins m_cursors.m_txc_block_coins -#define m_cur_output_txs m_cursors.m_txc_output_txs -#define m_cur_output_indices m_cursors.m_txc_output_indices -#define m_cur_output_amounts m_cursors.m_txc_output_amounts -#define m_cur_output_keys m_cursors.m_txc_output_keys -#define m_cur_txs m_cursors.m_txc_txs -#define m_cur_tx_heights m_cursors.m_txc_tx_heights -#define m_cur_tx_unlocks m_cursors.m_txc_tx_unlocks -#define m_cur_tx_outputs m_cursors.m_txc_tx_outputs -#define m_cur_spent_keys m_cursors.m_txc_spent_keys + MDB_cursor *m_txc_hf_versions; +} mdb_txn_cursors; + +#define m_cur_blocks m_cursors->m_txc_blocks +#define m_cur_block_heights m_cursors->m_txc_block_heights +#define m_cur_block_hashes m_cursors->m_txc_block_hashes +#define m_cur_block_timestamps m_cursors->m_txc_block_timestamps +#define m_cur_block_sizes m_cursors->m_txc_block_sizes +#define m_cur_block_diffs m_cursors->m_txc_block_diffs +#define m_cur_block_coins m_cursors->m_txc_block_coins +#define m_cur_output_txs m_cursors->m_txc_output_txs +#define m_cur_output_indices m_cursors->m_txc_output_indices +#define m_cur_output_amounts m_cursors->m_txc_output_amounts +#define m_cur_output_keys m_cursors->m_txc_output_keys +#define m_cur_txs m_cursors->m_txc_txs +#define m_cur_tx_heights m_cursors->m_txc_tx_heights +#define m_cur_tx_unlocks m_cursors->m_txc_tx_unlocks +#define m_cur_tx_outputs m_cursors->m_txc_tx_outputs +#define m_cur_spent_keys m_cursors->m_txc_spent_keys +#define m_cur_hf_versions m_cursors->m_txc_hf_versions + +typedef struct mdb_rflags +{ + bool m_rf_txn; + bool m_rf_blocks; + bool m_rf_block_heights; + bool m_rf_block_hashes; + bool m_rf_block_timestamps; + bool m_rf_block_sizes; + bool m_rf_block_diffs; + bool m_rf_block_coins; + bool m_rf_output_txs; + bool m_rf_output_indices; + bool m_rf_output_amounts; + bool m_rf_output_keys; + bool m_rf_txs; + bool m_rf_tx_heights; + bool m_rf_tx_unlocks; + bool m_rf_tx_outputs; + bool m_rf_spent_keys; + bool m_rf_hf_versions; +} mdb_rflags; + +typedef struct mdb_threadinfo +{ + MDB_txn *m_ti_rtxn; // per-thread read txn + mdb_txn_cursors m_ti_rcursors; // per-thread read cursors + mdb_rflags m_ti_rflags; // per-thread read state + + ~mdb_threadinfo(); +} mdb_threadinfo; struct mdb_txn_safe { @@ -234,9 +269,11 @@ public: virtual void batch_stop(); virtual void batch_abort(); - virtual void block_txn_start(); + virtual void block_txn_start(bool readonly); virtual void block_txn_stop(); virtual void block_txn_abort(); + virtual bool block_rtxn_start() const; + virtual void block_rtxn_stop() const; virtual void pop_block(block& blk, std::vector<transaction>& txs); @@ -355,7 +392,8 @@ private: bool m_batch_transactions; // support for batch transactions bool m_batch_active; // whether batch transaction is in progress - struct mdb_txn_cursors m_cursors; + mdb_txn_cursors m_wcursors; + mutable boost::thread_specific_ptr<mdb_threadinfo> m_tinfo; #if defined(__arm__) // force a value so it can compile with 32-bit ARM diff --git a/src/blockchain_utilities/CMakeLists.txt b/src/blockchain_utilities/CMakeLists.txt index ad2a5b40a..96b30b83b 100644 --- a/src/blockchain_utilities/CMakeLists.txt +++ b/src/blockchain_utilities/CMakeLists.txt @@ -99,6 +99,11 @@ target_link_libraries(blockchain_converter blockchain_db ${CMAKE_THREAD_LIBS_INIT}) +if(ARCH_WIDTH) + target_compile_definitions(blockchain_converter + PUBLIC -DARCH_WIDTH=${ARCH_WIDTH}) +endif() + add_dependencies(blockchain_converter version) set_property(TARGET blockchain_converter @@ -117,6 +122,11 @@ target_link_libraries(blockchain_import p2p ${CMAKE_THREAD_LIBS_INIT}) +if(ARCH_WIDTH) + target_compile_definitions(blockchain_import + PUBLIC -DARCH_WIDTH=${ARCH_WIDTH}) +endif() + add_dependencies(blockchain_import version) set_property(TARGET blockchain_import diff --git a/src/blockchain_utilities/README.md b/src/blockchain_utilities/README.md index d763bc4dd..9c69647ad 100644 --- a/src/blockchain_utilities/README.md +++ b/src/blockchain_utilities/README.md @@ -52,18 +52,68 @@ the `blockchain_import` command again, and it will restart from where it left of ## use default settings to import blockchain.raw into database $ blockchain_import -## fast import with large batch size, verification off -$ blockchain_import --batch-size 100000 --verify off +## fast import with large batch size, database mode "fastest", verification off +$ blockchain_import --batch-size 20000 --database lmdb#fastest --verify off + +``` + +### Import options + +`--input-file` +specifies input file path for importing + +default: `<data-dir>/export/blockchain.raw` + +`--output-file` +specifies output file path to export to + +default: `<data-dir>/export/blockchain.raw` + +`--block-stop` +stop at block number + +`--database <database type>` + +`--database <database type>#<flag(s)>` + +database type: `lmdb, berkeley, memory` + +flags: + +The flag after the # is interpreted as a composite mode/flag if there's only +one (no comma separated arguments). + +The composite mode represents multiple DB flags and support different database types: + +`safe, fast, fastest` + +Database-specific flags can be set instead. + +LMDB flags (more than one may be specified): + +`nosync, nometasync, writemap, mapasync, nordahead` + +BerkeleyDB flags (takes one): + +`txn_write_nosync, txn_nosync, txn_sync` + +``` +## Examples: +$ blockchain_import --database lmdb#fastest +$ blockchain_import --database berkeley#fastest -## LMDB flags can be set by appending them to the database type: -## flags: nosync, nometasync, writemap, mapasync $ blockchain_import --database lmdb#nosync $ blockchain_import --database lmdb#nosync,nometasync + +$ blockchain_import --database berkeley#txn_nosync ``` + ### Blockchain converter with batching `blockchain_converter` has also been updated and includes batching for faster writes. However, on lower RAM systems, this will be slower than using the exporter and importer utilities. The converter needs to keep the blockchain in memory for the duration of the conversion, like the original bitmonerod, thus leaving less memory available to the destination database to operate. +Due to higher resource use, it is recommended to use the importer with an exported file instead of the converter. + ```bash $ blockchain_converter --batch on --batch-size 20000 ``` diff --git a/src/blockchain_utilities/blockchain_converter.cpp b/src/blockchain_utilities/blockchain_converter.cpp index fdd369e79..7c33ec399 100644 --- a/src/blockchain_utilities/blockchain_converter.cpp +++ b/src/blockchain_utilities/blockchain_converter.cpp @@ -45,8 +45,6 @@ #include <iostream> -unsigned int epee::g_test_dbg_lock_sleep = 0; - namespace { @@ -57,12 +55,12 @@ bool opt_testnet = false; // number of blocks per batch transaction // adjustable through command-line argument according to available RAM -#if !defined(WIN32) +#if ARCH_WIDTH != 32 uint64_t db_batch_size_verify = 5000; #else // set a lower default batch size for Windows, pending possible LMDB issue with // large batch size. -uint64_t db_batch_size_verify = 1000; +uint64_t db_batch_size_verify = 100; #endif // converter only uses verify mode diff --git a/src/blockchain_utilities/blockchain_dump.cpp b/src/blockchain_utilities/blockchain_dump.cpp index 53dc22f8b..6fa5ce801 100644 --- a/src/blockchain_utilities/blockchain_dump.cpp +++ b/src/blockchain_utilities/blockchain_dump.cpp @@ -38,8 +38,6 @@ #include "common/command_line.h" #include "version.h" -unsigned int epee::g_test_dbg_lock_sleep = 0; - namespace po = boost::program_options; using namespace epee; // log_space diff --git a/src/blockchain_utilities/blockchain_export.cpp b/src/blockchain_utilities/blockchain_export.cpp index f5dea731a..964c610cd 100644 --- a/src/blockchain_utilities/blockchain_export.cpp +++ b/src/blockchain_utilities/blockchain_export.cpp @@ -29,15 +29,42 @@ #include "bootstrap_file.h" #include "blocksdat_file.h" #include "common/command_line.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/lmdb/db_lmdb.h" +#if defined(BERKELEY_DB) +#include "blockchain_db/berkeleydb/db_bdb.h" +#endif +#include "blockchain_db/db_types.h" #include "version.h" -unsigned int epee::g_test_dbg_lock_sleep = 0; - namespace po = boost::program_options; using namespace epee; // log_space +std::string join_set_strings(const std::unordered_set<std::string>& db_types_all, const char* delim) +{ + std::string result; + std::ostringstream s; + std::copy(db_types_all.begin(), db_types_all.end(), std::ostream_iterator<std::string>(s, delim)); + result = s.str(); + if (result.length() > 0) + result.erase(result.end()-strlen(delim), result.end()); + return result; +} + int main(int argc, char* argv[]) { +#if defined(BLOCKCHAIN_DB) && (BLOCKCHAIN_DB == DB_MEMORY) + std::string default_db_type = "memory"; +#else + std::string default_db_type = "lmdb"; +#endif + + std::unordered_set<std::string> db_types_all = cryptonote::blockchain_db_types; + db_types_all.insert("memory"); + + std::string available_dbs = join_set_strings(db_types_all, ", "); + available_dbs = "available: " + available_dbs; + uint32_t log_level = 0; uint64_t block_stop = 0; bool blocks_dat = false; @@ -58,6 +85,9 @@ int main(int argc, char* argv[]) , "Run on testnet." , false }; + const command_line::arg_descriptor<std::string> arg_database = { + "database", available_dbs.c_str(), default_db_type + }; const command_line::arg_descriptor<bool> arg_blocks_dat = {"blocksdat", "Output in blocks.dat format", blocks_dat}; @@ -66,6 +96,7 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_cmd_sett, arg_output_file); command_line::add_arg(desc_cmd_sett, arg_testnet_on); command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_database); command_line::add_arg(desc_cmd_sett, arg_block_stop); command_line::add_arg(desc_cmd_sett, arg_blocks_dat); @@ -107,6 +138,20 @@ int main(int argc, char* argv[]) auto data_dir_arg = opt_testnet ? command_line::arg_testnet_data_dir : command_line::arg_data_dir; m_config_folder = command_line::get_arg(vm, data_dir_arg); + std::string db_type = command_line::get_arg(vm, arg_database); + if (db_types_all.count(db_type) == 0) + { + std::cerr << "Invalid database type: " << db_type << std::endl; + return 1; + } +#if !defined(BERKELEY_DB) + if (db_type == "berkeley") + { + LOG_ERROR("BerkeleyDB support disabled."); + return false; + } +#endif + if (command_line::has_arg(vm, arg_output_file)) output_file_path = boost::filesystem::path(command_line::get_arg(vm, arg_output_file)); else @@ -138,17 +183,33 @@ int main(int argc, char* argv[]) tx_memory_pool m_mempool(*core_storage); core_storage = new Blockchain(m_mempool); - BlockchainDB* db = new BlockchainLMDB(); + int db_flags = 0; + + BlockchainDB* db = nullptr; + if (db_type == "lmdb") + { + db_flags |= MDB_RDONLY; + db = new BlockchainLMDB(); + } +#if defined(BERKELEY_DB) + else if (db_type == "berkeley") + db = new BlockchainBDB(); +#endif + else + { + LOG_ERROR("Attempted to use non-existent database type: " << db_type); + throw std::runtime_error("Attempting to use non-existent database type"); + } + LOG_PRINT_L0("database: " << db_type); + boost::filesystem::path folder(m_config_folder); folder /= db->get_db_name(); - int lmdb_flags = 0; - lmdb_flags |= MDB_RDONLY; const std::string filename = folder.string(); LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); try { - db->open(filename, lmdb_flags); + db->open(filename, db_flags); } catch (const std::exception& e) { diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp index daa5db2a3..1aaf2bddc 100644 --- a/src/blockchain_utilities/blockchain_import.cpp +++ b/src/blockchain_utilities/blockchain_import.cpp @@ -44,8 +44,6 @@ #include "fake_core.h" -unsigned int epee::g_test_dbg_lock_sleep = 0; - namespace { // CONFIG @@ -56,11 +54,11 @@ bool opt_testnet = true; // number of blocks per batch transaction // adjustable through command-line argument according to available RAM -#if !defined(WIN32) +#if ARCH_WIDTH != 32 uint64_t db_batch_size = 20000; #else // set a lower default batch size, pending possible LMDB issue with large transaction size -uint64_t db_batch_size = 1000; +uint64_t db_batch_size = 100; #endif // when verifying, use a smaller default batch size so progress is more diff --git a/src/blockchain_utilities/bootstrap_file.h b/src/blockchain_utilities/bootstrap_file.h index 1e1c66ba9..dd134f7ae 100644 --- a/src/blockchain_utilities/bootstrap_file.h +++ b/src/blockchain_utilities/bootstrap_file.h @@ -37,8 +37,6 @@ #include "cryptonote_core/cryptonote_basic.h" #include "cryptonote_core/blockchain_storage.h" #include "cryptonote_core/blockchain.h" -#include "blockchain_db/blockchain_db.h" -#include "blockchain_db/lmdb/db_lmdb.h" #include <algorithm> #include <cstdio> diff --git a/src/blockchain_utilities/cn_deserialize.cpp b/src/blockchain_utilities/cn_deserialize.cpp index 6c6288aec..bf02dc150 100644 --- a/src/blockchain_utilities/cn_deserialize.cpp +++ b/src/blockchain_utilities/cn_deserialize.cpp @@ -32,8 +32,6 @@ #include "common/command_line.h" #include "version.h" -unsigned int epee::g_test_dbg_lock_sleep = 0; - namespace po = boost::program_options; using namespace epee; // log_space diff --git a/src/blockchain_utilities/fake_core.h b/src/blockchain_utilities/fake_core.h index 20a3361b6..2fb5031d3 100644 --- a/src/blockchain_utilities/fake_core.h +++ b/src/blockchain_utilities/fake_core.h @@ -33,7 +33,9 @@ #include "cryptonote_core/blockchain_storage.h" // in-memory DB #include "blockchain_db/blockchain_db.h" #include "blockchain_db/lmdb/db_lmdb.h" +#if defined(BERKELEY_DB) #include "blockchain_db/berkeleydb/db_bdb.h" +#endif using namespace cryptonote; diff --git a/src/common/dns_utils.cpp b/src/common/dns_utils.cpp index eb7b6608b..e6e53a5c0 100644 --- a/src/common/dns_utils.cpp +++ b/src/common/dns_utils.cpp @@ -38,7 +38,7 @@ using namespace epee; namespace bf = boost::filesystem; -static std::mutex instance_lock; +static boost::mutex instance_lock; namespace { @@ -181,6 +181,17 @@ struct DNSResolverData ub_ctx* m_ub_context; }; +// work around for bug https://www.nlnetlabs.nl/bugs-script/show_bug.cgi?id=515 needed for it to compile on e.g. Debian 7 +class string_copy { +public: + string_copy(const char *s): str(strdup(s)) {} + ~string_copy() { free(str); } + operator char*() { return str; } + +public: + char *str; +}; + DNSResolver::DNSResolver() : m_data(new DNSResolverData()) { int use_dns_public = 0; @@ -201,9 +212,9 @@ DNSResolver::DNSResolver() : m_data(new DNSResolverData()) if (use_dns_public) { - ub_ctx_set_fwd(m_data->m_ub_context, dns_public_addr); - ub_ctx_set_option(m_data->m_ub_context, "do-udp:", "no"); - ub_ctx_set_option(m_data->m_ub_context, "do-tcp:", "yes"); + ub_ctx_set_fwd(m_data->m_ub_context, string_copy(dns_public_addr)); + ub_ctx_set_option(m_data->m_ub_context, string_copy("do-udp:"), string_copy("no")); + ub_ctx_set_option(m_data->m_ub_context, string_copy("do-tcp:"), string_copy("yes")); } else { // look for "/etc/resolv.conf" and "/etc/hosts" or platform equivalent @@ -211,24 +222,7 @@ DNSResolver::DNSResolver() : m_data(new DNSResolverData()) ub_ctx_hosts(m_data->m_ub_context, NULL); } - #ifdef DEVELOPER_LIBUNBOUND_OLD - #pragma message "Using the work around for old libunbound" - { // work around for bug https://www.nlnetlabs.nl/bugs-script/show_bug.cgi?id=515 needed for it to compile on e.g. Debian 7 - char * ds_copy = NULL; // this will be the writable copy of string that bugged version of libunbound requires - try { - char * ds_copy = strdup( ::get_builtin_ds() ); - ub_ctx_add_ta(m_data->m_ub_context, ds_copy); - } catch(...) { // probably not needed but to work correctly in every case... - if (ds_copy) { free(ds_copy); ds_copy=NULL; } // for the strdup - throw ; - } - if (ds_copy) { free(ds_copy); ds_copy=NULL; } // for the strdup - } - #else - // normal version for fixed libunbound - ub_ctx_add_ta(m_data->m_ub_context, ::get_builtin_ds() ); - #endif - + ub_ctx_add_ta(m_data->m_ub_context, string_copy(::get_builtin_ds())); } DNSResolver::~DNSResolver() @@ -258,7 +252,7 @@ std::vector<std::string> DNSResolver::get_record(const std::string& url, int rec ub_result_ptr result; // call DNS resolver, blocking. if return value not zero, something went wrong - if (!ub_resolve(m_data->m_ub_context, url.c_str(), record_type, DNS_CLASS_IN, &result)) + if (!ub_resolve(m_data->m_ub_context, string_copy(url.c_str()), record_type, DNS_CLASS_IN, &result)) { dnssec_available = (result->secure || (!result->secure && result->bogus)); dnssec_valid = result->secure && !result->bogus; @@ -304,7 +298,7 @@ std::string DNSResolver::get_dns_format_from_oa_address(const std::string& oa_ad DNSResolver& DNSResolver::instance() { - std::lock_guard<std::mutex> lock(instance_lock); + boost::lock_guard<boost::mutex> lock(instance_lock); static DNSResolver* staticInstance = NULL; if (staticInstance == NULL) diff --git a/src/common/util.h b/src/common/util.h index de0244c7f..7554b1df7 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -150,8 +150,8 @@ namespace tools /*! \brief calles m_handler */ static void handle_signal(int type) { - static std::mutex m_mutex; - std::unique_lock<std::mutex> lock(m_mutex); + static boost::mutex m_mutex; + boost::unique_lock<boost::mutex> lock(m_mutex); m_handler(type); } diff --git a/src/connectivity_tool/CMakeLists.txt b/src/connectivity_tool/CMakeLists.txt index 223d6e67f..0ade952d2 100644 --- a/src/connectivity_tool/CMakeLists.txt +++ b/src/connectivity_tool/CMakeLists.txt @@ -42,4 +42,5 @@ target_link_libraries(connectivity_tool ${CMAKE_THREAD_LIBS_INIT} ${Boost_PROGRAM_OPTIONS_LIBRARY} ${Boost_REGEX_LIBRARY} + ${Boost_CHRONO_LIBRARY} ${Boost_SYSTEM_LIBRARY}) diff --git a/src/connectivity_tool/conn_tool.cpp b/src/connectivity_tool/conn_tool.cpp index 9f00300ae..458d30cc3 100644 --- a/src/connectivity_tool/conn_tool.cpp +++ b/src/connectivity_tool/conn_tool.cpp @@ -49,8 +49,6 @@ namespace po = boost::program_options; using namespace cryptonote; using namespace nodetool; -unsigned int epee::g_test_dbg_lock_sleep = 0; - namespace { const command_line::arg_descriptor<std::string, true> arg_ip = {"ip", "set ip"}; diff --git a/src/crypto/crypto.cpp b/src/crypto/crypto.cpp index fa7b1b580..e47aab0f7 100644 --- a/src/crypto/crypto.cpp +++ b/src/crypto/crypto.cpp @@ -34,7 +34,8 @@ #include <cstdlib> #include <cstring> #include <memory> -#include <mutex> +#include <boost/thread/mutex.hpp> +#include <boost/thread/lock_guard.hpp> #include "common/varint.h" #include "warnings.h" @@ -52,8 +53,6 @@ namespace crypto { using std::abort; using std::int32_t; using std::int64_t; - using std::lock_guard; - using std::mutex; using std::size_t; using std::uint32_t; using std::uint64_t; @@ -63,7 +62,7 @@ namespace crypto { #include "random.h" } - mutex random_lock; + boost::mutex random_lock; static inline unsigned char *operator &(ec_point &point) { return &reinterpret_cast<unsigned char &>(point); @@ -100,7 +99,7 @@ namespace crypto { * */ secret_key crypto_ops::generate_keys(public_key &pub, secret_key &sec, const secret_key& recovery_key, bool recover) { - lock_guard<mutex> lock(random_lock); + boost::lock_guard<boost::mutex> lock(random_lock); ge_p3 point; secret_key rng; @@ -199,7 +198,7 @@ namespace crypto { }; void crypto_ops::generate_signature(const hash &prefix_hash, const public_key &pub, const secret_key &sec, signature &sig) { - lock_guard<mutex> lock(random_lock); + boost::lock_guard<boost::mutex> lock(random_lock); ge_p3 tmp3; ec_scalar k; s_comm buf; @@ -280,7 +279,7 @@ POP_WARNINGS const public_key *const *pubs, size_t pubs_count, const secret_key &sec, size_t sec_index, signature *sig) { - lock_guard<mutex> lock(random_lock); + boost::lock_guard<boost::mutex> lock(random_lock); size_t i; ge_p3 image_unp; ge_dsmp image_pre; diff --git a/src/crypto/crypto.h b/src/crypto/crypto.h index 360b571f3..883aa521a 100644 --- a/src/crypto/crypto.h +++ b/src/crypto/crypto.h @@ -31,7 +31,8 @@ #pragma once #include <cstddef> -#include <mutex> +#include <boost/thread/mutex.hpp> +#include <boost/thread/lock_guard.hpp> #include <vector> #include "common/pod-class.h" @@ -44,7 +45,7 @@ namespace crypto { #include "random.h" } - extern std::mutex random_lock; + extern boost::mutex random_lock; #pragma pack(push, 1) POD_CLASS ec_point { @@ -121,7 +122,7 @@ namespace crypto { template<typename T> typename std::enable_if<std::is_pod<T>::value, T>::type rand() { typename std::remove_cv<T>::type res; - std::lock_guard<std::mutex> lock(random_lock); + boost::lock_guard<boost::mutex> lock(random_lock); generate_random_bytes(sizeof(T), &res); return res; } diff --git a/src/cryptonote_core/account.cpp b/src/cryptonote_core/account.cpp index 0d2175ed7..c3f2b4446 100644 --- a/src/cryptonote_core/account.cpp +++ b/src/cryptonote_core/account.cpp @@ -93,9 +93,10 @@ DISABLE_VS_WARNINGS(4244 4345) return first; } //----------------------------------------------------------------- - void account_base::create_from_viewkey(const cryptonote::account_public_address& address, const crypto::secret_key& viewkey) + void account_base::create_from_keys(const cryptonote::account_public_address& address, const crypto::secret_key& spendkey, const crypto::secret_key& viewkey) { m_keys.m_account_address = address; + m_keys.m_spend_secret_key = spendkey; m_keys.m_view_secret_key = viewkey; struct tm timestamp; @@ -109,6 +110,13 @@ DISABLE_VS_WARNINGS(4244 4345) m_creation_timestamp = mktime(×tamp); } //----------------------------------------------------------------- + void account_base::create_from_viewkey(const cryptonote::account_public_address& address, const crypto::secret_key& viewkey) + { + crypto::secret_key fake; + memset(&fake, 0, sizeof(fake)); + create_from_keys(address, fake, viewkey); + } + //----------------------------------------------------------------- const account_keys& account_base::get_keys() const { return m_keys; diff --git a/src/cryptonote_core/account.h b/src/cryptonote_core/account.h index 38f61ebd4..41a119b07 100644 --- a/src/cryptonote_core/account.h +++ b/src/cryptonote_core/account.h @@ -58,6 +58,7 @@ namespace cryptonote public: account_base(); crypto::secret_key generate(const crypto::secret_key& recovery_key = crypto::secret_key(), bool recover = false, bool two_random = false); + void create_from_keys(const cryptonote::account_public_address& address, const crypto::secret_key& spendkey, const crypto::secret_key& viewkey); void create_from_viewkey(const cryptonote::account_public_address& address, const crypto::secret_key& viewkey); const account_keys& get_keys() const; std::string get_public_address_str(bool testnet) const; diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index a3eb21187..f540697dc 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -101,7 +101,7 @@ static const uint64_t testnet_hard_fork_version_1_till = 624633; //------------------------------------------------------------------ Blockchain::Blockchain(tx_memory_pool& tx_pool) : m_db(), m_tx_pool(tx_pool), m_hardfork(NULL), m_timestamps_and_difficulties_height(0), m_current_block_cumul_sz_limit(0), m_is_in_checkpoint_zone(false), - m_is_blockchain_storing(false), m_enforce_dns_checkpoints(false), m_max_prepare_blocks_threads(4), m_db_blocks_per_sync(1), m_db_sync_mode(db_async), m_fast_sync(true), m_sync_counter(0) + m_is_blockchain_storing(false), m_enforce_dns_checkpoints(false), m_max_prepare_blocks_threads(4), m_db_blocks_per_sync(1), m_db_sync_mode(db_async), m_fast_sync(true), m_show_time_stats(false), m_sync_counter(0) { LOG_PRINT_L3("Blockchain::" << __func__); } @@ -334,6 +334,7 @@ bool Blockchain::init(BlockchainDB* db, const bool testnet, const cryptonote::te m_db->fixup(); } + m_db->block_txn_start(true); // check how far behind we are uint64_t top_block_timestamp = m_db->get_top_block_timestamp(); uint64_t timestamp_diff = time(NULL) - top_block_timestamp; @@ -354,6 +355,7 @@ bool Blockchain::init(BlockchainDB* db, const bool testnet, const cryptonote::te #endif LOG_PRINT_GREEN("Blockchain initialized. last block: " << m_db->height() - 1 << ", " << epee::misc_utils::get_time_interval_string(timestamp_diff) << " time ago, current difficulty: " << get_difficulty_for_next_block(), LOG_LEVEL_0); + m_db->block_txn_stop(); return true; } @@ -549,6 +551,7 @@ bool Blockchain::get_short_chain_history(std::list<crypto::hash>& ids) const if(!sz) return true; + m_db->block_txn_start(true); bool genesis_included = false; uint64_t current_back_offset = 1; while(current_back_offset < sz) @@ -575,6 +578,7 @@ bool Blockchain::get_short_chain_history(std::list<crypto::hash>& ids) const { ids.push_back(m_db->get_block_hash_from_height(0)); } + m_db->block_txn_stop(); return true; } @@ -996,12 +1000,14 @@ void Blockchain::get_last_n_blocks_sizes(std::vector<size_t>& sz, size_t count) if(h == 0) return; + m_db->block_txn_start(true); // add size of last <count> blocks to vector <sz> (or less, if blockchain size < count) size_t start_offset = h - std::min<size_t>(h, count); for(size_t i = start_offset; i < h; i++) { sz.push_back(m_db->get_block_size(i)); } + m_db->block_txn_stop(); } //------------------------------------------------------------------ uint64_t Blockchain::get_current_cumulative_blocksize_limit() const @@ -1403,6 +1409,7 @@ bool Blockchain::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NO { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); + m_db->block_txn_start(true); rsp.current_blockchain_height = get_current_blockchain_height(); std::list<block> blocks; get_blocks(arg.blocks, blocks, rsp.missed_ids); @@ -1424,6 +1431,7 @@ bool Blockchain::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NO // as done below if any standalone transactions were requested // and missed. rsp.missed_ids.splice(rsp.missed_ids.end(), missed_tx_ids); + m_db->block_txn_stop(); return false; } @@ -1442,6 +1450,7 @@ bool Blockchain::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NO for (const auto& tx: txs) rsp.txs.push_back(t_serializable_object_to_blob(tx)); + m_db->block_txn_stop(); return true; } //------------------------------------------------------------------ @@ -1585,12 +1594,14 @@ bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qbloc return false; } + m_db->block_txn_start(true); // make sure that the last block in the request's block list matches // the genesis block auto gen_hash = m_db->get_block_hash_from_height(0); if(qblock_ids.back() != gen_hash) { LOG_PRINT_L1("Client sent wrong NOTIFY_REQUEST_CHAIN: genesis block missmatch: " << std::endl << "id: " << qblock_ids.back() << ", " << std::endl << "expected: " << gen_hash << "," << std::endl << " dropping connection"); + m_db->block_txn_abort(); return false; } @@ -1612,9 +1623,11 @@ bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qbloc catch (const std::exception& e) { LOG_PRINT_L1("Non-critical error trying to find block by hash in BlockchainDB, hash: " << *bl_it); + m_db->block_txn_abort(); return false; } } + m_db->block_txn_stop(); // this should be impossible, as we checked that we share the genesis block, // but just in case... @@ -2092,7 +2105,7 @@ bool Blockchain::check_tx_inputs(const transaction& tx, uint64_t* pmax_used_bloc std::vector < uint64_t > results; results.resize(tx.vin.size(), 0); - int threads = std::thread::hardware_concurrency(); + int threads = boost::thread::hardware_concurrency(); boost::asio::io_service ioservice; boost::thread_group threadpool; @@ -2427,9 +2440,12 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& CRITICAL_REGION_LOCAL(m_blockchain_lock); TIME_MEASURE_START(t1); + m_db->block_txn_start(true); if(bl.prev_id != get_tail_id()) { LOG_PRINT_L1("Block with id: " << id << std::endl << "has wrong prev_id: " << bl.prev_id << std::endl << "expected: " << get_tail_id()); +leave: + m_db->block_txn_stop(); return false; } @@ -2438,7 +2454,7 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& { LOG_PRINT_L1("Block with id: " << id << std::endl << "has old version: " << (unsigned)bl.major_version << std::endl << "current: " << (unsigned)m_hardfork->get_current_version()); bvc.m_verifivation_failed = true; - return false; + goto leave; } TIME_MEASURE_FINISH(t1); @@ -2450,7 +2466,7 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& { LOG_PRINT_L1("Block with id: " << id << std::endl << "has invalid timestamp: " << bl.timestamp); bvc.m_verifivation_failed = true; - return false; + goto leave; } TIME_MEASURE_FINISH(t2); @@ -2490,7 +2506,7 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& { LOG_PRINT_L1("Block with id is INVALID: " << id); bvc.m_verifivation_failed = true; - return false; + goto leave; } fast_check = true; } @@ -2511,7 +2527,7 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& { LOG_PRINT_L1("Block with id: " << id << std::endl << "does not have enough proof of work: " << proof_of_work << std::endl << "unexpected difficulty: " << current_diffic); bvc.m_verifivation_failed = true; - return false; + goto leave; } } @@ -2523,7 +2539,7 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& { LOG_ERROR("CHECKPOINT VALIDATION FAILED"); bvc.m_verifivation_failed = true; - return false; + goto leave; } } @@ -2538,7 +2554,7 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& { LOG_PRINT_L1("Block with id: " << id << " failed to pass prevalidation"); bvc.m_verifivation_failed = true; - return false; + goto leave; } size_t coinbase_blob_size = get_object_blobsize(bl.miner_tx); @@ -2574,7 +2590,7 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& LOG_PRINT_L1("Block with id: " << id << " attempting to add transaction already in blockchain with id: " << tx_id); bvc.m_verifivation_failed = true; return_tx_to_pool(txs); - return false; + goto leave; } TIME_MEASURE_FINISH(aa); @@ -2587,7 +2603,7 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& LOG_PRINT_L1("Block with id: " << id << " has at least one unknown transaction with id: " << tx_id); bvc.m_verifivation_failed = true; return_tx_to_pool(txs); - return false; + goto leave; } TIME_MEASURE_FINISH(bb); @@ -2624,7 +2640,7 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& LOG_PRINT_L1("Block with id " << id << " added as invalid because of wrong inputs in transactions"); bvc.m_verifivation_failed = true; return_tx_to_pool(txs); - return false; + goto leave; } } #if defined(PER_BLOCK_CHECKPOINT) @@ -2640,7 +2656,7 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& LOG_PRINT_L1("Block with id " << id << " added as invalid because of wrong inputs in transactions"); bvc.m_verifivation_failed = true; return_tx_to_pool(txs); - return false; + goto leave; } } #endif @@ -2660,7 +2676,7 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& LOG_PRINT_L1("Block with id: " << id << " has incorrect miner transaction"); bvc.m_verifivation_failed = true; return_tx_to_pool(txs); - return false; + goto leave; } TIME_MEASURE_FINISH(vmt); @@ -2678,6 +2694,7 @@ bool Blockchain::handle_block_to_main_chain(const block& bl, const crypto::hash& if(precomputed) block_processing_time += m_fake_pow_calc_time; + m_db->block_txn_stop(); TIME_MEASURE_START(addblock); uint64_t new_height = 0; if (!bvc.m_verifivation_failed) @@ -2754,10 +2771,12 @@ bool Blockchain::add_new_block(const block& bl_, block_verification_context& bvc crypto::hash id = get_block_hash(bl); CRITICAL_REGION_LOCAL(m_tx_pool);//to avoid deadlock lets lock tx_pool for whole add/reorganize process CRITICAL_REGION_LOCAL1(m_blockchain_lock); + m_db->block_txn_start(true); if(have_block(id)) { LOG_PRINT_L3("block with id = " << id << " already exists"); bvc.m_already_exists = true; + m_db->block_txn_stop(); return false; } @@ -2766,10 +2785,12 @@ bool Blockchain::add_new_block(const block& bl_, block_verification_context& bvc { //chain switching or wrong block bvc.m_added_to_main_chain = false; + m_db->block_txn_stop(); return handle_alternative_block(bl, id, bvc); //never relay alternative blocks } + m_db->block_txn_stop(); return handle_block_to_main_chain(bl, id, bvc); } //------------------------------------------------------------------ @@ -2777,6 +2798,7 @@ void Blockchain::check_against_checkpoints(const checkpoints& points, bool enfor { const auto& pts = points.get_points(); + m_db->batch_start(); for (const auto& pt : pts) { // if the checkpoint is for a block we don't have yet, move on @@ -2800,6 +2822,7 @@ void Blockchain::check_against_checkpoints(const checkpoints& points, bool enfor } } } + m_db->batch_stop(); } //------------------------------------------------------------------ // returns false if any of the checkpoints loading returns false. @@ -2942,7 +2965,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::list<block_complete_e return true; bool blocks_exist = false; - uint64_t threads = std::thread::hardware_concurrency(); + uint64_t threads = boost::thread::hardware_concurrency(); if (blocks_entry.size() > 1 && threads > 1 && m_max_prepare_blocks_threads > 1) { @@ -3142,7 +3165,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::list<block_complete_e // [output] stores all transactions for each tx_out_index::hash found std::vector<std::unordered_map<crypto::hash, cryptonote::transaction>> transactions(amounts.size()); - threads = std::thread::hardware_concurrency(); + threads = boost::thread::hardware_concurrency(); if (!m_db->can_thread_bulk_indices()) threads = 1; diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 2c485d791..6f0fe88a4 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -943,14 +943,14 @@ namespace cryptonote HardFork::State state = m_blockchain_storage.get_hard_fork_state(); switch (state) { case HardFork::LikelyForked: - LOG_PRINT_L0(ENDL + LOG_PRINT_RED_L0(ENDL << "**********************************************************************" << ENDL << "Last scheduled hard fork is too far in the past." << ENDL << "We are most likely forked from the network. Daemon update needed now." << ENDL << "**********************************************************************" << ENDL); break; case HardFork::UpdateNeeded: - LOG_PRINT_L0(ENDL + LOG_PRINT_RED_L0(ENDL << "**********************************************************************" << ENDL << "Last scheduled hard fork time shows a daemon update is needed now." << ENDL << "**********************************************************************" << ENDL); diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 3c6f56749..32f0b2ad4 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -198,7 +198,7 @@ namespace cryptonote std::string m_config_folder; cryptonote_protocol_stub m_protocol_stub; epee::math_helper::once_a_time_seconds<60*60*12, false> m_store_blockchain_interval; - epee::math_helper::once_a_time_seconds<60*60*2, false> m_fork_moaner; + epee::math_helper::once_a_time_seconds<60*60*2, true> m_fork_moaner; epee::math_helper::once_a_time_seconds<60*2, false> m_txpool_auto_relayer; //!< interval for checking re-relaying txpool transactions friend class tx_validate_inputs; std::atomic<bool> m_starter_message_showed; diff --git a/src/cryptonote_core/hardfork.cpp b/src/cryptonote_core/hardfork.cpp index e99736ff4..7e2e82c4a 100644 --- a/src/cryptonote_core/hardfork.cpp +++ b/src/cryptonote_core/hardfork.cpp @@ -254,8 +254,11 @@ bool HardFork::reorganize_from_chain_height(uint64_t height) bool HardFork::rescan_from_block_height(uint64_t height) { CRITICAL_REGION_LOCAL(lock); - if (height >= db.height()) + db.block_txn_start(true); + if (height >= db.height()) { + db.block_txn_stop(); return false; + } versions.clear(); @@ -273,6 +276,7 @@ bool HardFork::rescan_from_block_height(uint64_t height) current_fork_index = 0; while (current_fork_index + 1 < heights.size() && heights[current_fork_index].version != lastv) ++current_fork_index; + db.block_txn_stop(); return true; } diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.h b/src/cryptonote_protocol/cryptonote_protocol_handler.h index c2d428025..1c92958c9 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.h @@ -134,7 +134,7 @@ namespace cryptonote bool m_one_request = true; // static std::ofstream m_logreq; - std::mutex m_buffer_mutex; + boost::mutex m_buffer_mutex; double get_avg_block_size(); boost::circular_buffer<size_t> m_avg_buffer = boost::circular_buffer<size_t>(10); diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index ed6a66722..e79823d08 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -46,8 +46,6 @@ using namespace epee; #include <functional> -unsigned int epee::g_test_dbg_lock_sleep = 0; - namespace daemonize { struct t_internals { diff --git a/src/daemon/main.cpp b/src/daemon/main.cpp index 089d8fb69..0717fd89b 100644 --- a/src/daemon/main.cpp +++ b/src/daemon/main.cpp @@ -132,7 +132,7 @@ int main(int argc, char const * argv[]) return 0; } - epee::g_test_dbg_lock_sleep = command_line::get_arg(vm, command_line::arg_test_dbg_lock_sleep); + epee::debug::g_test_dbg_lock_sleep() = command_line::get_arg(vm, command_line::arg_test_dbg_lock_sleep); std::string db_type = command_line::get_arg(vm, command_line::arg_db_type); diff --git a/src/daemonizer/windows_service.cpp b/src/daemonizer/windows_service.cpp index 88f05a305..f8cc0c6c7 100644 --- a/src/daemonizer/windows_service.cpp +++ b/src/daemonizer/windows_service.cpp @@ -99,8 +99,8 @@ namespace { // to allow the user to read any output. void pause_to_display_admin_window_messages() { - std::chrono::milliseconds how_long{1500}; - std::this_thread::sleep_for(how_long); + boost::chrono::milliseconds how_long{1500}; + boost::this_thread::sleep_for(how_long); } } diff --git a/src/daemonizer/windows_service_runner.h b/src/daemonizer/windows_service_runner.h index 2d21509ec..f4258a215 100644 --- a/src/daemonizer/windows_service_runner.h +++ b/src/daemonizer/windows_service_runner.h @@ -56,7 +56,7 @@ namespace windows { private: SERVICE_STATUS_HANDLE m_status_handle{nullptr}; SERVICE_STATUS m_status{}; - std::mutex m_lock{}; + boost::mutex m_lock{}; std::string m_name; T_handler m_handler; diff --git a/src/miner/CMakeLists.txt b/src/miner/CMakeLists.txt index b09ad47aa..d065b9950 100644 --- a/src/miner/CMakeLists.txt +++ b/src/miner/CMakeLists.txt @@ -49,6 +49,7 @@ target_link_libraries(simpleminer ${Boost_FILESYSTEM_LIBRARY} ${Boost_PROGRAM_OPTIONS_LIBRARY} ${Boost_REGEX_LIBRARY} + ${Boost_CHRONO_LIBRARY} ${Boost_SYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} diff --git a/src/miner/simpleminer.cpp b/src/miner/simpleminer.cpp index e04ffbbcf..ba956d90b 100644 --- a/src/miner/simpleminer.cpp +++ b/src/miner/simpleminer.cpp @@ -41,7 +41,6 @@ using namespace epee; namespace po = boost::program_options; -unsigned int epee::g_test_dbg_lock_sleep = 0; int main(int argc, char** argv) { diff --git a/src/p2p/data_logger.cpp b/src/p2p/data_logger.cpp index f875cb8f0..ca0726c5f 100644 --- a/src/p2p/data_logger.cpp +++ b/src/p2p/data_logger.cpp @@ -31,6 +31,7 @@ #include <boost/chrono.hpp> #include <boost/filesystem.hpp> +#include <boost/thread.hpp> #include <chrono> #include "../../contrib/otshell_utils/utils.hpp" @@ -39,7 +40,7 @@ namespace epee namespace net_utils { data_logger &data_logger::get_instance() { - std::call_once(m_singleton, + boost::call_once(m_singleton, [] { _info_c("dbg/data","Creating singleton of data_logger"); if (m_state != data_logger_state::state_before_init) { _erro_c("dbg/data","Internal error in singleton"); throw std::runtime_error("data_logger singleton"); } @@ -60,7 +61,7 @@ namespace net_utils data_logger::data_logger() { _note_c("dbg/data","Starting data logger (for graphs data)"); if (m_state != data_logger_state::state_during_init) { _erro_c("dbg/data","Singleton ctor state"); throw std::runtime_error("data_logger ctor state"); } - std::lock_guard<std::mutex> lock(mMutex); // lock + boost::lock_guard<boost::mutex> lock(mMutex); // lock // prepare all the files for given data channels: mFilesMap["peers"] = data_logger::fileData("log/dr-monero/peers.data"); @@ -85,14 +86,14 @@ namespace net_utils _info_c("dbg/data","Creating thread for data logger"); // create timer thread m_thread_maybe_running=true; - std::shared_ptr<std::thread> logger_thread(new std::thread([&]() { + std::shared_ptr<boost::thread> logger_thread(new boost::thread([&]() { _info_c("dbg/data","Inside thread for data logger"); while (m_state == data_logger_state::state_during_init) { // wait for creation to be done (in other thread, in singleton) before actually running - std::this_thread::sleep_for(std::chrono::seconds(1)); + boost::this_thread::sleep_for(boost::chrono::seconds(1)); } _info_c("dbg/data","Inside thread for data logger - going into main loop"); while (m_state == data_logger_state::state_ready_to_use) { // run as long as we are not closing the single object - std::this_thread::sleep_for(std::chrono::seconds(1)); + boost::this_thread::sleep_for(boost::chrono::seconds(1)); saveToFile(); // save all the pending data } _info_c("dbg/data","Inside thread for data logger - done the main loop"); @@ -105,12 +106,12 @@ namespace net_utils data_logger::~data_logger() { _note_c("dbg/data","Destructor of the data logger"); { - std::lock_guard<std::mutex> lock(mMutex); + boost::lock_guard<boost::mutex> lock(mMutex); m_state = data_logger_state::state_dying; } _info_c("dbg/data","State was set to dying"); while(m_thread_maybe_running) { // wait for the thread to exit - std::this_thread::sleep_for(std::chrono::seconds(1)); + boost::this_thread::sleep_for(boost::chrono::seconds(1)); _info_c("dbg/data","Waiting for background thread to exit"); } _info_c("dbg/data","Thread exited"); @@ -122,7 +123,7 @@ namespace net_utils } void data_logger::add_data(std::string filename, unsigned int data) { - std::lock_guard<std::mutex> lock(mMutex); + boost::lock_guard<boost::mutex> lock(mMutex); if (m_state != data_logger_state::state_ready_to_use) { _info_c("dbg/data","Data logger is not ready, returning."); return; } if (mFilesMap.find(filename) == mFilesMap.end()) { // no such file/counter @@ -150,7 +151,7 @@ namespace net_utils void data_logger::saveToFile() { _dbg2_c("dbg/data","saving to files"); - std::lock_guard<std::mutex> lock(mMutex); + boost::lock_guard<boost::mutex> lock(mMutex); if (m_state != data_logger_state::state_ready_to_use) { _info_c("dbg/data","Data logger is not ready, returning."); return; } nOT::nUtils::cFilesystemUtils::CreateDirTree("log/dr-monero/net/"); for (auto &element : mFilesMap) @@ -193,7 +194,7 @@ namespace net_utils data_logger_state data_logger::m_state(data_logger_state::state_before_init); ///< (static) state of the singleton object std::atomic<bool> data_logger::m_save_graph(false); // (static) std::atomic<bool> data_logger::m_thread_maybe_running(false); // (static) -std::once_flag data_logger::m_singleton; // (static) +boost::once_flag data_logger::m_singleton; // (static) std::unique_ptr<data_logger> data_logger::m_obj; // (static) } // namespace diff --git a/src/p2p/data_logger.hpp b/src/p2p/data_logger.hpp index affc97f5f..148afc0ab 100644 --- a/src/p2p/data_logger.hpp +++ b/src/p2p/data_logger.hpp @@ -33,8 +33,9 @@ #include <map> #include <fstream> #include <memory> -#include <thread> -#include <mutex> +#include <boost/thread/thread.hpp> +#include <boost/thread/mutex.hpp> +#include <boost/thread/once.hpp> #include <atomic> namespace epee @@ -71,7 +72,7 @@ enum class data_logger_state { state_before_init, state_during_init, state_ready static bool is_dying(); private: - static std::once_flag m_singleton; ///< to guarantee singleton creates the object exactly once + static boost::once_flag m_singleton; ///< to guarantee singleton creates the object exactly once static data_logger_state m_state; ///< state of the singleton object static std::atomic<bool> m_thread_maybe_running; ///< is the background thread (more or less) running, or is it fully finished static std::unique_ptr<data_logger> m_obj; ///< the singleton object. Only use it via get_instance(). Can be killed by kill_instance() @@ -94,7 +95,7 @@ enum class data_logger_state { state_before_init, state_during_init, state_ready }; std::map<std::string, fileData> mFilesMap; - std::mutex mMutex; + boost::mutex mMutex; void saveToFile(); ///< write data to the target files. do not use this directly }; diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index 2ae849e2a..260dd813d 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -270,7 +270,7 @@ namespace nodetool bool m_offline; std::atomic<bool> m_save_graph; std::atomic<bool> is_closing; - std::unique_ptr<std::thread> mPeersLoggerThread; + std::unique_ptr<boost::thread> mPeersLoggerThread; //critical_section m_connections_lock; //connections_indexed_container m_connections; diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index 47a5dc6c3..6278db891 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -567,7 +567,7 @@ namespace nodetool bool node_server<t_payload_net_handler>::run() { // creating thread to log number of connections - mPeersLoggerThread.reset(new std::thread([&]() + mPeersLoggerThread.reset(new boost::thread([&]() { _note("Thread monitor number of peers - start"); while (!is_closing && !m_net_server.is_stop_signal_sent()) @@ -585,7 +585,7 @@ namespace nodetool break; epee::net_utils::data_logger::get_instance().add_data("peers", number_of_peers); - std::this_thread::sleep_for(std::chrono::seconds(1)); + boost::this_thread::sleep_for(boost::chrono::seconds(1)); } // main loop of thread _note("Thread monitor number of peers - done"); })); // lambda diff --git a/src/p2p/network_throttle-detail.cpp b/src/p2p/network_throttle-detail.cpp index 54c82dd8d..ed3c8e7b4 100644 --- a/src/p2p/network_throttle-detail.cpp +++ b/src/p2p/network_throttle-detail.cpp @@ -243,7 +243,7 @@ network_time_seconds network_throttle::get_sleep_time_after_tick(size_t packet_s void network_throttle::logger_handle_net(const std::string &filename, double time, size_t size) { if (! epee::net_utils::data_logger::m_save_graph) return; - std::mutex mutex; + boost::mutex mutex; mutex.lock(); { std::fstream file; file.open(filename.c_str(), std::ios::app | std::ios::out ); diff --git a/src/p2p/network_throttle.cpp b/src/p2p/network_throttle.cpp index 42f54964b..30538bb3c 100644 --- a/src/p2p/network_throttle.cpp +++ b/src/p2p/network_throttle.cpp @@ -67,9 +67,9 @@ namespace net_utils // ================================================================================================ // static: -std::mutex network_throttle_manager::m_lock_get_global_throttle_in; -std::mutex network_throttle_manager::m_lock_get_global_throttle_inreq; -std::mutex network_throttle_manager::m_lock_get_global_throttle_out; +boost::mutex network_throttle_manager::m_lock_get_global_throttle_in; +boost::mutex network_throttle_manager::m_lock_get_global_throttle_inreq; +boost::mutex network_throttle_manager::m_lock_get_global_throttle_out; int network_throttle_manager::xxx; @@ -77,27 +77,27 @@ int network_throttle_manager::xxx; // ================================================================================================ // methods: i_network_throttle & network_throttle_manager::get_global_throttle_in() { - std::call_once(m_once_get_global_throttle_in, [] { m_obj_get_global_throttle_in.reset(new network_throttle("in/all","<<< global-IN",10)); } ); + boost::call_once(m_once_get_global_throttle_in, [] { m_obj_get_global_throttle_in.reset(new network_throttle("in/all","<<< global-IN",10)); } ); return * m_obj_get_global_throttle_in; } -std::once_flag network_throttle_manager::m_once_get_global_throttle_in; +boost::once_flag network_throttle_manager::m_once_get_global_throttle_in; std::unique_ptr<i_network_throttle> network_throttle_manager::m_obj_get_global_throttle_in; i_network_throttle & network_throttle_manager::get_global_throttle_inreq() { - std::call_once(m_once_get_global_throttle_inreq, [] { m_obj_get_global_throttle_inreq.reset(new network_throttle("inreq/all", "<== global-IN-REQ",10)); } ); + boost::call_once(m_once_get_global_throttle_inreq, [] { m_obj_get_global_throttle_inreq.reset(new network_throttle("inreq/all", "<== global-IN-REQ",10)); } ); return * m_obj_get_global_throttle_inreq; } -std::once_flag network_throttle_manager::m_once_get_global_throttle_inreq; +boost::once_flag network_throttle_manager::m_once_get_global_throttle_inreq; std::unique_ptr<i_network_throttle> network_throttle_manager::m_obj_get_global_throttle_inreq; i_network_throttle & network_throttle_manager::get_global_throttle_out() { - std::call_once(m_once_get_global_throttle_out, [] { m_obj_get_global_throttle_out.reset(new network_throttle("out/all", ">>> global-OUT",10)); } ); + boost::call_once(m_once_get_global_throttle_out, [] { m_obj_get_global_throttle_out.reset(new network_throttle("out/all", ">>> global-OUT",10)); } ); return * m_obj_get_global_throttle_out; } -std::once_flag network_throttle_manager::m_once_get_global_throttle_out; +boost::once_flag network_throttle_manager::m_once_get_global_throttle_out; std::unique_ptr<i_network_throttle> network_throttle_manager::m_obj_get_global_throttle_out; diff --git a/src/p2p/network_throttle.hpp b/src/p2p/network_throttle.hpp index 6135ed72b..b954c5b3a 100644 --- a/src/p2p/network_throttle.hpp +++ b/src/p2p/network_throttle.hpp @@ -113,16 +113,16 @@ class network_throttle_manager { //protected: public: // XXX // [[note1]] - static std::once_flag m_once_get_global_throttle_in; - static std::once_flag m_once_get_global_throttle_inreq; // [[note2]] - static std::once_flag m_once_get_global_throttle_out; + static boost::once_flag m_once_get_global_throttle_in; + static boost::once_flag m_once_get_global_throttle_inreq; // [[note2]] + static boost::once_flag m_once_get_global_throttle_out; static std::unique_ptr<i_network_throttle> m_obj_get_global_throttle_in; static std::unique_ptr<i_network_throttle> m_obj_get_global_throttle_inreq; static std::unique_ptr<i_network_throttle> m_obj_get_global_throttle_out; - static std::mutex m_lock_get_global_throttle_in; - static std::mutex m_lock_get_global_throttle_inreq; - static std::mutex m_lock_get_global_throttle_out; + static boost::mutex m_lock_get_global_throttle_in; + static boost::mutex m_lock_get_global_throttle_inreq; + static boost::mutex m_lock_get_global_throttle_out; friend class cryptonote::cryptonote_protocol_handler_base; // FRIEND - to directly access global throttle-s. !! REMEMBER TO USE LOCKS! friend class connection_basic; // FRIEND - to directly access global throttle-s. !! REMEMBER TO USE LOCKS! diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index f4dcb6a35..3e5cdfcb6 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -73,15 +73,14 @@ typedef cryptonote::simple_wallet sw; #define EXTENDED_LOGS_FILE "wallet_details.log" -unsigned int epee::g_test_dbg_lock_sleep = 0; - #define DEFAULT_MIX 4 namespace { const command_line::arg_descriptor<std::string> arg_wallet_file = {"wallet-file", sw::tr("Use wallet <arg>"), ""}; const command_line::arg_descriptor<std::string> arg_generate_new_wallet = {"generate-new-wallet", sw::tr("Generate new wallet and save it to <arg> or <address>.wallet by default"), ""}; - const command_line::arg_descriptor<std::string> arg_generate_from_view_key = {"generate-from-view-key", sw::tr("Generate incoming-only wallet from (address:viewkey:filename) and save it to <filename>"), ""}; + const command_line::arg_descriptor<std::string> arg_generate_from_view_key = {"generate-from-view-key", sw::tr("Generate incoming-only wallet from view key"), ""}; + const command_line::arg_descriptor<std::string> arg_generate_from_keys = {"generate-from-keys", sw::tr("Generate wallet from private keys"), ""}; const command_line::arg_descriptor<std::string> arg_daemon_address = {"daemon-address", sw::tr("Use daemon instance at <host>:<port>"), ""}; const command_line::arg_descriptor<std::string> arg_daemon_host = {"daemon-host", sw::tr("Use daemon instance at host <arg> instead of localhost"), ""}; const command_line::arg_descriptor<std::string> arg_password = {"password", sw::tr("Wallet password"), "", true}; @@ -446,7 +445,7 @@ bool simple_wallet::set_auto_refresh(const std::vector<std::string> &args/* = st if (auto_refresh && !m_auto_refresh_run.load(std::memory_order_relaxed)) { m_auto_refresh_run.store(true, std::memory_order_relaxed); - m_auto_refresh_thread = std::thread([&]{wallet_refresh_thread();}); + m_auto_refresh_thread = boost::thread([&]{wallet_refresh_thread();}); } else if (!auto_refresh && m_auto_refresh_run.load(std::memory_order_relaxed)) { @@ -716,7 +715,7 @@ bool simple_wallet::ask_wallet_create_if_needed() // add logic to error out if new wallet requested but named wallet file exists if (keys_file_exists || wallet_file_exists) { - if (!m_generate_new.empty() || m_restore_deterministic_wallet || !m_generate_from_view_key.empty()) + if (!m_generate_new.empty() || m_restore_deterministic_wallet || !m_generate_from_view_key.empty() || !m_generate_from_keys.empty()) { fail_msg_writer() << tr("attempting to generate or restore wallet, but specified file(s) exist. Exiting to not risk overwriting."); return false; @@ -772,12 +771,12 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) return false; } - if((!m_generate_new.empty()) + (!m_wallet_file.empty()) + (!m_generate_from_view_key.empty()) > 1) + if((!m_generate_new.empty()) + (!m_wallet_file.empty()) + (!m_generate_from_view_key.empty()) + (!m_generate_from_keys.empty()) > 1) { - fail_msg_writer() << tr("can't specify more than one of --generate-new-wallet=\"wallet_name\", --wallet-file=\"wallet_name\" and --generate-from-view-key"); + fail_msg_writer() << tr("can't specify more than one of --generate-new-wallet=\"wallet_name\", --wallet-file=\"wallet_name\", --generate-from-view-key=\"wallet_name\" and --generate-from-keys=\"wallet_name\""); return false; } - else if (m_generate_new.empty() && m_wallet_file.empty() && m_generate_from_view_key.empty()) + else if (m_generate_new.empty() && m_wallet_file.empty() && m_generate_from_view_key.empty() && m_generate_from_keys.empty()) { if(!ask_wallet_create_if_needed()) return false; } @@ -832,7 +831,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) } } - if (!m_generate_new.empty() || m_restore_deterministic_wallet || !m_generate_from_view_key.empty()) + if (!m_generate_new.empty() || m_restore_deterministic_wallet || !m_generate_from_view_key.empty() || !m_generate_from_keys.empty()) { if (m_wallet_file.empty()) m_wallet_file = m_generate_new; // alias for simplicity later @@ -864,41 +863,108 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) } if (!m_generate_from_view_key.empty()) { - // split address:viewkey:filename triple - std::vector<std::string> parts; - boost::split(parts,m_generate_from_view_key, boost::is_any_of(":")); - if (parts.size() < 3) + // parse address + std::string address_string = command_line::input_line("Standard address: "); + if (address_string.empty()) { + fail_msg_writer() << tr("No data supplied, cancelled"); + return false; + } + cryptonote::account_public_address address; + bool has_payment_id; + crypto::hash8 new_payment_id; + if(!get_account_integrated_address_from_str(address, has_payment_id, new_payment_id, testnet, address_string)) + { + fail_msg_writer() << tr("failed to parse address"); + return false; + } + + // parse view secret key + std::string viewkey_string = command_line::input_line("View key: "); + if (viewkey_string.empty()) { + fail_msg_writer() << tr("No data supplied, cancelled"); + return false; + } + cryptonote::blobdata viewkey_data; + if(!epee::string_tools::parse_hexstr_to_binbuff(viewkey_string, viewkey_data)) { - fail_msg_writer() << tr("--generate-from-view-key needs a address:viewkey:filename triple"); + fail_msg_writer() << tr("failed to parse view key secret key"); return false; } + crypto::secret_key viewkey = *reinterpret_cast<const crypto::secret_key*>(viewkey_data.data()); + m_wallet_file = m_generate_from_view_key; + + // check the view key matches the given address + crypto::public_key pkey; + if (!crypto::secret_key_to_public_key(viewkey, pkey)) { + fail_msg_writer() << tr("failed to verify view key secret key"); + return false; + } + if (address.m_view_public_key != pkey) { + fail_msg_writer() << tr("view key does not match standard address"); + return false; + } + + bool r = new_wallet(m_wallet_file, pwd_container.password(), address, viewkey, testnet); + CHECK_AND_ASSERT_MES(r, false, tr("account creation failed")); + } + else if (!m_generate_from_keys.empty()) + { // parse address + std::string address_string = command_line::input_line("Standard address: "); + if (address_string.empty()) { + fail_msg_writer() << tr("No data supplied, cancelled"); + return false; + } cryptonote::account_public_address address; bool has_payment_id; crypto::hash8 new_payment_id; - if(!get_account_integrated_address_from_str(address, has_payment_id, new_payment_id, testnet, parts[0])) + if(!get_account_integrated_address_from_str(address, has_payment_id, new_payment_id, testnet, address_string)) { fail_msg_writer() << tr("failed to parse address"); return false; } + // parse spend secret key + std::string spendkey_string = command_line::input_line("Spend key: "); + if (spendkey_string.empty()) { + fail_msg_writer() << tr("No data supplied, cancelled"); + return false; + } + cryptonote::blobdata spendkey_data; + if(!epee::string_tools::parse_hexstr_to_binbuff(spendkey_string, spendkey_data)) + { + fail_msg_writer() << tr("failed to parse spend key secret key"); + return false; + } + crypto::secret_key spendkey = *reinterpret_cast<const crypto::secret_key*>(spendkey_data.data()); + // parse view secret key + std::string viewkey_string = command_line::input_line("View key: "); + if (viewkey_string.empty()) { + fail_msg_writer() << tr("No data supplied, cancelled"); + return false; + } cryptonote::blobdata viewkey_data; - if(!epee::string_tools::parse_hexstr_to_binbuff(parts[1], viewkey_data)) + if(!epee::string_tools::parse_hexstr_to_binbuff(viewkey_string, viewkey_data)) { fail_msg_writer() << tr("failed to parse view key secret key"); return false; } crypto::secret_key viewkey = *reinterpret_cast<const crypto::secret_key*>(viewkey_data.data()); - // parse filename - m_wallet_file = parts[2]; - for (size_t n = 3; n < parts.size(); ++n) - m_wallet_file += std::string(":") + parts[n]; + m_wallet_file = m_generate_from_keys; - // check the view key matches the given address + // check the spend and view keys match the given address crypto::public_key pkey; + if (!crypto::secret_key_to_public_key(spendkey, pkey)) { + fail_msg_writer() << tr("failed to verify spend key secret key"); + return false; + } + if (address.m_spend_public_key != pkey) { + fail_msg_writer() << tr("spend key does not match standard address"); + return false; + } if (!crypto::secret_key_to_public_key(viewkey, pkey)) { fail_msg_writer() << tr("failed to verify view key secret key"); return false; @@ -908,7 +974,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) return false; } - bool r = new_wallet(m_wallet_file, pwd_container.password(), address, viewkey, testnet); + bool r = new_wallet(m_wallet_file, pwd_container.password(), address, spendkey, viewkey, testnet); CHECK_AND_ASSERT_MES(r, false, tr("account creation failed")); } else @@ -940,6 +1006,7 @@ bool simple_wallet::handle_command_line(const boost::program_options::variables_ m_wallet_file = command_line::get_arg(vm, arg_wallet_file); m_generate_new = command_line::get_arg(vm, arg_generate_new_wallet); m_generate_from_view_key = command_line::get_arg(vm, arg_generate_from_view_key); + m_generate_from_keys = command_line::get_arg(vm, arg_generate_from_keys); m_daemon_address = command_line::get_arg(vm, arg_daemon_address); m_daemon_host = command_line::get_arg(vm, arg_daemon_host); m_daemon_port = command_line::get_arg(vm, arg_daemon_port); @@ -1098,6 +1165,31 @@ bool simple_wallet::new_wallet(const std::string &wallet_file, const std::string return true; } //---------------------------------------------------------------------------------------------------- +bool simple_wallet::new_wallet(const std::string &wallet_file, const std::string& password, const cryptonote::account_public_address& address, + const crypto::secret_key& spendkey, const crypto::secret_key& viewkey, bool testnet) +{ + m_wallet_file = wallet_file; + + m_wallet.reset(new tools::wallet2(testnet)); + m_wallet->callback(this); + + try + { + m_wallet->generate(wallet_file, password, address, spendkey, viewkey); + message_writer(epee::log_space::console_color_white, true) << tr("Generated new wallet: ") + << m_wallet->get_account().get_public_address_str(m_wallet->testnet()); + } + catch (const std::exception& e) + { + fail_msg_writer() << tr("failed to generate new wallet: ") << e.what(); + return false; + } + + m_wallet->init(m_daemon_address); + + return true; +} +//---------------------------------------------------------------------------------------------------- bool simple_wallet::open_wallet(const string &wallet_file, const std::string& password, bool testnet) { if (!tools::wallet2::wallet_valid_path_format(wallet_file)) @@ -1164,7 +1256,7 @@ bool simple_wallet::close_wallet() m_auto_refresh_run.store(false, std::memory_order_relaxed); m_wallet->stop(); { - std::unique_lock<std::mutex> lock(m_auto_refresh_mutex); + boost::unique_lock<boost::mutex> lock(m_auto_refresh_mutex); m_auto_refresh_cond.notify_one(); } m_auto_refresh_thread.join(); @@ -1249,7 +1341,7 @@ bool simple_wallet::start_mining(const std::vector<std::string>& args) req.miner_address = m_wallet->get_account().get_public_address_str(m_wallet->testnet()); bool ok = true; - size_t max_mining_threads_count = (std::max)(std::thread::hardware_concurrency(), static_cast<unsigned>(2)); + size_t max_mining_threads_count = (std::max)(boost::thread::hardware_concurrency(), static_cast<unsigned>(2)); if (0 == args.size()) { req.threads_count = 1; @@ -1366,7 +1458,7 @@ bool simple_wallet::refresh_main(uint64_t start_height, bool reset) m_auto_refresh_run.store(false, std::memory_order_relaxed); // stop any background refresh, and take over m_wallet->stop(); - std::unique_lock<std::mutex> lock(m_auto_refresh_mutex); + boost::unique_lock<boost::mutex> lock(m_auto_refresh_mutex); m_auto_refresh_cond.notify_one(); if (reset) @@ -1814,9 +1906,18 @@ bool simple_wallet::transfer_main(bool new_algorithm, const std::vector<std::str total_fee += ptx_vector[n].fee; } - std::string prompt_str = (boost::format(tr("Your transaction needs to be split into %llu transactions. " - "This will result in a transaction fee being applied to each transaction, for a total fee of %s. Is this okay? (Y/Yes/N/No)")) % - ((unsigned long long)ptx_vector.size()) % print_money(total_fee)).str(); + std::string prompt_str; + if (ptx_vector.size() > 1) + { + prompt_str = (boost::format(tr("Your transaction needs to be split into %llu transactions. " + "This will result in a transaction fee being applied to each transaction, for a total fee of %s. Is this okay? (Y/Yes/N/No)")) % + ((unsigned long long)ptx_vector.size()) % print_money(total_fee)).str(); + } + else + { + prompt_str = (boost::format(tr("The transaction fee is %s. Is this okay? (Y/Yes/N/No)")) % + print_money(total_fee)).str(); + } std::string accepted = command_line::input_line(prompt_str); if (accepted != "Y" && accepted != "y" && accepted != "Yes" && accepted != "yes") { @@ -2333,7 +2434,7 @@ void simple_wallet::wallet_refresh_thread() { while (true) { - std::unique_lock<std::mutex> lock(m_auto_refresh_mutex); + boost::unique_lock<boost::mutex> lock(m_auto_refresh_mutex); if (!m_auto_refresh_run.load(std::memory_order_relaxed)) break; m_auto_refresh_refreshing = true; @@ -2346,7 +2447,7 @@ void simple_wallet::wallet_refresh_thread() m_auto_refresh_refreshing = false; if (!m_auto_refresh_run.load(std::memory_order_relaxed)) break; - m_auto_refresh_cond.wait_for(lock, chrono::seconds(90)); + m_auto_refresh_cond.wait_for(lock, boost::chrono::seconds(90)); } } //---------------------------------------------------------------------------------------------------- @@ -2356,7 +2457,7 @@ bool simple_wallet::run() m_auto_refresh_run = m_wallet->auto_refresh(); if (m_auto_refresh_run) { - m_auto_refresh_thread = std::thread([&]{wallet_refresh_thread();}); + m_auto_refresh_thread = boost::thread([&]{wallet_refresh_thread();}); } else { @@ -2457,6 +2558,7 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_params, arg_wallet_file); command_line::add_arg(desc_params, arg_generate_new_wallet); command_line::add_arg(desc_params, arg_generate_from_view_key); + command_line::add_arg(desc_params, arg_generate_from_keys); command_line::add_arg(desc_params, arg_password); command_line::add_arg(desc_params, arg_password_file); command_line::add_arg(desc_params, arg_daemon_address); diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h index 05f42f8de..d1617dbf4 100644 --- a/src/simplewallet/simplewallet.h +++ b/src/simplewallet/simplewallet.h @@ -83,6 +83,8 @@ namespace cryptonote bool new_wallet(const std::string &wallet_file, const std::string& password, const crypto::secret_key& recovery_key, bool recover, bool two_random, bool testnet, const std::string &old_language); bool new_wallet(const std::string &wallet_file, const std::string& password, const cryptonote::account_public_address& address, + const crypto::secret_key& spendkey, const crypto::secret_key& viewkey, bool testnet); + bool new_wallet(const std::string &wallet_file, const std::string& password, const cryptonote::account_public_address& address, const crypto::secret_key& viewkey, bool testnet); bool open_wallet(const std::string &wallet_file, const std::string& password, bool testnet); bool close_wallet(); @@ -218,6 +220,7 @@ namespace cryptonote std::string m_wallet_file; std::string m_generate_new; std::string m_generate_from_view_key; + std::string m_generate_from_keys; std::string m_import_path; std::string m_electrum_seed; // electrum-style seed parameter @@ -239,9 +242,9 @@ namespace cryptonote std::atomic<bool> m_auto_refresh_run; bool m_auto_refresh_refreshing; - std::thread m_auto_refresh_thread; - std::mutex m_auto_refresh_mutex; - std::condition_variable m_auto_refresh_cond; + boost::thread m_auto_refresh_thread; + boost::mutex m_auto_refresh_mutex; + boost::condition_variable m_auto_refresh_cond; std::atomic<bool> m_in_manual_refresh; }; } diff --git a/src/version.cmake b/src/version.cmake index 4cbaa58e2..8c56b392c 100644 --- a/src/version.cmake +++ b/src/version.cmake @@ -72,7 +72,7 @@ else() message(STATUS "You are building a tagged release") set(VERSIONTAG "release") else() - message(STATUS "You are ahead or behind of a tagged release") + message(STATUS "You are ahead of or behind a tagged release") set(VERSIONTAG "${COMMIT}") endif() endif() diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index ce829b00f..389286ceb 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -204,7 +204,7 @@ void wallet2::process_new_transaction(const cryptonote::transaction& tx, uint64_ tx_pub_key = pub_key_field.pub_key; bool r = true; - int threads = std::thread::hardware_concurrency(); + int threads = boost::thread::hardware_concurrency(); if (miner_tx && m_refresh_type == RefreshNoCoinbase) { // assume coinbase isn't for us @@ -574,7 +574,7 @@ void wallet2::process_blocks(uint64_t start_height, const std::list<cryptonote:: size_t current_index = start_height; blocks_added = 0; - int threads = std::thread::hardware_concurrency(); + int threads = boost::thread::hardware_concurrency(); if (threads > 1) { std::vector<crypto::hash> round_block_hashes(threads); @@ -771,7 +771,7 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& re size_t try_count = 0; crypto::hash last_tx_hash_id = m_transfers.size() ? get_transaction_hash(m_transfers.back().m_tx) : null_hash; std::list<crypto::hash> short_chain_history; - std::thread pull_thread; + boost::thread pull_thread; uint64_t blocks_start_height; std::list<cryptonote::block_complete_entry> blocks; @@ -788,7 +788,7 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& re uint64_t next_blocks_start_height; std::list<cryptonote::block_complete_entry> next_blocks; bool error = false; - pull_thread = std::thread([&]{pull_next_blocks(start_height, next_blocks_start_height, short_chain_history, blocks, next_blocks, error);}); + pull_thread = boost::thread([&]{pull_next_blocks(start_height, next_blocks_start_height, short_chain_history, blocks, next_blocks, error);}); process_blocks(blocks_start_height, blocks, added_blocks); blocks_fetched += added_blocks; @@ -1186,6 +1186,41 @@ void wallet2::generate(const std::string& wallet_, const std::string& password, } /*! +* \brief Creates a wallet from a public address and a spend/view secret key pair. +* \param wallet_ Name of wallet file +* \param password Password of wallet file +* \param spendkey spend secret key +* \param viewkey view secret key +*/ +void wallet2::generate(const std::string& wallet_, const std::string& password, + const cryptonote::account_public_address &account_public_address, + const crypto::secret_key& spendkey, const crypto::secret_key& viewkey) +{ + clear(); + prepare_file_names(wallet_); + + boost::system::error_code ignored_ec; + THROW_WALLET_EXCEPTION_IF(boost::filesystem::exists(m_wallet_file, ignored_ec), error::file_exists, m_wallet_file); + THROW_WALLET_EXCEPTION_IF(boost::filesystem::exists(m_keys_file, ignored_ec), error::file_exists, m_keys_file); + + m_account.create_from_keys(account_public_address, spendkey, viewkey); + m_account_public_address = account_public_address; + m_watch_only = false; + + bool r = store_keys(m_keys_file, password, true); + THROW_WALLET_EXCEPTION_IF(!r, error::file_save_error, m_keys_file); + + r = file_io_utils::save_string_to_file(m_wallet_file + ".address.txt", m_account.get_public_address_str(m_testnet)); + if(!r) LOG_PRINT_RED_L0("String with address text not saved"); + + cryptonote::block b; + generate_genesis(b); + m_blockchain.push_back(get_block_hash(b)); + + store(); +} + +/*! * \brief Rewrites to the wallet file for wallet upgrade (doesn't generate key, assumes it's already there) * \param wallet_name Name of wallet file (should exist) * \param password Password for wallet file @@ -1277,7 +1312,7 @@ bool wallet2::prepare_file_names(const std::string& file_path) //---------------------------------------------------------------------------------------------------- bool wallet2::check_connection() { - std::lock_guard<std::mutex> lock(m_daemon_rpc_mutex); + boost::lock_guard<boost::mutex> lock(m_daemon_rpc_mutex); if(m_http_client.is_connected()) return true; diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index c9f00bfc9..c79da2e15 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -185,6 +185,16 @@ namespace tools const crypto::secret_key& recovery_param = crypto::secret_key(), bool recover = false, bool two_random = false); /*! + * \brief Creates a wallet from a public address and a spend/view secret key pair. + * \param wallet_ Name of wallet file + * \param password Password of wallet file + * \param viewkey view secret key + * \param spendkey spend secret key + */ + void generate(const std::string& wallet, const std::string& password, + const cryptonote::account_public_address &account_public_address, + const crypto::secret_key& spendkey, const crypto::secret_key& viewkey); + /*! * \brief Creates a watch only wallet from a public address and a view secret key. * \param wallet_ Name of wallet file * \param password Password of wallet file @@ -397,7 +407,7 @@ namespace tools std::atomic<bool> m_run; - std::mutex m_daemon_rpc_mutex; + boost::mutex m_daemon_rpc_mutex; i_wallet2_callback* m_callback; bool m_testnet; diff --git a/src/wallet/wallet_errors.h b/src/wallet/wallet_errors.h index 10d27651f..80482c1ba 100644 --- a/src/wallet/wallet_errors.h +++ b/src/wallet/wallet_errors.h @@ -31,6 +31,7 @@ #pragma once #include <stdexcept> +#include <system_error> #include <string> #include <vector> diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index e8e062e95..ac13d8021 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -461,6 +461,7 @@ namespace tools bool wallet_rpc_server::on_get_payments(const wallet_rpc::COMMAND_RPC_GET_PAYMENTS::request& req, wallet_rpc::COMMAND_RPC_GET_PAYMENTS::response& res, epee::json_rpc::error& er) { crypto::hash payment_id; + crypto::hash8 payment_id8; cryptonote::blobdata payment_id_blob; if(!epee::string_tools::parse_hexstr_to_binbuff(req.payment_id, payment_id_blob)) { @@ -469,14 +470,22 @@ namespace tools return false; } - if(sizeof(payment_id) != payment_id_blob.size()) - { - er.code = WALLET_RPC_ERROR_CODE_WRONG_PAYMENT_ID; - er.message = "Payment ID has invalid size"; - return false; - } - - payment_id = *reinterpret_cast<const crypto::hash*>(payment_id_blob.data()); + if(sizeof(payment_id) == payment_id_blob.size()) + { + payment_id = *reinterpret_cast<const crypto::hash*>(payment_id_blob.data()); + } + else if(sizeof(payment_id8) == payment_id_blob.size()) + { + payment_id8 = *reinterpret_cast<const crypto::hash8*>(payment_id_blob.data()); + memcpy(payment_id.data, payment_id8.data, 8); + memset(payment_id.data + 8, 0, 24); + } + else + { + er.code = WALLET_RPC_ERROR_CODE_WRONG_PAYMENT_ID; + er.message = "Payment ID has invalid size: " + req.payment_id; + return false; + } res.payments.clear(); std::list<wallet2::payment_details> payment_list; diff --git a/tests/core_proxy/core_proxy.cpp b/tests/core_proxy/core_proxy.cpp index 14a12e139..256faeccb 100644 --- a/tests/core_proxy/core_proxy.cpp +++ b/tests/core_proxy/core_proxy.cpp @@ -62,8 +62,6 @@ using namespace crypto; BOOST_CLASS_VERSION(nodetool::node_server<cryptonote::t_cryptonote_protocol_handler<tests::proxy_core> >, 1); -unsigned int epee::g_test_dbg_lock_sleep = 0; - int main(int argc, char* argv[]) { diff --git a/tests/core_tests/chaingen_main.cpp b/tests/core_tests/chaingen_main.cpp index 40bce1e4e..9f8a57821 100644 --- a/tests/core_tests/chaingen_main.cpp +++ b/tests/core_tests/chaingen_main.cpp @@ -44,8 +44,6 @@ namespace const command_line::arg_descriptor<bool> arg_test_transactions = {"test_transactions", ""}; } -unsigned int epee::g_test_dbg_lock_sleep = 0; - int main(int argc, char* argv[]) { TRY_ENTRY(); diff --git a/tests/functional_tests/main.cpp b/tests/functional_tests/main.cpp index 0233c50b5..58a2a5c90 100644 --- a/tests/functional_tests/main.cpp +++ b/tests/functional_tests/main.cpp @@ -55,8 +55,6 @@ namespace const command_line::arg_descriptor<size_t> arg_test_repeat_count = {"test_repeat_count", "", 1}; } -unsigned int epee::g_test_dbg_lock_sleep = 0; - int main(int argc, char* argv[]) { TRY_ENTRY(); diff --git a/tests/net_load_tests/clt.cpp b/tests/net_load_tests/clt.cpp index e6b5c0414..56089a4da 100644 --- a/tests/net_load_tests/clt.cpp +++ b/tests/net_load_tests/clt.cpp @@ -628,8 +628,6 @@ TEST_F(net_load_test_clt, permament_open_and_close_and_connections_closed_by_ser ASSERT_EQ(RESERVED_CONN_CNT, m_tcp_server.get_config_object().get_connections_count()); } -unsigned int epee::g_test_dbg_lock_sleep = 0; - int main(int argc, char** argv) { epee::debug::get_set_enable_assert(true, false); diff --git a/tests/net_load_tests/srv.cpp b/tests/net_load_tests/srv.cpp index 5a737e616..d8d3eae2e 100644 --- a/tests/net_load_tests/srv.cpp +++ b/tests/net_load_tests/srv.cpp @@ -213,8 +213,6 @@ namespace }; } -unsigned int epee::g_test_dbg_lock_sleep = 0; - int main(int argc, char** argv) { //set up logging options diff --git a/tests/performance_tests/main.cpp b/tests/performance_tests/main.cpp index 2e390fabc..84a51aa96 100644 --- a/tests/performance_tests/main.cpp +++ b/tests/performance_tests/main.cpp @@ -42,8 +42,6 @@ #include "generate_key_image_helper.h" #include "is_out_to_acc.h" -unsigned int epee::g_test_dbg_lock_sleep = 0; - int main(int argc, char** argv) { set_process_affinity(1); diff --git a/tests/unit_tests/address_from_url.cpp b/tests/unit_tests/address_from_url.cpp index 60f21201f..ff163dab9 100644 --- a/tests/unit_tests/address_from_url.cpp +++ b/tests/unit_tests/address_from_url.cpp @@ -82,7 +82,7 @@ TEST(AddressFromTXT, Failure) TEST(AddressFromURL, Success) { - std::string addr = "46BeWrHpwXmHDpDEUmZBWZfoQpdc6HaERCNmx1pEYL2rAcuwufPN9rXHHtyUA4QVy66qeFQkn6sfK8aHYjA3jk3o1Bv16em"; + const std::string addr = "44AFFq5kSiGBoZ4NMDwYtN18obc8AemS33DBLWs3H7otXft3XjrpDtQGv7SqSsaBYBb98uNbr2VBBEt7f2wfn3RVGQBEP3A"; bool dnssec_result = false; diff --git a/tests/unit_tests/hardfork.cpp b/tests/unit_tests/hardfork.cpp index 7432c03ec..50e0e5ae8 100644 --- a/tests/unit_tests/hardfork.cpp +++ b/tests/unit_tests/hardfork.cpp @@ -57,7 +57,7 @@ public: virtual void batch_start(uint64_t batch_num_blocks=0) {} virtual void batch_stop() {} virtual void set_batch_transactions(bool) {} - virtual void block_txn_start() {} + virtual void block_txn_start(bool readonly=false) {} virtual void block_txn_stop() {} virtual void block_txn_abort() {} virtual void drop_hard_fork_info() {} diff --git a/tests/unit_tests/main.cpp b/tests/unit_tests/main.cpp index b96129a3a..faaf9475a 100644 --- a/tests/unit_tests/main.cpp +++ b/tests/unit_tests/main.cpp @@ -32,8 +32,6 @@ #include "include_base_utils.h" -unsigned int epee::g_test_dbg_lock_sleep = 0; - int main(int argc, char** argv) { epee::debug::get_set_enable_assert(true, false); |