diff options
90 files changed, 2293 insertions, 851 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 73c5f6e93..cab853581 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -137,9 +137,24 @@ endif() if(ARCH_ID STREQUAL "ppc64le") set(PPC64LE 1) + set(PPC64 0) + set(PPC 0) + +endif() + +if(ARCH_ID STREQUAL "powerpc64" OR ARCH_ID STREQUAL "ppc64") + set(PPC64LE 0) + set(PPC64 1) + set(PPC 0) endif() -if(WIN32 OR ARM) +if(ARCH_ID STREQUAL "powerpc") + set(PPC64LE 0) + set(PPC64 0) + set(PPC 1) +endif() + +if(WIN32 OR ARM OR PPC64LE OR PPC64 OR PPC) set(OPT_FLAGS_RELEASE "-O2") else() set(OPT_FLAGS_RELEASE "-Ofast") @@ -452,9 +467,12 @@ link_directories(${LIBUNWIND_LIBRARY_DIRS}) # Final setup for libpcsc if (PCSC_FOUND) + message(STATUS "Using PCSC include dir at ${PCSC_INCLUDE_DIR}") add_definitions(-DHAVE_PCSC) include_directories(${PCSC_INCLUDE_DIR}) link_directories(${LIBPCSC_LIBRARY_DIRS}) +else (PCSC_FOUND) + message(STATUS "Could not find PCSC") endif() if(MSVC) @@ -474,7 +492,11 @@ else() if(ARCH STREQUAL "default") set(ARCH_FLAG "") elseif(PPC64LE) - set(ARCH_FLAG "-mcpu=${ARCH}") + set(ARCH_FLAG "-mcpu=power8") + elseif(PPC64) + set(ARCH_FLAG "-mcpu=970") + elseif(PPC) + set(ARCH_FLAG "-mcpu=7400") elseif(IOS AND ARCH STREQUAL "arm64") message(STATUS "IOS: Changing arch from arm64 to armv8") set(ARCH_FLAG "-march=armv8") @@ -584,12 +606,12 @@ else() message(STATUS "AES support explicitly disabled") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DNO_AES") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DNO_AES") - elseif(NOT ARM AND NOT PPC64LE) + elseif(NOT ARM AND NOT PPC64LE AND NOT PPC64 AND NOT PPC) message(STATUS "AES support enabled") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -maes") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -maes") - elseif(PPC64LE) - message(STATUS "AES support not available on ppc64le") + elseif(PPC64LE OR PPC64 OR PPC) + message(STATUS "AES support not available on POWER") elseif(ARM6) message(STATUS "AES support not available on ARMv6") elseif(ARM7) @@ -22,6 +22,10 @@ Portions Copyright (c) 2012-2013 The Cryptonote developers. ## Build +### IMPORTANT + +These builds are of the master branch, which is used for active development and can be either unstable or incompatible with release software. Please compile release branches. + | Operating System | Processor | Status | | --------------------- | -------- |--------| | Ubuntu 16.04 | i686 | [](https://build.getmonero.org/builders/monero-static-ubuntu-i686) @@ -103,7 +107,7 @@ Dates are provided in the format YYYY-MM-DD. | 1220516 | 2017-01-05 | v4 | v0.10.1 | v0.10.2.1 | Allow normal and RingCT transactions | | 1288616 | 2017-04-15 | v5 | v0.10.3.0 | v0.10.3.1 | Adjusted minimum blocksize and fee algorithm | | 1400000 | 2017-09-16 | v6 | v0.11.0.0 | v0.11.0.0 | Allow only RingCT transactions, allow only >= ringsize 5 | -| 1546000 | 2018-04-06 | v7 | v0.12.0.0 | v0.12.0.0 | Cryptonight variant 1, ringsize >= 7, sorted inputs +| 1546000 | 2018-04-06 | v7 | v0.12.0.0 | v0.12.2.0 | Cryptonight variant 1, ringsize >= 7, sorted inputs | XXXXXXX | 2018-10-XX | XX | XXXXXXXXX | XXXXXXXXX | X X's indicate that these details have not been determined as of commit date. @@ -112,49 +116,6 @@ X's indicate that these details have not been determined as of commit date. Approximately three months prior to a scheduled software upgrade, a branch from Master will be created with the new release version tag. Pull requests that address bugs should then be made to both Master and the new release branch. Pull requests that require extensive review and testing (generally, optimizations and new features) should *not* be made to the release branch. -## Installing Monero from a package - -Packages are available for - -* Ubuntu and [snap supported](https://snapcraft.io/docs/core/install) systems, via a community contributed build. - - snap install monero --beta - -Installing a snap is very quick. Snaps are secure. They are isolated with all of their dependencies. Snaps also auto update when a new version is released. - -* Arch Linux (via [AUR](https://aur.archlinux.org/)): - - Stable release: [`monero`](https://aur.archlinux.org/packages/monero) - - Bleeding edge: [`monero-git`](https://aur.archlinux.org/packages/monero-git) - -* Void Linux: - - xbps-install -S monero - -* GuixSD - - guix package -i monero - -* OS X via [Homebrew](http://brew.sh) - - brew tap sammy007/cryptonight - brew install monero --build-from-source - -* Docker - - # Build using all available cores - docker build -t monero . - - # or build using a specific number of cores (reduce RAM requirement) - docker build --build-arg NPROC=1 -t monero . - - # either run in foreground - docker run -it -v /monero/chain:/root/.bitmonero -v /monero/wallet:/wallet -p 18080:18080 monero - - # or in background - docker run -it -d -v /monero/chain:/root/.bitmonero -v /monero/wallet:/wallet -p 18080:18080 monero - -Packaging for your favorite distribution would be a welcome contribution! - ## Compiling Monero from source ### Dependencies @@ -187,11 +148,15 @@ library archives (`.a`). | GTest | 1.5 | YES | `libgtest-dev`^ | `gtest` | `gtest-devel` | YES | Test suite | | Doxygen | any | NO | `doxygen` | `doxygen` | `doxygen` | YES | Documentation | | Graphviz | any | NO | `graphviz` | `graphviz` | `graphviz` | YES | Documentation | +| pcsclite | ? | NO | `libpcsclite-dev` | ? | `pcsc-lite pcsc-lite-devel` | NO | Ledger | [^] On Debian/Ubuntu `libgtest-dev` only includes sources and headers. You must build the library binary manually. This can be done with the following command ```sudo apt-get install libgtest-dev && cd /usr/src/gtest && sudo cmake . && sudo make && sudo mv libg* /usr/lib/ ``` +Debian / Ubuntu one liner for all dependencies +``` sudo apt update && sudo apt install build-essential cmake pkg-config libboost-all-dev libssl-dev libzmq3-dev libunbound-dev libsodium-dev libminiupnpc-dev libunwind8-dev liblzma-dev libreadline6-dev libldns-dev libexpat1-dev doxygen graphviz libpcsclite-dev ``` + ### Cloning the repository Clone recursively to pull-in needed submodule(s): @@ -210,9 +175,10 @@ invokes cmake commands as needed. #### On Linux and OS X * Install the dependencies -* Change to the root of the source code directory and build: +* Change to the root of the source code directory, change to the most recent release branch, and build: cd monero + git checkout v0.12.2.0 make *Optional*: If your machine has several cores and enough memory, enable @@ -222,6 +188,12 @@ invokes cmake commands as needed. *Note*: If cmake can not find zmq.hpp file on OS X, installing `zmq.hpp` from https://github.com/zeromq/cppzmq to `/usr/local/include` should fix that error. + + *Note*: The instructions above will compile the most stable release of the + Monero software. If you would like to use and test the most recent software, + use ```git checkout master```. The master branch may contain updates that are + both unstable and incompatible with release software, though testing is always + encouraged. * The resulting executables can be found in `build/release/bin` @@ -268,7 +240,7 @@ Tested on a Raspberry Pi Zero with a clean install of minimal Raspbian Stretch ( ``` git clone https://github.com/monero-project/monero.git cd monero - git checkout tags/v0.11.1.0 + git checkout tags/v0.12.2.0 ``` * Build: ``` @@ -353,8 +325,22 @@ application. or `MinGW-w64-Win64 Shell` shortcut on 32-bit Windows. Note that if you are running 64-bit Windows, you will have both 64-bit and 32-bit MinGW shells. +**Cloning** + +* To git clone, run: + + git clone --recursive https://github.com/monero-project/monero.git + **Building** +* Change to the cloned directory, run: + + cd monero + +* If you would like a specific [version/tag](https://github.com/monero-project/monero/tags), do a git checkout for that version. eg. 'v0.12.1.0'. If you dont care about the version and just want binaries from master, skip this step: + + git checkout v0.12.1.0 + * If you are on a 64-bit system, run: make release-static-win64 @@ -503,6 +489,46 @@ By default, in either dynamically or statically linked builds, binaries target t * ```make release-static-win64``` builds binaries on 64-bit Windows portable across 64-bit Windows systems * ```make release-static-win32``` builds binaries on 64-bit or 32-bit Windows portable across 32-bit Windows systems +## Installing Monero from a package + +**DISCLAIMER: These packages are not part of this repository or maintained by this project's contributors, and as such, do not go through the same review process to ensure their trustworthiness and security.** + +Packages are available for + +* Ubuntu and [snap supported](https://snapcraft.io/docs/core/install) systems, via a community contributed build. + + snap install monero --beta + +Installing a snap is very quick. Snaps are secure. They are isolated with all of their dependencies. Snaps also auto update when a new version is released. + +* Arch Linux (via [AUR](https://aur.archlinux.org/)): + - Stable release: [`monero`](https://aur.archlinux.org/packages/monero) + - Bleeding edge: [`monero-git`](https://aur.archlinux.org/packages/monero-git) + +* Void Linux: + + xbps-install -S monero + +* GuixSD + + guix package -i monero + +* Docker + + # Build using all available cores + docker build -t monero . + + # or build using a specific number of cores (reduce RAM requirement) + docker build --build-arg NPROC=1 -t monero . + + # either run in foreground + docker run -it -v /monero/chain:/root/.bitmonero -v /monero/wallet:/wallet -p 18080:18080 monero + + # or in background + docker run -it -d -v /monero/chain:/root/.bitmonero -v /monero/wallet:/wallet -p 18080:18080 monero + +Packaging for your favorite distribution would be a welcome contribution! + ## Running monerod The build places the binary in `bin/` sub-directory within the build directory @@ -556,6 +582,8 @@ setting the following configuration parameters and environment variables: as well. * Do NOT pass `--detach` when running through torsocks with systemd, (see [utils/systemd/monerod.service](utils/systemd/monerod.service) for details). +* If you use the wallet with a Tor daemon via the loopback IP (eg, 127.0.0.1:9050), + then use `--untrusted-daemon` unless it is your own hidden service. Example command line to start monerod through Tor: diff --git a/cmake/FindPCSC.cmake b/cmake/FindPCSC.cmake index 8dd9d0e76..b5e8420e6 100644 --- a/cmake/FindPCSC.cmake +++ b/cmake/FindPCSC.cmake @@ -14,22 +14,31 @@ ENDIF (PCSC_INCLUDE_DIR AND PCSC_LIBRARIES) IF (NOT WIN32) FIND_PACKAGE(PkgConfig) PKG_CHECK_MODULES(PC_PCSC libpcsclite) -ENDIF (NOT WIN32) -FIND_PATH(PCSC_INCLUDE_DIR winscard.h - HINTS - /usr/include/PCSC - ${PC_PCSC_INCLUDEDIR} - ${PC_PCSC_INCLUDE_DIRS} - PATH_SUFFIXES PCSC + FIND_PATH(PCSC_INCLUDE_DIR winscard.h + HINTS + /usr/include/PCSC + ${PC_PCSC_INCLUDEDIR} + ${PC_PCSC_INCLUDE_DIRS} + PATH_SUFFIXES PCSC ) -FIND_LIBRARY(PCSC_LIBRARY NAMES pcsclite libpcsclite WinSCard PCSC - HINTS - ${PC_PCSC_LIBDIR} - ${PC_PCSC_LIBRARY_DIRS} + FIND_LIBRARY(PCSC_LIBRARY NAMES pcsclite libpcsclite PCSC + HINTS + ${PC_PCSC_LIBDIR} + ${PC_PCSC_LIBRARY_DIRS} ) +ELSE (NOT WIN32) + IF(BUILD_64 STREQUAL "ON") + set(PCSC_INCLUDE_DIR /mingw64/x86_64-w64-mingw32/include) + set(PCSC_LIBRARY /mingw64/x86_64-w64-mingw32/lib/libwinscard.a) + ELSE(BUILD_64 STREQUAL "ON") + set(PCSC_INCLUDE_DIR /mingw32/i686-w64-mingw32/include) + set(PCSC_LIBRARY /mingw32/i686-w64-mingw32/lib/libwinscard.a) + ENDIF(BUILD_64 STREQUAL "ON") +ENDIF (NOT WIN32) + # handle the QUIETLY and REQUIRED arguments and set PCSC_FOUND to TRUE if # all listed variables are TRUE INCLUDE(FindPackageHandleStandardArgs) diff --git a/cmake/FindReadline.cmake b/cmake/FindReadline.cmake index 87f8ccace..de9ddc46d 100644 --- a/cmake/FindReadline.cmake +++ b/cmake/FindReadline.cmake @@ -23,7 +23,7 @@ find_path(Readline_ROOT_DIR NAMES include/readline/readline.h - PATHS /opt/local/ /usr/local/ /usr/ + PATHS /usr/local/opt/readline/ /opt/local/ /usr/local/ /usr/ NO_DEFAULT_PATH ) diff --git a/contrib/epee/include/file_io_utils.h b/contrib/epee/include/file_io_utils.h index 196610674..4434f7383 100644 --- a/contrib/epee/include/file_io_utils.h +++ b/contrib/epee/include/file_io_utils.h @@ -28,7 +28,7 @@ #ifndef _FILE_IO_UTILS_H_ #define _FILE_IO_UTILS_H_ -#include <iostream> +#include <fstream> #include <boost/filesystem/path.hpp> #include <boost/filesystem/operations.hpp> #ifdef WIN32 @@ -128,7 +128,7 @@ namespace file_io_utils inline - bool load_file_to_string(const std::string& path_to_file, std::string& target_str) + bool load_file_to_string(const std::string& path_to_file, std::string& target_str, size_t max_size = 1000000000) { #ifdef WIN32 WCHAR wide_path[1000]; @@ -139,7 +139,7 @@ namespace file_io_utils if (file_handle == INVALID_HANDLE_VALUE) return false; DWORD file_size = GetFileSize(file_handle, NULL); - if ((file_size == INVALID_FILE_SIZE) || (file_size > 1000000000)) { + if ((file_size == INVALID_FILE_SIZE) || (uint64_t)file_size > (uint64_t)max_size) { CloseHandle(file_handle); return false; } @@ -159,7 +159,7 @@ namespace file_io_utils std::ifstream::pos_type file_size = fstream.tellg(); - if(file_size > 1000000000) + if((uint64_t)file_size > (uint64_t)max_size) // ensure a large domain for comparison, and negative -> too large return false;//don't go crazy size_t file_size_t = static_cast<size_t>(file_size); diff --git a/contrib/epee/include/misc_log_ex.h b/contrib/epee/include/misc_log_ex.h index 35ec0950b..530f8e636 100644 --- a/contrib/epee/include/misc_log_ex.h +++ b/contrib/epee/include/misc_log_ex.h @@ -34,6 +34,7 @@ #define MONERO_DEFAULT_LOG_CATEGORY "default" #define MAX_LOG_FILE_SIZE 104850000 // 100 MB - 7600 bytes +#define MAX_LOG_FILES 50 #define MCFATAL(cat,x) CLOG(FATAL,cat) << x #define MCERROR(cat,x) CLOG(ERROR,cat) << x @@ -105,7 +106,7 @@ #endif std::string mlog_get_default_log_path(const char *default_filename); -void mlog_configure(const std::string &filename_base, bool console, const std::size_t max_log_file_size = MAX_LOG_FILE_SIZE); +void mlog_configure(const std::string &filename_base, bool console, const std::size_t max_log_file_size = MAX_LOG_FILE_SIZE, const std::size_t max_log_files = MAX_LOG_FILES); void mlog_set_categories(const char *categories); std::string mlog_get_categories(); void mlog_set_log_level(int level); diff --git a/contrib/epee/include/net/abstract_tcp_server2.h b/contrib/epee/include/net/abstract_tcp_server2.h index 2f7325be5..7ca6ac872 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.h +++ b/contrib/epee/include/net/abstract_tcp_server2.h @@ -119,6 +119,7 @@ namespace net_utils //----------------- i_service_endpoint --------------------- virtual bool do_send(const void* ptr, size_t cb); ///< (see do_send from i_service_endpoint) virtual bool do_send_chunk(const void* ptr, size_t cb); ///< will send (or queue) a part of data + virtual bool send_done(); virtual bool close(); virtual bool call_run_once_service_io(); virtual bool request_callback(); @@ -137,8 +138,11 @@ namespace net_utils /// reset connection timeout timer and callback void reset_timer(boost::posix_time::milliseconds ms, bool add); - boost::posix_time::milliseconds get_default_time() const; - boost::posix_time::milliseconds get_timeout_from_bytes_read(size_t bytes) const; + boost::posix_time::milliseconds get_default_timeout(); + boost::posix_time::milliseconds get_timeout_from_bytes_read(size_t bytes); + + /// host connection count tracking + unsigned int host_count(const std::string &host, int delta = 0); /// Buffer for incoming data. boost::array<char, 8192> buffer_; @@ -165,6 +169,8 @@ namespace net_utils boost::asio::deadline_timer m_timer; bool m_local; + bool m_ready_to_close; + std::string m_host; public: void setRpcStation(); diff --git a/contrib/epee/include/net/abstract_tcp_server2.inl b/contrib/epee/include/net/abstract_tcp_server2.inl index 91a94c21e..134bb4199 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.inl +++ b/contrib/epee/include/net/abstract_tcp_server2.inl @@ -56,8 +56,8 @@ #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net" -#define DEFAULT_TIMEOUT_MS_LOCAL boost::posix_time::milliseconds(120000) // 2 minutes -#define DEFAULT_TIMEOUT_MS_REMOTE boost::posix_time::milliseconds(10000) // 10 seconds +#define DEFAULT_TIMEOUT_MS_LOCAL 1800000 // 30 minutes +#define DEFAULT_TIMEOUT_MS_REMOTE 300000 // 5 minutes #define TIMEOUT_EXTRA_MS_PER_BYTE 0.2 PRAGMA_WARNING_PUSH @@ -86,7 +86,8 @@ PRAGMA_WARNING_DISABLE_VS(4355) m_throttle_speed_in("speed_in", "throttle_speed_in"), m_throttle_speed_out("speed_out", "throttle_speed_out"), m_timer(io_service), - m_local(false) + m_local(false), + m_ready_to_close(false) { MDEBUG("test, connection constructor set m_connection_type="<<m_connection_type); } @@ -146,7 +147,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) context = boost::value_initialized<t_connection_context>(); const unsigned long ip_{boost::asio::detail::socket_ops::host_to_network_long(remote_ep.address().to_v4().to_ulong())}; - m_local = epee::net_utils::is_ip_loopback(ip_); + m_local = epee::net_utils::is_ip_loopback(ip_) || epee::net_utils::is_ip_local(ip_); // create a random uuid boost::uuids::uuid random_uuid; @@ -165,9 +166,12 @@ PRAGMA_WARNING_DISABLE_VS(4355) return false; } + m_host = context.m_remote_address.host_str(); + try { host_count(m_host, 1); } catch(...) { /* ignore */ } + m_protocol_handler.after_init_connection(); - reset_timer(get_default_time(), false); + reset_timer(get_default_timeout(), false); socket_.async_read_some(boost::asio::buffer(buffer_), strand_.wrap( @@ -324,6 +328,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) logger_handle_net_read(bytes_transferred); context.m_last_recv = time(NULL); context.m_recv_cnt += bytes_transferred; + m_ready_to_close = false; bool recv_res = m_protocol_handler.handle_recv(buffer_.data(), bytes_transferred); if(!recv_res) { @@ -356,6 +361,13 @@ PRAGMA_WARNING_DISABLE_VS(4355) _dbg3("[sock " << socket_.native_handle() << "] Some problems at read: " << e.message() << ':' << e.value()); shutdown(); } + else + { + _dbg3("[sock " << socket_.native_handle() << "] peer closed connection"); + if (m_ready_to_close) + shutdown(); + } + m_ready_to_close = true; } // If an error occurs then no new asynchronous operations are started. This // means that all shared_ptr references to the connection object will @@ -531,7 +543,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) if(m_send_que.size() > 1) { // active operation should be in progress, nothing to do, just wait last operation callback auto size_now = cb; - MDEBUG("do_send() NOW just queues: packet="<<size_now<<" B, is added to queue-size="<<m_send_que.size()); + MDEBUG("do_send_chunk() NOW just queues: packet="<<size_now<<" B, is added to queue-size="<<m_send_que.size()); //do_send_handler_delayed( ptr , size_now ); // (((H))) // empty function LOG_TRACE_CC(context, "[sock " << socket_.native_handle() << "] Async send requested " << m_send_que.front().size()); @@ -546,12 +558,12 @@ PRAGMA_WARNING_DISABLE_VS(4355) } auto size_now = m_send_que.front().size(); - MDEBUG("do_send() NOW SENSD: packet="<<size_now<<" B"); + MDEBUG("do_send_chunk() NOW SENSD: packet="<<size_now<<" B"); if (speed_limit_is_enabled()) do_send_handler_write( ptr , size_now ); // (((H))) CHECK_AND_ASSERT_MES( size_now == m_send_que.front().size(), false, "Unexpected queue size"); - reset_timer(get_default_time(), false); + reset_timer(get_default_timeout(), false); boost::asio::async_write(socket_, boost::asio::buffer(m_send_que.front().data(), size_now ) , //strand_.wrap( boost::bind(&connection<t_protocol_handler>::handle_write, self, _1, _2) @@ -566,29 +578,51 @@ PRAGMA_WARNING_DISABLE_VS(4355) return true; - CATCH_ENTRY_L0("connection<t_protocol_handler>::do_send", false); + CATCH_ENTRY_L0("connection<t_protocol_handler>::do_send_chunk", false); } // do_send_chunk //--------------------------------------------------------------------------------- template<class t_protocol_handler> - boost::posix_time::milliseconds connection<t_protocol_handler>::get_default_time() const + boost::posix_time::milliseconds connection<t_protocol_handler>::get_default_timeout() { + unsigned count; + try { count = host_count(m_host); } catch (...) { count = 0; } + const unsigned shift = std::min(std::max(count, 1u) - 1, 8u); + boost::posix_time::milliseconds timeout(0); if (m_local) - return DEFAULT_TIMEOUT_MS_LOCAL; + timeout = boost::posix_time::milliseconds(DEFAULT_TIMEOUT_MS_LOCAL >> shift); else - return DEFAULT_TIMEOUT_MS_REMOTE; + timeout = boost::posix_time::milliseconds(DEFAULT_TIMEOUT_MS_REMOTE >> shift); + return timeout; } //--------------------------------------------------------------------------------- template<class t_protocol_handler> - boost::posix_time::milliseconds connection<t_protocol_handler>::get_timeout_from_bytes_read(size_t bytes) const + boost::posix_time::milliseconds connection<t_protocol_handler>::get_timeout_from_bytes_read(size_t bytes) { boost::posix_time::milliseconds ms = (boost::posix_time::milliseconds)(unsigned)(bytes * TIMEOUT_EXTRA_MS_PER_BYTE); ms += m_timer.expires_from_now(); - if (ms > get_default_time()) - ms = get_default_time(); + if (ms > get_default_timeout()) + ms = get_default_timeout(); return ms; } //--------------------------------------------------------------------------------- template<class t_protocol_handler> + unsigned int connection<t_protocol_handler>::host_count(const std::string &host, int delta) + { + static boost::mutex hosts_mutex; + CRITICAL_REGION_LOCAL(hosts_mutex); + static std::map<std::string, unsigned int> hosts; + unsigned int &val = hosts[host]; + if (delta > 0) + MTRACE("New connection from host " << host << ": " << val); + else if (delta < 0) + MTRACE("Closed connection from host " << host << ": " << val); + CHECK_AND_ASSERT_THROW_MES(delta >= 0 || val >= (unsigned)-delta, "Count would go negative"); + CHECK_AND_ASSERT_THROW_MES(delta <= 0 || val <= std::numeric_limits<unsigned int>::max() - (unsigned)delta, "Count would wrap"); + val += delta; + return val; + } + //--------------------------------------------------------------------------------- + template<class t_protocol_handler> void connection<t_protocol_handler>::reset_timer(boost::posix_time::milliseconds ms, bool add) { if (m_connection_type != e_connection_type_RPC) @@ -621,6 +655,11 @@ PRAGMA_WARNING_DISABLE_VS(4355) socket_.shutdown(boost::asio::ip::tcp::socket::shutdown_both, ignored_ec); m_was_shutdown = true; m_protocol_handler.release_protocol(); + if (!m_host.empty()) + { + try { host_count(m_host, -1); } catch (...) { /* ignore */ } + m_host = ""; + } return true; } //--------------------------------------------------------------------------------- @@ -645,6 +684,15 @@ PRAGMA_WARNING_DISABLE_VS(4355) } //--------------------------------------------------------------------------------- template<class t_protocol_handler> + bool connection<t_protocol_handler>::send_done() + { + if (m_ready_to_close) + return close(); + m_ready_to_close = true; + return true; + } + //--------------------------------------------------------------------------------- + template<class t_protocol_handler> bool connection<t_protocol_handler>::cancel() { return close(); @@ -687,7 +735,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) }else { //have more data to send - reset_timer(get_default_time(), false); + reset_timer(get_default_timeout(), false); auto size_now = m_send_que.front().size(); MDEBUG("handle_write() NOW SENDS: packet="<<size_now<<" B" <<", from queue size="<<m_send_que.size()); if (speed_limit_is_enabled()) diff --git a/contrib/epee/include/net/http_protocol_handler.h b/contrib/epee/include/net/http_protocol_handler.h index e602fac2b..1780f2393 100644 --- a/contrib/epee/include/net/http_protocol_handler.h +++ b/contrib/epee/include/net/http_protocol_handler.h @@ -69,7 +69,7 @@ namespace net_utils typedef t_connection_context connection_context;//t_connection_context net_utils::connection_context_base connection_context; typedef http_server_config config_type; - simple_http_connection_handler(i_service_endpoint* psnd_hndlr, config_type& config); + simple_http_connection_handler(i_service_endpoint* psnd_hndlr, config_type& config, t_connection_context& conn_context); virtual ~simple_http_connection_handler(){} bool release_protocol() @@ -144,6 +144,7 @@ namespace net_utils size_t m_newlines; protected: i_service_endpoint* m_psnd_hndlr; + t_connection_context& m_conn_context; }; template<class t_connection_context> @@ -175,9 +176,8 @@ namespace net_utils typedef custum_handler_config<t_connection_context> config_type; http_custom_handler(i_service_endpoint* psnd_hndlr, config_type& config, t_connection_context& conn_context) - : simple_http_connection_handler<t_connection_context>(psnd_hndlr, config), + : simple_http_connection_handler<t_connection_context>(psnd_hndlr, config, conn_context), m_config(config), - m_conn_context(conn_context), m_auth(m_config.m_user ? http_server_auth{*m_config.m_user, config.rng} : http_server_auth{}) {} inline bool handle_request(const http_request_info& query_info, http_response_info& response) @@ -197,7 +197,7 @@ namespace net_utils response.m_response_comment = "OK"; response.m_body.clear(); - return m_config.m_phandler->handle_http_request(query_info, response, m_conn_context); + return m_config.m_phandler->handle_http_request(query_info, response, this->m_conn_context); } virtual bool thread_init() @@ -219,7 +219,6 @@ namespace net_utils private: //simple_http_connection_handler::config_type m_stub_config; config_type& m_config; - t_connection_context& m_conn_context; http_server_auth m_auth; }; } diff --git a/contrib/epee/include/net/http_protocol_handler.inl b/contrib/epee/include/net/http_protocol_handler.inl index f1da5067a..0bdba0bfe 100644 --- a/contrib/epee/include/net/http_protocol_handler.inl +++ b/contrib/epee/include/net/http_protocol_handler.inl @@ -196,16 +196,17 @@ namespace net_utils //-------------------------------------------------------------------------------------------- template<class t_connection_context> - simple_http_connection_handler<t_connection_context>::simple_http_connection_handler(i_service_endpoint* psnd_hndlr, config_type& config): + simple_http_connection_handler<t_connection_context>::simple_http_connection_handler(i_service_endpoint* psnd_hndlr, config_type& config, t_connection_context& conn_context): m_state(http_state_retriving_comand_line), m_body_transfer_type(http_body_transfer_undefined), - m_is_stop_handling(false), + m_is_stop_handling(false), m_len_summary(0), m_len_remain(0), - m_config(config), + m_config(config), m_want_close(false), m_newlines(0), - m_psnd_hndlr(psnd_hndlr) + m_psnd_hndlr(psnd_hndlr), + m_conn_context(conn_context) { } @@ -281,7 +282,7 @@ namespace net_utils m_is_stop_handling = true; if(m_cache.size() > HTTP_MAX_URI_LEN) { - LOG_ERROR("simple_http_connection_handler::handle_buff_out: Too long URI line"); + LOG_ERROR_CC(m_conn_context, "simple_http_connection_handler::handle_buff_out: Too long URI line"); m_state = http_state_error; return false; } @@ -295,7 +296,7 @@ namespace net_utils m_is_stop_handling = true; if(m_cache.size() > HTTP_MAX_HEADER_LEN) { - LOG_ERROR("simple_http_connection_handler::handle_buff_in: Too long header area"); + LOG_ERROR_CC(m_conn_context, "simple_http_connection_handler::handle_buff_in: Too long header area"); m_state = http_state_error; return false; } @@ -310,10 +311,10 @@ namespace net_utils case http_state_connection_close: return false; default: - LOG_ERROR("simple_http_connection_handler::handle_char_out: Wrong state: " << m_state); + LOG_ERROR_CC(m_conn_context, "simple_http_connection_handler::handle_char_out: Wrong state: " << m_state); return false; case http_state_error: - LOG_ERROR("simple_http_connection_handler::handle_char_out: Error state!!!"); + LOG_ERROR_CC(m_conn_context, "simple_http_connection_handler::handle_char_out: Error state!!!"); return false; } @@ -375,7 +376,7 @@ namespace net_utils }else { m_state = http_state_error; - LOG_ERROR("simple_http_connection_handler<t_connection_context>::handle_invoke_query_line(): Failed to match first line: " << m_cache); + LOG_ERROR_CC(m_conn_context, "simple_http_connection_handler<t_connection_context>::handle_invoke_query_line(): Failed to match first line: " << m_cache); return false; } @@ -399,14 +400,14 @@ namespace net_utils template<class t_connection_context> bool simple_http_connection_handler<t_connection_context>::analize_cached_request_header_and_invoke_state(size_t pos) { - //LOG_PRINT_L4("HTTP HEAD:\r\n" << m_cache.substr(0, pos)); + LOG_PRINT_L3("HTTP HEAD:\r\n" << m_cache.substr(0, pos)); m_query_info.m_full_request_buf_size = pos; m_query_info.m_request_head.assign(m_cache.begin(), m_cache.begin()+pos); if(!parse_cached_header(m_query_info.m_header_info, m_cache, pos)) { - LOG_ERROR("simple_http_connection_handler<t_connection_context>::analize_cached_request_header_and_invoke_state(): failed to anilize request header: " << m_cache); + LOG_ERROR_CC(m_conn_context, "simple_http_connection_handler<t_connection_context>::analize_cached_request_header_and_invoke_state(): failed to anilize request header: " << m_cache); m_state = http_state_error; return false; } @@ -422,7 +423,7 @@ namespace net_utils m_body_transfer_type = http_body_transfer_measure; if(!get_len_from_content_lenght(m_query_info.m_header_info.m_content_length, m_len_summary)) { - LOG_ERROR("simple_http_connection_handler<t_connection_context>::analize_cached_request_header_and_invoke_state(): Failed to get_len_from_content_lenght();, m_query_info.m_content_length="<<m_query_info.m_header_info.m_content_length); + LOG_ERROR_CC(m_conn_context, "simple_http_connection_handler<t_connection_context>::analize_cached_request_header_and_invoke_state(): Failed to get_len_from_content_lenght();, m_query_info.m_content_length="<<m_query_info.m_header_info.m_content_length); m_state = http_state_error; return false; } @@ -455,7 +456,7 @@ namespace net_utils case http_body_transfer_multipart: case http_body_transfer_undefined: default: - LOG_ERROR("simple_http_connection_handler<t_connection_context>::handle_retriving_query_body(): Unexpected m_body_query_type state:" << m_body_transfer_type); + LOG_ERROR_CC(m_conn_context, "simple_http_connection_handler<t_connection_context>::handle_retriving_query_body(): Unexpected m_body_query_type state:" << m_body_transfer_type); m_state = http_state_error; return false; } @@ -536,7 +537,7 @@ namespace net_utils body_info.m_etc_fields.push_back(std::pair<std::string, std::string>(result[field_etc_name], result[field_val])); else { - LOG_ERROR("simple_http_connection_handler<t_connection_context>::parse_cached_header() not matched last entry in:"<<m_cache_to_process); + LOG_ERROR_CC(m_conn_context, "simple_http_connection_handler<t_connection_context>::parse_cached_header() not matched last entry in:" << m_cache_to_process); } it_current_bound = result[(int)result.size()-1]. first; @@ -582,6 +583,7 @@ namespace net_utils m_psnd_hndlr->do_send((void*)response_data.data(), response_data.size()); if ((response.m_body.size() && (query_info.m_http_method != http::http_method_head)) || (query_info.m_http_method == http::http_method_options)) m_psnd_hndlr->do_send((void*)response.m_body.data(), response.m_body.size()); + m_psnd_hndlr->send_done(); return res; } //----------------------------------------------------------------------------------- diff --git a/contrib/epee/include/net/local_ip.h b/contrib/epee/include/net/local_ip.h index 0d458963c..52c5855b9 100644 --- a/contrib/epee/include/net/local_ip.h +++ b/contrib/epee/include/net/local_ip.h @@ -48,7 +48,7 @@ namespace epee if( (ip | 0xffffff00) == 0xffffffac) { - uint32_t second_num = (ip << 8) & 0xff000000; + uint32_t second_num = (ip >> 8) & 0xff; if(second_num >= 16 && second_num <= 31 ) return true; } diff --git a/contrib/epee/include/net/net_utils_base.h b/contrib/epee/include/net/net_utils_base.h index 7615786be..a133942fb 100644 --- a/contrib/epee/include/net/net_utils_base.h +++ b/contrib/epee/include/net/net_utils_base.h @@ -281,6 +281,7 @@ namespace net_utils { virtual bool do_send(const void* ptr, size_t cb)=0; virtual bool close()=0; + virtual bool send_done()=0; virtual bool call_run_once_service_io()=0; virtual bool request_callback()=0; virtual boost::asio::io_service& get_io_service()=0; diff --git a/contrib/epee/include/serialization/keyvalue_serialization_overloads.h b/contrib/epee/include/serialization/keyvalue_serialization_overloads.h index 7087136cc..09087f785 100644 --- a/contrib/epee/include/serialization/keyvalue_serialization_overloads.h +++ b/contrib/epee/include/serialization/keyvalue_serialization_overloads.h @@ -156,7 +156,7 @@ namespace epee typename stl_container::value_type* pelem = (typename stl_container::value_type*)buff.data(); CHECK_AND_ASSERT_MES(!(loaded_size%sizeof(typename stl_container::value_type)), false, - "size in blob " << loaded_size << " not have not zero modulo for sizeof(value_type) = " << sizeof(typename stl_container::value_type)); + "size in blob " << loaded_size << " not have not zero modulo for sizeof(value_type) = " << sizeof(typename stl_container::value_type) << ", type " << typeid(typename stl_container::value_type).name()); size_t count = (loaded_size/sizeof(typename stl_container::value_type)); for(size_t i = 0; i < count; i++) container.insert(container.end(), *(pelem++)); diff --git a/contrib/epee/src/mlog.cpp b/contrib/epee/src/mlog.cpp index fb0b4ac2b..e8248c958 100644 --- a/contrib/epee/src/mlog.cpp +++ b/contrib/epee/src/mlog.cpp @@ -47,6 +47,7 @@ using namespace epee; static std::string generate_log_filename(const char *base) { std::string filename(base); + static unsigned int fallback_counter = 0; char tmp[200]; struct tm tm; time_t now = time(NULL); @@ -56,7 +57,7 @@ static std::string generate_log_filename(const char *base) #else (!gmtime_r(&now, &tm)) #endif - strcpy(tmp, "unknown"); + snprintf(tmp, sizeof(tmp), "part-%u", ++fallback_counter); else strftime(tmp, sizeof(tmp), "%Y-%m-%d-%H-%M-%S", &tm); tmp[sizeof(tmp) - 1] = 0; @@ -116,7 +117,7 @@ static const char *get_default_categories(int level) return categories; } -void mlog_configure(const std::string &filename_base, bool console, const std::size_t max_log_file_size) +void mlog_configure(const std::string &filename_base, bool console, const std::size_t max_log_file_size, const std::size_t max_log_files) { el::Configurations c; c.setGlobally(el::ConfigurationType::Filename, filename_base); @@ -134,9 +135,58 @@ void mlog_configure(const std::string &filename_base, bool console, const std::s el::Loggers::addFlag(el::LoggingFlag::DisableApplicationAbortOnFatalLog); el::Loggers::addFlag(el::LoggingFlag::ColoredTerminalOutput); el::Loggers::addFlag(el::LoggingFlag::StrictLogFileSizeCheck); - el::Helpers::installPreRollOutCallback([filename_base](const char *name, size_t){ + el::Helpers::installPreRollOutCallback([filename_base, max_log_files](const char *name, size_t){ std::string rname = generate_log_filename(filename_base.c_str()); rename(name, rname.c_str()); + if (max_log_files != 0) + { + std::vector<boost::filesystem::path> found_files; + const boost::filesystem::directory_iterator end_itr; + for (boost::filesystem::directory_iterator iter(boost::filesystem::path(filename_base).parent_path()); iter != end_itr; ++iter) + { + const std::string filename = iter->path().string(); + if (filename.size() >= filename_base.size() && std::memcmp(filename.data(), filename_base.data(), filename_base.size()) == 0) + { + found_files.push_back(iter->path()); + } + } + if (found_files.size() >= max_log_files) + { + std::sort(found_files.begin(), found_files.end(), [](const boost::filesystem::path &a, const boost::filesystem::path &b) { + boost::system::error_code ec; + std::time_t ta = boost::filesystem::last_write_time(boost::filesystem::path(a), ec); + if (ec) + { + MERROR("Failed to get timestamp from " << a << ": " << ec); + ta = std::time(nullptr); + } + std::time_t tb = boost::filesystem::last_write_time(boost::filesystem::path(b), ec); + if (ec) + { + MERROR("Failed to get timestamp from " << b << ": " << ec); + tb = std::time(nullptr); + } + static_assert(std::is_integral<time_t>(), "bad time_t"); + return ta < tb; + }); + for (size_t i = 0; i <= found_files.size() - max_log_files; ++i) + { + try + { + boost::system::error_code ec; + boost::filesystem::remove(found_files[i], ec); + if (ec) + { + MERROR("Failed to remove " << found_files[i] << ": " << ec); + } + } + catch (const std::exception &e) + { + MERROR("Failed to remove " << found_files[i] << ": " << e.what()); + } + } + } + } }); mlog_set_common_prefix(); const char *monero_log = getenv("MONERO_LOGS"); diff --git a/contrib/snap/setup/gui/icon.png b/contrib/snap/setup/gui/icon.png Binary files differindex 17b8bd47b..b7e821270 100644 --- a/contrib/snap/setup/gui/icon.png +++ b/contrib/snap/setup/gui/icon.png diff --git a/external/easylogging++/easylogging++.cc b/external/easylogging++/easylogging++.cc index 212a1822d..8a5dc91e9 100644 --- a/external/easylogging++/easylogging++.cc +++ b/external/easylogging++/easylogging++.cc @@ -1967,10 +1967,12 @@ void VRegistry::setCategories(const char* categories, bool clear) { base::threading::ScopedLock scopedLock(lock()); auto insert = [&](std::stringstream& ss, Level level) { m_categories.push_back(std::make_pair(ss.str(), level)); + m_cached_allowed_categories.clear(); }; if (clear) { m_categories.clear(); + m_cached_allowed_categories.clear(); m_categoriesString.clear(); } if (!m_categoriesString.empty()) @@ -2033,15 +2035,22 @@ static int priority(Level level) { bool VRegistry::allowed(Level level, const char* category) { base::threading::ScopedLock scopedLock(lock()); + const std::string scategory = category; + const std::map<std::string, int>::const_iterator it = m_cached_allowed_categories.find(scategory); + if (it != m_cached_allowed_categories.end()) + return priority(level) <= it->second; if (m_categories.empty() || category == nullptr) { return false; } else { std::deque<std::pair<std::string, Level>>::const_reverse_iterator it = m_categories.rbegin(); for (; it != m_categories.rend(); ++it) { if (base::utils::Str::wildCardMatch(category, it->first.c_str())) { - return priority(level) <= priority(it->second); + const int p = priority(it->second); + m_cached_allowed_categories.insert(std::make_pair(std::move(scategory), p)); + return priority(level) <= p; } } + m_cached_allowed_categories.insert(std::make_pair(std::move(scategory), -1)); return false; } } diff --git a/external/easylogging++/easylogging++.h b/external/easylogging++/easylogging++.h index 3270bd607..6b8b4fc35 100644 --- a/external/easylogging++/easylogging++.h +++ b/external/easylogging++/easylogging++.h @@ -2485,6 +2485,7 @@ class VRegistry : base::NoCopy, public base::threading::ThreadSafe { inline void clearCategories(void) { base::threading::ScopedLock scopedLock(lock()); m_categories.clear(); + m_cached_allowed_categories.clear(); } inline void clearModules(void) { @@ -2526,6 +2527,7 @@ class VRegistry : base::NoCopy, public base::threading::ThreadSafe { base::type::EnumType* m_pFlags; std::map<std::string, base::type::VerboseLevel> m_modules; std::deque<std::pair<std::string, Level>> m_categories; + std::map<std::string, int> m_cached_allowed_categories; std::string m_categoriesString; std::string m_filenameCommonPrefix; }; diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 19ba32340..564016fc9 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -148,6 +148,7 @@ struct txpool_tx_meta_t uint8_t relayed; uint8_t do_not_relay; uint8_t double_spend_seen: 1; + uint8_t bf_padding: 7; uint8_t padding[76]; // till 192 bytes }; diff --git a/src/blockchain_utilities/blockchain_blackball.cpp b/src/blockchain_utilities/blockchain_blackball.cpp index 95eb2f73d..52d2938cd 100644 --- a/src/blockchain_utilities/blockchain_blackball.cpp +++ b/src/blockchain_utilities/blockchain_blackball.cpp @@ -235,7 +235,7 @@ int main(int argc, char* argv[]) "blackball-db-dir", "Specify blackball database directory", get_default_db_path(), {{ &arg_testnet_on, &arg_stagenet_on }}, - [](std::array<bool, 2> testnet_stagenet, bool defaulted, std::string val) { + [](std::array<bool, 2> testnet_stagenet, bool defaulted, std::string val)->std::string { if (testnet_stagenet[0]) return (boost::filesystem::path(val) / "testnet").string(); else if (testnet_stagenet[1]) diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp index caa549c13..c76641598 100644 --- a/src/blockchain_utilities/blockchain_import.cpp +++ b/src/blockchain_utilities/blockchain_import.cpp @@ -164,7 +164,7 @@ int pop_blocks(cryptonote::core& core, int num_blocks) return num_blocks; } -int check_flush(cryptonote::core &core, std::list<block_complete_entry> &blocks, bool force) +int check_flush(cryptonote::core &core, std::vector<block_complete_entry> &blocks, bool force) { if (blocks.empty()) return 0; @@ -176,7 +176,7 @@ int check_flush(cryptonote::core &core, std::list<block_complete_entry> &blocks, if (!force && new_height % HASH_OF_HASHES_STEP) return 0; - std::list<crypto::hash> hashes; + std::vector<crypto::hash> hashes; for (const auto &b: blocks) { cryptonote::block block; @@ -312,7 +312,7 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path MINFO("Reading blockchain from bootstrap file..."); std::cout << ENDL; - std::list<block_complete_entry> blocks; + std::vector<block_complete_entry> blocks; // Skip to start_height before we start adding. { @@ -437,7 +437,7 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path { cryptonote::blobdata block; cryptonote::block_to_blob(bp.block, block); - std::list<cryptonote::blobdata> txs; + std::vector<cryptonote::blobdata> txs; for (const auto &tx: bp.txs) { txs.push_back(cryptonote::blobdata()); @@ -593,8 +593,8 @@ int main(int argc, char* argv[]) const command_line::arg_descriptor<std::string> arg_database = { "database", available_dbs.c_str(), default_db_type }; - const command_line::arg_descriptor<bool> arg_verify = {"guard-against-pwnage", - "Verify blocks and transactions during import (only disable if you exported the file yourself)", true}; + const command_line::arg_descriptor<bool> arg_noverify = {"dangerous-unverified-import", + "Blindly trust the import file and use potentially malicious blocks and transactions during import (only enable if you exported the file yourself)", false}; const command_line::arg_descriptor<bool> arg_batch = {"batch", "Batch transactions for faster import", true}; const command_line::arg_descriptor<bool> arg_resume = {"resume", @@ -614,7 +614,7 @@ int main(int argc, char* argv[]) // call add_options() directly for these arguments since // command_line helpers support only boolean switch, not boolean argument desc_cmd_sett.add_options() - (arg_verify.name, make_semantic(arg_verify), arg_verify.description) + (arg_noverify.name, make_semantic(arg_noverify), arg_noverify.description) (arg_batch.name, make_semantic(arg_batch), arg_batch.description) (arg_resume.name, make_semantic(arg_resume), arg_resume.description) ; @@ -633,7 +633,7 @@ int main(int argc, char* argv[]) if (! r) return 1; - opt_verify = command_line::get_arg(vm, arg_verify); + opt_verify = !command_line::get_arg(vm, arg_noverify); opt_batch = command_line::get_arg(vm, arg_batch); opt_resume = command_line::get_arg(vm, arg_resume); block_stop = command_line::get_arg(vm, arg_block_stop); @@ -738,6 +738,18 @@ int main(int argc, char* argv[]) MINFO("bootstrap file path: " << import_file_path); MINFO("database path: " << m_config_folder); + if (!opt_verify) + { + MCLOG_RED(el::Level::Warning, "global", "\n" + "Import is set to proceed WITHOUT VERIFICATION.\n" + "This is a DANGEROUS operation: if the file was tampered with in transit, or obtained from a malicious source,\n" + "you could end up with a compromised database. It is recommended to NOT use " << arg_noverify.name << ".\n" + "*****************************************************************************************\n" + "You have 90 seconds to press ^C or terminate this program before unverified import starts\n" + "*****************************************************************************************"); + sleep(90); + } + cryptonote::cryptonote_protocol_stub pr; //TODO: stub only for this kind of test, make real validation of relayed objects cryptonote::core core(&pr); diff --git a/src/common/password.cpp b/src/common/password.cpp index 9336a14fc..3ce2ba42a 100644 --- a/src/common/password.cpp +++ b/src/common/password.cpp @@ -164,7 +164,7 @@ namespace while (true) { if (message) - std::cout << message <<": "; + std::cout << message <<": " << std::flush; if (!read_from_tty(pass1)) return false; if (verify) diff --git a/src/common/threadpool.cpp b/src/common/threadpool.cpp index 51e071577..6b69e2a12 100644 --- a/src/common/threadpool.cpp +++ b/src/common/threadpool.cpp @@ -36,16 +36,17 @@ #include "common/util.h" static __thread int depth = 0; +static __thread bool is_leaf = false; namespace tools { -threadpool::threadpool() : running(true), active(0) { +threadpool::threadpool(unsigned int max_threads) : running(true), active(0) { boost::thread::attributes attrs; attrs.set_stack_size(THREAD_STACK_SIZE); - max = tools::get_max_concurrency(); - size_t i = max; + max = max_threads ? max_threads : tools::get_max_concurrency(); + size_t i = max ? max - 1 : 0; while(i--) { - threads.push_back(boost::thread(attrs, boost::bind(&threadpool::run, this))); + threads.push_back(boost::thread(attrs, boost::bind(&threadpool::run, this, false))); } } @@ -60,25 +61,30 @@ threadpool::~threadpool() { } } -void threadpool::submit(waiter *obj, std::function<void()> f) { - entry e = {obj, f}; +void threadpool::submit(waiter *obj, std::function<void()> f, bool leaf) { + CHECK_AND_ASSERT_THROW_MES(!is_leaf, "A leaf routine is using a thread pool"); boost::unique_lock<boost::mutex> lock(mutex); - if ((active == max && !queue.empty()) || depth > 0) { + if (!leaf && ((active == max && !queue.empty()) || depth > 0)) { // if all available threads are already running // and there's work waiting, just run in current thread lock.unlock(); ++depth; + is_leaf = leaf; f(); --depth; + is_leaf = false; } else { if (obj) obj->inc(); - queue.push_back(e); + if (leaf) + queue.push_front({obj, f, leaf}); + else + queue.push_back({obj, f, leaf}); has_work.notify_one(); } } -int threadpool::get_max_concurrency() { +unsigned int threadpool::get_max_concurrency() const { return max; } @@ -91,7 +97,7 @@ threadpool::waiter::~waiter() } try { - wait(); + wait(NULL); } catch (const std::exception &e) { @@ -99,9 +105,12 @@ threadpool::waiter::~waiter() } } -void threadpool::waiter::wait() { +void threadpool::waiter::wait(threadpool *tpool) { + if (tpool) + tpool->run(true); boost::unique_lock<boost::mutex> lock(mt); - while(num) cv.wait(lock); + while(num) + cv.wait(lock); } void threadpool::waiter::inc() { @@ -113,15 +122,19 @@ void threadpool::waiter::dec() { const boost::unique_lock<boost::mutex> lock(mt); num--; if (!num) - cv.notify_one(); + cv.notify_all(); } -void threadpool::run() { +void threadpool::run(bool flush) { boost::unique_lock<boost::mutex> lock(mutex); while (running) { entry e; while(queue.empty() && running) + { + if (flush) + return; has_work.wait(lock); + } if (!running) break; active++; @@ -129,8 +142,10 @@ void threadpool::run() { queue.pop_front(); lock.unlock(); ++depth; + is_leaf = e.leaf; e.f(); --depth; + is_leaf = false; if (e.wo) e.wo->dec(); diff --git a/src/common/threadpool.h b/src/common/threadpool.h index 34152541c..a43e38a76 100644 --- a/src/common/threadpool.h +++ b/src/common/threadpool.h @@ -46,6 +46,9 @@ public: static threadpool instance; return instance; } + static threadpool *getNewForUnitTests(unsigned max_threads = 0) { + return new threadpool(max_threads); + } // The waiter lets the caller know when all of its // tasks are completed. @@ -56,7 +59,7 @@ public: public: void inc(); void dec(); - void wait(); //! Wait for a set of tasks to finish. + void wait(threadpool *tpool); //! Wait for a set of tasks to finish. waiter() : num(0){} ~waiter(); }; @@ -64,25 +67,27 @@ public: // Submit a task to the pool. The waiter pointer may be // NULL if the caller doesn't care to wait for the // task to finish. - void submit(waiter *waiter, std::function<void()> f); + void submit(waiter *waiter, std::function<void()> f, bool leaf = false); + + unsigned int get_max_concurrency() const; - int get_max_concurrency(); + ~threadpool(); private: - threadpool(); - ~threadpool(); + threadpool(unsigned int max_threads = 0); typedef struct entry { waiter *wo; std::function<void()> f; + bool leaf; } entry; std::deque<entry> queue; boost::condition_variable has_work; boost::mutex mutex; std::vector<boost::thread> threads; - int active; - int max; + unsigned int active; + unsigned int max; bool running; - void run(); + void run(bool flush = false); }; } diff --git a/src/common/util.cpp b/src/common/util.cpp index 008610117..329352e94 100644 --- a/src/common/util.cpp +++ b/src/common/util.cpp @@ -727,6 +727,13 @@ std::string get_nix_version_display_string() bool is_local_address(const std::string &address) { + // always assume Tor/I2P addresses to be untrusted by default + if (boost::ends_with(address, ".onion") || boost::ends_with(address, ".i2p")) + { + MDEBUG("Address '" << address << "' is Tor/I2P, non local"); + return false; + } + // extract host epee::net_utils::http::url_content u_c; if (!epee::net_utils::parse_url(address, u_c)) @@ -820,4 +827,22 @@ std::string get_nix_version_display_string() return false; return true; } + + boost::optional<std::pair<uint32_t, uint32_t>> parse_subaddress_lookahead(const std::string& str) + { + auto pos = str.find(":"); + bool r = pos != std::string::npos; + uint32_t major; + r = r && epee::string_tools::get_xtype_from_string(major, str.substr(0, pos)); + uint32_t minor; + r = r && epee::string_tools::get_xtype_from_string(minor, str.substr(pos + 1)); + if (r) + { + return std::make_pair(major, minor); + } + else + { + return {}; + } + } } diff --git a/src/common/util.h b/src/common/util.h index 7caf0e3c5..dc426830b 100644 --- a/src/common/util.h +++ b/src/common/util.h @@ -32,6 +32,7 @@ #include <boost/thread/locks.hpp> #include <boost/thread/mutex.hpp> +#include <boost/optional.hpp> #include <system_error> #include <csignal> #include <cstdio> @@ -214,4 +215,6 @@ namespace tools bool sha256sum(const std::string &filename, crypto::hash &hash); bool is_hdd(const char *path); + + boost::optional<std::pair<uint32_t, uint32_t>> parse_subaddress_lookahead(const std::string& str); } diff --git a/src/cryptonote_basic/account.cpp b/src/cryptonote_basic/account.cpp index bab991d19..aac6ec22b 100644 --- a/src/cryptonote_basic/account.cpp +++ b/src/cryptonote_basic/account.cpp @@ -157,7 +157,7 @@ DISABLE_VS_WARNINGS(4244 4345) void account_base::create_from_viewkey(const cryptonote::account_public_address& address, const crypto::secret_key& viewkey) { crypto::secret_key fake; - memset(&fake, 0, sizeof(fake)); + memset(&unwrap(fake), 0, sizeof(fake)); create_from_keys(address, fake, viewkey); } //----------------------------------------------------------------- diff --git a/src/cryptonote_basic/connection_context.h b/src/cryptonote_basic/connection_context.h index 5cd1709ab..eb73ab0ea 100644 --- a/src/cryptonote_basic/connection_context.h +++ b/src/cryptonote_basic/connection_context.h @@ -52,7 +52,7 @@ namespace cryptonote }; state m_state; - std::list<crypto::hash> m_needed_objects; + std::vector<crypto::hash> m_needed_objects; std::unordered_set<crypto::hash> m_requested_objects; uint64_t m_remote_blockchain_height; uint64_t m_last_response_height; @@ -67,15 +67,15 @@ namespace cryptonote switch (s) { case cryptonote_connection_context::state_before_handshake: - return "state_before_handshake"; + return "before_handshake"; case cryptonote_connection_context::state_synchronizing: - return "state_synchronizing"; + return "synchronizing"; case cryptonote_connection_context::state_standby: - return "state_standby"; + return "standby"; case cryptonote_connection_context::state_idle: - return "state_idle"; + return "idle"; case cryptonote_connection_context::state_normal: - return "state_normal"; + return "normal"; default: return "unknown"; } diff --git a/src/cryptonote_basic/miner.cpp b/src/cryptonote_basic/miner.cpp index 2c777f5a2..3a3222f9b 100644 --- a/src/cryptonote_basic/miner.cpp +++ b/src/cryptonote_basic/miner.cpp @@ -328,6 +328,11 @@ namespace cryptonote LOG_PRINT_L0("Background mining controller thread started" ); } + if(get_ignore_battery()) + { + MINFO("Ignoring battery"); + } + return true; } //----------------------------------------------------------------------------------------------------- diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index d4509e4a3..092a649e5 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -242,6 +242,7 @@ bool Blockchain::scan_outputkeys_for_indexes(size_t tx_version, const txin_to_ke MDEBUG("Additional outputs needed: " << absolute_offsets.size() - outputs.size()); std::vector < uint64_t > add_offsets; std::vector<output_data_t> add_outputs; + add_outputs.reserve(absolute_offsets.size() - outputs.size()); for (size_t i = outputs.size(); i < absolute_offsets.size(); i++) add_offsets.push_back(absolute_offsets[i]); try @@ -433,7 +434,7 @@ bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline m_db->block_txn_stop(); uint64_t num_popped_blocks = 0; - while (true) + while (!m_db->is_read_only()) { const uint64_t top_height = m_db->height() - 1; const crypto::hash top_id = m_db->top_block_hash(); @@ -816,7 +817,7 @@ difficulty_type Blockchain::get_difficulty_for_next_block() // then when the next block difficulty is queried, push the latest height data and // pop the oldest one from the list. This only requires 1x read per height instead // of doing 735 (DIFFICULTY_BLOCKS_COUNT). - if (m_timestamps_and_difficulties_height != 0 && ((height - m_timestamps_and_difficulties_height) == 1)) + if (m_timestamps_and_difficulties_height != 0 && ((height - m_timestamps_and_difficulties_height) == 1) && m_timestamps.size() >= DIFFICULTY_BLOCKS_COUNT) { uint64_t index = height - 1; m_timestamps.push_back(m_db->get_block_timestamp(index)); @@ -839,6 +840,11 @@ difficulty_type Blockchain::get_difficulty_for_next_block() timestamps.clear(); difficulties.clear(); + if (height > offset) + { + timestamps.reserve(height - offset); + difficulties.reserve(height - offset); + } for (; offset < height; offset++) { timestamps.push_back(m_db->get_block_timestamp(offset)); @@ -1159,6 +1165,7 @@ void Blockchain::get_last_n_blocks_sizes(std::vector<size_t>& sz, size_t count) m_db->block_txn_start(true); // add size of last <count> blocks to vector <sz> (or less, if blockchain size < count) size_t start_offset = h - std::min<size_t>(h, count); + sz.reserve(sz.size() + h - start_offset); for(size_t i = start_offset; i < h; i++) { sz.push_back(m_db->get_block_size(i)); @@ -1356,6 +1363,7 @@ bool Blockchain::complete_timestamps_vector(uint64_t start_top_height, std::vect size_t need_elements = BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW - timestamps.size(); CHECK_AND_ASSERT_MES(start_top_height < m_db->height(), false, "internal error: passed start_height not < " << " m_db->height() -- " << start_top_height << " >= " << m_db->height()); size_t stop_offset = start_top_height > need_elements ? start_top_height - need_elements : 0; + timestamps.reserve(timestamps.size() + start_top_height - stop_offset); while (start_top_height != stop_offset) { timestamps.push_back(m_db->get_block_timestamp(start_top_height)); @@ -1555,7 +1563,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id return true; } //------------------------------------------------------------------ -bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::list<std::pair<cryptonote::blobdata,block>>& blocks, std::list<cryptonote::blobdata>& txs) const +bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::vector<std::pair<cryptonote::blobdata,block>>& blocks, std::vector<cryptonote::blobdata>& txs) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -1569,7 +1577,7 @@ bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::list<std:: for(const auto& blk : blocks) { - std::list<crypto::hash> missed_ids; + std::vector<crypto::hash> missed_ids; get_transactions_blobs(blk.second.tx_hashes, txs, missed_ids); CHECK_AND_ASSERT_MES(!missed_ids.size(), false, "has missed transactions in own block in main blockchain"); } @@ -1577,14 +1585,16 @@ bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::list<std:: return true; } //------------------------------------------------------------------ -bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::list<std::pair<cryptonote::blobdata,block>>& blocks) const +bool Blockchain::get_blocks(uint64_t start_offset, size_t count, std::vector<std::pair<cryptonote::blobdata,block>>& blocks) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); - if(start_offset >= m_db->height()) + const uint64_t height = m_db->height(); + if(start_offset >= height) return false; - for(size_t i = start_offset; i < start_offset + count && i < m_db->height();i++) + blocks.reserve(blocks.size() + height - start_offset); + for(size_t i = start_offset; i < start_offset + count && i < height;i++) { blocks.push_back(std::make_pair(m_db->get_block_blob_from_height(i), block())); if (!parse_and_validate_block_from_blob(blocks.back().first, blocks.back().second)) @@ -1609,17 +1619,20 @@ bool Blockchain::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NO CRITICAL_REGION_LOCAL(m_blockchain_lock); m_db->block_txn_start(true); rsp.current_blockchain_height = get_current_blockchain_height(); - std::list<std::pair<cryptonote::blobdata,block>> blocks; + std::vector<std::pair<cryptonote::blobdata,block>> blocks; get_blocks(arg.blocks, blocks, rsp.missed_ids); - for (const auto& bl: blocks) + for (auto& bl: blocks) { - std::list<crypto::hash> missed_tx_ids; - std::list<cryptonote::blobdata> txs; + std::vector<crypto::hash> missed_tx_ids; + std::vector<cryptonote::blobdata> txs; + + rsp.blocks.push_back(block_complete_entry()); + block_complete_entry& e = rsp.blocks.back(); // FIXME: s/rsp.missed_ids/missed_tx_id/ ? Seems like rsp.missed_ids // is for missed blocks, not missed transactions as well. - get_transactions_blobs(bl.second.tx_hashes, txs, missed_tx_ids); + get_transactions_blobs(bl.second.tx_hashes, e.txs, missed_tx_ids); if (missed_tx_ids.size() != 0) { @@ -1631,35 +1644,28 @@ bool Blockchain::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NO // append missed transaction hashes to response missed_ids field, // as done below if any standalone transactions were requested // and missed. - rsp.missed_ids.splice(rsp.missed_ids.end(), missed_tx_ids); - m_db->block_txn_stop(); + rsp.missed_ids.insert(rsp.missed_ids.end(), missed_tx_ids.begin(), missed_tx_ids.end()); + m_db->block_txn_stop(); return false; } - rsp.blocks.push_back(block_complete_entry()); - block_complete_entry& e = rsp.blocks.back(); //pack block - e.block = bl.first; - //pack transactions - for (const cryptonote::blobdata& tx: txs) - e.txs.push_back(tx); - } - //get another transactions, if need - std::list<cryptonote::blobdata> txs; - get_transactions_blobs(arg.txs, txs, rsp.missed_ids); - //pack aside transactions - for (const auto& tx: txs) - rsp.txs.push_back(tx); + e.block = std::move(bl.first); + } + //get and pack other transactions, if needed + std::vector<cryptonote::blobdata> txs; + get_transactions_blobs(arg.txs, rsp.txs, rsp.missed_ids); m_db->block_txn_stop(); return true; } //------------------------------------------------------------------ -bool Blockchain::get_alternative_blocks(std::list<block>& blocks) const +bool Blockchain::get_alternative_blocks(std::vector<block>& blocks) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); + blocks.reserve(m_alternative_chains.size()); for (const auto& alt_bl: m_alternative_chains) { blocks.push_back(alt_bl.second.bl); @@ -1949,14 +1955,21 @@ bool Blockchain::get_outs(const COMMAND_RPC_GET_OUTPUTS_BIN::request& req, COMMA res.outs.clear(); res.outs.reserve(req.outputs.size()); - for (const auto &i: req.outputs) + try { - // get tx_hash, tx_out_index from DB - const output_data_t od = m_db->get_output_key(i.amount, i.index); - tx_out_index toi = m_db->get_output_tx_and_index(i.amount, i.index); - bool unlocked = is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first)); + for (const auto &i: req.outputs) + { + // get tx_hash, tx_out_index from DB + const output_data_t od = m_db->get_output_key(i.amount, i.index); + tx_out_index toi = m_db->get_output_tx_and_index(i.amount, i.index); + bool unlocked = is_tx_spendtime_unlocked(m_db->get_tx_unlock_time(toi.first)); - res.outs.push_back({od.pubkey, od.commitment, unlocked, od.height, toi.first}); + res.outs.push_back({od.pubkey, od.commitment, unlocked, od.height, toi.first}); + } + } + catch (const std::exception &e) + { + return false; } return true; } @@ -2072,6 +2085,9 @@ uint64_t Blockchain::block_difficulty(uint64_t i) const return 0; } //------------------------------------------------------------------ +template<typename T> void reserve_container(std::vector<T> &v, size_t N) { v.reserve(N); } +template<typename T> void reserve_container(std::list<T> &v, size_t N) { } +//------------------------------------------------------------------ //TODO: return type should be void, throw on exception // alternatively, return true only if no blocks missed template<class t_ids_container, class t_blocks_container, class t_missed_container> @@ -2080,20 +2096,24 @@ bool Blockchain::get_blocks(const t_ids_container& block_ids, t_blocks_container LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); + reserve_container(blocks, block_ids.size()); for (const auto& block_hash : block_ids) { try { - blocks.push_back(std::make_pair(m_db->get_block_blob(block_hash), block())); - if (!parse_and_validate_block_from_blob(blocks.back().first, blocks.back().second)) + uint64_t height = 0; + if (m_db->block_exists(block_hash, &height)) { - LOG_ERROR("Invalid block"); - return false; + blocks.push_back(std::make_pair(m_db->get_block_blob_from_height(height), block())); + if (!parse_and_validate_block_from_blob(blocks.back().first, blocks.back().second)) + { + LOG_ERROR("Invalid block: " << block_hash); + blocks.pop_back(); + missed_bs.push_back(block_hash); + } } - } - catch (const BLOCK_DNE& e) - { - missed_bs.push_back(block_hash); + else + missed_bs.push_back(block_hash); } catch (const std::exception& e) { @@ -2111,6 +2131,7 @@ bool Blockchain::get_transactions_blobs(const t_ids_container& txs_ids, t_tx_con LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); + reserve_container(txs, txs_ids.size()); for (const auto& tx_hash : txs_ids) { try @@ -2137,6 +2158,7 @@ bool Blockchain::get_transactions(const t_ids_container& txs_ids, t_tx_container LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); + reserve_container(txs, txs_ids.size()); for (const auto& tx_hash : txs_ids) { try @@ -2165,7 +2187,7 @@ bool Blockchain::get_transactions(const t_ids_container& txs_ids, t_tx_container // Find the split point between us and foreign blockchain and return // (by reference) the most recent common block hash along with up to // BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT additional (more recent) hashes. -bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, std::list<crypto::hash>& hashes, uint64_t& start_height, uint64_t& current_height) const +bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, std::vector<crypto::hash>& hashes, uint64_t& start_height, uint64_t& current_height) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -2179,6 +2201,7 @@ bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qbloc m_db->block_txn_start(true); current_height = get_current_blockchain_height(); size_t count = 0; + hashes.reserve(std::max((size_t)(current_height - start_height), (size_t)BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT)); for(size_t i = start_height; i < current_height && count < BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT; i++, count++) { hashes.push_back(m_db->get_block_hash_from_height(i)); @@ -2194,7 +2217,7 @@ bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qbloc CRITICAL_REGION_LOCAL(m_blockchain_lock); bool result = find_blockchain_supplement(qblock_ids, resp.m_block_ids, resp.start_height, resp.total_height); - resp.cumulative_difficulty = m_db->get_block_cumulative_difficulty(m_db->height() - 1); + resp.cumulative_difficulty = m_db->get_block_cumulative_difficulty(resp.total_height - 1); return result; } @@ -2203,7 +2226,7 @@ bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qbloc // find split point between ours and foreign blockchain (or start at // blockchain height <req_start_block>), and return up to max_count FULL // blocks by reference. -bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::list<std::pair<cryptonote::blobdata, std::list<cryptonote::blobdata> > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, size_t max_count) const +bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_count) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -2229,18 +2252,28 @@ bool Blockchain::find_blockchain_supplement(const uint64_t req_start_block, cons m_db->block_txn_start(true); total_height = get_current_blockchain_height(); size_t count = 0, size = 0; + blocks.reserve(std::min(std::min(max_count, (size_t)10000), (size_t)(total_height - start_height))); for(size_t i = start_height; i < total_height && count < max_count && (size < FIND_BLOCKCHAIN_SUPPLEMENT_MAX_SIZE || count < 3); i++, count++) { blocks.resize(blocks.size()+1); - blocks.back().first = m_db->get_block_blob_from_height(i); + blocks.back().first.first = m_db->get_block_blob_from_height(i); block b; - CHECK_AND_ASSERT_MES(parse_and_validate_block_from_blob(blocks.back().first, b), false, "internal error, invalid block"); - std::list<crypto::hash> mis; - get_transactions_blobs(b.tx_hashes, blocks.back().second, mis, pruned); + CHECK_AND_ASSERT_MES(parse_and_validate_block_from_blob(blocks.back().first.first, b), false, "internal error, invalid block"); + blocks.back().first.second = get_miner_tx_hash ? cryptonote::get_transaction_hash(b.miner_tx) : crypto::null_hash; + std::vector<crypto::hash> mis; + std::vector<cryptonote::blobdata> txs; + get_transactions_blobs(b.tx_hashes, txs, mis, pruned); CHECK_AND_ASSERT_MES(!mis.size(), false, "internal error, transaction from block not found"); - size += blocks.back().first.size(); - for (const auto &t: blocks.back().second) + size += blocks.back().first.first.size(); + for (const auto &t: txs) size += t.size(); + + CHECK_AND_ASSERT_MES(txs.size() == b.tx_hashes.size(), false, "mismatched sizes of b.tx_hashes and txs"); + blocks.back().second.reserve(txs.size()); + for (size_t i = 0; i < txs.size(); ++i) + { + blocks.back().second.push_back(std::make_pair(b.tx_hashes[i], std::move(txs[i]))); + } } m_db->block_txn_stop(); return true; @@ -2271,19 +2304,19 @@ bool Blockchain::have_block(const crypto::hash& id) const if(m_db->block_exists(id)) { - LOG_PRINT_L3("block exists in main chain"); + LOG_PRINT_L2("block " << id << " found in main chain"); return true; } if(m_alternative_chains.count(id)) { - LOG_PRINT_L3("block found in m_alternative_chains"); + LOG_PRINT_L2("block " << id << " found in m_alternative_chains"); return true; } if(m_invalid_blocks.count(id)) { - LOG_PRINT_L3("block found in m_invalid_blocks"); + LOG_PRINT_L2("block " << id << " found in m_invalid_blocks"); return true; } @@ -2774,7 +2807,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, { // ND: Speedup // 1. Thread ring signature verification if possible. - tpool.submit(&waiter, boost::bind(&Blockchain::check_ring_signature, this, std::cref(tx_prefix_hash), std::cref(in_to_key.k_image), std::cref(pubkeys[sig_index]), std::cref(tx.signatures[sig_index]), std::ref(results[sig_index]))); + tpool.submit(&waiter, boost::bind(&Blockchain::check_ring_signature, this, std::cref(tx_prefix_hash), std::cref(in_to_key.k_image), std::cref(pubkeys[sig_index]), std::cref(tx.signatures[sig_index]), std::ref(results[sig_index])), true); } else { @@ -2798,7 +2831,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, sig_index++; } if (tx.version == 1 && threads > 1) - waiter.wait(); + waiter.wait(&tpool); if (tx.version == 1) { @@ -2970,6 +3003,7 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, void Blockchain::check_ring_signature(const crypto::hash &tx_prefix_hash, const crypto::key_image &key_image, const std::vector<rct::ctkey> &pubkeys, const std::vector<crypto::signature>& sig, uint64_t &result) { std::vector<const crypto::public_key *> p_output_keys; + p_output_keys.reserve(pubkeys.size()); for (auto &key : pubkeys) { // rct::key and crypto::public_key have the same structure, avoid object ctor/memcpy @@ -3066,6 +3100,7 @@ uint64_t Blockchain::get_dynamic_per_kb_fee_estimate(uint64_t grace_blocks) cons const uint64_t min_block_size = get_min_block_size(version); std::vector<size_t> sz; get_last_n_blocks_sizes(sz, CRYPTONOTE_REWARD_BLOCKS_WINDOW - grace_blocks); + sz.reserve(grace_blocks); for (size_t i = 0; i < grace_blocks; ++i) sz.push_back(min_block_size); @@ -3223,6 +3258,7 @@ bool Blockchain::check_block_timestamp(const block& b, uint64_t& median_ts) cons // need most recent 60 blocks, get index of first of those size_t offset = h - BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW; + timestamps.reserve(h - offset); for(;offset < h; ++offset) { timestamps.push_back(m_db->get_block_timestamp(offset)); @@ -3249,7 +3285,7 @@ void Blockchain::return_tx_to_pool(std::vector<transaction> &txs) } } //------------------------------------------------------------------ -bool Blockchain::flush_txes_from_pool(const std::list<crypto::hash> &txids) +bool Blockchain::flush_txes_from_pool(const std::vector<crypto::hash> &txids) { CRITICAL_REGION_LOCAL(m_tx_pool); @@ -3439,6 +3475,7 @@ leave: // Iterate over the block's transaction hashes, grabbing each // from the tx_pool and validating them. Each is then added // to txs. Keys spent in each are added to <keys> by the double spend check. + txs.reserve(bl.tx_hashes.size()); for (const crypto::hash& tx_id : bl.tx_hashes) { transaction tx; @@ -3852,7 +3889,7 @@ void Blockchain::output_scan_worker(const uint64_t amount, const std::vector<uin } } -uint64_t Blockchain::prevalidate_block_hashes(uint64_t height, const std::list<crypto::hash> &hashes) +uint64_t Blockchain::prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes) { // new: . . . . . X X X X X . . . . . . // pre: A A A A B B B B C C C C D D D D @@ -3955,7 +3992,7 @@ uint64_t Blockchain::prevalidate_block_hashes(uint64_t height, const std::list<c // vs [k_image, output_keys] (m_scan_table). This is faster because it takes advantage of bulk queries // and is threaded if possible. The table (m_scan_table) will be used later when querying output // keys. -bool Blockchain::prepare_handle_incoming_blocks(const std::list<block_complete_entry> &blocks_entry) +bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete_entry> &blocks_entry) { MTRACE("Blockchain::" << __func__); TIME_MEASURE_START(prepare); @@ -4021,6 +4058,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::list<block_complete_e for (uint64_t i = 0; i < threads; i++) { + blocks[i].reserve(batches + 1); for (int j = 0; j < batches; j++) { block block; @@ -4079,11 +4117,11 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::list<block_complete_e tools::threadpool::waiter waiter; for (uint64_t i = 0; i < threads; i++) { - tpool.submit(&waiter, boost::bind(&Blockchain::block_longhash_worker, this, thread_height, std::cref(blocks[i]), std::ref(maps[i]))); + tpool.submit(&waiter, boost::bind(&Blockchain::block_longhash_worker, this, thread_height, std::cref(blocks[i]), std::ref(maps[i])), true); thread_height += blocks[i].size(); } - waiter.wait(); + waiter.wait(&tpool); if (m_cancel) return false; @@ -4218,9 +4256,9 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::list<block_complete_e for (size_t i = 0; i < amounts.size(); i++) { uint64_t amount = amounts[i]; - tpool.submit(&waiter, boost::bind(&Blockchain::output_scan_worker, this, amount, std::cref(offset_map[amount]), std::ref(tx_map[amount]), std::ref(transactions[i]))); + tpool.submit(&waiter, boost::bind(&Blockchain::output_scan_worker, this, amount, std::cref(offset_map[amount]), std::ref(tx_map[amount]), std::ref(transactions[i])), true); } - waiter.wait(); + waiter.wait(&tpool); } else { @@ -4484,7 +4522,7 @@ void Blockchain::load_compiled_in_block_hashes() // for tx hashes will fail in handle_block_to_main_chain(..) CRITICAL_REGION_LOCAL(m_tx_pool); - std::list<transaction> txs; + std::vector<transaction> txs; m_tx_pool.get_transactions(txs); size_t blob_size; @@ -4547,6 +4585,6 @@ bool Blockchain::for_all_outputs(uint64_t amount, std::function<bool(uint64_t he } namespace cryptonote { -template bool Blockchain::get_transactions(const std::vector<crypto::hash>&, std::list<transaction>&, std::list<crypto::hash>&) const; -template bool Blockchain::get_transactions_blobs(const std::vector<crypto::hash>&, std::list<cryptonote::blobdata>&, std::list<crypto::hash>&, bool) const; +template bool Blockchain::get_transactions(const std::vector<crypto::hash>&, std::vector<transaction>&, std::vector<crypto::hash>&) const; +template bool Blockchain::get_transactions_blobs(const std::vector<crypto::hash>&, std::vector<cryptonote::blobdata>&, std::vector<crypto::hash>&, bool) const; } diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index ef736d1e7..36d6b8609 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -157,7 +157,7 @@ namespace cryptonote * * @return false if start_offset > blockchain height, else true */ - bool get_blocks(uint64_t start_offset, size_t count, std::list<std::pair<cryptonote::blobdata,block>>& blocks, std::list<cryptonote::blobdata>& txs) const; + bool get_blocks(uint64_t start_offset, size_t count, std::vector<std::pair<cryptonote::blobdata,block>>& blocks, std::vector<cryptonote::blobdata>& txs) const; /** * @brief get blocks from blocks based on start height and count @@ -168,7 +168,7 @@ namespace cryptonote * * @return false if start_offset > blockchain height, else true */ - bool get_blocks(uint64_t start_offset, size_t count, std::list<std::pair<cryptonote::blobdata,block>>& blocks) const; + bool get_blocks(uint64_t start_offset, size_t count, std::vector<std::pair<cryptonote::blobdata,block>>& blocks) const; /** * @brief compiles a list of all blocks stored as alternative chains @@ -177,7 +177,7 @@ namespace cryptonote * * @return true */ - bool get_alternative_blocks(std::list<block>& blocks) const; + bool get_alternative_blocks(std::vector<block>& blocks) const; /** * @brief returns the number of alternative blocks stored @@ -213,7 +213,7 @@ namespace cryptonote * * @return false on erroneous blocks, else true */ - bool prepare_handle_incoming_blocks(const std::list<block_complete_entry> &blocks); + bool prepare_handle_incoming_blocks(const std::vector<block_complete_entry> &blocks); /** * @brief incoming blocks post-processing, cleanup, and disk sync @@ -373,7 +373,7 @@ namespace cryptonote * * @return true if a block found in common, else false */ - bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, std::list<crypto::hash>& hashes, uint64_t& start_height, uint64_t& current_height) const; + bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, std::vector<crypto::hash>& hashes, uint64_t& start_height, uint64_t& current_height) const; /** * @brief get recent block hashes for a foreign chain @@ -420,7 +420,7 @@ namespace cryptonote * * @return true if a block found in common or req_start_block specified, else false */ - bool find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::list<std::pair<cryptonote::blobdata, std::list<cryptonote::blobdata> > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, size_t max_count) const; + bool find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_count) const; /** * @brief retrieves a set of blocks and their transactions, and possibly other transactions @@ -829,7 +829,7 @@ namespace cryptonote * * @return false if any removals fail, otherwise true */ - bool flush_txes_from_pool(const std::list<crypto::hash> &txids); + bool flush_txes_from_pool(const std::vector<crypto::hash> &txids); /** * @brief return a histogram of outputs on the blockchain @@ -952,7 +952,7 @@ namespace cryptonote bool is_within_compiled_block_hash_area(uint64_t height) const; bool is_within_compiled_block_hash_area() const { return is_within_compiled_block_hash_area(m_db->height()); } - uint64_t prevalidate_block_hashes(uint64_t height, const std::list<crypto::hash> &hashes); + uint64_t prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes); void lock(); void unlock(); diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 7d34415c4..910bf0c1f 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -170,7 +170,6 @@ namespace cryptonote m_last_dns_checkpoints_update(0), m_last_json_checkpoints_update(0), m_disable_dns_checkpoints(false), - m_threadpool(tools::threadpool::getInstance()), m_update_download(0), m_nettype(UNDEFINED) { @@ -324,19 +323,19 @@ namespace cryptonote top_id = m_blockchain_storage.get_tail_id(height); } //----------------------------------------------------------------------------------------------- - bool core::get_blocks(uint64_t start_offset, size_t count, std::list<std::pair<cryptonote::blobdata,block>>& blocks, std::list<cryptonote::blobdata>& txs) const + bool core::get_blocks(uint64_t start_offset, size_t count, std::vector<std::pair<cryptonote::blobdata,block>>& blocks, std::vector<cryptonote::blobdata>& txs) const { return m_blockchain_storage.get_blocks(start_offset, count, blocks, txs); } //----------------------------------------------------------------------------------------------- - bool core::get_blocks(uint64_t start_offset, size_t count, std::list<std::pair<cryptonote::blobdata,block>>& blocks) const + bool core::get_blocks(uint64_t start_offset, size_t count, std::vector<std::pair<cryptonote::blobdata,block>>& blocks) const { return m_blockchain_storage.get_blocks(start_offset, count, blocks); } //----------------------------------------------------------------------------------------------- - bool core::get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks) const + bool core::get_blocks(uint64_t start_offset, size_t count, std::vector<block>& blocks) const { - std::list<std::pair<cryptonote::blobdata, cryptonote::block>> bs; + std::vector<std::pair<cryptonote::blobdata, cryptonote::block>> bs; if (!m_blockchain_storage.get_blocks(start_offset, count, bs)) return false; for (const auto &b: bs) @@ -344,7 +343,7 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- - bool core::get_transactions(const std::vector<crypto::hash>& txs_ids, std::list<cryptonote::blobdata>& txs, std::list<crypto::hash>& missed_txs) const + bool core::get_transactions(const std::vector<crypto::hash>& txs_ids, std::vector<cryptonote::blobdata>& txs, std::vector<crypto::hash>& missed_txs) const { return m_blockchain_storage.get_transactions_blobs(txs_ids, txs, missed_txs); } @@ -355,12 +354,12 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- - bool core::get_transactions(const std::vector<crypto::hash>& txs_ids, std::list<transaction>& txs, std::list<crypto::hash>& missed_txs) const + bool core::get_transactions(const std::vector<crypto::hash>& txs_ids, std::vector<transaction>& txs, std::vector<crypto::hash>& missed_txs) const { return m_blockchain_storage.get_transactions(txs_ids, txs, missed_txs); } //----------------------------------------------------------------------------------------------- - bool core::get_alternative_blocks(std::list<block>& blocks) const + bool core::get_alternative_blocks(std::vector<block>& blocks) const { return m_blockchain_storage.get_alternative_blocks(blocks); } @@ -673,18 +672,20 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- - bool core::handle_incoming_txs(const std::list<blobdata>& tx_blobs, std::vector<tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) + bool core::handle_incoming_txs(const std::vector<blobdata>& tx_blobs, std::vector<tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { TRY_ENTRY(); + CRITICAL_REGION_LOCAL(m_incoming_tx_lock); struct result { bool res; cryptonote::transaction tx; crypto::hash hash; crypto::hash prefix_hash; bool in_txpool; bool in_blockchain; }; std::vector<result> results(tx_blobs.size()); tvc.resize(tx_blobs.size()); + tools::threadpool& tpool = tools::threadpool::getInstance(); tools::threadpool::waiter waiter; - std::list<blobdata>::const_iterator it = tx_blobs.begin(); + std::vector<blobdata>::const_iterator it = tx_blobs.begin(); for (size_t i = 0; i < tx_blobs.size(); i++, ++it) { - m_threadpool.submit(&waiter, [&, i, it] { + tpool.submit(&waiter, [&, i, it] { try { results[i].res = handle_incoming_tx_pre(*it, tvc[i], results[i].tx, results[i].hash, results[i].prefix_hash, keeped_by_block, relayed, do_not_relay); @@ -696,7 +697,7 @@ namespace cryptonote } }); } - waiter.wait(); + waiter.wait(&tpool); it = tx_blobs.begin(); for (size_t i = 0; i < tx_blobs.size(); i++, ++it) { if (!results[i].res) @@ -711,7 +712,7 @@ namespace cryptonote } else { - m_threadpool.submit(&waiter, [&, i, it] { + tpool.submit(&waiter, [&, i, it] { try { results[i].res = handle_incoming_tx_post(*it, tvc[i], results[i].tx, results[i].hash, results[i].prefix_hash, keeped_by_block, relayed, do_not_relay); @@ -724,7 +725,7 @@ namespace cryptonote }); } } - waiter.wait(); + waiter.wait(&tpool); bool ok = true; it = tx_blobs.begin(); @@ -751,7 +752,7 @@ namespace cryptonote //----------------------------------------------------------------------------------------------- bool core::handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { - std::list<cryptonote::blobdata> tx_blobs; + std::vector<cryptonote::blobdata> tx_blobs; tx_blobs.push_back(tx_blob); std::vector<tx_verification_context> tvcv(1); bool r = handle_incoming_txs(tx_blobs, tvcv, keeped_by_block, relayed, do_not_relay); @@ -917,8 +918,8 @@ namespace cryptonote const uint64_t end = start_offset + count - 1; m_blockchain_storage.for_blocks_range(start_offset, end, [this, &emission_amount, &total_fee_amount](uint64_t, const crypto::hash& hash, const block& b){ - std::list<transaction> txs; - std::list<crypto::hash> missed_txs; + std::vector<transaction> txs; + std::vector<crypto::hash> missed_txs; uint64_t coinbase_amount = get_outs_money_amount(b.miner_tx); this->get_transactions(b.tx_hashes, txs, missed_txs); uint64_t tx_fee_amount = 0; @@ -1014,7 +1015,7 @@ namespace cryptonote bool core::relay_txpool_transactions() { // we attempt to relay txes that should be relayed, but were not - std::list<std::pair<crypto::hash, cryptonote::blobdata>> txs; + std::vector<std::pair<crypto::hash, cryptonote::blobdata>> txs; if (m_mempool.get_relayable_transactions(txs) && !txs.empty()) { cryptonote_connection_context fake_context = AUTO_VAL_INIT(fake_context); @@ -1032,7 +1033,7 @@ namespace cryptonote //----------------------------------------------------------------------------------------------- void core::on_transaction_relayed(const cryptonote::blobdata& tx_blob) { - std::list<std::pair<crypto::hash, cryptonote::blobdata>> txs; + std::vector<std::pair<crypto::hash, cryptonote::blobdata>> txs; cryptonote::transaction tx; crypto::hash tx_hash, tx_prefix_hash; if (!parse_and_validate_tx_from_blob(tx_blob, tx, tx_hash, tx_prefix_hash)) @@ -1054,9 +1055,9 @@ namespace cryptonote return m_blockchain_storage.find_blockchain_supplement(qblock_ids, resp); } //----------------------------------------------------------------------------------------------- - bool core::find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::list<std::pair<cryptonote::blobdata, std::list<cryptonote::blobdata> > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, size_t max_count) const + bool core::find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_count) const { - return m_blockchain_storage.find_blockchain_supplement(req_start_block, qblock_ids, blocks, total_height, start_height, pruned, max_count); + return m_blockchain_storage.find_blockchain_supplement(req_start_block, qblock_ids, blocks, total_height, start_height, pruned, get_miner_tx_hash, max_count); } //----------------------------------------------------------------------------------------------- bool core::get_random_outs_for_amounts(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res) const @@ -1111,7 +1112,7 @@ namespace cryptonote { block_verification_context bvc = boost::value_initialized<block_verification_context>(); m_miner.pause(); - std::list<block_complete_entry> blocks; + std::vector<block_complete_entry> blocks; try { blocks.push_back(get_block_complete_entry(b, m_mempool)); @@ -1135,8 +1136,8 @@ namespace cryptonote cryptonote_connection_context exclude_context = boost::value_initialized<cryptonote_connection_context>(); NOTIFY_NEW_BLOCK::request arg = AUTO_VAL_INIT(arg); arg.current_blockchain_height = m_blockchain_storage.get_current_blockchain_height(); - std::list<crypto::hash> missed_txs; - std::list<cryptonote::blobdata> txs; + std::vector<crypto::hash> missed_txs; + std::vector<cryptonote::blobdata> txs; m_blockchain_storage.get_transactions_blobs(b.tx_hashes, txs, missed_txs); if(missed_txs.size() && m_blockchain_storage.get_block_id_by_height(get_block_height(b)) != get_block_hash(b)) { @@ -1172,7 +1173,7 @@ namespace cryptonote } //----------------------------------------------------------------------------------------------- - bool core::prepare_handle_incoming_blocks(const std::list<block_complete_entry> &blocks) + bool core::prepare_handle_incoming_blocks(const std::vector<block_complete_entry> &blocks) { m_incoming_tx_lock.lock(); m_blockchain_storage.prepare_handle_incoming_blocks(blocks); @@ -1265,7 +1266,7 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- - bool core::get_pool_transactions(std::list<transaction>& txs, bool include_sensitive_data) const + bool core::get_pool_transactions(std::vector<transaction>& txs, bool include_sensitive_data) const { m_mempool.get_transactions(txs, include_sensitive_data); return true; @@ -1553,7 +1554,7 @@ namespace cryptonote return m_target_blockchain_height; } //----------------------------------------------------------------------------------------------- - uint64_t core::prevalidate_block_hashes(uint64_t height, const std::list<crypto::hash> &hashes) + uint64_t core::prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes) { return get_blockchain_storage().prevalidate_block_hashes(height, hashes); } diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 567966d48..03000383e 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -39,7 +39,6 @@ #include "cryptonote_protocol/cryptonote_protocol_handler_common.h" #include "storages/portable_storage_template_helper.h" #include "common/download.h" -#include "common/threadpool.h" #include "common/command_line.h" #include "tx_pool.h" #include "blockchain.h" @@ -134,7 +133,7 @@ namespace cryptonote * * @return true if the transactions made it to the transaction pool, otherwise false */ - bool handle_incoming_txs(const std::list<blobdata>& tx_blobs, std::vector<tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); + bool handle_incoming_txs(const std::vector<blobdata>& tx_blobs, std::vector<tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); /** * @brief handles an incoming block @@ -157,7 +156,7 @@ namespace cryptonote * * @note see Blockchain::prepare_handle_incoming_blocks */ - bool prepare_handle_incoming_blocks(const std::list<block_complete_entry> &blocks); + bool prepare_handle_incoming_blocks(const std::vector<block_complete_entry> &blocks); /** * @copydoc Blockchain::cleanup_handle_incoming_blocks @@ -309,25 +308,25 @@ namespace cryptonote void get_blockchain_top(uint64_t& height, crypto::hash& top_id) const; /** - * @copydoc Blockchain::get_blocks(uint64_t, size_t, std::list<std::pair<cryptonote::blobdata,block>>&, std::list<transaction>&) const + * @copydoc Blockchain::get_blocks(uint64_t, size_t, std::vector<std::pair<cryptonote::blobdata,block>>&, std::vector<transaction>&) const * - * @note see Blockchain::get_blocks(uint64_t, size_t, std::list<std::pair<cryptonote::blobdata,block>>&, std::list<transaction>&) const + * @note see Blockchain::get_blocks(uint64_t, size_t, std::vector<std::pair<cryptonote::blobdata,block>>&, std::vector<transaction>&) const */ - bool get_blocks(uint64_t start_offset, size_t count, std::list<std::pair<cryptonote::blobdata,block>>& blocks, std::list<cryptonote::blobdata>& txs) const; + bool get_blocks(uint64_t start_offset, size_t count, std::vector<std::pair<cryptonote::blobdata,block>>& blocks, std::vector<cryptonote::blobdata>& txs) const; /** - * @copydoc Blockchain::get_blocks(uint64_t, size_t, std::list<std::pair<cryptonote::blobdata,block>>&) const + * @copydoc Blockchain::get_blocks(uint64_t, size_t, std::vector<std::pair<cryptonote::blobdata,block>>&) const * - * @note see Blockchain::get_blocks(uint64_t, size_t, std::list<std::pair<cryptonote::blobdata,block>>&) const + * @note see Blockchain::get_blocks(uint64_t, size_t, std::vector<std::pair<cryptonote::blobdata,block>>&) const */ - bool get_blocks(uint64_t start_offset, size_t count, std::list<std::pair<cryptonote::blobdata,block>>& blocks) const; + bool get_blocks(uint64_t start_offset, size_t count, std::vector<std::pair<cryptonote::blobdata,block>>& blocks) const; /** - * @copydoc Blockchain::get_blocks(uint64_t, size_t, std::list<std::pair<cryptonote::blobdata,block>>&) const + * @copydoc Blockchain::get_blocks(uint64_t, size_t, std::vector<std::pair<cryptonote::blobdata,block>>&) const * - * @note see Blockchain::get_blocks(uint64_t, size_t, std::list<std::pair<cryptonote::blobdata,block>>&) const + * @note see Blockchain::get_blocks(uint64_t, size_t, std::vector<std::pair<cryptonote::blobdata,block>>&) const */ - bool get_blocks(uint64_t start_offset, size_t count, std::list<block>& blocks) const; + bool get_blocks(uint64_t start_offset, size_t count, std::vector<block>& blocks) const; /** * @copydoc Blockchain::get_blocks(const t_ids_container&, t_blocks_container&, t_missed_container&) const @@ -352,14 +351,14 @@ namespace cryptonote * * @note see Blockchain::get_transactions */ - bool get_transactions(const std::vector<crypto::hash>& txs_ids, std::list<cryptonote::blobdata>& txs, std::list<crypto::hash>& missed_txs) const; + bool get_transactions(const std::vector<crypto::hash>& txs_ids, std::vector<cryptonote::blobdata>& txs, std::vector<crypto::hash>& missed_txs) const; /** * @copydoc Blockchain::get_transactions * * @note see Blockchain::get_transactions */ - bool get_transactions(const std::vector<crypto::hash>& txs_ids, std::list<transaction>& txs, std::list<crypto::hash>& missed_txs) const; + bool get_transactions(const std::vector<crypto::hash>& txs_ids, std::vector<transaction>& txs, std::vector<crypto::hash>& missed_txs) const; /** * @copydoc Blockchain::get_block_by_hash @@ -371,9 +370,9 @@ namespace cryptonote /** * @copydoc Blockchain::get_alternative_blocks * - * @note see Blockchain::get_alternative_blocks(std::list<block>&) const + * @note see Blockchain::get_alternative_blocks(std::vector<block>&) const */ - bool get_alternative_blocks(std::list<block>& blocks) const; + bool get_alternative_blocks(std::vector<block>& blocks) const; /** * @copydoc Blockchain::get_alternative_blocks_count @@ -430,7 +429,7 @@ namespace cryptonote * * @note see tx_memory_pool::get_transactions */ - bool get_pool_transactions(std::list<transaction>& txs, bool include_unrelayed_txes = true) const; + bool get_pool_transactions(std::vector<transaction>& txs, bool include_unrelayed_txes = true) const; /** * @copydoc tx_memory_pool::get_txpool_backlog @@ -513,11 +512,11 @@ namespace cryptonote bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const; /** - * @copydoc Blockchain::find_blockchain_supplement(const uint64_t, const std::list<crypto::hash>&, std::list<std::pair<cryptonote::blobdata, std::list<cryptonote::blobdata> > >&, uint64_t&, uint64_t&, size_t) const + * @copydoc Blockchain::find_blockchain_supplement(const uint64_t, const std::list<crypto::hash>&, std::vector<std::pair<cryptonote::blobdata, std::vector<cryptonote::blobdata> > >&, uint64_t&, uint64_t&, size_t) const * - * @note see Blockchain::find_blockchain_supplement(const uint64_t, const std::list<crypto::hash>&, std::list<std::pair<cryptonote::blobdata, std::list<transaction> > >&, uint64_t&, uint64_t&, size_t) const + * @note see Blockchain::find_blockchain_supplement(const uint64_t, const std::list<crypto::hash>&, std::vector<std::pair<cryptonote::blobdata, std::vector<transaction> > >&, uint64_t&, uint64_t&, size_t) const */ - bool find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::list<std::pair<cryptonote::blobdata, std::list<cryptonote::blobdata> > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, size_t max_count) const; + bool find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_count) const; /** * @brief gets some stats about the daemon @@ -764,7 +763,7 @@ namespace cryptonote * * @return number of usable blocks */ - uint64_t prevalidate_block_hashes(uint64_t height, const std::list<crypto::hash> &hashes); + uint64_t prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes); /** * @brief get free disk space on the blockchain partition @@ -991,8 +990,6 @@ namespace cryptonote std::unordered_set<crypto::hash> bad_semantics_txes[2]; boost::mutex bad_semantics_txes_lock; - tools::threadpool& m_threadpool; - enum { UPDATES_DISABLED, UPDATES_NOTIFY, diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index bf1fe476e..8dee2b922 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -239,6 +239,7 @@ namespace cryptonote meta.relayed = relayed; meta.do_not_relay = do_not_relay; meta.double_spend_seen = have_tx_keyimges_as_spent(tx); + meta.bf_padding = 0; memset(meta.padding, 0, sizeof(meta.padding)); try { @@ -278,6 +279,7 @@ namespace cryptonote meta.relayed = relayed; meta.do_not_relay = do_not_relay; meta.double_spend_seen = false; + meta.bf_padding = 0; memset(meta.padding, 0, sizeof(meta.padding)); try @@ -556,11 +558,12 @@ namespace cryptonote } //--------------------------------------------------------------------------------- //TODO: investigate whether boolean return is appropriate - bool tx_memory_pool::get_relayable_transactions(std::list<std::pair<crypto::hash, cryptonote::blobdata>> &txs) const + bool tx_memory_pool::get_relayable_transactions(std::vector<std::pair<crypto::hash, cryptonote::blobdata>> &txs) const { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); const uint64_t now = time(NULL); + txs.reserve(m_blockchain.get_txpool_tx_count()); m_blockchain.for_all_txpool_txes([this, now, &txs](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *){ // 0 fee transactions are never relayed if(meta.fee > 0 && !meta.do_not_relay && now - meta.last_relayed_time > get_relay_delay(now, meta.receive_time)) @@ -588,7 +591,7 @@ namespace cryptonote return true; } //--------------------------------------------------------------------------------- - void tx_memory_pool::set_relayed(const std::list<std::pair<crypto::hash, cryptonote::blobdata>> &txs) + void tx_memory_pool::set_relayed(const std::vector<std::pair<crypto::hash, cryptonote::blobdata>> &txs) { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); @@ -621,10 +624,11 @@ namespace cryptonote return m_blockchain.get_txpool_tx_count(include_unrelayed_txes); } //--------------------------------------------------------------------------------- - void tx_memory_pool::get_transactions(std::list<transaction>& txs, bool include_unrelayed_txes) const + void tx_memory_pool::get_transactions(std::vector<transaction>& txs, bool include_unrelayed_txes) const { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); + txs.reserve(m_blockchain.get_txpool_tx_count(include_unrelayed_txes)); m_blockchain.for_all_txpool_txes([&txs](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ transaction tx; if (!parse_and_validate_tx_from_blob(*bd, tx)) @@ -642,6 +646,7 @@ namespace cryptonote { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); + txs.reserve(m_blockchain.get_txpool_tx_count(include_unrelayed_txes)); m_blockchain.for_all_txpool_txes([&txs](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ txs.push_back(txid); return true; @@ -653,6 +658,7 @@ namespace cryptonote CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); const uint64_t now = time(NULL); + backlog.reserve(m_blockchain.get_txpool_tx_count(include_unrelayed_txes)); m_blockchain.for_all_txpool_txes([&backlog, now](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ backlog.push_back({meta.blob_size, meta.fee, meta.receive_time - now}); return true; @@ -741,6 +747,8 @@ namespace cryptonote { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); + tx_infos.reserve(m_blockchain.get_txpool_tx_count()); + key_image_infos.reserve(m_blockchain.get_txpool_tx_count()); m_blockchain.for_all_txpool_txes([&tx_infos, key_image_infos, include_sensitive_data](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ tx_info txi; txi.id_hash = epee::string_tools::pod_to_hex(txid); @@ -811,6 +819,8 @@ namespace cryptonote { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); + tx_infos.reserve(m_blockchain.get_txpool_tx_count()); + key_image_infos.reserve(m_blockchain.get_txpool_tx_count()); m_blockchain.for_all_txpool_txes([&tx_infos, key_image_infos](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){ cryptonote::rpc::tx_in_pool txi; txi.tx_hash = txid; @@ -927,8 +937,26 @@ namespace cryptonote m_transactions_lock.unlock(); } //--------------------------------------------------------------------------------- - bool tx_memory_pool::is_transaction_ready_to_go(txpool_tx_meta_t& txd, transaction &tx) const + bool tx_memory_pool::is_transaction_ready_to_go(txpool_tx_meta_t& txd, const cryptonote::blobdata &txblob, transaction &tx) const { + struct transction_parser + { + transction_parser(const cryptonote::blobdata &txblob, transaction &tx): txblob(txblob), tx(tx), parsed(false) {} + cryptonote::transaction &operator()() + { + if (!parsed) + { + if (!parse_and_validate_tx_from_blob(txblob, tx)) + throw std::runtime_error("failed to parse transaction blob"); + parsed = true; + } + return tx; + } + const cryptonote::blobdata &txblob; + transaction &tx; + bool parsed; + } lazy_tx(txblob, tx); + //not the best implementation at this time, sorry :( //check is ring_signature already checked ? if(txd.max_used_block_id == null_hash) @@ -938,7 +966,7 @@ namespace cryptonote return false;//we already sure that this tx is broken for this height tx_verification_context tvc; - if(!m_blockchain.check_tx_inputs(tx, txd.max_used_block_height, txd.max_used_block_id, tvc)) + if(!m_blockchain.check_tx_inputs(lazy_tx(), txd.max_used_block_height, txd.max_used_block_id, tvc)) { txd.last_failed_height = m_blockchain.get_current_blockchain_height()-1; txd.last_failed_id = m_blockchain.get_block_id_by_height(txd.last_failed_height); @@ -955,7 +983,7 @@ namespace cryptonote return false; //check ring signature again, it is possible (with very small chance) that this transaction become again valid tx_verification_context tvc; - if(!m_blockchain.check_tx_inputs(tx, txd.max_used_block_height, txd.max_used_block_id, tvc)) + if(!m_blockchain.check_tx_inputs(lazy_tx(), txd.max_used_block_height, txd.max_used_block_id, tvc)) { txd.last_failed_height = m_blockchain.get_current_blockchain_height()-1; txd.last_failed_id = m_blockchain.get_block_id_by_height(txd.last_failed_height); @@ -964,7 +992,7 @@ namespace cryptonote } } //if we here, transaction seems valid, but, anyway, check for key_images collisions with blockchain, just to be sure - if(m_blockchain.have_tx_keyimges_as_spent(tx)) + if(m_blockchain.have_tx_keyimges_as_spent(lazy_tx())) { txd.double_spend_seen = true; return false; @@ -1140,18 +1168,21 @@ namespace cryptonote cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(sorted_it->second); cryptonote::transaction tx; - if (!parse_and_validate_tx_from_blob(txblob, tx)) - { - MERROR("Failed to parse tx from txpool"); - sorted_it++; - continue; - } // Skip transactions that are not ready to be // included into the blockchain or that are // missing key images const cryptonote::txpool_tx_meta_t original_meta = meta; - bool ready = is_transaction_ready_to_go(meta, tx); + bool ready = false; + try + { + ready = is_transaction_ready_to_go(meta, txblob, tx); + } + catch (const std::exception &e) + { + MERROR("Failed to check transaction readiness: " << e.what()); + // continue, not fatal + } if (memcmp(&original_meta, &meta, sizeof(meta))) { try diff --git a/src/cryptonote_core/tx_pool.h b/src/cryptonote_core/tx_pool.h index 19cd83ed9..5ccb71196 100644 --- a/src/cryptonote_core/tx_pool.h +++ b/src/cryptonote_core/tx_pool.h @@ -237,7 +237,7 @@ namespace cryptonote * @param include_unrelayed_txes include unrelayed txes in the result * */ - void get_transactions(std::list<transaction>& txs, bool include_unrelayed_txes = true) const; + void get_transactions(std::vector<transaction>& txs, bool include_unrelayed_txes = true) const; /** * @brief get a list of all transaction hashes in the pool @@ -324,14 +324,14 @@ namespace cryptonote * * @return true */ - bool get_relayable_transactions(std::list<std::pair<crypto::hash, cryptonote::blobdata>>& txs) const; + bool get_relayable_transactions(std::vector<std::pair<crypto::hash, cryptonote::blobdata>>& txs) const; /** * @brief tell the pool that certain transactions were just relayed * * @param txs the list of transactions (and their hashes) */ - void set_relayed(const std::list<std::pair<crypto::hash, cryptonote::blobdata>>& txs); + void set_relayed(const std::vector<std::pair<crypto::hash, cryptonote::blobdata>>& txs); /** * @brief get the total number of transactions in the pool @@ -499,10 +499,12 @@ namespace cryptonote * @brief check if a transaction is a valid candidate for inclusion in a block * * @param txd the transaction to check (and info about it) + * @param txblob the transaction blob to check + * @param tx the parsed transaction, if successful * * @return true if the transaction is good to go, otherwise false */ - bool is_transaction_ready_to_go(txpool_tx_meta_t& txd, transaction &tx) const; + bool is_transaction_ready_to_go(txpool_tx_meta_t& txd, const cryptonote::blobdata &txblob, transaction &tx) const; /** * @brief mark all transactions double spending the one passed diff --git a/src/cryptonote_protocol/block_queue.cpp b/src/cryptonote_protocol/block_queue.cpp index 9ae33d540..c39d67ceb 100644 --- a/src/cryptonote_protocol/block_queue.cpp +++ b/src/cryptonote_protocol/block_queue.cpp @@ -50,10 +50,10 @@ namespace std { namespace cryptonote { -void block_queue::add_blocks(uint64_t height, std::list<cryptonote::block_complete_entry> bcel, const boost::uuids::uuid &connection_id, float rate, size_t size) +void block_queue::add_blocks(uint64_t height, std::vector<cryptonote::block_complete_entry> bcel, const boost::uuids::uuid &connection_id, float rate, size_t size) { boost::unique_lock<boost::recursive_mutex> lock(mutex); - std::list<crypto::hash> hashes; + std::vector<crypto::hash> hashes; bool has_hashes = remove_span(height, &hashes); blocks.insert(span(height, std::move(bcel), connection_id, rate, size)); if (has_hashes) @@ -97,7 +97,7 @@ void block_queue::flush_stale_spans(const std::set<boost::uuids::uuid> &live_con } } -bool block_queue::remove_span(uint64_t start_block_height, std::list<crypto::hash> *hashes) +bool block_queue::remove_span(uint64_t start_block_height, std::vector<crypto::hash> *hashes) { boost::unique_lock<boost::recursive_mutex> lock(mutex); for (block_map::iterator i = blocks.begin(); i != blocks.end(); ++i) @@ -172,7 +172,7 @@ bool block_queue::requested(const crypto::hash &hash) const return false; } -std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, const std::list<crypto::hash> &block_hashes, boost::posix_time::ptime time) +std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, const std::vector<crypto::hash> &block_hashes, boost::posix_time::ptime time) { boost::unique_lock<boost::recursive_mutex> lock(mutex); @@ -183,14 +183,14 @@ std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_hei } uint64_t span_start_height = last_block_height - block_hashes.size() + 1; - std::list<crypto::hash>::const_iterator i = block_hashes.begin(); + std::vector<crypto::hash>::const_iterator i = block_hashes.begin(); while (i != block_hashes.end() && requested(*i)) { ++i; ++span_start_height; } uint64_t span_length = 0; - std::list<crypto::hash> hashes; + std::vector<crypto::hash> hashes; while (i != block_hashes.end() && span_length < max_blocks) { hashes.push_back(*i); @@ -230,7 +230,7 @@ std::pair<uint64_t, uint64_t> block_queue::get_start_gap_span() const return std::make_pair(current_height + 1, first_span_height - current_height - 1); } -std::pair<uint64_t, uint64_t> block_queue::get_next_span_if_scheduled(std::list<crypto::hash> &hashes, boost::uuids::uuid &connection_id, boost::posix_time::ptime &time) const +std::pair<uint64_t, uint64_t> block_queue::get_next_span_if_scheduled(std::vector<crypto::hash> &hashes, boost::uuids::uuid &connection_id, boost::posix_time::ptime &time) const { boost::unique_lock<boost::recursive_mutex> lock(mutex); if (blocks.empty()) @@ -248,7 +248,7 @@ std::pair<uint64_t, uint64_t> block_queue::get_next_span_if_scheduled(std::list< return std::make_pair(i->start_block_height, i->nblocks); } -void block_queue::set_span_hashes(uint64_t start_height, const boost::uuids::uuid &connection_id, std::list<crypto::hash> hashes) +void block_queue::set_span_hashes(uint64_t start_height, const boost::uuids::uuid &connection_id, std::vector<crypto::hash> hashes) { boost::unique_lock<boost::recursive_mutex> lock(mutex); for (block_map::iterator i = blocks.begin(); i != blocks.end(); ++i) @@ -264,7 +264,7 @@ void block_queue::set_span_hashes(uint64_t start_height, const boost::uuids::uui } } -bool block_queue::get_next_span(uint64_t &height, std::list<cryptonote::block_complete_entry> &bcel, boost::uuids::uuid &connection_id, bool filled) const +bool block_queue::get_next_span(uint64_t &height, std::vector<cryptonote::block_complete_entry> &bcel, boost::uuids::uuid &connection_id, bool filled) const { boost::unique_lock<boost::recursive_mutex> lock(mutex); if (blocks.empty()) diff --git a/src/cryptonote_protocol/block_queue.h b/src/cryptonote_protocol/block_queue.h index 69ddaa435..9059e89ac 100644 --- a/src/cryptonote_protocol/block_queue.h +++ b/src/cryptonote_protocol/block_queue.h @@ -31,7 +31,7 @@ #pragma once #include <string> -#include <list> +#include <vector> #include <set> #include <boost/thread/recursive_mutex.hpp> #include <boost/uuid/uuid.hpp> @@ -49,15 +49,15 @@ namespace cryptonote struct span { uint64_t start_block_height; - std::list<crypto::hash> hashes; - std::list<cryptonote::block_complete_entry> blocks; + std::vector<crypto::hash> hashes; + std::vector<cryptonote::block_complete_entry> blocks; boost::uuids::uuid connection_id; uint64_t nblocks; float rate; size_t size; boost::posix_time::ptime time; - span(uint64_t start_block_height, std::list<cryptonote::block_complete_entry> blocks, const boost::uuids::uuid &connection_id, float rate, size_t size): + span(uint64_t start_block_height, std::vector<cryptonote::block_complete_entry> blocks, const boost::uuids::uuid &connection_id, float rate, size_t size): start_block_height(start_block_height), blocks(std::move(blocks)), connection_id(connection_id), nblocks(this->blocks.size()), rate(rate), size(size), time() {} span(uint64_t start_block_height, uint64_t nblocks, const boost::uuids::uuid &connection_id, boost::posix_time::ptime time): start_block_height(start_block_height), connection_id(connection_id), nblocks(nblocks), rate(0.0f), size(0), time(time) {} @@ -67,21 +67,21 @@ namespace cryptonote typedef std::set<span> block_map; public: - void add_blocks(uint64_t height, std::list<cryptonote::block_complete_entry> bcel, const boost::uuids::uuid &connection_id, float rate, size_t size); + void add_blocks(uint64_t height, std::vector<cryptonote::block_complete_entry> bcel, const boost::uuids::uuid &connection_id, float rate, size_t size); void add_blocks(uint64_t height, uint64_t nblocks, const boost::uuids::uuid &connection_id, boost::posix_time::ptime time = boost::date_time::min_date_time); void flush_spans(const boost::uuids::uuid &connection_id, bool all = false); void flush_stale_spans(const std::set<boost::uuids::uuid> &live_connections); - bool remove_span(uint64_t start_block_height, std::list<crypto::hash> *hashes = NULL); + bool remove_span(uint64_t start_block_height, std::vector<crypto::hash> *hashes = NULL); void remove_spans(const boost::uuids::uuid &connection_id, uint64_t start_block_height); uint64_t get_max_block_height() const; void print() const; std::string get_overview() const; - std::pair<uint64_t, uint64_t> reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, const std::list<crypto::hash> &block_hashes, boost::posix_time::ptime time = boost::posix_time::microsec_clock::universal_time()); + std::pair<uint64_t, uint64_t> reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, const std::vector<crypto::hash> &block_hashes, boost::posix_time::ptime time = boost::posix_time::microsec_clock::universal_time()); bool is_blockchain_placeholder(const span &span) const; std::pair<uint64_t, uint64_t> get_start_gap_span() const; - std::pair<uint64_t, uint64_t> get_next_span_if_scheduled(std::list<crypto::hash> &hashes, boost::uuids::uuid &connection_id, boost::posix_time::ptime &time) const; - void set_span_hashes(uint64_t start_height, const boost::uuids::uuid &connection_id, std::list<crypto::hash> hashes); - bool get_next_span(uint64_t &height, std::list<cryptonote::block_complete_entry> &bcel, boost::uuids::uuid &connection_id, bool filled = true) const; + std::pair<uint64_t, uint64_t> get_next_span_if_scheduled(std::vector<crypto::hash> &hashes, boost::uuids::uuid &connection_id, boost::posix_time::ptime &time) const; + void set_span_hashes(uint64_t start_height, const boost::uuids::uuid &connection_id, std::vector<crypto::hash> hashes); + bool get_next_span(uint64_t &height, std::vector<cryptonote::block_complete_entry> &bcel, boost::uuids::uuid &connection_id, bool filled = true) const; bool has_next_span(const boost::uuids::uuid &connection_id, bool &filled) const; size_t get_data_size() const; size_t get_num_filled_spans_prefix() const; diff --git a/src/cryptonote_protocol/cryptonote_protocol_defs.h b/src/cryptonote_protocol/cryptonote_protocol_defs.h index cf0043287..db159f0f4 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_defs.h +++ b/src/cryptonote_protocol/cryptonote_protocol_defs.h @@ -109,7 +109,7 @@ namespace cryptonote struct block_complete_entry { blobdata block; - std::list<blobdata> txs; + std::vector<blobdata> txs; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(block) KV_SERIALIZE(txs) @@ -145,7 +145,7 @@ namespace cryptonote struct request { - std::list<blobdata> txs; + std::vector<blobdata> txs; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(txs) @@ -161,8 +161,8 @@ namespace cryptonote struct request { - std::list<crypto::hash> txs; - std::list<crypto::hash> blocks; + std::vector<crypto::hash> txs; + std::vector<crypto::hash> blocks; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE_CONTAINER_POD_AS_BLOB(txs) @@ -177,9 +177,9 @@ namespace cryptonote struct request { - std::list<blobdata> txs; - std::list<block_complete_entry> blocks; - std::list<crypto::hash> missed_ids; + std::vector<blobdata> txs; + std::vector<block_complete_entry> blocks; + std::vector<crypto::hash> missed_ids; uint64_t current_blockchain_height; BEGIN_KV_SERIALIZE_MAP() @@ -230,7 +230,7 @@ namespace cryptonote uint64_t start_height; uint64_t total_height; uint64_t cumulative_difficulty; - std::list<crypto::hash> m_block_ids; + std::vector<crypto::hash> m_block_ids; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(start_height) diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index 2e1df8078..56aa1dc06 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -351,7 +351,7 @@ namespace cryptonote return 1; } m_core.pause_mine(); - std::list<block_complete_entry> blocks; + std::vector<block_complete_entry> blocks; blocks.push_back(arg.b); m_core.prepare_handle_incoming_blocks(blocks); for(auto tx_blob_it = arg.b.txs.begin(); tx_blob_it!=arg.b.txs.end();tx_blob_it++) @@ -438,7 +438,7 @@ namespace cryptonote } } - std::list<blobdata> have_tx; + std::vector<blobdata> have_tx; // Instead of requesting missing transactions by hash like BTC, // we do it by index (thanks to a suggestion from moneromooo) because @@ -578,8 +578,8 @@ namespace cryptonote else { std::vector<crypto::hash> tx_ids; - std::list<transaction> txes; - std::list<crypto::hash> missing; + std::vector<transaction> txes; + std::vector<crypto::hash> missing; tx_ids.push_back(tx_hash); if (m_core.get_transactions(tx_ids, txes, missing) && missing.empty()) { @@ -626,7 +626,7 @@ namespace cryptonote b.block = arg.b.block; b.txs = have_tx; - std::list<block_complete_entry> blocks; + std::vector<block_complete_entry> blocks; blocks.push_back(b); m_core.prepare_handle_incoming_blocks(blocks); @@ -687,8 +687,8 @@ namespace cryptonote { MLOG_P2P_MESSAGE("Received NOTIFY_REQUEST_FLUFFY_MISSING_TX (" << arg.missing_tx_indices.size() << " txes), block hash " << arg.block_hash); - std::list<std::pair<cryptonote::blobdata, block>> local_blocks; - std::list<cryptonote::blobdata> local_txs; + std::vector<std::pair<cryptonote::blobdata, block>> local_blocks; + std::vector<cryptonote::blobdata> local_txs; block b; if (!m_core.get_block_by_hash(arg.block_hash, b)) @@ -725,8 +725,8 @@ namespace cryptonote } } - std::list<cryptonote::transaction> txs; - std::list<crypto::hash> missed; + std::vector<cryptonote::transaction> txs; + std::vector<crypto::hash> missed; if (!m_core.get_transactions(txids, txs, missed)) { LOG_ERROR_CCONTEXT("Failed to handle request NOTIFY_REQUEST_FLUFFY_MISSING_TX, " @@ -774,10 +774,12 @@ namespace cryptonote return 1; } - for(auto tx_blob_it = arg.txs.begin(); tx_blob_it!=arg.txs.end();) + std::vector<cryptonote::blobdata> newtxs; + newtxs.reserve(arg.txs.size()); + for (size_t i = 0; i < arg.txs.size(); ++i) { cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc); - m_core.handle_incoming_tx(*tx_blob_it, tvc, false, true, false); + m_core.handle_incoming_tx(arg.txs[i], tvc, false, true, false); if(tvc.m_verifivation_failed) { LOG_PRINT_CCONTEXT_L1("Tx verification failed, dropping connection"); @@ -785,10 +787,9 @@ namespace cryptonote return 1; } if(tvc.m_should_be_relayed) - ++tx_blob_it; - else - arg.txs.erase(tx_blob_it++); + newtxs.push_back(std::move(arg.txs[i])); } + arg.txs = std::move(newtxs); if(arg.txs.size()) { @@ -996,7 +997,7 @@ skip: { const uint64_t previous_height = m_core.get_current_blockchain_height(); uint64_t start_height; - std::list<cryptonote::block_complete_entry> blocks; + std::vector<cryptonote::block_complete_entry> blocks; boost::uuids::uuid span_connection_id; if (!m_block_queue.get_next_span(start_height, blocks, span_connection_id)) { @@ -1070,7 +1071,7 @@ skip: LOG_ERROR_CCONTEXT("Internal error: tvc.size() != block_entry.txs.size()"); return 1; } - std::list<blobdata>::const_iterator it = block_entry.txs.begin(); + std::vector<blobdata>::const_iterator it = block_entry.txs.begin(); for (size_t i = 0; i < tvc.size(); ++i, ++it) { if(tvc[i].m_verifivation_failed) @@ -1248,7 +1249,7 @@ skip: template<class t_core> bool t_cryptonote_protocol_handler<t_core>::should_download_next_span(cryptonote_connection_context& context) const { - std::list<crypto::hash> hashes; + std::vector<crypto::hash> hashes; boost::uuids::uuid span_connection_id; boost::posix_time::ptime request_time; std::pair<uint64_t, uint64_t> span; @@ -1267,7 +1268,7 @@ skip: // we might be in a weird case where there is a filled next span, // but it starts higher than the current height uint64_t height; - std::list<cryptonote::block_complete_entry> bcel; + std::vector<cryptonote::block_complete_entry> bcel; if (!m_block_queue.get_next_span(height, bcel, span_connection_id, true)) return false; if (height > m_core.get_current_blockchain_height()) @@ -1415,7 +1416,7 @@ skip: { if (span.second == 0) { - std::list<crypto::hash> hashes; + std::vector<crypto::hash> hashes; boost::uuids::uuid span_connection_id; boost::posix_time::ptime time; span = m_block_queue.get_next_span_if_scheduled(hashes, span_connection_id, time); @@ -1441,14 +1442,18 @@ skip: goto skip; } // take out blocks we already have - while (!context.m_needed_objects.empty() && m_core.have_block(context.m_needed_objects.front())) + size_t skip = 0; + while (skip < context.m_needed_objects.size() && m_core.have_block(context.m_needed_objects[skip])) { // if we're popping the last hash, record it so we can ask again from that hash, // this prevents never being able to progress on peers we get old hash lists from - if (context.m_needed_objects.size() == 1) - context.m_last_known_hash = context.m_needed_objects.front(); - context.m_needed_objects.pop_front(); + if (skip + 1 == context.m_needed_objects.size()) + context.m_last_known_hash = context.m_needed_objects[skip]; + ++skip; } + if (skip > 0) + context.m_needed_objects = std::vector<crypto::hash>(context.m_needed_objects.begin() + skip, context.m_needed_objects.end()); + const uint64_t first_block_height = context.m_last_response_height - context.m_needed_objects.size() + 1; span = m_block_queue.reserve_span(first_block_height, context.m_last_response_height, count_limit, context.m_connection_id, context.m_needed_objects); MDEBUG(context << " span from " << first_block_height << ": " << span.first << "/" << span.second); @@ -1456,7 +1461,7 @@ skip: if (span.second == 0 && !force_next_span) { MDEBUG(context << " still no span reserved, we may be in the corner case of next span scheduled and everything else scheduled/filled"); - std::list<crypto::hash> hashes; + std::vector<crypto::hash> hashes; boost::uuids::uuid span_connection_id; boost::posix_time::ptime time; span = m_block_queue.get_next_span_if_scheduled(hashes, span_connection_id, time); @@ -1487,23 +1492,21 @@ skip: MERROR("ERROR: skip " << skip << ", m_needed_objects " << context.m_needed_objects.size() << ", first_context_block_height" << first_context_block_height); return false; } - while (skip--) - context.m_needed_objects.pop_front(); + if (skip > 0) + context.m_needed_objects = std::vector<crypto::hash>(context.m_needed_objects.begin() + skip, context.m_needed_objects.end()); if (context.m_needed_objects.size() < span.second) { MERROR("ERROR: span " << span.first << "/" << span.second << ", m_needed_objects " << context.m_needed_objects.size()); return false; } - auto it = context.m_needed_objects.begin(); for (size_t n = 0; n < span.second; ++n) { - req.blocks.push_back(*it); + req.blocks.push_back(context.m_needed_objects[n]); ++count; - context.m_requested_objects.insert(*it); - auto j = it++; - context.m_needed_objects.erase(j); + context.m_requested_objects.insert(context.m_needed_objects[n]); } + context.m_needed_objects = std::vector<crypto::hash>(context.m_needed_objects.begin() + span.second, context.m_needed_objects.end()); } context.m_last_request_time = boost::posix_time::microsec_clock::universal_time(); @@ -1664,7 +1667,7 @@ skip: { NOTIFY_NEW_FLUFFY_BLOCK::request fluffy_arg = AUTO_VAL_INIT(fluffy_arg); fluffy_arg.current_blockchain_height = arg.current_blockchain_height; - std::list<blobdata> fluffy_txs; + std::vector<blobdata> fluffy_txs; fluffy_arg.b = arg.b; fluffy_arg.b.txs = fluffy_txs; @@ -1712,6 +1715,9 @@ skip: template<class t_core> void t_cryptonote_protocol_handler<t_core>::drop_connection(cryptonote_connection_context &context, bool add_fail, bool flush_all_spans) { + LOG_DEBUG_CC(context, "dropping connection id " << context.m_connection_id << + ", add_fail " << add_fail << ", flush_all_spans " << flush_all_spans); + if (add_fail) m_p2p->add_host_fail(context.m_remote_address); diff --git a/src/daemon/command_line_args.h b/src/daemon/command_line_args.h index 4673590aa..cba71bf3b 100644 --- a/src/daemon/command_line_args.h +++ b/src/daemon/command_line_args.h @@ -72,6 +72,11 @@ namespace daemon_args , "Specify maximum log file size [B]" , MAX_LOG_FILE_SIZE }; + const command_line::arg_descriptor<std::size_t> arg_max_log_files = { + "max-log-files" + , "Specify maximum number of rotated log files to be saved (no limit by setting to 0)" + , MAX_LOG_FILES + }; const command_line::arg_descriptor<std::string> arg_log_level = { "log-level" , "" diff --git a/src/daemon/command_parser_executor.cpp b/src/daemon/command_parser_executor.cpp index 34d9fb4c8..aa688294d 100644 --- a/src/daemon/command_parser_executor.cpp +++ b/src/daemon/command_parser_executor.cpp @@ -27,6 +27,7 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "common/dns_utils.h" +#include "common/command_line.h" #include "version.h" #include "daemon/command_parser_executor.h" @@ -326,12 +327,26 @@ bool t_command_parser_executor::start_mining(const std::vector<std::string>& arg if(args.size() == 4) { - ignore_battery = args[3] == "true"; + if(args[3] == "true" || command_line::is_yes(args[3]) || args[3] == "1") + { + ignore_battery = true; + } + else if(args[3] != "false" && !command_line::is_no(args[3]) && args[3] != "0") + { + return false; + } } if(args.size() >= 3) { - do_background_mining = args[2] == "true"; + if(args[2] == "true" || command_line::is_yes(args[2]) || args[2] == "1") + { + do_background_mining = true; + } + else if(args[2] != "false" && !command_line::is_no(args[2]) && args[2] != "0") + { + return false; + } } if(args.size() >= 2) diff --git a/src/daemon/main.cpp b/src/daemon/main.cpp index 49494e889..88bb1fd0c 100644 --- a/src/daemon/main.cpp +++ b/src/daemon/main.cpp @@ -84,6 +84,7 @@ int main(int argc, char const * argv[]) command_line::add_arg(core_settings, daemon_args::arg_log_file); command_line::add_arg(core_settings, daemon_args::arg_log_level); command_line::add_arg(core_settings, daemon_args::arg_max_log_file_size); + command_line::add_arg(core_settings, daemon_args::arg_max_log_files); command_line::add_arg(core_settings, daemon_args::arg_max_concurrency); command_line::add_arg(core_settings, daemon_args::arg_zmq_rpc_bind_ip); command_line::add_arg(core_settings, daemon_args::arg_zmq_rpc_bind_port); @@ -203,7 +204,7 @@ int main(int argc, char const * argv[]) if (!command_line::is_arg_defaulted(vm, daemon_args::arg_log_file)) log_file_path = command_line::get_arg(vm, daemon_args::arg_log_file); log_file_path = bf::absolute(log_file_path, relative_path_base); - mlog_configure(log_file_path.string(), true, command_line::get_arg(vm, daemon_args::arg_max_log_file_size)); + mlog_configure(log_file_path.string(), true, command_line::get_arg(vm, daemon_args::arg_max_log_file_size), command_line::get_arg(vm, daemon_args::arg_max_log_files)); // Set log level if (!command_line::is_arg_defaulted(vm, daemon_args::arg_log_level)) @@ -262,6 +263,9 @@ int main(int argc, char const * argv[]) } else { +#ifdef HAVE_READLINE + rdln::suspend_readline pause_readline; +#endif std::cerr << "Unknown command: " << command.front() << std::endl; return 1; } diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp index 487853441..956c84a01 100644 --- a/src/daemon/rpc_command_executor.cpp +++ b/src/daemon/rpc_command_executor.cpp @@ -976,7 +976,7 @@ bool t_rpc_command_executor::print_transaction_pool_stats() { } else { - memset(&res.pool_stats, 0, sizeof(res.pool_stats)); + res.pool_stats = {}; if (!m_rpc_server->on_get_transaction_pool_stats(req, res, false) || res.status != CORE_RPC_STATUS_OK) { tools::fail_msg_writer() << make_error(fail_message, res.status); @@ -1895,7 +1895,7 @@ bool t_rpc_command_executor::sync_info() for (const auto &s: res.spans) if (s.rate > 0.0f && s.connection_id == p.info.connection_id) nblocks += s.nblocks, size += s.size; - tools::success_msg_writer() << address << " " << epee::string_tools::pad_string(p.info.peer_id, 16, '0', true) << " " << p.info.height << " " << p.info.current_download << " kB/s, " << nblocks << " blocks / " << size/1e6 << " MB queued"; + tools::success_msg_writer() << address << " " << epee::string_tools::pad_string(p.info.peer_id, 16, '0', true) << " " << epee::string_tools::pad_string(p.info.state, 16) << " " << p.info.height << " " << p.info.current_download << " kB/s, " << nblocks << " blocks / " << size/1e6 << " MB queued"; } uint64_t total_size = 0; diff --git a/src/debug_utilities/cn_deserialize.cpp b/src/debug_utilities/cn_deserialize.cpp index dfbd3b864..6c09b0f18 100644 --- a/src/debug_utilities/cn_deserialize.cpp +++ b/src/debug_utilities/cn_deserialize.cpp @@ -27,6 +27,8 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <boost/filesystem.hpp> +#include <boost/algorithm/string/join.hpp> +#include <boost/range/adaptor/transformed.hpp> #include "cryptonote_basic/cryptonote_basic.h" #include "cryptonote_basic/tx_extra.h" #include "cryptonote_core/blockchain.h" @@ -51,6 +53,7 @@ static void print_extra_fields(const std::vector<cryptonote::tx_extra_field> &fi else if (typeid(cryptonote::tx_extra_pub_key) == fields[n].type()) std::cout << "extra pub key: " << boost::get<cryptonote::tx_extra_pub_key>(fields[n]).pub_key; else if (typeid(cryptonote::tx_extra_nonce) == fields[n].type()) std::cout << "extra nonce: " << epee::string_tools::buff_to_hex_nodelimer(boost::get<cryptonote::tx_extra_nonce>(fields[n]).nonce); else if (typeid(cryptonote::tx_extra_merge_mining_tag) == fields[n].type()) std::cout << "extra merge mining tag: depth " << boost::get<cryptonote::tx_extra_merge_mining_tag>(fields[n]).depth << ", merkle root " << boost::get<cryptonote::tx_extra_merge_mining_tag>(fields[n]).merkle_root; + else if (typeid(cryptonote::tx_extra_additional_pub_keys) == fields[n].type()) std::cout << "additional tx pubkeys: " << boost::join(boost::get<cryptonote::tx_extra_additional_pub_keys>(fields[n]).data | boost::adaptors::transformed([](const crypto::public_key &key){ return epee::string_tools::pod_to_hex(key); }), ", " ); else if (typeid(cryptonote::tx_extra_mysterious_minergate) == fields[n].type()) std::cout << "extra minergate custom: " << epee::string_tools::buff_to_hex_nodelimer(boost::get<cryptonote::tx_extra_mysterious_minergate>(fields[n]).data); else std::cout << "unknown"; std::cout << std::endl; diff --git a/src/debug_utilities/object_sizes.cpp b/src/debug_utilities/object_sizes.cpp index a3d037a59..ab8839636 100644 --- a/src/debug_utilities/object_sizes.cpp +++ b/src/debug_utilities/object_sizes.cpp @@ -94,6 +94,11 @@ int main(int argc, char* argv[]) SL(nodetool::node_server<cryptonote::t_cryptonote_protocol_handler<cryptonote::core>>); SL(nodetool::p2p_connection_context_t<cryptonote::t_cryptonote_protocol_handler<cryptonote::core>::connection_context>); SL(nodetool::network_address_old); + SL(nodetool::peerlist_entry_base<nodetool::network_address_old>); + + SL(nodetool::network_config); + SL(nodetool::basic_node_data); + SL(cryptonote::CORE_SYNC_DATA); SL(tools::wallet2::transfer_details); SL(tools::wallet2::payment_details); diff --git a/src/device/device_ledger.cpp b/src/device/device_ledger.cpp index 3b9ab6744..c70422887 100644 --- a/src/device/device_ledger.cpp +++ b/src/device/device_ledger.cpp @@ -48,6 +48,15 @@ namespace hw { /* ===================================================================== */ /* === Debug ==== */ /* ===================================================================== */ + #ifdef WIN32 + static char *pcsc_stringify_error(LONG rv) { + static __thread char out[20]; + sprintf_s(out, sizeof(out), "0x%08lX", rv); + + return out; + } + #endif + void set_apdu_verbose(bool verbose) { apdu_verbose = verbose; } @@ -396,7 +405,7 @@ namespace hw { } } - if (mszReaders) { + if (rv == SCARD_S_SUCCESS && mszReaders) { #ifdef SCARD_AUTOALLOCATE SCardFreeMemory(this->hContext, mszReaders); #else diff --git a/src/device/device_ledger.hpp b/src/device/device_ledger.hpp index f1fcaab87..b62bdf959 100644 --- a/src/device/device_ledger.hpp +++ b/src/device/device_ledger.hpp @@ -33,8 +33,13 @@ #include <cstddef> #include <string> #include "device.hpp" +#ifdef WIN32 +#include <winscard.h> +#define MAX_ATR_SIZE 33 +#else #include <PCSC/winscard.h> #include <PCSC/wintypes.h> +#endif #include <boost/thread/mutex.hpp> #include <boost/thread/recursive_mutex.hpp> diff --git a/src/mnemonics/electrum-words.cpp b/src/mnemonics/electrum-words.cpp index 6a2a3e0c4..7dd09ecb9 100644 --- a/src/mnemonics/electrum-words.cpp +++ b/src/mnemonics/electrum-words.cpp @@ -67,6 +67,9 @@ #include "language_base.h" #include "singleton.h" +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "mnemonic" + namespace { uint32_t create_checksum_index(const std::vector<std::string> &word_list, @@ -152,6 +155,7 @@ namespace if (full_match) { *language = *it1; + MINFO("Full match for language " << (*language)->get_english_language_name()); return true; } // Some didn't match. Clear the index array. @@ -164,9 +168,11 @@ namespace if (fallback) { *language = fallback; + MINFO("Fallback match for language " << (*language)->get_english_language_name()); return true; } + MINFO("No match found"); return false; } @@ -217,7 +223,9 @@ namespace checksum; std::string trimmed_last_word = last_word.length() > unique_prefix_length ? Language::utf8prefix(last_word, unique_prefix_length) : last_word; - return trimmed_checksum == trimmed_last_word; + bool ret = trimmed_checksum == trimmed_last_word; + MINFO("Checksum is %s" << (ret ? "valid" : "invalid")); + return ret; } } @@ -253,7 +261,10 @@ namespace crypto boost::split(seed, words, boost::is_any_of(" "), boost::token_compress_on); if (len % 4) + { + MERROR("Invalid seed: not a multiple of 4"); return false; + } bool has_checksum = true; if (len) @@ -263,6 +274,7 @@ namespace crypto if (seed.size() != expected/2 && seed.size() != expected && seed.size() != expected + 1) { + MERROR("Invalid seed: unexpected number of words"); return false; } @@ -274,6 +286,7 @@ namespace crypto Language::Base *language; if (!find_seed_language(seed, has_checksum, matched_indices, &language)) { + MERROR("Invalid seed: language not found"); return false; } language_name = language->get_language_name(); @@ -284,6 +297,7 @@ namespace crypto if (!checksum_test(seed, language->get_unique_prefix_length())) { // Checksum fail + MERROR("Invalid seed: invalid checksum"); return false; } seed.pop_back(); @@ -300,7 +314,11 @@ namespace crypto val = w1 + word_list_length * (((word_list_length - w1) + w2) % word_list_length) + word_list_length * word_list_length * (((word_list_length - w2) + w3) % word_list_length); - if (!(val % word_list_length == w1)) return false; + if (!(val % word_list_length == w1)) + { + MERROR("Invalid seed: mumble mumble"); + return false; + } dst.append((const char*)&val, 4); // copy 4 bytes to position } @@ -332,9 +350,15 @@ namespace crypto { std::string s; if (!words_to_bytes(words, s, sizeof(dst), true, language_name)) + { + MERROR("Invalid seed: failed to convert words to bytes"); return false; + } if (s.size() != sizeof(dst)) + { + MERROR("Invalid seed: wrong output size"); return false; + } dst = *(const crypto::secret_key*)s.data(); return true; } diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index 4606f66ee..90e2f78b1 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -244,6 +244,7 @@ namespace nodetool bool check_connection_and_handshake_with_peer(const epee::net_utils::network_address& na, uint64_t last_seen_stamp); bool gray_peerlist_housekeeping(); + bool check_incoming_connections(); void kill() { ///< will be called e.g. from deinit() _info("Killing the net_node"); @@ -307,6 +308,7 @@ namespace nodetool epee::math_helper::once_a_time_seconds<1> m_connections_maker_interval; epee::math_helper::once_a_time_seconds<60*30, false> m_peerlist_store_interval; epee::math_helper::once_a_time_seconds<60> m_gray_peerlist_housekeeping_interval; + epee::math_helper::once_a_time_seconds<900, false> m_incoming_connections_interval; std::string m_bind_ip; std::string m_port; @@ -316,6 +318,7 @@ namespace nodetool std::list<epee::net_utils::network_address> m_priority_peers; std::vector<epee::net_utils::network_address> m_exclusive_peers; std::vector<epee::net_utils::network_address> m_seed_nodes; + bool m_fallback_seed_nodes_added; std::list<nodetool::peerlist_entry> m_command_line_peers; uint64_t m_peer_livetime; //keep connections to initiate some interactions diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index 501be3894..85470f799 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -425,6 +425,7 @@ namespace nodetool bool res = handle_command_line(vm); CHECK_AND_ASSERT_MES(res, false, "Failed to handle command line"); + m_fallback_seed_nodes_added = false; if (m_nettype == cryptonote::TESTNET) { memcpy(&m_network_id, &::config::testnet::NETWORK_ID, 16); @@ -518,6 +519,7 @@ namespace nodetool for (const auto &peer: get_seed_nodes(cryptonote::MAINNET)) full_addrs.insert(peer); + m_fallback_seed_nodes_added = true; } } } @@ -1154,7 +1156,6 @@ namespace nodetool size_t try_count = 0; size_t current_index = crypto::rand<size_t>()%m_seed_nodes.size(); - bool fallback_nodes_added = false; while(true) { if(m_net_server.is_stop_signal_sent()) @@ -1164,15 +1165,21 @@ namespace nodetool break; if(++try_count > m_seed_nodes.size()) { - if (!fallback_nodes_added) + if (!m_fallback_seed_nodes_added) { MWARNING("Failed to connect to any of seed peers, trying fallback seeds"); + current_index = m_seed_nodes.size(); for (const auto &peer: get_seed_nodes(m_nettype)) { MDEBUG("Fallback seed node: " << peer); append_net_address(m_seed_nodes, peer, cryptonote::get_config(m_nettype).P2P_DEFAULT_PORT); } - fallback_nodes_added = true; + m_fallback_seed_nodes_added = true; + if (current_index == m_seed_nodes.size()) + { + MWARNING("No fallback seeds, continuing without seeds"); + break; + } // continue for another few cycles } else @@ -1313,6 +1320,20 @@ namespace nodetool m_connections_maker_interval.do_call(boost::bind(&node_server<t_payload_net_handler>::connections_maker, this)); m_gray_peerlist_housekeeping_interval.do_call(boost::bind(&node_server<t_payload_net_handler>::gray_peerlist_housekeeping, this)); m_peerlist_store_interval.do_call(boost::bind(&node_server<t_payload_net_handler>::store_config, this)); + m_incoming_connections_interval.do_call(boost::bind(&node_server<t_payload_net_handler>::check_incoming_connections, this)); + return true; + } + //----------------------------------------------------------------------------------- + template<class t_payload_net_handler> + bool node_server<t_payload_net_handler>::check_incoming_connections() + { + if (m_offline || m_hide_my_port) + return true; + if (get_incoming_connections_count() == 0) + { + const el::Level level = el::Level::Warning; + MCLOG_RED(level, "global", "No incoming connections - check firewalls/routers allow port " << get_this_peer_port()); + } return true; } //----------------------------------------------------------------------------------- diff --git a/src/ringct/rctOps.cpp b/src/ringct/rctOps.cpp index cc46d0aa7..68cc43128 100644 --- a/src/ringct/rctOps.cpp +++ b/src/ringct/rctOps.cpp @@ -134,12 +134,9 @@ namespace rct { } key zeroCommit(xmr_amount amount) { - key mask = identity(); - mask = scalarmultBase(mask); key am = d2h(amount); key bH = scalarmultH(am); - addKeys(mask, mask, bH); - return mask; + return addKeys(G, bH); } key commit(xmr_amount amount, const key &mask) { diff --git a/src/ringct/rctOps.h b/src/ringct/rctOps.h index 3f8f6955c..f8889af5c 100644 --- a/src/ringct/rctOps.h +++ b/src/ringct/rctOps.h @@ -62,6 +62,7 @@ namespace rct { static const key Z = { {0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 } }; static const key I = { {0x01, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 , 0x00, 0x00, 0x00,0x00 } }; static const key L = { {0xed, 0xd3, 0xf5, 0x5c, 0x1a, 0x63, 0x12, 0x58, 0xd6, 0x9c, 0xf7, 0xa2, 0xde, 0xf9, 0xde, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10 } }; + static const key G = { {0x58, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66 } }; //Creates a zero scalar inline key zero() { return Z; } diff --git a/src/ringct/rctSigs.cpp b/src/ringct/rctSigs.cpp index 777b4d13a..cc6fbe738 100644 --- a/src/ringct/rctSigs.cpp +++ b/src/ringct/rctSigs.cpp @@ -862,9 +862,9 @@ namespace rct { results[i] = verBulletproof(rv.p.bulletproofs[i]); else results[i] = verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]); - }); + }, true); } - waiter.wait(); + waiter.wait(&tpool); for (size_t i = 0; i < rv.outPk.size(); ++i) { if (!results[i]) { @@ -970,9 +970,9 @@ namespace rct { results[i] = verBulletproof(rv.p.bulletproofs[i]); else results[i] = verRange(rv.outPk[i].mask, rv.p.rangeSigs[i]); - }); + }, true); } - waiter.wait(); + waiter.wait(&tpool); for (size_t i = 0; i < results.size(); ++i) { if (!results[i]) { @@ -989,9 +989,9 @@ namespace rct { for (size_t i = 0 ; i < rv.mixRing.size() ; i++) { tpool.submit(&waiter, [&, i] { results[i] = verRctMGSimple(message, rv.p.MGs[i], rv.mixRing[i], pseudoOuts[i]); - }); + }, true); } - waiter.wait(); + waiter.wait(&tpool); for (size_t i = 0; i < results.size(); ++i) { if (!results[i]) { diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index dc7b6b30f..b55b1994b 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -226,47 +226,47 @@ namespace cryptonote if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_BLOCKS_FAST>(invoke_http_mode::BIN, "/getblocks.bin", req, res, r)) return r; - std::list<std::pair<cryptonote::blobdata, std::list<cryptonote::blobdata> > > bs; + std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > > bs; - if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, bs, res.current_height, res.start_height, req.prune, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT)) + if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, bs, res.current_height, res.start_height, req.prune, !req.no_miner_tx, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT)) { res.status = "Failed"; return false; } size_t pruned_size = 0, unpruned_size = 0, ntxes = 0; + res.blocks.reserve(bs.size()); + res.output_indices.reserve(bs.size()); for(auto& bd: bs) { res.blocks.resize(res.blocks.size()+1); - res.blocks.back().block = bd.first; - pruned_size += bd.first.size(); - unpruned_size += bd.first.size(); + res.blocks.back().block = bd.first.first; + pruned_size += bd.first.first.size(); + unpruned_size += bd.first.first.size(); res.output_indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices()); res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices()); - block b; - if (!parse_and_validate_block_from_blob(bd.first, b)) + if (!req.no_miner_tx) { - res.status = "Invalid block"; - return false; - } - bool r = m_core.get_tx_outputs_gindexs(get_transaction_hash(b.miner_tx), res.output_indices.back().indices.back().indices); - if (!r) - { - res.status = "Failed"; - return false; + bool r = m_core.get_tx_outputs_gindexs(bd.first.second, res.output_indices.back().indices.back().indices); + if (!r) + { + res.status = "Failed"; + return false; + } } - size_t txidx = 0; ntxes += bd.second.size(); - for (std::list<cryptonote::blobdata>::iterator i = bd.second.begin(); i != bd.second.end(); ++i) + res.blocks.back().txs.reserve(bd.second.size()); + res.output_indices.back().indices.reserve(bd.second.size()); + for (std::vector<std::pair<crypto::hash, cryptonote::blobdata>>::iterator i = bd.second.begin(); i != bd.second.end(); ++i) { - unpruned_size += i->size(); - res.blocks.back().txs.push_back(std::move(*i)); - i->clear(); - i->shrink_to_fit(); + unpruned_size += i->second.size(); + res.blocks.back().txs.push_back(std::move(i->second)); + i->second.clear(); + i->second.shrink_to_fit(); pruned_size += res.blocks.back().txs.back().size(); res.output_indices.back().indices.push_back(COMMAND_RPC_GET_BLOCKS_FAST::tx_output_indices()); - bool r = m_core.get_tx_outputs_gindexs(b.tx_hashes[txidx++], res.output_indices.back().indices.back().indices); + bool r = m_core.get_tx_outputs_gindexs(i->first, res.output_indices.back().indices.back().indices); if (!r) { res.status = "Failed"; @@ -286,7 +286,7 @@ namespace cryptonote if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_GET_ALT_BLOCKS_HASHES>(invoke_http_mode::JON, "/get_alt_blocks_hashes", req, res, r)) return r; - std::list<block> blks; + std::vector<block> blks; if(!m_core.get_alternative_blocks(blks)) { @@ -328,8 +328,8 @@ namespace cryptonote res.status = "Error retrieving block at height " + std::to_string(height); return true; } - std::list<transaction> txs; - std::list<crypto::hash> missed_txs; + std::vector<transaction> txs; + std::vector<crypto::hash> missed_txs; m_core.get_transactions(blk.tx_hashes, txs, missed_txs); res.blocks.resize(res.blocks.size() + 1); res.blocks.back().block = block_to_blob(blk); @@ -544,8 +544,8 @@ namespace cryptonote } vh.push_back(*reinterpret_cast<const crypto::hash*>(b.data())); } - std::list<crypto::hash> missed_txs; - std::list<transaction> txs; + std::vector<crypto::hash> missed_txs; + std::vector<transaction> txs; bool r = m_core.get_transactions(vh, txs, missed_txs); if(!r) { @@ -566,25 +566,26 @@ namespace cryptonote if(r) { // sort to match original request - std::list<transaction> sorted_txs; + std::vector<transaction> sorted_txs; std::vector<tx_info>::const_iterator i; + unsigned txs_processed = 0; for (const crypto::hash &h: vh) { if (std::find(missed_txs.begin(), missed_txs.end(), h) == missed_txs.end()) { - if (txs.empty()) + if (txs.size() == txs_processed) { res.status = "Failed: internal error - txs is empty"; return true; } // core returns the ones it finds in the right order - if (get_transaction_hash(txs.front()) != h) + if (get_transaction_hash(txs[txs_processed]) != h) { res.status = "Failed: tx hash mismatch"; return true; } - sorted_txs.push_back(std::move(txs.front())); - txs.pop_front(); + sorted_txs.push_back(std::move(txs[txs_processed])); + ++txs_processed; } else if ((i = std::find_if(pool_tx_info.begin(), pool_tx_info.end(), [h](const tx_info &txi) { return epee::string_tools::pod_to_hex(h) == txi.id_hash; })) != pool_tx_info.end()) { @@ -595,7 +596,7 @@ namespace cryptonote return true; } sorted_txs.push_back(tx); - missed_txs.remove(h); + missed_txs.erase(std::find(missed_txs.begin(), missed_txs.end(), h)); pool_tx_hashes.insert(h); const std::string hash_string = epee::string_tools::pod_to_hex(h); for (const auto &ti: pool_tx_info) @@ -614,7 +615,7 @@ namespace cryptonote LOG_PRINT_L2("Found " << found_in_pool << "/" << vh.size() << " transactions in the pool"); } - std::list<std::string>::const_iterator txhi = req.txs_hashes.begin(); + std::vector<std::string>::const_iterator txhi = req.txs_hashes.begin(); std::vector<crypto::hash>::const_iterator vhi = vh.begin(); for(auto& tx: txs) { @@ -1655,10 +1656,10 @@ namespace cryptonote PERF_TIMER(on_flush_txpool); bool failed = false; - std::list<crypto::hash> txids; + std::vector<crypto::hash> txids; if (req.txids.empty()) { - std::list<transaction> pool_txs; + std::vector<transaction> pool_txs; bool r = m_core.get_pool_transactions(pool_txs); if (!r) { diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index 70e186848..49b730149 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -83,10 +83,12 @@ namespace cryptonote std::list<crypto::hash> block_ids; //*first 10 blocks id goes sequential, next goes in pow(2,n) offset, like 2, 4, 8, 16, 32, 64 and so on, and the last one is always genesis block */ uint64_t start_height; bool prune; + bool no_miner_tx; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE_CONTAINER_POD_AS_BLOB(block_ids) KV_SERIALIZE(start_height) KV_SERIALIZE(prune) + KV_SERIALIZE_OPT(no_miner_tx, false) END_KV_SERIALIZE_MAP() }; @@ -110,7 +112,7 @@ namespace cryptonote struct response { - std::list<block_complete_entry> blocks; + std::vector<block_complete_entry> blocks; uint64_t start_height; uint64_t current_height; std::string status; @@ -188,7 +190,7 @@ namespace cryptonote struct response { - std::list<crypto::hash> m_block_ids; + std::vector<crypto::hash> m_block_ids; uint64_t start_height; uint64_t current_height; std::string status; @@ -273,7 +275,7 @@ namespace cryptonote uint64_t total_received; uint64_t total_received_unlocked = 0; // OpenMonero only uint64_t scanned_height; - std::list<transaction> transactions; + std::vector<transaction> transactions; uint64_t blockchain_height; uint64_t scanned_block_height; std::string status; @@ -561,7 +563,7 @@ namespace cryptonote { struct request { - std::list<std::string> txs_hashes; + std::vector<std::string> txs_hashes; bool decode_as_json; bool prune; @@ -598,11 +600,11 @@ namespace cryptonote struct response { // older compatibility stuff - std::list<std::string> txs_as_hex; //transactions blobs as hex (old compat) - std::list<std::string> txs_as_json; //transactions decoded as json (old compat) + std::vector<std::string> txs_as_hex; //transactions blobs as hex (old compat) + std::vector<std::string> txs_as_json; //transactions decoded as json (old compat) // in both old and new - std::list<std::string> missed_tx; //not found transactions + std::vector<std::string> missed_tx; //not found transactions // new style std::vector<entry> txs; @@ -1565,6 +1567,8 @@ namespace cryptonote std::vector<txpool_histo> histo; uint32_t num_double_spends; + txpool_stats(): bytes_total(0), bytes_min(0), bytes_max(0), bytes_med(0), fee_total(0), oldest(0), txs_total(0), num_failing(0), num_10m(0), num_not_relayed(0), histo_98pc(0), num_double_spends(0) {} + BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(bytes_total) KV_SERIALIZE(bytes_min) @@ -1929,7 +1933,7 @@ namespace cryptonote { struct request { - std::list<std::string> txids; + std::vector<std::string> txids; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(txids) @@ -2146,7 +2150,7 @@ namespace cryptonote { struct request { - std::list<std::string> txids; + std::vector<std::string> txids; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(txids) diff --git a/src/rpc/daemon_handler.cpp b/src/rpc/daemon_handler.cpp index 39f169cdf..55858cc2a 100644 --- a/src/rpc/daemon_handler.cpp +++ b/src/rpc/daemon_handler.cpp @@ -50,9 +50,9 @@ namespace rpc void DaemonHandler::handle(const GetBlocksFast::Request& req, GetBlocksFast::Response& res) { - std::list<std::pair<blobdata, std::list<blobdata> > > blocks; + std::vector<std::pair<std::pair<blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, blobdata> > > > blocks; - if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, blocks, res.current_height, res.start_height, req.prune, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT)) + if(!m_core.find_blockchain_supplement(req.start_height, req.block_ids, blocks, res.current_height, res.start_height, req.prune, true, COMMAND_RPC_GET_BLOCKS_FAST_MAX_COUNT)) { res.status = Message::STATUS_FAILED; res.error_details = "core::find_blockchain_supplement() returned false"; @@ -62,9 +62,6 @@ namespace rpc res.blocks.resize(blocks.size()); res.output_indices.resize(blocks.size()); - //TODO: really need to switch uses of std::list to std::vector unless - // it's a huge performance concern - auto it = blocks.begin(); uint64_t block_count = 0; @@ -72,7 +69,7 @@ namespace rpc { cryptonote::rpc::block_with_transactions& bwt = res.blocks[block_count]; - if (!parse_and_validate_block_from_blob(it->first, bwt.block)) + if (!parse_and_validate_block_from_blob(it->first.first, bwt.block)) { res.blocks.clear(); res.output_indices.clear(); @@ -89,11 +86,11 @@ namespace rpc res.error_details = "incorrect number of transactions retrieved for block"; return; } - std::list<transaction> txs; - for (const auto& blob : it->second) + std::vector<transaction> txs; + for (const auto& p : it->second) { txs.resize(txs.size() + 1); - if (!parse_and_validate_tx_from_blob(blob, txs.back())) + if (!parse_and_validate_tx_from_blob(p.second, txs.back())) { res.blocks.clear(); res.output_indices.clear(); @@ -163,10 +160,10 @@ namespace rpc void DaemonHandler::handle(const GetTransactions::Request& req, GetTransactions::Response& res) { - std::list<cryptonote::transaction> found_txs; - std::list<crypto::hash> missed_hashes; + std::vector<cryptonote::transaction> found_txs_vec; + std::vector<crypto::hash> missed_vec; - bool r = m_core.get_transactions(req.tx_hashes, found_txs, missed_hashes); + bool r = m_core.get_transactions(req.tx_hashes, found_txs_vec, missed_vec); // TODO: consider fixing core::get_transactions to not hide exceptions if (!r) @@ -176,20 +173,7 @@ namespace rpc return; } - size_t num_found = found_txs.size(); - - // std::list is annoying - std::vector<cryptonote::transaction> found_txs_vec - { - std::make_move_iterator(std::begin(found_txs)), - std::make_move_iterator(std::end(found_txs)) - }; - - std::vector<crypto::hash> missed_vec - { - std::make_move_iterator(std::begin(missed_hashes)), - std::make_move_iterator(std::end(missed_hashes)) - }; + size_t num_found = found_txs_vec.size(); std::vector<uint64_t> heights(num_found); std::vector<bool> in_pool(num_found, false); @@ -204,7 +188,7 @@ namespace rpc // if any missing from blockchain, check in tx pool if (!missed_vec.empty()) { - std::list<cryptonote::transaction> pool_txs; + std::vector<cryptonote::transaction> pool_txs; m_core.get_pool_transactions(pool_txs); diff --git a/src/rpc/daemon_messages.h b/src/rpc/daemon_messages.h index 1495c845f..8fff369df 100644 --- a/src/rpc/daemon_messages.h +++ b/src/rpc/daemon_messages.h @@ -106,7 +106,7 @@ BEGIN_RPC_MESSAGE_CLASS(GetHashesFast); RPC_MESSAGE_MEMBER(uint64_t, start_height); END_RPC_MESSAGE_REQUEST; BEGIN_RPC_MESSAGE_RESPONSE; - RPC_MESSAGE_MEMBER(std::list<crypto::hash>, hashes); + RPC_MESSAGE_MEMBER(std::vector<crypto::hash>, hashes); RPC_MESSAGE_MEMBER(uint64_t, start_height); RPC_MESSAGE_MEMBER(uint64_t, current_height); END_RPC_MESSAGE_RESPONSE; diff --git a/src/rpc/rpc_args.cpp b/src/rpc/rpc_args.cpp index d4a6138ba..d4044d11b 100644 --- a/src/rpc/rpc_args.cpp +++ b/src/rpc/rpc_args.cpp @@ -102,7 +102,7 @@ namespace cryptonote { if (!config.login) { - LOG_ERROR(arg.rpc_access_control_origins.name << tr(" requires RFC server password --") << arg.rpc_login.name << tr(" cannot be empty")); + LOG_ERROR(arg.rpc_access_control_origins.name << tr(" requires RPC server password --") << arg.rpc_login.name << tr(" cannot be empty")); return boost::none; } diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index b530082d7..e82c1a3c7 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -90,8 +90,6 @@ typedef cryptonote::simple_wallet sw; #define MIN_RING_SIZE 7 // Used to inform user about min ring size -- does not track actual protocol -#define OUTPUT_EXPORT_FILE_MAGIC "Monero output export\003" - #define LOCK_IDLE_SCOPE() \ bool auto_refresh_enabled = m_auto_refresh_enabled.load(std::memory_order_relaxed); \ m_auto_refresh_enabled.store(false, std::memory_order_relaxed); \ @@ -381,21 +379,10 @@ namespace boost::optional<std::pair<uint32_t, uint32_t>> parse_subaddress_lookahead(const std::string& str) { - auto pos = str.find(":"); - bool r = pos != std::string::npos; - uint32_t major; - r = r && epee::string_tools::get_xtype_from_string(major, str.substr(0, pos)); - uint32_t minor; - r = r && epee::string_tools::get_xtype_from_string(minor, str.substr(pos + 1)); - if (r) - { - return std::make_pair(major, minor); - } - else - { + auto r = tools::parse_subaddress_lookahead(str); + if (!r) fail_msg_writer() << tr("invalid format for subaddress lookahead; must be <major>:<minor>"); - return {}; - } + return r; } void handle_transfer_exception(const std::exception_ptr &e, bool trusted_daemon) @@ -508,7 +495,7 @@ namespace } if (warn_of_possible_attack) - fail_msg_writer() << tr("There was an error, which could mean the node may be trying to get you to retry creating a transaction, and zero in on which outputs you own. Or it could be a bona fide error. It may be prudent to disconnect from this node, and not try to send a tranasction immediately. Alternatively, connect to another node so the original node cannot correlate information."); + fail_msg_writer() << tr("There was an error, which could mean the node may be trying to get you to retry creating a transaction, and zero in on which outputs you own. Or it could be a bona fide error. It may be prudent to disconnect from this node, and not try to send a transaction immediately. Alternatively, connect to another node so the original node cannot correlate information."); } bool check_file_overwrite(const std::string &filename) @@ -1371,9 +1358,117 @@ bool simple_wallet::print_ring(const std::vector<std::string> &args) bool simple_wallet::set_ring(const std::vector<std::string> &args) { crypto::key_image key_image; + + // try filename first + if (args.size() == 1) + { + if (!epee::file_io_utils::is_file_exist(args[0])) + { + fail_msg_writer() << tr("File doesn't exist"); + return true; + } + + char str[4096]; + std::unique_ptr<FILE, tools::close_file> f(fopen(args[0].c_str(), "r")); + if (f) + { + while (!feof(f.get())) + { + if (!fgets(str, sizeof(str), f.get())) + break; + const size_t len = strlen(str); + if (len > 0 && str[len - 1] == '\n') + str[len - 1] = 0; + if (!str[0]) + continue; + char key_image_str[65], type_str[9]; + int read_after_key_image = 0, read = 0; + int fields = sscanf(str, "%64[abcdefABCDEF0123456789] %n%8s %n", key_image_str, &read_after_key_image, type_str, &read); + if (fields != 2) + { + fail_msg_writer() << tr("Invalid ring specification: ") << str; + continue; + } + key_image_str[64] = 0; + type_str[8] = 0; + crypto::key_image key_image; + if (read_after_key_image == 0 || !epee::string_tools::hex_to_pod(key_image_str, key_image)) + { + fail_msg_writer() << tr("Invalid key image: ") << str; + continue; + } + if (read == read_after_key_image+8 || (strcmp(type_str, "absolute") && strcmp(type_str, "relative"))) + { + fail_msg_writer() << tr("Invalid ring type, expected relative or abosolute: ") << str; + continue; + } + bool relative = !strcmp(type_str, "relative"); + if (read < 0 || (size_t)read > strlen(str)) + { + fail_msg_writer() << tr("Error reading line: ") << str; + continue; + } + bool valid = true; + std::vector<uint64_t> ring; + const char *ptr = str + read; + while (*ptr) + { + unsigned long offset; + int elements = sscanf(ptr, "%lu %n", &offset, &read); + if (elements == 0 || read <= 0 || (size_t)read > strlen(str)) + { + fail_msg_writer() << tr("Error reading line: ") << str; + valid = false; + break; + } + ring.push_back(offset); + ptr += read; + MGINFO("read offset: " << offset); + } + if (!valid) + continue; + if (ring.empty()) + { + fail_msg_writer() << tr("Invalid ring: ") << str; + continue; + } + if (relative) + { + for (size_t n = 1; n < ring.size(); ++n) + { + if (ring[n] <= 0) + { + fail_msg_writer() << tr("Invalid relative ring: ") << str; + valid = false; + break; + } + } + } + else + { + for (size_t n = 1; n < ring.size(); ++n) + { + if (ring[n] <= ring[n-1]) + { + fail_msg_writer() << tr("Invalid absolute ring: ") << str; + valid = false; + break; + } + } + } + if (!valid) + continue; + if (!m_wallet->set_ring(key_image, ring, relative)) + fail_msg_writer() << tr("Failed to set ring for key image: ") << key_image << ". " << tr("Continuing."); + } + f.reset(); + } + return true; + } + if (args.size() < 3) { - fail_msg_writer() << tr("usage: set_ring <key_image> absolute|relative <index> [<index>...]"); + fail_msg_writer() << tr("usage: set_ring <filename> | ( <key_image> absolute|relative <index> [<index>...] )"); return true; } @@ -2028,7 +2123,7 @@ simple_wallet::simple_wallet() tr("Stop mining in the daemon.")); m_cmd_binder.set_handler("set_daemon", boost::bind(&simple_wallet::set_daemon, this, _1), - tr("set_daemon <host>[:<port>]"), + tr("set_daemon <host>[:<port>] [trusted|untrusted]"), tr("Set another daemon to connect to.")); m_cmd_binder.set_handler("save_bc", boost::bind(&simple_wallet::save_bc, this, _1), @@ -2082,8 +2177,8 @@ simple_wallet::simple_wallet() tr("Donate <amount> to the development team (donate.getmonero.org).")); m_cmd_binder.set_handler("sign_transfer", boost::bind(&simple_wallet::sign_transfer, this, _1), - tr("sign_transfer [export]"), - tr("Sign a transaction from a file.")); + tr("sign_transfer [export_raw]"), + tr("Sign a transaction from a file. If the parameter \"export_raw\" is specified, transaction raw hex data suitable for the daemon RPC /sendrawtransaction is exported.")); m_cmd_binder.set_handler("submit_transfer", boost::bind(&simple_wallet::submit_transfer, this, _1), tr("Submit a signed transaction from a file.")); @@ -2153,7 +2248,7 @@ simple_wallet::simple_wallet() "refresh-type <full|optimize-coinbase|no-coinbase|default>\n " " Set the wallet's refresh behaviour.\n " "priority [0|1|2|3|4]\n " - " Set the fee too default/unimportant/normal/elevated/priority.\n " + " Set the fee to default/unimportant/normal/elevated/priority.\n " "confirm-missing-payment-id <1|0>\n " "ask-password <1|0>\n " "unit <monero|millinero|micronero|nanonero|piconero>\n " @@ -2325,7 +2420,7 @@ simple_wallet::simple_wallet() tr("Print the ring(s) used to spend a given key image or transaction (if the ring size is > 1)")); m_cmd_binder.set_handler("set_ring", boost::bind(&simple_wallet::set_ring, this, _1), - tr("set_ring <key_image> absolute|relative <index> [<index>...]"), + tr("set_ring <filename> | ( <key_image> absolute|relative <index> [<index>...] )"), tr("Set the ring used for a given key image, so it can be reused in a fork")); m_cmd_binder.set_handler("save_known_rings", boost::bind(&simple_wallet::save_known_rings, this, _1), @@ -2451,8 +2546,24 @@ bool simple_wallet::set_log(const std::vector<std::string> &args) fail_msg_writer() << tr("usage: set_log <log_level_number_0-4> | <categories>"); return true; } - if (!args.empty()) - mlog_set_log(args[0].c_str()); + if(!args.empty()) + { + uint16_t level = 0; + if(epee::string_tools::get_xtype_from_string(level, args[0])) + { + if(4 < level) + { + fail_msg_writer() << tr("wrong number range, use: set_log <log_level_number_0-4> | <categories>"); + return true; + } + mlog_set_log_level(level); + } + else + { + mlog_set_log(args[0].c_str()); + } + } + success_msg_writer() << "New log categories: " << mlog_get_categories(); return true; } @@ -3685,7 +3796,6 @@ bool simple_wallet::start_mining(const std::vector<std::string>& args) req.miner_address = m_wallet->get_account().get_public_address_str(m_wallet->nettype()); bool ok = true; - size_t max_mining_threads_count = (std::max)(tools::get_max_concurrency(), static_cast<unsigned>(2)); size_t arg_size = args.size(); if(arg_size >= 3) { @@ -3701,7 +3811,7 @@ bool simple_wallet::start_mining(const std::vector<std::string>& args) { uint16_t num = 1; ok = string_tools::get_xtype_from_string(num, args[0]); - ok = ok && (1 <= num && num <= max_mining_threads_count); + ok = ok && 1 <= num; req.threads_count = num; } else @@ -3711,8 +3821,7 @@ bool simple_wallet::start_mining(const std::vector<std::string>& args) if (!ok) { - fail_msg_writer() << tr("invalid arguments. Please use start_mining [<number_of_threads>] [do_bg_mining] [ignore_battery], " - "<number_of_threads> should be from 1 to ") << max_mining_threads_count; + fail_msg_writer() << tr("invalid arguments. Please use start_mining [<number_of_threads>] [do_bg_mining] [ignore_battery]"); return true; } @@ -3778,6 +3887,33 @@ bool simple_wallet::set_daemon(const std::vector<std::string>& args) } LOCK_IDLE_SCOPE(); m_wallet->init(daemon_url); + + if (args.size() == 2) + { + if (args[1] == "trusted") + m_trusted_daemon = true; + else if (args[1] == "untrusted") + m_trusted_daemon = false; + else + { + fail_msg_writer() << tr("Expected trusted or untrusted, got ") << args[1] << ": assuming untrusted"; + m_trusted_daemon = false; + } + } + else + { + m_trusted_daemon = false; + try + { + if (tools::is_local_address(m_wallet->get_daemon_address())) + { + MINFO(tr("Daemon is local, assuming trusted")); + m_trusted_daemon = true; + } + } + catch (const std::exception &e) { } + } + success_msg_writer() << boost::format("Daemon set to %s, %s") % daemon_url % (*m_trusted_daemon ? tr("trusted") : tr("untrusted")); } else { fail_msg_writer() << tr("This does not seem to be a valid daemon URL."); } @@ -3941,6 +4077,8 @@ bool simple_wallet::show_balance_unlocked(bool detailed) std::string extra; if (m_wallet->has_multisig_partial_key_images()) extra = tr(" (Some owned outputs have partial key images - import_multisig_info needed)"); + else if (m_wallet->has_unknown_key_images()) + extra += tr(" (Some owned outputs have missing key images - import_key_images needed)"); success_msg_writer() << tr("Currently selected account: [") << m_current_subaddress_account << tr("] ") << m_wallet->get_subaddress_label({m_current_subaddress_account, 0}); const std::string tag = m_wallet->get_account_tags().second[m_current_subaddress_account]; success_msg_writer() << tr("Tag: ") << (tag.empty() ? std::string{tr("(No tag assigned)")} : tag); @@ -4437,6 +4575,7 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri } vector<cryptonote::tx_destination_entry> dsts; + size_t num_subaddresses = 0; for (size_t i = 0; i < local_args.size(); i += 2) { cryptonote::address_parse_info info; @@ -4448,6 +4587,7 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri } de.addr = info.address; de.is_subaddress = info.is_subaddress; + num_subaddresses += info.is_subaddress; if (info.has_payment_id) { @@ -4480,7 +4620,7 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri } // prompt is there is no payment id and confirmation is required - if (!payment_id_seen && m_wallet->confirm_missing_payment_id()) + if (!payment_id_seen && m_wallet->confirm_missing_payment_id() && dsts.size() > num_subaddresses) { std::string accepted = input_line(tr("No payment id is included with this transaction. Is this okay? (Y/Yes/N/No): ")); if (std::cin.eof()) @@ -4933,7 +5073,7 @@ bool simple_wallet::sweep_main(uint64_t below, const std::vector<std::string> &a } // prompt is there is no payment id and confirmation is required - if (!payment_id_seen && m_wallet->confirm_missing_payment_id()) + if (!payment_id_seen && m_wallet->confirm_missing_payment_id() && !info.is_subaddress) { std::string accepted = input_line(tr("No payment id is included with this transaction. Is this okay? (Y/Yes/N/No): ")); if (std::cin.eof()) @@ -5146,7 +5286,7 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) } // prompt if there is no payment id and confirmation is required - if (!payment_id_seen && m_wallet->confirm_missing_payment_id()) + if (!payment_id_seen && m_wallet->confirm_missing_payment_id() && !info.is_subaddress) { std::string accepted = input_line(tr("No payment id is included with this transaction. Is this okay? (Y/Yes/N/No): ")); if (std::cin.eof()) @@ -5299,7 +5439,7 @@ bool simple_wallet::donate(const std::vector<std::string> &args_) local_args.push_back(amount_str); if (!payment_id_str.empty()) local_args.push_back(payment_id_str); - message_writer() << tr("Donating ") << amount_str << " to The Monero Project (donate.getmonero.org or "<< MONERO_DONATION_ADDR <<")."; + message_writer() << (boost::format(tr("Donating %s %s to The Monero Project (donate.getmonero.org or %s).")) % amount_str % cryptonote::get_unit(cryptonote::get_default_decimal_point()) % MONERO_DONATION_ADDR).str(); transfer_new(local_args); return true; } @@ -5469,9 +5609,9 @@ bool simple_wallet::sign_transfer(const std::vector<std::string> &args_) fail_msg_writer() << tr("This is a watch only wallet"); return true; } - if (args_.size() > 1 || (args_.size() == 1 && args_[0] != "export")) + if (args_.size() > 1 || (args_.size() == 1 && args_[0] != "export_raw")) { - fail_msg_writer() << tr("usage: sign_transfer [export]"); + fail_msg_writer() << tr("usage: sign_transfer [export_raw]"); return true; } if (m_wallet->ask_password() && !get_and_verify_password()) { return true; } @@ -6016,10 +6156,7 @@ static std::string get_human_readable_timestamp(uint64_t ts) #endif uint64_t now = time(NULL); uint64_t diff = ts > now ? ts - now : now - ts; - if (diff > 24*3600) - strftime(buffer, sizeof(buffer), "%Y-%m-%d", &tm); - else - strftime(buffer, sizeof(buffer), "%I:%M:%S %p", &tm); + strftime(buffer, sizeof(buffer), "%Y-%m-%d %H:%M:%S", &tm); return std::string(buffer); } //---------------------------------------------------------------------------------------------------- @@ -6131,7 +6268,7 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_) if (payment_id.substr(16).find_first_not_of('0') == std::string::npos) payment_id = payment_id.substr(0,16); std::string note = m_wallet->get_tx_note(pd.m_tx_hash); - output.insert(std::make_pair(pd.m_block_height, std::make_pair(true, (boost::format("%16.16s %20.20s %s %s %d %s %s") % get_human_readable_timestamp(pd.m_timestamp) % print_money(pd.m_amount) % string_tools::pod_to_hex(pd.m_tx_hash) % payment_id % pd.m_subaddr_index.minor % "-" % note).str()))); + output.insert(std::make_pair(pd.m_block_height, std::make_pair(true, (boost::format("%25.25s %20.20s %s %s %d %s %s") % get_human_readable_timestamp(pd.m_timestamp) % print_money(pd.m_amount) % string_tools::pod_to_hex(pd.m_tx_hash) % payment_id % pd.m_subaddr_index.minor % "-" % note).str()))); } } @@ -6164,7 +6301,7 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_) if (payment_id.substr(16).find_first_not_of('0') == std::string::npos) payment_id = payment_id.substr(0,16); std::string note = m_wallet->get_tx_note(i->first); - output.insert(std::make_pair(pd.m_block_height, std::make_pair(false, (boost::format("%16.16s %20.20s %s %s %14.14s %s %s - %s") % get_human_readable_timestamp(pd.m_timestamp) % print_money(pd.m_amount_in - change - fee) % string_tools::pod_to_hex(i->first) % payment_id % print_money(fee) % dests % print_subaddr_indices(pd.m_subaddr_indices) % note).str()))); + output.insert(std::make_pair(pd.m_block_height, std::make_pair(false, (boost::format("%25.25s %20.20s %s %s %14.14s %s %s - %s") % get_human_readable_timestamp(pd.m_timestamp) % print_money(pd.m_amount_in - change - fee) % string_tools::pod_to_hex(i->first) % payment_id % print_money(fee) % dests % print_subaddr_indices(pd.m_subaddr_indices) % note).str()))); } } @@ -6190,7 +6327,7 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_) std::string double_spend_note; if (i->second.m_double_spend_seen) double_spend_note = tr("[Double spend seen on the network: this transaction may or may not end up being mined] "); - message_writer() << (boost::format("%8.8s %6.6s %16.16s %20.20s %s %s %d %s %s%s") % "pool" % "in" % get_human_readable_timestamp(pd.m_timestamp) % print_money(pd.m_amount) % string_tools::pod_to_hex(pd.m_tx_hash) % payment_id % pd.m_subaddr_index.minor % "-" % note % double_spend_note).str(); + message_writer() << (boost::format("%8.8s %6.6s %25.25s %20.20s %s %s %d %s %s%s") % "pool" % "in" % get_human_readable_timestamp(pd.m_timestamp) % print_money(pd.m_amount) % string_tools::pod_to_hex(pd.m_tx_hash) % payment_id % pd.m_subaddr_index.minor % "-" % note % double_spend_note).str(); } } catch (const std::exception& e) @@ -6213,7 +6350,7 @@ bool simple_wallet::show_transfers(const std::vector<std::string> &args_) std::string note = m_wallet->get_tx_note(i->first); bool is_failed = pd.m_state == tools::wallet2::unconfirmed_transfer_details::failed; if ((failed && is_failed) || (!is_failed && pending)) { - message_writer() << (boost::format("%8.8s %6.6s %16.16s %20.20s %s %s %14.14s %s - %s") % (is_failed ? tr("failed") : tr("pending")) % tr("out") % get_human_readable_timestamp(pd.m_timestamp) % print_money(amount - pd.m_change - fee) % string_tools::pod_to_hex(i->first) % payment_id % print_money(fee) % print_subaddr_indices(pd.m_subaddr_indices) % note).str(); + message_writer() << (boost::format("%8.8s %6.6s %25.25s %20.20s %s %s %14.14s %s - %s") % (is_failed ? tr("failed") : tr("pending")) % tr("out") % get_human_readable_timestamp(pd.m_timestamp) % print_money(amount - pd.m_change - fee) % string_tools::pod_to_hex(i->first) % payment_id % print_money(fee) % print_subaddr_indices(pd.m_subaddr_indices) % note).str(); } } } @@ -7182,19 +7319,8 @@ bool simple_wallet::export_outputs(const std::vector<std::string> &args) LOCK_IDLE_SCOPE(); try { - std::vector<tools::wallet2::transfer_details> outs = m_wallet->export_outputs(); - - std::stringstream oss; - boost::archive::portable_binary_oarchive ar(oss); - ar << outs; - - std::string magic(OUTPUT_EXPORT_FILE_MAGIC, strlen(OUTPUT_EXPORT_FILE_MAGIC)); - const cryptonote::account_public_address &keys = m_wallet->get_account().get_keys().m_account_address; - std::string header; - header += std::string((const char *)&keys.m_spend_public_key, sizeof(crypto::public_key)); - header += std::string((const char *)&keys.m_view_public_key, sizeof(crypto::public_key)); - std::string ciphertext = m_wallet->encrypt_with_view_secret_key(header + oss.str()); - bool r = epee::file_io_utils::save_string_to_file(filename, magic + ciphertext); + std::string data = m_wallet->export_outputs_to_str(); + bool r = epee::file_io_utils::save_string_to_file(filename, data); if (!r) { fail_msg_writer() << tr("failed to save file ") << filename; @@ -7233,63 +7359,16 @@ bool simple_wallet::import_outputs(const std::vector<std::string> &args) fail_msg_writer() << tr("failed to read file ") << filename; return true; } - const size_t magiclen = strlen(OUTPUT_EXPORT_FILE_MAGIC); - if (data.size() < magiclen || memcmp(data.data(), OUTPUT_EXPORT_FILE_MAGIC, magiclen)) - { - fail_msg_writer() << "Bad output export file magic in " << filename; - return true; - } - - try - { - data = m_wallet->decrypt_with_view_secret_key(std::string(data, magiclen)); - } - catch (const std::exception &e) - { - fail_msg_writer() << "Failed to decrypt " << filename << ": " << e.what(); - return true; - } - - const size_t headerlen = 2 * sizeof(crypto::public_key); - if (data.size() < headerlen) - { - fail_msg_writer() << "Bad data size from file " << filename; - return true; - } - const crypto::public_key &public_spend_key = *(const crypto::public_key*)&data[0]; - const crypto::public_key &public_view_key = *(const crypto::public_key*)&data[sizeof(crypto::public_key)]; - const cryptonote::account_public_address &keys = m_wallet->get_account().get_keys().m_account_address; - if (public_spend_key != keys.m_spend_public_key || public_view_key != keys.m_view_public_key) - { - fail_msg_writer() << "Outputs from " << filename << " are for a different account"; - return true; - } try { - std::string body(data, headerlen); - std::stringstream iss; - iss << body; - std::vector<tools::wallet2::transfer_details> outputs; - try - { - boost::archive::portable_binary_iarchive ar(iss); - ar >> outputs; - } - catch (...) - { - iss.str(""); - iss << body; - boost::archive::binary_iarchive ar(iss); - ar >> outputs; - } LOCK_IDLE_SCOPE(); - size_t n_outputs = m_wallet->import_outputs(outputs); + size_t n_outputs = m_wallet->import_outputs_from_str(data); success_msg_writer() << boost::lexical_cast<std::string>(n_outputs) << " outputs imported"; } catch (const std::exception &e) { - fail_msg_writer() << "Failed to import outputs: " << e.what(); + fail_msg_writer() << "Failed to import outputs " << filename << ": " << e.what(); return true; } @@ -7529,7 +7608,7 @@ int main(int argc, char* argv[]) std::tie(vm, should_terminate) = wallet_args::main( argc, argv, "monero-wallet-cli [--wallet-file=<file>|--generate-new-wallet=<file>] [<COMMAND>]", - sw::tr("This is the command line monero wallet. It needs to connect to a monero\ndaemon to work correctly.\nWARNING: Do not reuse your Monero keys on an another fork, UNLESS this fork has key reuse mitigations built in. Doing so will harm your privacy."), + sw::tr("This is the command line monero wallet. It needs to connect to a monero\ndaemon to work correctly.\nWARNING: Do not reuse your Monero keys on another fork, UNLESS this fork has key reuse mitigations built in. Doing so will harm your privacy."), desc_params, positional_options, [](const std::string &s, bool emphasis){ tools::scoped_message_writer(emphasis ? epee::console_color_white : epee::console_color_default, true) << s; }, diff --git a/src/wallet/api/wallet.cpp b/src/wallet/api/wallet.cpp index 908e2db2e..c7dbd29e4 100644 --- a/src/wallet/api/wallet.cpp +++ b/src/wallet/api/wallet.cpp @@ -372,6 +372,7 @@ WalletImpl::WalletImpl(NetworkType nettype) , m_trustedDaemon(false) , m_wallet2Callback(nullptr) , m_recoveringFromSeed(false) + , m_recoveringFromDevice(false) , m_synchronized(false) , m_rebuildWalletCache(false) , m_is_connected(false) @@ -419,6 +420,7 @@ bool WalletImpl::create(const std::string &path, const std::string &password, co clearStatus(); m_recoveringFromSeed = false; + m_recoveringFromDevice = false; bool keys_file_exists; bool wallet_file_exists; tools::wallet2::wallet_exists(path, keys_file_exists, wallet_file_exists); @@ -554,18 +556,26 @@ bool WalletImpl::recoverFromKeysWithPassword(const std::string &path, } // parse view secret key + bool has_viewkey = true; + crypto::secret_key viewkey; if (viewkey_string.empty()) { - setStatusError(tr("No view key supplied, cancelled")); - return false; + if(has_spendkey) { + has_viewkey = false; + } + else { + setStatusError(tr("Neither view key nor spend key supplied, cancelled")); + return false; + } } - cryptonote::blobdata viewkey_data; - if(!epee::string_tools::parse_hexstr_to_binbuff(viewkey_string, viewkey_data) || viewkey_data.size() != sizeof(crypto::secret_key)) - { - setStatusError(tr("failed to parse secret view key")); - return false; + if(has_viewkey) { + cryptonote::blobdata viewkey_data; + if(!epee::string_tools::parse_hexstr_to_binbuff(viewkey_string, viewkey_data) || viewkey_data.size() != sizeof(crypto::secret_key)) + { + setStatusError(tr("failed to parse secret view key")); + return false; + } + viewkey = *reinterpret_cast<const crypto::secret_key*>(viewkey_data.data()); } - crypto::secret_key viewkey = *reinterpret_cast<const crypto::secret_key*>(viewkey_data.data()); - // check the spend and view keys match the given address crypto::public_key pkey; if(has_spendkey) { @@ -578,26 +588,32 @@ bool WalletImpl::recoverFromKeysWithPassword(const std::string &path, return false; } } - if (!crypto::secret_key_to_public_key(viewkey, pkey)) { - setStatusError(tr("failed to verify secret view key")); - return false; - } - if (info.address.m_view_public_key != pkey) { - setStatusError(tr("view key does not match address")); - return false; + if(has_viewkey) { + if (!crypto::secret_key_to_public_key(viewkey, pkey)) { + setStatusError(tr("failed to verify secret view key")); + return false; + } + if (info.address.m_view_public_key != pkey) { + setStatusError(tr("view key does not match address")); + return false; + } } try { - if (has_spendkey) { + if (has_spendkey && has_viewkey) { m_wallet->generate(path, password, info.address, spendkey, viewkey); - setSeedLanguage(language); - LOG_PRINT_L1("Generated new wallet from keys with seed language: " + language); + LOG_PRINT_L1("Generated new wallet from spend key and view key"); } - else { + if(!has_spendkey && has_viewkey) { m_wallet->generate(path, password, info.address, viewkey); LOG_PRINT_L1("Generated new view only wallet from keys"); } + if(has_spendkey && !has_viewkey) { + m_wallet->generate(path, password, spendkey, true, false, false); + setSeedLanguage(language); + LOG_PRINT_L1("Generated deterministic wallet from spend key with seed language: " + language); + } } catch (const std::exception& e) { @@ -607,11 +623,28 @@ bool WalletImpl::recoverFromKeysWithPassword(const std::string &path, return true; } +bool WalletImpl::recoverFromDevice(const std::string &path, const std::string &password, const std::string &device_name) +{ + clearStatus(); + m_recoveringFromSeed = false; + m_recoveringFromDevice = true; + try + { + m_wallet->restore(path, password, device_name); + LOG_PRINT_L1("Generated new wallet from device: " + device_name); + } + catch (const std::exception& e) { + setStatusError(string(tr("failed to generate new wallet: ")) + e.what()); + return false; + } + return true; +} bool WalletImpl::open(const std::string &path, const std::string &password) { clearStatus(); m_recoveringFromSeed = false; + m_recoveringFromDevice = false; try { // TODO: handle "deprecated" // Check if wallet cache exists @@ -649,6 +682,7 @@ bool WalletImpl::recover(const std::string &path, const std::string &password, c } m_recoveringFromSeed = true; + m_recoveringFromDevice = false; crypto::secret_key recovery_key; std::string old_language; if (!crypto::ElectrumWords::words_to_bytes(seed, recovery_key, old_language)) { @@ -870,6 +904,16 @@ void WalletImpl::setRecoveringFromSeed(bool recoveringFromSeed) m_recoveringFromSeed = recoveringFromSeed; } +void WalletImpl::setRecoveringFromDevice(bool recoveringFromDevice) +{ + m_recoveringFromDevice = recoveringFromDevice; +} + +void WalletImpl::setSubaddressLookahead(uint32_t major, uint32_t minor) +{ + m_wallet->set_subaddress_lookahead(major, minor); +} + uint64_t WalletImpl::balance(uint32_t accountIndex) const { return m_wallet->balance(accountIndex); @@ -1896,7 +1940,7 @@ void WalletImpl::refreshThreadFunc() // if auto refresh enabled, we wait for the "m_refreshIntervalSeconds" interval. // if not - we wait forever if (m_refreshIntervalMillis > 0) { - boost::posix_time::milliseconds wait_for_ms(m_refreshIntervalMillis); + boost::posix_time::milliseconds wait_for_ms(m_refreshIntervalMillis.load()); m_refreshCV.timed_wait(lock, wait_for_ms); } else { m_refreshCV.wait(lock); @@ -1982,7 +2026,7 @@ bool WalletImpl::isNewWallet() const // with the daemon (pull hashes instead of pull blocks). // If wallet cache is rebuilt, creation height stored in .keys is used. // Watch only wallet is a copy of an existing wallet. - return !(blockChainHeight() > 1 || m_recoveringFromSeed || m_rebuildWalletCache) && !watchOnly(); + return !(blockChainHeight() > 1 || m_recoveringFromSeed || m_recoveringFromDevice || m_rebuildWalletCache) && !watchOnly(); } bool WalletImpl::doInit(const string &daemon_address, uint64_t upper_transaction_size_limit, bool ssl) diff --git a/src/wallet/api/wallet.h b/src/wallet/api/wallet.h index 813ca4b30..08232cafd 100644 --- a/src/wallet/api/wallet.h +++ b/src/wallet/api/wallet.h @@ -76,6 +76,9 @@ public: const std::string &address_string, const std::string &viewkey_string, const std::string &spendkey_string = ""); + bool recoverFromDevice(const std::string &path, + const std::string &password, + const std::string &device_name); bool close(bool store = true); std::string seed() const; std::string getSeedLanguage() const; @@ -115,6 +118,8 @@ public: void setRefreshFromBlockHeight(uint64_t refresh_from_block_height); uint64_t getRefreshFromBlockHeight() const { return m_wallet->get_refresh_from_block_height(); }; void setRecoveringFromSeed(bool recoveringFromSeed); + void setRecoveringFromDevice(bool recoveringFromDevice) override; + void setSubaddressLookahead(uint32_t major, uint32_t minor) override; bool watchOnly() const; bool rescanSpent(); NetworkType nettype() const {return static_cast<NetworkType>(m_wallet->nettype());} @@ -232,6 +237,7 @@ private: // so it shouldn't be considered as new and pull blocks (slow-refresh) // instead of pulling hashes (fast-refresh) std::atomic<bool> m_recoveringFromSeed; + std::atomic<bool> m_recoveringFromDevice; std::atomic<bool> m_synchronized; std::atomic<bool> m_rebuildWalletCache; // cache connection status to avoid unnecessary RPC calls diff --git a/src/wallet/api/wallet2_api.h b/src/wallet/api/wallet2_api.h index 5b99bd975..f54255e91 100644 --- a/src/wallet/api/wallet2_api.h +++ b/src/wallet/api/wallet2_api.h @@ -509,6 +509,21 @@ struct Wallet */ virtual void setRecoveringFromSeed(bool recoveringFromSeed) = 0; + /*! + * \brief setRecoveringFromDevice - set state to recovering from device + * + * \param recoveringFromDevice - true/false + */ + virtual void setRecoveringFromDevice(bool recoveringFromDevice) = 0; + + /*! + * \brief setSubaddressLookahead - set size of subaddress lookahead + * + * \param major - size fot the major index + * \param minor - size fot the minor index + */ + virtual void setSubaddressLookahead(uint32_t major, uint32_t minor) = 0; + /** * @brief connectToDaemon - connects to the daemon. TODO: check if it can be removed * @return @@ -1015,6 +1030,23 @@ struct WalletManager } /*! + * \brief creates wallet using hardware device. + * \param path Name of wallet file to be created + * \param password Password of wallet file + * \param nettype Network type + * \param deviceName Device name + * \param restoreHeight restore from start height (0 sets to current height) + * \param subaddressLookahead Size of subaddress lookahead (empty sets to some default low value) + * \return Wallet instance (Wallet::status() needs to be called to check if recovered successfully) + */ + virtual Wallet * createWalletFromDevice(const std::string &path, + const std::string &password, + NetworkType nettype, + const std::string &deviceName, + uint64_t restoreHeight = 0, + const std::string &subaddressLookahead = "") = 0; + + /*! * \brief Closes wallet. In case operation succeeded, wallet object deleted. in case operation failed, wallet object not deleted * \param wallet previously opened / created wallet instance * \return None diff --git a/src/wallet/api/wallet_manager.cpp b/src/wallet/api/wallet_manager.cpp index a63716576..99eadc82f 100644 --- a/src/wallet/api/wallet_manager.cpp +++ b/src/wallet/api/wallet_manager.cpp @@ -114,6 +114,26 @@ Wallet *WalletManagerImpl::createWalletFromKeys(const std::string &path, return wallet; } +Wallet *WalletManagerImpl::createWalletFromDevice(const std::string &path, + const std::string &password, + NetworkType nettype, + const std::string &deviceName, + uint64_t restoreHeight, + const std::string &subaddressLookahead) +{ + WalletImpl * wallet = new WalletImpl(nettype); + if(restoreHeight > 0){ + wallet->setRefreshFromBlockHeight(restoreHeight); + } + auto lookahead = tools::parse_subaddress_lookahead(subaddressLookahead); + if (lookahead) + { + wallet->setSubaddressLookahead(lookahead->first, lookahead->second); + } + wallet->recoverFromDevice(path, password, deviceName); + return wallet; +} + bool WalletManagerImpl::closeWallet(Wallet *wallet, bool store) { WalletImpl * wallet_ = dynamic_cast<WalletImpl*>(wallet); diff --git a/src/wallet/api/wallet_manager.h b/src/wallet/api/wallet_manager.h index 26238b658..19aad9ee3 100644 --- a/src/wallet/api/wallet_manager.h +++ b/src/wallet/api/wallet_manager.h @@ -64,6 +64,12 @@ public: const std::string &addressString, const std::string &viewKeyString, const std::string &spendKeyString = ""); + virtual Wallet * createWalletFromDevice(const std::string &path, + const std::string &password, + NetworkType nettype, + const std::string &deviceName, + uint64_t restoreHeight = 0, + const std::string &subaddressLookahead = "") override; virtual bool closeWallet(Wallet *wallet, bool store = true); bool walletExists(const std::string &path); bool verifyWalletPassword(const std::string &keys_file_name, const std::string &password, bool no_spend_key) const; diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index aa0702716..9492cc085 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -110,6 +110,8 @@ using namespace cryptonote; #define MULTISIG_EXPORT_FILE_MAGIC "Monero multisig export\001" +#define OUTPUT_EXPORT_FILE_MAGIC "Monero output export\003" + #define SEGREGATION_FORK_HEIGHT 1546000 #define TESTNET_SEGREGATION_FORK_HEIGHT 1000000 #define STAGENET_SEGREGATION_FORK_HEIGHT 1000000 @@ -117,6 +119,7 @@ using namespace cryptonote; static const std::string MULTISIG_SIGNATURE_MAGIC = "SigMultisigPkV1"; +std::atomic<unsigned int> tools::wallet2::key_ref::refs(0); namespace { @@ -768,8 +771,6 @@ bool wallet2::init(std::string daemon_address, boost::optional<epee::net_utils:: m_daemon_address = std::move(daemon_address); m_daemon_login = std::move(daemon_login); // When switching from light wallet to full wallet, we need to reset the height we got from lw node. - if(m_light_wallet) - m_local_bc_height = m_blockchain.size(); return m_http_client.set_server(get_daemon_address(), get_daemon_login(), ssl); } //---------------------------------------------------------------------------------------------------- @@ -1034,6 +1035,23 @@ void wallet2::check_acc_out_precomp(const tx_out &o, const crypto::key_derivatio tx_scan_info.error = false; } //---------------------------------------------------------------------------------------------------- +void wallet2::check_acc_out_precomp(const tx_out &o, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, size_t i, const is_out_data *is_out_data, tx_scan_info_t &tx_scan_info) const +{ + if (!is_out_data || i >= is_out_data->received.size()) + return check_acc_out_precomp(o, derivation, additional_derivations, i, tx_scan_info); + + tx_scan_info.received = is_out_data->received[i]; + if(tx_scan_info.received) + { + tx_scan_info.money_transfered = o.amount; // may be 0 for ringct outputs + } + else + { + tx_scan_info.money_transfered = 0; + } + tx_scan_info.error = false; +} +//---------------------------------------------------------------------------------------------------- static uint64_t decodeRct(const rct::rctSig & rv, const crypto::key_derivation &derivation, unsigned int i, rct::key & mask, hw::device &hwdev) { crypto::secret_key scalar1; @@ -1087,16 +1105,48 @@ void wallet2::scan_output(const cryptonote::transaction &tx, const crypto::publi ++num_vouts_received; } //---------------------------------------------------------------------------------------------------- -void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector<uint64_t> &o_indices, uint64_t height, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen) +void wallet2::cache_tx_data(const cryptonote::transaction& tx, const crypto::hash &txid, tx_cache_data &tx_cache_data) const { - //ensure device is let in NONE mode in any case - hw::device &hwdev = m_account.get_device(); - - boost::unique_lock<hw::device> hwdev_lock (hwdev); - hw::reset_mode rst(hwdev); - hwdev_lock.unlock(); + const cryptonote::account_keys& keys = m_account.get_keys(); - // In this function, tx (probably) only contains the base information + if(!parse_tx_extra(tx.extra, tx_cache_data.tx_extra_fields)) + { + // Extra may only be partially parsed, it's OK if tx_extra_fields contains public key + LOG_PRINT_L0("Transaction extra has unsupported format: " << txid); + tx_cache_data.tx_extra_fields.clear(); + return; + } + + // Don't try to extract tx public key if tx has no ouputs + const bool is_miner = tx.vin.size() == 1 && tx.vin[0].type() == typeid(cryptonote::txin_gen); + if (!is_miner || m_refresh_type != RefreshType::RefreshNoCoinbase) + { + const size_t rec_size = is_miner && m_refresh_type == RefreshType::RefreshOptimizeCoinbase ? 1 : tx.vout.size(); + if (!tx.vout.empty()) + { + // if tx.vout is not empty, we loop through all tx pubkeys + const std::vector<boost::optional<cryptonote::subaddress_receive_info>> rec(rec_size, boost::none); + + tx_extra_pub_key pub_key_field; + size_t pk_index = 0; + while (find_tx_extra_field_by_type(tx_cache_data.tx_extra_fields, pub_key_field, pk_index++)) + tx_cache_data.primary.push_back({pub_key_field.pub_key, {}, rec}); + + // additional tx pubkeys and derivations for multi-destination transfers involving one or more subaddresses + tx_extra_additional_pub_keys additional_tx_pub_keys; + std::vector<crypto::key_derivation> additional_derivations; + if (find_tx_extra_field_by_type(tx_cache_data.tx_extra_fields, additional_tx_pub_keys)) + { + for (size_t i = 0; i < additional_tx_pub_keys.data.size(); ++i) + tx_cache_data.additional.push_back({additional_tx_pub_keys.data[i], {}, {}}); + } + } + } +} +//---------------------------------------------------------------------------------------------------- +void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector<uint64_t> &o_indices, uint64_t height, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen, const tx_cache_data &tx_cache_data) +{ + // In this function, tx (probably) only contains the base information // (that is, the prunable stuff may or may not be included) if (!miner_tx && !pool) process_unconfirmed(txid, tx, height); @@ -1104,12 +1154,16 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote std::unordered_map<cryptonote::subaddress_index, uint64_t> tx_money_got_in_outs; // per receiving subaddress index crypto::public_key tx_pub_key = null_pkey; - std::vector<tx_extra_field> tx_extra_fields; - if(!parse_tx_extra(tx.extra, tx_extra_fields)) + std::vector<tx_extra_field> local_tx_extra_fields; + if (tx_cache_data.tx_extra_fields.empty()) { - // Extra may only be partially parsed, it's OK if tx_extra_fields contains public key - LOG_PRINT_L0("Transaction extra has unsupported format: " << txid); + if(!parse_tx_extra(tx.extra, local_tx_extra_fields)) + { + // Extra may only be partially parsed, it's OK if tx_extra_fields contains public key + LOG_PRINT_L0("Transaction extra has unsupported format: " << txid); + } } + const std::vector<tx_extra_field> &tx_extra_fields = tx_cache_data.tx_extra_fields.empty() ? local_tx_extra_fields : tx_cache_data.tx_extra_fields; // Don't try to extract tx public key if tx has no ouputs size_t pk_index = 0; @@ -1128,6 +1182,11 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote m_callback->on_skip_transaction(height, txid, tx); break; } + if (!tx_cache_data.primary.empty()) + { + THROW_WALLET_EXCEPTION_IF(tx_cache_data.primary.size() < pk_index || pub_key_field.pub_key != tx_cache_data.primary[pk_index - 1].pkey, + error::wallet_internal_error, "tx_cache_data is out of sync"); + } int num_vouts_received = 0; tx_pub_key = pub_key_field.pub_key; @@ -1136,28 +1195,49 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote const cryptonote::account_keys& keys = m_account.get_keys(); crypto::key_derivation derivation; - hwdev_lock.lock(); - hwdev.set_mode(hw::device::TRANSACTION_PARSE); - if (!hwdev.generate_key_derivation(tx_pub_key, keys.m_view_secret_key, derivation)) + std::vector<crypto::key_derivation> additional_derivations; + tx_extra_additional_pub_keys additional_tx_pub_keys; + const wallet2::is_out_data *is_out_data_ptr = NULL; + if (tx_cache_data.primary.empty()) { - MWARNING("Failed to generate key derivation from tx pubkey, skipping"); - static_assert(sizeof(derivation) == sizeof(rct::key), "Mismatched sizes of key_derivation and rct::key"); - memcpy(&derivation, rct::identity().bytes, sizeof(derivation)); - } + hw::device &hwdev = m_account.get_device(); + boost::unique_lock<hw::device> hwdev_lock (hwdev); + hw::reset_mode rst(hwdev); - // additional tx pubkeys and derivations for multi-destination transfers involving one or more subaddresses - std::vector<crypto::public_key> additional_tx_pub_keys = get_additional_tx_pub_keys_from_extra(tx); - std::vector<crypto::key_derivation> additional_derivations; - for (size_t i = 0; i < additional_tx_pub_keys.size(); ++i) + hwdev.set_mode(hw::device::TRANSACTION_PARSE); + if (!hwdev.generate_key_derivation(tx_pub_key, keys.m_view_secret_key, derivation)) + { + MWARNING("Failed to generate key derivation from tx pubkey in " << txid << ", skipping"); + static_assert(sizeof(derivation) == sizeof(rct::key), "Mismatched sizes of key_derivation and rct::key"); + memcpy(&derivation, rct::identity().bytes, sizeof(derivation)); + } + + // additional tx pubkeys and derivations for multi-destination transfers involving one or more subaddresses + if (find_tx_extra_field_by_type(tx_extra_fields, additional_tx_pub_keys)) + { + for (size_t i = 0; i < additional_tx_pub_keys.data.size(); ++i) + { + additional_derivations.push_back({}); + if (!hwdev.generate_key_derivation(additional_tx_pub_keys.data[i], keys.m_view_secret_key, additional_derivations.back())) + { + MWARNING("Failed to generate key derivation from additional tx pubkey in " << txid << ", skipping"); + memcpy(&additional_derivations.back(), rct::identity().bytes, sizeof(crypto::key_derivation)); + } + } + } + } + else { - additional_derivations.push_back({}); - if (!hwdev.generate_key_derivation(additional_tx_pub_keys[i], keys.m_view_secret_key, additional_derivations.back())) + THROW_WALLET_EXCEPTION_IF(pk_index - 1 >= tx_cache_data.primary.size(), + error::wallet_internal_error, "pk_index out of range of tx_cache_data"); + is_out_data_ptr = &tx_cache_data.primary[pk_index - 1]; + derivation = tx_cache_data.primary[pk_index - 1].derivation; + for (size_t n = 0; n < tx_cache_data.additional.size(); ++n) { - MWARNING("Failed to generate key derivation from tx pubkey, skipping"); - additional_derivations.pop_back(); + additional_tx_pub_keys.data.push_back(tx_cache_data.additional[n].pkey); + additional_derivations.push_back(tx_cache_data.additional[n].derivation); } } - hwdev_lock.unlock(); if (miner_tx && m_refresh_type == RefreshNoCoinbase) { @@ -1165,7 +1245,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote } else if (miner_tx && m_refresh_type == RefreshOptimizeCoinbase) { - check_acc_out_precomp(tx.vout[0], derivation, additional_derivations, 0, tx_scan_info[0]); + check_acc_out_precomp(tx.vout[0], derivation, additional_derivations, 0, is_out_data_ptr, tx_scan_info[0]); THROW_WALLET_EXCEPTION_IF(tx_scan_info[0].error, error::acc_outs_lookup_error, tx, tx_pub_key, m_account.get_keys()); // this assumes that the miner tx pays a single address @@ -1176,59 +1256,59 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote for (size_t i = 1; i < tx.vout.size(); ++i) { tpool.submit(&waiter, boost::bind(&wallet2::check_acc_out_precomp, this, std::cref(tx.vout[i]), std::cref(derivation), std::cref(additional_derivations), i, - std::ref(tx_scan_info[i]))); + std::cref(is_out_data_ptr), std::ref(tx_scan_info[i])), true); } - waiter.wait(); + waiter.wait(&tpool); // then scan all outputs from 0 - hwdev_lock.lock(); + hw::device &hwdev = m_account.get_device(); + boost::unique_lock<hw::device> hwdev_lock (hwdev); hwdev.set_mode(hw::device::NONE); for (size_t i = 0; i < tx.vout.size(); ++i) { THROW_WALLET_EXCEPTION_IF(tx_scan_info[i].error, error::acc_outs_lookup_error, tx, tx_pub_key, m_account.get_keys()); if (tx_scan_info[i].received) { - hwdev.conceal_derivation(tx_scan_info[i].received->derivation, tx_pub_key, additional_tx_pub_keys, derivation, additional_derivations); + hwdev.conceal_derivation(tx_scan_info[i].received->derivation, tx_pub_key, additional_tx_pub_keys.data, derivation, additional_derivations); scan_output(tx, tx_pub_key, i, tx_scan_info[i], num_vouts_received, tx_money_got_in_outs, outs); } } - hwdev_lock.unlock(); } } - else if (tx.vout.size() > 1 && tools::threadpool::getInstance().get_max_concurrency() > 1) + else if (tx.vout.size() > 1 && tools::threadpool::getInstance().get_max_concurrency() > 1 && !is_out_data_ptr) { for (size_t i = 0; i < tx.vout.size(); ++i) { tpool.submit(&waiter, boost::bind(&wallet2::check_acc_out_precomp, this, std::cref(tx.vout[i]), std::cref(derivation), std::cref(additional_derivations), i, - std::ref(tx_scan_info[i]))); + std::cref(is_out_data_ptr), std::ref(tx_scan_info[i])), true); } - waiter.wait(); + waiter.wait(&tpool); - hwdev_lock.lock(); + hw::device &hwdev = m_account.get_device(); + boost::unique_lock<hw::device> hwdev_lock (hwdev); hwdev.set_mode(hw::device::NONE); for (size_t i = 0; i < tx.vout.size(); ++i) { THROW_WALLET_EXCEPTION_IF(tx_scan_info[i].error, error::acc_outs_lookup_error, tx, tx_pub_key, m_account.get_keys()); if (tx_scan_info[i].received) { - hwdev.conceal_derivation(tx_scan_info[i].received->derivation, tx_pub_key, additional_tx_pub_keys, derivation, additional_derivations); + hwdev.conceal_derivation(tx_scan_info[i].received->derivation, tx_pub_key, additional_tx_pub_keys.data, derivation, additional_derivations); scan_output(tx, tx_pub_key, i, tx_scan_info[i], num_vouts_received, tx_money_got_in_outs, outs); } } - hwdev_lock.unlock(); } else { for (size_t i = 0; i < tx.vout.size(); ++i) { - check_acc_out_precomp(tx.vout[i], derivation, additional_derivations, i, tx_scan_info[i]); + check_acc_out_precomp(tx.vout[i], derivation, additional_derivations, i, is_out_data_ptr, tx_scan_info[i]); THROW_WALLET_EXCEPTION_IF(tx_scan_info[i].error, error::acc_outs_lookup_error, tx, tx_pub_key, m_account.get_keys()); if (tx_scan_info[i].received) { - hwdev_lock.lock(); + hw::device &hwdev = m_account.get_device(); + boost::unique_lock<hw::device> hwdev_lock (hwdev); hwdev.set_mode(hw::device::NONE); - hwdev.conceal_derivation(tx_scan_info[i].received->derivation, tx_pub_key, additional_tx_pub_keys, derivation, additional_derivations); + hwdev.conceal_derivation(tx_scan_info[i].received->derivation, tx_pub_key, additional_tx_pub_keys.data, derivation, additional_derivations); scan_output(tx, tx_pub_key, i, tx_scan_info[i], num_vouts_received, tx_money_got_in_outs, outs); - hwdev_lock.unlock(); } } } @@ -1568,12 +1648,11 @@ void wallet2::process_outgoing(const crypto::hash &txid, const cryptonote::trans add_rings(tx); } //---------------------------------------------------------------------------------------------------- -void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const crypto::hash& bl_id, uint64_t height, const cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices &o_indices) +void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const parsed_block &parsed_block, const crypto::hash& bl_id, uint64_t height, const std::vector<tx_cache_data> &tx_cache_data, size_t tx_cache_data_offset) { - size_t txidx = 0; - THROW_WALLET_EXCEPTION_IF(bche.txs.size() + 1 != o_indices.indices.size(), error::wallet_internal_error, + THROW_WALLET_EXCEPTION_IF(bche.txs.size() + 1 != parsed_block.o_indices.indices.size(), error::wallet_internal_error, "block transactions=" + std::to_string(bche.txs.size()) + - " not match with daemon response size=" + std::to_string(o_indices.indices.size())); + " not match with daemon response size=" + std::to_string(parsed_block.o_indices.indices.size())); //handle transactions from new block @@ -1581,29 +1660,26 @@ void wallet2::process_new_blockchain_entry(const cryptonote::block& b, const cry if(b.timestamp + 60*60*24 > m_account.get_createtime() && height >= m_refresh_from_block_height) { TIME_MEASURE_START(miner_tx_handle_time); - process_new_transaction(get_transaction_hash(b.miner_tx), b.miner_tx, o_indices.indices[txidx++].indices, height, b.timestamp, true, false, false); + if (m_refresh_type != RefreshNoCoinbase) + process_new_transaction(get_transaction_hash(b.miner_tx), b.miner_tx, parsed_block.o_indices.indices[0].indices, height, b.timestamp, true, false, false, tx_cache_data[tx_cache_data_offset]); + ++tx_cache_data_offset; TIME_MEASURE_FINISH(miner_tx_handle_time); TIME_MEASURE_START(txs_handle_time); THROW_WALLET_EXCEPTION_IF(bche.txs.size() != b.tx_hashes.size(), error::wallet_internal_error, "Wrong amount of transactions for block"); - size_t idx = 0; - for (const auto& txblob: bche.txs) + THROW_WALLET_EXCEPTION_IF(bche.txs.size() != parsed_block.txes.size(), error::wallet_internal_error, "Wrong amount of transactions for block"); + for (size_t idx = 0; idx < b.tx_hashes.size(); ++idx) { - cryptonote::transaction tx; - bool r = parse_and_validate_tx_base_from_blob(txblob, tx); - THROW_WALLET_EXCEPTION_IF(!r, error::tx_parse_error, txblob); - process_new_transaction(b.tx_hashes[idx], tx, o_indices.indices[txidx++].indices, height, b.timestamp, false, false, false); - ++idx; + process_new_transaction(b.tx_hashes[idx], parsed_block.txes[idx], parsed_block.o_indices.indices[idx+1].indices, height, b.timestamp, false, false, false, tx_cache_data[tx_cache_data_offset++]); } TIME_MEASURE_FINISH(txs_handle_time); LOG_PRINT_L2("Processed block: " << bl_id << ", height " << height << ", " << miner_tx_handle_time + txs_handle_time << "(" << miner_tx_handle_time << "/" << txs_handle_time <<")ms"); }else { - if (!(height % 100)) + if (!(height % 128)) LOG_PRINT_L2( "Skipped block by timestamp, height: " << height << ", block time " << b.timestamp << ", account time " << m_account.get_createtime()); } m_blockchain.push_back(bl_id); - ++m_local_bc_height; if (0 != m_callback) m_callback->on_new_block(height, b); @@ -1648,7 +1724,7 @@ void wallet2::parse_block_round(const cryptonote::blobdata &blob, cryptonote::bl bl_id = get_block_hash(bl); } //---------------------------------------------------------------------------------------------------- -void wallet2::pull_blocks(uint64_t start_height, uint64_t &blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::list<cryptonote::block_complete_entry> &blocks, std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices) +void wallet2::pull_blocks(uint64_t start_height, uint64_t &blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices) { cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::request req = AUTO_VAL_INIT(req); cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::response res = AUTO_VAL_INIT(res); @@ -1683,6 +1759,7 @@ void wallet2::pull_blocks(uint64_t start_height, uint64_t &blocks_start_height, } req.start_height = start_height; + req.no_miner_tx = m_refresh_type == RefreshNoCoinbase; m_daemon_rpc_mutex.lock(); bool r = net_utils::invoke_http_bin("/getblocks.bin", req, res, m_http_client, rpc_timeout); m_daemon_rpc_mutex.unlock(); @@ -1694,11 +1771,11 @@ void wallet2::pull_blocks(uint64_t start_height, uint64_t &blocks_start_height, boost::lexical_cast<std::string>(res.output_indices.size()) + ") sizes from daemon"); blocks_start_height = res.start_height; - blocks = res.blocks; - o_indices = res.output_indices; + blocks = std::move(res.blocks); + o_indices = std::move(res.output_indices); } //---------------------------------------------------------------------------------------------------- -void wallet2::pull_hashes(uint64_t start_height, uint64_t &blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::list<crypto::hash> &hashes) +void wallet2::pull_hashes(uint64_t start_height, uint64_t &blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<crypto::hash> &hashes) { cryptonote::COMMAND_RPC_GET_HASHES_FAST::request req = AUTO_VAL_INIT(req); cryptonote::COMMAND_RPC_GET_HASHES_FAST::response res = AUTO_VAL_INIT(res); @@ -1713,88 +1790,117 @@ void wallet2::pull_hashes(uint64_t start_height, uint64_t &blocks_start_height, THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::get_hashes_error, res.status); blocks_start_height = res.start_height; - hashes = res.m_block_ids; + hashes = std::move(res.m_block_ids); } //---------------------------------------------------------------------------------------------------- -void wallet2::process_blocks(uint64_t start_height, const std::list<cryptonote::block_complete_entry> &blocks, const std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices, uint64_t& blocks_added) +void wallet2::process_parsed_blocks(uint64_t start_height, const std::vector<cryptonote::block_complete_entry> &blocks, const std::vector<parsed_block> &parsed_blocks, uint64_t& blocks_added) { size_t current_index = start_height; blocks_added = 0; - size_t tx_o_indices_idx = 0; - THROW_WALLET_EXCEPTION_IF(blocks.size() != o_indices.size(), error::wallet_internal_error, "size mismatch"); + THROW_WALLET_EXCEPTION_IF(blocks.size() != parsed_blocks.size(), error::wallet_internal_error, "size mismatch"); THROW_WALLET_EXCEPTION_IF(!m_blockchain.is_in_bounds(current_index), error::wallet_internal_error, "Index out of bounds of hashchain"); tools::threadpool& tpool = tools::threadpool::getInstance(); - int threads = tpool.get_max_concurrency(); - if (threads > 1) + tools::threadpool::waiter waiter; + + size_t num_txes = 0; + std::vector<tx_cache_data> tx_cache_data; + for (size_t i = 0; i < blocks.size(); ++i) + num_txes += 1 + parsed_blocks[i].txes.size(); + tx_cache_data.resize(num_txes); + size_t txidx = 0; + for (size_t i = 0; i < blocks.size(); ++i) { - std::vector<crypto::hash> round_block_hashes(threads); - std::vector<cryptonote::block> round_blocks(threads); - std::deque<bool> error(threads); - size_t blocks_size = blocks.size(); - std::list<block_complete_entry>::const_iterator blocki = blocks.begin(); - for (size_t b = 0; b < blocks_size; b += threads) + THROW_WALLET_EXCEPTION_IF(parsed_blocks[i].txes.size() != parsed_blocks[i].block.tx_hashes.size(), + error::wallet_internal_error, "Mismatched parsed_blocks[i].txes.size() and parsed_blocks[i].block.tx_hashes.size()"); + if (m_refresh_type != RefreshNoCoinbase) + tpool.submit(&waiter, [&, i, txidx](){ cache_tx_data(parsed_blocks[i].block.miner_tx, get_transaction_hash(parsed_blocks[i].block.miner_tx), tx_cache_data[txidx]); }); + ++txidx; + for (size_t idx = 0; idx < parsed_blocks[i].txes.size(); ++idx) { - size_t round_size = std::min((size_t)threads, blocks_size - b); - tools::threadpool::waiter waiter; + tpool.submit(&waiter, [&, i, idx, txidx](){ cache_tx_data(parsed_blocks[i].txes[idx], parsed_blocks[i].block.tx_hashes[idx], tx_cache_data[txidx]); }); + ++txidx; + } + } + THROW_WALLET_EXCEPTION_IF(txidx != num_txes, error::wallet_internal_error, "txidx does not match tx_cache_data size"); + waiter.wait(&tpool); - std::list<block_complete_entry>::const_iterator tmpblocki = blocki; - for (size_t i = 0; i < round_size; ++i) - { - tpool.submit(&waiter, boost::bind(&wallet2::parse_block_round, this, std::cref(tmpblocki->block), - std::ref(round_blocks[i]), std::ref(round_block_hashes[i]), std::ref(error[i]))); - ++tmpblocki; - } - waiter.wait(); - tmpblocki = blocki; - for (size_t i = 0; i < round_size; ++i) - { - THROW_WALLET_EXCEPTION_IF(error[i], error::block_parse_error, tmpblocki->block); - ++tmpblocki; - } - for (size_t i = 0; i < round_size; ++i) - { - const crypto::hash &bl_id = round_block_hashes[i]; - cryptonote::block &bl = round_blocks[i]; + hw::device &hwdev = m_account.get_device(); + hw::reset_mode rst(hwdev); + hwdev.set_mode(hw::device::TRANSACTION_PARSE); + const cryptonote::account_keys &keys = m_account.get_keys(); - if(current_index >= m_blockchain.size()) - { - process_new_blockchain_entry(bl, *blocki, bl_id, current_index, o_indices[b+i]); - ++blocks_added; - } - else if(bl_id != m_blockchain[current_index]) - { - //split detected here !!! - THROW_WALLET_EXCEPTION_IF(current_index == start_height, error::wallet_internal_error, - "wrong daemon response: split starts from the first block in response " + string_tools::pod_to_hex(bl_id) + - " (height " + std::to_string(start_height) + "), local block id at this height: " + - string_tools::pod_to_hex(m_blockchain[current_index])); - - detach_blockchain(current_index); - process_new_blockchain_entry(bl, *blocki, bl_id, current_index, o_indices[b+i]); - } - else + auto gender = [&](wallet2::is_out_data &iod) { + boost::unique_lock<hw::device> hwdev_lock(hwdev); + if (!hwdev.generate_key_derivation(iod.pkey, keys.m_view_secret_key, iod.derivation)) + { + MWARNING("Failed to generate key derivation from tx pubkey, skipping"); + static_assert(sizeof(iod.derivation) == sizeof(rct::key), "Mismatched sizes of key_derivation and rct::key"); + memcpy(&iod.derivation, rct::identity().bytes, sizeof(iod.derivation)); + } + }; + + for (auto &slot: tx_cache_data) + { + for (auto &iod: slot.primary) + tpool.submit(&waiter, [&gender, &iod]() { gender(iod); }, true); + for (auto &iod: slot.additional) + tpool.submit(&waiter, [&gender, &iod]() { gender(iod); }, true); + } + waiter.wait(&tpool); + + auto geniod = [&](const cryptonote::transaction &tx, size_t n_vouts, size_t txidx) { + for (size_t k = 0; k < n_vouts; ++k) + { + const auto &o = tx.vout[k]; + if (o.target.type() == typeid(cryptonote::txout_to_key)) + { + std::vector<crypto::key_derivation> additional_derivations; + for (const auto &iod: tx_cache_data[txidx].additional) + additional_derivations.push_back(iod.derivation); + const auto &key = boost::get<txout_to_key>(o.target).key; + for (size_t l = 0; l < tx_cache_data[txidx].primary.size(); ++l) { - LOG_PRINT_L2("Block is already in blockchain: " << string_tools::pod_to_hex(bl_id)); + THROW_WALLET_EXCEPTION_IF(tx_cache_data[txidx].primary[l].received.size() != n_vouts, + error::wallet_internal_error, "Unexpected received array size"); + tx_cache_data[txidx].primary[l].received[k] = is_out_to_acc_precomp(m_subaddresses, key, tx_cache_data[txidx].primary[l].derivation, additional_derivations, k, hwdev); + additional_derivations.clear(); } - ++current_index; - ++blocki; } } - } - else + }; + + txidx = 0; + for (size_t i = 0; i < blocks.size(); ++i) { - for(auto& bl_entry: blocks) + if (m_refresh_type != RefreshType::RefreshNoCoinbase) + { + THROW_WALLET_EXCEPTION_IF(txidx >= tx_cache_data.size(), error::wallet_internal_error, "txidx out of range"); + const size_t n_vouts = m_refresh_type == RefreshType::RefreshOptimizeCoinbase ? 1 : parsed_blocks[i].block.miner_tx.vout.size(); + tpool.submit(&waiter, [&, i, txidx](){ geniod(parsed_blocks[i].block.miner_tx, n_vouts, txidx); }, true); + } + ++txidx; + for (size_t j = 0; j < parsed_blocks[i].txes.size(); ++j) + { + THROW_WALLET_EXCEPTION_IF(txidx >= tx_cache_data.size(), error::wallet_internal_error, "txidx out of range"); + tpool.submit(&waiter, [&, i, j, txidx](){ geniod(parsed_blocks[i].txes[j], parsed_blocks[i].txes[j].vout.size(), txidx); }, true); + ++txidx; + } + } + THROW_WALLET_EXCEPTION_IF(txidx != tx_cache_data.size(), error::wallet_internal_error, "txidx did not reach expected value"); + waiter.wait(&tpool); + hwdev.set_mode(hw::device::NONE); + + size_t tx_cache_data_offset = 0; + for (size_t i = 0; i < blocks.size(); ++i) { - cryptonote::block bl; - bool r = cryptonote::parse_and_validate_block_from_blob(bl_entry.block, bl); - THROW_WALLET_EXCEPTION_IF(!r, error::block_parse_error, bl_entry.block); + const crypto::hash &bl_id = parsed_blocks[i].hash; + const cryptonote::block &bl = parsed_blocks[i].block; - crypto::hash bl_id = get_block_hash(bl); if(current_index >= m_blockchain.size()) { - process_new_blockchain_entry(bl, bl_entry, bl_id, current_index, o_indices[tx_o_indices_idx]); + process_new_blockchain_entry(bl, blocks[i], parsed_blocks[i], bl_id, current_index, tx_cache_data, tx_cache_data_offset); ++blocks_added; } else if(bl_id != m_blockchain[current_index]) @@ -1806,16 +1912,14 @@ void wallet2::process_blocks(uint64_t start_height, const std::list<cryptonote:: string_tools::pod_to_hex(m_blockchain[current_index])); detach_blockchain(current_index); - process_new_blockchain_entry(bl, bl_entry, bl_id, current_index, o_indices[tx_o_indices_idx]); + process_new_blockchain_entry(bl, blocks[i], parsed_blocks[i], bl_id, current_index, tx_cache_data, tx_cache_data_offset); } else { LOG_PRINT_L2("Block is already in blockchain: " << string_tools::pod_to_hex(bl_id)); } - ++current_index; - ++tx_o_indices_idx; - } + tx_cache_data_offset += 1 + parsed_blocks[i].txes.size(); } } //---------------------------------------------------------------------------------------------------- @@ -1831,7 +1935,7 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched) refresh(start_height, blocks_fetched, received_money); } //---------------------------------------------------------------------------------------------------- -void wallet2::pull_next_blocks(uint64_t start_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, const std::list<cryptonote::block_complete_entry> &prev_blocks, std::list<cryptonote::block_complete_entry> &blocks, std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices, bool &error) +void wallet2::pull_and_parse_next_blocks(uint64_t start_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, const std::vector<cryptonote::block_complete_entry> &prev_blocks, const std::vector<parsed_block> &prev_parsed_blocks, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<parsed_block> &parsed_blocks, bool &error) { error = false; @@ -1840,18 +1944,53 @@ void wallet2::pull_next_blocks(uint64_t start_height, uint64_t &blocks_start_hei drop_from_short_history(short_chain_history, 3); // prepend the last 3 blocks, should be enough to guard against a block or two's reorg - cryptonote::block bl; - std::list<cryptonote::block_complete_entry>::const_reverse_iterator i = prev_blocks.rbegin(); + std::vector<parsed_block>::const_reverse_iterator i = prev_parsed_blocks.rbegin(); for (size_t n = 0; n < std::min((size_t)3, prev_blocks.size()); ++n) { - bool ok = cryptonote::parse_and_validate_block_from_blob(i->block, bl); - THROW_WALLET_EXCEPTION_IF(!ok, error::block_parse_error, i->block); - short_chain_history.push_front(cryptonote::get_block_hash(bl)); + short_chain_history.push_front(i->hash); ++i; } // pull the new blocks + std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> o_indices; pull_blocks(start_height, blocks_start_height, short_chain_history, blocks, o_indices); + THROW_WALLET_EXCEPTION_IF(blocks.size() != o_indices.size(), error::wallet_internal_error, "Mismatched sizes of blocks and o_indices"); + + tools::threadpool& tpool = tools::threadpool::getInstance(); + tools::threadpool::waiter waiter; + parsed_blocks.resize(blocks.size()); + for (size_t i = 0; i < blocks.size(); ++i) + { + tpool.submit(&waiter, boost::bind(&wallet2::parse_block_round, this, std::cref(blocks[i].block), + std::ref(parsed_blocks[i].block), std::ref(parsed_blocks[i].hash), std::ref(parsed_blocks[i].error)), true); + } + waiter.wait(&tpool); + for (size_t i = 0; i < blocks.size(); ++i) + { + if (parsed_blocks[i].error) + { + error = true; + break; + } + parsed_blocks[i].o_indices = std::move(o_indices[i]); + } + + boost::mutex error_lock; + for (size_t i = 0; i < blocks.size(); ++i) + { + parsed_blocks[i].txes.resize(blocks[i].txs.size()); + for (size_t j = 0; j < blocks[i].txs.size(); ++j) + { + tpool.submit(&waiter, [&, i, j](){ + if (!parse_and_validate_tx_base_from_blob(blocks[i].txs[j], parsed_blocks[i].txes[j])) + { + boost::unique_lock<boost::mutex> lock(error_lock); + error = true; + } + }, true); + } + } + waiter.wait(&tpool); } catch(...) { @@ -2064,7 +2203,7 @@ void wallet2::update_pool_state(bool refreshed) [tx_hash](const std::pair<crypto::hash, bool> &e) { return e.first == tx_hash; }); if (i != txids.end()) { - process_new_transaction(tx_hash, tx, std::vector<uint64_t>(), 0, time(NULL), false, true, tx_entry.double_spend_seen); + process_new_transaction(tx_hash, tx, std::vector<uint64_t>(), 0, time(NULL), false, true, tx_entry.double_spend_seen, {}); m_scanned_pool_txs[0].insert(tx_hash); if (m_scanned_pool_txs[0].size() > 5000) { @@ -2108,7 +2247,7 @@ void wallet2::update_pool_state(bool refreshed) //---------------------------------------------------------------------------------------------------- void wallet2::fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history) { - std::list<crypto::hash> hashes; + std::vector<crypto::hash> hashes; const uint64_t checkpoint_height = m_checkpoints.get_max_height(); if (stop_height > checkpoint_height && m_blockchain.size()-1 < checkpoint_height) @@ -2118,7 +2257,6 @@ void wallet2::fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, while (missing_blocks-- > 0) m_blockchain.push_back(crypto::null_hash); // maybe a bit suboptimal, but deque won't do huge reallocs like vector m_blockchain.push_back(m_checkpoints.get_points().at(checkpoint_height)); - m_local_bc_height = m_blockchain.size(); short_chain_history.clear(); get_short_chain_history(short_chain_history); } @@ -2136,7 +2274,7 @@ void wallet2::fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, } if (hashes.size() + current_index < stop_height) { drop_from_short_history(short_chain_history, 3); - std::list<crypto::hash>::iterator right = hashes.end(); + std::vector<crypto::hash>::iterator right = hashes.end(); // prepend 3 more for (int i = 0; i<3; i++) { right--; @@ -2148,10 +2286,9 @@ void wallet2::fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, { if(current_index >= m_blockchain.size()) { - if (!(current_index % 1000)) + if (!(current_index % 1024)) LOG_PRINT_L2( "Skipped block by height: " << current_index); m_blockchain.push_back(bl_id); - ++m_local_bc_height; if (0 != m_callback) { // FIXME: this isn't right, but simplewallet just logs that we got a block. @@ -2199,6 +2336,8 @@ bool wallet2::delete_address_book_row(std::size_t row_id) { //---------------------------------------------------------------------------------------------------- void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& received_money) { + key_ref kref(*this); + if(m_light_wallet) { // MyMonero get_address_info needs to be called occasionally to trigger wallet sync. @@ -2212,7 +2351,6 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& re // Update lw heights m_light_wallet_scanned_block_height = res.scanned_block_height; m_light_wallet_blockchain_height = res.blockchain_height; - m_local_bc_height = res.blockchain_height; // If new height - call new_block callback if(m_light_wallet_blockchain_height != prev_height) { @@ -2241,8 +2379,8 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& re tools::threadpool& tpool = tools::threadpool::getInstance(); tools::threadpool::waiter waiter; uint64_t blocks_start_height; - std::list<cryptonote::block_complete_entry> blocks; - std::vector<COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> o_indices; + std::vector<cryptonote::block_complete_entry> blocks; + std::vector<parsed_block> parsed_blocks; bool refreshed = false; // pull the first set of blocks @@ -2263,31 +2401,34 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& re // If stop() is called during fast refresh we don't need to continue if(!m_run.load(std::memory_order_relaxed)) return; - pull_blocks(start_height, blocks_start_height, short_chain_history, blocks, o_indices); // always reset start_height to 0 to force short_chain_ history to be used on // subsequent pulls in this refresh. start_height = 0; + bool first = true; while(m_run.load(std::memory_order_relaxed)) { try { // pull the next set of blocks while we're processing the current one uint64_t next_blocks_start_height; - std::list<cryptonote::block_complete_entry> next_blocks; - std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> next_o_indices; + std::vector<cryptonote::block_complete_entry> next_blocks; + std::vector<parsed_block> next_parsed_blocks; bool error = false; - if (blocks.empty()) + if (!first && blocks.empty()) { refreshed = false; break; } - tpool.submit(&waiter, [&]{pull_next_blocks(start_height, next_blocks_start_height, short_chain_history, blocks, next_blocks, next_o_indices, error);}); + tpool.submit(&waiter, [&]{pull_and_parse_next_blocks(start_height, next_blocks_start_height, short_chain_history, blocks, parsed_blocks, next_blocks, next_parsed_blocks, error);}); - process_blocks(blocks_start_height, blocks, o_indices, added_blocks); - blocks_fetched += added_blocks; - waiter.wait(); - if(blocks_start_height == next_blocks_start_height) + if (!first) + { + process_parsed_blocks(blocks_start_height, blocks, parsed_blocks, added_blocks); + blocks_fetched += added_blocks; + } + waiter.wait(&tpool); + if(!first && blocks_start_height == next_blocks_start_height) { m_node_rpc_proxy.set_height(m_blockchain.size()); refreshed = true; @@ -2296,8 +2437,9 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& re // switch to the new blocks from the daemon blocks_start_height = next_blocks_start_height; - blocks = next_blocks; - o_indices = next_o_indices; + blocks = std::move(next_blocks); + parsed_blocks = std::move(next_parsed_blocks); + first = false; // handle error from async fetching thread if (error) @@ -2308,7 +2450,7 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& re catch (const std::exception&) { blocks_fetched += added_blocks; - waiter.wait(); + waiter.wait(&tpool); if(try_count < 3) { LOG_PRINT_L1("Another try pull_blocks (try_count=" << try_count << ")..."); @@ -2463,7 +2605,6 @@ void wallet2::detach_blockchain(uint64_t height) size_t blocks_detached = m_blockchain.size() - height; m_blockchain.crop(height); - m_local_bc_height -= blocks_detached; for (auto it = m_payments.begin(); it != m_payments.end(); ) { @@ -2505,7 +2646,6 @@ bool wallet2::clear() m_scanned_pool_txs[0].clear(); m_scanned_pool_txs[1].clear(); m_address_book.clear(); - m_local_bc_height = 1; m_subaddresses.clear(); m_subaddress_labels.clear(); return true; @@ -3272,6 +3412,12 @@ void wallet2::restore(const std::string& wallet_, const epee::wipeable_string& p cryptonote::block b; generate_genesis(b); m_blockchain.push_back(get_block_hash(b)); + if (m_subaddress_lookahead_major == SUBADDRESS_LOOKAHEAD_MAJOR && m_subaddress_lookahead_minor == SUBADDRESS_LOOKAHEAD_MINOR) + { + // the default lookahead setting (50:200) is clearly too much for hardware wallet + m_subaddress_lookahead_major = 5; + m_subaddress_lookahead_minor = 20; + } add_subaddress_account(tr("Primary account")); if (!wallet_.empty()) { store(); @@ -3623,6 +3769,14 @@ bool wallet2::has_multisig_partial_key_images() const return false; } +bool wallet2::has_unknown_key_images() const +{ + for (const auto &td: m_transfers) + if (!td.m_key_image_known) + return true; + return false; +} + /*! * \brief Rewrites to the wallet file for wallet upgrade (doesn't generate key, assumes it's already there) * \param wallet_name Name of wallet file (should exist) @@ -3785,7 +3939,7 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass { wallet2::cache_file_data cache_file_data; std::string buf; - bool r = epee::file_io_utils::load_file_to_string(m_wallet_file, buf); + bool r = epee::file_io_utils::load_file_to_string(m_wallet_file, buf, std::numeric_limits<size_t>::max()); THROW_WALLET_EXCEPTION_IF(!r, error::file_read_error, m_wallet_file); // try to read it as an encrypted cache @@ -3873,8 +4027,6 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass if (get_num_subaddress_accounts() == 0) add_subaddress_account(tr("Primary account")); - m_local_bc_height = m_blockchain.size(); - try { find_and_save_rings(false); @@ -4254,7 +4406,6 @@ void wallet2::rescan_blockchain(bool refresh) crypto::hash genesis_hash = get_block_hash(genesis); m_blockchain.push_back(genesis_hash); add_subaddress_account(tr("Primary account")); - m_local_bc_height = 1; if (refresh) this->refresh(); @@ -4270,7 +4421,7 @@ bool wallet2::is_transfer_unlocked(uint64_t unlock_time, uint64_t block_height) if(!is_tx_spendtime_unlocked(unlock_time, block_height)) return false; - if(block_height + CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE > m_local_bc_height) + if(block_height + CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE > get_blockchain_current_height()) return false; return true; @@ -4281,7 +4432,7 @@ bool wallet2::is_tx_spendtime_unlocked(uint64_t unlock_time, uint64_t block_heig if(unlock_time < CRYPTONOTE_MAX_BLOCK_NUMBER) { //interpret as block index - if(m_local_bc_height-1 + CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_BLOCKS >= unlock_time) + if(get_blockchain_current_height()-1 + CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_BLOCKS >= unlock_time) return true; else return false; @@ -4666,6 +4817,15 @@ void wallet2::commit_tx(std::vector<pending_tx>& ptx_vector) bool wallet2::save_tx(const std::vector<pending_tx>& ptx_vector, const std::string &filename) const { LOG_PRINT_L0("saving " << ptx_vector.size() << " transactions"); + std::string ciphertext = dump_tx_to_str(ptx_vector); + if (ciphertext.empty()) + return false; + return epee::file_io_utils::save_string_to_file(filename, ciphertext); +} +//---------------------------------------------------------------------------------------------------- +std::string wallet2::dump_tx_to_str(const std::vector<pending_tx> &ptx_vector) const +{ + LOG_PRINT_L0("saving " << ptx_vector.size() << " transactions"); unsigned_tx_set txs; for (auto &tx: ptx_vector) { @@ -4685,11 +4845,11 @@ bool wallet2::save_tx(const std::vector<pending_tx>& ptx_vector, const std::stri } catch (...) { - return false; + return std::string(); } LOG_PRINT_L2("Saving unsigned tx data: " << oss.str()); std::string ciphertext = encrypt_with_view_secret_key(oss.str()); - return epee::file_io_utils::save_string_to_file(filename, std::string(UNSIGNED_TX_PREFIX) + ciphertext); + return std::string(UNSIGNED_TX_PREFIX) + ciphertext; } //---------------------------------------------------------------------------------------------------- bool wallet2::load_unsigned_tx(const std::string &unsigned_filename, unsigned_tx_set &exported_txs) const @@ -4707,10 +4867,17 @@ bool wallet2::load_unsigned_tx(const std::string &unsigned_filename, unsigned_tx LOG_PRINT_L0("Failed to load from " << unsigned_filename); return false; } + + return parse_unsigned_tx_from_str(s, exported_txs); +} +//---------------------------------------------------------------------------------------------------- +bool wallet2::parse_unsigned_tx_from_str(const std::string &unsigned_tx_st, unsigned_tx_set &exported_txs) const +{ + std::string s = unsigned_tx_st; const size_t magiclen = strlen(UNSIGNED_TX_PREFIX) - 1; if (strncmp(s.c_str(), UNSIGNED_TX_PREFIX, magiclen)) { - LOG_PRINT_L0("Bad magic from " << unsigned_filename); + LOG_PRINT_L0("Bad magic from unsigned tx"); return false; } s = s.substr(magiclen); @@ -4726,7 +4893,7 @@ bool wallet2::load_unsigned_tx(const std::string &unsigned_filename, unsigned_tx } catch (...) { - LOG_PRINT_L0("Failed to parse data from " << unsigned_filename); + LOG_PRINT_L0("Failed to parse data from unsigned tx"); return false; } } @@ -4743,19 +4910,19 @@ bool wallet2::load_unsigned_tx(const std::string &unsigned_filename, unsigned_tx } catch (...) { - LOG_PRINT_L0("Failed to parse data from " << unsigned_filename); + LOG_PRINT_L0("Failed to parse data from unsigned tx"); return false; } } catch (const std::exception &e) { - LOG_PRINT_L0("Failed to decrypt " << unsigned_filename << ": " << e.what()); + LOG_PRINT_L0("Failed to decrypt unsigned tx: " << e.what()); return false; } } else { - LOG_PRINT_L0("Unsupported version in " << unsigned_filename); + LOG_PRINT_L0("Unsupported version in unsigned tx"); return false; } LOG_PRINT_L1("Loaded tx unsigned data from binary: " << exported_txs.txes.size() << " transactions"); @@ -4776,14 +4943,12 @@ bool wallet2::sign_tx(const std::string &unsigned_filename, const std::string &s } return sign_tx(exported_txs, signed_filename, txs, export_raw); } - //---------------------------------------------------------------------------------------------------- -bool wallet2::sign_tx(unsigned_tx_set &exported_txs, const std::string &signed_filename, std::vector<wallet2::pending_tx> &txs, bool export_raw) +bool wallet2::sign_tx(unsigned_tx_set &exported_txs, std::vector<wallet2::pending_tx> &txs, signed_tx_set &signed_txes) { import_outputs(exported_txs.transfers); // sign the transactions - signed_tx_set signed_txes; for (size_t n = 0; n < exported_txs.txes.size(); ++n) { tools::wallet2::tx_construction_data &sd = exported_txs.txes[n]; @@ -4791,11 +4956,10 @@ bool wallet2::sign_tx(unsigned_tx_set &exported_txs, const std::string &signed_f LOG_PRINT_L1(" " << (n+1) << ": " << sd.sources.size() << " inputs, ring size " << sd.sources[0].outputs.size()); signed_txes.ptx.push_back(pending_tx()); tools::wallet2::pending_tx &ptx = signed_txes.ptx.back(); - bool bulletproof = sd.use_rct && !ptx.tx.rct_signatures.p.bulletproofs.empty(); crypto::secret_key tx_key; std::vector<crypto::secret_key> additional_tx_keys; rct::multisig_out msout; - bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sd.sources, sd.splitted_dsts, sd.change_dts.addr, sd.extra, ptx.tx, sd.unlock_time, tx_key, additional_tx_keys, sd.use_rct, bulletproof, m_multisig ? &msout : NULL); + bool r = cryptonote::construct_tx_and_get_tx_key(m_account.get_keys(), m_subaddresses, sd.sources, sd.splitted_dsts, sd.change_dts.addr, sd.extra, ptx.tx, sd.unlock_time, tx_key, additional_tx_keys, sd.use_rct, sd.use_bulletproofs, m_multisig ? &msout : NULL); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sd.sources, sd.splitted_dsts, sd.unlock_time, m_nettype); // we don't test tx size, because we don't know the current limit, due to not having a blockchain, // and it's a bit pointless to fail there anyway, since it'd be a (good) guess only. We sign anyway, @@ -4845,19 +5009,20 @@ bool wallet2::sign_tx(unsigned_tx_set &exported_txs, const std::string &signed_f signed_txes.key_images[i] = m_transfers[i].m_key_image; } - // save as binary - std::ostringstream oss; - boost::archive::portable_binary_oarchive ar(oss); - try - { - ar << signed_txes; - } - catch(...) + return true; +} +//---------------------------------------------------------------------------------------------------- +bool wallet2::sign_tx(unsigned_tx_set &exported_txs, const std::string &signed_filename, std::vector<wallet2::pending_tx> &txs, bool export_raw) +{ + // sign the transactions + signed_tx_set signed_txes; + std::string ciphertext = sign_tx_dump_to_str(exported_txs, txs, signed_txes); + if (ciphertext.empty()) { + LOG_PRINT_L0("Failed to sign unsigned_tx_set"); return false; } - LOG_PRINT_L3("Saving signed tx data (with encryption): " << oss.str()); - std::string ciphertext = encrypt_with_view_secret_key(oss.str()); + if (!epee::file_io_utils::save_string_to_file(signed_filename, std::string(SIGNED_TX_PREFIX) + ciphertext)) { LOG_PRINT_L0("Failed to save file to " << signed_filename); @@ -4880,6 +5045,32 @@ bool wallet2::sign_tx(unsigned_tx_set &exported_txs, const std::string &signed_f return true; } //---------------------------------------------------------------------------------------------------- +std::string wallet2::sign_tx_dump_to_str(unsigned_tx_set &exported_txs, std::vector<wallet2::pending_tx> &ptx, signed_tx_set &signed_txes) +{ + // sign the transactions + bool r = sign_tx(exported_txs, ptx, signed_txes); + if (!r) + { + LOG_PRINT_L0("Failed to sign unsigned_tx_set"); + return std::string(); + } + + // save as binary + std::ostringstream oss; + boost::archive::portable_binary_oarchive ar(oss); + try + { + ar << signed_txes; + } + catch(...) + { + return std::string(); + } + LOG_PRINT_L3("Saving signed tx data (with encryption): " << oss.str()); + std::string ciphertext = encrypt_with_view_secret_key(oss.str()); + return std::string(SIGNED_TX_PREFIX) + ciphertext; +} +//---------------------------------------------------------------------------------------------------- bool wallet2::load_tx(const std::string &signed_filename, std::vector<tools::wallet2::pending_tx> &ptx, std::function<bool(const signed_tx_set&)> accept_func) { std::string s; @@ -4897,10 +5088,20 @@ bool wallet2::load_tx(const std::string &signed_filename, std::vector<tools::wal LOG_PRINT_L0("Failed to load from " << signed_filename); return false; } + + return parse_tx_from_str(s, ptx, accept_func); +} +//---------------------------------------------------------------------------------------------------- +bool wallet2::parse_tx_from_str(const std::string &signed_tx_st, std::vector<tools::wallet2::pending_tx> &ptx, std::function<bool(const signed_tx_set &)> accept_func) +{ + std::string s = signed_tx_st; + boost::system::error_code errcode; + signed_tx_set signed_txs; + const size_t magiclen = strlen(SIGNED_TX_PREFIX) - 1; if (strncmp(s.c_str(), SIGNED_TX_PREFIX, magiclen)) { - LOG_PRINT_L0("Bad magic from " << signed_filename); + LOG_PRINT_L0("Bad magic from signed transaction"); return false; } s = s.substr(magiclen); @@ -4916,7 +5117,7 @@ bool wallet2::load_tx(const std::string &signed_filename, std::vector<tools::wal } catch (...) { - LOG_PRINT_L0("Failed to parse data from " << signed_filename); + LOG_PRINT_L0("Failed to parse data from signed transaction"); return false; } } @@ -4933,23 +5134,23 @@ bool wallet2::load_tx(const std::string &signed_filename, std::vector<tools::wal } catch (...) { - LOG_PRINT_L0("Failed to parse decrypted data from " << signed_filename); + LOG_PRINT_L0("Failed to parse decrypted data from signed transaction"); return false; } } catch (const std::exception &e) { - LOG_PRINT_L0("Failed to decrypt " << signed_filename << ": " << e.what()); + LOG_PRINT_L0("Failed to decrypt signed transaction: " << e.what()); return false; } } else { - LOG_PRINT_L0("Unsupported version in " << signed_filename); + LOG_PRINT_L0("Unsupported version in signed transaction"); return false; } LOG_PRINT_L0("Loaded signed tx data from binary: " << signed_txs.ptx.size() << " transactions"); - for (auto &ptx: signed_txs.ptx) LOG_PRINT_L0(cryptonote::obj_to_json_str(ptx.tx)); + for (auto &c_ptx: signed_txs.ptx) LOG_PRINT_L0(cryptonote::obj_to_json_str(c_ptx.tx)); if (accept_func && !accept_func(signed_txs)) { @@ -5171,8 +5372,7 @@ bool wallet2::sign_multisig_tx(multisig_tx_set &exported_txs, std::vector<crypto cryptonote::transaction tx; rct::multisig_out msout = ptx.multisig_sigs.front().msout; auto sources = sd.sources; - const bool bulletproof = sd.use_rct && (ptx.tx.rct_signatures.type == rct::RCTTypeFullBulletproof || ptx.tx.rct_signatures.type == rct::RCTTypeSimpleBulletproof); - bool r = cryptonote::construct_tx_with_tx_key(m_account.get_keys(), m_subaddresses, sources, sd.splitted_dsts, ptx.change_dts.addr, sd.extra, tx, sd.unlock_time, ptx.tx_key, ptx.additional_tx_keys, sd.use_rct, bulletproof, &msout, false); + bool r = cryptonote::construct_tx_with_tx_key(m_account.get_keys(), m_subaddresses, sources, sd.splitted_dsts, ptx.change_dts.addr, sd.extra, tx, sd.unlock_time, ptx.tx_key, ptx.additional_tx_keys, sd.use_rct, sd.use_bulletproofs, &msout, false); THROW_WALLET_EXCEPTION_IF(!r, error::tx_not_constructed, sd.sources, sd.splitted_dsts, sd.unlock_time, m_nettype); THROW_WALLET_EXCEPTION_IF(get_transaction_prefix_hash (tx) != get_transaction_prefix_hash(ptx.tx), @@ -5565,6 +5765,24 @@ bool wallet2::set_ring_database(const std::string &filename) return true; } +crypto::chacha_key wallet2::get_ringdb_key() +{ + if (!m_ringdb_key) + { + MINFO("caching ringdb key"); + crypto::chacha_key key; + generate_chacha_key_from_secret_keys(key); + m_ringdb_key = key; + } + return *m_ringdb_key; +} + +void wallet2::clear_ringdb_key() +{ + MINFO("clearing ringdb key"); + m_ringdb_key = boost::none; +} + bool wallet2::add_rings(const crypto::chacha_key &key, const cryptonote::transaction_prefix &tx) { if (!m_ringdb) @@ -5575,9 +5793,8 @@ bool wallet2::add_rings(const crypto::chacha_key &key, const cryptonote::transac bool wallet2::add_rings(const cryptonote::transaction_prefix &tx) { - crypto::chacha_key key; - generate_chacha_key_from_secret_keys(key); - try { return add_rings(key, tx); } + key_ref kref(*this); + try { return add_rings(get_ringdb_key(), tx); } catch (const std::exception &e) { return false; } } @@ -5585,9 +5802,8 @@ bool wallet2::remove_rings(const cryptonote::transaction_prefix &tx) { if (!m_ringdb) return false; - crypto::chacha_key key; - generate_chacha_key_from_secret_keys(key); - try { return m_ringdb->remove_rings(key, tx); } + key_ref kref(*this); + try { return m_ringdb->remove_rings(get_ringdb_key(), tx); } catch (const std::exception &e) { return false; } } @@ -5624,10 +5840,8 @@ bool wallet2::get_rings(const crypto::hash &txid, std::vector<std::pair<crypto:: bool wallet2::get_ring(const crypto::key_image &key_image, std::vector<uint64_t> &outs) { - crypto::chacha_key key; - generate_chacha_key_from_secret_keys(key); - - try { return get_ring(key, key_image, outs); } + key_ref kref(*this); + try { return get_ring(get_ringdb_key(), key_image, outs); } catch (const std::exception &e) { return false; } } @@ -5636,10 +5850,8 @@ bool wallet2::set_ring(const crypto::key_image &key_image, const std::vector<uin if (!m_ringdb) return false; - crypto::chacha_key key; - generate_chacha_key_from_secret_keys(key); - - try { return m_ringdb->set_ring(key, key_image, outs, relative); } + key_ref kref(*this); + try { return m_ringdb->set_ring(get_ringdb_key(), key_image, outs, relative); } catch (const std::exception &e) { return false; } } @@ -5650,6 +5862,7 @@ bool wallet2::find_and_save_rings(bool force) if (!m_ringdb) return false; + key_ref kref(*this); COMMAND_RPC_GET_TRANSACTIONS::request req = AUTO_VAL_INIT(req); COMMAND_RPC_GET_TRANSACTIONS::response res = AUTO_VAL_INIT(res); @@ -5667,9 +5880,6 @@ bool wallet2::find_and_save_rings(bool force) MDEBUG("Found " << std::to_string(txs_hashes.size()) << " transactions"); - crypto::chacha_key key; - generate_chacha_key_from_secret_keys(key); - // get those transactions from the daemon static const size_t SLICE_SIZE = 200; for (size_t slice = 0; slice < txs_hashes.size(); slice += SLICE_SIZE) @@ -5706,7 +5916,7 @@ bool wallet2::find_and_save_rings(bool force) crypto::hash tx_hash, tx_prefix_hash; THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(bd, tx, tx_hash, tx_prefix_hash), error::wallet_internal_error, "failed to parse tx from blob"); THROW_WALLET_EXCEPTION_IF(epee::string_tools::pod_to_hex(tx_hash) != tx_info.tx_hash, error::wallet_internal_error, "txid mismatch"); - THROW_WALLET_EXCEPTION_IF(!add_rings(key, tx), error::wallet_internal_error, "Failed to save ring"); + THROW_WALLET_EXCEPTION_IF(!add_rings(get_ringdb_key(), tx), error::wallet_internal_error, "Failed to save ring"); } } @@ -5885,9 +6095,6 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> return; } - crypto::chacha_key key; - generate_chacha_key_from_secret_keys(key); - if (fake_outputs_count > 0) { uint64_t segregation_fork_height = get_segregation_fork_height(); @@ -6065,7 +6272,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> if (td.m_key_image_known && !td.m_key_image_partial) { std::vector<uint64_t> ring; - if (get_ring(key, td.m_key_image, ring)) + if (get_ring(get_ringdb_key(), td.m_key_image, ring)) { MINFO("This output has a known ring, reusing (size " << ring.size() << ")"); THROW_WALLET_EXCEPTION_IF(ring.size() > fake_outputs_count + 1, error::wallet_internal_error, @@ -6257,7 +6464,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> if (td.m_key_image_known && !td.m_key_image_partial) { std::vector<uint64_t> ring; - if (get_ring(key, td.m_key_image, ring)) + if (get_ring(get_ringdb_key(), td.m_key_image, ring)) { for (uint64_t out: ring) { @@ -6475,6 +6682,7 @@ void wallet2::transfer_selected(const std::vector<cryptonote::tx_destination_ent ptx.construction_data.extra = tx.extra; ptx.construction_data.unlock_time = unlock_time; ptx.construction_data.use_rct = false; + ptx.construction_data.use_bulletproofs = false; ptx.construction_data.dests = dsts; // record which subaddress indices are being used as inputs ptx.construction_data.subaddr_account = subaddr_account; @@ -6730,6 +6938,7 @@ void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry ptx.construction_data.extra = tx.extra; ptx.construction_data.unlock_time = unlock_time; ptx.construction_data.use_rct = true; + ptx.construction_data.use_bulletproofs = !tx.rct_signatures.p.bulletproofs.empty(); ptx.construction_data.dests = dsts; // record which subaddress indices are being used as inputs ptx.construction_data.subaddr_account = subaddr_account; @@ -8431,8 +8640,9 @@ bool wallet2::check_spend_proof(const crypto::hash &txid, const std::string &mes } std::vector<std::vector<crypto::signature>> signatures = { std::vector<crypto::signature>(1) }; const size_t sig_len = tools::base58::encode(std::string((const char *)&signatures[0][0], sizeof(crypto::signature))).size(); - THROW_WALLET_EXCEPTION_IF(sig_str.size() != header_len + num_sigs * sig_len, - error::wallet_internal_error, "incorrect signature size"); + if( sig_str.size() != header_len + num_sigs * sig_len ) { + return false; + } // decode base58 signatures.clear(); @@ -9162,9 +9372,9 @@ uint64_t wallet2::get_daemon_blockchain_target_height(string &err) uint64_t wallet2::get_approximate_blockchain_height() const { // time of v2 fork - const time_t fork_time = m_nettype == TESTNET ? 1448285909 : m_nettype == STAGENET ? (time_t)-1/*TODO*/ : 1458748658; + const time_t fork_time = m_nettype == TESTNET ? 1448285909 : m_nettype == STAGENET ? 1520937818 : 1458748658; // v2 fork block - const uint64_t fork_block = m_nettype == TESTNET ? 624634 : m_nettype == STAGENET ? (uint64_t)-1/*TODO*/ : 1009827; + const uint64_t fork_block = m_nettype == TESTNET ? 624634 : m_nettype == STAGENET ? 32000 : 1009827; // avg seconds per block const int seconds_per_block = DIFFICULTY_TARGET_V2; // Calculated blockchain height @@ -9797,7 +10007,6 @@ void wallet2::import_blockchain(const std::tuple<size_t, crypto::hash, std::vect generate_genesis(genesis); crypto::hash genesis_hash = get_block_hash(genesis); check_genesis(genesis_hash); - m_local_bc_height = m_blockchain.size(); } //---------------------------------------------------------------------------------------------------- std::vector<tools::wallet2::transfer_details> wallet2::export_outputs() const @@ -9815,6 +10024,23 @@ std::vector<tools::wallet2::transfer_details> wallet2::export_outputs() const return outs; } //---------------------------------------------------------------------------------------------------- +std::string wallet2::export_outputs_to_str() const +{ + std::vector<tools::wallet2::transfer_details> outs = export_outputs(); + + std::stringstream oss; + boost::archive::portable_binary_oarchive ar(oss); + ar << outs; + + std::string magic(OUTPUT_EXPORT_FILE_MAGIC, strlen(OUTPUT_EXPORT_FILE_MAGIC)); + const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address; + std::string header; + header += std::string((const char *)&keys.m_spend_public_key, sizeof(crypto::public_key)); + header += std::string((const char *)&keys.m_view_public_key, sizeof(crypto::public_key)); + std::string ciphertext = encrypt_with_view_secret_key(header + oss.str()); + return magic + ciphertext; +} +//---------------------------------------------------------------------------------------------------- size_t wallet2::import_outputs(const std::vector<tools::wallet2::transfer_details> &outputs) { m_transfers.clear(); @@ -9847,6 +10073,67 @@ size_t wallet2::import_outputs(const std::vector<tools::wallet2::transfer_detail return m_transfers.size(); } //---------------------------------------------------------------------------------------------------- +size_t wallet2::import_outputs_from_str(const std::string &outputs_st) +{ + std::string data = outputs_st; + const size_t magiclen = strlen(OUTPUT_EXPORT_FILE_MAGIC); + if (data.size() < magiclen || memcmp(data.data(), OUTPUT_EXPORT_FILE_MAGIC, magiclen)) + { + THROW_WALLET_EXCEPTION(error::wallet_internal_error, std::string("Bad magic from outputs")); + } + + try + { + data = decrypt_with_view_secret_key(std::string(data, magiclen)); + } + catch (const std::exception &e) + { + THROW_WALLET_EXCEPTION(error::wallet_internal_error, std::string("Failed to decrypt outputs: ") + e.what()); + } + + const size_t headerlen = 2 * sizeof(crypto::public_key); + if (data.size() < headerlen) + { + THROW_WALLET_EXCEPTION(error::wallet_internal_error, std::string("Bad data size for outputs")); + } + const crypto::public_key &public_spend_key = *(const crypto::public_key*)&data[0]; + const crypto::public_key &public_view_key = *(const crypto::public_key*)&data[sizeof(crypto::public_key)]; + const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address; + if (public_spend_key != keys.m_spend_public_key || public_view_key != keys.m_view_public_key) + { + THROW_WALLET_EXCEPTION(error::wallet_internal_error, std::string("Outputs from are for a different account")); + } + + size_t imported_outputs = 0; + try + { + std::string body(data, headerlen); + std::stringstream iss; + iss << body; + std::vector<tools::wallet2::transfer_details> outputs; + try + { + boost::archive::portable_binary_iarchive ar(iss); + ar >> outputs; + } + catch (...) + { + iss.str(""); + iss << body; + boost::archive::binary_iarchive ar(iss); + ar >> outputs; + } + + imported_outputs = import_outputs(outputs); + } + catch (const std::exception &e) + { + THROW_WALLET_EXCEPTION(error::wallet_internal_error, std::string("Failed to import outputs") + e.what()); + } + + return imported_outputs; +} +//---------------------------------------------------------------------------------------------------- crypto::public_key wallet2::get_multisig_signer_public_key(const crypto::secret_key &spend_skey) const { crypto::public_key pkey; diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 40f6e08d9..2da6dd21a 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -320,6 +320,7 @@ namespace tools std::vector<uint8_t> extra; uint64_t unlock_time; bool use_rct; + bool use_bulletproofs; std::vector<cryptonote::tx_destination_entry> dests; // original setup, does not include change uint32_t subaddr_account; // subaddress account of your wallet to be used in this transfer std::set<uint32_t> subaddr_indices; // set of address indices used as inputs in this transfer @@ -332,6 +333,7 @@ namespace tools FIELD(extra) FIELD(unlock_time) FIELD(use_rct) + FIELD(use_bulletproofs) FIELD(dests) FIELD(subaddr_account) FIELD(subaddr_indices) @@ -452,6 +454,39 @@ namespace tools typedef std::tuple<uint64_t, crypto::public_key, rct::key> get_outs_entry; + struct parsed_block + { + crypto::hash hash; + cryptonote::block block; + std::vector<cryptonote::transaction> txes; + cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices o_indices; + bool error; + }; + + struct is_out_data + { + crypto::public_key pkey; + crypto::key_derivation derivation; + std::vector<boost::optional<cryptonote::subaddress_receive_info>> received; + }; + + struct tx_cache_data + { + std::vector<cryptonote::tx_extra_field> tx_extra_fields; + std::vector<is_out_data> primary; + std::vector<is_out_data> additional; + }; + + struct key_ref + { + key_ref(tools::wallet2 &w): wallet(w) { ++refs; } + ~key_ref() { if (!--refs) wallet.clear_ringdb_key(); } + + private: + tools::wallet2 &wallet; + static std::atomic<unsigned int> refs; + }; + /*! * \brief Generates a wallet or restores one. * \param wallet_ Name of wallet file @@ -655,6 +690,7 @@ namespace tools bool watch_only() const { return m_watch_only; } bool multisig(bool *ready = NULL, uint32_t *threshold = NULL, uint32_t *total = NULL) const; bool has_multisig_partial_key_images() const; + bool has_unknown_key_images() const; bool get_multisig_seed(std::string& seed, const epee::wipeable_string &passphrase = std::string(), bool raw = true) const; bool key_on_device() const { return m_key_on_device; } @@ -684,6 +720,7 @@ namespace tools void commit_tx(pending_tx& ptx_vector); void commit_tx(std::vector<pending_tx>& ptx_vector); bool save_tx(const std::vector<pending_tx>& ptx_vector, const std::string &filename) const; + std::string dump_tx_to_str(const std::vector<pending_tx> &ptx_vector) const; std::string save_multisig_tx(multisig_tx_set txs); bool save_multisig_tx(const multisig_tx_set &txs, const std::string &filename); std::string save_multisig_tx(const std::vector<pending_tx>& ptx_vector); @@ -693,9 +730,13 @@ namespace tools bool sign_tx(const std::string &unsigned_filename, const std::string &signed_filename, std::vector<wallet2::pending_tx> &ptx, std::function<bool(const unsigned_tx_set&)> accept_func = NULL, bool export_raw = false); // sign unsigned tx. Takes unsigned_tx_set as argument. Used by GUI bool sign_tx(unsigned_tx_set &exported_txs, const std::string &signed_filename, std::vector<wallet2::pending_tx> &ptx, bool export_raw = false); + bool sign_tx(unsigned_tx_set &exported_txs, std::vector<wallet2::pending_tx> &ptx, signed_tx_set &signed_txs); + std::string sign_tx_dump_to_str(unsigned_tx_set &exported_txs, std::vector<wallet2::pending_tx> &ptx, signed_tx_set &signed_txes); // load unsigned_tx_set from file. bool load_unsigned_tx(const std::string &unsigned_filename, unsigned_tx_set &exported_txs) const; + bool parse_unsigned_tx_from_str(const std::string &unsigned_tx_st, unsigned_tx_set &exported_txs) const; bool load_tx(const std::string &signed_filename, std::vector<tools::wallet2::pending_tx> &ptx, std::function<bool(const signed_tx_set&)> accept_func = NULL); + bool parse_tx_from_str(const std::string &signed_tx_st, std::vector<tools::wallet2::pending_tx> &ptx, std::function<bool(const signed_tx_set &)> accept_func); std::vector<pending_tx> create_transactions(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, bool trusted_daemon); std::vector<wallet2::pending_tx> create_transactions_2(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices, bool trusted_daemon); // pass subaddr_indices by value on purpose std::vector<wallet2::pending_tx> create_transactions_all(uint64_t below, const cryptonote::account_public_address &address, bool is_subaddress, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices, bool trusted_daemon); @@ -716,7 +757,7 @@ namespace tools void get_unconfirmed_payments_out(std::list<std::pair<crypto::hash,wallet2::unconfirmed_transfer_details>>& unconfirmed_payments, const boost::optional<uint32_t>& subaddr_account = boost::none, const std::set<uint32_t>& subaddr_indices = {}) const; void get_unconfirmed_payments(std::list<std::pair<crypto::hash,wallet2::pool_payment_details>>& unconfirmed_payments, const boost::optional<uint32_t>& subaddr_account = boost::none, const std::set<uint32_t>& subaddr_indices = {}) const; - uint64_t get_blockchain_current_height() const { return m_local_bc_height; } + uint64_t get_blockchain_current_height() const { return m_light_wallet_blockchain_height ? m_light_wallet_blockchain_height : m_blockchain.size(); } void rescan_spent(); void rescan_blockchain(bool refresh = true); bool is_transfer_unlocked(const transfer_details& td) const; @@ -987,7 +1028,9 @@ namespace tools // Import/Export wallet data std::vector<tools::wallet2::transfer_details> export_outputs() const; + std::string export_outputs_to_str() const; size_t import_outputs(const std::vector<tools::wallet2::transfer_details> &outputs); + size_t import_outputs_from_str(const std::string &outputs_st); payment_container export_payments() const; void import_payments(const payment_container &payments); void import_payments_out(const std::list<std::pair<crypto::hash,wallet2::confirmed_transfer_details>> &confirmed_payments); @@ -1109,17 +1152,17 @@ namespace tools * \param password Password of wallet file */ bool load_keys(const std::string& keys_file_name, const epee::wipeable_string& password); - void process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector<uint64_t> &o_indices, uint64_t height, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen); - void process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const crypto::hash& bl_id, uint64_t height, const cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices &o_indices); + void process_new_transaction(const crypto::hash &txid, const cryptonote::transaction& tx, const std::vector<uint64_t> &o_indices, uint64_t height, uint64_t ts, bool miner_tx, bool pool, bool double_spend_seen, const tx_cache_data &tx_cache_data); + void process_new_blockchain_entry(const cryptonote::block& b, const cryptonote::block_complete_entry& bche, const parsed_block &parsed_block, const crypto::hash& bl_id, uint64_t height, const std::vector<tx_cache_data> &tx_cache_data, size_t tx_cache_data_offset); void detach_blockchain(uint64_t height); void get_short_chain_history(std::list<crypto::hash>& ids) const; bool is_tx_spendtime_unlocked(uint64_t unlock_time, uint64_t block_height) const; bool clear(); - void pull_blocks(uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::list<cryptonote::block_complete_entry> &blocks, std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices); - void pull_hashes(uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::list<crypto::hash> &hashes); + void pull_blocks(uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices); + void pull_hashes(uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<crypto::hash> &hashes); void fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history); - void pull_next_blocks(uint64_t start_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, const std::list<cryptonote::block_complete_entry> &prev_blocks, std::list<cryptonote::block_complete_entry> &blocks, std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices, bool &error); - void process_blocks(uint64_t start_height, const std::list<cryptonote::block_complete_entry> &blocks, const std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices, uint64_t& blocks_added); + void pull_and_parse_next_blocks(uint64_t start_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, const std::vector<cryptonote::block_complete_entry> &prev_blocks, const std::vector<parsed_block> &prev_parsed_blocks, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<parsed_block> &parsed_blocks, bool &error); + void process_parsed_blocks(uint64_t start_height, const std::vector<cryptonote::block_complete_entry> &blocks, const std::vector<parsed_block> &parsed_blocks, uint64_t& blocks_added); uint64_t select_transfers(uint64_t needed_money, std::vector<size_t> unused_transfers_indices, std::vector<size_t>& selected_transfers, bool trusted_daemon) const; bool prepare_file_names(const std::string& file_path); void process_unconfirmed(const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t height); @@ -1130,6 +1173,7 @@ namespace tools bool generate_chacha_key_from_secret_keys(crypto::chacha_key &key) const; crypto::hash get_payment_id(const pending_tx &ptx) const; void check_acc_out_precomp(const cryptonote::tx_out &o, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, size_t i, tx_scan_info_t &tx_scan_info) const; + void check_acc_out_precomp(const cryptonote::tx_out &o, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, size_t i, const is_out_data *is_out_data, tx_scan_info_t &tx_scan_info) const; void parse_block_round(const cryptonote::blobdata &blob, cryptonote::block &bl, crypto::hash &bl_id, bool &error) const; uint64_t get_upper_transaction_size_limit() const; std::vector<uint64_t> get_unspent_amounts_vector() const; @@ -1154,11 +1198,16 @@ namespace tools bool add_rings(const cryptonote::transaction_prefix &tx); bool remove_rings(const cryptonote::transaction_prefix &tx); bool get_ring(const crypto::chacha_key &key, const crypto::key_image &key_image, std::vector<uint64_t> &outs); + crypto::chacha_key get_ringdb_key(); + void cache_ringdb_key(); + void clear_ringdb_key(); bool get_output_distribution(uint64_t &start_height, std::vector<uint64_t> &distribution); uint64_t get_segregation_fork_height() const; + void cache_tx_data(const cryptonote::transaction& tx, const crypto::hash &txid, tx_cache_data &tx_cache_data) const; + cryptonote::account_base m_account; boost::optional<epee::net_utils::http::login> m_daemon_login; std::string m_daemon_address; @@ -1166,7 +1215,6 @@ namespace tools std::string m_keys_file; epee::net_utils::http::http_simple_client m_http_client; hashchain m_blockchain; - std::atomic<uint64_t> m_local_bc_height; //temporary workaround std::unordered_map<crypto::hash, unconfirmed_transfer_details> m_unconfirmed_txs; std::unordered_map<crypto::hash, confirmed_transfer_details> m_confirmed_txs; std::unordered_multimap<crypto::hash, pool_payment_details> m_unconfirmed_payments; @@ -1249,6 +1297,7 @@ namespace tools std::string m_ring_database; bool m_ring_history_saved; std::unique_ptr<ringdb> m_ringdb; + boost::optional<crypto::chacha_key> m_ringdb_key; }; } BOOST_CLASS_VERSION(tools::wallet2, 24) @@ -1264,7 +1313,7 @@ BOOST_CLASS_VERSION(tools::wallet2::address_book_row, 17) BOOST_CLASS_VERSION(tools::wallet2::reserve_proof_entry, 0) BOOST_CLASS_VERSION(tools::wallet2::unsigned_tx_set, 0) BOOST_CLASS_VERSION(tools::wallet2::signed_tx_set, 0) -BOOST_CLASS_VERSION(tools::wallet2::tx_construction_data, 2) +BOOST_CLASS_VERSION(tools::wallet2::tx_construction_data, 3) BOOST_CLASS_VERSION(tools::wallet2::pending_tx, 3) BOOST_CLASS_VERSION(tools::wallet2::multisig_sig, 0) @@ -1611,6 +1660,9 @@ namespace boost if (ver < 2) return; a & x.selected_transfers; + if (ver < 3) + return; + a & x.use_bulletproofs; } template <class Archive> @@ -1896,6 +1948,7 @@ namespace tools ptx.construction_data.extra = tx.extra; ptx.construction_data.unlock_time = unlock_time; ptx.construction_data.use_rct = false; + ptx.construction_data.use_bulletproofs = false; ptx.construction_data.dests = dsts; // record which subaddress indices are being used as inputs ptx.construction_data.subaddr_account = subaddr_account; diff --git a/src/wallet/wallet_args.cpp b/src/wallet/wallet_args.cpp index 6311e7700..a629eb506 100644 --- a/src/wallet/wallet_args.cpp +++ b/src/wallet/wallet_args.cpp @@ -101,6 +101,7 @@ namespace wallet_args const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""}; const command_line::arg_descriptor<std::size_t> arg_max_log_file_size = {"max-log-file-size", "Specify maximum log file size [B]", MAX_LOG_FILE_SIZE}; + const command_line::arg_descriptor<std::size_t> arg_max_log_files = {"max-log-files", "Specify maximum number of rotated log files to be saved (no limit by setting to 0)", MAX_LOG_FILES}; const command_line::arg_descriptor<uint32_t> arg_max_concurrency = {"max-concurrency", wallet_args::tr("Max number of threads to use for a parallel job"), DEFAULT_MAX_CONCURRENCY}; const command_line::arg_descriptor<std::string> arg_log_file = {"log-file", wallet_args::tr("Specify log file"), ""}; const command_line::arg_descriptor<std::string> arg_config_file = {"config-file", wallet_args::tr("Config file"), "", true}; @@ -119,6 +120,7 @@ namespace wallet_args command_line::add_arg(desc_params, arg_log_file); command_line::add_arg(desc_params, arg_log_level); command_line::add_arg(desc_params, arg_max_log_file_size); + command_line::add_arg(desc_params, arg_max_log_files); command_line::add_arg(desc_params, arg_max_concurrency); command_line::add_arg(desc_params, arg_config_file); @@ -180,11 +182,15 @@ namespace wallet_args log_path = command_line::get_arg(vm, arg_log_file); else log_path = mlog_get_default_log_path(default_log_name); - mlog_configure(log_path, log_to_console, command_line::get_arg(vm, arg_max_log_file_size)); + mlog_configure(log_path, log_to_console, command_line::get_arg(vm, arg_max_log_file_size), command_line::get_arg(vm, arg_max_log_files)); if (!command_line::is_arg_defaulted(vm, arg_log_level)) { mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); } + else if (!log_to_console) + { + mlog_set_categories(""); + } if (notice) Print(print) << notice << ENDL; diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index dc1beef7b..7f7d33642 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -712,7 +712,7 @@ namespace tools //------------------------------------------------------------------------------------------------------------------------------ template<typename Ts, typename Tu> bool wallet_rpc_server::fill_response(std::vector<tools::wallet2::pending_tx> &ptx_vector, - bool get_tx_key, Ts& tx_key, Tu &amount, Tu &fee, std::string &multisig_txset, bool do_not_relay, + bool get_tx_key, Ts& tx_key, Tu &amount, Tu &fee, std::string &multisig_txset, std::string &unsigned_txset, bool do_not_relay, Ts &tx_hash, bool get_tx_hex, Ts &tx_blob, bool get_tx_metadata, Ts &tx_metadata, epee::json_rpc::error &er) { for (const auto & ptx : ptx_vector) @@ -741,7 +741,16 @@ namespace tools } else { - if (!do_not_relay) + if (m_wallet->watch_only()){ + unsigned_txset = epee::string_tools::buff_to_hex_nodelimer(m_wallet->dump_tx_to_str(ptx_vector)); + if (unsigned_txset.empty()) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Failed to save unsigned tx set after creation"; + return false; + } + } + else if (!do_not_relay) m_wallet->commit_tx(ptx_vector); // populate response with tx hashes @@ -811,7 +820,7 @@ namespace tools return false; } - return fill_response(ptx_vector, req.get_tx_key, res.tx_key, res.amount, res.fee, res.multisig_txset, req.do_not_relay, + return fill_response(ptx_vector, req.get_tx_key, res.tx_key, res.amount, res.fee, res.multisig_txset, res.unsigned_txset, req.do_not_relay, res.tx_hash, req.get_tx_hex, res.tx_blob, req.get_tx_metadata, res.tx_metadata, er); } catch (const std::exception& e) @@ -858,7 +867,7 @@ namespace tools std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_2(dsts, mixin, req.unlock_time, priority, extra, req.account_index, req.subaddr_indices, m_trusted_daemon); LOG_PRINT_L2("on_transfer_split called create_transactions_2"); - return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.multisig_txset, req.do_not_relay, + return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay, res.tx_hash_list, req.get_tx_hex, res.tx_blob_list, req.get_tx_metadata, res.tx_metadata_list, er); } catch (const std::exception& e) @@ -869,6 +878,141 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_sign_transfer(const wallet_rpc::COMMAND_RPC_SIGN_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_SIGN_TRANSFER::response& res, epee::json_rpc::error& er) + { + if (!m_wallet) return not_open(er); + if (m_wallet->restricted()) + { + er.code = WALLET_RPC_ERROR_CODE_DENIED; + er.message = "Command unavailable in restricted mode."; + return false; + } + if (m_wallet->key_on_device()) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "command not supported by HW wallet"; + return false; + } + if(m_wallet->watch_only()) + { + er.code = WALLET_RPC_ERROR_CODE_WATCH_ONLY; + er.message = "command not supported by watch-only wallet"; + return false; + } + + cryptonote::blobdata blob; + if (!epee::string_tools::parse_hexstr_to_binbuff(req.unsigned_txset, blob)) + { + er.code = WALLET_RPC_ERROR_CODE_BAD_HEX; + er.message = "Failed to parse hex."; + return false; + } + + tools::wallet2::unsigned_tx_set exported_txs; + if(!m_wallet->parse_unsigned_tx_from_str(blob, exported_txs)) + { + er.code = WALLET_RPC_ERROR_CODE_BAD_UNSIGNED_TX_DATA; + er.message = "cannot load unsigned_txset"; + return false; + } + + std::vector<tools::wallet2::pending_tx> ptxs; + try + { + tools::wallet2::signed_tx_set signed_txs; + std::string ciphertext = m_wallet->sign_tx_dump_to_str(exported_txs, ptxs, signed_txs); + if (ciphertext.empty()) + { + er.code = WALLET_RPC_ERROR_CODE_SIGN_UNSIGNED; + er.message = "Failed to sign unsigned tx"; + return false; + } + + res.signed_txset = epee::string_tools::buff_to_hex_nodelimer(ciphertext); + } + catch (const std::exception &e) + { + er.code = WALLET_RPC_ERROR_CODE_SIGN_UNSIGNED; + er.message = std::string("Failed to sign unsigned tx: ") + e.what(); + return false; + } + + for (auto &ptx: ptxs) + { + res.tx_hash_list.push_back(epee::string_tools::pod_to_hex(cryptonote::get_transaction_hash(ptx.tx))); + } + + if (req.export_raw) + { + for (auto &ptx: ptxs) + { + res.tx_raw_list.push_back(epee::string_tools::buff_to_hex_nodelimer(cryptonote::tx_to_blob(ptx.tx))); + } + } + + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_submit_transfer(const wallet_rpc::COMMAND_RPC_SUBMIT_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_SUBMIT_TRANSFER::response& res, epee::json_rpc::error& er) + { + if (!m_wallet) return not_open(er); + if (m_wallet->restricted()) + { + er.code = WALLET_RPC_ERROR_CODE_DENIED; + er.message = "Command unavailable in restricted mode."; + return false; + } + if (m_wallet->key_on_device()) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "command not supported by HW wallet"; + return false; + } + + cryptonote::blobdata blob; + if (!epee::string_tools::parse_hexstr_to_binbuff(req.tx_data_hex, blob)) + { + er.code = WALLET_RPC_ERROR_CODE_BAD_HEX; + er.message = "Failed to parse hex."; + return false; + } + + std::vector<tools::wallet2::pending_tx> ptx_vector; + try + { + bool r = m_wallet->parse_tx_from_str(blob, ptx_vector, NULL); + if (!r) + { + er.code = WALLET_RPC_ERROR_CODE_BAD_SIGNED_TX_DATA; + er.message = "Failed to parse signed tx data."; + return false; + } + } + catch (const std::exception &e) + { + er.code = WALLET_RPC_ERROR_CODE_BAD_SIGNED_TX_DATA; + er.message = std::string("Failed to parse signed tx: ") + e.what(); + return false; + } + + try + { + for (auto &ptx: ptx_vector) + { + m_wallet->commit_tx(ptx); + res.tx_hash_list.push_back(epee::string_tools::pod_to_hex(cryptonote::get_transaction_hash(ptx.tx))); + } + } + catch (const std::exception &e) + { + er.code = WALLET_RPC_ERROR_CODE_SIGNED_SUBMISSION; + er.message = std::string("Failed to submit signed tx: ") + e.what(); + return false; + } + + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_sweep_dust(const wallet_rpc::COMMAND_RPC_SWEEP_DUST::request& req, wallet_rpc::COMMAND_RPC_SWEEP_DUST::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); @@ -883,7 +1027,7 @@ namespace tools { std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_unmixable_sweep_transactions(m_trusted_daemon); - return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.multisig_txset, req.do_not_relay, + return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay, res.tx_hash_list, req.get_tx_hex, res.tx_blob_list, req.get_tx_metadata, res.tx_metadata_list, er); } catch (const std::exception& e) @@ -931,7 +1075,7 @@ namespace tools uint32_t priority = m_wallet->adjust_priority(req.priority); std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_all(req.below_amount, dsts[0].addr, dsts[0].is_subaddress, mixin, req.unlock_time, priority, extra, req.account_index, req.subaddr_indices, m_trusted_daemon); - return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.multisig_txset, req.do_not_relay, + return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay, res.tx_hash_list, req.get_tx_hex, res.tx_blob_list, req.get_tx_metadata, res.tx_metadata_list, er); } catch (const std::exception& e) @@ -1007,7 +1151,7 @@ namespace tools return false; } - return fill_response(ptx_vector, req.get_tx_key, res.tx_key, res.amount, res.fee, res.multisig_txset, req.do_not_relay, + return fill_response(ptx_vector, req.get_tx_key, res.tx_key, res.amount, res.fee, res.multisig_txset, res.unsigned_txset, req.do_not_relay, res.tx_hash, req.get_tx_hex, res.tx_blob, req.get_tx_metadata, res.tx_metadata, er); } catch (const std::exception& e) @@ -1974,6 +2118,72 @@ namespace tools return false; } //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_export_outputs(const wallet_rpc::COMMAND_RPC_EXPORT_OUTPUTS::request& req, wallet_rpc::COMMAND_RPC_EXPORT_OUTPUTS::response& res, epee::json_rpc::error& er) + { + if (!m_wallet) return not_open(er); + if (m_wallet->restricted()) + { + er.code = WALLET_RPC_ERROR_CODE_DENIED; + er.message = "Command unavailable in restricted mode."; + return false; + } + if (m_wallet->key_on_device()) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "command not supported by HW wallet"; + return false; + } + + try + { + res.outputs_data_hex = epee::string_tools::buff_to_hex_nodelimer(m_wallet->export_outputs_to_str()); + } + catch (const std::exception &e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_import_outputs(const wallet_rpc::COMMAND_RPC_IMPORT_OUTPUTS::request& req, wallet_rpc::COMMAND_RPC_IMPORT_OUTPUTS::response& res, epee::json_rpc::error& er) + { + if (!m_wallet) return not_open(er); + if (m_wallet->restricted()) + { + er.code = WALLET_RPC_ERROR_CODE_DENIED; + er.message = "Command unavailable in restricted mode."; + return false; + } + if (m_wallet->key_on_device()) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "command not supported by HW wallet"; + return false; + } + + cryptonote::blobdata blob; + if (!epee::string_tools::parse_hexstr_to_binbuff(req.outputs_data_hex, blob)) + { + er.code = WALLET_RPC_ERROR_CODE_BAD_HEX; + er.message = "Failed to parse hex."; + return false; + } + + try + { + res.num_imported = m_wallet->import_outputs_from_str(blob); + } + catch (const std::exception &e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_export_key_images(const wallet_rpc::COMMAND_RPC_EXPORT_KEY_IMAGES::request& req, wallet_rpc::COMMAND_RPC_EXPORT_KEY_IMAGES::response& res, epee::json_rpc::error& er) { if (!m_wallet) return not_open(er); diff --git a/src/wallet/wallet_rpc_server.h b/src/wallet/wallet_rpc_server.h index cb1a274b6..9cb67c593 100644 --- a/src/wallet/wallet_rpc_server.h +++ b/src/wallet/wallet_rpc_server.h @@ -84,6 +84,8 @@ namespace tools MAP_JON_RPC_WE("getheight", on_getheight, wallet_rpc::COMMAND_RPC_GET_HEIGHT) MAP_JON_RPC_WE("transfer", on_transfer, wallet_rpc::COMMAND_RPC_TRANSFER) MAP_JON_RPC_WE("transfer_split", on_transfer_split, wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT) + MAP_JON_RPC_WE("sign_transfer", on_sign_transfer, wallet_rpc::COMMAND_RPC_SIGN_TRANSFER) + MAP_JON_RPC_WE("submit_transfer", on_submit_transfer, wallet_rpc::COMMAND_RPC_SUBMIT_TRANSFER) MAP_JON_RPC_WE("sweep_dust", on_sweep_dust, wallet_rpc::COMMAND_RPC_SWEEP_DUST) MAP_JON_RPC_WE("sweep_unmixable", on_sweep_dust, wallet_rpc::COMMAND_RPC_SWEEP_DUST) MAP_JON_RPC_WE("sweep_all", on_sweep_all, wallet_rpc::COMMAND_RPC_SWEEP_ALL) @@ -114,6 +116,8 @@ namespace tools MAP_JON_RPC_WE("get_transfer_by_txid", on_get_transfer_by_txid, wallet_rpc::COMMAND_RPC_GET_TRANSFER_BY_TXID) MAP_JON_RPC_WE("sign", on_sign, wallet_rpc::COMMAND_RPC_SIGN) MAP_JON_RPC_WE("verify", on_verify, wallet_rpc::COMMAND_RPC_VERIFY) + MAP_JON_RPC_WE("export_outputs", on_export_outputs, wallet_rpc::COMMAND_RPC_EXPORT_OUTPUTS) + MAP_JON_RPC_WE("import_outputs", on_import_outputs, wallet_rpc::COMMAND_RPC_IMPORT_OUTPUTS) MAP_JON_RPC_WE("export_key_images", on_export_key_images, wallet_rpc::COMMAND_RPC_EXPORT_KEY_IMAGES) MAP_JON_RPC_WE("import_key_images", on_import_key_images, wallet_rpc::COMMAND_RPC_IMPORT_KEY_IMAGES) MAP_JON_RPC_WE("make_uri", on_make_uri, wallet_rpc::COMMAND_RPC_MAKE_URI) @@ -155,6 +159,8 @@ namespace tools bool validate_transfer(const std::list<wallet_rpc::transfer_destination>& destinations, const std::string& payment_id, std::vector<cryptonote::tx_destination_entry>& dsts, std::vector<uint8_t>& extra, bool at_least_one_destination, epee::json_rpc::error& er); bool on_transfer(const wallet_rpc::COMMAND_RPC_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_TRANSFER::response& res, epee::json_rpc::error& er); bool on_transfer_split(const wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT::request& req, wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT::response& res, epee::json_rpc::error& er); + bool on_sign_transfer(const wallet_rpc::COMMAND_RPC_SIGN_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_SIGN_TRANSFER::response& res, epee::json_rpc::error& er); + bool on_submit_transfer(const wallet_rpc::COMMAND_RPC_SUBMIT_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_SUBMIT_TRANSFER::response& res, epee::json_rpc::error& er); bool on_sweep_dust(const wallet_rpc::COMMAND_RPC_SWEEP_DUST::request& req, wallet_rpc::COMMAND_RPC_SWEEP_DUST::response& res, epee::json_rpc::error& er); bool on_sweep_all(const wallet_rpc::COMMAND_RPC_SWEEP_ALL::request& req, wallet_rpc::COMMAND_RPC_SWEEP_ALL::response& res, epee::json_rpc::error& er); bool on_sweep_single(const wallet_rpc::COMMAND_RPC_SWEEP_SINGLE::request& req, wallet_rpc::COMMAND_RPC_SWEEP_SINGLE::response& res, epee::json_rpc::error& er); @@ -183,6 +189,8 @@ namespace tools bool on_get_transfer_by_txid(const wallet_rpc::COMMAND_RPC_GET_TRANSFER_BY_TXID::request& req, wallet_rpc::COMMAND_RPC_GET_TRANSFER_BY_TXID::response& res, epee::json_rpc::error& er); bool on_sign(const wallet_rpc::COMMAND_RPC_SIGN::request& req, wallet_rpc::COMMAND_RPC_SIGN::response& res, epee::json_rpc::error& er); bool on_verify(const wallet_rpc::COMMAND_RPC_VERIFY::request& req, wallet_rpc::COMMAND_RPC_VERIFY::response& res, epee::json_rpc::error& er); + bool on_export_outputs(const wallet_rpc::COMMAND_RPC_EXPORT_OUTPUTS::request& req, wallet_rpc::COMMAND_RPC_EXPORT_OUTPUTS::response& res, epee::json_rpc::error& er); + bool on_import_outputs(const wallet_rpc::COMMAND_RPC_IMPORT_OUTPUTS::request& req, wallet_rpc::COMMAND_RPC_IMPORT_OUTPUTS::response& res, epee::json_rpc::error& er); bool on_export_key_images(const wallet_rpc::COMMAND_RPC_EXPORT_KEY_IMAGES::request& req, wallet_rpc::COMMAND_RPC_EXPORT_KEY_IMAGES::response& res, epee::json_rpc::error& er); bool on_import_key_images(const wallet_rpc::COMMAND_RPC_IMPORT_KEY_IMAGES::request& req, wallet_rpc::COMMAND_RPC_IMPORT_KEY_IMAGES::response& res, epee::json_rpc::error& er); bool on_make_uri(const wallet_rpc::COMMAND_RPC_MAKE_URI::request& req, wallet_rpc::COMMAND_RPC_MAKE_URI::response& res, epee::json_rpc::error& er); @@ -219,7 +227,7 @@ namespace tools template<typename Ts, typename Tu> bool fill_response(std::vector<tools::wallet2::pending_tx> &ptx_vector, - bool get_tx_key, Ts& tx_key, Tu &amount, Tu &fee, std::string &multisig_txset, bool do_not_relay, + bool get_tx_key, Ts& tx_key, Tu &amount, Tu &fee, std::string &multisig_txset, std::string &unsigned_txset, bool do_not_relay, Ts &tx_hash, bool get_tx_hex, Ts &tx_blob, bool get_tx_metadata, Ts &tx_metadata, epee::json_rpc::error &er); wallet2 *m_wallet; diff --git a/src/wallet/wallet_rpc_server_commands_defs.h b/src/wallet/wallet_rpc_server_commands_defs.h index d44aa459f..96e135f01 100644 --- a/src/wallet/wallet_rpc_server_commands_defs.h +++ b/src/wallet/wallet_rpc_server_commands_defs.h @@ -430,6 +430,7 @@ namespace wallet_rpc std::string tx_blob; std::string tx_metadata; std::string multisig_txset; + std::string unsigned_txset; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tx_hash) @@ -440,6 +441,7 @@ namespace wallet_rpc KV_SERIALIZE(tx_blob) KV_SERIALIZE(tx_metadata) KV_SERIALIZE(multisig_txset) + KV_SERIALIZE(unsigned_txset) END_KV_SERIALIZE_MAP() }; }; @@ -495,6 +497,7 @@ namespace wallet_rpc std::list<std::string> tx_blob_list; std::list<std::string> tx_metadata_list; std::string multisig_txset; + std::string unsigned_txset; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tx_hash_list) @@ -504,6 +507,55 @@ namespace wallet_rpc KV_SERIALIZE(tx_blob_list) KV_SERIALIZE(tx_metadata_list) KV_SERIALIZE(multisig_txset) + KV_SERIALIZE(unsigned_txset) + END_KV_SERIALIZE_MAP() + }; + }; + + struct COMMAND_RPC_SIGN_TRANSFER + { + struct request + { + std::string unsigned_txset; + bool export_raw; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(unsigned_txset) + KV_SERIALIZE_OPT(export_raw, false) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string signed_txset; + std::list<std::string> tx_hash_list; + std::list<std::string> tx_raw_list; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(signed_txset) + KV_SERIALIZE(tx_hash_list) + KV_SERIALIZE(tx_raw_list) + END_KV_SERIALIZE_MAP() + }; + }; + + struct COMMAND_RPC_SUBMIT_TRANSFER + { + struct request + { + std::string tx_data_hex; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(tx_data_hex) + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::list<std::string> tx_hash_list; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(tx_hash_list) END_KV_SERIALIZE_MAP() }; }; @@ -543,6 +595,7 @@ namespace wallet_rpc std::list<std::string> tx_blob_list; std::list<std::string> tx_metadata_list; std::string multisig_txset; + std::string unsigned_txset; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tx_hash_list) @@ -552,6 +605,7 @@ namespace wallet_rpc KV_SERIALIZE(tx_blob_list) KV_SERIALIZE(tx_metadata_list) KV_SERIALIZE(multisig_txset) + KV_SERIALIZE(unsigned_txset) END_KV_SERIALIZE_MAP() }; }; @@ -609,6 +663,7 @@ namespace wallet_rpc std::list<std::string> tx_blob_list; std::list<std::string> tx_metadata_list; std::string multisig_txset; + std::string unsigned_txset; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tx_hash_list) @@ -618,6 +673,7 @@ namespace wallet_rpc KV_SERIALIZE(tx_blob_list) KV_SERIALIZE(tx_metadata_list) KV_SERIALIZE(multisig_txset) + KV_SERIALIZE(unsigned_txset) END_KV_SERIALIZE_MAP() }; }; @@ -662,6 +718,7 @@ namespace wallet_rpc std::string tx_blob; std::string tx_metadata; std::string multisig_txset; + std::string unsigned_txset; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tx_hash) @@ -671,6 +728,7 @@ namespace wallet_rpc KV_SERIALIZE(tx_blob) KV_SERIALIZE(tx_metadata) KV_SERIALIZE(multisig_txset) + KV_SERIALIZE(unsigned_txset) END_KV_SERIALIZE_MAP() }; }; @@ -1375,6 +1433,45 @@ namespace wallet_rpc }; }; + struct COMMAND_RPC_EXPORT_OUTPUTS + { + struct request + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + + struct response + { + std::string outputs_data_hex; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(outputs_data_hex); + END_KV_SERIALIZE_MAP() + }; + }; + + struct COMMAND_RPC_IMPORT_OUTPUTS + { + struct request + { + std::string outputs_data_hex; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(outputs_data_hex); + END_KV_SERIALIZE_MAP() + }; + + struct response + { + uint64_t num_imported; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(num_imported); + END_KV_SERIALIZE_MAP() + }; + }; + struct COMMAND_RPC_EXPORT_KEY_IMAGES { struct request diff --git a/src/wallet/wallet_rpc_server_error_codes.h b/src/wallet/wallet_rpc_server_error_codes.h index d47467940..f127ae240 100644 --- a/src/wallet/wallet_rpc_server_error_codes.h +++ b/src/wallet/wallet_rpc_server_error_codes.h @@ -69,3 +69,7 @@ #define WALLET_RPC_ERROR_CODE_MULTISIG_SUBMISSION -36 #define WALLET_RPC_ERROR_CODE_NOT_ENOUGH_UNLOCKED_MONEY -37 #define WALLET_RPC_ERROR_CODE_NO_DAEMON_CONNECTION -38 +#define WALLET_RPC_ERROR_CODE_BAD_UNSIGNED_TX_DATA -39 +#define WALLET_RPC_ERROR_CODE_BAD_SIGNED_TX_DATA -40 +#define WALLET_RPC_ERROR_CODE_SIGNED_SUBMISSION -41 +#define WALLET_RPC_ERROR_CODE_SIGN_UNSIGNED -42 diff --git a/tests/core_proxy/core_proxy.cpp b/tests/core_proxy/core_proxy.cpp index f0a1eb5ce..17e552714 100644 --- a/tests/core_proxy/core_proxy.cpp +++ b/tests/core_proxy/core_proxy.cpp @@ -184,7 +184,7 @@ bool tests::proxy_core::handle_incoming_tx(const cryptonote::blobdata& tx_blob, return true; } -bool tests::proxy_core::handle_incoming_txs(const std::list<blobdata>& tx_blobs, std::vector<tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) +bool tests::proxy_core::handle_incoming_txs(const std::vector<blobdata>& tx_blobs, std::vector<tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { tvc.resize(tx_blobs.size()); size_t i = 0; diff --git a/tests/core_proxy/core_proxy.h b/tests/core_proxy/core_proxy.h index 8b7ac4291..7d36a0f68 100644 --- a/tests/core_proxy/core_proxy.h +++ b/tests/core_proxy/core_proxy.h @@ -76,7 +76,7 @@ namespace tests bool have_block(const crypto::hash& id); void get_blockchain_top(uint64_t& height, crypto::hash& top_id); bool handle_incoming_tx(const cryptonote::blobdata& tx_blob, cryptonote::tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); - bool handle_incoming_txs(const std::list<cryptonote::blobdata>& tx_blobs, std::vector<cryptonote::tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); + bool handle_incoming_txs(const std::vector<cryptonote::blobdata>& tx_blobs, std::vector<cryptonote::tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay); bool handle_incoming_block(const cryptonote::blobdata& block_blob, cryptonote::block_verification_context& bvc, bool update_miner_blocktemplate = true); void pause_mine(){} void resume_mine(){} @@ -86,7 +86,7 @@ namespace tests cryptonote::Blockchain &get_blockchain_storage() { throw std::runtime_error("Called invalid member function: please never call get_blockchain_storage on the TESTING class proxy_core."); } bool get_test_drop_download() {return true;} bool get_test_drop_download_height() {return true;} - bool prepare_handle_incoming_blocks(const std::list<cryptonote::block_complete_entry> &blocks) { return true; } + bool prepare_handle_incoming_blocks(const std::vector<cryptonote::block_complete_entry> &blocks) { return true; } bool cleanup_handle_incoming_blocks(bool force_sync = false) { return true; } uint64_t get_target_blockchain_height() const { return 1; } size_t get_block_sync_size(uint64_t height) const { return BLOCKS_SYNCHRONIZING_DEFAULT_COUNT; } @@ -94,8 +94,8 @@ namespace tests cryptonote::network_type get_nettype() const { return cryptonote::MAINNET; } bool get_pool_transaction(const crypto::hash& id, cryptonote::blobdata& tx_blob) const { return false; } bool pool_has_tx(const crypto::hash &txid) const { return false; } - bool get_blocks(uint64_t start_offset, size_t count, std::list<std::pair<cryptonote::blobdata, cryptonote::block>>& blocks, std::list<cryptonote::blobdata>& txs) const { return false; } - bool get_transactions(const std::vector<crypto::hash>& txs_ids, std::list<cryptonote::transaction>& txs, std::list<crypto::hash>& missed_txs) const { return false; } + bool get_blocks(uint64_t start_offset, size_t count, std::vector<std::pair<cryptonote::blobdata, cryptonote::block>>& blocks, std::vector<cryptonote::blobdata>& txs) const { return false; } + bool get_transactions(const std::vector<crypto::hash>& txs_ids, std::vector<cryptonote::transaction>& txs, std::vector<crypto::hash>& missed_txs) const { return false; } bool get_block_by_hash(const crypto::hash &h, cryptonote::block &blk, bool *orphan = NULL) const { return false; } uint8_t get_ideal_hard_fork_version() const { return 0; } uint8_t get_ideal_hard_fork_version(uint64_t height) const { return 0; } @@ -103,6 +103,6 @@ namespace tests uint64_t get_earliest_ideal_height_for_version(uint8_t version) const { return 0; } cryptonote::difficulty_type get_block_cumulative_difficulty(uint64_t height) const { return 0; } bool fluffy_blocks_enabled() const { return false; } - uint64_t prevalidate_block_hashes(uint64_t height, const std::list<crypto::hash> &hashes) { return 0; } + uint64_t prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes) { return 0; } }; } diff --git a/tests/core_tests/chain_switch_1.cpp b/tests/core_tests/chain_switch_1.cpp index c5017a0df..18a813b19 100644 --- a/tests/core_tests/chain_switch_1.cpp +++ b/tests/core_tests/chain_switch_1.cpp @@ -128,7 +128,7 @@ bool gen_chain_switch_1::check_split_not_switched(cryptonote::core& c, size_t ev m_recipient_account_3 = boost::get<account_base>(events[3]); m_recipient_account_4 = boost::get<account_base>(events[4]); - std::list<block> blocks; + std::vector<block> blocks; bool r = c.get_blocks(0, 10000, blocks); CHECK_TEST_CONDITION(r); CHECK_EQ(5 + 2 * CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW, blocks.size()); @@ -145,7 +145,7 @@ bool gen_chain_switch_1::check_split_not_switched(cryptonote::core& c, size_t ev CHECK_EQ(MK_COINS(14), get_balance(m_recipient_account_3, chain, mtx)); CHECK_EQ(MK_COINS(3), get_balance(m_recipient_account_4, chain, mtx)); - std::list<transaction> tx_pool; + std::vector<transaction> tx_pool; r = c.get_pool_transactions(tx_pool); CHECK_TEST_CONDITION(r); CHECK_EQ(1, tx_pool.size()); @@ -166,7 +166,7 @@ bool gen_chain_switch_1::check_split_switched(cryptonote::core& c, size_t ev_ind { DEFINE_TESTS_ERROR_CONTEXT("gen_chain_switch_1::check_split_switched"); - std::list<block> blocks; + std::vector<block> blocks; bool r = c.get_blocks(0, 10000, blocks); CHECK_TEST_CONDITION(r); CHECK_EQ(6 + 2 * CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW, blocks.size()); @@ -175,7 +175,7 @@ bool gen_chain_switch_1::check_split_switched(cryptonote::core& c, size_t ev_ind CHECK_TEST_CONDITION(std::equal(blocks.begin(), it, m_chain_1.begin())); CHECK_TEST_CONDITION(blocks.back() == boost::get<block>(events[24 + 2 * CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW])); // blk_7 - std::list<block> alt_blocks; + std::vector<block> alt_blocks; r = c.get_alternative_blocks(alt_blocks); CHECK_TEST_CONDITION(r); CHECK_EQ(2, c.get_alternative_blocks_count()); @@ -195,7 +195,7 @@ bool gen_chain_switch_1::check_split_switched(cryptonote::core& c, size_t ev_ind CHECK_EQ(MK_COINS(14), get_balance(m_recipient_account_3, chain, mtx)); CHECK_EQ(MK_COINS(16), get_balance(m_recipient_account_4, chain, mtx)); - std::list<transaction> tx_pool; + std::vector<transaction> tx_pool; r = c.get_pool_transactions(tx_pool); CHECK_TEST_CONDITION(r); CHECK_EQ(1, tx_pool.size()); diff --git a/tests/core_tests/chain_switch_1.h b/tests/core_tests/chain_switch_1.h index 5a035bf06..989b6df11 100644 --- a/tests/core_tests/chain_switch_1.h +++ b/tests/core_tests/chain_switch_1.h @@ -45,12 +45,12 @@ public: bool check_split_switched(cryptonote::core& c, size_t ev_index, const std::vector<test_event_entry>& events); private: - std::list<cryptonote::block> m_chain_1; + std::vector<cryptonote::block> m_chain_1; cryptonote::account_base m_recipient_account_1; cryptonote::account_base m_recipient_account_2; cryptonote::account_base m_recipient_account_3; cryptonote::account_base m_recipient_account_4; - std::list<cryptonote::transaction> m_tx_pool; + std::vector<cryptonote::transaction> m_tx_pool; }; diff --git a/tests/core_tests/chaingen.h b/tests/core_tests/chaingen.h index 6a723d56f..201da4fa0 100644 --- a/tests/core_tests/chaingen.h +++ b/tests/core_tests/chaingen.h @@ -481,7 +481,7 @@ inline bool do_replay_events(std::vector<test_event_entry>& events) MERROR("Failed to flush txpool"); return false; } - c.get_blockchain_storage().flush_txes_from_pool(std::list<crypto::hash>(pool_txs.begin(), pool_txs.end())); + c.get_blockchain_storage().flush_txes_from_pool(pool_txs); t_test_class validator; bool ret = replay_events_through_core<t_test_class>(c, events, validator); diff --git a/tests/core_tests/chaingen001.cpp b/tests/core_tests/chaingen001.cpp index e84bfb924..a76cf1592 100644 --- a/tests/core_tests/chaingen001.cpp +++ b/tests/core_tests/chaingen001.cpp @@ -78,7 +78,7 @@ bool one_block::verify_1(cryptonote::core& c, size_t ev_index, const std::vector //CHECK_TEST_CONDITION(get_block_reward(0) == get_balance(alice, events, chain, mtx)); // check height - std::list<cryptonote::block> blocks; + std::vector<cryptonote::block> blocks; std::list<crypto::public_key> outs; bool r = c.get_blocks(0, 100, blocks); //c.get_outs(100, outs); diff --git a/tests/core_tests/double_spend.cpp b/tests/core_tests/double_spend.cpp index 7ed62cf6d..c60ea885e 100644 --- a/tests/core_tests/double_spend.cpp +++ b/tests/core_tests/double_spend.cpp @@ -73,7 +73,7 @@ bool gen_double_spend_in_different_chains::check_double_spend(cryptonote::core& { DEFINE_TESTS_ERROR_CONTEXT("gen_double_spend_in_different_chains::check_double_spend"); - std::list<block> block_list; + std::vector<block> block_list; bool r = c.get_blocks(0, 100 + 2 * CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW, block_list); CHECK_TEST_CONDITION(r); diff --git a/tests/core_tests/double_spend.inl b/tests/core_tests/double_spend.inl index 0c58fb018..d02147065 100644 --- a/tests/core_tests/double_spend.inl +++ b/tests/core_tests/double_spend.inl @@ -64,7 +64,7 @@ bool gen_double_spend_base<concrete_test>::check_block_verification_context(cons template<class concrete_test> bool gen_double_spend_base<concrete_test>::mark_last_valid_block(cryptonote::core& c, size_t /*ev_index*/, const std::vector<test_event_entry>& /*events*/) { - std::list<cryptonote::block> block_list; + std::vector<cryptonote::block> block_list; bool r = c.get_blocks(c.get_current_blockchain_height() - 1, 1, block_list); CHECK_AND_ASSERT_MES(r, false, "core::get_blocks failed"); m_last_valid_block = block_list.back(); @@ -96,7 +96,7 @@ bool gen_double_spend_base<concrete_test>::check_double_spend(cryptonote::core& } CHECK_NOT_EQ(invalid_index_value, m_invalid_block_index); - std::list<cryptonote::block> block_list; + std::vector<cryptonote::block> block_list; bool r = c.get_blocks(0, 100 + 2 * CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW, block_list); CHECK_TEST_CONDITION(r); CHECK_TEST_CONDITION(m_last_valid_block == block_list.back()); diff --git a/tests/core_tests/ring_signature_1.cpp b/tests/core_tests/ring_signature_1.cpp index 38eb1cf9b..8b2b943cc 100644 --- a/tests/core_tests/ring_signature_1.cpp +++ b/tests/core_tests/ring_signature_1.cpp @@ -101,7 +101,7 @@ bool gen_ring_signature_1::check_balances_1(cryptonote::core& c, size_t ev_index m_bob_account = boost::get<account_base>(events[3]); m_alice_account = boost::get<account_base>(events[4]); - std::list<block> blocks; + std::vector<block> blocks; bool r = c.get_blocks(0, 100 + 2 * CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW, blocks); CHECK_TEST_CONDITION(r); @@ -119,7 +119,7 @@ bool gen_ring_signature_1::check_balances_2(cryptonote::core& c, size_t ev_index { DEFINE_TESTS_ERROR_CONTEXT("gen_ring_signature_1::check_balances_2"); - std::list<block> blocks; + std::vector<block> blocks; bool r = c.get_blocks(0, 100 + 2 * CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW, blocks); CHECK_TEST_CONDITION(r); @@ -182,7 +182,7 @@ bool gen_ring_signature_2::check_balances_1(cryptonote::core& c, size_t ev_index m_bob_account = boost::get<account_base>(events[1]); m_alice_account = boost::get<account_base>(events[2]); - std::list<block> blocks; + std::vector<block> blocks; bool r = c.get_blocks(0, 100 + 2 * CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW, blocks); CHECK_TEST_CONDITION(r); @@ -200,7 +200,7 @@ bool gen_ring_signature_2::check_balances_2(cryptonote::core& c, size_t ev_index { DEFINE_TESTS_ERROR_CONTEXT("gen_ring_signature_2::check_balances_2"); - std::list<block> blocks; + std::vector<block> blocks; bool r = c.get_blocks(0, 100 + 2 * CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW, blocks); CHECK_TEST_CONDITION(r); @@ -292,7 +292,7 @@ bool gen_ring_signature_big::check_balances_1(cryptonote::core& c, size_t ev_ind m_bob_account = boost::get<account_base>(events[1]); m_alice_account = boost::get<account_base>(events[1 + m_test_size]); - std::list<block> blocks; + std::vector<block> blocks; bool r = c.get_blocks(0, 2 * m_test_size + CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW, blocks); CHECK_TEST_CONDITION(r); @@ -317,7 +317,7 @@ bool gen_ring_signature_big::check_balances_2(cryptonote::core& c, size_t ev_ind { DEFINE_TESTS_ERROR_CONTEXT("gen_ring_signature_big::check_balances_2"); - std::list<block> blocks; + std::vector<block> blocks; bool r = c.get_blocks(0, 2 * m_test_size + CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW, blocks); CHECK_TEST_CONDITION(r); diff --git a/tests/fuzz/levin.cpp b/tests/fuzz/levin.cpp index 6a164dda9..4ced1837f 100644 --- a/tests/fuzz/levin.cpp +++ b/tests/fuzz/levin.cpp @@ -158,6 +158,7 @@ namespace } virtual bool close() { return true; } + virtual bool send_done() { return true; } virtual bool call_run_once_service_io() { return true; } virtual bool request_callback() { return true; } virtual boost::asio::io_service& get_io_service() { return m_io_service; } diff --git a/tests/unit_tests/CMakeLists.txt b/tests/unit_tests/CMakeLists.txt index 8cc074bb2..6d79ba74b 100644 --- a/tests/unit_tests/CMakeLists.txt +++ b/tests/unit_tests/CMakeLists.txt @@ -62,6 +62,7 @@ set(unit_tests_sources test_tx_utils.cpp test_peerlist.cpp test_protocol_pack.cpp + threadpool.cpp hardfork.cpp unbound.cpp uri.cpp diff --git a/tests/unit_tests/ban.cpp b/tests/unit_tests/ban.cpp index 15bc0bce3..e3dbdaef1 100644 --- a/tests/unit_tests/ban.cpp +++ b/tests/unit_tests/ban.cpp @@ -55,7 +55,7 @@ public: bool have_block(const crypto::hash& id) const {return true;} void get_blockchain_top(uint64_t& height, crypto::hash& top_id)const{height=0;top_id=crypto::null_hash;} bool handle_incoming_tx(const cryptonote::blobdata& tx_blob, cryptonote::tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { return true; } - bool handle_incoming_txs(const std::list<cryptonote::blobdata>& tx_blob, std::vector<cryptonote::tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { return true; } + bool handle_incoming_txs(const std::vector<cryptonote::blobdata>& tx_blob, std::vector<cryptonote::tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { return true; } bool handle_incoming_block(const cryptonote::blobdata& block_blob, cryptonote::block_verification_context& bvc, bool update_miner_blocktemplate = true) { return true; } void pause_mine(){} void resume_mine(){} @@ -65,7 +65,7 @@ public: cryptonote::blockchain_storage &get_blockchain_storage() { throw std::runtime_error("Called invalid member function: please never call get_blockchain_storage on the TESTING class test_core."); } bool get_test_drop_download() const {return true;} bool get_test_drop_download_height() const {return true;} - bool prepare_handle_incoming_blocks(const std::list<cryptonote::block_complete_entry> &blocks) { return true; } + bool prepare_handle_incoming_blocks(const std::vector<cryptonote::block_complete_entry> &blocks) { return true; } bool cleanup_handle_incoming_blocks(bool force_sync = false) { return true; } uint64_t get_target_blockchain_height() const { return 1; } size_t get_block_sync_size(uint64_t height) const { return BLOCKS_SYNCHRONIZING_DEFAULT_COUNT; } @@ -73,8 +73,8 @@ public: cryptonote::network_type get_nettype() const { return cryptonote::MAINNET; } bool get_pool_transaction(const crypto::hash& id, cryptonote::blobdata& tx_blob) const { return false; } bool pool_has_tx(const crypto::hash &txid) const { return false; } - bool get_blocks(uint64_t start_offset, size_t count, std::list<std::pair<cryptonote::blobdata, cryptonote::block>>& blocks, std::list<cryptonote::blobdata>& txs) const { return false; } - bool get_transactions(const std::vector<crypto::hash>& txs_ids, std::list<cryptonote::transaction>& txs, std::list<crypto::hash>& missed_txs) const { return false; } + bool get_blocks(uint64_t start_offset, size_t count, std::vector<std::pair<cryptonote::blobdata, cryptonote::block>>& blocks, std::vector<cryptonote::blobdata>& txs) const { return false; } + bool get_transactions(const std::vector<crypto::hash>& txs_ids, std::vector<cryptonote::transaction>& txs, std::vector<crypto::hash>& missed_txs) const { return false; } bool get_block_by_hash(const crypto::hash &h, cryptonote::block &blk, bool *orphan = NULL) const { return false; } uint8_t get_ideal_hard_fork_version() const { return 0; } uint8_t get_ideal_hard_fork_version(uint64_t height) const { return 0; } @@ -82,7 +82,7 @@ public: uint64_t get_earliest_ideal_height_for_version(uint8_t version) const { return 0; } cryptonote::difficulty_type get_block_cumulative_difficulty(uint64_t height) const { return 0; } bool fluffy_blocks_enabled() const { return false; } - uint64_t prevalidate_block_hashes(uint64_t height, const std::list<crypto::hash> &hashes) { return 0; } + uint64_t prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes) { return 0; } void stop() {} }; diff --git a/tests/unit_tests/epee_levin_protocol_handler_async.cpp b/tests/unit_tests/epee_levin_protocol_handler_async.cpp index 38a8360d7..72d8f3205 100644 --- a/tests/unit_tests/epee_levin_protocol_handler_async.cpp +++ b/tests/unit_tests/epee_levin_protocol_handler_async.cpp @@ -150,6 +150,7 @@ namespace } virtual bool close() { /*std::cout << "test_connection::close()" << std::endl; */return true; } + virtual bool send_done() { /*std::cout << "test_connection::send_done()" << std::endl; */return true; } virtual bool call_run_once_service_io() { std::cout << "test_connection::call_run_once_service_io()" << std::endl; return true; } virtual bool request_callback() { std::cout << "test_connection::request_callback()" << std::endl; return true; } virtual boost::asio::io_service& get_io_service() { std::cout << "test_connection::get_io_service()" << std::endl; return m_io_service; } diff --git a/tests/unit_tests/epee_utils.cpp b/tests/unit_tests/epee_utils.cpp index 3969f50f6..3474000d8 100644 --- a/tests/unit_tests/epee_utils.cpp +++ b/tests/unit_tests/epee_utils.cpp @@ -45,6 +45,7 @@ #include "boost/archive/portable_binary_oarchive.hpp" #include "hex.h" #include "net/net_utils_base.h" +#include "net/local_ip.h" #include "p2p/net_peerlist_boost_serialization.h" #include "span.h" #include "string_tools.h" @@ -648,3 +649,34 @@ TEST(NetUtils, NetworkAddress) EXPECT_THROW(address1.as<epee::net_utils::ipv4_network_address>(), std::bad_cast); EXPECT_NO_THROW(address1.as<custom_address>()); } + +static bool is_local(const char *s) +{ + uint32_t ip; + CHECK_AND_ASSERT_THROW_MES(epee::string_tools::get_ip_int32_from_string(ip, s), std::string("Invalid IP address: ") + s); + return epee::net_utils::is_ip_local(ip); +} + +TEST(NetUtils, PrivateRanges) +{ + ASSERT_EQ(is_local("10.0.0.0"), true); + ASSERT_EQ(is_local("10.255.0.0"), true); + ASSERT_EQ(is_local("127.0.0.0"), false); // loopback is not considered local + ASSERT_EQ(is_local("192.167.255.255"), false); + ASSERT_EQ(is_local("192.168.0.0"), true); + ASSERT_EQ(is_local("192.168.255.255"), true); + ASSERT_EQ(is_local("192.169.0.0"), false); + ASSERT_EQ(is_local("172.0.0.0"), false); + ASSERT_EQ(is_local("172.15.255.255"), false); + ASSERT_EQ(is_local("172.16.0.0"), true); + ASSERT_EQ(is_local("172.16.255.255"), true); + ASSERT_EQ(is_local("172.31.255.255"), true); + ASSERT_EQ(is_local("172.32.0.0"), false); + ASSERT_EQ(is_local("0.0.0.0"), false); + ASSERT_EQ(is_local("255.255.255.254"), false); + ASSERT_EQ(is_local("11.255.255.255"), false); + ASSERT_EQ(is_local("0.0.0.10"), false); + ASSERT_EQ(is_local("0.0.168.192"), false); + ASSERT_EQ(is_local("0.0.30.172"), false); + ASSERT_EQ(is_local("0.0.30.127"), false); +} diff --git a/tests/unit_tests/ringct.cpp b/tests/unit_tests/ringct.cpp index b7fcbbcab..0f4bd3edf 100644 --- a/tests/unit_tests/ringct.cpp +++ b/tests/unit_tests/ringct.cpp @@ -1061,3 +1061,13 @@ TEST(ringct, key_ostream) out.str() ); } + +TEST(ringct, zeroCommmit) +{ + static const uint64_t amount = crypto::rand<uint64_t>(); + const rct::key z = rct::zeroCommit(amount); + const rct::key a = rct::scalarmultBase(rct::identity()); + const rct::key b = rct::scalarmultH(rct::d2h(amount)); + const rct::key manual = rct::addKeys(a, b); + ASSERT_EQ(z, manual); +} diff --git a/tests/unit_tests/threadpool.cpp b/tests/unit_tests/threadpool.cpp new file mode 100644 index 000000000..1307cd738 --- /dev/null +++ b/tests/unit_tests/threadpool.cpp @@ -0,0 +1,146 @@ +// Copyright (c) 2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <atomic> +#include "gtest/gtest.h" +#include "misc_language.h" +#include "common/threadpool.h" + +TEST(threadpool, wait_nothing) +{ + std::shared_ptr<tools::threadpool> tpool(tools::threadpool::getNewForUnitTests()); + tools::threadpool::waiter waiter; + waiter.wait(tpool.get()); +} + +TEST(threadpool, wait_waits) +{ + std::shared_ptr<tools::threadpool> tpool(tools::threadpool::getNewForUnitTests()); + tools::threadpool::waiter waiter; + std::atomic<bool> b(false); + tpool->submit(&waiter, [&b](){ epee::misc_utils::sleep_no_w(1000); b = true; }); + ASSERT_FALSE(b); + waiter.wait(tpool.get()); + ASSERT_TRUE(b); +} + +TEST(threadpool, one_thread) +{ + std::shared_ptr<tools::threadpool> tpool(tools::threadpool::getNewForUnitTests(1)); + tools::threadpool::waiter waiter; + + std::atomic<unsigned int> counter(0); + for (size_t n = 0; n < 4096; ++n) + { + tpool->submit(&waiter, [&counter](){++counter;}); + } + waiter.wait(tpool.get()); + ASSERT_EQ(counter, 4096); +} + +TEST(threadpool, many_threads) +{ + std::shared_ptr<tools::threadpool> tpool(tools::threadpool::getNewForUnitTests(256)); + tools::threadpool::waiter waiter; + + std::atomic<unsigned int> counter(0); + for (size_t n = 0; n < 4096; ++n) + { + tpool->submit(&waiter, [&counter](){++counter;}); + } + waiter.wait(tpool.get()); + ASSERT_EQ(counter, 4096); +} + +static uint64_t fibonacci(std::shared_ptr<tools::threadpool> tpool, uint64_t n) +{ + if (n <= 1) + return n; + uint64_t f1, f2; + tools::threadpool::waiter waiter; + tpool->submit(&waiter, [&tpool, &f1, n](){ f1 = fibonacci(tpool, n-1); }); + tpool->submit(&waiter, [&tpool, &f2, n](){ f2 = fibonacci(tpool, n-2); }); + waiter.wait(tpool.get()); + return f1 + f2; +} + +TEST(threadpool, reentrency) +{ + std::shared_ptr<tools::threadpool> tpool(tools::threadpool::getNewForUnitTests(4)); + tools::threadpool::waiter waiter; + + uint64_t f = fibonacci(tpool, 13); + waiter.wait(tpool.get()); + ASSERT_EQ(f, 233); +} + +TEST(threadpool, reentrancy) +{ + std::shared_ptr<tools::threadpool> tpool(tools::threadpool::getNewForUnitTests(4)); + tools::threadpool::waiter waiter; + + uint64_t f = fibonacci(tpool, 13); + waiter.wait(tpool.get()); + ASSERT_EQ(f, 233); +} + +TEST(threadpool, leaf_throws) +{ + std::shared_ptr<tools::threadpool> tpool(tools::threadpool::getNewForUnitTests()); + tools::threadpool::waiter waiter; + + bool thrown = false, executed = false; + tpool->submit(&waiter, [&](){ + try { tpool->submit(&waiter, [&](){ executed = true; }); } + catch(const std::exception &e) { thrown = true; } + }, true); + waiter.wait(tpool.get()); + ASSERT_TRUE(thrown); + ASSERT_FALSE(executed); +} + +TEST(threadpool, leaf_reentrancy) +{ + std::shared_ptr<tools::threadpool> tpool(tools::threadpool::getNewForUnitTests(4)); + tools::threadpool::waiter waiter; + + std::atomic<int> counter(0); + for (int i = 0; i < 1000; ++i) + { + tpool->submit(&waiter, [&](){ + tools::threadpool::waiter waiter; + for (int j = 0; j < 500; ++j) + { + tpool->submit(&waiter, [&](){ ++counter; }, true); + } + waiter.wait(tpool.get()); + }); + } + waiter.wait(tpool.get()); + ASSERT_EQ(counter, 500000); +} diff --git a/translations/monero.ts b/translations/monero.ts index 1c2723608..5f0ba0c7c 100644 --- a/translations/monero.ts +++ b/translations/monero.ts @@ -461,7 +461,7 @@ </message> <message> <location filename="../src/rpc/rpc_args.cpp" line="105"/> - <source> requires RFC server password --</source> + <source> requires RPC server password --</source> <translation type="unfinished"></translation> </message> </context> @@ -1678,7 +1678,7 @@ If the "tag_description" argument is specified, the tag <tag_name&g refresh-type <full|optimize-coinbase|no-coinbase|default> Set the wallet's refresh behaviour. priority [0|1|2|3|4] - Set the fee too default/unimportant/normal/elevated/priority. + Set the fee to default/unimportant/normal/elevated/priority. confirm-missing-payment-id <1|0> ask-password <1|0> unit <monero|millinero|micronero|nanonero|piconero> diff --git a/translations/monero_fr.ts b/translations/monero_fr.ts index a52d1a11f..957d2a382 100644 --- a/translations/monero_fr.ts +++ b/translations/monero_fr.ts @@ -461,7 +461,7 @@ </message> <message> <location filename="../src/rpc/rpc_args.cpp" line="105"/> - <source> requires RFC server password --</source> + <source> requires RPC server password --</source> <translation> nécessite le mot de passe du serveur RPC --</translation> </message> </context> @@ -1699,7 +1699,7 @@ Si l'argument "tag_description" est spécifié, le texte arbitrai refresh-type <full|optimize-coinbase|no-coinbase|default> Set the wallet's refresh behaviour. priority [0|1|2|3|4] - Set the fee too default/unimportant/normal/elevated/priority. + Set the fee to default/unimportant/normal/elevated/priority. confirm-missing-payment-id <1|0> ask-password <1|0> unit <monero|millinero|micronero|nanonero|piconero> diff --git a/translations/monero_it.ts b/translations/monero_it.ts index d14a33d60..d4b1695f3 100644 --- a/translations/monero_it.ts +++ b/translations/monero_it.ts @@ -461,8 +461,8 @@ </message> <message> <location filename="../src/rpc/rpc_args.cpp" line="105"/> - <source> requires RFC server password --</source> - <translation> richiede la password del server RFC --</translation> + <source> requires RPC server password --</source> + <translation type="unfinished"></translation> </message> </context> <context> @@ -1582,7 +1582,7 @@ If the "tag_description" argument is specified, the tag <tag_name&g refresh-type <full|optimize-coinbase|no-coinbase|default> Set the wallet's refresh behaviour. priority [0|1|2|3|4] - Set the fee too default/unimportant/normal/elevated/priority. + Set the fee to default/unimportant/normal/elevated/priority. confirm-missing-payment-id <1|0> ask-password <1|0> unit <monero|millinero|micronero|nanonero|piconero> diff --git a/translations/monero_sv.ts b/translations/monero_sv.ts index e99076960..3c5ddf9af 100644 --- a/translations/monero_sv.ts +++ b/translations/monero_sv.ts @@ -461,7 +461,7 @@ </message> <message> <location filename="../src/rpc/rpc_args.cpp" line="105"/> - <source> requires RFC server password --</source> + <source> requires RPC server password --</source> <translation> kräver lösenord till RPC-server --</translation> </message> </context> @@ -1699,7 +1699,7 @@ Om argumentet "tag_description" anges, så tilldelas taggen <taggna refresh-type <full|optimize-coinbase|no-coinbase|default> Set the wallet's refresh behaviour. priority [0|1|2|3|4] - Set the fee too default/unimportant/normal/elevated/priority. + Set the fee to default/unimportant/normal/elevated/priority. confirm-missing-payment-id <1|0> ask-password <1|0> unit <monero|millinero|micronero|nanonero|piconero> |
