diff options
117 files changed, 3713 insertions, 3796 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 60fcf130e..9f628049b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,6 +27,9 @@ # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + list(INSERT CMAKE_MODULE_PATH 0 "${CMAKE_SOURCE_DIR}/cmake") include(CheckCCompilerFlag) @@ -34,6 +37,7 @@ include(CheckCXXCompilerFlag) include(CheckLinkerFlag) include(CheckLibraryExists) include(CheckFunctionExists) +include(FindPythonInterp) if (IOS) INCLUDE(CmakeLists_IOS.txt) @@ -356,54 +360,9 @@ endif() # memory was the default in Cryptonote before Monero implemented LMDB, it still works but is unnecessary. # set(DATABASE memory) set(DATABASE lmdb) - -if (DEFINED ENV{DATABASE}) - set(DATABASE $ENV{DATABASE}) - message(STATUS "DATABASE set: ${DATABASE}") -else() - message(STATUS "Could not find DATABASE in env (not required unless you want to change database type from default: ${DATABASE})") -endif() - -set(BERKELEY_DB_OVERRIDE 0) -if (DEFINED ENV{BERKELEY_DB}) - set(BERKELEY_DB_OVERRIDE 1) - set(BERKELEY_DB $ENV{BERKELEY_DB}) -elseif() - set(BERKELEY_DB 0) -endif() - -if (DATABASE STREQUAL "lmdb") - message(STATUS "Using LMDB as default DB type") - set(BLOCKCHAIN_DB DB_LMDB) - add_definitions("-DDEFAULT_DB_TYPE=\"lmdb\"") -elseif (DATABASE STREQUAL "berkeleydb") - find_package(BerkeleyDB) - if(NOT BERKELEY_DB) - die("Found BerkeleyDB includes, but could not find BerkeleyDB library. Please make sure you have installed libdb and libdb-dev / libdb++-dev or the equivalent.") - else() - message(STATUS "Found BerkeleyDB include (db.h) in ${BERKELEY_DB_INCLUDE_DIR}") - if(BERKELEY_DB_LIBRARIES) - message(STATUS "Found BerkeleyDB shared library") - set(BDB_STATIC false CACHE BOOL "BDB Static flag") - set(BDB_INCLUDE ${BERKELEY_DB_INCLUDE_DIR} CACHE STRING "BDB include path") - set(BDB_LIBRARY ${BERKELEY_DB_LIBRARIES} CACHE STRING "BDB library name") - set(BDB_LIBRARY_DIRS "" CACHE STRING "BDB Library dirs") - set(BERKELEY_DB 1) - else() - die("Found BerkeleyDB includes, but could not find BerkeleyDB library. Please make sure you have installed libdb and libdb-dev / libdb++-dev or the equivalent.") - endif() - endif() - - message(STATUS "Using Berkeley DB as default DB type") - add_definitions("-DDEFAULT_DB_TYPE=\"berkeley\"") -else() - die("Invalid database type: ${DATABASE}") -endif() - -if(BERKELEY_DB) - add_definitions("-DBERKELEY_DB") -endif() - +message(STATUS "Using LMDB as default DB type") +set(BLOCKCHAIN_DB DB_LMDB) +add_definitions("-DDEFAULT_DB_TYPE=\"lmdb\"") add_definitions("-DBLOCKCHAIN_DB=${BLOCKCHAIN_DB}") # Can't install hook in static build on OSX, because OSX linker does not support --wrap @@ -505,11 +464,6 @@ link_directories(${EASYLOGGING_LIBRARY_DIRS}) # Final setup for liblmdb include_directories(${LMDB_INCLUDE}) -# Final setup for Berkeley DB -if (BERKELEY_DB) - include_directories(${BDB_INCLUDE}) -endif() - # Final setup for libunwind include_directories(${LIBUNWIND_INCLUDE}) link_directories(${LIBUNWIND_LIBRARY_DIRS}) @@ -978,14 +932,14 @@ if(CMAKE_C_COMPILER_ID STREQUAL "Clang" AND ARCH_WIDTH EQUAL "32" AND NOT IOS AN endif() endif() -find_path(ZMQ_INCLUDE_PATH zmq.hpp) +find_path(ZMQ_INCLUDE_PATH zmq.h) find_library(ZMQ_LIB zmq) find_library(PGM_LIBRARY pgm) find_library(NORM_LIBRARY norm) find_library(SODIUM_LIBRARY sodium) if(NOT ZMQ_INCLUDE_PATH) - message(FATAL_ERROR "Could not find required header zmq.hpp") + message(FATAL_ERROR "Could not find required header zmq.h") endif() if(NOT ZMQ_LIB) message(FATAL_ERROR "Could not find required libzmq") @@ -1048,3 +1002,13 @@ option(INSTALL_VENDORED_LIBUNBOUND "Install libunbound binary built from source CHECK_C_COMPILER_FLAG(-std=c11 HAVE_C11) + +find_package(PythonInterp) +find_program(iwyu_tool_path NAMES iwyu_tool.py iwyu_tool) +if (iwyu_tool_path AND PYTHONINTERP_FOUND) + add_custom_target(iwyu + COMMAND "${PYTHON_EXECUTABLE}" "${iwyu_tool_path}" -p "${CMAKE_BINARY_DIR}" -- --no_fwd_decls + COMMENT "Running include-what-you-use tool" + VERBATIM + ) +endif() @@ -63,6 +63,10 @@ debug-test: mkdir -p $(builddir)/debug cd $(builddir)/debug && cmake -D BUILD_TESTS=ON -D CMAKE_BUILD_TYPE=Debug $(topdir) && $(MAKE) && $(MAKE) ARGS="-E libwallet_api_tests" test +debug-test-asan: + mkdir -p $(builddir)/debug + cd $(builddir)/debug && cmake -D BUILD_TESTS=ON -D SANITIZE=ON -D CMAKE_BUILD_TYPE=Debug $(topdir) && $(MAKE) && $(MAKE) ARGS="-E libwallet_api_tests" test + debug-test-trezor: mkdir -p $(builddir)/debug cd $(builddir)/debug && cmake -D BUILD_TESTS=ON -D TREZOR_DEBUG=ON -D CMAKE_BUILD_TYPE=Debug $(topdir) && $(MAKE) && $(MAKE) ARGS="-E libwallet_api_tests" test @@ -22,6 +22,9 @@ Portions Copyright (c) 2012-2013 The Cryptonote developers. - [Release staging schedule and protocol](#release-staging-schedule-and-protocol) - [Compiling Monero from source](#compiling-monero-from-source) - [Dependencies](#dependencies) + - [Internationalization](#Internationalization) + - [Using Tor](#Using Tor) + - [Debugging](#Debugging) - [Known issues](#known-issues) ## Development resources @@ -178,7 +181,7 @@ library archives (`.a`). | pkg-config | any | NO | `pkg-config` | `base-devel` | `pkgconf` | NO | | | Boost | 1.58 | NO | `libboost-all-dev` | `boost` | `boost-devel` | NO | C++ libraries | | OpenSSL | basically any | NO | `libssl-dev` | `openssl` | `openssl-devel` | NO | sha256 sum | -| libzmq | 3.0.0 | NO | `libzmq3-dev` | `zeromq` | `cppzmq-devel` | NO | ZeroMQ library | +| libzmq | 3.0.0 | NO | `libzmq3-dev` | `zeromq` | `zeromq-devel` | NO | ZeroMQ library | | OpenPGM | ? | NO | `libpgm-dev` | `libpgm` | `openpgm-devel` | NO | For ZeroMQ | | libnorm[2] | ? | NO | `libnorm-dev` | | ` | YES | For ZeroMQ | | libunbound | 1.4.16 | YES | `libunbound-dev` | `unbound` | `unbound-devel` | NO | DNS resolver | @@ -550,13 +553,12 @@ The produced binaries still link libc dynamically. If the binary is compiled on Packages are available for -* Ubuntu and [snap supported](https://snapcraft.io/docs/core/install) systems, via a community contributed build. +* Debian Bullseye and Sid ```bash - snap install monero --beta + sudo apt install monero ``` - -Installing a snap is very quick. Snaps are secure. They are isolated with all of their dependencies. Snaps also auto update when a new version is released. +More info and versions in the [Debian package tracker](https://tracker.debian.org/pkg/monero). * Arch Linux (via [AUR](https://aur.archlinux.org/)): - Stable release: [`monero`](https://aur.archlinux.org/packages/monero) diff --git a/cmake/FindBerkeleyDB.cmake b/cmake/FindBerkeleyDB.cmake deleted file mode 100644 index 916d8f9cc..000000000 --- a/cmake/FindBerkeleyDB.cmake +++ /dev/null @@ -1,25 +0,0 @@ -# - Try to find Berkeley DB -# Once done this will define -# -# BERKELEY_DB_FOUND - system has Berkeley DB -# BERKELEY_DB_INCLUDE_DIR - the Berkeley DB include directory -# BERKELEY_DB_LIBRARIES - Link these to use Berkeley DB -# BERKELEY_DB_DEFINITIONS - Compiler switches required for using Berkeley DB - -# Copyright (c) 2006, Alexander Dymo, <adymo@kdevelop.org> -# -# Redistribution and use is allowed according to the terms of the BSD license. -# For details see the accompanying COPYING-CMAKE-SCRIPTS file. - -find_path(BERKELEY_DB_INCLUDE_DIR db_cxx.h - /usr/include/db4 - /usr/local/include/db4 -) - -find_library(BERKELEY_DB_LIBRARIES NAMES db_cxx ) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(Berkeley "Could not find Berkeley DB >= 4.1" BERKELEY_DB_INCLUDE_DIR BERKELEY_DB_LIBRARIES) -# show the BERKELEY_DB_INCLUDE_DIR and BERKELEY_DB_LIBRARIES variables only in the advanced view -mark_as_advanced(BERKELEY_DB_INCLUDE_DIR BERKELEY_DB_LIBRARIES ) - diff --git a/contrib/depends/packages/packages.mk b/contrib/depends/packages/packages.mk index 1e5a74670..f4b9c6407 100644 --- a/contrib/depends/packages/packages.mk +++ b/contrib/depends/packages/packages.mk @@ -12,7 +12,7 @@ packages += gtest endif ifneq ($(host_arch),riscv64) - packages += unwind +linux_packages += unwind endif ifeq ($(host_os),mingw32) diff --git a/contrib/epee/include/int-util.h b/contrib/epee/include/int-util.h index 0ed6505ff..8ef5be40a 100644 --- a/contrib/epee/include/int-util.h +++ b/contrib/epee/include/int-util.h @@ -129,9 +129,12 @@ static inline uint32_t div128_32(uint64_t dividend_hi, uint64_t dividend_lo, uin return remainder; } +#define IDENT16(x) ((uint16_t) (x)) #define IDENT32(x) ((uint32_t) (x)) #define IDENT64(x) ((uint64_t) (x)) +#define SWAP16(x) ((((uint16_t) (x) & 0x00ff) << 8) | \ + (((uint16_t) (x) & 0xff00) >> 8)) #define SWAP32(x) ((((uint32_t) (x) & 0x000000ff) << 24) | \ (((uint32_t) (x) & 0x0000ff00) << 8) | \ (((uint32_t) (x) & 0x00ff0000) >> 8) | \ @@ -145,10 +148,18 @@ static inline uint32_t div128_32(uint64_t dividend_hi, uint64_t dividend_lo, uin (((uint64_t) (x) & 0x00ff000000000000) >> 40) | \ (((uint64_t) (x) & 0xff00000000000000) >> 56)) +static inline uint16_t ident16(uint16_t x) { return x; } static inline uint32_t ident32(uint32_t x) { return x; } static inline uint64_t ident64(uint64_t x) { return x; } #ifndef __OpenBSD__ +# if defined(__ANDROID__) && defined(__swap16) && !defined(swap16) +# define swap16 __swap16 +# elif !defined(swap16) +static inline uint16_t swap16(uint16_t x) { + return ((x & 0x00ff) << 8) | ((x & 0xff00) >> 8); +} +# endif # if defined(__ANDROID__) && defined(__swap32) && !defined(swap32) # define swap32 __swap32 # elif !defined(swap32) @@ -176,6 +187,12 @@ static inline uint64_t swap64(uint64_t x) { static inline void mem_inplace_ident(void *mem UNUSED, size_t n UNUSED) { } #undef UNUSED +static inline void mem_inplace_swap16(void *mem, size_t n) { + size_t i; + for (i = 0; i < n; i++) { + ((uint16_t *) mem)[i] = swap16(((const uint16_t *) mem)[i]); + } +} static inline void mem_inplace_swap32(void *mem, size_t n) { size_t i; for (i = 0; i < n; i++) { @@ -189,6 +206,9 @@ static inline void mem_inplace_swap64(void *mem, size_t n) { } } +static inline void memcpy_ident16(void *dst, const void *src, size_t n) { + memcpy(dst, src, 2 * n); +} static inline void memcpy_ident32(void *dst, const void *src, size_t n) { memcpy(dst, src, 4 * n); } @@ -196,6 +216,12 @@ static inline void memcpy_ident64(void *dst, const void *src, size_t n) { memcpy(dst, src, 8 * n); } +static inline void memcpy_swap16(void *dst, const void *src, size_t n) { + size_t i; + for (i = 0; i < n; i++) { + ((uint16_t *) dst)[i] = swap16(((const uint16_t *) src)[i]); + } +} static inline void memcpy_swap32(void *dst, const void *src, size_t n) { size_t i; for (i = 0; i < n; i++) { @@ -220,6 +246,14 @@ static_assert(false, "BYTE_ORDER is undefined. Perhaps, GNU extensions are not e #endif #if BYTE_ORDER == LITTLE_ENDIAN +#define SWAP16LE IDENT16 +#define SWAP16BE SWAP16 +#define swap16le ident16 +#define swap16be swap16 +#define mem_inplace_swap16le mem_inplace_ident +#define mem_inplace_swap16be mem_inplace_swap16 +#define memcpy_swap16le memcpy_ident16 +#define memcpy_swap16be memcpy_swap16 #define SWAP32LE IDENT32 #define SWAP32BE SWAP32 #define swap32le ident32 @@ -239,6 +273,14 @@ static_assert(false, "BYTE_ORDER is undefined. Perhaps, GNU extensions are not e #endif #if BYTE_ORDER == BIG_ENDIAN +#define SWAP16BE IDENT16 +#define SWAP16LE SWAP16 +#define swap16be ident16 +#define swap16le swap16 +#define mem_inplace_swap16be mem_inplace_ident +#define mem_inplace_swap16le mem_inplace_swap16 +#define memcpy_swap16be memcpy_ident16 +#define memcpy_swap16le memcpy_swap16 #define SWAP32BE IDENT32 #define SWAP32LE SWAP32 #define swap32be ident32 diff --git a/contrib/epee/include/misc_log_ex.h b/contrib/epee/include/misc_log_ex.h index 602b6a371..0b216f2c4 100644 --- a/contrib/epee/include/misc_log_ex.h +++ b/contrib/epee/include/misc_log_ex.h @@ -38,29 +38,29 @@ #define MAX_LOG_FILE_SIZE 104850000 // 100 MB - 7600 bytes #define MAX_LOG_FILES 50 -#define MCLOG_TYPE(level, cat, type, x) do { \ +#define MCLOG_TYPE(level, cat, color, type, x) do { \ if (ELPP->vRegistry()->allowed(level, cat)) { \ - el::base::Writer(level, __FILE__, __LINE__, ELPP_FUNC, type).construct(cat) << x; \ + el::base::Writer(level, color, __FILE__, __LINE__, ELPP_FUNC, type).construct(cat) << x; \ } \ } while (0) -#define MCLOG(level, cat, x) MCLOG_TYPE(level, cat, el::base::DispatchAction::NormalLog, x) -#define MCLOG_FILE(level, cat, x) MCLOG_TYPE(level, cat, el::base::DispatchAction::FileOnlyLog, x) +#define MCLOG(level, cat, color, x) MCLOG_TYPE(level, cat, color, el::base::DispatchAction::NormalLog, x) +#define MCLOG_FILE(level, cat, x) MCLOG_TYPE(level, cat, el::Color::Default, el::base::DispatchAction::FileOnlyLog, x) -#define MCFATAL(cat,x) MCLOG(el::Level::Fatal,cat, x) -#define MCERROR(cat,x) MCLOG(el::Level::Error,cat, x) -#define MCWARNING(cat,x) MCLOG(el::Level::Warning,cat, x) -#define MCINFO(cat,x) MCLOG(el::Level::Info,cat, x) -#define MCDEBUG(cat,x) MCLOG(el::Level::Debug,cat, x) -#define MCTRACE(cat,x) MCLOG(el::Level::Trace,cat, x) +#define MCFATAL(cat,x) MCLOG(el::Level::Fatal,cat, el::Color::Default, x) +#define MCERROR(cat,x) MCLOG(el::Level::Error,cat, el::Color::Default, x) +#define MCWARNING(cat,x) MCLOG(el::Level::Warning,cat, el::Color::Default, x) +#define MCINFO(cat,x) MCLOG(el::Level::Info,cat, el::Color::Default, x) +#define MCDEBUG(cat,x) MCLOG(el::Level::Debug,cat, el::Color::Default, x) +#define MCTRACE(cat,x) MCLOG(el::Level::Trace,cat, el::Color::Default, x) -#define MCLOG_COLOR(level,cat,color,x) MCLOG(level,cat,"\033[1;" color "m" << x << "\033[0m") -#define MCLOG_RED(level,cat,x) MCLOG_COLOR(level,cat,"31",x) -#define MCLOG_GREEN(level,cat,x) MCLOG_COLOR(level,cat,"32",x) -#define MCLOG_YELLOW(level,cat,x) MCLOG_COLOR(level,cat,"33",x) -#define MCLOG_BLUE(level,cat,x) MCLOG_COLOR(level,cat,"34",x) -#define MCLOG_MAGENTA(level,cat,x) MCLOG_COLOR(level,cat,"35",x) -#define MCLOG_CYAN(level,cat,x) MCLOG_COLOR(level,cat,"36",x) +#define MCLOG_COLOR(level,cat,color,x) MCLOG(level,cat,color,x) +#define MCLOG_RED(level,cat,x) MCLOG_COLOR(level,cat,el::Color::Red,x) +#define MCLOG_GREEN(level,cat,x) MCLOG_COLOR(level,cat,el::Color::Green,x) +#define MCLOG_YELLOW(level,cat,x) MCLOG_COLOR(level,cat,el::Color::Yellow,x) +#define MCLOG_BLUE(level,cat,x) MCLOG_COLOR(level,cat,el::Color::Blue,x) +#define MCLOG_MAGENTA(level,cat,x) MCLOG_COLOR(level,cat,el::Color::Magenta,x) +#define MCLOG_CYAN(level,cat,x) MCLOG_COLOR(level,cat,el::Color::Cyan,x) #define MLOG_RED(level,x) MCLOG_RED(level,MONERO_DEFAULT_LOG_CATEGORY,x) #define MLOG_GREEN(level,x) MCLOG_GREEN(level,MONERO_DEFAULT_LOG_CATEGORY,x) @@ -75,7 +75,7 @@ #define MINFO(x) MCINFO(MONERO_DEFAULT_LOG_CATEGORY,x) #define MDEBUG(x) MCDEBUG(MONERO_DEFAULT_LOG_CATEGORY,x) #define MTRACE(x) MCTRACE(MONERO_DEFAULT_LOG_CATEGORY,x) -#define MLOG(level,x) MCLOG(level,MONERO_DEFAULT_LOG_CATEGORY,x) +#define MLOG(level,x) MCLOG(level,MONERO_DEFAULT_LOG_CATEGORY,el::Color::Default,x) #define MGINFO(x) MCINFO("global",x) #define MGINFO_RED(x) MCLOG_RED(el::Level::Info, "global",x) @@ -85,14 +85,14 @@ #define MGINFO_MAGENTA(x) MCLOG_MAGENTA(el::Level::Info, "global",x) #define MGINFO_CYAN(x) MCLOG_CYAN(el::Level::Info, "global",x) -#define IFLOG(level, cat, type, init, x) \ +#define IFLOG(level, cat, color, type, init, x) \ do { \ if (ELPP->vRegistry()->allowed(level, cat)) { \ init; \ - el::base::Writer(level, __FILE__, __LINE__, ELPP_FUNC, type).construct(cat) << x; \ + el::base::Writer(level, color, __FILE__, __LINE__, ELPP_FUNC, type).construct(cat) << x; \ } \ } while(0) -#define MIDEBUG(init, x) IFLOG(el::Level::Debug, MONERO_DEFAULT_LOG_CATEGORY, el::base::DispatchAction::NormalLog, init, x) +#define MIDEBUG(init, x) IFLOG(el::Level::Debug, MONERO_DEFAULT_LOG_CATEGORY, el::Color::Default, el::base::DispatchAction::NormalLog, init, x) #define LOG_ERROR(x) MERROR(x) diff --git a/contrib/epee/include/net/abstract_tcp_server2.inl b/contrib/epee/include/net/abstract_tcp_server2.inl index 12a87071a..8d96e4a84 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.inl +++ b/contrib/epee/include/net/abstract_tcp_server2.inl @@ -154,7 +154,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) } else { - const auto ip_{remote_ep.address().to_v6()}; + const auto ip_ = remote_ep.address().to_v6(); return start(is_income, is_multithreaded, ipv6_network_address{ip_, remote_ep.port()}); } CATCH_ENTRY_L0("connection<t_protocol_handler>::start()", false); diff --git a/contrib/epee/include/net/local_ip.h b/contrib/epee/include/net/local_ip.h index ce74e1cd3..246cf6ad8 100644 --- a/contrib/epee/include/net/local_ip.h +++ b/contrib/epee/include/net/local_ip.h @@ -30,6 +30,11 @@ #include <string> #include <boost/algorithm/string/predicate.hpp> #include <boost/asio/ip/address_v6.hpp> +#include "int-util.h" + +// IP addresses are kept in network byte order +// Masks below are little endian +// -> convert from network byte order to host byte order before comparing namespace epee { @@ -62,6 +67,7 @@ namespace epee inline bool is_ip_local(uint32_t ip) { + ip = SWAP32LE(ip); /* local ip area 10.0.0.0 — 10.255.255.255 @@ -85,6 +91,7 @@ namespace epee inline bool is_ip_loopback(uint32_t ip) { + ip = SWAP32LE(ip); if( (ip | 0xffffff00) == 0xffffff7f) return true; //MAKE_IP diff --git a/contrib/epee/include/net/net_utils_base.h b/contrib/epee/include/net/net_utils_base.h index dd80fae8b..028e605d7 100644 --- a/contrib/epee/include/net/net_utils_base.h +++ b/contrib/epee/include/net/net_utils_base.h @@ -38,6 +38,7 @@ #include "enums.h" #include "misc_log_ex.h" #include "serialization/keyvalue_serialization.h" +#include "int-util.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net" @@ -91,7 +92,20 @@ namespace net_utils static constexpr bool is_blockable() noexcept { return true; } BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(m_ip) + if (is_store) + { + KV_SERIALIZE_VAL_POD_AS_BLOB_N(m_ip, "ip") + uint32_t ip = SWAP32LE(this_ref.m_ip); + epee::serialization::selector<is_store>::serialize(ip, stg, hparent_section, "m_ip"); + } + else + { + if (!epee::serialization::selector<is_store>::serialize_t_val_as_blob(this_ref.m_ip, stg, hparent_section, "ip")) + { + KV_SERIALIZE(m_ip) + const_cast<ipv4_network_address&>(this_ref).m_ip = SWAP32LE(this_ref.m_ip); + } + } KV_SERIALIZE(m_port) END_KV_SERIALIZE_MAP() }; diff --git a/contrib/epee/include/storages/portable_storage_bin_utils.h b/contrib/epee/include/storages/portable_storage_bin_utils.h new file mode 100644 index 000000000..bcde64487 --- /dev/null +++ b/contrib/epee/include/storages/portable_storage_bin_utils.h @@ -0,0 +1,46 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include "int-util.h" + +template<typename T> T convert_swapper(T t) { return t; } +template<> inline uint16_t convert_swapper(uint16_t t) { return SWAP16LE(t); } +template<> inline int16_t convert_swapper(int16_t t) { return SWAP16LE((uint16_t&)t); } +template<> inline uint32_t convert_swapper(uint32_t t) { return SWAP32LE(t); } +template<> inline int32_t convert_swapper(int32_t t) { return SWAP32LE((uint32_t&)t); } +template<> inline uint64_t convert_swapper(uint64_t t) { return SWAP64LE(t); } +template<> inline int64_t convert_swapper(int64_t t) { return SWAP64LE((uint64_t&)t); } +template<> inline double convert_swapper(double t) { union { uint64_t u; double d; } u; u.d = t; u.u = SWAP64LE(u.u); return u.d; } + +#if BYTE_ORDER == BIG_ENDIAN +#define CONVERT_POD(x) convert_swapper(x) +#else +#define CONVERT_POD(x) (x) +#endif diff --git a/contrib/epee/include/storages/portable_storage_from_bin.h b/contrib/epee/include/storages/portable_storage_from_bin.h index e0a32b3ca..c0b6cc7b1 100644 --- a/contrib/epee/include/storages/portable_storage_from_bin.h +++ b/contrib/epee/include/storages/portable_storage_from_bin.h @@ -30,6 +30,7 @@ #include "misc_language.h" #include "portable_storage_base.h" +#include "portable_storage_bin_utils.h" #ifdef EPEE_PORTABLE_STORAGE_RECURSION_LIMIT #define EPEE_PORTABLE_STORAGE_RECURSION_LIMIT_INTERNAL EPEE_PORTABLE_STORAGE_RECURSION_LIMIT @@ -117,6 +118,7 @@ namespace epee RECURSION_LIMITATION(); static_assert(std::is_pod<t_pod_type>::value, "POD type expected"); read(&pod_val, sizeof(pod_val)); + pod_val = CONVERT_POD(pod_val); } template<class t_type> @@ -140,7 +142,7 @@ namespace epee sa.reserve(size); //TODO: add some optimization here later while(size--) - sa.m_array.push_back(read<type_name>()); + sa.m_array.push_back(read<type_name>()); return storage_entry(array_entry(sa)); } diff --git a/contrib/epee/include/storages/portable_storage_to_bin.h b/contrib/epee/include/storages/portable_storage_to_bin.h index 9501bbc2a..137497e19 100644 --- a/contrib/epee/include/storages/portable_storage_to_bin.h +++ b/contrib/epee/include/storages/portable_storage_to_bin.h @@ -31,6 +31,7 @@ #include "pragma_comp_defs.h" #include "misc_language.h" #include "portable_storage_base.h" +#include "portable_storage_bin_utils.h" namespace epee { @@ -40,8 +41,9 @@ namespace epee template<class pack_value, class t_stream> size_t pack_varint_t(t_stream& strm, uint8_t type_or, size_t& pv) { - pack_value v = (*((pack_value*)&pv)) << 2; + pack_value v = pv << 2; v |= type_or; + v = CONVERT_POD(v); strm.write((const char*)&v, sizeof(pack_value)); return sizeof(pack_value); } @@ -93,8 +95,11 @@ namespace epee uint8_t type = contained_type|SERIALIZE_FLAG_ARRAY; m_strm.write((const char*)&type, 1); pack_varint(m_strm, arr_pod.m_array.size()); - for(const t_pod_type& x: arr_pod.m_array) + for(t_pod_type x: arr_pod.m_array) + { + x = CONVERT_POD(x); m_strm.write((const char*)&x, sizeof(t_pod_type)); + } return true; } @@ -147,7 +152,8 @@ namespace epee bool pack_pod_type(uint8_t type, const pod_type& v) { m_strm.write((const char*)&type, 1); - m_strm.write((const char*)&v, sizeof(pod_type)); + pod_type v0 = CONVERT_POD(v); + m_strm.write((const char*)&v0, sizeof(pod_type)); return true; } //section, array_entry diff --git a/contrib/epee/src/mlog.cpp b/contrib/epee/src/mlog.cpp index 4c6ad5516..0cf579840 100644 --- a/contrib/epee/src/mlog.cpp +++ b/contrib/epee/src/mlog.cpp @@ -109,7 +109,7 @@ static const char *get_default_categories(int level) categories = "*:DEBUG"; break; case 3: - categories = "*:TRACE"; + categories = "*:TRACE,*.dump:DEBUG"; break; case 4: categories = "*:TRACE"; @@ -472,4 +472,40 @@ void reset_console_color() { } +static void mlog(el::Level level, const char *category, const char *format, va_list ap) +{ + int size = 0; + char *p = NULL; + va_list apc; + + /* Determine required size */ + va_copy(apc, ap); + size = vsnprintf(p, size, format, apc); + va_end(apc); + if (size < 0) + return; + + size++; /* For '\0' */ + p = (char*)malloc(size); + if (p == NULL) + return; + + size = vsnprintf(p, size, format, ap); + if (size < 0) + { + free(p); + return; + } + + MCLOG(level, category, el::Color::Default, p); + free(p); +} + +void mfatal(const char *category, const char *fmt, ...) { va_list ap; va_start(ap, fmt); mlog(el::Level::Fatal, category, fmt, ap); va_end(ap); } +void merror(const char *category, const char *fmt, ...) { va_list ap; va_start(ap, fmt); mlog(el::Level::Error, category, fmt, ap); va_end(ap); } +void mwarning(const char *category, const char *fmt, ...) { va_list ap; va_start(ap, fmt); mlog(el::Level::Warning, category, fmt, ap); va_end(ap); } +void minfo(const char *category, const char *fmt, ...) { va_list ap; va_start(ap, fmt); mlog(el::Level::Info, category, fmt, ap); va_end(ap); } +void mdebug(const char *category, const char *fmt, ...) { va_list ap; va_start(ap, fmt); mlog(el::Level::Debug, category, fmt, ap); va_end(ap); } +void mtrace(const char *category, const char *fmt, ...) { va_list ap; va_start(ap, fmt); mlog(el::Level::Trace, category, fmt, ap); va_end(ap); } + #endif //_MLOG_H_ diff --git a/contrib/lsan/lsan.supp b/contrib/lsan/lsan.supp new file mode 100644 index 000000000..4aef353c6 --- /dev/null +++ b/contrib/lsan/lsan.supp @@ -0,0 +1 @@ +leak:slow_hash_allocate_state diff --git a/external/boost/archive/portable_binary_iarchive.hpp b/external/boost/archive/portable_binary_iarchive.hpp index bd19599f3..d17e95c05 100644 --- a/external/boost/archive/portable_binary_iarchive.hpp +++ b/external/boost/archive/portable_binary_iarchive.hpp @@ -258,7 +258,7 @@ portable_binary_iarchive::load_impl(boost::intmax_t & l, char maxsize){ this->primitive_base_t::load_binary(cptr, size);
#if BOOST_ENDIAN_BIG_BYTE
- if(m_flags & endian_little)
+ if((m_flags & endian_little) || (!(m_flags & endian_big)))
#else
if(m_flags & endian_big)
#endif
@@ -343,6 +343,8 @@ portable_binary_iarchive::init(unsigned int flags){ );
#endif
}
+ if (!(m_flags & (endian_little | endian_big)))
+ m_flags |= endian_little;
unsigned char x;
load(x);
m_flags = x << CHAR_BIT;
diff --git a/external/boost/archive/portable_binary_oarchive.hpp b/external/boost/archive/portable_binary_oarchive.hpp index 783c7f7c9..a0ac0a9b5 100644 --- a/external/boost/archive/portable_binary_oarchive.hpp +++ b/external/boost/archive/portable_binary_oarchive.hpp @@ -171,7 +171,7 @@ protected: void init(unsigned int flags);
public:
- portable_binary_oarchive(std::ostream & os, unsigned flags = 0) :
+ portable_binary_oarchive(std::ostream & os, unsigned flags = endian_little) :
primitive_base_t(
* os.rdbuf(),
0 != (flags & boost::archive::no_codecvt)
diff --git a/external/db_drivers/liblmdb/mdb_dump.c b/external/db_drivers/liblmdb/mdb_dump.c index b7737f12d..068dab5a8 100644 --- a/external/db_drivers/liblmdb/mdb_dump.c +++ b/external/db_drivers/liblmdb/mdb_dump.c @@ -64,6 +64,8 @@ static void text(MDB_val *v) end = c + v->mv_size; while (c < end) { if (isprint(*c)) { + if (*c == '\\') + putchar('\\'); putchar(*c); } else { putchar('\\'); diff --git a/external/db_drivers/liblmdb/mdb_load.c b/external/db_drivers/liblmdb/mdb_load.c index ad911c088..e900ae660 100644 --- a/external/db_drivers/liblmdb/mdb_load.c +++ b/external/db_drivers/liblmdb/mdb_load.c @@ -236,7 +236,7 @@ badend: while (c2 < end) { if (*c2 == '\\') { if (c2[1] == '\\') { - c1++; c2 += 2; + *c1++ = *c2; } else { if (c2+3 > end || !isxdigit(c2[1]) || !isxdigit(c2[2])) { Eof = 1; @@ -244,8 +244,8 @@ badend: return EOF; } *c1++ = unhex(++c2); - c2 += 2; } + c2 += 2; } else { /* copies are redundant when no escapes were used */ *c1++ = *c2++; diff --git a/external/easylogging++/easylogging++.cc b/external/easylogging++/easylogging++.cc index 2b4c7bbbf..a5a4a64b7 100644 --- a/external/easylogging++/easylogging++.cc +++ b/external/easylogging++/easylogging++.cc @@ -18,6 +18,7 @@ #include "easylogging++.h" #include <unistd.h> +#include <boost/algorithm/string.hpp> #if defined(AUTO_INITIALIZE_EASYLOGGINGPP) INITIALIZE_EASYLOGGINGPP @@ -133,6 +134,50 @@ static void abort(int status, const std::string& reason) { #endif // defined(ELPP_COMPILER_MSVC) && defined(_M_IX86) && defined(_DEBUG) } +static el::Color colorFromLevel(el::Level level) +{ + switch (level) + { + case Level::Error: case Level::Fatal: return el::Color::Red; + case Level::Warning: return el::Color::Yellow; + case Level::Debug: return el::Color::Green; + case Level::Info: return el::Color::Cyan; + case Level::Trace: return el::Color::Magenta; + default: return el::Color::Default; + } +} + +static void setConsoleColor(el::Color color, bool bright) +{ +#if ELPP_OS_WINDOWS + HANDLE h_stdout = GetStdHandle(STD_OUTPUT_HANDLE); + switch (color) + { + case el::Color::Red: SetConsoleTextAttribute(h_stdout, FOREGROUND_RED | (bright ? FOREGROUND_INTENSITY:0)); break; + case el::Color::Green: SetConsoleTextAttribute(h_stdout, FOREGROUND_GREEN | (bright ? FOREGROUND_INTENSITY:0)); break; + case el::Color::Yellow: SetConsoleTextAttribute(h_stdout, FOREGROUND_RED | FOREGROUND_GREEN | (bright ? FOREGROUND_INTENSITY:0)); break; + case el::Color::Blue: SetConsoleTextAttribute(h_stdout, FOREGROUND_BLUE | (bright ? FOREGROUND_INTENSITY:0)); break; + case el::Color::Magenta: SetConsoleTextAttribute(h_stdout, FOREGROUND_RED | FOREGROUND_BLUE | (bright ? FOREGROUND_INTENSITY:0)); break; + case el::Color::Cyan: SetConsoleTextAttribute(h_stdout, FOREGROUND_GREEN | FOREGROUND_BLUE | (bright ? FOREGROUND_INTENSITY:0)); break; + case el::Color::Default: default: SetConsoleTextAttribute(h_stdout, FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE | (bright ? FOREGROUND_INTENSITY:0)); break; + } +#else + if (ELPP->hasFlag(LoggingFlag::ColoredTerminalOutput)) + { + switch (color) + { + case el::Color::Red: ELPP_COUT << (bright ? "\033[1;31m" : "\033[0;31m"); break; + case el::Color::Green: ELPP_COUT << (bright ? "\033[1;32m" : "\033[0;32m"); break; + case el::Color::Yellow: ELPP_COUT << (bright ? "\033[1;33m" : "\033[0;33m"); break; + case el::Color::Blue: ELPP_COUT << (bright ? "\033[1;34m" : "\033[0;34m"); break; + case el::Color::Magenta: ELPP_COUT << (bright ? "\033[1;35m" : "\033[0;35m"); break; + case el::Color::Cyan: ELPP_COUT << (bright ? "\033[1;36m" : "\033[0;36m"); break; + case el::Color::Default: default: ELPP_COUT << "\033[0m"; break; + } + } +#endif +} + } // namespace utils } // namespace base @@ -609,19 +654,34 @@ void Configurations::unsafeSetGlobally(ConfigurationType configurationType, cons // LogBuilder -void LogBuilder::convertToColoredOutput(base::type::string_t* logLine, Level level) { +void LogBuilder::convertToColoredOutput(base::type::string_t* logLine, Level level, Color color) { if (!m_termSupportsColor) return; const base::type::char_t* resetColor = ELPP_LITERAL("\x1b[0m"); - if (level == Level::Error || level == Level::Fatal) - *logLine = ELPP_LITERAL("\x1b[31m") + *logLine + resetColor; - else if (level == Level::Warning) - *logLine = ELPP_LITERAL("\x1b[33m") + *logLine + resetColor; - else if (level == Level::Debug) - *logLine = ELPP_LITERAL("\x1b[32m") + *logLine + resetColor; - else if (level == Level::Info) - *logLine = ELPP_LITERAL("\x1b[36m") + *logLine + resetColor; - else if (level == Level::Trace) - *logLine = ELPP_LITERAL("\x1b[35m") + *logLine + resetColor; + if (color == Color::Red) + *logLine = ELPP_LITERAL("\x1b[1;31m") + *logLine + resetColor; + else if (color == Color::Yellow) + *logLine = ELPP_LITERAL("\x1b[1;33m") + *logLine + resetColor; + else if (color == Color::Green) + *logLine = ELPP_LITERAL("\x1b[1;32m") + *logLine + resetColor; + else if (color == Color::Cyan) + *logLine = ELPP_LITERAL("\x1b[1;36m") + *logLine + resetColor; + else if (color == Color::Magenta) + *logLine = ELPP_LITERAL("\x1b[1;35m") + *logLine + resetColor; + else if (color == Color::Blue) + *logLine = ELPP_LITERAL("\x1b[1;34m") + *logLine + resetColor; + else if (color == Color::Default) + { + if (level == Level::Error || level == Level::Fatal) + *logLine = ELPP_LITERAL("\x1b[31m") + *logLine + resetColor; + else if (level == Level::Warning) + *logLine = ELPP_LITERAL("\x1b[33m") + *logLine + resetColor; + else if (level == Level::Debug) + *logLine = ELPP_LITERAL("\x1b[32m") + *logLine + resetColor; + else if (level == Level::Info) + *logLine = ELPP_LITERAL("\x1b[36m") + *logLine + resetColor; + else if (level == Level::Trace) + *logLine = ELPP_LITERAL("\x1b[35m") + *logLine + resetColor; + } } // Logger @@ -2372,13 +2432,32 @@ void DefaultLogDispatchCallback::handle(const LogDispatchData* data) { m_data = data; base::TypedConfigurations* tc = m_data->logMessage()->logger()->typedConfigurations(); const base::LogFormat* logFormat = &tc->logFormat(m_data->logMessage()->level()); - dispatch(base::utils::DateTime::getDateTime(logFormat->dateTimeFormat().c_str(), &tc->subsecondPrecision(m_data->logMessage()->level())) - + "\t" + convertToChar(m_data->logMessage()->level()) + " " + m_data->logMessage()->message() + "\n", - m_data->logMessage()->logger()->logBuilder()->build(m_data->logMessage(), - m_data->dispatchAction() == base::DispatchAction::NormalLog || m_data->dispatchAction() == base::DispatchAction::FileOnlyLog)); + + const auto &logmsg = m_data->logMessage(); + const auto msg = logmsg->message(); + if (strchr(msg.c_str(), '\n')) + { + std::vector<std::string> v; + boost::split(v, msg, boost::is_any_of("\n")); + for (const std::string &s: v) + { + LogMessage msgline(logmsg->level(), logmsg->color(), logmsg->file(), logmsg->line(), logmsg->func(), logmsg->verboseLevel(), logmsg->logger(), &s); + dispatch(base::utils::DateTime::getDateTime(logFormat->dateTimeFormat().c_str(), &tc->subsecondPrecision(m_data->logMessage()->level())) + "\t" + convertToChar(m_data->logMessage()->level()) + " ", + s + "\n", + m_data->logMessage()->logger()->logBuilder()->build(&msgline, + m_data->dispatchAction() == base::DispatchAction::NormalLog || m_data->dispatchAction() == base::DispatchAction::FileOnlyLog)); + } + } + else + { + dispatch(base::utils::DateTime::getDateTime(logFormat->dateTimeFormat().c_str(), &tc->subsecondPrecision(m_data->logMessage()->level())) + + "\t" + convertToChar(m_data->logMessage()->level()) + " ", m_data->logMessage()->message() + "\n", + m_data->logMessage()->logger()->logBuilder()->build(m_data->logMessage(), + m_data->dispatchAction() == base::DispatchAction::NormalLog || m_data->dispatchAction() == base::DispatchAction::FileOnlyLog)); + } } -void DefaultLogDispatchCallback::dispatch(base::type::string_t&& rawLine, base::type::string_t&& logLine) { +void DefaultLogDispatchCallback::dispatch(base::type::string_t&& rawLinePrefix, base::type::string_t&& rawLinePayload, base::type::string_t&& logLine) { if (m_data->dispatchAction() == base::DispatchAction::NormalLog || m_data->dispatchAction() == base::DispatchAction::FileOnlyLog) { if (m_data->logMessage()->logger()->m_typedConfigurations->toFile(m_data->logMessage()->level())) { base::type::fstream_t* fs = m_data->logMessage()->logger()->m_typedConfigurations->fileStream( @@ -2404,9 +2483,14 @@ void DefaultLogDispatchCallback::dispatch(base::type::string_t&& rawLine, base:: } if (m_data->dispatchAction() != base::DispatchAction::FileOnlyLog) { if (m_data->logMessage()->logger()->m_typedConfigurations->toStandardOutput(m_data->logMessage()->level())) { - if (ELPP->hasFlag(LoggingFlag::ColoredTerminalOutput)) - m_data->logMessage()->logger()->logBuilder()->convertToColoredOutput(&rawLine, m_data->logMessage()->level()); - ELPP_COUT << ELPP_COUT_LINE(rawLine); + const el::Level level = m_data->logMessage()->level(); + const el::Color color = m_data->logMessage()->color(); + el::base::utils::setConsoleColor(el::base::utils::colorFromLevel(level), false); + ELPP_COUT << rawLinePrefix; + el::base::utils::setConsoleColor(color == el::Color::Default ? el::base::utils::colorFromLevel(level): color, color != el::Color::Default); + ELPP_COUT << rawLinePayload; + el::base::utils::setConsoleColor(el::Color::Default, false); + ELPP_COUT << std::flush; } } } @@ -2447,7 +2531,7 @@ void AsyncLogDispatchCallback::handle(const LogDispatchData* data) { if ((data->dispatchAction() == base::DispatchAction::NormalLog || data->dispatchAction() == base::DispatchAction::FileOnlyLog) && data->logMessage()->logger()->typedConfigurations()->toStandardOutput(data->logMessage()->level())) { if (ELPP->hasFlag(LoggingFlag::ColoredTerminalOutput)) - data->logMessage()->logger()->logBuilder()->convertToColoredOutput(&logLine, data->logMessage()->level()); + data->logMessage()->logger()->logBuilder()->convertToColoredOutput(&logLine, data->logMessage()->level(), data->logMessage()->color()); ELPP_COUT << ELPP_COUT_LINE(logLine); } // Save resources and only queue if we want to write to file otherwise just ignore handler @@ -2739,7 +2823,7 @@ void Writer::initializeLogger(const std::string& loggerId, bool lookup, bool nee ELPP->registeredLoggers()->get(std::string(base::consts::kDefaultLoggerId)); } } - Writer(Level::Debug, m_file, m_line, m_func).construct(1, base::consts::kDefaultLoggerId) + Writer(Level::Debug, Color::Default, m_file, m_line, m_func).construct(1, base::consts::kDefaultLoggerId) << "Logger [" << loggerId << "] is not registered yet!"; m_proceed = false; } else { @@ -2813,7 +2897,7 @@ void Writer::processDispatch() { void Writer::triggerDispatch(void) { if (m_proceed) { if (m_msg == nullptr) { - LogMessage msg(m_level, m_file, m_line, m_func, m_verboseLevel, + LogMessage msg(m_level, m_color, m_file, m_line, m_func, m_verboseLevel, m_logger); base::LogDispatcher(m_proceed, &msg, m_dispatchAction).dispatch(); } else { @@ -2826,7 +2910,7 @@ void Writer::triggerDispatch(void) { } if (m_proceed && m_level == Level::Fatal && !ELPP->hasFlag(LoggingFlag::DisableApplicationAbortOnFatalLog)) { - base::Writer(Level::Warning, m_file, m_line, m_func).construct(1, base::consts::kDefaultLoggerId) + base::Writer(Level::Warning, Color::Default, m_file, m_line, m_func).construct(1, base::consts::kDefaultLoggerId) << "Aborting application. Reason: Fatal log at [" << m_file << ":" << m_line << "]"; std::stringstream reasonStream; reasonStream << "Fatal log at [" << m_file << ":" << m_line << "]" diff --git a/external/easylogging++/easylogging++.h b/external/easylogging++/easylogging++.h index f1fa2cb0d..a10b0c8e6 100644 --- a/external/easylogging++/easylogging++.h +++ b/external/easylogging++/easylogging++.h @@ -604,6 +604,15 @@ enum class Level : base::type::EnumType { /// @brief Represents unknown level Unknown = 1010 }; +enum class Color : base::type::EnumType { + Default, + Red, + Green, + Yellow, + Blue, + Magenta, + Cyan, +}; } // namespace el namespace std { template<> struct hash<el::Level> { @@ -2225,7 +2234,7 @@ class LogBuilder : base::NoCopy { ELPP_INTERNAL_INFO(3, "Destroying log builder...") } virtual base::type::string_t build(const LogMessage* logMessage, bool appendNewLine) const = 0; - void convertToColoredOutput(base::type::string_t* logLine, Level level); + void convertToColoredOutput(base::type::string_t* logLine, Level level, Color color); private: bool m_termSupportsColor; friend class el::base::DefaultLogDispatchCallback; @@ -2503,14 +2512,17 @@ class VRegistry : base::NoCopy, public base::threading::ThreadSafe { } // namespace base class LogMessage { public: - LogMessage(Level level, const std::string& file, base::type::LineNumber line, const std::string& func, - base::type::VerboseLevel verboseLevel, Logger* logger) : - m_level(level), m_file(file), m_line(line), m_func(func), - m_verboseLevel(verboseLevel), m_logger(logger), m_message(logger->stream().str()) { + LogMessage(Level level, Color color, const std::string& file, base::type::LineNumber line, const std::string& func, + base::type::VerboseLevel verboseLevel, Logger* logger, const base::type::string_t *msg = nullptr) : + m_level(level), m_color(color), m_file(file), m_line(line), m_func(func), + m_verboseLevel(verboseLevel), m_logger(logger), m_message(msg ? *msg : logger->stream().str()) { } inline Level level(void) const { return m_level; } + inline Color color(void) const { + return m_color; + } inline const std::string& file(void) const { return m_file; } @@ -2531,6 +2543,7 @@ class LogMessage { } private: Level m_level; + Color m_color; std::string m_file; base::type::LineNumber m_line; std::string m_func; @@ -2781,7 +2794,7 @@ class DefaultLogDispatchCallback : public LogDispatchCallback { void handle(const LogDispatchData* data); private: const LogDispatchData* m_data; - void dispatch(base::type::string_t&& rawLine, base::type::string_t&& logLine); + void dispatch(base::type::string_t&& rawLinePrefix, base::type::string_t&& rawLinePayload, base::type::string_t&& logLine); }; #if ELPP_ASYNC_LOGGING class AsyncLogDispatchCallback : public LogDispatchCallback { @@ -3242,10 +3255,10 @@ class NullWriter : base::NoCopy { /// @brief Main entry point of each logging class Writer : base::NoCopy { public: - Writer(Level level, const char* file, base::type::LineNumber line, + Writer(Level level, Color color, const char* file, base::type::LineNumber line, const char* func, base::DispatchAction dispatchAction = base::DispatchAction::NormalLog, base::type::VerboseLevel verboseLevel = 0) : - m_msg(nullptr), m_level(level), m_file(file), m_line(line), m_func(func), m_verboseLevel(verboseLevel), + m_msg(nullptr), m_level(level), m_color(color), m_file(file), m_line(line), m_func(func), m_verboseLevel(verboseLevel), m_logger(nullptr), m_proceed(false), m_dispatchAction(dispatchAction) { } @@ -3299,6 +3312,7 @@ class Writer : base::NoCopy { protected: LogMessage* m_msg; Level m_level; + Color m_color; const char* m_file; const base::type::LineNumber m_line; const char* m_func; @@ -3317,10 +3331,10 @@ class Writer : base::NoCopy { }; class PErrorWriter : public base::Writer { public: - PErrorWriter(Level level, const char* file, base::type::LineNumber line, + PErrorWriter(Level level, Color color, const char* file, base::type::LineNumber line, const char* func, base::DispatchAction dispatchAction = base::DispatchAction::NormalLog, base::type::VerboseLevel verboseLevel = 0) : - base::Writer(level, file, line, func, dispatchAction, verboseLevel) { + base::Writer(level, color, file, line, func, dispatchAction, verboseLevel) { } virtual ~PErrorWriter(void); @@ -3353,14 +3367,14 @@ template <typename T> void Logger::log_(Level level, int vlevel, const T& log) { if (level == Level::Verbose) { if (ELPP->vRegistry()->allowed(vlevel, __FILE__)) { - base::Writer(Level::Verbose, "FILE", 0, "FUNCTION", + base::Writer(Level::Verbose, Color::Default, "FILE", 0, "FUNCTION", base::DispatchAction::NormalLog, vlevel).construct(this, false) << log; } else { stream().str(ELPP_LITERAL("")); releaseLock(); } } else { - base::Writer(level, "FILE", 0, "FUNCTION").construct(this, false) << log; + base::Writer(level, Color::Default, "FILE", 0, "FUNCTION").construct(this, false) << log; } } template <typename T, typename... Args> @@ -3460,18 +3474,18 @@ LOGGER_LEVEL_WRITERS_DISABLED(trace, Level::Trace) #endif // ELPP_COMPILER_MSVC #define el_resolveVALength(_0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, N, ...) N #define ELPP_WRITE_LOG(writer, level, dispatchAction, ...) \ -writer(level, __FILE__, __LINE__, ELPP_FUNC, dispatchAction).construct(el_getVALength(__VA_ARGS__), __VA_ARGS__) +writer(level, el::Color::Default, __FILE__, __LINE__, ELPP_FUNC, dispatchAction).construct(el_getVALength(__VA_ARGS__), __VA_ARGS__) #define ELPP_WRITE_LOG_IF(writer, condition, level, dispatchAction, ...) if (condition) \ -writer(level, __FILE__, __LINE__, ELPP_FUNC, dispatchAction).construct(el_getVALength(__VA_ARGS__), __VA_ARGS__) +writer(level, el::Color::Default, __FILE__, __LINE__, ELPP_FUNC, dispatchAction).construct(el_getVALength(__VA_ARGS__), __VA_ARGS__) #define ELPP_WRITE_LOG_EVERY_N(writer, occasion, level, dispatchAction, ...) \ ELPP->validateEveryNCounter(__FILE__, __LINE__, occasion) && \ -writer(level, __FILE__, __LINE__, ELPP_FUNC, dispatchAction).construct(el_getVALength(__VA_ARGS__), __VA_ARGS__) +writer(level, el::Color::Default, __FILE__, __LINE__, ELPP_FUNC, dispatchAction).construct(el_getVALength(__VA_ARGS__), __VA_ARGS__) #define ELPP_WRITE_LOG_AFTER_N(writer, n, level, dispatchAction, ...) \ ELPP->validateAfterNCounter(__FILE__, __LINE__, n) && \ -writer(level, __FILE__, __LINE__, ELPP_FUNC, dispatchAction).construct(el_getVALength(__VA_ARGS__), __VA_ARGS__) +writer(level, el::Color::Default, __FILE__, __LINE__, ELPP_FUNC, dispatchAction).construct(el_getVALength(__VA_ARGS__), __VA_ARGS__) #define ELPP_WRITE_LOG_N_TIMES(writer, n, level, dispatchAction, ...) \ ELPP->validateNTimesCounter(__FILE__, __LINE__, n) && \ -writer(level, __FILE__, __LINE__, ELPP_FUNC, dispatchAction).construct(el_getVALength(__VA_ARGS__), __VA_ARGS__) +writer(level, el::Color::Default, __FILE__, __LINE__, ELPP_FUNC, dispatchAction).construct(el_getVALength(__VA_ARGS__), __VA_ARGS__) #if defined(ELPP_FEATURE_ALL) || defined(ELPP_FEATURE_PERFORMANCE_TRACKING) class PerformanceTrackingData { public: diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8a21763c8..9bab56200 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -112,6 +112,7 @@ add_subdirectory(cryptonote_core) add_subdirectory(lmdb) add_subdirectory(multisig) add_subdirectory(net) +add_subdirectory(hardforks) if(NOT IOS) add_subdirectory(blockchain_db) endif() diff --git a/src/blockchain_db/CMakeLists.txt b/src/blockchain_db/CMakeLists.txt index db161fa5e..0a21e4920 100644 --- a/src/blockchain_db/CMakeLists.txt +++ b/src/blockchain_db/CMakeLists.txt @@ -31,14 +31,6 @@ set(blockchain_db_sources lmdb/db_lmdb.cpp ) -if (BERKELEY_DB) - set(blockchain_db_sources - ${blockchain_db_sources} - berkeleydb/db_bdb.cpp - ) -endif() - - set(blockchain_db_headers) set(blockchain_db_private_headers @@ -46,13 +38,6 @@ set(blockchain_db_private_headers lmdb/db_lmdb.h ) -if (BERKELEY_DB) - set(blockchain_db_private_headers - ${blockchain_db_private_headers} - berkeleydb/db_bdb.h - ) -endif() - monero_private_headers(blockchain_db ${crypto_private_headers}) monero_add_library(blockchain_db @@ -65,7 +50,6 @@ target_link_libraries(blockchain_db cncrypto ringct ${LMDB_LIBRARY} - ${BDB_LIBRARY} ${Boost_FILESYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY} PRIVATE diff --git a/src/blockchain_db/berkeleydb/db_bdb.cpp b/src/blockchain_db/berkeleydb/db_bdb.cpp deleted file mode 100644 index d138a1e7e..000000000 --- a/src/blockchain_db/berkeleydb/db_bdb.cpp +++ /dev/null @@ -1,2296 +0,0 @@ -// Copyright (c) 2014-2019, The Monero Project -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without modification, are -// permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this list of -// conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright notice, this list -// of conditions and the following disclaimer in the documentation and/or other -// materials provided with the distribution. -// -// 3. Neither the name of the copyright holder nor the names of its contributors may be -// used to endorse or promote products derived from this software without specific -// prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL -// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF -// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -#include "db_bdb.h" - -#include <boost/filesystem.hpp> -#include <memory> // std::unique_ptr -#include <cstring> // memcpy - -#include "cryptonote_basic/cryptonote_format_utils.h" -#include "crypto/crypto.h" -#include "profile_tools.h" - -using epee::string_tools::pod_to_hex; -#define DB_DEFAULT_TX (m_write_txn != nullptr ? *m_write_txn : (DbTxn*) nullptr) - -// Increase when the DB changes in a non backward compatible way, and there -// is no automatic conversion, so that a full resync is needed. -#define VERSION 0 - -namespace -{ - -template <typename T> -inline void throw0(const T &e) -{ - LOG_PRINT_L0(e.what()); - throw e; -} - -template <typename T> -inline void throw1(const T &e) -{ - LOG_PRINT_L1(e.what()); - throw e; -} - -// cursor needs to be closed when it goes out of scope, -// this helps if the function using it throws -struct bdb_cur -{ - bdb_cur(DbTxn* txn, Db* dbi) - { - if (dbi->cursor(txn, &m_cur, 0)) - throw0(cryptonote::DB_ERROR("Error opening db cursor")); - done = false; - } - - ~bdb_cur() - { - close(); - } - - operator Dbc*() - { - return m_cur; - } - operator Dbc**() - { - return &m_cur; - } - Dbc* operator->() - { - return m_cur; - } - - void close() - { - if (!done) - { - m_cur->close(); - done = true; - } - } - -private: - Dbc* m_cur; - bool done; -}; - -const char* const BDB_BLOCKS = "blocks"; -const char* const BDB_BLOCK_TIMESTAMPS = "block_timestamps"; -const char* const BDB_BLOCK_HEIGHTS = "block_heights"; -const char* const BDB_BLOCK_HASHES = "block_hashes"; -const char* const BDB_BLOCK_SIZES = "block_sizes"; -const char* const BDB_BLOCK_DIFFS = "block_diffs"; -const char* const BDB_BLOCK_COINS = "block_coins"; - -const char* const BDB_TXS = "txs"; -const char* const BDB_TX_UNLOCKS = "tx_unlocks"; -const char* const BDB_TX_HEIGHTS = "tx_heights"; -const char* const BDB_TX_OUTPUTS = "tx_outputs"; - -const char* const BDB_OUTPUT_TXS = "output_txs"; -const char* const BDB_OUTPUT_INDICES = "output_indices"; -const char* const BDB_OUTPUT_AMOUNTS = "output_amounts"; -const char* const BDB_OUTPUT_KEYS = "output_keys"; - -const char* const BDB_SPENT_KEYS = "spent_keys"; - -const char* const BDB_HF_STARTING_HEIGHTS = "hf_starting_heights"; -const char* const BDB_HF_VERSIONS = "hf_versions"; - -const char* const BDB_PROPERTIES = "properties"; - -const unsigned int MB = 1024 * 1024; -// ND: FIXME: db keeps running out of locks when doing full syncs. Possible bug??? Set it to 5K for now. -const unsigned int DB_MAX_LOCKS = 5000; -const unsigned int DB_BUFFER_LENGTH = 32 * MB; -// 256MB cache adjust as necessary using DB_CONFIG -const unsigned int DB_DEF_CACHESIZE = 256 * MB; - -#if defined(BDB_BULK_CAN_THREAD) -const unsigned int DB_BUFFER_COUNT = tools::get_max_concurrency(); -#else -const unsigned int DB_BUFFER_COUNT = 1; -#endif - -template<typename T> -struct Dbt_copy: public Dbt -{ - Dbt_copy(const T &t) : - t_copy(t) - { - init(); - } - - Dbt_copy() - { - init(); - } - - void init() - { - set_data(&t_copy); - set_size(sizeof(T)); - set_ulen(sizeof(T)); - set_flags(DB_DBT_USERMEM); - } - - operator T() - { - return t_copy; - } -private: - T t_copy; -}; - -template<> -struct Dbt_copy<cryptonote::blobdata>: public Dbt -{ - Dbt_copy(const cryptonote::blobdata &bd) : - m_data(new char[bd.size()]) - { - memcpy(m_data.get(), bd.data(), bd.size()); - set_data(m_data.get()); - set_size(bd.size()); - set_ulen(bd.size()); - set_flags(DB_DBT_USERMEM); - } -private: - std::unique_ptr<char[]> m_data; -}; - -template<> -struct Dbt_copy<const char*>: public Dbt -{ - Dbt_copy(const char *s) : - m_data(strdup(s)) - { - size_t len = strlen(s) + 1; // include the NUL, makes it easier for compare - set_data(m_data.get()); - set_size(len); - set_ulen(len); - set_flags(DB_DBT_USERMEM); - } -private: - std::unique_ptr<char[]> m_data; -}; - -struct Dbt_safe : public Dbt -{ - Dbt_safe() - { - set_data(NULL); - set_flags(DB_DBT_MALLOC); - } - ~Dbt_safe() - { - void* buf = get_data(); - if (buf != NULL) - { - free(buf); - } - } -}; - -} // anonymous namespace - -namespace cryptonote -{ - -void BlockchainBDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const crypto::hash& blk_hash) -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - Dbt_copy<crypto::hash> val_h(blk_hash); - if (m_block_heights->exists(DB_DEFAULT_TX, &val_h, 0) == 0) - throw1(BLOCK_EXISTS("Attempting to add block that's already in the db")); - - if (m_height > 0) - { - Dbt_copy<crypto::hash> parent_key(blk.prev_id); - Dbt_copy<uint32_t> parent_h; - if (m_block_heights->get(DB_DEFAULT_TX, &parent_key, &parent_h, 0)) - { - LOG_PRINT_L3("m_height: " << m_height); - LOG_PRINT_L3("parent_key: " << blk.prev_id); - throw0(DB_ERROR("Failed to get top block hash to check for new block's parent")); - } - uint32_t parent_height = parent_h; - if (parent_height != m_height) - throw0(BLOCK_PARENT_DNE("Top block is not new block's parent")); - } - - Dbt_copy<uint32_t> key(m_height + 1); - - Dbt_copy<blobdata> blob(block_to_blob(blk)); - auto res = m_blocks->put(DB_DEFAULT_TX, &key, &blob, 0); - if (res) - throw0(DB_ERROR("Failed to add block blob to db transaction.")); - - Dbt_copy<size_t> sz(block_weight); - if (m_block_sizes->put(DB_DEFAULT_TX, &key, &sz, 0)) - throw0(DB_ERROR("Failed to add block size to db transaction.")); - - Dbt_copy<uint64_t> ts(blk.timestamp); - if (m_block_timestamps->put(DB_DEFAULT_TX, &key, &ts, 0)) - throw0(DB_ERROR("Failed to add block timestamp to db transaction.")); - - Dbt_copy<difficulty_type> diff(cumulative_difficulty); - if (m_block_diffs->put(DB_DEFAULT_TX, &key, &diff, 0)) - throw0(DB_ERROR("Failed to add block cumulative difficulty to db transaction.")); - - Dbt_copy<uint64_t> coinsgen(coins_generated); - if (m_block_coins->put(DB_DEFAULT_TX, &key, &coinsgen, 0)) - throw0(DB_ERROR("Failed to add block total generated coins to db transaction.")); - - if (m_block_heights->put(DB_DEFAULT_TX, &val_h, &key, 0)) - throw0(DB_ERROR("Failed to add block height by hash to db transaction.")); - - if (m_block_hashes->put(DB_DEFAULT_TX, &key, &val_h, 0)) - throw0(DB_ERROR("Failed to add block hash to db transaction.")); -} - -void BlockchainBDB::remove_block() -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - if (m_height == 0) - throw0(BLOCK_DNE ("Attempting to remove block from an empty blockchain")); - - Dbt_copy<uint32_t> k(m_height); - Dbt_copy<crypto::hash> h; - if (m_block_hashes->get(DB_DEFAULT_TX, &k, &h, 0)) - throw1(BLOCK_DNE("Attempting to remove block that's not in the db")); - - if (m_blocks->del(DB_DEFAULT_TX, &k, 0)) - throw1(DB_ERROR("Failed to add removal of block to db transaction")); - - if (m_block_sizes->del(DB_DEFAULT_TX, &k, 0)) - throw1(DB_ERROR("Failed to add removal of block size to db transaction")); - - if (m_block_diffs->del(DB_DEFAULT_TX, &k, 0)) - throw1(DB_ERROR("Failed to add removal of block cumulative difficulty to db transaction")); - - if (m_block_coins->del(DB_DEFAULT_TX, &k, 0)) - throw1(DB_ERROR("Failed to add removal of block total generated coins to db transaction")); - - if (m_block_timestamps->del(DB_DEFAULT_TX, &k, 0)) - throw1(DB_ERROR("Failed to add removal of block timestamp to db transaction")); - - if (m_block_heights->del(DB_DEFAULT_TX, &h, 0)) - throw1(DB_ERROR("Failed to add removal of block height by hash to db transaction")); - - if (m_block_hashes->del(DB_DEFAULT_TX, &k, 0)) - throw1(DB_ERROR("Failed to add removal of block hash to db transaction")); -} - -void BlockchainBDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - Dbt_copy<crypto::hash> val_h(tx_hash); - - if (m_txs->exists(DB_DEFAULT_TX, &val_h, 0) == 0) - throw1(TX_EXISTS("Attempting to add transaction that's already in the db")); - - Dbt_copy<blobdata> blob(tx_to_blob(tx)); - if (m_txs->put(DB_DEFAULT_TX, &val_h, &blob, 0)) - throw0(DB_ERROR("Failed to add tx blob to db transaction")); - - Dbt_copy<uint64_t> height(m_height + 1); - if (m_tx_heights->put(DB_DEFAULT_TX, &val_h, &height, 0)) - throw0(DB_ERROR("Failed to add tx block height to db transaction")); - - Dbt_copy<uint64_t> unlock_time(tx.unlock_time); - if (m_tx_unlocks->put(DB_DEFAULT_TX, &val_h, &unlock_time, 0)) - throw0(DB_ERROR("Failed to add tx unlock time to db transaction")); -} - -void BlockchainBDB::remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - Dbt_copy<crypto::hash> val_h(tx_hash); - if (m_txs->exists(DB_DEFAULT_TX, &val_h, 0)) - throw1(TX_DNE("Attempting to remove transaction that isn't in the db")); - - if (m_txs->del(DB_DEFAULT_TX, &val_h, 0)) - throw1(DB_ERROR("Failed to add removal of tx to db transaction")); - if (m_tx_unlocks->del(DB_DEFAULT_TX, &val_h, 0)) - throw1(DB_ERROR("Failed to add removal of tx unlock time to db transaction")); - if (m_tx_heights->del(DB_DEFAULT_TX, &val_h, 0)) - throw1(DB_ERROR("Failed to add removal of tx block height to db transaction")); - - remove_tx_outputs(tx_hash, tx); - - auto result = m_tx_outputs->del(DB_DEFAULT_TX, &val_h, 0); - if (result == DB_NOTFOUND) - LOG_PRINT_L1("tx has no outputs to remove: " << tx_hash); - else if (result) - throw1(DB_ERROR("Failed to add removal of tx outputs to db transaction")); -} - -void BlockchainBDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time, const rct::key *commitment) -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - Dbt_copy<uint32_t> k(m_num_outputs + 1); - Dbt_copy<crypto::hash> v(tx_hash); - - if (m_output_txs->put(DB_DEFAULT_TX, &k, &v, 0)) - throw0(DB_ERROR("Failed to add output tx hash to db transaction")); - if (m_tx_outputs->put(DB_DEFAULT_TX, &v, &k, 0)) - throw0(DB_ERROR("Failed to add tx output index to db transaction")); - - Dbt_copy<uint64_t> val_local_index(local_index); - if (m_output_indices->put(DB_DEFAULT_TX, &k, &val_local_index, 0)) - throw0(DB_ERROR("Failed to add tx output index to db transaction")); - - Dbt_copy<uint64_t> val_amount(tx_output.amount); - if (m_output_amounts->put(DB_DEFAULT_TX, &val_amount, &k, 0)) - throw0(DB_ERROR("Failed to add output amount to db transaction.")); - - if (tx_output.target.type() == typeid(txout_to_key)) - { - output_data_t od; - od.pubkey = boost::get < txout_to_key > (tx_output.target).key; - od.unlock_time = unlock_time; - od.height = m_height; - - Dbt_copy<output_data_t> data(od); - if (m_output_keys->put(DB_DEFAULT_TX, &k, &data, 0)) - throw0(DB_ERROR("Failed to add output pubkey to db transaction")); - } - else - { - throw0(DB_ERROR("Wrong output type: expected txout_to_key")); - } - - m_num_outputs++; -} - -void BlockchainBDB::remove_tx_outputs(const crypto::hash& tx_hash, const transaction& tx) -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - - bdb_cur cur(DB_DEFAULT_TX, m_tx_outputs); - - Dbt_copy<crypto::hash> k(tx_hash); - Dbt_copy<uint32_t> v; - - auto result = cur->get(&k, &v, DB_SET); - if (result == DB_NOTFOUND) - { - LOG_PRINT_L2("tx has no outputs, so no global output indices"); - } - else if (result) - { - throw0(DB_ERROR("DB error attempting to get an output")); - } - else - { - result = cur->get(&k, &v, DB_NEXT_NODUP); - if (result != 0 && result != DB_NOTFOUND) - throw0(DB_ERROR("DB error attempting to get next non-duplicate tx hash")); - - if (result == 0) - result = cur->get(&k, &v, DB_PREV); - else if (result == DB_NOTFOUND) - result = cur->get(&k, &v, DB_LAST); - - db_recno_t num_elems = 0; - cur->count(&num_elems, 0); - - // remove in order: from newest to oldest - for (uint64_t i = num_elems; i > 0; --i) - { - const tx_out tx_output = tx.vout[i-1]; - remove_output(v, tx_output.amount); - if (i > 1) - { - cur->get(&k, &v, DB_PREV_DUP); - } - } - } - - cur.close(); -} - -// TODO: probably remove this function -void BlockchainBDB::remove_output(const tx_out& tx_output) -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__ << " (unused version - does nothing)"); - return; -} - -void BlockchainBDB::remove_output(const uint64_t& out_index, const uint64_t amount) -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - Dbt_copy<uint32_t> k(out_index); - - auto result = m_output_indices->del(DB_DEFAULT_TX, &k, 0); - if (result == DB_NOTFOUND) - { - LOG_PRINT_L0("Unexpected: global output index not found in m_output_indices"); - } - else if (result) - { - throw1(DB_ERROR("Error adding removal of output tx index to db transaction")); - } - - result = m_output_txs->del(DB_DEFAULT_TX, &k, 0); - // if (result != 0 && result != DB_NOTFOUND) - // throw1(DB_ERROR("Error adding removal of output tx hash to db transaction")); - if (result == DB_NOTFOUND) - { - LOG_PRINT_L0("Unexpected: global output index not found in m_output_txs"); - } - else if (result) - { - throw1(DB_ERROR("Error adding removal of output tx hash to db transaction")); - } - - result = m_output_keys->del(DB_DEFAULT_TX, &k, 0); - if (result == DB_NOTFOUND) - { - LOG_PRINT_L0("Unexpected: global output index not found in m_output_keys"); - } - else if (result) - throw1(DB_ERROR("Error adding removal of output pubkey to db transaction")); - - remove_amount_output_index(amount, out_index); - - m_num_outputs--; -} - -void BlockchainBDB::remove_amount_output_index(const uint64_t amount, const uint64_t global_output_index) -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - bdb_cur cur(DB_DEFAULT_TX, m_output_amounts); - - Dbt_copy<uint64_t> k(amount); - Dbt_copy<uint32_t> v; - - auto result = cur->get(&k, &v, DB_SET); - if (result == DB_NOTFOUND) - throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); - else if (result) - throw0(DB_ERROR("DB error attempting to get an output")); - - db_recno_t num_elems = 0; - cur->count(&num_elems, 0); - - // workaround for Berkeley DB to start at end of k's duplicate list: - // if next key exists: - // - move cursor to start of next key's duplicate list, then move back one - // duplicate element to reach the end of the original key's duplicate - // list. - // - // else if the next key doesn't exist: - // - that means we're already on the last key. - // - move cursor to last element in the db, which is the last element of - // the desired key's duplicate list. - - result = cur->get(&k, &v, DB_NEXT_NODUP); - if (result != 0 && result != DB_NOTFOUND) - throw0(DB_ERROR("DB error attempting to get next non-duplicate output amount")); - - if (result == 0) - result = cur->get(&k, &v, DB_PREV); - else if (result == DB_NOTFOUND) - result = cur->get(&k, &v, DB_LAST); - - bool found_index = false; - uint64_t amount_output_index = 0; - uint64_t goi = 0; - - for (uint64_t i = num_elems; i > 0; --i) - { - goi = v; - if (goi == global_output_index) - { - amount_output_index = i-1; - found_index = true; - break; - } - if (i > 1) - cur->get(&k, &v, DB_PREV_DUP); - } - - if (found_index) - { - // found the amount output index - // now delete it - result = cur->del(0); - if (result) - throw0(DB_ERROR(std::string("Error deleting amount output index ").append(boost::lexical_cast<std::string>(amount_output_index)).c_str())); - } - else - { - // not found - throw1(OUTPUT_DNE("Failed to find amount output index")); - } - cur.close(); -} - -void BlockchainBDB::add_spent_key(const crypto::key_image& k_image) -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - Dbt_copy<crypto::key_image> val_key(k_image); - if (m_spent_keys->exists(DB_DEFAULT_TX, &val_key, 0) == 0) - throw1(KEY_IMAGE_EXISTS("Attempting to add spent key image that's already in the db")); - - Dbt_copy<char> val('\0'); - if (m_spent_keys->put(DB_DEFAULT_TX, &val_key, &val, 0)) - throw1(DB_ERROR("Error adding spent key image to db transaction.")); -} - -void BlockchainBDB::remove_spent_key(const crypto::key_image& k_image) -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - Dbt_copy<crypto::key_image> k(k_image); - auto result = m_spent_keys->del(DB_DEFAULT_TX, &k, 0); - if (result != 0 && result != DB_NOTFOUND) - throw1(DB_ERROR("Error adding removal of key image to db transaction")); -} - -bool BlockchainBDB::for_all_key_images(std::function<bool(const crypto::key_image&)> f) const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - bdb_cur cur(DB_DEFAULT_TX, m_spent_keys); - - Dbt_copy<crypto::key_image> k; - Dbt_copy<char> v; - bool ret = true; - int result; - while ((result = cur->get(&k, &v, DB_NEXT)) == 0) - { - if (!f(k)) - { - ret = false; - break; - } - } - if (result != DB_NOTFOUND) - ret = false; - - cur.close(); - return ret; -} - -bool BlockchainBDB::for_all_blocks(std::function<bool(uint64_t, const crypto::hash&, const cryptonote::block&)> f) const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - bdb_cur cur(DB_DEFAULT_TX, m_blocks); - - Dbt_copy<uint32_t> k; - Dbt_safe v; - bool ret = true; - int result; - while ((result = cur->get(&k, &v, DB_NEXT)) == 0) - { - uint64_t height = k - 1; - blobdata bd; - bd.assign(reinterpret_cast<char*>(v.get_data()), v.get_size()); - block b; - if (!parse_and_validate_block_from_blob(bd, b)) - throw0(DB_ERROR("Failed to parse block from blob retrieved from the db")); - crypto::hash hash; - if (!get_block_hash(b, hash)) - throw0(DB_ERROR("Failed to get block hash from blob retrieved from the db")); - if (!f(height, hash, b)) - { - ret = false; - break; - } - } - if (result != DB_NOTFOUND) - ret = false; - - cur.close(); - return ret; -} - -bool BlockchainBDB::for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)> f, bool pruned) const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - bdb_cur cur(DB_DEFAULT_TX, m_txs); - - Dbt_copy<crypto::hash> k; - Dbt_safe v; - bool ret = true; - int result; - while ((result = cur->get(&k, &v, DB_NEXT)) == 0) - { - blobdata bd; - bd.assign(reinterpret_cast<char*>(v.get_data()), v.get_size()); - transaction tx; - if (!parse_and_validate_tx_from_blob(bd, tx)) - throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); - if (!f(k, tx)) - { - ret = false; - break; - } - } - if (result != DB_NOTFOUND) - ret = false; - - cur.close(); - return ret; -} - -bool BlockchainBDB::for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, size_t tx_idx)> f) const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - bdb_cur cur(DB_DEFAULT_TX, m_output_amounts); - - Dbt_copy<uint64_t> k; - Dbt_copy<uint32_t> v; - bool ret = true; - int result; - while ((result = cur->get(&k, &v, DB_NEXT)) == 0) - { - uint32_t global_index = v - 1; - tx_out_index toi = get_output_tx_and_index_from_global(global_index); - if (!f(k, toi.first, toi.second)) - { - ret = false; - break; - } - } - if (result != DB_NOTFOUND) - ret = false; - - cur.close(); - return ret; -} - -uint64_t BlockchainBDB::get_output_global_index(const uint64_t& amount, const uint64_t& index) -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - std::vector < uint64_t > offsets; - std::vector < uint64_t > global_indices; - offsets.push_back(index); - get_output_global_indices(amount, offsets, global_indices); - if (!global_indices.size()) - throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); - - return global_indices[0]; -} - -void BlockchainBDB::check_open() const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - if (!m_open) - throw0(DB_ERROR("DB operation attempted on a not-open DB instance")); -} - -BlockchainBDB::~BlockchainBDB() -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - - if (m_open) - { - close(); - } -} - -BlockchainBDB::BlockchainBDB(bool batch_transactions) : - BlockchainDB(), - m_buffer(DB_BUFFER_COUNT, DB_BUFFER_LENGTH) -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - // initialize folder to something "safe" just in case - // someone accidentally misuses this class... - m_folder = "thishsouldnotexistbecauseitisgibberish"; - m_run_checkpoint = 0; - m_batch_transactions = batch_transactions; - m_write_txn = nullptr; - m_height = 0; - - m_hardfork = nullptr; -} - -void BlockchainBDB::open(const std::string& filename, const int db_flags) -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - - if (m_open) - throw0(DB_OPEN_FAILURE("Attempted to open db, but it's already open")); - - boost::filesystem::path direc(filename); - if (boost::filesystem::exists(direc)) - { - if (!boost::filesystem::is_directory(direc)) - throw0(DB_OPEN_FAILURE("DB needs a directory path, but a file was passed")); - } - else - { - if (!boost::filesystem::create_directories(direc)) - throw0(DB_OPEN_FAILURE(std::string("Failed to create directory ").append(filename).c_str())); - } - - m_folder = filename; - - try - { - - //Create BerkeleyDB environment - m_env = new DbEnv(0); // no flags needed for DbEnv - - uint32_t db_env_open_flags = DB_CREATE | DB_INIT_MPOOL | DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_TXN | DB_RECOVER | DB_THREAD; - - // Set some default values for these parameters. - // They can be overridden using the DB_CONFIG file. - m_env->set_cachesize(0, DB_DEF_CACHESIZE, 1); - m_env->set_lk_max_locks(DB_MAX_LOCKS); - m_env->set_lk_max_lockers(DB_MAX_LOCKS); - m_env->set_lk_max_objects(DB_MAX_LOCKS); - - #ifndef __OpenBSD__ //OpenBSD's DB package is too old to support this feature - if(m_auto_remove_logs) - m_env->log_set_config(DB_LOG_AUTO_REMOVE, 1); - #endif - - // last parameter left 0, files will be created with default rw access - m_env->open(filename.c_str(), db_env_open_flags, 0); - m_env->set_flags(db_flags, 1); - - // begin transaction to init dbs - bdb_txn_safe txn; - m_env->txn_begin(NULL, txn, 0); - - // create Dbs in the environment - m_blocks = new Db(m_env, 0); - m_block_heights = new Db(m_env, 0); - m_block_hashes = new Db(m_env, 0); - m_block_timestamps = new Db(m_env, 0); - m_block_sizes = new Db(m_env, 0); - m_block_diffs = new Db(m_env, 0); - m_block_coins = new Db(m_env, 0); - - m_txs = new Db(m_env, 0); - m_tx_unlocks = new Db(m_env, 0); - m_tx_heights = new Db(m_env, 0); - m_tx_outputs = new Db(m_env, 0); - - m_output_txs = new Db(m_env, 0); - m_output_indices = new Db(m_env, 0); - m_output_amounts = new Db(m_env, 0); - m_output_keys = new Db(m_env, 0); - - m_spent_keys = new Db(m_env, 0); - - m_hf_starting_heights = new Db(m_env, 0); - m_hf_versions = new Db(m_env, 0); - - m_properties = new Db(m_env, 0); - - // Tell DB about Dbs that need duplicate support - // Note: no need to tell about sorting, - // as the default is insertion order, which we want - m_tx_outputs->set_flags(DB_DUP); - m_output_amounts->set_flags(DB_DUP); - - // Tell DB about fixed-size values. - m_block_hashes->set_re_len(sizeof(crypto::hash)); - m_block_timestamps->set_re_len(sizeof(uint64_t)); - m_block_sizes->set_re_len(sizeof(size_t)); // should really store block size as uint64_t... - m_block_diffs->set_re_len(sizeof(difficulty_type)); - m_block_coins->set_re_len(sizeof(uint64_t)); - - m_output_txs->set_re_len(sizeof(crypto::hash)); - m_output_indices->set_re_len(sizeof(uint64_t)); - m_output_keys->set_re_len(sizeof(output_data_t)); - - m_hf_starting_heights->set_re_len(sizeof(uint64_t)); - m_hf_versions->set_re_len(sizeof(uint8_t)); - - //TODO: Find out if we need to do Db::set_flags(DB_RENUMBER) - // for the RECNO databases. We shouldn't as we're only - // inserting/removing from the end, but we'll see. - - // open Dbs in the environment - // m_tx_outputs and m_output_amounts must be DB_HASH or DB_BTREE - // because they need duplicate entry support. The rest are DB_RECNO, - // as it seems that will be the most performant choice. - m_blocks->open(txn, BDB_BLOCKS, NULL, DB_RECNO, DB_CREATE, 0); - - m_block_timestamps->open(txn, BDB_BLOCK_TIMESTAMPS, NULL, DB_RECNO, DB_CREATE, 0); - m_block_heights->open(txn, BDB_BLOCK_HEIGHTS, NULL, DB_HASH, DB_CREATE, 0); - m_block_hashes->open(txn, BDB_BLOCK_HASHES, NULL, DB_RECNO, DB_CREATE, 0); - m_block_sizes->open(txn, BDB_BLOCK_SIZES, NULL, DB_RECNO, DB_CREATE, 0); - m_block_diffs->open(txn, BDB_BLOCK_DIFFS, NULL, DB_RECNO, DB_CREATE, 0); - m_block_coins->open(txn, BDB_BLOCK_COINS, NULL, DB_RECNO, DB_CREATE, 0); - - m_txs->open(txn, BDB_TXS, NULL, DB_HASH, DB_CREATE, 0); - m_tx_unlocks->open(txn, BDB_TX_UNLOCKS, NULL, DB_HASH, DB_CREATE, 0); - m_tx_heights->open(txn, BDB_TX_HEIGHTS, NULL, DB_HASH, DB_CREATE, 0); - m_tx_outputs->open(txn, BDB_TX_OUTPUTS, NULL, DB_HASH, DB_CREATE, 0); - - m_output_txs->open(txn, BDB_OUTPUT_TXS, NULL, DB_RECNO, DB_CREATE, 0); - m_output_indices->open(txn, BDB_OUTPUT_INDICES, NULL, DB_RECNO, DB_CREATE, 0); - m_output_amounts->open(txn, BDB_OUTPUT_AMOUNTS, NULL, DB_HASH, DB_CREATE, 0); - m_output_keys->open(txn, BDB_OUTPUT_KEYS, NULL, DB_RECNO, DB_CREATE, 0); - - m_spent_keys->open(txn, BDB_SPENT_KEYS, NULL, DB_HASH, DB_CREATE, 0); - - m_hf_starting_heights->open(txn, BDB_HF_STARTING_HEIGHTS, NULL, DB_RECNO, DB_CREATE, 0); - m_hf_versions->open(txn, BDB_HF_VERSIONS, NULL, DB_RECNO, DB_CREATE, 0); - - m_properties->open(txn, BDB_PROPERTIES, NULL, DB_HASH, DB_CREATE, 0); - - txn.commit(); - - DB_BTREE_STAT* stats; - - // DB_FAST_STAT can apparently cause an incorrect number of records - // to be returned. The flag should be set to 0 instead if this proves - // to be the case. - - // ND: The bug above can occur when a block is popped and the application - // exits without pushing a new block to the db. Set txn to NULL and DB_FAST_STAT - // to zero (0) for reliability. - m_blocks->stat(NULL, &stats, 0); - m_height = stats->bt_nkeys; - free(stats); - - // see above comment about DB_FAST_STAT - m_output_indices->stat(NULL, &stats, 0); - m_num_outputs = stats->bt_nkeys; - free(stats); - - // checks for compatibility - bool compatible = true; - - Dbt_copy<const char*> key("version"); - Dbt_copy<uint32_t> result; - auto get_result = m_properties->get(DB_DEFAULT_TX, &key, &result, 0); - if (get_result == 0) - { - if (result > VERSION) - { - LOG_PRINT_RED_L0("Existing BerkeleyDB database was made by a later version. We don't know how it will change yet."); - compatible = false; - } -#if VERSION > 0 - else if (result < VERSION) - { - compatible = false; - } -#endif - } - else - { - // if not found, but we're on version 0, it's fine. If the DB's empty, it's fine too. - if (VERSION > 0 && m_height > 0) - compatible = false; - } - - if (!compatible) - { - m_open = false; - LOG_PRINT_RED_L0("Existing BerkeleyDB database is incompatible with this version."); - LOG_PRINT_RED_L0("Please delete the existing database and resync."); - return; - } - - if (1 /* this can't be set readonly atm */) - { - // only write version on an empty DB - if (m_height == 0) - { - Dbt_copy<const char*> k("version"); - Dbt_copy<uint32_t> v(VERSION); - auto put_result = m_properties->put(DB_DEFAULT_TX, &k, &v, 0); - if (put_result != 0) - { - m_open = false; - LOG_PRINT_RED_L0("Failed to write version to database."); - return; - } - } - } - - // run checkpoint thread - m_run_checkpoint = true; - m_checkpoint_thread.reset(new boost::thread(&BlockchainBDB::checkpoint_worker, this)); - } - catch (const std::exception& e) - { - throw0(DB_OPEN_FAILURE(e.what())); - } - - m_open = true; -} - -void BlockchainBDB::close() -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - this->sync(); - - m_run_checkpoint = false; - m_checkpoint_thread->join(); - m_checkpoint_thread.reset(); - - // FIXME: not yet thread safe!!! Use with care. - m_open = false; - - // DB_FORCESYNC is only available on newer version of libdb. - // The libdb doc says using the DB_FORCESYNC flag to DB_ENV->close - // is "similar to calling the DB->close(0) method to close each - // database handle". So this is what we do here as a fallback. -#ifdef DB_FORCESYNC - m_env->close(DB_FORCESYNC); -#else - m_blocks->close(0); - m_block_heights->close(0); - m_block_hashes->close(0); - m_block_timestamps->close(0); - m_block_sizes->close(0); - m_block_diffs->close(0); - m_block_coins->close(0); - - m_txs->close(0); - m_tx_unlocks->close(0); - m_tx_heights->close(0); - m_tx_outputs->close(0); - - m_output_txs->close(0); - m_output_indices->close(0); - m_output_amounts->close(0); - m_output_keys->close(0); - - m_spent_keys->close(0); - - m_hf_starting_heights->close(0); - m_hf_versions->close(0); - - m_properties->close(0); - - m_env->close(0); -#endif -} - -void BlockchainBDB::sync() -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - try - { - m_blocks->sync(0); - m_block_heights->sync(0); - m_block_hashes->sync(0); - m_block_timestamps->sync(0); - m_block_sizes->sync(0); - m_block_diffs->sync(0); - m_block_coins->sync(0); - - m_txs->sync(0); - m_tx_unlocks->sync(0); - m_tx_heights->sync(0); - m_tx_outputs->sync(0); - - m_output_txs->sync(0); - m_output_indices->sync(0); - m_output_amounts->sync(0); - m_output_keys->sync(0); - - m_spent_keys->sync(0); - - if (m_hf_starting_heights != nullptr) - m_hf_starting_heights->sync(0); - if (m_hf_versions != nullptr) - m_hf_versions->sync(0); - - m_properties->sync(0); - } - catch (const std::exception& e) - { - throw0(DB_ERROR(std::string("Failed to sync database: ").append(e.what()).c_str())); - } -} - -void BlockchainBDB::reset() -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - bdb_txn_safe txn; - if (m_env->txn_begin(NULL, txn, 0)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - m_write_txn = &txn; - try - { - uint32_t count; - - m_blocks->truncate(*m_write_txn, &count, 0); - m_block_heights->truncate(*m_write_txn, &count, 0); - m_block_hashes->truncate(*m_write_txn, &count, 0); - m_block_timestamps->truncate(*m_write_txn, &count, 0); - m_block_sizes->truncate(*m_write_txn, &count, 0); - m_block_diffs->truncate(*m_write_txn, &count, 0); - m_block_coins->truncate(*m_write_txn, &count, 0); - - m_txs->truncate(*m_write_txn, &count, 0); - m_tx_unlocks->truncate(*m_write_txn, &count, 0); - m_tx_heights->truncate(*m_write_txn, &count, 0); - m_tx_outputs->truncate(*m_write_txn, &count, 0); - - m_output_txs->truncate(*m_write_txn, &count, 0); - m_output_indices->truncate(*m_write_txn, &count, 0); - m_output_amounts->truncate(*m_write_txn, &count, 0); - m_output_keys->truncate(*m_write_txn, &count, 0); - - m_spent_keys->truncate(*m_write_txn, &count, 0); - - m_hf_starting_heights->truncate(*m_write_txn, &count, 0); - m_hf_versions->truncate(*m_write_txn, &count, 0); - - m_properties->truncate(*m_write_txn, &count, 0); - } - catch (const std::exception& e) - { - throw0(DB_ERROR(std::string("Failed to reset database: ").append(e.what()).c_str())); - } - m_write_txn = NULL; -} - -std::vector<std::string> BlockchainBDB::get_filenames() const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - std::vector<std::string> filenames; - - char *fname, *dbname; - const char **pfname, **pdbname; - - pfname = (const char **)&fname; - pdbname = (const char **)&dbname; - - m_blocks->get_dbname(pfname, pdbname); - filenames.push_back(fname); - - m_block_heights->get_dbname(pfname, pdbname); - filenames.push_back(fname); - - m_block_hashes->get_dbname(pfname, pdbname); - filenames.push_back(fname); - - m_block_timestamps->get_dbname(pfname, pdbname); - filenames.push_back(fname); - - m_block_sizes->get_dbname(pfname, pdbname); - filenames.push_back(fname); - - m_block_diffs->get_dbname(pfname, pdbname); - filenames.push_back(fname); - - m_block_coins->get_dbname(pfname, pdbname); - filenames.push_back(fname); - - m_txs->get_dbname(pfname, pdbname); - filenames.push_back(fname); - - m_tx_unlocks->get_dbname(pfname, pdbname); - filenames.push_back(fname); - - m_tx_heights->get_dbname(pfname, pdbname); - filenames.push_back(fname); - - m_tx_outputs->get_dbname(pfname, pdbname); - filenames.push_back(fname); - - m_output_txs->get_dbname(pfname, pdbname); - filenames.push_back(fname); - - m_output_indices->get_dbname(pfname, pdbname); - filenames.push_back(fname); - - m_output_amounts->get_dbname(pfname, pdbname); - filenames.push_back(fname); - - m_output_keys->get_dbname(pfname, pdbname); - filenames.push_back(fname); - - m_spent_keys->get_dbname(pfname, pdbname); - filenames.push_back(fname); - - m_hf_starting_heights->get_dbname(pfname, pdbname); - filenames.push_back(fname); - - m_hf_versions->get_dbname(pfname, pdbname); - filenames.push_back(fname); - - m_properties->get_dbname(pfname, pdbname); - filenames.push_back(fname); - - std::vector<std::string> full_paths; - - for (auto& filename : filenames) - { - boost::filesystem::path p(m_folder); - p /= filename; - full_paths.push_back(p.string()); - } - - return full_paths; -} - -bool BlockchainBDB::remove_data_file(const std::string& folder) -{ - return true; -} - -std::string BlockchainBDB::get_db_name() const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - - return std::string("BerkeleyDB"); -} - -// TODO: this? -bool BlockchainBDB::lock() -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - return false; -} - -// TODO: this? -void BlockchainBDB::unlock() -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); -} - -bool BlockchainBDB::block_exists(const crypto::hash& h, uint64_t *height) const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - Dbt_copy<crypto::hash> key(h); - - auto get_result = m_block_heights->exists(DB_DEFAULT_TX, &key, 0); - if (get_result == DB_NOTFOUND) - { - LOG_PRINT_L3("Block with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); - return false; - } - else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch block index from hash")); - - if (height) - *height = get_result - 1; - - return true; -} - -block BlockchainBDB::get_block(const crypto::hash& h) const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - return get_block_from_height(get_block_height(h)); -} - -uint64_t BlockchainBDB::get_block_height(const crypto::hash& h) const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - Dbt_copy<crypto::hash> key(h); - Dbt_copy<uint32_t> result; - - auto get_result = m_block_heights->get(DB_DEFAULT_TX, &key, &result, 0); - if (get_result == DB_NOTFOUND) - throw1(BLOCK_DNE("Attempted to retrieve non-existent block height")); - else if (get_result) - throw0(DB_ERROR("Error attempting to retrieve a block height from the db")); - - return result - 1; -} - -block_header BlockchainBDB::get_block_header(const crypto::hash& h) const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - // block_header object is automatically cast from block object - return get_block(h); -} - -block BlockchainBDB::get_block_from_height(const uint64_t& height) const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - Dbt_copy<uint32_t> key(height + 1); - Dbt_safe result; - auto get_result = m_blocks->get(DB_DEFAULT_TX, &key, &result, 0); - if (get_result == DB_NOTFOUND) - { - throw0(BLOCK_DNE(std::string("Attempt to get block from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- block not in db").c_str())); - } - else if (get_result) - throw0(DB_ERROR("Error attempting to retrieve a block from the db")); - - blobdata bd; - bd.assign(reinterpret_cast<char*>(result.get_data()), result.get_size()); - - block b; - if (!parse_and_validate_block_from_blob(bd, b)) - throw0(DB_ERROR("Failed to parse block from blob retrieved from the db")); - - return b; -} - -uint64_t BlockchainBDB::get_block_timestamp(const uint64_t& height) const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - Dbt_copy<uint32_t> key(height + 1); - Dbt_copy<uint64_t> result; - auto get_result = m_block_timestamps->get(DB_DEFAULT_TX, &key, &result, 0); - if (get_result == DB_NOTFOUND) - { - throw0(BLOCK_DNE(std::string("Attempt to get timestamp from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- timestamp not in db").c_str())); - } - else if (get_result) - throw0(DB_ERROR("Error attempting to retrieve a timestamp from the db")); - - return result; -} - -uint64_t BlockchainBDB::get_top_block_timestamp() const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - // if no blocks, return 0 - if (m_height == 0) - { - return 0; - } - - return get_block_timestamp(m_height - 1); -} - -size_t BlockchainBDB::get_block_weight(const uint64_t& height) const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - Dbt_copy<uint32_t> key(height + 1); - Dbt_copy<size_t> result; - auto get_result = m_block_sizes->get(DB_DEFAULT_TX, &key, &result, 0); - if (get_result == DB_NOTFOUND) - { - throw0(BLOCK_DNE(std::string("Attempt to get block size from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- block size not in db").c_str())); - } - else if (get_result) - throw0(DB_ERROR("Error attempting to retrieve a block size from the db")); - - return result; -} - -difficulty_type BlockchainBDB::get_block_cumulative_difficulty(const uint64_t& height) const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__ << " height: " << height); - check_open(); - - Dbt_copy<uint32_t> key(height + 1); - Dbt_copy<difficulty_type> result; - auto get_result = m_block_diffs->get(DB_DEFAULT_TX, &key, &result, 0); - if (get_result == DB_NOTFOUND) - { - throw0(BLOCK_DNE(std::string("Attempt to get cumulative difficulty from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- difficulty not in db").c_str())); - } - else if (get_result) - throw0(DB_ERROR("Error attempting to retrieve a cumulative difficulty from the db")); - - return result; -} - -difficulty_type BlockchainBDB::get_block_difficulty(const uint64_t& height) const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - difficulty_type diff1 = 0; - difficulty_type diff2 = 0; - - diff1 = get_block_cumulative_difficulty(height); - if (height != 0) - { - diff2 = get_block_cumulative_difficulty(height - 1); - } - - return diff1 - diff2; -} - -uint64_t BlockchainBDB::get_block_already_generated_coins(const uint64_t& height) const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - Dbt_copy<uint32_t> key(height + 1); - Dbt_copy<uint64_t> result; - auto get_result = m_block_coins->get(DB_DEFAULT_TX, &key, &result, 0); - if (get_result == DB_NOTFOUND) - { - throw0(BLOCK_DNE(std::string("Attempt to get generated coins from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- block size not in db").c_str())); - } - else if (get_result) - throw0(DB_ERROR("Error attempting to retrieve a total generated coins from the db")); - - return result; -} - -crypto::hash BlockchainBDB::get_block_hash_from_height(const uint64_t& height) const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - Dbt_copy<uint32_t> key(height + 1); - Dbt_copy<crypto::hash> result; - auto get_result = m_block_hashes->get(DB_DEFAULT_TX, &key, &result, 0); - if (get_result == DB_NOTFOUND) - { - throw0(BLOCK_DNE(std::string("Attempt to get hash from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- hash not in db").c_str())); - } - else if (get_result) - throw0(DB_ERROR("Error attempting to retrieve a block hash from the db.")); - - return result; -} - -std::vector<block> BlockchainBDB::get_blocks_range(const uint64_t& h1, const uint64_t& h2) const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - std::vector<block> v; - - for (uint64_t height = h1; height <= h2; ++height) - { - v.push_back(get_block_from_height(height)); - } - - return v; -} - -std::vector<crypto::hash> BlockchainBDB::get_hashes_range(const uint64_t& h1, const uint64_t& h2) const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - std::vector<crypto::hash> v; - - for (uint64_t height = h1; height <= h2; ++height) - { - v.push_back(get_block_hash_from_height(height)); - } - - return v; -} - -crypto::hash BlockchainBDB::top_block_hash() const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - if (m_height > 0) - { - return get_block_hash_from_height(m_height - 1); - } - - return null_hash; -} - -block BlockchainBDB::get_top_block() const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - if (m_height > 0) - { - return get_block_from_height(m_height - 1); - } - - block b; - return b; -} - -uint64_t BlockchainBDB::height() const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - return m_height; -} - -bool BlockchainBDB::tx_exists(const crypto::hash& h) const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - Dbt_copy<crypto::hash> key(h); - - TIME_MEASURE_START(time1); - auto get_result = m_txs->exists(DB_DEFAULT_TX, &key, 0); - TIME_MEASURE_FINISH(time1); - time_tx_exists += time1; - if (get_result == DB_NOTFOUND) - { - LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db"); - return false; - } - else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch transaction from hash")); - - return true; -} - -uint64_t BlockchainBDB::get_tx_unlock_time(const crypto::hash& h) const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - Dbt_copy<crypto::hash> key(h); - Dbt_copy<uint64_t> result; - auto get_result = m_tx_unlocks->get(DB_DEFAULT_TX, &key, &result, 0); - if (get_result == DB_NOTFOUND) - throw1(TX_DNE(std::string("tx unlock time with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); - else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch tx unlock time from hash")); - - return result; -} - -transaction BlockchainBDB::get_tx(const crypto::hash& h) const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - Dbt_copy<crypto::hash> key(h); - Dbt_safe result; - auto get_result = m_txs->get(DB_DEFAULT_TX, &key, &result, 0); - if (get_result == DB_NOTFOUND) - throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); - else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch tx from hash")); - - blobdata bd; - bd.assign(reinterpret_cast<char*>(result.get_data()), result.get_size()); - - transaction tx; - if (!parse_and_validate_tx_from_blob(bd, tx)) - throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db")); - - return tx; -} - -uint64_t BlockchainBDB::get_tx_count() const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - DB_BTREE_STAT* stats; - - // DB_FAST_STAT can apparently cause an incorrect number of records - // to be returned. The flag should be set to 0 instead if this proves - // to be the case. - m_txs->stat(DB_DEFAULT_TX, &stats, DB_FAST_STAT); - auto num_txs = stats->bt_nkeys; - delete stats; - - return num_txs; -} - -std::vector<transaction> BlockchainBDB::get_tx_list(const std::vector<crypto::hash>& hlist) const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - std::vector<transaction> v; - -for (auto& h : hlist) - { - v.push_back(get_tx(h)); - } - - return v; -} - -uint64_t BlockchainBDB::get_tx_block_height(const crypto::hash& h) const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - Dbt_copy<crypto::hash> key(h); - Dbt_copy<uint64_t> result; - auto get_result = m_tx_heights->get(DB_DEFAULT_TX, &key, &result, 0); - if (get_result == DB_NOTFOUND) - { - throw1(TX_DNE(std::string("tx height with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str())); - } - else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch tx height from hash")); - - return (uint64_t)result - 1; -} - -uint64_t BlockchainBDB::get_num_outputs(const uint64_t& amount) const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - bdb_cur cur(DB_DEFAULT_TX, m_output_amounts); - - Dbt_copy<uint64_t> k(amount); - Dbt_copy<uint32_t> v; - auto result = cur->get(&k, &v, DB_SET); - if (result == DB_NOTFOUND) - { - return 0; - } - else if (result) - throw0(DB_ERROR("DB error attempting to get number of outputs of an amount")); - - db_recno_t num_elems = 0; - cur->count(&num_elems, 0); - - cur.close(); - - return num_elems; -} - -output_data_t BlockchainBDB::get_output_key(const uint64_t& global_index) const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - Dbt_copy<uint32_t> k(global_index); - Dbt_copy<output_data_t> v; - auto get_result = m_output_keys->get(DB_DEFAULT_TX, &k, &v, 0); - if (get_result == DB_NOTFOUND) - throw1(OUTPUT_DNE("Attempting to get output pubkey by global index, but key does not exist")); - else if (get_result) - throw0(DB_ERROR("Error attempting to retrieve an output pubkey from the db")); - - return v; -} - -output_data_t BlockchainBDB::get_output_key(const uint64_t& amount, const uint64_t& index) const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - uint64_t glob_index = get_output_global_index(amount, index); - return get_output_key(glob_index); -} - -tx_out_index BlockchainBDB::get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - std::vector < uint64_t > offsets; - std::vector<tx_out_index> indices; - offsets.push_back(index); - get_output_tx_and_index(amount, offsets, indices); - if (!indices.size()) - throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); - - return indices[0]; -} - -std::vector<uint64_t> BlockchainBDB::get_tx_amount_output_indices(const crypto::hash& h) const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - std::vector<uint64_t> index_vec; - std::vector<uint64_t> index_vec2; - - // get the transaction's global output indices first - index_vec = get_tx_output_indices(h); - // these are next used to obtain the amount output indices - - transaction tx = get_tx(h); - - uint64_t i = 0; - uint64_t global_index; - for (const auto& vout : tx.vout) - { - uint64_t amount = vout.amount; - - global_index = index_vec[i]; - - bdb_cur cur(DB_DEFAULT_TX, m_output_amounts); - - Dbt_copy<uint64_t> k(amount); - Dbt_copy<uint32_t> v; - - auto result = cur->get(&k, &v, DB_SET); - if (result == DB_NOTFOUND) - throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); - else if (result) - throw0(DB_ERROR("DB error attempting to get an output")); - - db_recno_t num_elems = 0; - cur->count(&num_elems, 0); - - uint64_t amount_output_index = 0; - uint64_t output_index = 0; - bool found_index = false; - for (uint64_t j = 0; j < num_elems; ++j) - { - output_index = v; - if (output_index == global_index) - { - amount_output_index = j; - found_index = true; - break; - } - cur->get(&k, &v, DB_NEXT_DUP); - } - if (found_index) - { - index_vec2.push_back(amount_output_index); - } - else - { - // not found - cur.close(); - throw1(OUTPUT_DNE("specified output not found in db")); - } - - cur.close(); - ++i; - } - - return index_vec2; -} - - -tx_out_index BlockchainBDB::get_output_tx_and_index_from_global(const uint64_t& index) const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - Dbt_copy<uint32_t> k(index); - Dbt_copy<crypto::hash > v; - - auto get_result = m_output_txs->get(DB_DEFAULT_TX, &k, &v, 0); - if (get_result == DB_NOTFOUND) - throw1(OUTPUT_DNE("output with given index not in db")); - else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch output tx hash")); - - crypto::hash tx_hash = v; - - Dbt_copy<uint64_t> result; - get_result = m_output_indices->get(DB_DEFAULT_TX, &k, &result, 0); - if (get_result == DB_NOTFOUND) - throw1(OUTPUT_DNE("output with given index not in db")); - else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch output tx index")); - - return tx_out_index(tx_hash, result); -} - -bool BlockchainBDB::has_key_image(const crypto::key_image& img) const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - Dbt_copy<crypto::key_image> val_key(img); - if (m_spent_keys->exists(DB_DEFAULT_TX, &val_key, 0) == 0) - { - return true; - } - - return false; -} - -// Ostensibly BerkeleyDB has batch transaction support built-in, -// so the following few functions will be NOP. - -bool BlockchainBDB::batch_start(uint64_t batch_num_blocks) -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - return false; -} - -void BlockchainBDB::batch_commit() -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); -} - -void BlockchainBDB::batch_stop() -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); -} - -void BlockchainBDB::batch_abort() -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); -} - -void BlockchainBDB::set_batch_transactions(bool batch_transactions) -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - m_batch_transactions = batch_transactions; - LOG_PRINT_L3("batch transactions " << (m_batch_transactions ? "enabled" : "disabled")); -} - -void BlockchainBDB::block_txn_start(bool readonly) -{ - // TODO -} - -void BlockchainBDB::block_txn_stop() -{ - // TODO -} - -void BlockchainBDB::block_txn_abort() -{ - // TODO -} - -uint64_t BlockchainBDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const std::vector<transaction>& txs) -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - bdb_txn_safe txn; - if (m_env->txn_begin(NULL, txn, 0)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - m_write_txn = &txn; - - uint64_t num_outputs = m_num_outputs; - try - { - BlockchainDB::add_block(blk, block_weight, cumulative_difficulty, coins_generated, txs); - m_write_txn = NULL; - - TIME_MEASURE_START(time1); - txn.commit(); - TIME_MEASURE_FINISH(time1); - time_commit1 += time1; - } - catch (const std::exception& e) - { - m_num_outputs = num_outputs; - m_write_txn = NULL; - throw; - } - - return ++m_height; -} - -void BlockchainBDB::pop_block(block& blk, std::vector<transaction>& txs) -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - bdb_txn_safe txn; - if (m_env->txn_begin(NULL, txn, 0)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - m_write_txn = &txn; - - uint64_t num_outputs = m_num_outputs; - try - { - BlockchainDB::pop_block(blk, txs); - - m_write_txn = NULL; - txn.commit(); - } - catch (...) - { - m_num_outputs = num_outputs; - m_write_txn = NULL; - throw; - } - - --m_height; -} - -void BlockchainBDB::get_output_tx_and_index_from_global(const std::vector<uint64_t> &global_indices, std::vector<tx_out_index> &tx_out_indices) const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - tx_out_indices.clear(); - - for (const uint64_t &index : global_indices) - { - Dbt_copy<uint32_t> k(index); - Dbt_copy<crypto::hash> v; - - auto get_result = m_output_txs->get(DB_DEFAULT_TX, &k, &v, 0); - if (get_result == DB_NOTFOUND) - throw1(OUTPUT_DNE("output with given index not in db")); - else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch output tx hash")); - - crypto::hash tx_hash = v; - - Dbt_copy<uint64_t> result; - get_result = m_output_indices->get(DB_DEFAULT_TX, &k, &result, 0); - if (get_result == DB_NOTFOUND) - throw1(OUTPUT_DNE("output with given index not in db")); - else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch output tx index")); - auto hashindex = tx_out_index(tx_hash, result); - tx_out_indices.push_back(hashindex); - } -} - -void BlockchainBDB::get_output_global_indices(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<uint64_t> &global_indices) -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - TIME_MEASURE_START(txx); - check_open(); - - bdb_cur cur(DB_DEFAULT_TX, m_output_amounts); - uint64_t max = 0; - for (const uint64_t& index : offsets) - { - if (index > max) - max = index; - } - - // get returned keypairs count -#define DB_COUNT_RECORDS(dbt, cnt) \ - do { \ - uint32_t *_p = (uint32_t *) ((uint8_t *)(dbt)->data + \ - (dbt)->ulen - sizeof(uint32_t)); \ - cnt = 0; \ - while(*_p != (uint32_t) -1) { \ - _p -= 2; \ - ++cnt; \ - } \ - } while(0); \ - - Dbt_copy<uint64_t> k(amount); - Dbt_copy<uint32_t> v; - uint64_t buflen = 0; - uint64_t t_dbmul = 0; - uint64_t t_dbscan = 0; - - auto result = cur->get(&k, &v, DB_SET); - if (result == DB_NOTFOUND) - throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); - else if (result) - throw0(DB_ERROR("DB error attempting to get an output")); - - db_recno_t num_elems = 0; - cur->count(&num_elems, 0); - - if (max <= 1 && num_elems <= max) - throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found")); - - TIME_MEASURE_START(db2); - if (max <= 1) - { - for (const uint64_t& index : offsets) - { - TIME_MEASURE_START(t_seek); - - auto result = cur->get(&k, &v, DB_SET); - if (result == DB_NOTFOUND) - throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); - else if (result) - throw0(DB_ERROR("DB error attempting to get an output")); - - for (uint64_t i = 0; i < index; ++i) - cur->get(&k, &v, DB_NEXT_DUP); - - uint64_t glob_index = v; - - LOG_PRINT_L3("L0->v: " << glob_index); - global_indices.push_back(glob_index); - - TIME_MEASURE_FINISH(t_seek); - } - } - else - { - // setup a 256KB minimum buffer size - uint32_t pagesize = 256 * 1024; - - // Retrieve only a suitable portion of the kvp data, up to somewhere near - // the maximum offset value being retrieved - buflen = (max + 1) * 4 * sizeof(uint64_t); - buflen = ((buflen / pagesize) + ((buflen % pagesize) > 0 ? 1 : 0)) * pagesize; - - bool nomem = false; - Dbt data; - - bool singlebuff = buflen <= m_buffer.get_buffer_size(); - buflen = buflen < m_buffer.get_buffer_size() ? buflen : m_buffer.get_buffer_size(); - bdb_safe_buffer_t::type buffer = nullptr; - bdb_safe_buffer_autolock<bdb_safe_buffer_t> lock(m_buffer, buffer); - - data.set_data(buffer); - data.set_ulen(buflen); - data.set_size(buflen); - data.set_flags(DB_DBT_USERMEM); - - uint32_t curcount = 0; - uint32_t blockstart = 0; - for (const uint64_t& index : offsets) - { - if (index >= num_elems) - { - LOG_PRINT_L1("Index: " << index << " Elems: " << num_elems << " partial results found for get_output_tx_and_index"); - break; - } - - // fixme! for whatever reason, the first call to DB_MULTIPLE | DB_SET does not - // retrieve the first value. - if (index <= 1 || nomem) - { - auto result = cur->get(&k, &v, DB_SET); - if (result == DB_NOTFOUND) - { - throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found")); - } - else if (result) - { - throw0(DB_ERROR("DB error attempting to get an output")); - } - - for (uint64_t i = 0; i < index; ++i) - cur->get(&k, &v, DB_NEXT_DUP); - } - else - { - while (index >= curcount) - { - TIME_MEASURE_START(t_db1); - try - { - cur->get(&k, &data, DB_MULTIPLE | (curcount == 0 ? DB_SET : DB_NEXT_DUP)); - blockstart = curcount; - - int count = 0; - DB_COUNT_RECORDS((DBT * ) &data, count); - curcount += count; - } - catch (const std::exception &e) - { - cur.close(); - throw0(DB_ERROR(std::string("Failed on DB_MULTIPLE: ").append(e.what()).c_str())); - } - - TIME_MEASURE_FINISH(t_db1); - t_dbmul += t_db1; - if (singlebuff) - break; - } - - LOG_PRINT_L3("Records returned: " << curcount << " Index: " << index); - TIME_MEASURE_START(t_db2); - DBT *pdata = (DBT *) &data; - - uint8_t *value; - uint64_t dlen = 0; - - void *pbase = ((uint8_t *) (pdata->data)) + pdata->ulen - sizeof(uint32_t); - uint32_t *p = (uint32_t *) pbase; - if (*p == (uint32_t) -1) - { - value = NULL; - } - else - { - p -= (index - blockstart) * 2; // index * 4 + 2; <- if DB_MULTIPLE_KEY - value = (uint8_t *) pdata->data + *p--; - dlen = *p--; - if (value == (uint8_t *) pdata->data) - value = NULL; - } - - if (value != NULL) - { - v = dlen == sizeof(uint64_t) ? *((uint64_t *) value) : *((uint32_t *) value); - } - TIME_MEASURE_FINISH(t_db2); - t_dbscan += t_db2; - } - - uint64_t glob_index = v; - - LOG_PRINT_L3("L1->v: " << glob_index); - global_indices.push_back(glob_index); - } - } - TIME_MEASURE_FINISH(db2); - - cur.close(); - - TIME_MEASURE_FINISH(txx); - - LOG_PRINT_L3("blen: " << buflen << " txx: " << txx << " db1: " << t_dbmul << " db2: " << t_dbscan); - -} - -void BlockchainBDB::get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs) -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - TIME_MEASURE_START(txx); - outputs.clear(); - - std::vector < uint64_t > global_indices; - get_output_global_indices(amount, offsets, global_indices); - - TIME_MEASURE_START(db3); - if (global_indices.size() > 0) - { - for (const uint64_t &index : global_indices) - { - Dbt_copy<uint32_t> k(index); - Dbt_copy<output_data_t> v; - - auto get_result = m_output_keys->get(DB_DEFAULT_TX, &k, &v, 0); - if (get_result == DB_NOTFOUND) - throw1(OUTPUT_DNE("output with given index not in db")); - else if (get_result) - throw0(DB_ERROR("DB error attempting to fetch output tx hash")); - - output_data_t data = *(output_data_t *) v.get_data(); - outputs.push_back(data); - } - } - - TIME_MEASURE_FINISH(txx); - LOG_PRINT_L3("db3: " << db3); -} - -void BlockchainBDB::get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices) -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - std::vector < uint64_t > global_indices; - get_output_global_indices(amount, offsets, global_indices); - - TIME_MEASURE_START(db3); - if (global_indices.size() > 0) - get_output_tx_and_index_from_global(global_indices, indices); - TIME_MEASURE_FINISH(db3); - - LOG_PRINT_L3("db3: " << db3); -} - -std::map<uint64_t, uint64_t>::BlockchainBDB::get_output_histogram(const std::vector<uint64_t> &amounts) const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - throw1(DB_ERROR("Not implemented.")); -} - -void BlockchainBDB::check_hard_fork_info() -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - if (m_hf_versions == nullptr) - { - LOG_PRINT_L0("hf versions DB not open, so not checking"); - return; - } - - DB_BTREE_STAT* db_stat1, * db_stat2; - - // DB_FAST_STAT can apparently cause an incorrect number of records - // to be returned. The flag should be set to 0 instead if this proves - // to be the case. - - // Set txn to NULL and DB_FAST_STAT to zero (0) for reliability. - m_blocks->stat(NULL, &db_stat1, 0); - m_hf_versions->stat(NULL, &db_stat2, 0); - if (db_stat1->bt_nkeys != db_stat2->bt_nkeys) - { - LOG_PRINT_L0("num blocks " << db_stat1->bt_nkeys << " != " << "num hf_versions " << db_stat2->bt_nkeys << " - will clear the two hard fork DBs"); - - bdb_txn_safe txn; - bdb_txn_safe* txn_ptr = &txn; - if (m_write_txn) - txn_ptr = m_write_txn; - else - { - if (m_env->txn_begin(NULL, txn, 0)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - } - - try - { - uint32_t count; - m_hf_starting_heights->truncate(*txn_ptr, &count, 0); - LOG_PRINT_L0("hf_starting_heights count: " << count); - m_hf_versions->truncate(*txn_ptr, &count, 0); - LOG_PRINT_L0("hf_versions count: " << count); - - if (!m_write_txn) - txn.commit(); - } - catch (const std::exception& e) - { - throw0(DB_ERROR(std::string("Failed to clear two hard fork DBs: ").append(e.what()).c_str())); - } - } - delete db_stat1; - delete db_stat2; -} - -void BlockchainBDB::drop_hard_fork_info() -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - bdb_txn_safe txn; - bdb_txn_safe* txn_ptr = &txn; - if (m_write_txn) - txn_ptr = m_write_txn; - else - { - if (m_env->txn_begin(NULL, txn, 0)) - throw0(DB_ERROR("Failed to create a transaction for the db")); - } - - try - { - m_hf_starting_heights->close(0); - m_hf_versions->close(0); - m_hf_starting_heights = nullptr; - m_hf_versions = nullptr; - if (m_env->dbremove(*txn_ptr, BDB_HF_STARTING_HEIGHTS, NULL, 0) != 0) - LOG_ERROR("Error removing hf_starting_heights"); - if (m_env->dbremove(*txn_ptr, BDB_HF_VERSIONS, NULL, 0) != 0) - LOG_ERROR("Error removing hf_versions"); - - if (!m_write_txn) - txn.commit(); - } - catch (const std::exception& e) - { - throw0(DB_ERROR(std::string("Failed to drop hard fork info: ").append(e.what()).c_str())); - } -} - -void BlockchainBDB::set_hard_fork_version(uint64_t height, uint8_t version) -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - Dbt_copy<uint32_t> val_key(height + 1); - Dbt_copy<uint8_t> val(version); - if (m_hf_versions->put(DB_DEFAULT_TX, &val_key, &val, 0)) - throw1(DB_ERROR("Error adding hard fork version to db transaction.")); -} - -uint8_t BlockchainBDB::get_hard_fork_version(uint64_t height) const -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - check_open(); - - Dbt_copy<uint32_t> key(height + 1); - Dbt_copy<uint8_t> result; - - auto get_result = m_hf_versions->get(DB_DEFAULT_TX, &key, &result, 0); - if (get_result == DB_NOTFOUND || get_result == DB_KEYEMPTY) - throw0(OUTPUT_DNE("Error attempting to retrieve hard fork version from the db")); - else if (get_result) - throw0(DB_ERROR("Error attempting to retrieve hard fork version from the db")); - - return result; -} - -void BlockchainBDB::checkpoint_worker() const -{ - LOG_PRINT_L0("Entering BDB checkpoint thread."); - int count = 0; - while(m_run_checkpoint && m_open) - { - // sleep every second, so we don't delay exit condition m_run_checkpoint = false - sleep(1); - // checkpoint every 5 minutes - if(count++ >= 300) - { - count = 0; - if(m_env->txn_checkpoint(0, 0, 0) != 0) - { - LOG_PRINT_L0("BDB txn_checkpoint failed."); - break; - } - } - } - LOG_PRINT_L0("Leaving BDB checkpoint thread."); -} - -bool BlockchainBDB::is_read_only() const -{ - return false; -} - -void BlockchainBDB::fixup() -{ - LOG_PRINT_L3("BlockchainBDB::" << __func__); - // Always call parent as well - BlockchainDB::fixup(); -} - -} // namespace cryptonote diff --git a/src/blockchain_db/berkeleydb/db_bdb.h b/src/blockchain_db/berkeleydb/db_bdb.h deleted file mode 100644 index 3ae90efe1..000000000 --- a/src/blockchain_db/berkeleydb/db_bdb.h +++ /dev/null @@ -1,452 +0,0 @@ -// Copyright (c) 2014-2019, The Monero Project -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without modification, are -// permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this list of -// conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright notice, this list -// of conditions and the following disclaimer in the documentation and/or other -// materials provided with the distribution. -// -// 3. Neither the name of the copyright holder nor the names of its contributors may be -// used to endorse or promote products derived from this software without specific -// prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY -// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL -// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF -// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -#include <db_cxx.h> - -#include "blockchain_db/blockchain_db.h" -#include "cryptonote_basic/blobdatatype.h" // for type blobdata - -#include <unordered_map> -#include <condition_variable> - -// ND: Enables multi-threaded bulk reads for when getting indices. -// TODO: Disabled for now, as it doesn't seem to provide noticeable improvements (??. Reason: TBD. -// #define BDB_BULK_CAN_THREAD -namespace cryptonote -{ - -struct bdb_txn_safe -{ - bdb_txn_safe() : m_txn(NULL) { } - ~bdb_txn_safe() - { - LOG_PRINT_L3("bdb_txn_safe: destructor"); - - if (m_txn != NULL) - abort(); - } - - void commit(std::string message = "") - { - if (message.size() == 0) - { - message = "Failed to commit a transaction to the db"; - } - - if (m_txn->commit(0)) - { - m_txn = NULL; - LOG_PRINT_L0(message); - throw DB_ERROR(message.c_str()); - } - m_txn = NULL; - } - - void abort() - { - LOG_PRINT_L3("bdb_txn_safe: abort()"); - if(m_txn != NULL) - { - m_txn->abort(); - m_txn = NULL; - } - else - { - LOG_PRINT_L0("WARNING: bdb_txn_safe: abort() called, but m_txn is NULL"); - } - } - - operator DbTxn*() - { - return m_txn; - } - - operator DbTxn**() - { - return &m_txn; - } -private: - DbTxn* m_txn; -}; - -// ND: Class to handle buffer management when doing bulk queries -// (DB_MULTIPLE). Allocates buffers then handles thread queuing -// so a fixed set of buffers can be used (instead of allocating -// every time a bulk query is needed). -template <typename T> -class bdb_safe_buffer -{ - // limit the number of buffers to 8 - const size_t MaxAllowedBuffers = 8; -public: - bdb_safe_buffer(size_t num_buffers, size_t count) - { - if(num_buffers > MaxAllowedBuffers) - num_buffers = MaxAllowedBuffers; - - set_count(num_buffers); - for (size_t i = 0; i < num_buffers; i++) - m_buffers.push_back((T) malloc(sizeof(T) * count)); - m_buffer_count = count; - } - - ~bdb_safe_buffer() - { - for (size_t i = 0; i < m_buffers.size(); i++) - { - if (m_buffers[i]) - { - free(m_buffers[i]); - m_buffers[i] = nullptr; - } - } - - m_buffers.resize(0); - } - - T acquire_buffer() - { - boost::unique_lock<boost::mutex> lock(m_lock); - m_cv.wait(lock, [&]{ return m_count > 0; }); - - --m_count; - size_t index = -1; - for (size_t i = 0; i < m_open_slot.size(); i++) - { - if (m_open_slot[i]) - { - m_open_slot[i] = false; - index = i; - break; - } - } - - assert(index >= 0); - - T buffer = m_buffers[index]; - m_buffer_map.emplace(buffer, index); - return buffer; - } - - void release_buffer(T buffer) - { - boost::unique_lock<boost::mutex> lock(m_lock); - - assert(buffer != nullptr); - auto it = m_buffer_map.find(buffer); - if (it != m_buffer_map.end()) - { - auto index = it->second; - - assert(index < m_open_slot.size()); - assert(m_open_slot[index] == false); - assert(m_count < m_open_slot.size()); - - ++m_count; - m_open_slot[index] = true; - m_buffer_map.erase(it); - m_cv.notify_one(); - } - } - - size_t get_buffer_size() const - { - return m_buffer_count * sizeof(T); - } - - size_t get_buffer_count() const - { - return m_buffer_count; - } - - typedef T type; - -private: - void set_count(size_t count) - { - assert(count > 0); - m_open_slot.resize(count, true); - m_count = count; - } - - std::vector<T> m_buffers; - std::unordered_map<T, size_t> m_buffer_map; - - boost::condition_variable m_cv; - std::vector<bool> m_open_slot; - size_t m_count; - boost::mutex m_lock; - - size_t m_buffer_count; -}; - -template <typename T> -class bdb_safe_buffer_autolock -{ -public: - bdb_safe_buffer_autolock(T &safe_buffer, typename T::type &buffer) : - m_safe_buffer(safe_buffer), m_buffer(nullptr) - { - m_buffer = m_safe_buffer.acquire_buffer(); - buffer = m_buffer; - } - - ~bdb_safe_buffer_autolock() - { - if (m_buffer != nullptr) - { - m_safe_buffer.release_buffer(m_buffer); - m_buffer = nullptr; - } - } -private: - T &m_safe_buffer; - typename T::type m_buffer; -}; - -class BlockchainBDB : public BlockchainDB -{ -public: - BlockchainBDB(bool batch_transactions=false); - ~BlockchainBDB(); - - virtual void open(const std::string& filename, const int db_flags); - - virtual void close(); - - virtual void sync(); - - virtual void reset(); - - virtual std::vector<std::string> get_filenames() const; - - virtual bool remove_data_file(const std::string& folder); - - virtual std::string get_db_name() const; - - virtual bool lock(); - - virtual void unlock(); - - virtual bool block_exists(const crypto::hash& h, uint64_t *height = NULL) const; - - virtual block get_block(const crypto::hash& h) const; - - virtual uint64_t get_block_height(const crypto::hash& h) const; - - virtual block_header get_block_header(const crypto::hash& h) const; - - virtual block get_block_from_height(const uint64_t& height) const; - - virtual uint64_t get_block_timestamp(const uint64_t& height) const; - - virtual uint64_t get_top_block_timestamp() const; - - virtual size_t get_block_weight(const uint64_t& height) const; - - virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const; - - virtual difficulty_type get_block_difficulty(const uint64_t& height) const; - - virtual uint64_t get_block_already_generated_coins(const uint64_t& height) const; - - virtual crypto::hash get_block_hash_from_height(const uint64_t& height) const; - - virtual std::vector<block> get_blocks_range(const uint64_t& h1, const uint64_t& h2) const; - - virtual std::vector<crypto::hash> get_hashes_range(const uint64_t& h1, const uint64_t& h2) const; - - virtual crypto::hash top_block_hash() const; - - virtual block get_top_block() const; - - virtual uint64_t height() const; - - virtual bool tx_exists(const crypto::hash& h) const; - - virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const; - - virtual transaction get_tx(const crypto::hash& h) const; - - virtual uint64_t get_tx_count() const; - - virtual std::vector<transaction> get_tx_list(const std::vector<crypto::hash>& hlist) const; - - virtual uint64_t get_tx_block_height(const crypto::hash& h) const; - - virtual uint64_t get_num_outputs(const uint64_t& amount) const; - - virtual uint64_t get_indexing_base() const { return 1; } - - virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index); - virtual void get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs); - - virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const; - virtual void get_output_tx_and_index_from_global(const std::vector<uint64_t> &global_indices, - std::vector<tx_out_index> &tx_out_indices) const; - - virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index); - virtual void get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices); - - virtual std::vector<uint64_t> get_tx_amount_output_indices(const crypto::hash& h) const; - - virtual bool has_key_image(const crypto::key_image& img) const; - - virtual uint64_t add_block( const block& blk - , size_t block_weight - , const difficulty_type& cumulative_difficulty - , const uint64_t& coins_generated - , const std::vector<transaction>& txs - ); - - virtual void set_batch_transactions(bool batch_transactions); - virtual bool batch_start(uint64_t batch_num_blocks=0); - virtual void batch_commit(); - virtual void batch_stop(); - virtual void batch_abort(); - - virtual void block_txn_start(bool readonly); - virtual void block_txn_stop(); - virtual void block_txn_abort(); - - virtual void pop_block(block& blk, std::vector<transaction>& txs); - -#if defined(BDB_BULK_CAN_THREAD) - virtual bool can_thread_bulk_indices() const { return true; } -#else - virtual bool can_thread_bulk_indices() const { return false; } -#endif - - /** - * @brief return a histogram of outputs on the blockchain - * - * @param amounts optional set of amounts to lookup - * - * @return a set of amount/instances - */ - std::map<uint64_t, uint64_t> get_output_histogram(const std::vector<uint64_t> &amounts) const; - -private: - virtual void add_block( const block& blk - , size_t block_weight - , const difficulty_type& cumulative_difficulty - , const uint64_t& coins_generated - , const crypto::hash& block_hash - ); - - virtual void remove_block(); - - virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash); - - virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx); - - virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time, const rct::key *commitment); - - virtual void remove_output(const tx_out& tx_output); - - void remove_tx_outputs(const crypto::hash& tx_hash, const transaction& tx); - - void remove_output(const uint64_t& out_index, const uint64_t amount); - void remove_amount_output_index(const uint64_t amount, const uint64_t global_output_index); - - virtual void add_spent_key(const crypto::key_image& k_image); - - virtual void remove_spent_key(const crypto::key_image& k_image); - - void get_output_global_indices(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<uint64_t> &global_indices); - - virtual bool for_all_key_images(std::function<bool(const crypto::key_image&)>) const; - virtual bool for_all_blocks(std::function<bool(uint64_t, const crypto::hash&, const cryptonote::block&)>) const; - virtual bool for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)>, bool pruned) const; - virtual bool for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, size_t tx_idx)> f) const; - - // Hard fork related storage - virtual void set_hard_fork_version(uint64_t height, uint8_t version); - virtual uint8_t get_hard_fork_version(uint64_t height) const; - virtual void check_hard_fork_info(); - virtual void drop_hard_fork_info(); - - /** - * @brief get the global index of the index-th output of the given amount - * - * @param amount the output amount - * @param index the index into the set of outputs of that amount - * - * @return the global index of the desired output - */ - uint64_t get_output_global_index(const uint64_t& amount, const uint64_t& index); - output_data_t get_output_key(const uint64_t& global_index) const; - void checkpoint_worker() const; - void check_open() const; - - virtual bool is_read_only() const; - - // - // fix up anything that may be wrong due to past bugs - virtual void fixup(); - - bool m_run_checkpoint; - std::unique_ptr<boost::thread> m_checkpoint_thread; - typedef bdb_safe_buffer<void *> bdb_safe_buffer_t; - bdb_safe_buffer_t m_buffer; - - DbEnv* m_env; - - Db* m_blocks; - Db* m_block_heights; - Db* m_block_hashes; - Db* m_block_timestamps; - Db* m_block_sizes; - Db* m_block_diffs; - Db* m_block_coins; - - Db* m_txs; - Db* m_tx_unlocks; - Db* m_tx_heights; - Db* m_tx_outputs; - - Db* m_output_txs; - Db* m_output_indices; - Db* m_output_amounts; - Db* m_output_keys; - - Db* m_spent_keys; - - Db* m_hf_starting_heights; - Db* m_hf_versions; - - Db* m_properties; - - uint64_t m_height; - uint64_t m_num_outputs; - std::string m_folder; - bdb_txn_safe *m_write_txn; - - bool m_batch_transactions; // support for batch transactions -}; - -} // namespace cryptonote diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp index 2b039f557..63ac38a88 100644 --- a/src/blockchain_db/blockchain_db.cpp +++ b/src/blockchain_db/blockchain_db.cpp @@ -35,17 +35,6 @@ #include "ringct/rctOps.h" #include "lmdb/db_lmdb.h" -#ifdef BERKELEY_DB -#include "berkeleydb/db_bdb.h" -#endif - -static const char *db_types[] = { - "lmdb", -#ifdef BERKELEY_DB - "berkeley", -#endif - NULL -}; #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "blockchain.db" @@ -55,36 +44,6 @@ using epee::string_tools::pod_to_hex; namespace cryptonote { -bool blockchain_valid_db_type(const std::string& db_type) -{ - int i; - for (i=0; db_types[i]; i++) - { - if (db_types[i] == db_type) - return true; - } - return false; -} - -std::string blockchain_db_types(const std::string& sep) -{ - int i; - std::string ret = ""; - for (i=0; db_types[i]; i++) - { - if (i) - ret += sep; - ret += db_types[i]; - } - return ret; -} - -std::string arg_db_type_description = "Specify database type, available: " + cryptonote::blockchain_db_types(", "); -const command_line::arg_descriptor<std::string> arg_db_type = { - "db-type" -, arg_db_type_description.c_str() -, DEFAULT_DB_TYPE -}; const command_line::arg_descriptor<std::string> arg_db_sync_mode = { "db-sync-mode" , "Specify sync option, using format [safe|fast|fastest]:[sync|async]:[<nblocks_per_sync>[blocks]|<nbytes_per_sync>[bytes]]." @@ -96,20 +55,13 @@ const command_line::arg_descriptor<bool> arg_db_salvage = { , false }; -BlockchainDB *new_db(const std::string& db_type) +BlockchainDB *new_db() { - if (db_type == "lmdb") - return new BlockchainLMDB(); -#if defined(BERKELEY_DB) - if (db_type == "berkeley") - return new BlockchainBDB(); -#endif - return NULL; + return new BlockchainLMDB(); } void BlockchainDB::init_options(boost::program_options::options_description& desc) { - command_line::add_arg(desc, arg_db_type); command_line::add_arg(desc, arg_db_sync_mode); command_line::add_arg(desc, arg_db_salvage); } diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index bb4de3ce6..8a6695cd8 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -102,7 +102,6 @@ namespace cryptonote /** a pair of <transaction hash, output index>, typedef for convenience */ typedef std::pair<crypto::hash, uint64_t> tx_out_index; -extern const command_line::arg_descriptor<std::string> arg_db_type; extern const command_line::arg_descriptor<std::string> arg_db_sync_mode; extern const command_line::arg_descriptor<bool, false> arg_db_salvage; @@ -1826,7 +1825,7 @@ private: class db_rtxn_guard: public db_txn_guard { public: db_rtxn_guard(BlockchainDB *db): db_txn_guard(db, true) {} }; class db_wtxn_guard: public db_txn_guard { public: db_wtxn_guard(BlockchainDB *db): db_txn_guard(db, false) {} }; -BlockchainDB *new_db(const std::string& db_type); +BlockchainDB *new_db(); } // namespace cryptonote diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 1ede7af62..760e380a9 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -595,7 +595,7 @@ bool BlockchainLMDB::need_resize(uint64_t threshold_size) const MDEBUG("Space remaining: " << mei.me_mapsize - size_used); MDEBUG("Size threshold: " << threshold_size); float resize_percent = RESIZE_PERCENT; - MDEBUG(boost::format("Percent used: %.04f Percent threshold: %.04f") % ((double)size_used/mei.me_mapsize) % resize_percent); + MDEBUG(boost::format("Percent used: %.04f Percent threshold: %.04f") % (100.*size_used/mei.me_mapsize) % (100.*resize_percent)); if (threshold_size > 0) { diff --git a/src/blockchain_utilities/blockchain_ancestry.cpp b/src/blockchain_utilities/blockchain_ancestry.cpp index a6ee0573f..db6d6f7d7 100644 --- a/src/blockchain_utilities/blockchain_ancestry.cpp +++ b/src/blockchain_utilities/blockchain_ancestry.cpp @@ -40,7 +40,6 @@ #include "cryptonote_core/cryptonote_core.h" #include "cryptonote_core/blockchain.h" #include "blockchain_db/blockchain_db.h" -#include "blockchain_db/db_types.h" #include "version.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -336,11 +335,6 @@ int main(int argc, char* argv[]) epee::string_tools::set_module_name_and_folder(argv[0]); - std::string default_db_type = "lmdb"; - - std::string available_dbs = cryptonote::blockchain_db_types(", "); - available_dbs = "available: " + available_dbs; - uint32_t log_level = 0; tools::on_startup(); @@ -350,9 +344,6 @@ int main(int argc, char* argv[]) po::options_description desc_cmd_only("Command line options"); po::options_description desc_cmd_sett("Command line options and settings options"); const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""}; - const command_line::arg_descriptor<std::string> arg_database = { - "database", available_dbs.c_str(), default_db_type - }; const command_line::arg_descriptor<std::string> arg_txid = {"txid", "Get ancestry for this txid", ""}; const command_line::arg_descriptor<std::string> arg_output = {"output", "Get ancestry for this output (amount/offset format)", ""}; const command_line::arg_descriptor<uint64_t> arg_height = {"height", "Get ancestry for all txes at this height", 0}; @@ -367,7 +358,6 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); command_line::add_arg(desc_cmd_sett, arg_log_level); - command_line::add_arg(desc_cmd_sett, arg_database); command_line::add_arg(desc_cmd_sett, arg_txid); command_line::add_arg(desc_cmd_sett, arg_output); command_line::add_arg(desc_cmd_sett, arg_height); @@ -446,13 +436,6 @@ int main(int argc, char* argv[]) } } - std::string db_type = command_line::get_arg(vm, arg_database); - if (!cryptonote::blockchain_valid_db_type(db_type)) - { - std::cerr << "Invalid database type: " << db_type << std::endl; - return 1; - } - // If we wanted to use the memory pool, we would set up a fake_core. // Use Blockchain instead of lower-level BlockchainDB for two reasons: @@ -468,13 +451,13 @@ int main(int argc, char* argv[]) std::unique_ptr<Blockchain> core_storage; tx_memory_pool m_mempool(*core_storage); core_storage.reset(new Blockchain(m_mempool)); - BlockchainDB *db = new_db(db_type); + BlockchainDB *db = new_db(); if (db == NULL) { - LOG_ERROR("Attempted to use non-existent database type: " << db_type); - throw std::runtime_error("Attempting to use non-existent database type"); + LOG_ERROR("Failed to initialize a database"); + throw std::runtime_error("Failed to initialize a database"); } - LOG_PRINT_L0("database: " << db_type); + LOG_PRINT_L0("database: LMDB"); const std::string filename = (boost::filesystem::path(opt_data_dir) / db->get_db_name()).string(); LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); diff --git a/src/blockchain_utilities/blockchain_blackball.cpp b/src/blockchain_utilities/blockchain_blackball.cpp index f824d93a6..4a5712921 100644 --- a/src/blockchain_utilities/blockchain_blackball.cpp +++ b/src/blockchain_utilities/blockchain_blackball.cpp @@ -39,7 +39,6 @@ #include "cryptonote_core/cryptonote_core.h" #include "cryptonote_core/blockchain.h" #include "blockchain_db/blockchain_db.h" -#include "blockchain_db/db_types.h" #include "wallet/ringdb.h" #include "version.h" @@ -415,6 +414,115 @@ static bool for_all_transactions(const std::string &filename, uint64_t &start_id return fret; } +static bool for_all_transactions(const std::string &filename, const uint64_t &start_idx, uint64_t &n_txes, const std::function<bool(bool, uint64_t, const cryptonote::transaction_prefix&)> &f) +{ + MDB_env *env; + MDB_dbi dbi_blocks, dbi_txs; + MDB_txn *txn; + MDB_cursor *cur_blocks, *cur_txs; + int dbr; + bool tx_active = false; + MDB_val k; + MDB_val v; + + dbr = mdb_env_create(&env); + if (dbr) throw std::runtime_error("Failed to create LDMB environment: " + std::string(mdb_strerror(dbr))); + dbr = mdb_env_set_maxdbs(env, 3); + if (dbr) throw std::runtime_error("Failed to set max env dbs: " + std::string(mdb_strerror(dbr))); + const std::string actual_filename = filename; + dbr = mdb_env_open(env, actual_filename.c_str(), 0, 0664); + if (dbr) throw std::runtime_error("Failed to open rings database file '" + + actual_filename + "': " + std::string(mdb_strerror(dbr))); + + dbr = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + if (dbr) throw std::runtime_error("Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + epee::misc_utils::auto_scope_leave_caller txn_dtor = epee::misc_utils::create_scope_leave_handler([&](){if (tx_active) mdb_txn_abort(txn);}); + tx_active = true; + + dbr = mdb_dbi_open(txn, "blocks", MDB_INTEGERKEY, &dbi_blocks); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + dbr = mdb_dbi_open(txn, "txs_pruned", MDB_INTEGERKEY, &dbi_txs); + if (dbr) throw std::runtime_error("Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr))); + + dbr = mdb_cursor_open(txn, dbi_blocks, &cur_blocks); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + dbr = mdb_cursor_open(txn, dbi_txs, &cur_txs); + if (dbr) throw std::runtime_error("Failed to create LMDB cursor: " + std::string(mdb_strerror(dbr))); + + MDB_stat stat; + dbr = mdb_stat(txn, dbi_blocks, &stat); + if (dbr) throw std::runtime_error("Failed to query txs stat: " + std::string(mdb_strerror(dbr))); + uint64_t n_blocks = stat.ms_entries; + dbr = mdb_stat(txn, dbi_txs, &stat); + if (dbr) throw std::runtime_error("Failed to query txs stat: " + std::string(mdb_strerror(dbr))); + n_txes = stat.ms_entries; + + bool fret = true; + + MDB_cursor_op op_blocks = MDB_FIRST; + MDB_cursor_op op_txs = MDB_FIRST; + uint64_t tx_idx = 0; + while (1) + { + int ret = mdb_cursor_get(cur_blocks, &k, &v, op_blocks); + op_blocks = MDB_NEXT; + if (ret == MDB_NOTFOUND) + break; + if (ret) + throw std::runtime_error("Failed to enumerate blocks: " + std::string(mdb_strerror(ret))); + + if (k.mv_size != sizeof(uint64_t)) + throw std::runtime_error("Bad key size"); + uint64_t height = *(const uint64_t*)k.mv_data; + blobdata bd; + bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size); + block b; + if (!parse_and_validate_block_from_blob(bd, b)) + throw std::runtime_error("Failed to parse block from blob retrieved from the db"); + + ret = mdb_cursor_get(cur_txs, &k, &v, op_txs); + if (ret) + throw std::runtime_error("Failed to fetch transaction " + string_tools::pod_to_hex(get_transaction_hash(b.miner_tx)) + ": " + std::string(mdb_strerror(ret))); + op_txs = MDB_NEXT; + + bool last_block = height == n_blocks - 1; + if (start_idx <= tx_idx++ && !f(last_block && b.tx_hashes.empty(), height, b.miner_tx)) + { + fret = false; + break; + } + for (size_t i = 0; i < b.tx_hashes.size(); ++i) + { + const crypto::hash& txid = b.tx_hashes[i]; + ret = mdb_cursor_get(cur_txs, &k, &v, op_txs); + if (ret) + throw std::runtime_error("Failed to fetch transaction " + string_tools::pod_to_hex(txid) + ": " + std::string(mdb_strerror(ret))); + if (start_idx <= tx_idx++) + { + cryptonote::transaction_prefix tx; + bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size); + CHECK_AND_ASSERT_MES(parse_and_validate_tx_prefix_from_blob(bd, tx), false, "Failed to parse transaction from blob"); + if (!f(last_block && i == b.tx_hashes.size() - 1, height, tx)) + { + fret = false; + break; + } + } + } + if (!fret) + break; + } + + mdb_cursor_close(cur_blocks); + mdb_cursor_close(cur_txs); + mdb_txn_commit(txn); + tx_active = false; + mdb_dbi_close(env, dbi_blocks); + mdb_dbi_close(env, dbi_txs); + mdb_env_close(env); + return fret; +} + static uint64_t find_first_diverging_transaction(const std::string &first_filename, const std::string &second_filename) { MDB_env *env[2]; @@ -1061,11 +1169,6 @@ int main(int argc, char* argv[]) epee::string_tools::set_module_name_and_folder(argv[0]); - std::string default_db_type = "lmdb"; - - std::string available_dbs = cryptonote::blockchain_db_types(", "); - available_dbs = "available: " + available_dbs; - uint32_t log_level = 0; tools::on_startup(); @@ -1079,9 +1182,6 @@ int main(int argc, char* argv[]) get_default_db_path(), }; const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""}; - const command_line::arg_descriptor<std::string> arg_database = { - "database", available_dbs.c_str(), default_db_type - }; const command_line::arg_descriptor<bool> arg_rct_only = {"rct-only", "Only work on ringCT outputs", false}; const command_line::arg_descriptor<bool> arg_check_subsets = {"check-subsets", "Check ring subsets (very expensive)", false}; const command_line::arg_descriptor<bool> arg_verbose = {"verbose", "Verbose output)", false}; @@ -1094,10 +1194,10 @@ int main(int argc, char* argv[]) const command_line::arg_descriptor<std::string> arg_extra_spent_list = {"extra-spent-list", "Optional list of known spent outputs",""}; const command_line::arg_descriptor<std::string> arg_export = {"export", "Filename to export the backball list to"}; const command_line::arg_descriptor<bool> arg_force_chain_reaction_pass = {"force-chain-reaction-pass", "Run the chain reaction pass even if no new blockchain data was processed"}; + const command_line::arg_descriptor<bool> arg_historical_stat = {"historical-stat", "Report historical stat of spent outputs for every 10000 blocks window"}; command_line::add_arg(desc_cmd_sett, arg_blackball_db_dir); command_line::add_arg(desc_cmd_sett, arg_log_level); - command_line::add_arg(desc_cmd_sett, arg_database); command_line::add_arg(desc_cmd_sett, arg_rct_only); command_line::add_arg(desc_cmd_sett, arg_check_subsets); command_line::add_arg(desc_cmd_sett, arg_verbose); @@ -1105,6 +1205,7 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_cmd_sett, arg_extra_spent_list); command_line::add_arg(desc_cmd_sett, arg_export); command_line::add_arg(desc_cmd_sett, arg_force_chain_reaction_pass); + command_line::add_arg(desc_cmd_sett, arg_historical_stat); command_line::add_arg(desc_cmd_sett, arg_inputs); command_line::add_arg(desc_cmd_only, command_line::arg_help); @@ -1145,16 +1246,11 @@ int main(int argc, char* argv[]) bool opt_check_subsets = command_line::get_arg(vm, arg_check_subsets); bool opt_verbose = command_line::get_arg(vm, arg_verbose); bool opt_force_chain_reaction_pass = command_line::get_arg(vm, arg_force_chain_reaction_pass); + bool opt_historical_stat = command_line::get_arg(vm, arg_historical_stat); std::string opt_export = command_line::get_arg(vm, arg_export); std::string extra_spent_list = command_line::get_arg(vm, arg_extra_spent_list); std::vector<std::pair<uint64_t, uint64_t>> extra_spent_outputs = extra_spent_list.empty() ? std::vector<std::pair<uint64_t, uint64_t>>() : load_outputs(extra_spent_list); - std::string db_type = command_line::get_arg(vm, arg_database); - if (!cryptonote::blockchain_valid_db_type(db_type)) - { - std::cerr << "Invalid database type: " << db_type << std::endl; - return 1; - } std::string db_sync_mode = command_line::get_arg(vm, arg_db_sync_mode); if (!parse_db_sync_mode(db_sync_mode)) @@ -1196,6 +1292,69 @@ int main(int argc, char* argv[]) MDB_cursor *cur0; open_db(inputs[0], &env0, &txn0, &cur0, &dbi0); + std::vector<output_data> work_spent; + + if (opt_historical_stat) + { + if (!start_blackballed_outputs) + { + MINFO("Spent outputs database is empty. Either you haven't run the analysis mode yet, or there is really no output marked as spent."); + goto skip_secondary_passes; + } + MDB_txn *txn; + int dbr = mdb_txn_begin(env, NULL, 0, &txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LMDB transaction: " + std::string(mdb_strerror(dbr))); + MDB_cursor *cur; + dbr = mdb_cursor_open(txn, dbi_spent, &cur); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB cursor: " + std::string(mdb_strerror(dbr))); + + const uint64_t STAT_WINDOW = 10000; + uint64_t outs_total = 0; + uint64_t outs_spent = 0; + std::unordered_map<uint64_t, uint64_t> outs_per_amount; + uint64_t start_idx = 0, n_txes; + uint64_t prev_height = 0; + for_all_transactions(inputs[0], start_idx, n_txes, [&](bool last_tx, uint64_t height, const cryptonote::transaction_prefix &tx)->bool + { + if (height != prev_height) + { + if (height % 100 == 0) std::cout << "\r" << height << ": " << (100.0f * outs_spent / outs_total) << "% ( " << outs_spent << " / " << outs_total << " ) \r" << std::flush; + if (height % STAT_WINDOW == 0) + { + uint64_t window_front = (height / STAT_WINDOW - 1) * STAT_WINDOW; + uint64_t window_back = window_front + STAT_WINDOW - 1; + LOG_PRINT_L0(window_front << "-" << window_back << ": " << (100.0f * outs_spent / outs_total) << "% ( " << outs_spent << " / " << outs_total << " )"); + outs_total = outs_spent = 0; + } + } + prev_height = height; + for (const auto &out: tx.vout) + { + ++outs_total; + CHECK_AND_ASSERT_THROW_MES(out.target.type() == typeid(txout_to_key), "Out target type is not txout_to_key: height=" + std::to_string(height)); + uint64_t out_global_index = outs_per_amount[out.amount]++; + if (is_output_spent(cur, output_data(out.amount, out_global_index))) + ++outs_spent; + } + if (last_tx) + { + uint64_t window_front = (height / STAT_WINDOW) * STAT_WINDOW; + uint64_t window_back = height; + LOG_PRINT_L0(window_front << "-" << window_back << ": " << (100.0f * outs_spent / outs_total) << "% ( " << outs_spent << " / " << outs_total << " )"); + } + if (stop_requested) + { + MINFO("Stopping scan..."); + return false; + } + return true; + }); + mdb_cursor_close(cur); + dbr = mdb_txn_commit(txn); + CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to commit txn creating/opening database: " + std::string(mdb_strerror(dbr))); + goto skip_secondary_passes; + } + if (!extra_spent_outputs.empty()) { MINFO("Adding " << extra_spent_outputs.size() << " extra spent outputs"); @@ -1432,8 +1591,6 @@ int main(int argc, char* argv[]) break; } - std::vector<output_data> work_spent; - if (stop_requested) goto skip_secondary_passes; diff --git a/src/blockchain_utilities/blockchain_depth.cpp b/src/blockchain_utilities/blockchain_depth.cpp index 8be83ee67..153f5f7c6 100644 --- a/src/blockchain_utilities/blockchain_depth.cpp +++ b/src/blockchain_utilities/blockchain_depth.cpp @@ -34,7 +34,6 @@ #include "cryptonote_core/cryptonote_core.h" #include "cryptonote_core/blockchain.h" #include "blockchain_db/blockchain_db.h" -#include "blockchain_db/db_types.h" #include "version.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -50,11 +49,6 @@ int main(int argc, char* argv[]) epee::string_tools::set_module_name_and_folder(argv[0]); - std::string default_db_type = "lmdb"; - - std::string available_dbs = cryptonote::blockchain_db_types(", "); - available_dbs = "available: " + available_dbs; - uint32_t log_level = 0; tools::on_startup(); @@ -64,9 +58,6 @@ int main(int argc, char* argv[]) po::options_description desc_cmd_only("Command line options"); po::options_description desc_cmd_sett("Command line options and settings options"); const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""}; - const command_line::arg_descriptor<std::string> arg_database = { - "database", available_dbs.c_str(), default_db_type - }; const command_line::arg_descriptor<std::string> arg_txid = {"txid", "Get min depth for this txid", ""}; const command_line::arg_descriptor<uint64_t> arg_height = {"height", "Get min depth for all txes at this height", 0}; const command_line::arg_descriptor<bool> arg_include_coinbase = {"include-coinbase", "Include coinbase in the average", false}; @@ -75,7 +66,6 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); command_line::add_arg(desc_cmd_sett, arg_log_level); - command_line::add_arg(desc_cmd_sett, arg_database); command_line::add_arg(desc_cmd_sett, arg_txid); command_line::add_arg(desc_cmd_sett, arg_height); command_line::add_arg(desc_cmd_sett, arg_include_coinbase); @@ -133,13 +123,6 @@ int main(int argc, char* argv[]) } } - std::string db_type = command_line::get_arg(vm, arg_database); - if (!cryptonote::blockchain_valid_db_type(db_type)) - { - std::cerr << "Invalid database type: " << db_type << std::endl; - return 1; - } - // If we wanted to use the memory pool, we would set up a fake_core. // Use Blockchain instead of lower-level BlockchainDB for two reasons: @@ -155,13 +138,13 @@ int main(int argc, char* argv[]) std::unique_ptr<Blockchain> core_storage; tx_memory_pool m_mempool(*core_storage); core_storage.reset(new Blockchain(m_mempool)); - BlockchainDB *db = new_db(db_type); + BlockchainDB *db = new_db(); if (db == NULL) { - LOG_ERROR("Attempted to use non-existent database type: " << db_type); - throw std::runtime_error("Attempting to use non-existent database type"); + LOG_ERROR("Failed to initialize a database"); + throw std::runtime_error("Failed to initialize a database"); } - LOG_PRINT_L0("database: " << db_type); + LOG_PRINT_L0("database: LMDB"); const std::string filename = (boost::filesystem::path(opt_data_dir) / db->get_db_name()).string(); LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); diff --git a/src/blockchain_utilities/blockchain_export.cpp b/src/blockchain_utilities/blockchain_export.cpp index 85566efca..b180f88de 100644 --- a/src/blockchain_utilities/blockchain_export.cpp +++ b/src/blockchain_utilities/blockchain_export.cpp @@ -32,7 +32,6 @@ #include "cryptonote_core/tx_pool.h" #include "cryptonote_core/cryptonote_core.h" #include "blockchain_db/blockchain_db.h" -#include "blockchain_db/db_types.h" #include "version.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -47,11 +46,6 @@ int main(int argc, char* argv[]) epee::string_tools::set_module_name_and_folder(argv[0]); - std::string default_db_type = "lmdb"; - - std::string available_dbs = cryptonote::blockchain_db_types(", "); - available_dbs = "available: " + available_dbs; - uint32_t log_level = 0; uint64_t block_stop = 0; bool blocks_dat = false; @@ -65,9 +59,6 @@ int main(int argc, char* argv[]) const command_line::arg_descriptor<std::string> arg_output_file = {"output-file", "Specify output file", "", true}; const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""}; const command_line::arg_descriptor<uint64_t> arg_block_stop = {"block-stop", "Stop at block number", block_stop}; - const command_line::arg_descriptor<std::string> arg_database = { - "database", available_dbs.c_str(), default_db_type - }; const command_line::arg_descriptor<bool> arg_blocks_dat = {"blocksdat", "Output in blocks.dat format", blocks_dat}; @@ -76,7 +67,6 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); command_line::add_arg(desc_cmd_sett, arg_log_level); - command_line::add_arg(desc_cmd_sett, arg_database); command_line::add_arg(desc_cmd_sett, arg_block_stop); command_line::add_arg(desc_cmd_sett, arg_blocks_dat); @@ -124,13 +114,6 @@ int main(int argc, char* argv[]) m_config_folder = command_line::get_arg(vm, cryptonote::arg_data_dir); - std::string db_type = command_line::get_arg(vm, arg_database); - if (!cryptonote::blockchain_valid_db_type(db_type)) - { - std::cerr << "Invalid database type: " << db_type << std::endl; - return 1; - } - if (command_line::has_arg(vm, arg_output_file)) output_file_path = boost::filesystem::path(command_line::get_arg(vm, arg_output_file)); else @@ -153,13 +136,13 @@ int main(int argc, char* argv[]) tx_memory_pool m_mempool(*core_storage); core_storage = new Blockchain(m_mempool); - BlockchainDB* db = new_db(db_type); + BlockchainDB* db = new_db(); if (db == NULL) { - LOG_ERROR("Attempted to use non-existent database type: " << db_type); - throw std::runtime_error("Attempting to use non-existent database type"); + LOG_ERROR("Failed to initialize a database"); + throw std::runtime_error("Failed to initialize a database"); } - LOG_PRINT_L0("database: " << db_type); + LOG_PRINT_L0("database: LMDB"); boost::filesystem::path folder(m_config_folder); folder /= db->get_db_name(); diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp index cb9154f29..a285c2bd0 100644 --- a/src/blockchain_utilities/blockchain_import.cpp +++ b/src/blockchain_utilities/blockchain_import.cpp @@ -42,7 +42,6 @@ #include "serialization/binary_utils.h" // dump_binary(), parse_binary() #include "serialization/json_utils.h" // dump_json() #include "include_base_utils.h" -#include "blockchain_db/db_types.h" #include "cryptonote_core/cryptonote_core.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -93,44 +92,6 @@ int get_db_flags_from_mode(const std::string& db_mode) return db_flags; } -int parse_db_arguments(const std::string& db_arg_str, std::string& db_type, int& db_flags) -{ - std::vector<std::string> db_args; - boost::split(db_args, db_arg_str, boost::is_any_of("#")); - db_type = db_args.front(); - boost::algorithm::trim(db_type); - - if (db_args.size() == 1) - { - return 0; - } - else if (db_args.size() > 2) - { - std::cerr << "unrecognized database argument format: " << db_arg_str << ENDL; - return 1; - } - - std::string db_arg_str2 = db_args[1]; - boost::split(db_args, db_arg_str2, boost::is_any_of(",")); - - // optionally use a composite mode instead of individual flags - const std::unordered_set<std::string> db_modes {"safe", "fast", "fastest"}; - std::string db_mode; - if (db_args.size() == 1) - { - if (db_modes.count(db_args[0]) > 0) - { - db_mode = db_args[0]; - } - } - if (! db_mode.empty()) - { - db_flags = get_db_flags_from_mode(db_mode); - } - return 0; -} - - int pop_blocks(cryptonote::core& core, int num_blocks) { bool use_batch = opt_batch; @@ -225,7 +186,7 @@ int check_flush(cryptonote::core &core, std::vector<block_complete_entry> &block // process block - block_verification_context bvc = boost::value_initialized<block_verification_context>(); + block_verification_context bvc = {}; core.handle_incoming_block(block_entry.block, pblocks.empty() ? NULL : &pblocks[blockidx++], bvc, false); // <--- process block @@ -594,11 +555,6 @@ int main(int argc, char* argv[]) epee::string_tools::set_module_name_and_folder(argv[0]); - std::string default_db_type = "lmdb"; - - std::string available_dbs = cryptonote::blockchain_db_types(", "); - available_dbs = "available: " + available_dbs; - uint32_t log_level = 0; uint64_t num_blocks = 0; uint64_t block_stop = 0; @@ -622,9 +578,6 @@ int main(int argc, char* argv[]) , "Count blocks in bootstrap file and exit" , false }; - const command_line::arg_descriptor<std::string> arg_database = { - "database", available_dbs.c_str(), default_db_type - }; const command_line::arg_descriptor<bool> arg_noverify = {"dangerous-unverified-import", "Blindly trust the import file and use potentially malicious blocks and transactions during import (only enable if you exported the file yourself)", false}; const command_line::arg_descriptor<bool> arg_batch = {"batch", @@ -634,7 +587,6 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_cmd_sett, arg_input_file); command_line::add_arg(desc_cmd_sett, arg_log_level); - command_line::add_arg(desc_cmd_sett, arg_database); command_line::add_arg(desc_cmd_sett, arg_batch_size); command_line::add_arg(desc_cmd_sett, arg_block_stop); @@ -709,7 +661,6 @@ int main(int argc, char* argv[]) return 1; } m_config_folder = command_line::get_arg(vm, cryptonote::arg_data_dir); - db_arg_str = command_line::get_arg(vm, arg_database); mlog_configure(mlog_get_default_log_path("monero-blockchain-import.log"), true); if (!command_line::is_arg_defaulted(vm, arg_log_level)) @@ -735,25 +686,7 @@ int main(int argc, char* argv[]) return 0; } - - std::string db_type; - int db_flags = 0; - int res = 0; - res = parse_db_arguments(db_arg_str, db_type, db_flags); - if (res) - { - std::cerr << "Error parsing database argument(s)" << ENDL; - return 1; - } - - if (!cryptonote::blockchain_valid_db_type(db_type)) - { - std::cerr << "Invalid database type: " << db_type << std::endl; - return 1; - } - - MINFO("database: " << db_type); - MINFO("database flags: " << db_flags); + MINFO("database: LMDB"); MINFO("verify: " << std::boolalpha << opt_verify << std::noboolalpha); if (opt_batch) { diff --git a/src/blockchain_utilities/blockchain_prune.cpp b/src/blockchain_utilities/blockchain_prune.cpp index 8e13f2c04..9a9d58c46 100644 --- a/src/blockchain_utilities/blockchain_prune.cpp +++ b/src/blockchain_utilities/blockchain_prune.cpp @@ -35,7 +35,6 @@ #include "cryptonote_core/blockchain.h" #include "blockchain_db/blockchain_db.h" #include "blockchain_db/lmdb/db_lmdb.h" -#include "blockchain_db/db_types.h" #include "version.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -441,11 +440,6 @@ int main(int argc, char* argv[]) epee::string_tools::set_module_name_and_folder(argv[0]); - std::string default_db_type = "lmdb"; - - std::string available_dbs = cryptonote::blockchain_db_types(", "); - available_dbs = "available: " + available_dbs; - uint32_t log_level = 0; tools::on_startup(); @@ -455,9 +449,6 @@ int main(int argc, char* argv[]) po::options_description desc_cmd_only("Command line options"); po::options_description desc_cmd_sett("Command line options and settings options"); const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""}; - const command_line::arg_descriptor<std::string> arg_database = { - "database", available_dbs.c_str(), default_db_type - }; const command_line::arg_descriptor<std::string> arg_db_sync_mode = { "db-sync-mode" , "Specify sync option, using format [safe|fast|fastest]:[nrecords_per_sync]." @@ -469,7 +460,6 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); command_line::add_arg(desc_cmd_sett, arg_log_level); - command_line::add_arg(desc_cmd_sett, arg_database); command_line::add_arg(desc_cmd_sett, arg_db_sync_mode); command_line::add_arg(desc_cmd_sett, arg_copy_pruned_database); command_line::add_arg(desc_cmd_only, command_line::arg_help); @@ -511,18 +501,6 @@ int main(int argc, char* argv[]) while (boost::ends_with(data_dir, "/") || boost::ends_with(data_dir, "\\")) data_dir.pop_back(); - std::string db_type = command_line::get_arg(vm, arg_database); - if (!cryptonote::blockchain_valid_db_type(db_type)) - { - MERROR("Invalid database type: " << db_type); - return 1; - } - if (db_type != "lmdb") - { - MERROR("Unsupported database type: " << db_type << ". Only lmdb is supported"); - return 1; - } - std::string db_sync_mode = command_line::get_arg(vm, arg_db_sync_mode); uint64_t db_flags = 0; if (!parse_db_sync_mode(db_sync_mode, db_flags)) @@ -552,13 +530,12 @@ int main(int argc, char* argv[]) { core_storage[n].reset(new Blockchain(m_mempool)); - BlockchainDB* db = new_db(db_type); + BlockchainDB* db = new_db(); if (db == NULL) { - MERROR("Attempted to use non-existent database type: " << db_type); - throw std::runtime_error("Attempting to use non-existent database type"); + MERROR("Failed to initialize a database"); + throw std::runtime_error("Failed to initialize a database"); } - MDEBUG("database: " << db_type); if (n == 1) { diff --git a/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp b/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp index 2d49b6ecd..cee24d4da 100644 --- a/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp +++ b/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp @@ -33,7 +33,6 @@ #include "cryptonote_core/cryptonote_core.h" #include "cryptonote_core/blockchain.h" #include "blockchain_db/blockchain_db.h" -#include "blockchain_db/db_types.h" #include "version.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -102,11 +101,6 @@ int main(int argc, char* argv[]) epee::string_tools::set_module_name_and_folder(argv[0]); - std::string default_db_type = "lmdb"; - - std::string available_dbs = cryptonote::blockchain_db_types(", "); - available_dbs = "available: " + available_dbs; - uint32_t log_level = 0; tools::on_startup(); @@ -114,9 +108,6 @@ int main(int argc, char* argv[]) po::options_description desc_cmd_only("Command line options"); po::options_description desc_cmd_sett("Command line options and settings options"); const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""}; - const command_line::arg_descriptor<std::string> arg_database = { - "database", available_dbs.c_str(), default_db_type - }; const command_line::arg_descriptor<bool> arg_verbose = {"verbose", "Verbose output", false}; const command_line::arg_descriptor<bool> arg_dry_run = {"dry-run", "Do not actually prune", false}; const command_line::arg_descriptor<std::string> arg_input = {"input", "Path to the known spent outputs file"}; @@ -125,7 +116,6 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); command_line::add_arg(desc_cmd_sett, arg_log_level); - command_line::add_arg(desc_cmd_sett, arg_database); command_line::add_arg(desc_cmd_sett, arg_verbose); command_line::add_arg(desc_cmd_sett, arg_dry_run); command_line::add_arg(desc_cmd_sett, arg_input); @@ -167,26 +157,18 @@ int main(int argc, char* argv[]) bool opt_verbose = command_line::get_arg(vm, arg_verbose); bool opt_dry_run = command_line::get_arg(vm, arg_dry_run); - std::string db_type = command_line::get_arg(vm, arg_database); - if (!cryptonote::blockchain_valid_db_type(db_type)) - { - std::cerr << "Invalid database type: " << db_type << std::endl; - return 1; - } - const std::string input = command_line::get_arg(vm, arg_input); LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); std::unique_ptr<Blockchain> core_storage; tx_memory_pool m_mempool(*core_storage); core_storage.reset(new Blockchain(m_mempool)); - BlockchainDB *db = new_db(db_type); + BlockchainDB *db = new_db(); if (db == NULL) { - LOG_ERROR("Attempted to use non-existent database type: " << db_type); - throw std::runtime_error("Attempting to use non-existent database type"); + LOG_ERROR("Failed to initialize a database"); + throw std::runtime_error("Failed to initialize a database"); } - LOG_PRINT_L0("database: " << db_type); const std::string filename = (boost::filesystem::path(opt_data_dir) / db->get_db_name()).string(); LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); diff --git a/src/blockchain_utilities/blockchain_stats.cpp b/src/blockchain_utilities/blockchain_stats.cpp index 33c26277e..2f66d54aa 100644 --- a/src/blockchain_utilities/blockchain_stats.cpp +++ b/src/blockchain_utilities/blockchain_stats.cpp @@ -34,7 +34,6 @@ #include "cryptonote_core/cryptonote_core.h" #include "cryptonote_core/blockchain.h" #include "blockchain_db/blockchain_db.h" -#include "blockchain_db/db_types.h" #include "version.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -52,11 +51,6 @@ int main(int argc, char* argv[]) epee::string_tools::set_module_name_and_folder(argv[0]); - std::string default_db_type = "lmdb"; - - std::string available_dbs = cryptonote::blockchain_db_types(", "); - available_dbs = "available: " + available_dbs; - uint32_t log_level = 0; uint64_t block_start = 0; uint64_t block_stop = 0; @@ -68,9 +62,6 @@ int main(int argc, char* argv[]) po::options_description desc_cmd_only("Command line options"); po::options_description desc_cmd_sett("Command line options and settings options"); const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""}; - const command_line::arg_descriptor<std::string> arg_database = { - "database", available_dbs.c_str(), default_db_type - }; const command_line::arg_descriptor<uint64_t> arg_block_start = {"block-start", "start at block number", block_start}; const command_line::arg_descriptor<uint64_t> arg_block_stop = {"block-stop", "Stop at block number", block_stop}; const command_line::arg_descriptor<bool> arg_inputs = {"with-inputs", "with input stats", false}; @@ -82,7 +73,6 @@ int main(int argc, char* argv[]) command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); command_line::add_arg(desc_cmd_sett, arg_log_level); - command_line::add_arg(desc_cmd_sett, arg_database); command_line::add_arg(desc_cmd_sett, arg_block_start); command_line::add_arg(desc_cmd_sett, arg_block_stop); command_line::add_arg(desc_cmd_sett, arg_inputs); @@ -131,24 +121,16 @@ int main(int argc, char* argv[]) bool do_ringsize = command_line::get_arg(vm, arg_ringsize); bool do_hours = command_line::get_arg(vm, arg_hours); - std::string db_type = command_line::get_arg(vm, arg_database); - if (!cryptonote::blockchain_valid_db_type(db_type)) - { - std::cerr << "Invalid database type: " << db_type << std::endl; - return 1; - } - LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); std::unique_ptr<Blockchain> core_storage; tx_memory_pool m_mempool(*core_storage); core_storage.reset(new Blockchain(m_mempool)); - BlockchainDB *db = new_db(db_type); + BlockchainDB *db = new_db(); if (db == NULL) { - LOG_ERROR("Attempted to use non-existent database type: " << db_type); - throw std::runtime_error("Attempting to use non-existent database type"); + LOG_ERROR("Failed to initialize a database"); + throw std::runtime_error("Failed to initialize a database"); } - LOG_PRINT_L0("database: " << db_type); const std::string filename = (boost::filesystem::path(opt_data_dir) / db->get_db_name()).string(); LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); diff --git a/src/blockchain_utilities/blockchain_usage.cpp b/src/blockchain_utilities/blockchain_usage.cpp index bd73350b3..2fa56452b 100644 --- a/src/blockchain_utilities/blockchain_usage.cpp +++ b/src/blockchain_utilities/blockchain_usage.cpp @@ -34,7 +34,6 @@ #include "cryptonote_core/cryptonote_core.h" #include "cryptonote_core/blockchain.h" #include "blockchain_db/blockchain_db.h" -#include "blockchain_db/db_types.h" #include "version.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -82,11 +81,6 @@ int main(int argc, char* argv[]) epee::string_tools::set_module_name_and_folder(argv[0]); - std::string default_db_type = "lmdb"; - - std::string available_dbs = cryptonote::blockchain_db_types(", "); - available_dbs = "available: " + available_dbs; - uint32_t log_level = 0; tools::on_startup(); @@ -96,16 +90,12 @@ int main(int argc, char* argv[]) po::options_description desc_cmd_only("Command line options"); po::options_description desc_cmd_sett("Command line options and settings options"); const command_line::arg_descriptor<std::string> arg_log_level = {"log-level", "0-4 or categories", ""}; - const command_line::arg_descriptor<std::string> arg_database = { - "database", available_dbs.c_str(), default_db_type - }; const command_line::arg_descriptor<bool> arg_rct_only = {"rct-only", "Only work on ringCT outputs", false}; const command_line::arg_descriptor<std::string> arg_input = {"input", ""}; command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); command_line::add_arg(desc_cmd_sett, arg_log_level); - command_line::add_arg(desc_cmd_sett, arg_database); command_line::add_arg(desc_cmd_sett, arg_rct_only); command_line::add_arg(desc_cmd_sett, arg_input); command_line::add_arg(desc_cmd_only, command_line::arg_help); @@ -147,13 +137,6 @@ int main(int argc, char* argv[]) network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; bool opt_rct_only = command_line::get_arg(vm, arg_rct_only); - std::string db_type = command_line::get_arg(vm, arg_database); - if (!cryptonote::blockchain_valid_db_type(db_type)) - { - std::cerr << "Invalid database type: " << db_type << std::endl; - return 1; - } - // If we wanted to use the memory pool, we would set up a fake_core. // Use Blockchain instead of lower-level BlockchainDB for two reasons: @@ -170,13 +153,13 @@ int main(int argc, char* argv[]) std::unique_ptr<Blockchain> core_storage; tx_memory_pool m_mempool(*core_storage); core_storage.reset(new Blockchain(m_mempool)); - BlockchainDB* db = new_db(db_type); + BlockchainDB* db = new_db(); if (db == NULL) { - LOG_ERROR("Attempted to use non-existent database type: " << db_type); - throw std::runtime_error("Attempting to use non-existent database type"); + LOG_ERROR("Failed to initialize a database"); + throw std::runtime_error("Failed to initialize a database"); } - LOG_PRINT_L0("database: " << db_type); + LOG_PRINT_L0("database: LMDB"); const std::string filename = input; LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); diff --git a/src/common/perf_timer.cpp b/src/common/perf_timer.cpp index 189eb85eb..4408170d1 100644 --- a/src/common/perf_timer.cpp +++ b/src/common/perf_timer.cpp @@ -34,7 +34,7 @@ #define MONERO_DEFAULT_LOG_CATEGORY "perf" #define PERF_LOG_ALWAYS(level, cat, x) \ - el::base::Writer(level, __FILE__, __LINE__, ELPP_FUNC, el::base::DispatchAction::FileOnlyLog).construct(cat) << x + el::base::Writer(level, el::Color::Default, __FILE__, __LINE__, ELPP_FUNC, el::base::DispatchAction::FileOnlyLog).construct(cat) << x #define PERF_LOG(level, cat, x) \ do { \ if (ELPP->vRegistry()->allowed(level, cat)) PERF_LOG_ALWAYS(level, cat, x); \ diff --git a/src/crypto/hash.h b/src/crypto/hash.h index 17071923d..27184fa53 100644 --- a/src/crypto/hash.h +++ b/src/crypto/hash.h @@ -32,7 +32,6 @@ #include <stddef.h> #include <iostream> -#include <boost/utility/value_init.hpp> #include "common/pod-class.h" #include "generic-ops.h" @@ -90,8 +89,8 @@ namespace crypto { epee::to_hex::formatted(o, epee::as_byte_span(v)); return o; } - const static crypto::hash null_hash = boost::value_initialized<crypto::hash>(); - const static crypto::hash8 null_hash8 = boost::value_initialized<crypto::hash8>(); + constexpr static crypto::hash null_hash = {}; + constexpr static crypto::hash8 null_hash8 = {}; } CRYPTO_MAKE_HASHABLE(hash) diff --git a/src/crypto/slow-hash.c b/src/crypto/slow-hash.c index 1fa819b57..647471513 100644 --- a/src/crypto/slow-hash.c +++ b/src/crypto/slow-hash.c @@ -136,8 +136,8 @@ static inline int use_v4_jit(void) { \ U64(b)[2] = state.hs.w[8] ^ state.hs.w[10]; \ U64(b)[3] = state.hs.w[9] ^ state.hs.w[11]; \ - division_result = state.hs.w[12]; \ - sqrt_result = state.hs.w[13]; \ + division_result = SWAP64LE(state.hs.w[12]); \ + sqrt_result = SWAP64LE(state.hs.w[13]); \ } while (0) #define VARIANT2_PORTABLE_INIT() \ @@ -210,7 +210,7 @@ static inline int use_v4_jit(void) uint64_t b0[2]; \ memcpy_swap64le(b0, b, 2); \ chunk2[0] = SWAP64LE(chunk1_old[0] + b0[0]); \ - chunk2[1] = SWAP64LE(SWAP64LE(chunk1_old[1]) + b0[1]); \ + chunk2[1] = SWAP64LE(chunk1_old[1] + b0[1]); \ if (variant >= 4) \ { \ uint64_t out_copy[2]; \ diff --git a/src/cryptonote_basic/difficulty.h b/src/cryptonote_basic/difficulty.h index 02ed89e5a..771deb04c 100644 --- a/src/cryptonote_basic/difficulty.h +++ b/src/cryptonote_basic/difficulty.h @@ -34,7 +34,6 @@ #include <vector> #include <string> #include <boost/multiprecision/cpp_int.hpp> - #include "crypto/hash.h" namespace cryptonote diff --git a/src/cryptonote_basic/hardfork.cpp b/src/cryptonote_basic/hardfork.cpp index 98158a513..dfeca27b4 100644 --- a/src/cryptonote_basic/hardfork.cpp +++ b/src/cryptonote_basic/hardfork.cpp @@ -87,7 +87,7 @@ bool HardFork::add_fork(uint8_t version, uint64_t height, uint8_t threshold, tim } if (threshold > 100) return false; - heights.push_back(Params(version, height, threshold, time)); + heights.push_back(hardfork_t(version, height, threshold, time)); return true; } @@ -171,7 +171,7 @@ void HardFork::init() // add a placeholder for the default version, to avoid special cases if (heights.empty()) - heights.push_back(Params(original_version, 0, 0, 0)); + heights.push_back(hardfork_t(original_version, 0, 0, 0)); versions.clear(); for (size_t n = 0; n < 256; ++n) diff --git a/src/cryptonote_basic/hardfork.h b/src/cryptonote_basic/hardfork.h index 123978b12..987dcc75a 100644 --- a/src/cryptonote_basic/hardfork.h +++ b/src/cryptonote_basic/hardfork.h @@ -29,6 +29,7 @@ #pragma once #include "syncobj.h" +#include "hardforks/hardforks.h" #include "cryptonote_basic/cryptonote_basic.h" namespace cryptonote @@ -230,14 +231,6 @@ namespace cryptonote */ uint64_t get_window_size() const { return window_size; } - struct Params { - uint8_t version; - uint8_t threshold; - uint64_t height; - time_t time; - Params(uint8_t version, uint64_t height, uint8_t threshold, time_t time): version(version), threshold(threshold), height(height), time(time) {} - }; - private: uint8_t get_block_version(uint64_t height) const; @@ -262,7 +255,7 @@ namespace cryptonote uint8_t original_version; uint64_t original_version_till_height; - std::vector<Params> heights; + std::vector<hardfork_t> heights; std::deque<uint8_t> versions; /* rolling window of the last N blocks' versions */ unsigned int last_versions[256]; /* count of the block versions in the last N blocks */ diff --git a/src/cryptonote_basic/miner.cpp b/src/cryptonote_basic/miner.cpp index 2dad2795e..856cccdeb 100644 --- a/src/cryptonote_basic/miner.cpp +++ b/src/cryptonote_basic/miner.cpp @@ -30,7 +30,6 @@ #include <sstream> #include <numeric> -#include <boost/utility/value_init.hpp> #include <boost/interprocess/detail/atomic.hpp> #include <boost/algorithm/string.hpp> #include "misc_language.h" @@ -100,7 +99,7 @@ namespace cryptonote miner::miner(i_miner_handler* phandler):m_stop(1), - m_template(boost::value_initialized<block>()), + m_template{}, m_template_no(0), m_diffic(0), m_thread_index(0), diff --git a/src/cryptonote_config.h b/src/cryptonote_config.h index 173b454f6..53c487d02 100644 --- a/src/cryptonote_config.h +++ b/src/cryptonote_config.h @@ -160,6 +160,10 @@ #define HF_VERSION_SMALLER_BP 10 #define HF_VERSION_LONG_TERM_BLOCK_WEIGHT 10 #define HF_VERSION_MIN_2_OUTPUTS 12 +#define HF_VERSION_MIN_V2_COINBASE_TX 12 +#define HF_VERSION_SAME_MIXIN 12 +#define HF_VERSION_REJECT_SIGS_IN_COINBASE 12 +#define HF_VERSION_ENFORCE_MIN_AGE 12 #define PER_KB_FEE_QUANTIZATION_DECIMALS 8 diff --git a/src/cryptonote_core/CMakeLists.txt b/src/cryptonote_core/CMakeLists.txt index 2cbe89b01..cb3875878 100644 --- a/src/cryptonote_core/CMakeLists.txt +++ b/src/cryptonote_core/CMakeLists.txt @@ -58,6 +58,7 @@ target_link_libraries(cryptonote_core multisig ringct device + hardforks ${Boost_DATE_TIME_LIBRARY} ${Boost_PROGRAM_OPTIONS_LIBRARY} ${Boost_SERIALIZATION_LIBRARY} diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 8ed0e526a..5cf4952ae 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -41,6 +41,7 @@ #include "cryptonote_basic/cryptonote_boost_serialization.h" #include "cryptonote_config.h" #include "cryptonote_basic/miner.h" +#include "hardforks/hardforks.h" #include "misc_language.h" #include "profile_tools.h" #include "file_io_utils.h" @@ -83,95 +84,6 @@ DISABLE_VS_WARNINGS(4267) // used to overestimate the block reward when estimating a per kB to use #define BLOCK_REWARD_OVERESTIMATE (10 * 1000000000000) -static const struct { - uint8_t version; - uint64_t height; - uint8_t threshold; - time_t time; -} mainnet_hard_forks[] = { - // version 1 from the start of the blockchain - { 1, 1, 0, 1341378000 }, - - // version 2 starts from block 1009827, which is on or around the 20th of March, 2016. Fork time finalised on 2015-09-20. No fork voting occurs for the v2 fork. - { 2, 1009827, 0, 1442763710 }, - - // version 3 starts from block 1141317, which is on or around the 24th of September, 2016. Fork time finalised on 2016-03-21. - { 3, 1141317, 0, 1458558528 }, - - // version 4 starts from block 1220516, which is on or around the 5th of January, 2017. Fork time finalised on 2016-09-18. - { 4, 1220516, 0, 1483574400 }, - - // version 5 starts from block 1288616, which is on or around the 15th of April, 2017. Fork time finalised on 2017-03-14. - { 5, 1288616, 0, 1489520158 }, - - // version 6 starts from block 1400000, which is on or around the 16th of September, 2017. Fork time finalised on 2017-08-18. - { 6, 1400000, 0, 1503046577 }, - - // version 7 starts from block 1546000, which is on or around the 6th of April, 2018. Fork time finalised on 2018-03-17. - { 7, 1546000, 0, 1521303150 }, - - // version 8 starts from block 1685555, which is on or around the 18th of October, 2018. Fork time finalised on 2018-09-02. - { 8, 1685555, 0, 1535889547 }, - - // version 9 starts from block 1686275, which is on or around the 19th of October, 2018. Fork time finalised on 2018-09-02. - { 9, 1686275, 0, 1535889548 }, - - // version 10 starts from block 1788000, which is on or around the 9th of March, 2019. Fork time finalised on 2019-02-10. - { 10, 1788000, 0, 1549792439 }, - - // version 11 starts from block 1788720, which is on or around the 10th of March, 2019. Fork time finalised on 2019-02-15. - { 11, 1788720, 0, 1550225678 }, -}; -static const uint64_t mainnet_hard_fork_version_1_till = 1009826; - -static const struct { - uint8_t version; - uint64_t height; - uint8_t threshold; - time_t time; -} testnet_hard_forks[] = { - // version 1 from the start of the blockchain - { 1, 1, 0, 1341378000 }, - - // version 2 starts from block 624634, which is on or around the 23rd of November, 2015. Fork time finalised on 2015-11-20. No fork voting occurs for the v2 fork. - { 2, 624634, 0, 1445355000 }, - - // versions 3-5 were passed in rapid succession from September 18th, 2016 - { 3, 800500, 0, 1472415034 }, - { 4, 801219, 0, 1472415035 }, - { 5, 802660, 0, 1472415036 + 86400*180 }, // add 5 months on testnet to shut the update warning up since there's a large gap to v6 - - { 6, 971400, 0, 1501709789 }, - { 7, 1057027, 0, 1512211236 }, - { 8, 1057058, 0, 1533211200 }, - { 9, 1057778, 0, 1533297600 }, - { 10, 1154318, 0, 1550153694 }, - { 11, 1155038, 0, 1550225678 }, -}; -static const uint64_t testnet_hard_fork_version_1_till = 624633; - -static const struct { - uint8_t version; - uint64_t height; - uint8_t threshold; - time_t time; -} stagenet_hard_forks[] = { - // version 1 from the start of the blockchain - { 1, 1, 0, 1341378000 }, - - // versions 2-7 in rapid succession from March 13th, 2018 - { 2, 32000, 0, 1521000000 }, - { 3, 33000, 0, 1521120000 }, - { 4, 34000, 0, 1521240000 }, - { 5, 35000, 0, 1521360000 }, - { 6, 36000, 0, 1521480000 }, - { 7, 37000, 0, 1521600000 }, - { 8, 176456, 0, 1537821770 }, - { 9, 177176, 0, 1537821771 }, - { 10, 269000, 0, 1550153694 }, - { 11, 269720, 0, 1550225678 }, -}; - //------------------------------------------------------------------ Blockchain::Blockchain(tx_memory_pool& tx_pool) : m_db(), m_tx_pool(tx_pool), m_hardfork(NULL), m_timestamps_and_difficulties_height(0), m_current_block_cumul_weight_limit(0), m_current_block_cumul_weight_median(0), @@ -403,17 +315,17 @@ bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline } else if (m_nettype == TESTNET) { - for (size_t n = 0; n < sizeof(testnet_hard_forks) / sizeof(testnet_hard_forks[0]); ++n) + for (size_t n = 0; n < num_testnet_hard_forks; ++n) m_hardfork->add_fork(testnet_hard_forks[n].version, testnet_hard_forks[n].height, testnet_hard_forks[n].threshold, testnet_hard_forks[n].time); } else if (m_nettype == STAGENET) { - for (size_t n = 0; n < sizeof(stagenet_hard_forks) / sizeof(stagenet_hard_forks[0]); ++n) + for (size_t n = 0; n < num_stagenet_hard_forks; ++n) m_hardfork->add_fork(stagenet_hard_forks[n].version, stagenet_hard_forks[n].height, stagenet_hard_forks[n].threshold, stagenet_hard_forks[n].time); } else { - for (size_t n = 0; n < sizeof(mainnet_hard_forks) / sizeof(mainnet_hard_forks[0]); ++n) + for (size_t n = 0; n < num_mainnet_hard_forks; ++n) m_hardfork->add_fork(mainnet_hard_forks[n].version, mainnet_hard_forks[n].height, mainnet_hard_forks[n].threshold, mainnet_hard_forks[n].time); } m_hardfork->init(); @@ -428,7 +340,7 @@ bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline { MINFO("Blockchain not loaded, generating genesis block."); block bl; - block_verification_context bvc = boost::value_initialized<block_verification_context>(); + block_verification_context bvc = {}; generate_genesis_block(bl, get_config(m_nettype).GENESIS_TX, get_config(m_nettype).GENESIS_NONCE); db_wtxn_guard wtxn_guard(m_db); add_new_block(bl, bvc); @@ -616,7 +528,7 @@ bool Blockchain::deinit() // It starts a batch and calls private method pop_block_from_blockchain(). void Blockchain::pop_blocks(uint64_t nblocks) { - uint64_t i; + uint64_t i = 0; CRITICAL_REGION_LOCAL(m_tx_pool); CRITICAL_REGION_LOCAL1(m_blockchain_lock); @@ -627,9 +539,10 @@ void Blockchain::pop_blocks(uint64_t nblocks) const uint64_t blockchain_height = m_db->height(); if (blockchain_height > 0) nblocks = std::min(nblocks, blockchain_height - 1); - for (i=0; i < nblocks; ++i) + while (i < nblocks) { pop_block_from_blockchain(); + ++i; } } catch (const std::exception& e) @@ -736,7 +649,7 @@ bool Blockchain::reset_and_set_genesis_block(const block& b) m_hardfork->init(); db_wtxn_guard wtxn_guard(m_db); - block_verification_context bvc = boost::value_initialized<block_verification_context>(); + block_verification_context bvc = {}; add_new_block(b, bvc); if (!update_next_cumulative_weight_limit()) return false; @@ -1006,7 +919,7 @@ bool Blockchain::rollback_blockchain_switching(std::list<block>& original_chain, //return back original chain for (auto& bl : original_chain) { - block_verification_context bvc = boost::value_initialized<block_verification_context>(); + block_verification_context bvc = {}; bool r = handle_block_to_main_chain(bl, bvc); CHECK_AND_ASSERT_MES(r && bvc.m_added_to_main_chain, false, "PANIC! failed to add (again) block while chain switching during the rollback!"); } @@ -1055,7 +968,7 @@ bool Blockchain::switch_to_alternative_blockchain(std::list<block_extended_info> for(auto alt_ch_iter = alt_chain.begin(); alt_ch_iter != alt_chain.end(); alt_ch_iter++) { const auto &bei = *alt_ch_iter; - block_verification_context bvc = boost::value_initialized<block_verification_context>(); + block_verification_context bvc = {}; // add block to main chain bool r = handle_block_to_main_chain(bei.bl, bvc); @@ -1098,7 +1011,7 @@ bool Blockchain::switch_to_alternative_blockchain(std::list<block_extended_info> //pushing old chain as alternative chain for (auto& old_ch_ent : disconnected_chain) { - block_verification_context bvc = boost::value_initialized<block_verification_context>(); + block_verification_context bvc = {}; bool r = handle_alternative_block(old_ch_ent, get_block_hash(old_ch_ent), bvc); if(!r) { @@ -1201,11 +1114,19 @@ difficulty_type Blockchain::get_next_difficulty_for_alternative_chain(const std: // one input, of type txin_gen, with height set to the block's height // correct miner tx unlock time // a non-overflowing tx amount (dubious necessity on this check) -bool Blockchain::prevalidate_miner_transaction(const block& b, uint64_t height) +bool Blockchain::prevalidate_miner_transaction(const block& b, uint64_t height, uint8_t hf_version) { LOG_PRINT_L3("Blockchain::" << __func__); CHECK_AND_ASSERT_MES(b.miner_tx.vin.size() == 1, false, "coinbase transaction in the block has no inputs"); CHECK_AND_ASSERT_MES(b.miner_tx.vin[0].type() == typeid(txin_gen), false, "coinbase transaction in the block has the wrong type"); + CHECK_AND_ASSERT_MES(b.miner_tx.version > 1 || hf_version < HF_VERSION_MIN_V2_COINBASE_TX, false, "Invalid coinbase transaction version"); + + // for v2 txes (ringct), we only accept empty rct signatures for miner transactions, + if (hf_version >= HF_VERSION_REJECT_SIGS_IN_COINBASE && b.miner_tx.version >= 2) + { + CHECK_AND_ASSERT_MES(b.miner_tx.rct_signatures.type == rct::RCTTypeNull, false, "RingCT signatures not allowed in coinbase transactions"); + } + if(boost::get<txin_gen>(b.miner_tx.vin[0]).height != height) { MWARNING("The miner transaction in block has invalid height: " << boost::get<txin_gen>(b.miner_tx.vin[0]).height << ", expected: " << height); @@ -1412,7 +1333,7 @@ bool Blockchain::create_block_template(block& b, const crypto::hash *from_block, //we have new block in alternative chain std::list<block_extended_info> alt_chain; - block_verification_context bvc = boost::value_initialized<block_verification_context>(); + block_verification_context bvc = {}; std::vector<uint64_t> timestamps; if (!build_alt_chain(*from_block, alt_chain, timestamps, bvc)) return false; @@ -1446,7 +1367,7 @@ bool Blockchain::create_block_template(block& b, const crypto::hash *from_block, } // FIXME: consider moving away from block_extended_info at some point - block_extended_info bei = boost::value_initialized<block_extended_info>(); + block_extended_info bei = {}; bei.bl = b; bei.height = alt_chain.size() ? prev_data.height + 1 : m_db->get_block_height(*from_block) + 1; @@ -1712,6 +1633,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id } // this is a cheap test + const uint8_t hf_version = m_hardfork->get_ideal_version(block_height); if (!m_hardfork->check_for_height(b, block_height)) { LOG_PRINT_L1("Block with id: " << id << std::endl << "has old version for height " << block_height); @@ -1733,7 +1655,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id return false; // FIXME: consider moving away from block_extended_info at some point - block_extended_info bei = boost::value_initialized<block_extended_info>(); + block_extended_info bei = {}; bei.bl = b; const uint64_t prev_height = alt_chain.size() ? prev_data.height : m_db->get_block_height(b.prev_id); bei.height = prev_height + 1; @@ -1770,7 +1692,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id return false; } - if(!prevalidate_miner_transaction(b, bei.height)) + if(!prevalidate_miner_transaction(b, bei.height, hf_version)) { MERROR_VER("Block with id: " << epee::string_tools::pod_to_hex(id) << " (as alternative) has incorrect miner transaction."); bvc.m_verifivation_failed = true; @@ -2858,7 +2780,8 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, if (hf_version >= 2) { size_t n_unmixable = 0, n_mixable = 0; - size_t mixin = std::numeric_limits<size_t>::max(); + size_t min_actual_mixin = std::numeric_limits<size_t>::max(); + size_t max_actual_mixin = 0; const size_t min_mixin = hf_version >= HF_VERSION_MIN_MIXIN_10 ? 10 : hf_version >= HF_VERSION_MIN_MIXIN_6 ? 6 : hf_version >= HF_VERSION_MIN_MIXIN_4 ? 4 : 2; for (const auto& txin : tx.vin) { @@ -2883,29 +2806,43 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, else ++n_mixable; } - if (in_to_key.key_offsets.size() - 1 < mixin) - mixin = in_to_key.key_offsets.size() - 1; + size_t ring_mixin = in_to_key.key_offsets.size() - 1; + if (ring_mixin < min_actual_mixin) + min_actual_mixin = ring_mixin; + if (ring_mixin > max_actual_mixin) + max_actual_mixin = ring_mixin; + } + } + MDEBUG("Mixin: " << min_actual_mixin << "-" << max_actual_mixin); + + if (hf_version >= HF_VERSION_SAME_MIXIN) + { + if (min_actual_mixin != max_actual_mixin) + { + MERROR_VER("Tx " << get_transaction_hash(tx) << " has varying ring size (" << (min_actual_mixin + 1) << "-" << (max_actual_mixin + 1) << "), it should be constant"); + tvc.m_low_mixin = true; + return false; } } - if (((hf_version == HF_VERSION_MIN_MIXIN_10 || hf_version == HF_VERSION_MIN_MIXIN_10+1) && mixin != 10) || (hf_version >= HF_VERSION_MIN_MIXIN_10+2 && mixin > 10)) + if (((hf_version == HF_VERSION_MIN_MIXIN_10 || hf_version == HF_VERSION_MIN_MIXIN_10+1) && min_actual_mixin != 10) || (hf_version >= HF_VERSION_MIN_MIXIN_10+2 && min_actual_mixin > 10)) { - MERROR_VER("Tx " << get_transaction_hash(tx) << " has invalid ring size (" << (mixin + 1) << "), it should be 11"); + MERROR_VER("Tx " << get_transaction_hash(tx) << " has invalid ring size (" << (min_actual_mixin + 1) << "), it should be 11"); tvc.m_low_mixin = true; return false; } - if (mixin < min_mixin) + if (min_actual_mixin < min_mixin) { if (n_unmixable == 0) { - MERROR_VER("Tx " << get_transaction_hash(tx) << " has too low ring size (" << (mixin + 1) << "), and no unmixable inputs"); + MERROR_VER("Tx " << get_transaction_hash(tx) << " has too low ring size (" << (min_actual_mixin + 1) << "), and no unmixable inputs"); tvc.m_low_mixin = true; return false; } if (n_mixable > 1) { - MERROR_VER("Tx " << get_transaction_hash(tx) << " has too low ring size (" << (mixin + 1) << "), and more than one mixable input with unmixable inputs"); + MERROR_VER("Tx " << get_transaction_hash(tx) << " has too low ring size (" << (min_actual_mixin + 1) << "), and more than one mixable input with unmixable inputs"); tvc.m_low_mixin = true; return false; } @@ -2957,6 +2894,9 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, const auto waiter_guard = epee::misc_utils::create_scope_leave_handler([&]() { waiter.wait(&tpool); }); int threads = tpool.get_max_concurrency(); + uint64_t max_used_block_height = 0; + if (!pmax_used_block_height) + pmax_used_block_height = &max_used_block_height; for (const auto& txin : tx.vin) { // make sure output being spent is of type txin_to_key, rather than @@ -3023,6 +2963,13 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, if (tx.version == 1 && threads > 1) waiter.wait(&tpool); + // enforce min output age + if (hf_version >= HF_VERSION_ENFORCE_MIN_AGE) + { + CHECK_AND_ASSERT_MES(*pmax_used_block_height + CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE <= m_db->height(), + false, "Transaction spends at least one output which is too young"); + } + if (tx.version == 1) { if (threads > 1) @@ -3588,9 +3535,10 @@ leave: } // this is a cheap test + const uint8_t hf_version = get_current_hard_fork_version(); if (!m_hardfork->check(bl)) { - MERROR_VER("Block with id: " << id << std::endl << "has old version: " << (unsigned)bl.major_version << std::endl << "current: " << (unsigned)m_hardfork->get_current_version()); + MERROR_VER("Block with id: " << id << std::endl << "has old version: " << (unsigned)bl.major_version << std::endl << "current: " << (unsigned)hf_version); bvc.m_verifivation_failed = true; goto leave; } @@ -3695,7 +3643,7 @@ leave: TIME_MEASURE_START(t3); // sanity check basic miner tx properties; - if(!prevalidate_miner_transaction(bl, blockchain_height)) + if(!prevalidate_miner_transaction(bl, blockchain_height, hf_version)) { MERROR_VER("Block with id: " << id << " failed to pass prevalidation"); bvc.m_verifivation_failed = true; @@ -4776,39 +4724,6 @@ HardFork::State Blockchain::get_hard_fork_state() const return m_hardfork->get_state(); } -const std::vector<HardFork::Params>& Blockchain::get_hard_fork_heights(network_type nettype) -{ - static const std::vector<HardFork::Params> mainnet_heights = []() - { - std::vector<HardFork::Params> heights; - for (const auto& i : mainnet_hard_forks) - heights.emplace_back(i.version, i.height, i.threshold, i.time); - return heights; - }(); - static const std::vector<HardFork::Params> testnet_heights = []() - { - std::vector<HardFork::Params> heights; - for (const auto& i : testnet_hard_forks) - heights.emplace_back(i.version, i.height, i.threshold, i.time); - return heights; - }(); - static const std::vector<HardFork::Params> stagenet_heights = []() - { - std::vector<HardFork::Params> heights; - for (const auto& i : stagenet_hard_forks) - heights.emplace_back(i.version, i.height, i.threshold, i.time); - return heights; - }(); - static const std::vector<HardFork::Params> dummy; - switch (nettype) - { - case MAINNET: return mainnet_heights; - case TESTNET: return testnet_heights; - case STAGENET: return stagenet_heights; - default: return dummy; - } -} - bool Blockchain::get_hard_fork_voting_info(uint8_t version, uint32_t &window, uint32_t &votes, uint32_t &threshold, uint64_t &earliest_height, uint8_t &voting) const { return m_hardfork->get_voting_info(version, window, votes, threshold, earliest_height, voting); diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index f58059a6d..178b2cb24 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -763,13 +763,6 @@ namespace cryptonote HardFork::State get_hard_fork_state() const; /** - * @brief gets the hardfork heights of given network - * - * @return the HardFork object - */ - static const std::vector<HardFork::Params>& get_hard_fork_heights(network_type nettype); - - /** * @brief gets the current hardfork version in use/voted for * * @return the version @@ -1245,10 +1238,11 @@ namespace cryptonote * * @param b the block containing the miner transaction * @param height the height at which the block will be added + * @param hf_version the consensus rules to apply * * @return false if anything is found wrong with the miner transaction, otherwise true */ - bool prevalidate_miner_transaction(const block& b, uint64_t height); + bool prevalidate_miner_transaction(const block& b, uint64_t height, uint8_t hf_version); /** * @brief validates a miner (coinbase) transaction diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index a3a92ab60..e34ab8fbd 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -51,6 +51,7 @@ using namespace epee; #include "blockchain_db/blockchain_db.h" #include "ringct/rctSigs.h" #include "common/notify.h" +#include "hardforks/hardforks.h" #include "version.h" #undef MONERO_DEFAULT_LOG_CATEGORY @@ -454,7 +455,6 @@ namespace cryptonote bool r = handle_command_line(vm); CHECK_AND_ASSERT_MES(r, false, "Failed to handle command line"); - std::string db_type = command_line::get_arg(vm, cryptonote::arg_db_type); std::string db_sync_mode = command_line::get_arg(vm, cryptonote::arg_db_sync_mode); bool db_salvage = command_line::get_arg(vm, cryptonote::arg_db_salvage) != 0; bool fast_sync = command_line::get_arg(vm, arg_fast_block_sync) != 0; @@ -488,10 +488,10 @@ namespace cryptonote // folder might not be a directory, etc, etc catch (...) { } - std::unique_ptr<BlockchainDB> db(new_db(db_type)); + std::unique_ptr<BlockchainDB> db(new_db()); if (db == NULL) { - LOG_ERROR("Attempted to use non-existent database type"); + LOG_ERROR("Failed to initialize a database"); return false; } @@ -634,7 +634,7 @@ namespace cryptonote MERROR("Failed to parse block rate notify spec: " << e.what()); } - const std::pair<uint8_t, uint64_t> regtest_hard_forks[3] = {std::make_pair(1, 0), std::make_pair(Blockchain::get_hard_fork_heights(MAINNET).back().version, 1), std::make_pair(0, 0)}; + const std::pair<uint8_t, uint64_t> regtest_hard_forks[3] = {std::make_pair(1, 0), std::make_pair(mainnet_hard_forks[num_mainnet_hard_forks-1].version, 1), std::make_pair(0, 0)}; const cryptonote::test_options regtest_test_options = { regtest_hard_forks, 0 @@ -746,7 +746,7 @@ namespace cryptonote //----------------------------------------------------------------------------------------------- bool core::handle_incoming_tx_pre(const blobdata& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, bool keeped_by_block, bool relayed, bool do_not_relay) { - tvc = boost::value_initialized<tx_verification_context>(); + tvc = {}; if(tx_blob.size() > get_max_tx_size()) { @@ -1345,7 +1345,7 @@ namespace cryptonote //----------------------------------------------------------------------------------------------- bool core::handle_block_found(block& b, block_verification_context &bvc) { - bvc = boost::value_initialized<block_verification_context>(); + bvc = {}; m_miner.pause(); std::vector<block_complete_entry> blocks; try @@ -1374,7 +1374,7 @@ namespace cryptonote CHECK_AND_ASSERT_MES(!bvc.m_verifivation_failed, false, "mined block failed verification"); if(bvc.m_added_to_main_chain) { - cryptonote_connection_context exclude_context = boost::value_initialized<cryptonote_connection_context>(); + cryptonote_connection_context exclude_context = {}; NOTIFY_NEW_BLOCK::request arg = AUTO_VAL_INIT(arg); arg.current_blockchain_height = m_blockchain_storage.get_current_blockchain_height(); std::vector<crypto::hash> missed_txs; @@ -1442,7 +1442,7 @@ namespace cryptonote { TRY_ENTRY(); - bvc = boost::value_initialized<block_verification_context>(); + bvc = {}; if (!check_incoming_block_size(block_blob)) { @@ -1630,18 +1630,18 @@ namespace cryptonote return true; HardFork::State state = m_blockchain_storage.get_hard_fork_state(); - const el::Level level = el::Level::Warning; + el::Level level; switch (state) { case HardFork::LikelyForked: + level = el::Level::Warning; MCLOG_RED(level, "global", "**********************************************************************"); MCLOG_RED(level, "global", "Last scheduled hard fork is too far in the past."); MCLOG_RED(level, "global", "We are most likely forked from the network. Daemon update needed now."); MCLOG_RED(level, "global", "**********************************************************************"); break; case HardFork::UpdateNeeded: - MCLOG_RED(level, "global", "**********************************************************************"); - MCLOG_RED(level, "global", "Last scheduled hard fork time shows a daemon update is needed soon."); - MCLOG_RED(level, "global", "**********************************************************************"); + level = el::Level::Info; + MCLOG(level, "global", el::Color::Default, "Last scheduled hard fork time suggests a daemon update will be released within the next couple months."); break; default: break; diff --git a/src/cryptonote_core/cryptonote_tx_utils.cpp b/src/cryptonote_core/cryptonote_tx_utils.cpp index 4cf71e558..955d6a4b0 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.cpp +++ b/src/cryptonote_core/cryptonote_tx_utils.cpp @@ -647,7 +647,7 @@ namespace cryptonote ) { //genesis block - bl = boost::value_initialized<block>(); + bl = {}; blobdata tx_bl; bool r = string_tools::parse_hexstr_to_binbuff(genesis_tx, tx_bl); diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index 65115ee72..6d1ad0405 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -55,7 +55,7 @@ if (ELPP->vRegistry()->allowed(level, cat)) { \ init; \ if (test) \ - el::base::Writer(level, __FILE__, __LINE__, ELPP_FUNC, el::base::DispatchAction::NormalLog).construct(cat) << x; \ + el::base::Writer(level, el::Color::Default, __FILE__, __LINE__, ELPP_FUNC, el::base::DispatchAction::NormalLog).construct(cat) << x; \ } \ } while(0) @@ -134,7 +134,7 @@ namespace cryptonote if(context.m_state == cryptonote_connection_context::state_synchronizing) { - NOTIFY_REQUEST_CHAIN::request r = boost::value_initialized<NOTIFY_REQUEST_CHAIN::request>(); + NOTIFY_REQUEST_CHAIN::request r = {}; context.m_needed_objects.clear(); m_core.get_short_chain_history(r.block_ids); handler_request_blocks_history( r.block_ids ); // change the limit(?), sleep(?) @@ -370,7 +370,7 @@ namespace cryptonote uint64_t max_block_height = std::max(hshd.current_height,m_core.get_current_blockchain_height()); uint64_t last_block_v1 = m_core.get_nettype() == TESTNET ? 624633 : m_core.get_nettype() == MAINNET ? 1009826 : (uint64_t)-1; uint64_t diff_v2 = max_block_height > last_block_v1 ? std::min(abs_diff, max_block_height - last_block_v1) : 0; - MCLOG(is_inital ? el::Level::Info : el::Level::Debug, "global", context << "Sync data returned a new top block candidate: " << m_core.get_current_blockchain_height() << " -> " << hshd.current_height + MCLOG(is_inital ? el::Level::Info : el::Level::Debug, "global", el::Color::Yellow, context << "Sync data returned a new top block candidate: " << m_core.get_current_blockchain_height() << " -> " << hshd.current_height << " [Your node is " << abs_diff << " blocks (" << ((abs_diff - diff_v2) / (24 * 60 * 60 / DIFFICULTY_TARGET_V1)) + (diff_v2 / (24 * 60 * 60 / DIFFICULTY_TARGET_V2)) << " days) " << (0 <= diff ? std::string("behind") : std::string("ahead")) << "] " << ENDL << "SYNCHRONIZATION started"); @@ -426,7 +426,7 @@ namespace cryptonote template<class t_core> bool t_cryptonote_protocol_handler<t_core>::get_payload_sync_data(blobdata& data) { - CORE_SYNC_DATA hsd = boost::value_initialized<CORE_SYNC_DATA>(); + CORE_SYNC_DATA hsd = {}; get_payload_sync_data(hsd); epee::serialization::store_t_to_binary(hsd, data); return true; @@ -468,7 +468,7 @@ namespace cryptonote } } - block_verification_context bvc = boost::value_initialized<block_verification_context>(); + block_verification_context bvc = {}; m_core.handle_incoming_block(arg.b.block, pblocks.empty() ? NULL : &pblocks[0], bvc); // got block from handle_notify_new_block if (!m_core.cleanup_handle_incoming_blocks(true)) { @@ -491,7 +491,7 @@ namespace cryptonote { context.m_needed_objects.clear(); context.m_state = cryptonote_connection_context::state_synchronizing; - NOTIFY_REQUEST_CHAIN::request r = boost::value_initialized<NOTIFY_REQUEST_CHAIN::request>(); + NOTIFY_REQUEST_CHAIN::request r = {}; m_core.get_short_chain_history(r.block_ids); handler_request_blocks_history( r.block_ids ); // change the limit(?), sleep(?) MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); @@ -740,7 +740,7 @@ namespace cryptonote return 1; } - block_verification_context bvc = boost::value_initialized<block_verification_context>(); + block_verification_context bvc = {}; m_core.handle_incoming_block(arg.b.block, pblocks.empty() ? NULL : &pblocks[0], bvc); // got block from handle_notify_new_block if (!m_core.cleanup_handle_incoming_blocks(true)) { @@ -768,7 +768,7 @@ namespace cryptonote { context.m_needed_objects.clear(); context.m_state = cryptonote_connection_context::state_synchronizing; - NOTIFY_REQUEST_CHAIN::request r = boost::value_initialized<NOTIFY_REQUEST_CHAIN::request>(); + NOTIFY_REQUEST_CHAIN::request r = {}; m_core.get_short_chain_history(r.block_ids); handler_request_blocks_history( r.block_ids ); // change the limit(?), sleep(?) MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() ); @@ -1306,7 +1306,7 @@ namespace cryptonote // process block TIME_MEASURE_START(block_process_time); - block_verification_context bvc = boost::value_initialized<block_verification_context>(); + block_verification_context bvc = {}; m_core.handle_incoming_block(block_entry.block, pblocks.empty() ? NULL : &pblocks[blockidx], bvc, false); // <--- process block @@ -2010,7 +2010,7 @@ skip: if(context.m_last_response_height < context.m_remote_blockchain_height-1) {//we have to fetch more objects ids, request blockchain entry - NOTIFY_REQUEST_CHAIN::request r = boost::value_initialized<NOTIFY_REQUEST_CHAIN::request>(); + NOTIFY_REQUEST_CHAIN::request r = {}; m_core.get_short_chain_history(r.block_ids); CHECK_AND_ASSERT_MES(!r.block_ids.empty(), false, "Short chain history is empty"); diff --git a/src/daemon/command_parser_executor.cpp b/src/daemon/command_parser_executor.cpp index 924447701..d47823735 100644 --- a/src/daemon/command_parser_executor.cpp +++ b/src/daemon/command_parser_executor.cpp @@ -51,12 +51,14 @@ bool t_command_parser_executor::print_peer_list(const std::vector<std::string>& { if (args.size() > 3) { - std::cout << "use: print_pl [white] [gray] [<limit>]" << std::endl; + std::cout << "use: print_pl [white] [gray] [<limit>] [pruned] [publicrpc]" << std::endl; return true; } bool white = false; bool gray = false; + bool pruned = false; + bool publicrpc = false; size_t limit = 0; for (size_t i = 0; i < args.size(); ++i) { @@ -68,6 +70,14 @@ bool t_command_parser_executor::print_peer_list(const std::vector<std::string>& { gray = true; } + else if (args[i] == "pruned") + { + pruned = true; + } + else if (args[i] == "publicrpc") + { + publicrpc = true; + } else if (!epee::string_tools::get_xtype_from_string(limit, args[i])) { std::cout << "unexpected argument: " << args[i] << std::endl; @@ -76,7 +86,7 @@ bool t_command_parser_executor::print_peer_list(const std::vector<std::string>& } const bool print_both = !white && !gray; - return m_executor.print_peer_list(white | print_both, gray | print_both, limit); + return m_executor.print_peer_list(white | print_both, gray | print_both, limit, pruned, publicrpc); } bool t_command_parser_executor::print_peer_list_stats(const std::vector<std::string>& args) @@ -813,4 +823,18 @@ bool t_command_parser_executor::check_blockchain_pruning(const std::vector<std:: return m_executor.check_blockchain_pruning(); } +bool t_command_parser_executor::set_bootstrap_daemon(const std::vector<std::string>& args) +{ + const size_t args_count = args.size(); + if (args_count < 1 || args_count > 3) + { + return false; + } + + return m_executor.set_bootstrap_daemon( + args[0] != "none" ? args[0] : std::string(), + args_count > 1 ? args[1] : std::string(), + args_count > 2 ? args[2] : std::string()); +} + } // namespace daemonize diff --git a/src/daemon/command_parser_executor.h b/src/daemon/command_parser_executor.h index d39bc1c9b..25587dea8 100644 --- a/src/daemon/command_parser_executor.h +++ b/src/daemon/command_parser_executor.h @@ -150,6 +150,8 @@ public: bool check_blockchain_pruning(const std::vector<std::string>& args); bool print_net_stats(const std::vector<std::string>& args); + + bool set_bootstrap_daemon(const std::vector<std::string>& args); }; } // namespace daemonize diff --git a/src/daemon/command_server.cpp b/src/daemon/command_server.cpp index aecdda52c..757e072a4 100644 --- a/src/daemon/command_server.cpp +++ b/src/daemon/command_server.cpp @@ -310,6 +310,13 @@ t_command_server::t_command_server( , std::bind(&t_command_parser_executor::check_blockchain_pruning, &m_parser, p::_1) , "Check the blockchain pruning." ); + m_command_lookup.set_handler( + "set_bootstrap_daemon" + , std::bind(&t_command_parser_executor::set_bootstrap_daemon, &m_parser, p::_1) + , "set_bootstrap_daemon (auto | none | host[:port] [username] [password])" + , "URL of a 'bootstrap' remote daemon that the connected wallets can use while this daemon is still not fully synced.\n" + "Use 'auto' to enable automatic public nodes discovering and bootstrap daemon switching" + ); } bool t_command_server::process_command_str(const std::string& cmd) diff --git a/src/daemon/main.cpp b/src/daemon/main.cpp index cb288071e..8fa983fe5 100644 --- a/src/daemon/main.cpp +++ b/src/daemon/main.cpp @@ -44,7 +44,6 @@ #include "rpc/core_rpc_server.h" #include "rpc/rpc_args.h" #include "daemon/command_line_args.h" -#include "blockchain_db/db_types.h" #include "version.h" #ifdef STACK_TRACE @@ -224,16 +223,6 @@ int main(int argc, char const * argv[]) return 1; } - std::string db_type = command_line::get_arg(vm, cryptonote::arg_db_type); - - // verify that blockchaindb type is valid - if(!cryptonote::blockchain_valid_db_type(db_type)) - { - std::cout << "Invalid database type (" << db_type << "), available types are: " << - cryptonote::blockchain_db_types(", ") << std::endl; - return 0; - } - // data_dir // default: e.g. ~/.bitmonero/ or ~/.bitmonero/testnet // if data-dir argument given: diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp index 4d3debed6..014865730 100644 --- a/src/daemon/rpc_command_executor.cpp +++ b/src/daemon/rpc_command_executor.cpp @@ -60,13 +60,18 @@ namespace { } } - void print_peer(std::string const & prefix, cryptonote::peer const & peer) + void print_peer(std::string const & prefix, cryptonote::peer const & peer, bool pruned_only, bool publicrpc_only) { + if (pruned_only && peer.pruning_seed == 0) + return; + if (publicrpc_only && peer.rpc_port == 0) + return; + time_t now; time(&now); time_t last_seen = static_cast<time_t>(peer.last_seen); - std::string elapsed = epee::misc_utils::get_time_interval_string(now - last_seen); + std::string elapsed = peer.last_seen == 0 ? "never" : epee::misc_utils::get_time_interval_string(now - last_seen); std::string id_str = epee::string_tools::pad_string(epee::string_tools::to_string_hex(peer.id), 16, '0', true); std::string port_str; epee::string_tools::xtype_to_string(peer.port, port_str); @@ -169,7 +174,7 @@ t_rpc_command_executor::~t_rpc_command_executor() } } -bool t_rpc_command_executor::print_peer_list(bool white, bool gray, size_t limit) { +bool t_rpc_command_executor::print_peer_list(bool white, bool gray, size_t limit, bool pruned_only, bool publicrpc_only) { cryptonote::COMMAND_RPC_GET_PEER_LIST::request req; cryptonote::COMMAND_RPC_GET_PEER_LIST::response res; @@ -196,7 +201,7 @@ bool t_rpc_command_executor::print_peer_list(bool white, bool gray, size_t limit const auto end = limit ? peer + std::min(limit, res.white_list.size()) : res.white_list.cend(); for (; peer != end; ++peer) { - print_peer("white", *peer); + print_peer("white", *peer, pruned_only, publicrpc_only); } } @@ -206,7 +211,7 @@ bool t_rpc_command_executor::print_peer_list(bool white, bool gray, size_t limit const auto end = limit ? peer + std::min(limit, res.gray_list.size()) : res.gray_list.cend(); for (; peer != end; ++peer) { - print_peer("gray", *peer); + print_peer("gray", *peer, pruned_only, publicrpc_only); } } @@ -573,9 +578,9 @@ bool t_rpc_command_executor::mining_status() { tools::msg_writer() << "Mining at " << get_mining_speed(mres.speed) << " with " << mres.threads_count << " threads"; } + tools::msg_writer() << "PoW algorithm: " << mres.pow_algorithm; if (mres.active || mres.is_background_mining_enabled) { - tools::msg_writer() << "PoW algorithm: " << mres.pow_algorithm; tools::msg_writer() << "Mining address: " << mres.address; } @@ -2320,4 +2325,40 @@ bool t_rpc_command_executor::check_blockchain_pruning() return true; } +bool t_rpc_command_executor::set_bootstrap_daemon( + const std::string &address, + const std::string &username, + const std::string &password) +{ + cryptonote::COMMAND_RPC_SET_BOOTSTRAP_DAEMON::request req; + cryptonote::COMMAND_RPC_SET_BOOTSTRAP_DAEMON::response res; + const std::string fail_message = "Unsuccessful"; + + req.address = address; + req.username = username; + req.password = password; + + if (m_is_rpc) + { + if (!m_rpc_client->rpc_request(req, res, "/set_bootstrap_daemon", fail_message)) + { + return true; + } + } + else + { + if (!m_rpc_server->on_set_bootstrap_daemon(req, res) || res.status != CORE_RPC_STATUS_OK) + { + tools::fail_msg_writer() << make_error(fail_message, res.status); + return true; + } + } + + tools::success_msg_writer() + << "Successfully set bootstrap daemon address to " + << (!req.address.empty() ? req.address : "none"); + + return true; +} + }// namespace daemonize diff --git a/src/daemon/rpc_command_executor.h b/src/daemon/rpc_command_executor.h index f3ed48319..af7081ef3 100644 --- a/src/daemon/rpc_command_executor.h +++ b/src/daemon/rpc_command_executor.h @@ -69,7 +69,7 @@ public: ~t_rpc_command_executor(); - bool print_peer_list(bool white = true, bool gray = true, size_t limit = 0); + bool print_peer_list(bool white = true, bool gray = true, size_t limit = 0, bool pruned_only = false, bool publicrpc_only = false); bool print_peer_list_stats(); @@ -162,6 +162,11 @@ public: bool check_blockchain_pruning(); bool print_net_stats(); + + bool set_bootstrap_daemon( + const std::string &address, + const std::string &username, + const std::string &password); }; } // namespace daemonize diff --git a/src/device/device_ledger.cpp b/src/device/device_ledger.cpp index 2d91b881b..15fe560d1 100644 --- a/src/device/device_ledger.cpp +++ b/src/device/device_ledger.cpp @@ -418,10 +418,10 @@ namespace hw { #ifdef DEBUG_HWDEVICE cryptonote::account_public_address pubkey; this->get_public_address(pubkey); - #endif crypto::secret_key vkey; crypto::secret_key skey; this->get_secret_keys(vkey,skey); + #endif return true; } diff --git a/src/hardforks/CMakeLists.txt b/src/hardforks/CMakeLists.txt new file mode 100644 index 000000000..bd2d14ceb --- /dev/null +++ b/src/hardforks/CMakeLists.txt @@ -0,0 +1,47 @@ +# Copyright (c) 2014-2019, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set(hardforks_sources + hardforks.cpp) + +set(hardforks_headers + hardforks.h) + +set(hardforks_private_headers) + +monero_private_headers(hardforks + ${hardforks_private_headers}) +monero_add_library(hardforks + ${hardforks_sources} + ${hardforks_headers} + ${hardforks_private_headers}) +target_link_libraries(hardforks + PUBLIC + version + PRIVATE + ${EXTRA_LIBRARIES}) diff --git a/src/hardforks/hardforks.cpp b/src/hardforks/hardforks.cpp new file mode 100644 index 000000000..3cb148c60 --- /dev/null +++ b/src/hardforks/hardforks.cpp @@ -0,0 +1,109 @@ +// Copyright (c) 2014-2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "hardforks.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "blockchain.hardforks" + +const hardfork_t mainnet_hard_forks[] = { + // version 1 from the start of the blockchain + { 1, 1, 0, 1341378000 }, + + // version 2 starts from block 1009827, which is on or around the 20th of March, 2016. Fork time finalised on 2015-09-20. No fork voting occurs for the v2 fork. + { 2, 1009827, 0, 1442763710 }, + + // version 3 starts from block 1141317, which is on or around the 24th of September, 2016. Fork time finalised on 2016-03-21. + { 3, 1141317, 0, 1458558528 }, + + // version 4 starts from block 1220516, which is on or around the 5th of January, 2017. Fork time finalised on 2016-09-18. + { 4, 1220516, 0, 1483574400 }, + + // version 5 starts from block 1288616, which is on or around the 15th of April, 2017. Fork time finalised on 2017-03-14. + { 5, 1288616, 0, 1489520158 }, + + // version 6 starts from block 1400000, which is on or around the 16th of September, 2017. Fork time finalised on 2017-08-18. + { 6, 1400000, 0, 1503046577 }, + + // version 7 starts from block 1546000, which is on or around the 6th of April, 2018. Fork time finalised on 2018-03-17. + { 7, 1546000, 0, 1521303150 }, + + // version 8 starts from block 1685555, which is on or around the 18th of October, 2018. Fork time finalised on 2018-09-02. + { 8, 1685555, 0, 1535889547 }, + + // version 9 starts from block 1686275, which is on or around the 19th of October, 2018. Fork time finalised on 2018-09-02. + { 9, 1686275, 0, 1535889548 }, + + // version 10 starts from block 1788000, which is on or around the 9th of March, 2019. Fork time finalised on 2019-02-10. + { 10, 1788000, 0, 1549792439 }, + + // version 11 starts from block 1788720, which is on or around the 10th of March, 2019. Fork time finalised on 2019-02-15. + { 11, 1788720, 0, 1550225678 }, +}; +const size_t num_mainnet_hard_forks = sizeof(mainnet_hard_forks) / sizeof(mainnet_hard_forks[0]); +const uint64_t mainnet_hard_fork_version_1_till = 1009826; + +const hardfork_t testnet_hard_forks[] = { + // version 1 from the start of the blockchain + { 1, 1, 0, 1341378000 }, + + // version 2 starts from block 624634, which is on or around the 23rd of November, 2015. Fork time finalised on 2015-11-20. No fork voting occurs for the v2 fork. + { 2, 624634, 0, 1445355000 }, + + // versions 3-5 were passed in rapid succession from September 18th, 2016 + { 3, 800500, 0, 1472415034 }, + { 4, 801219, 0, 1472415035 }, + { 5, 802660, 0, 1472415036 + 86400*180 }, // add 5 months on testnet to shut the update warning up since there's a large gap to v6 + + { 6, 971400, 0, 1501709789 }, + { 7, 1057027, 0, 1512211236 }, + { 8, 1057058, 0, 1533211200 }, + { 9, 1057778, 0, 1533297600 }, + { 10, 1154318, 0, 1550153694 }, + { 11, 1155038, 0, 1550225678 }, +}; +const size_t num_testnet_hard_forks = sizeof(testnet_hard_forks) / sizeof(testnet_hard_forks[0]); +const uint64_t testnet_hard_fork_version_1_till = 624633; + +const hardfork_t stagenet_hard_forks[] = { + // version 1 from the start of the blockchain + { 1, 1, 0, 1341378000 }, + + // versions 2-7 in rapid succession from March 13th, 2018 + { 2, 32000, 0, 1521000000 }, + { 3, 33000, 0, 1521120000 }, + { 4, 34000, 0, 1521240000 }, + { 5, 35000, 0, 1521360000 }, + { 6, 36000, 0, 1521480000 }, + { 7, 37000, 0, 1521600000 }, + { 8, 176456, 0, 1537821770 }, + { 9, 177176, 0, 1537821771 }, + { 10, 269000, 0, 1550153694 }, + { 11, 269720, 0, 1550225678 }, +}; +const size_t num_stagenet_hard_forks = sizeof(stagenet_hard_forks) / sizeof(stagenet_hard_forks[0]); diff --git a/src/blockchain_db/db_types.h b/src/hardforks/hardforks.h index 04cadbb10..e7bceca42 100644 --- a/src/blockchain_db/db_types.h +++ b/src/hardforks/hardforks.h @@ -25,12 +25,28 @@ // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + #pragma once -namespace cryptonote +#include <stdint.h> +#include <time.h> + +struct hardfork_t { - bool blockchain_valid_db_type(const std::string& db_type); - std::string blockchain_db_types(const std::string& sep); -} // namespace cryptonote + uint8_t version; + uint64_t height; + uint8_t threshold; + time_t time; + hardfork_t(uint8_t version, uint64_t height, uint8_t threshold, time_t time): version(version), height(height), threshold(threshold), time(time) {} +}; + +extern const hardfork_t mainnet_hard_forks[]; +extern const uint64_t mainnet_hard_fork_version_1_till; +extern const size_t num_mainnet_hard_forks; + +extern const hardfork_t testnet_hard_forks[]; +extern const uint64_t testnet_hard_fork_version_1_till; +extern const size_t num_testnet_hard_forks; + +extern const hardfork_t stagenet_hard_forks[]; +extern const size_t num_stagenet_hard_forks; diff --git a/src/net/CMakeLists.txt b/src/net/CMakeLists.txt index 24b707f77..339587ffa 100644 --- a/src/net/CMakeLists.txt +++ b/src/net/CMakeLists.txt @@ -26,8 +26,10 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -set(net_sources dandelionpp.cpp error.cpp i2p_address.cpp parse.cpp socks.cpp socks_connect.cpp tor_address.cpp) -set(net_headers dandelionpp.h error.h i2p_address.h parse.h socks.h socks_connect.h tor_address.h) +set(net_sources dandelionpp.cpp error.cpp i2p_address.cpp parse.cpp socks.cpp + socks_connect.cpp tor_address.cpp zmq.cpp) +set(net_headers dandelionpp.h error.h i2p_address.h parse.h socks.h socks_connect.h + tor_address.h zmq.h) monero_add_library(net ${net_sources} ${net_headers}) target_link_libraries(net common epee ${Boost_ASIO_LIBRARY}) diff --git a/src/net/zmq.cpp b/src/net/zmq.cpp new file mode 100644 index 000000000..d02a22983 --- /dev/null +++ b/src/net/zmq.cpp @@ -0,0 +1,188 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "net/zmq.h" + +#include <cassert> +#include <cerrno> +#include <limits> +#include <utility> + +namespace net +{ +namespace zmq +{ + const std::error_category& error_category() noexcept + { + struct category final : std::error_category + { + virtual const char* name() const noexcept override final + { + return "error::error_category()"; + } + + virtual std::string message(int value) const override final + { + char const* const msg = zmq_strerror(value); + if (msg) + return msg; + return "zmq_strerror failure"; + } + + virtual std::error_condition default_error_condition(int value) const noexcept override final + { + // maps specific errors to generic `std::errc` cases. + switch (value) + { + case EFSM: + case ETERM: + break; + default: + /* zmq is using cerrno errors. C++ spec indicates that + `std::errc` values must be identical to the cerrno value. + So just map every zmq specific error to the generic errc + equivalent. zmq extensions must be in the switch or they + map to a non-existent errc enum value. */ + return std::errc(value); + } + return std::error_condition{value, *this}; + } + + }; + static const category instance{}; + return instance; + } + + void terminate::call(void* ptr) noexcept + { + assert(ptr != nullptr); // see header + while (zmq_term(ptr)) + { + if (zmq_errno() != EINTR) + break; + } + } + + namespace + { + //! RAII wrapper for `zmq_msg_t`. + class message + { + zmq_msg_t handle_; + + public: + message() noexcept + : handle_() + { + zmq_msg_init(handle()); + } + + message(message&& rhs) = delete; + message(const message& rhs) = delete; + message& operator=(message&& rhs) = delete; + message& operator=(const message& rhs) = delete; + + ~message() noexcept + { + zmq_msg_close(handle()); + } + + zmq_msg_t* handle() noexcept + { + return std::addressof(handle_); + } + + const char* data() noexcept + { + return static_cast<const char*>(zmq_msg_data(handle())); + } + + std::size_t size() noexcept + { + return zmq_msg_size(handle()); + } + }; + + struct do_receive + { + /* ZMQ documentation states that message parts are atomic - either + all are received or none are. Looking through ZMQ code and + Github discussions indicates that after part 1 is returned, + `EAGAIN` cannot be returned to meet these guarantees. Unit tests + verify (for the `inproc://` case) that this is the behavior. + Therefore, read errors after the first part are treated as a + failure for the entire message (probably `ETERM`). */ + int operator()(std::string& payload, void* const socket, const int flags) const + { + static constexpr const int max_out = std::numeric_limits<int>::max(); + const std::string::size_type initial = payload.size(); + message part{}; + for (;;) + { + int last = 0; + if ((last = zmq_msg_recv(part.handle(), socket, flags)) < 0) + return last; + + payload.append(part.data(), part.size()); + if (!zmq_msg_more(part.handle())) + break; + } + const std::string::size_type added = payload.size() - initial; + return unsigned(max_out) < added ? max_out : int(added); + } + }; + + template<typename F, typename... T> + expect<void> retry_op(F op, T&&... args) noexcept(noexcept(op(args...))) + { + for (;;) + { + if (0 <= op(args...)) + return success(); + + const int error = zmq_errno(); + if (error != EINTR) + return make_error_code(error); + } + } + } // anonymous + + expect<std::string> receive(void* const socket, const int flags) + { + std::string payload{}; + MONERO_CHECK(retry_op(do_receive{}, payload, socket, flags)); + return {std::move(payload)}; + } + + expect<void> send(const epee::span<const std::uint8_t> payload, void* const socket, const int flags) noexcept + { + return retry_op(zmq_send, socket, payload.data(), payload.size(), flags); + } +} // zmq +} // net + diff --git a/src/net/zmq.h b/src/net/zmq.h new file mode 100644 index 000000000..c6a7fd743 --- /dev/null +++ b/src/net/zmq.h @@ -0,0 +1,136 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <memory> +#include <string> +#include <system_error> +#include <zmq.h> + +#include "common/expect.h" +#include "span.h" + +//! If the expression is less than 0, return the current ZMQ error code. +#define MONERO_ZMQ_CHECK(...) \ + do \ + { \ + if (( __VA_ARGS__ ) < 0) \ + return {::net::zmq::get_error_code()}; \ + } while (0) + +//! Print a message followed by the current ZMQ error message. +#define MONERO_LOG_ZMQ_ERROR(...) \ + do \ + { \ + MERROR( __VA_ARGS__ << ": " << ::net::zmq::get_error_code().message()); \ + } while (0) + +//! Throw an exception with a custom `msg`, current ZMQ error code, filename, and line number. +#define MONERO_ZMQ_THROW(msg) \ + MONERO_THROW( ::net::zmq::get_error_code(), msg ) + +namespace net +{ +namespace zmq +{ + //! \return Category for ZMQ errors. + const std::error_category& error_category() noexcept; + + //! \return `code` (usally from zmq_errno()`) using `net::zmq::error_category()`. + inline std::error_code make_error_code(int code) noexcept + { + return std::error_code{code, error_category()}; + } + + //! \return Error from `zmq_errno()` using `net::zmq::error_category()`. + inline std::error_code get_error_code() noexcept + { + return make_error_code(zmq_errno()); + } + + //! Calls `zmq_term` + class terminate + { + static void call(void* ptr) noexcept; + public: + void operator()(void* ptr) const noexcept + { + if (ptr) + call(ptr); + } + }; + + //! Calls `zmq_close` + struct close + { + void operator()(void* ptr) const noexcept + { + if (ptr) + zmq_close(ptr); + } + }; + + //! Unique ZMQ context handle, calls `zmq_term` on destruction. + using context = std::unique_ptr<void, terminate>; + + //! Unique ZMQ socket handle, calls `zmq_close` on destruction. + using socket = std::unique_ptr<void, close>; + + /*! Read all parts of the next message on `socket`. Blocks until the entire + next message (all parts) are read, or until `zmq_term` is called on the + `zmq_context` associated with `socket`. If the context is terminated, + `make_error_code(ETERM)` is returned. + + \note This will automatically retry on `EINTR`, so exiting on + interrupts requires context termination. + \note If non-blocking behavior is requested on `socket` or by `flags`, + then `net::zmq::make_error_code(EAGAIN)` will be returned if this + would block. + + \param socket Handle created with `zmq_socket`. + \param flags See `zmq_msg_read` for possible flags. + \return Message payload read from `socket` or ZMQ error. */ + expect<std::string> receive(void* socket, int flags = 0); + + /*! Sends `payload` on `socket`. Blocks until the entire message is queued + for sending, or until `zmq_term` is called on the `zmq_context` + associated with `socket`. If the context is terminated, + `make_error_code(ETERM)` is returned. + + \note This will automatically retry on `EINTR`, so exiting on + interrupts requires context termination. + \note If non-blocking behavior is requested on `socket` or by `flags`, + then `net::zmq::make_error_code(EAGAIN)` will be returned if this + would block. + + \param payload sent as one message on `socket`. + \param socket Handle created with `zmq_socket`. + \param flags See `zmq_send` for possible flags. + \return `success()` if sent, otherwise ZMQ error. */ + expect<void> send(epee::span<const std::uint8_t> payload, void* socket, int flags = 0) noexcept; +} // zmq +} // net diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index 231175dd2..255a1fc1f 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -355,8 +355,7 @@ namespace nodetool bool get_local_node_data(basic_node_data& node_data, const network_zone& zone); //bool get_local_handshake_data(handshake_data& hshd); - bool merge_peerlist_with_local(const std::vector<peerlist_entry>& bs); - bool fix_time_delta(std::vector<peerlist_entry>& local_peerlist, time_t local_time, int64_t& delta); + bool sanitize_peerlist(std::vector<peerlist_entry>& local_peerlist); bool connections_maker(); bool peer_sync_idle_maker(); diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index 41ca19917..97a18b519 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -1446,7 +1446,7 @@ namespace nodetool if (skipped == 0 || !filtered.empty()) break; if (skipped) - MGINFO("Skipping " << skipped << " possible peers as they share a class B with existing peers"); + MINFO("Skipping " << skipped << " possible peers as they share a class B with existing peers"); } if (filtered.empty()) { @@ -1841,21 +1841,32 @@ namespace nodetool } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::fix_time_delta(std::vector<peerlist_entry>& local_peerlist, time_t local_time, int64_t& delta) + bool node_server<t_payload_net_handler>::sanitize_peerlist(std::vector<peerlist_entry>& local_peerlist) { - //fix time delta - time_t now = 0; - time(&now); - delta = now - local_time; - - for(peerlist_entry& be: local_peerlist) + for (size_t i = 0; i < local_peerlist.size(); ++i) { - if(be.last_seen > local_time) + bool ignore = false; + peerlist_entry &be = local_peerlist[i]; + epee::net_utils::network_address &na = be.adr; + if (na.is_loopback() || na.is_local()) { - MWARNING("FOUND FUTURE peerlist for entry " << be.adr.str() << " last_seen: " << be.last_seen << ", local_time(on remote node):" << local_time); - return false; + ignore = true; + } + else if (be.adr.get_type_id() == epee::net_utils::ipv4_network_address::get_type_id()) + { + const epee::net_utils::ipv4_network_address &ipv4 = na.as<const epee::net_utils::ipv4_network_address>(); + if (ipv4.ip() == 0) + ignore = true; + } + if (ignore) + { + MDEBUG("Ignoring " << be.adr.str()); + std::swap(local_peerlist[i], local_peerlist[local_peerlist.size() - 1]); + local_peerlist.resize(local_peerlist.size() - 1); + --i; + continue; } - be.last_seen += delta; + #ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED be.pruning_seed = tools::make_pruning_seed(1 + (be.adr.as<epee::net_utils::ipv4_network_address>().ip()) % (1ul << CRYPTONOTE_PRUNING_LOG_STRIPES), CRYPTONOTE_PRUNING_LOG_STRIPES); #endif @@ -1866,9 +1877,8 @@ namespace nodetool template<class t_payload_net_handler> bool node_server<t_payload_net_handler>::handle_remote_peerlist(const std::vector<peerlist_entry>& peerlist, time_t local_time, const epee::net_utils::connection_context_base& context) { - int64_t delta = 0; std::vector<peerlist_entry> peerlist_ = peerlist; - if(!fix_time_delta(peerlist_, local_time, delta)) + if(!sanitize_peerlist(peerlist_)) return false; const epee::net_utils::zone zone = context.m_remote_address.get_zone(); @@ -1881,8 +1891,8 @@ namespace nodetool } } - LOG_DEBUG_CC(context, "REMOTE PEERLIST: TIME_DELTA: " << delta << ", remote peerlist size=" << peerlist_.size()); - LOG_DEBUG_CC(context, "REMOTE PEERLIST: " << print_peerlist_to_string(peerlist_)); + LOG_DEBUG_CC(context, "REMOTE PEERLIST: remote peerlist size=" << peerlist_.size()); + LOG_DEBUG_CC(context, "REMOTE PEERLIST: " << ENDL << print_peerlist_to_string(peerlist_)); return m_network_zones.at(context.m_remote_address.get_zone()).m_peerlist.merge_peerlist(peerlist_); } //----------------------------------------------------------------------------------- @@ -2308,6 +2318,15 @@ namespace nodetool network_zone& zone = m_network_zones.at(context.m_remote_address.get_zone()); + // test only the remote end's zone, otherwise an attacker could connect to you on clearnet + // and pass in a tor connection's peer id, and deduce the two are the same if you reject it + if(arg.node_data.peer_id == zone.m_config.m_peer_id) + { + LOG_DEBUG_CC(context, "Connection to self detected, dropping connection"); + drop_connection(context); + return 1; + } + if (zone.m_current_number_of_in_peers >= zone.m_config.m_net_config.max_in_connection_count) // in peers limit { LOG_WARNING_CC(context, "COMMAND_HANDSHAKE came, but already have max incoming connections, so dropping this one."); @@ -2334,7 +2353,7 @@ namespace nodetool context.m_in_timedsync = false; context.m_rpc_port = arg.node_data.rpc_port; - if(arg.node_data.peer_id != zone.m_config.m_peer_id && arg.node_data.my_port && zone.m_can_pingback) + if(arg.node_data.my_port && zone.m_can_pingback) { peerid_type peer_id_l = arg.node_data.peer_id; uint32_t port_l = arg.node_data.my_port; diff --git a/src/p2p/net_peerlist_boost_serialization.h b/src/p2p/net_peerlist_boost_serialization.h index 05eb36e65..c2773981c 100644 --- a/src/p2p/net_peerlist_boost_serialization.h +++ b/src/p2p/net_peerlist_boost_serialization.h @@ -95,7 +95,9 @@ namespace boost { uint32_t ip{na.ip()}; uint16_t port{na.port()}; + ip = SWAP32LE(ip); a & ip; + ip = SWAP32LE(ip); a & port; if (!typename Archive::is_saving()) na = epee::net_utils::ipv4_network_address{ip, port}; diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt index d98670e05..116e7f568 100644 --- a/src/rpc/CMakeLists.txt +++ b/src/rpc/CMakeLists.txt @@ -26,6 +26,8 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +include_directories(SYSTEM ${ZMQ_INCLUDE_PATH}) + set(rpc_base_sources rpc_args.cpp) diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 706776ae0..7d6472ddb 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -103,6 +103,7 @@ namespace cryptonote ) : m_core(cr) , m_p2p(p2p) + , m_was_bootstrap_ever_used(false) {} //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::set_bootstrap_daemon(const std::string &address, const std::string &username_password) @@ -1049,7 +1050,8 @@ namespace cryptonote res.block_reward = lMiner.get_block_reward(); } const account_public_address& lMiningAdr = lMiner.get_mining_address(); - res.address = get_account_address_as_str(nettype(), false, lMiningAdr); + if (lMiner.is_mining() || lMiner.get_is_background_mining_enabled()) + res.address = get_account_address_as_str(nettype(), false, lMiningAdr); const uint8_t major_version = m_core.get_blockchain_storage().get_current_hard_fork_version(); const unsigned variant = major_version >= 7 ? major_version - 6 : 0; switch (variant) @@ -1554,7 +1556,7 @@ namespace cryptonote template_req.reserve_size = 1; template_req.wallet_address = req.wallet_address; template_req.prev_block = req.prev_block; - submit_req.push_back(boost::value_initialized<std::string>()); + submit_req.push_back(std::string{}); res.height = m_core.get_blockchain_storage().get_current_blockchain_height(); for(size_t i = 0; i < req.amount_of_blocks; i++) diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index 325ac4343..39995c206 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -86,8 +86,8 @@ namespace cryptonote // whether they can talk to a given daemon without having to know in // advance which version they will stop working with // Don't go over 32767 for any of these -#define CORE_RPC_VERSION_MAJOR 2 -#define CORE_RPC_VERSION_MINOR 10 +#define CORE_RPC_VERSION_MAJOR 3 +#define CORE_RPC_VERSION_MINOR 0 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR) @@ -255,56 +255,6 @@ namespace cryptonote }; typedef epee::misc_utils::struct_init<response_t> response; }; - - //----------------------------------------------- - struct COMMAND_RPC_GET_RANDOM_OUTS - { - struct request_t - { - std::vector<std::string> amounts; - uint32_t count; - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(amounts) - KV_SERIALIZE(count) - END_KV_SERIALIZE_MAP() - }; - typedef epee::misc_utils::struct_init<request_t> request; - - - struct output { - std::string public_key; - uint64_t global_index; - std::string rct; // 64+64+64 characters long (<rct commit> + <encrypted mask> + <rct amount>) - - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(public_key) - KV_SERIALIZE(global_index) - KV_SERIALIZE(rct) - END_KV_SERIALIZE_MAP() - }; - - struct amount_out { - uint64_t amount; - std::vector<output> outputs; - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(amount) - KV_SERIALIZE(outputs) - END_KV_SERIALIZE_MAP() - - }; - - struct response_t - { - std::vector<amount_out> amount_outs; - std::string Error; - BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(amount_outs) - KV_SERIALIZE(Error) - END_KV_SERIALIZE_MAP() - }; - typedef epee::misc_utils::struct_init<response_t> response; - }; //----------------------------------------------- struct COMMAND_RPC_SUBMIT_RAW_TX { @@ -1566,7 +1516,7 @@ namespace cryptonote KV_SERIALIZE(num_10m) KV_SERIALIZE(num_not_relayed) KV_SERIALIZE(histo_98pc) - KV_SERIALIZE_CONTAINER_POD_AS_BLOB(histo) + KV_SERIALIZE(histo) KV_SERIALIZE(num_double_spends) END_KV_SERIALIZE_MAP() }; diff --git a/src/rpc/zmq_server.cpp b/src/rpc/zmq_server.cpp index 668a2e5cd..1ee55673e 100644 --- a/src/rpc/zmq_server.cpp +++ b/src/rpc/zmq_server.cpp @@ -28,18 +28,29 @@ #include "zmq_server.h" +#include <chrono> +#include <cstdint> +#include <system_error> + namespace cryptonote { +namespace +{ + constexpr const int num_zmq_threads = 1; + constexpr const std::int64_t max_message_size = 10 * 1024 * 1024; // 10 MiB + constexpr const std::chrono::seconds linger_timeout{2}; // wait period for pending out messages +} + namespace rpc { ZmqServer::ZmqServer(RpcHandler& h) : handler(h), - stop_signal(false), - running(false), - context(DEFAULT_NUM_ZMQ_THREADS) // TODO: make this configurable + context(zmq_init(num_zmq_threads)) { + if (!context) + MONERO_ZMQ_THROW("Unable to create ZMQ context"); } ZmqServer::~ZmqServer() @@ -48,71 +59,88 @@ ZmqServer::~ZmqServer() void ZmqServer::serve() { - - while (1) + try { - try + // socket must close before `zmq_term` will exit. + const net::zmq::socket socket = std::move(rep_socket); + if (!socket) { - zmq::message_t message; - - if (!rep_socket) - { - throw std::runtime_error("ZMQ RPC server reply socket is null"); - } - while (rep_socket->recv(&message, 0)) - { - std::string message_string(reinterpret_cast<const char *>(message.data()), message.size()); - - MDEBUG(std::string("Received RPC request: \"") + message_string + "\""); - - std::string response = handler.handle(message_string); - - zmq::message_t reply(response.size()); - memcpy((void *) reply.data(), response.c_str(), response.size()); - - rep_socket->send(reply); - MDEBUG(std::string("Sent RPC reply: \"") + response + "\""); - - } - } - catch (const boost::thread_interrupted& e) - { - MDEBUG("ZMQ Server thread interrupted."); + MERROR("ZMQ RPC server reply socket is null"); + return; } - catch (const zmq::error_t& e) + + while (1) { - MERROR(std::string("ZMQ error: ") + e.what()); + const std::string message = MONERO_UNWRAP(net::zmq::receive(socket.get())); + MDEBUG("Received RPC request: \"" << message << "\""); + const std::string& response = handler.handle(message); + + MONERO_UNWRAP(net::zmq::send(epee::strspan<std::uint8_t>(response), socket.get())); + MDEBUG("Sent RPC reply: \"" << response << "\""); } - boost::this_thread::interruption_point(); + } + catch (const std::system_error& e) + { + if (e.code() != net::zmq::make_error_code(ETERM)) + MERROR("ZMQ RPC Server Error: " << e.what()); + } + catch (const std::exception& e) + { + MERROR("ZMQ RPC Server Error: " << e.what()); + } + catch (...) + { + MERROR("Unknown error in ZMQ RPC server"); } } -bool ZmqServer::addIPCSocket(std::string address, std::string port) +bool ZmqServer::addIPCSocket(const boost::string_ref address, const boost::string_ref port) { MERROR("ZmqServer::addIPCSocket not yet implemented!"); return false; } -bool ZmqServer::addTCPSocket(std::string address, std::string port) +bool ZmqServer::addTCPSocket(boost::string_ref address, boost::string_ref port) { - try + if (!context) { - std::string addr_prefix("tcp://"); + MERROR("ZMQ RPC Server already shutdown"); + return false; + } - rep_socket.reset(new zmq::socket_t(context, ZMQ_REP)); + rep_socket.reset(zmq_socket(context.get(), ZMQ_REP)); + if (!rep_socket) + { + MONERO_LOG_ZMQ_ERROR("ZMQ RPC Server socket create failed"); + return false; + } - rep_socket->setsockopt(ZMQ_RCVTIMEO, &DEFAULT_RPC_RECV_TIMEOUT_MS, sizeof(DEFAULT_RPC_RECV_TIMEOUT_MS)); + if (zmq_setsockopt(rep_socket.get(), ZMQ_MAXMSGSIZE, std::addressof(max_message_size), sizeof(max_message_size)) != 0) + { + MONERO_LOG_ZMQ_ERROR("Failed to set maximum incoming message size"); + return false; + } - if (address.empty()) - address = "*"; - if (port.empty()) - port = "*"; - std::string bind_address = addr_prefix + address + std::string(":") + port; - rep_socket->bind(bind_address.c_str()); + static constexpr const int linger_value = std::chrono::milliseconds{linger_timeout}.count(); + if (zmq_setsockopt(rep_socket.get(), ZMQ_LINGER, std::addressof(linger_value), sizeof(linger_value)) != 0) + { + MONERO_LOG_ZMQ_ERROR("Failed to set linger timeout"); + return false; } - catch (const std::exception& e) + + if (address.empty()) + address = "*"; + if (port.empty()) + port = "*"; + + std::string bind_address = "tcp://"; + bind_address.append(address.data(), address.size()); + bind_address += ":"; + bind_address.append(port.data(), port.size()); + + if (zmq_bind(rep_socket.get(), bind_address.c_str()) < 0) { - MERROR(std::string("Error creating ZMQ Socket: ") + e.what()); + MONERO_LOG_ZMQ_ERROR("ZMQ RPC Server bind failed"); return false; } return true; @@ -120,22 +148,16 @@ bool ZmqServer::addTCPSocket(std::string address, std::string port) void ZmqServer::run() { - running = true; run_thread = boost::thread(boost::bind(&ZmqServer::serve, this)); } void ZmqServer::stop() { - if (!running) return; - - stop_signal = true; + if (!run_thread.joinable()) + return; - run_thread.interrupt(); + context.reset(); // destroying context terminates all calls run_thread.join(); - - running = false; - - return; } diff --git a/src/rpc/zmq_server.h b/src/rpc/zmq_server.h index 1b1e4c7cf..ce7892dab 100644 --- a/src/rpc/zmq_server.h +++ b/src/rpc/zmq_server.h @@ -29,12 +29,10 @@ #pragma once #include <boost/thread/thread.hpp> -#include <zmq.hpp> -#include <string> -#include <memory> +#include <boost/utility/string_ref.hpp> #include "common/command_line.h" - +#include "net/zmq.h" #include "rpc_handler.h" namespace cryptonote @@ -43,9 +41,6 @@ namespace cryptonote namespace rpc { -static constexpr int DEFAULT_NUM_ZMQ_THREADS = 1; -static constexpr int DEFAULT_RPC_RECV_TIMEOUT_MS = 1000; - class ZmqServer { public: @@ -58,8 +53,8 @@ class ZmqServer void serve(); - bool addIPCSocket(std::string address, std::string port); - bool addTCPSocket(std::string address, std::string port); + bool addIPCSocket(boost::string_ref address, boost::string_ref port); + bool addTCPSocket(boost::string_ref address, boost::string_ref port); void run(); void stop(); @@ -67,14 +62,11 @@ class ZmqServer private: RpcHandler& handler; - volatile bool stop_signal; - volatile bool running; - - zmq::context_t context; + net::zmq::context context; boost::thread run_thread; - std::unique_ptr<zmq::socket_t> rep_socket; + net::zmq::socket rep_socket; }; diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index af8d7526d..fe384e529 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -2283,9 +2283,16 @@ bool simple_wallet::set_default_ring_size(const std::vector<std::string> &args/* } if (ring_size != 0 && ring_size != DEFAULT_MIX+1) - message_writer() << tr("WARNING: this is a non default ring size, which may harm your privacy. Default is recommended."); - else if (ring_size == DEFAULT_MIX) - message_writer() << tr("WARNING: from v8, ring size will be fixed and this setting will be ignored."); + { + if (m_wallet->use_fork_rules(8, 0)) + { + message_writer() << tr("WARNING: from v8, ring size will be fixed and this setting will be ignored."); + } + else + { + message_writer() << tr("WARNING: this is a non default ring size, which may harm your privacy. Default is recommended."); + } + } const auto pwd_container = get_and_verify_password(); if (pwd_container) @@ -4196,7 +4203,22 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) } if (!m_wallet->is_trusted_daemon()) - message_writer() << (boost::format(tr("Warning: using an untrusted daemon at %s, privacy will be lessened")) % m_wallet->get_daemon_address()).str(); + { + message_writer(console_color_red, true) << (boost::format(tr("Warning: using an untrusted daemon at %s")) % m_wallet->get_daemon_address()).str(); + message_writer(console_color_red, true) << boost::format(tr("Using a third party daemon can be detrimental to your security and privacy")); + bool ssl = false; + if (m_wallet->check_connection(NULL, &ssl) && !ssl) + message_writer(console_color_red, true) << boost::format(tr("Using your own without SSL exposes your RPC traffic to monitoring")); + message_writer(console_color_red, true) << boost::format(tr("You are strongly encouraged to connect to the Monero network using your own daemon")); + message_writer(console_color_red, true) << boost::format(tr("If you or someone you trust are operating this daemon, you can use --trusted-daemon")); + + COMMAND_RPC_GET_INFO::request req; + COMMAND_RPC_GET_INFO::response res; + bool r = m_wallet->invoke_http_json("/get_info", req, res); + std::string err = interpret_rpc_response(r, res.status); + if (r && err.empty() && (res.was_bootstrap_ever_used || !res.bootstrap_daemon_address.empty())) + message_writer(console_color_red, true) << boost::format(tr("Moreover, a daemon is also less secure when running in bootstrap mode")); + } if (m_wallet->get_ring_database().empty()) fail_msg_writer() << tr("Failed to initialize ring database: privacy enhancing features will be inactive"); @@ -4446,7 +4468,7 @@ boost::optional<epee::wipeable_string> simple_wallet::new_wallet(const boost::pr } success_msg_writer() << "**********************************************************************"; - return std::move(password); + return password; } //---------------------------------------------------------------------------------------------------- boost::optional<epee::wipeable_string> simple_wallet::new_wallet(const boost::program_options::variables_map& vm, @@ -4495,7 +4517,7 @@ boost::optional<epee::wipeable_string> simple_wallet::new_wallet(const boost::pr } - return std::move(password); + return password; } //---------------------------------------------------------------------------------------------------- @@ -4538,7 +4560,7 @@ boost::optional<epee::wipeable_string> simple_wallet::new_wallet(const boost::pr return {}; } - return std::move(password); + return password; } //---------------------------------------------------------------------------------------------------- boost::optional<epee::wipeable_string> simple_wallet::new_wallet(const boost::program_options::variables_map& vm, @@ -4593,7 +4615,7 @@ boost::optional<epee::wipeable_string> simple_wallet::new_wallet(const boost::pr return {}; } - return std::move(password); + return password; } //---------------------------------------------------------------------------------------------------- boost::optional<epee::wipeable_string> simple_wallet::open_wallet(const boost::program_options::variables_map& vm) @@ -4696,7 +4718,7 @@ boost::optional<epee::wipeable_string> simple_wallet::open_wallet(const boost::p tr("Use the \"help\" command to see the list of available commands.\n") << tr("Use \"help <command>\" to see a command's documentation.\n") << "**********************************************************************"; - return std::move(password); + return password; } //---------------------------------------------------------------------------------------------------- bool simple_wallet::close_wallet() @@ -5320,14 +5342,14 @@ bool simple_wallet::show_balance_unlocked(bool detailed) const std::string tag = m_wallet->get_account_tags().second[m_current_subaddress_account]; success_msg_writer() << tr("Tag: ") << (tag.empty() ? std::string{tr("(No tag assigned)")} : tag); uint64_t blocks_to_unlock; - uint64_t unlocked_balance = m_wallet->unlocked_balance(m_current_subaddress_account, &blocks_to_unlock); + uint64_t unlocked_balance = m_wallet->unlocked_balance(m_current_subaddress_account, false, &blocks_to_unlock); std::string unlock_time_message; if (blocks_to_unlock > 0) unlock_time_message = (boost::format(" (%lu block(s) to unlock)") % blocks_to_unlock).str(); - success_msg_writer() << tr("Balance: ") << print_money(m_wallet->balance(m_current_subaddress_account)) << ", " + success_msg_writer() << tr("Balance: ") << print_money(m_wallet->balance(m_current_subaddress_account, false)) << ", " << tr("unlocked balance: ") << print_money(unlocked_balance) << unlock_time_message << extra; - std::map<uint32_t, uint64_t> balance_per_subaddress = m_wallet->balance_per_subaddress(m_current_subaddress_account); - std::map<uint32_t, std::pair<uint64_t, uint64_t>> unlocked_balance_per_subaddress = m_wallet->unlocked_balance_per_subaddress(m_current_subaddress_account); + std::map<uint32_t, uint64_t> balance_per_subaddress = m_wallet->balance_per_subaddress(m_current_subaddress_account, false); + std::map<uint32_t, std::pair<uint64_t, uint64_t>> unlocked_balance_per_subaddress = m_wallet->unlocked_balance_per_subaddress(m_current_subaddress_account, false); if (!detailed || balance_per_subaddress.empty()) return true; success_msg_writer() << tr("Balance per address:"); @@ -5859,14 +5881,11 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri priority = m_wallet->adjust_priority(priority); - size_t fake_outs_count = 0; + size_t fake_outs_count = DEFAULT_MIX; if(local_args.size() > 0) { size_t ring_size; if(!epee::string_tools::get_xtype_from_string(ring_size, local_args[0])) { - fake_outs_count = m_wallet->default_mixin(); - if (fake_outs_count == 0) - fake_outs_count = DEFAULT_MIX; } else if (ring_size == 0) { @@ -6480,14 +6499,11 @@ bool simple_wallet::sweep_main(uint64_t below, bool locked, const std::vector<st priority = m_wallet->adjust_priority(priority); - size_t fake_outs_count = 0; + size_t fake_outs_count = DEFAULT_MIX; if(local_args.size() > 0) { size_t ring_size; if(!epee::string_tools::get_xtype_from_string(ring_size, local_args[0])) { - fake_outs_count = m_wallet->default_mixin(); - if (fake_outs_count == 0) - fake_outs_count = DEFAULT_MIX; } else if (ring_size == 0) { @@ -6779,14 +6795,11 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) priority = m_wallet->adjust_priority(priority); - size_t fake_outs_count = 0; + size_t fake_outs_count = DEFAULT_MIX; if(local_args.size() > 0) { size_t ring_size; if(!epee::string_tools::get_xtype_from_string(ring_size, local_args[0])) { - fake_outs_count = m_wallet->default_mixin(); - if (fake_outs_count == 0) - fake_outs_count = DEFAULT_MIX; } else if (ring_size == 0) { @@ -8728,7 +8741,7 @@ void simple_wallet::print_accounts() print_accounts(""); if (num_untagged_accounts < m_wallet->get_num_subaddress_accounts()) - success_msg_writer() << tr("\nGrand total:\n Balance: ") << print_money(m_wallet->balance_all()) << tr(", unlocked balance: ") << print_money(m_wallet->unlocked_balance_all()); + success_msg_writer() << tr("\nGrand total:\n Balance: ") << print_money(m_wallet->balance_all(false)) << tr(", unlocked balance: ") << print_money(m_wallet->unlocked_balance_all(false)); } //---------------------------------------------------------------------------------------------------- void simple_wallet::print_accounts(const std::string& tag) @@ -8758,11 +8771,11 @@ void simple_wallet::print_accounts(const std::string& tag) % (m_current_subaddress_account == account_index ? '*' : ' ') % account_index % m_wallet->get_subaddress_as_str({account_index, 0}).substr(0, 6) - % print_money(m_wallet->balance(account_index)) - % print_money(m_wallet->unlocked_balance(account_index)) + % print_money(m_wallet->balance(account_index, false)) + % print_money(m_wallet->unlocked_balance(account_index, false)) % m_wallet->get_subaddress_label({account_index, 0}); - total_balance += m_wallet->balance(account_index); - total_unlocked_balance += m_wallet->unlocked_balance(account_index); + total_balance += m_wallet->balance(account_index, false); + total_unlocked_balance += m_wallet->unlocked_balance(account_index, false); } success_msg_writer() << tr("----------------------------------------------------------------------------------"); success_msg_writer() << boost::format(tr("%15s %21s %21s")) % "Total" % print_money(total_balance) % print_money(total_unlocked_balance); diff --git a/src/wallet/api/subaddress_account.cpp b/src/wallet/api/subaddress_account.cpp index 9bc9d1d91..eaaddc11f 100644 --- a/src/wallet/api/subaddress_account.cpp +++ b/src/wallet/api/subaddress_account.cpp @@ -64,8 +64,8 @@ void SubaddressAccountImpl::refresh() i, m_wallet->m_wallet->get_subaddress_as_str({i,0}), m_wallet->m_wallet->get_subaddress_label({i,0}), - cryptonote::print_money(m_wallet->m_wallet->balance(i)), - cryptonote::print_money(m_wallet->m_wallet->unlocked_balance(i)) + cryptonote::print_money(m_wallet->m_wallet->balance(i, false)), + cryptonote::print_money(m_wallet->m_wallet->unlocked_balance(i, false)) )); } } diff --git a/src/wallet/api/wallet.cpp b/src/wallet/api/wallet.cpp index e632b8d23..7120485d5 100644 --- a/src/wallet/api/wallet.cpp +++ b/src/wallet/api/wallet.cpp @@ -993,12 +993,12 @@ void WalletImpl::setSubaddressLookahead(uint32_t major, uint32_t minor) uint64_t WalletImpl::balance(uint32_t accountIndex) const { - return m_wallet->balance(accountIndex); + return m_wallet->balance(accountIndex, false); } uint64_t WalletImpl::unlockedBalance(uint32_t accountIndex) const { - return m_wallet->unlocked_balance(accountIndex); + return m_wallet->unlocked_balance(accountIndex, false); } uint64_t WalletImpl::blockChainHeight() const diff --git a/src/wallet/message_store.cpp b/src/wallet/message_store.cpp index 96d4ef3ce..6e2cb933f 100644 --- a/src/wallet/message_store.cpp +++ b/src/wallet/message_store.cpp @@ -699,7 +699,7 @@ void message_store::write_to_file(const multisig_wallet_state &state, const std: crypto::chacha_key key; crypto::generate_chacha_key(&state.view_secret_key, sizeof(crypto::secret_key), key, 1); - file_data write_file_data = boost::value_initialized<file_data>(); + file_data write_file_data = {}; write_file_data.magic_string = "MMS"; write_file_data.file_version = 0; write_file_data.iv = crypto::rand<crypto::chacha_iv>(); diff --git a/src/wallet/ringdb.cpp b/src/wallet/ringdb.cpp index 8da95de7b..b7efdd75c 100644 --- a/src/wallet/ringdb.cpp +++ b/src/wallet/ringdb.cpp @@ -424,7 +424,7 @@ bool ringdb::blackball_worker(const std::vector<std::pair<uint64_t, uint64_t>> & { case BLACKBALL_BLACKBALL: MDEBUG("Marking output " << output.first << "/" << output.second << " as spent"); - dbr = mdb_cursor_put(cursor, &key, &data, MDB_APPENDDUP); + dbr = mdb_cursor_put(cursor, &key, &data, MDB_NODUPDATA); if (dbr == MDB_KEYEXIST) dbr = 0; break; diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index b99fc12e2..b85e805de 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -32,7 +32,6 @@ #include <tuple> #include <boost/format.hpp> #include <boost/optional/optional.hpp> -#include <boost/utility/value_init.hpp> #include <boost/algorithm/string/classification.hpp> #include <boost/algorithm/string/trim.hpp> #include <boost/algorithm/string/split.hpp> @@ -1543,6 +1542,7 @@ bool wallet2::is_deprecated() const //---------------------------------------------------------------------------------------------------- void wallet2::set_spent(size_t idx, uint64_t height) { + CHECK_AND_ASSERT_THROW_MES(idx < m_transfers.size(), "Invalid index"); transfer_details &td = m_transfers[idx]; LOG_PRINT_L2("Setting SPENT at " << height << ": ki " << td.m_key_image << ", amount " << print_money(td.m_amount)); td.m_spent = true; @@ -1551,12 +1551,32 @@ void wallet2::set_spent(size_t idx, uint64_t height) //---------------------------------------------------------------------------------------------------- void wallet2::set_unspent(size_t idx) { + CHECK_AND_ASSERT_THROW_MES(idx < m_transfers.size(), "Invalid index"); transfer_details &td = m_transfers[idx]; LOG_PRINT_L2("Setting UNSPENT: ki " << td.m_key_image << ", amount " << print_money(td.m_amount)); td.m_spent = false; td.m_spent_height = 0; } //---------------------------------------------------------------------------------------------------- +bool wallet2::is_spent(const transfer_details &td, bool strict) const +{ + if (strict) + { + return td.m_spent && td.m_spent_height > 0; + } + else + { + return td.m_spent; + } +} +//---------------------------------------------------------------------------------------------------- +bool wallet2::is_spent(size_t idx, bool strict) const +{ + CHECK_AND_ASSERT_THROW_MES(idx < m_transfers.size(), "Invalid index"); + const transfer_details &td = m_transfers[idx]; + return is_spent(td, strict); +} +//---------------------------------------------------------------------------------------------------- void wallet2::freeze(size_t idx) { CHECK_AND_ASSERT_THROW_MES(idx < m_transfers.size(), "Invalid transfer_details index"); @@ -1978,7 +1998,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote uint64_t amount = tx.vout[o].amount ? tx.vout[o].amount : tx_scan_info[o].amount; if (!pool) { - m_transfers.push_back(boost::value_initialized<transfer_details>()); + m_transfers.push_back(transfer_details{}); transfer_details& td = m_transfers.back(); td.m_block_height = height; td.m_internal_output_index = o; @@ -3059,6 +3079,21 @@ bool wallet2::add_address_book_row(const cryptonote::account_public_address &add return false; } +bool wallet2::set_address_book_row(size_t row_id, const cryptonote::account_public_address &address, const crypto::hash &payment_id, const std::string &description, bool is_subaddress) +{ + wallet2::address_book_row a; + a.m_address = address; + a.m_payment_id = payment_id; + a.m_description = description; + a.m_is_subaddress = is_subaddress; + + const auto size = m_address_book.size(); + if (row_id >= size) + return false; + m_address_book[row_id] = a; + return true; +} + bool wallet2::delete_address_book_row(std::size_t row_id) { if(m_address_book.size() <= row_id) return false; @@ -3294,7 +3329,7 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo m_first_refresh_done = true; - LOG_PRINT_L1("Refresh done, blocks received: " << blocks_fetched << ", balance (all accounts): " << print_money(balance_all()) << ", unlocked: " << print_money(unlocked_balance_all())); + LOG_PRINT_L1("Refresh done, blocks received: " << blocks_fetched << ", balance (all accounts): " << print_money(balance_all(false)) << ", unlocked: " << print_money(unlocked_balance_all(false))); } //---------------------------------------------------------------------------------------------------- bool wallet2::refresh(bool trusted_daemon, uint64_t & blocks_fetched, bool& received_money, bool& ok) @@ -3536,7 +3571,7 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable bool r = epee::serialization::store_t_to_binary(account, account_data); CHECK_AND_ASSERT_MES(r, false, "failed to serialize wallet keys"); - wallet2::keys_file_data keys_file_data = boost::value_initialized<wallet2::keys_file_data>(); + wallet2::keys_file_data keys_file_data = {}; // Create a JSON object with "key_data" and "seed_language" as keys. rapidjson::Document json; @@ -5509,7 +5544,7 @@ void wallet2::store_to(const std::string &path, const epee::wipeable_string &pas boost::archive::portable_binary_oarchive ar(oss); ar << *this; - wallet2::cache_file_data cache_file_data = boost::value_initialized<wallet2::cache_file_data>(); + wallet2::cache_file_data cache_file_data = {}; cache_file_data.cache_data = oss.str(); std::string cipher; cipher.resize(cache_file_data.cache_data.size()); @@ -5594,24 +5629,24 @@ void wallet2::store_to(const std::string &path, const epee::wipeable_string &pas } //---------------------------------------------------------------------------------------------------- -uint64_t wallet2::balance(uint32_t index_major) const +uint64_t wallet2::balance(uint32_t index_major, bool strict) const { uint64_t amount = 0; if(m_light_wallet) return m_light_wallet_unlocked_balance; - for (const auto& i : balance_per_subaddress(index_major)) + for (const auto& i : balance_per_subaddress(index_major, strict)) amount += i.second; return amount; } //---------------------------------------------------------------------------------------------------- -uint64_t wallet2::unlocked_balance(uint32_t index_major, uint64_t *blocks_to_unlock) const +uint64_t wallet2::unlocked_balance(uint32_t index_major, bool strict, uint64_t *blocks_to_unlock) const { uint64_t amount = 0; if (blocks_to_unlock) *blocks_to_unlock = 0; if(m_light_wallet) return m_light_wallet_balance; - for (const auto& i : unlocked_balance_per_subaddress(index_major)) + for (const auto& i : unlocked_balance_per_subaddress(index_major, strict)) { amount += i.second.first; if (blocks_to_unlock && i.second.second > *blocks_to_unlock) @@ -5620,12 +5655,12 @@ uint64_t wallet2::unlocked_balance(uint32_t index_major, uint64_t *blocks_to_unl return amount; } //---------------------------------------------------------------------------------------------------- -std::map<uint32_t, uint64_t> wallet2::balance_per_subaddress(uint32_t index_major) const +std::map<uint32_t, uint64_t> wallet2::balance_per_subaddress(uint32_t index_major, bool strict) const { std::map<uint32_t, uint64_t> amount_per_subaddr; for (const auto& td: m_transfers) { - if (td.m_subaddr_index.major == index_major && !td.m_spent && !td.m_frozen) + if (td.m_subaddr_index.major == index_major && !is_spent(td, strict) && !td.m_frozen) { auto found = amount_per_subaddr.find(td.m_subaddr_index.minor); if (found == amount_per_subaddr.end()) @@ -5634,8 +5669,10 @@ std::map<uint32_t, uint64_t> wallet2::balance_per_subaddress(uint32_t index_majo found->second += td.amount(); } } - for (const auto& utx: m_unconfirmed_txs) + if (!strict) { + for (const auto& utx: m_unconfirmed_txs) + { if (utx.second.m_subaddr_account == index_major && utx.second.m_state != wallet2::unconfirmed_transfer_details::failed) { // all changes go to 0-th subaddress (in the current subaddress account) @@ -5645,17 +5682,18 @@ std::map<uint32_t, uint64_t> wallet2::balance_per_subaddress(uint32_t index_majo else found->second += utx.second.m_change; } + } } return amount_per_subaddr; } //---------------------------------------------------------------------------------------------------- -std::map<uint32_t, std::pair<uint64_t, uint64_t>> wallet2::unlocked_balance_per_subaddress(uint32_t index_major) const +std::map<uint32_t, std::pair<uint64_t, uint64_t>> wallet2::unlocked_balance_per_subaddress(uint32_t index_major, bool strict) const { std::map<uint32_t, std::pair<uint64_t, uint64_t>> amount_per_subaddr; const uint64_t blockchain_height = get_blockchain_current_height(); for(const transfer_details& td: m_transfers) { - if(td.m_subaddr_index.major == index_major && !td.m_spent && !td.m_frozen) + if(td.m_subaddr_index.major == index_major && !is_spent(td, strict) && !td.m_frozen) { uint64_t amount = 0, blocks_to_unlock = 0; if (is_transfer_unlocked(td)) @@ -5684,15 +5722,15 @@ std::map<uint32_t, std::pair<uint64_t, uint64_t>> wallet2::unlocked_balance_per_ return amount_per_subaddr; } //---------------------------------------------------------------------------------------------------- -uint64_t wallet2::balance_all() const +uint64_t wallet2::balance_all(bool strict) const { uint64_t r = 0; for (uint32_t index_major = 0; index_major < get_num_subaddress_accounts(); ++index_major) - r += balance(index_major); + r += balance(index_major, strict); return r; } //---------------------------------------------------------------------------------------------------- -uint64_t wallet2::unlocked_balance_all(uint64_t *blocks_to_unlock) const +uint64_t wallet2::unlocked_balance_all(bool strict, uint64_t *blocks_to_unlock) const { uint64_t r = 0; if (blocks_to_unlock) @@ -5700,7 +5738,7 @@ uint64_t wallet2::unlocked_balance_all(uint64_t *blocks_to_unlock) const for (uint32_t index_major = 0; index_major < get_num_subaddress_accounts(); ++index_major) { uint64_t local_blocks_to_unlock; - r += unlocked_balance(index_major, blocks_to_unlock ? &local_blocks_to_unlock : NULL); + r += unlocked_balance(index_major, strict, blocks_to_unlock ? &local_blocks_to_unlock : NULL); if (blocks_to_unlock) *blocks_to_unlock = std::max(*blocks_to_unlock, local_blocks_to_unlock); } @@ -6177,8 +6215,8 @@ void wallet2::commit_tx(pending_tx& ptx) //fee includes dust if dust policy specified it. LOG_PRINT_L1("Transaction successfully sent. <" << txid << ">" << ENDL << "Commission: " << print_money(ptx.fee) << " (dust sent to dust addr: " << print_money((ptx.dust_added_to_fee ? 0 : ptx.dust)) << ")" << ENDL - << "Balance: " << print_money(balance(ptx.construction_data.subaddr_account)) << ENDL - << "Unlocked: " << print_money(unlocked_balance(ptx.construction_data.subaddr_account)) << ENDL + << "Balance: " << print_money(balance(ptx.construction_data.subaddr_account, false)) << ENDL + << "Unlocked: " << print_money(unlocked_balance(ptx.construction_data.subaddr_account, false)) << ENDL << "Please, wait for confirmation for your balance to be unlocked."); } @@ -6346,7 +6384,7 @@ bool wallet2::sign_tx(unsigned_tx_set &exported_txs, std::vector<wallet2::pendin // normally, the tx keys are saved in commit_tx, when the tx is actually sent to the daemon. // we can't do that here since the tx will be sent from the compromised wallet, which we don't want // to see that info, so we save it here - if (store_tx_info() && ptx.tx_key != crypto::null_skey) + if (store_tx_info() && tx_key != crypto::null_skey) { const crypto::hash txid = get_transaction_hash(ptx.tx); m_tx_keys.insert(std::make_pair(txid, tx_key)); @@ -7258,9 +7296,7 @@ bool wallet2::unset_ring(const crypto::hash &txid) bool ok = invoke_http_json("/gettransactions", req, res, rpc_timeout); m_daemon_rpc_mutex.unlock(); THROW_WALLET_EXCEPTION_IF(!ok, error::wallet_internal_error, "Failed to get transaction from daemon"); - if (res.txs.empty()) - return false; - THROW_WALLET_EXCEPTION_IF(res.txs.size(), error::wallet_internal_error, "Failed to get transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(res.txs.size() != 1, error::wallet_internal_error, "Failed to get transaction from daemon"); cryptonote::transaction tx; crypto::hash tx_hash; @@ -7435,8 +7471,8 @@ void wallet2::light_wallet_get_outs(std::vector<std::vector<tools::wallet2::get_ MDEBUG("LIGHTWALLET - Getting random outs"); - cryptonote::COMMAND_RPC_GET_RANDOM_OUTS::request oreq; - cryptonote::COMMAND_RPC_GET_RANDOM_OUTS::response ores; + tools::COMMAND_RPC_GET_RANDOM_OUTS::request oreq; + tools::COMMAND_RPC_GET_RANDOM_OUTS::response ores; size_t light_wallet_requested_outputs_count = (size_t)((fake_outputs_count + 1) * 1.5 + 1); @@ -8561,7 +8597,7 @@ std::vector<size_t> wallet2::pick_preferred_rct_inputs(uint64_t needed_money, ui for (size_t i = 0; i < m_transfers.size(); ++i) { const transfer_details& td = m_transfers[i]; - if (!td.m_spent && !td.m_frozen && td.is_rct() && td.amount() >= needed_money && is_transfer_unlocked(td) && td.m_subaddr_index.major == subaddr_account && subaddr_indices.count(td.m_subaddr_index.minor) == 1) + if (!is_spent(td, false) && !td.m_frozen && td.is_rct() && td.amount() >= needed_money && is_transfer_unlocked(td) && td.m_subaddr_index.major == subaddr_account && subaddr_indices.count(td.m_subaddr_index.minor) == 1) { LOG_PRINT_L2("We can use " << i << " alone: " << print_money(td.amount())); picks.push_back(i); @@ -8576,13 +8612,13 @@ std::vector<size_t> wallet2::pick_preferred_rct_inputs(uint64_t needed_money, ui for (size_t i = 0; i < m_transfers.size(); ++i) { const transfer_details& td = m_transfers[i]; - if (!td.m_spent && !td.m_frozen && !td.m_key_image_partial && td.is_rct() && is_transfer_unlocked(td) && td.m_subaddr_index.major == subaddr_account && subaddr_indices.count(td.m_subaddr_index.minor) == 1) + if (!is_spent(td, false) && !td.m_frozen && !td.m_key_image_partial && td.is_rct() && is_transfer_unlocked(td) && td.m_subaddr_index.major == subaddr_account && subaddr_indices.count(td.m_subaddr_index.minor) == 1) { LOG_PRINT_L2("Considering input " << i << ", " << print_money(td.amount())); for (size_t j = i + 1; j < m_transfers.size(); ++j) { const transfer_details& td2 = m_transfers[j]; - if (!td2.m_spent && !td2.m_frozen && !td.m_key_image_partial && td2.is_rct() && td.amount() + td2.amount() >= needed_money && is_transfer_unlocked(td2) && td2.m_subaddr_index == td.m_subaddr_index) + if (!is_spent(td2, false) && !td2.m_frozen && !td.m_key_image_partial && td2.is_rct() && td.amount() + td2.amount() >= needed_money && is_transfer_unlocked(td2) && td2.m_subaddr_index == td.m_subaddr_index) { // update our picks if those outputs are less related than any we // already found. If the same, don't update, and oldest suitable outputs @@ -8791,7 +8827,7 @@ void wallet2::light_wallet_get_unspent_outs() if(!add_transfer) continue; - m_transfers.push_back(boost::value_initialized<transfer_details>()); + m_transfers.push_back(transfer_details{}); transfer_details& td = m_transfers.back(); td.m_block_height = o.height; @@ -9237,8 +9273,8 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp // throw if attempting a transaction with no money THROW_WALLET_EXCEPTION_IF(needed_money == 0, error::zero_destination); - std::map<uint32_t, std::pair<uint64_t, uint64_t>> unlocked_balance_per_subaddr = unlocked_balance_per_subaddress(subaddr_account); - std::map<uint32_t, uint64_t> balance_per_subaddr = balance_per_subaddress(subaddr_account); + std::map<uint32_t, std::pair<uint64_t, uint64_t>> unlocked_balance_per_subaddr = unlocked_balance_per_subaddress(subaddr_account, false); + std::map<uint32_t, uint64_t> balance_per_subaddr = balance_per_subaddress(subaddr_account, false); if (subaddr_indices.empty()) // "index=<N1>[,<N2>,...]" wasn't specified -> use all the indices with non-zero unlocked balance { @@ -9284,7 +9320,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp MDEBUG("Ignoring output " << i << " of amount " << print_money(td.amount()) << " which is below threshold " << print_money(fractional_threshold)); continue; } - if (!td.m_spent && !td.m_frozen && !td.m_key_image_partial && (use_rct ? true : !td.is_rct()) && is_transfer_unlocked(td) && td.m_subaddr_index.major == subaddr_account && subaddr_indices.count(td.m_subaddr_index.minor) == 1) + if (!is_spent(td, false) && !td.m_frozen && !td.m_key_image_partial && (use_rct ? true : !td.is_rct()) && is_transfer_unlocked(td) && td.m_subaddr_index.major == subaddr_account && subaddr_indices.count(td.m_subaddr_index.minor) == 1) { const uint32_t index_minor = td.m_subaddr_index.minor; auto find_predicate = [&index_minor](const std::pair<uint32_t, std::vector<size_t>>& x) { return x.first == index_minor; }; @@ -9411,7 +9447,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp // if we need to spend money and don't have any left, we fail if (unused_dust_indices->empty() && unused_transfers_indices->empty()) { LOG_PRINT_L2("No more outputs to choose from"); - THROW_WALLET_EXCEPTION_IF(1, error::tx_not_possible, unlocked_balance(subaddr_account), needed_money, accumulated_fee + needed_fee); + THROW_WALLET_EXCEPTION_IF(1, error::tx_not_possible, unlocked_balance(subaddr_account, false), needed_money, accumulated_fee + needed_fee); } // get a random unspent output and use it to pay part (or all) of the current destination (and maybe next one, etc) @@ -9629,7 +9665,7 @@ skip_tx: if (adding_fee) { LOG_PRINT_L1("We ran out of outputs while trying to gather final fee"); - THROW_WALLET_EXCEPTION_IF(1, error::tx_not_possible, unlocked_balance(subaddr_account), needed_money, accumulated_fee + needed_fee); + THROW_WALLET_EXCEPTION_IF(1, error::tx_not_possible, unlocked_balance(subaddr_account, false), needed_money, accumulated_fee + needed_fee); } LOG_PRINT_L1("Done creating " << txes.size() << " transactions, " << print_money(accumulated_fee) << @@ -9764,7 +9800,18 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_all(uint64_t below std::vector<size_t> unused_dust_indices; const bool use_rct = use_fork_rules(4, 0); - THROW_WALLET_EXCEPTION_IF(unlocked_balance(subaddr_account) == 0, error::wallet_internal_error, "No unlocked balance in the entire wallet"); + // determine threshold for fractional amount + const bool use_per_byte_fee = use_fork_rules(HF_VERSION_PER_BYTE_FEE, 0); + const bool bulletproof = use_fork_rules(get_bulletproof_fork(), 0); + const uint64_t base_fee = get_base_fee(); + const uint64_t fee_multiplier = get_fee_multiplier(priority, get_fee_algorithm()); + const size_t tx_weight_one_ring = estimate_tx_weight(use_rct, 1, fake_outs_count, 2, 0, bulletproof); + const size_t tx_weight_two_rings = estimate_tx_weight(use_rct, 2, fake_outs_count, 2, 0, bulletproof); + THROW_WALLET_EXCEPTION_IF(tx_weight_one_ring > tx_weight_two_rings, error::wallet_internal_error, "Estimated tx weight with 1 input is larger than with 2 inputs!"); + const size_t tx_weight_per_ring = tx_weight_two_rings - tx_weight_one_ring; + const uint64_t fractional_threshold = (fee_multiplier * base_fee * tx_weight_per_ring) / (use_per_byte_fee ? 1 : 1024); + + THROW_WALLET_EXCEPTION_IF(unlocked_balance(subaddr_account, false) == 0, error::wallet_internal_error, "No unlocked balance in the entire wallet"); std::map<uint32_t, std::pair<std::vector<size_t>, std::vector<size_t>>> unused_transfer_dust_indices_per_subaddr; @@ -9773,7 +9820,12 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_all(uint64_t below for (size_t i = 0; i < m_transfers.size(); ++i) { const transfer_details& td = m_transfers[i]; - if (!td.m_spent && !td.m_frozen && !td.m_key_image_partial && (use_rct ? true : !td.is_rct()) && is_transfer_unlocked(td) && td.m_subaddr_index.major == subaddr_account && (subaddr_indices.empty() || subaddr_indices.count(td.m_subaddr_index.minor) == 1)) + if (m_ignore_fractional_outputs && td.amount() < fractional_threshold) + { + MDEBUG("Ignoring output " << i << " of amount " << print_money(td.amount()) << " which is below threshold " << print_money(fractional_threshold)); + continue; + } + if (!is_spent(td, false) && !td.m_frozen && !td.m_key_image_partial && (use_rct ? true : !td.is_rct()) && is_transfer_unlocked(td) && td.m_subaddr_index.major == subaddr_account && (subaddr_indices.empty() || subaddr_indices.count(td.m_subaddr_index.minor) == 1)) { fund_found = true; if (below == 0 || td.amount() < below) @@ -9821,7 +9873,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_single(const crypt for (size_t i = 0; i < m_transfers.size(); ++i) { const transfer_details& td = m_transfers[i]; - if (td.m_key_image_known && td.m_key_image == ki && !td.m_spent && !td.m_frozen && (use_rct ? true : !td.is_rct()) && is_transfer_unlocked(td)) + if (td.m_key_image_known && td.m_key_image == ki && !is_spent(td, false) && !td.m_frozen && (use_rct ? true : !td.is_rct()) && is_transfer_unlocked(td)) { if (td.is_rct() || is_valid_decomposed_amount(td.amount())) unused_transfers_indices.push_back(i); @@ -10187,7 +10239,7 @@ std::vector<size_t> wallet2::select_available_outputs(const std::function<bool(c size_t n = 0; for (transfer_container::const_iterator i = m_transfers.begin(); i != m_transfers.end(); ++i, ++n) { - if (i->m_spent) + if (is_spent(*i, false)) continue; if (i->m_frozen) continue; @@ -10201,12 +10253,12 @@ std::vector<size_t> wallet2::select_available_outputs(const std::function<bool(c return outputs; } //---------------------------------------------------------------------------------------------------- -std::vector<uint64_t> wallet2::get_unspent_amounts_vector() const +std::vector<uint64_t> wallet2::get_unspent_amounts_vector(bool strict) const { std::set<uint64_t> set; for (const auto &td: m_transfers) { - if (!td.m_spent && !td.m_frozen) + if (!is_spent(td, strict) && !td.m_frozen) set.insert(td.is_rct() ? 0 : td.amount()); } std::vector<uint64_t> vector; @@ -10224,7 +10276,7 @@ std::vector<size_t> wallet2::select_available_outputs_from_histogram(uint64_t co cryptonote::COMMAND_RPC_GET_OUTPUT_HISTOGRAM::response resp_t = AUTO_VAL_INIT(resp_t); m_daemon_rpc_mutex.lock(); if (is_trusted_daemon()) - req_t.amounts = get_unspent_amounts_vector(); + req_t.amounts = get_unspent_amounts_vector(false); req_t.min_count = count; req_t.max_count = 0; req_t.unlocked = unlocked; @@ -11122,8 +11174,8 @@ bool wallet2::check_tx_proof(const cryptonote::transaction &tx, const cryptonote std::string wallet2::get_reserve_proof(const boost::optional<std::pair<uint32_t, uint64_t>> &account_minreserve, const std::string &message) { THROW_WALLET_EXCEPTION_IF(m_watch_only || m_multisig, error::wallet_internal_error, "Reserve proof can only be generated by a full wallet"); - THROW_WALLET_EXCEPTION_IF(balance_all() == 0, error::wallet_internal_error, "Zero balance"); - THROW_WALLET_EXCEPTION_IF(account_minreserve && balance(account_minreserve->first) < account_minreserve->second, error::wallet_internal_error, + THROW_WALLET_EXCEPTION_IF(balance_all(true) == 0, error::wallet_internal_error, "Zero balance"); + THROW_WALLET_EXCEPTION_IF(account_minreserve && balance(account_minreserve->first, true) < account_minreserve->second, error::wallet_internal_error, "Not enough balance in this account for the requested minimum reserve amount"); // determine which outputs to include in the proof @@ -11131,7 +11183,7 @@ std::string wallet2::get_reserve_proof(const boost::optional<std::pair<uint32_t, for (size_t i = 0; i < m_transfers.size(); ++i) { const transfer_details &td = m_transfers[i]; - if (!td.m_spent && !td.m_frozen && (!account_minreserve || account_minreserve->first == td.m_subaddr_index.major)) + if (!is_spent(td, true) && !td.m_frozen && (!account_minreserve || account_minreserve->first == td.m_subaddr_index.major)) selected_transfers.push_back(i); } diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 5607602e7..1469b4c00 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -804,14 +804,14 @@ private: bool reconnect_device(); // locked & unlocked balance of given or current subaddress account - uint64_t balance(uint32_t subaddr_index_major) const; - uint64_t unlocked_balance(uint32_t subaddr_index_major, uint64_t *blocks_to_unlock = NULL) const; + uint64_t balance(uint32_t subaddr_index_major, bool strict) const; + uint64_t unlocked_balance(uint32_t subaddr_index_major, bool strict, uint64_t *blocks_to_unlock = NULL) const; // locked & unlocked balance per subaddress of given or current subaddress account - std::map<uint32_t, uint64_t> balance_per_subaddress(uint32_t subaddr_index_major) const; - std::map<uint32_t, std::pair<uint64_t, uint64_t>> unlocked_balance_per_subaddress(uint32_t subaddr_index_major) const; + std::map<uint32_t, uint64_t> balance_per_subaddress(uint32_t subaddr_index_major, bool strict) const; + std::map<uint32_t, std::pair<uint64_t, uint64_t>> unlocked_balance_per_subaddress(uint32_t subaddr_index_major, bool strict) const; // all locked & unlocked balances of all subaddress accounts - uint64_t balance_all() const; - uint64_t unlocked_balance_all(uint64_t *blocks_to_unlock = NULL) const; + uint64_t balance_all(bool strict) const; + uint64_t unlocked_balance_all(bool strict, uint64_t *blocks_to_unlock = NULL) const; template<typename T> void transfer_selected(const std::vector<cryptonote::tx_destination_entry>& dsts, const std::vector<size_t>& selected_transfers, size_t fake_outputs_count, std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, @@ -1098,6 +1098,7 @@ private: */ std::vector<address_book_row> get_address_book() const { return m_address_book; } bool add_address_book_row(const cryptonote::account_public_address &address, const crypto::hash &payment_id, const std::string &description, bool is_subaddress); + bool set_address_book_row(size_t row_id, const cryptonote::account_public_address &address, const crypto::hash &payment_id, const std::string &description, bool is_subaddress); bool delete_address_book_row(std::size_t row_id); uint64_t get_num_rct_outputs(); @@ -1377,12 +1378,14 @@ private: void check_acc_out_precomp_once(const cryptonote::tx_out &o, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, size_t i, const is_out_data *is_out_data, tx_scan_info_t &tx_scan_info, bool &already_seen) const; void parse_block_round(const cryptonote::blobdata &blob, cryptonote::block &bl, crypto::hash &bl_id, bool &error) const; uint64_t get_upper_transaction_weight_limit() const; - std::vector<uint64_t> get_unspent_amounts_vector() const; + std::vector<uint64_t> get_unspent_amounts_vector(bool strict) const; uint64_t get_dynamic_base_fee_estimate() const; float get_output_relatedness(const transfer_details &td0, const transfer_details &td1) const; std::vector<size_t> pick_preferred_rct_inputs(uint64_t needed_money, uint32_t subaddr_account, const std::set<uint32_t> &subaddr_indices) const; void set_spent(size_t idx, uint64_t height); void set_unspent(size_t idx); + bool is_spent(const transfer_details &td, bool strict = true) const; + bool is_spent(size_t idx, bool strict = true) const; void get_outs(std::vector<std::vector<get_outs_entry>> &outs, const std::vector<size_t> &selected_transfers, size_t fake_outputs_count); bool tx_add_fake_output(std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, uint64_t global_index, const crypto::public_key& tx_public_key, const rct::key& mask, uint64_t real_index, bool unlocked) const; bool should_pick_a_second_output(bool use_rct, size_t n_transfers, const std::vector<size_t> &unused_transfers_indices, const std::vector<size_t> &unused_dust_indices) const; diff --git a/src/wallet/wallet_light_rpc.h b/src/wallet/wallet_light_rpc.h index 1d35cec33..c2a7dc021 100644 --- a/src/wallet/wallet_light_rpc.h +++ b/src/wallet/wallet_light_rpc.h @@ -317,4 +317,51 @@ namespace tools typedef epee::misc_utils::struct_init<response_t> response; }; //----------------------------------------------- + struct COMMAND_RPC_GET_RANDOM_OUTS + { + struct request_t + { + std::vector<std::string> amounts; + uint32_t count; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(amounts) + KV_SERIALIZE(count) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<request_t> request; + + struct output { + std::string public_key; + uint64_t global_index; + std::string rct; // 64+64+64 characters long (<rct commit> + <encrypted mask> + <rct amount>) + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(public_key) + KV_SERIALIZE(global_index) + KV_SERIALIZE(rct) + END_KV_SERIALIZE_MAP() + }; + + struct amount_out { + uint64_t amount; + std::vector<output> outputs; + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(amount) + KV_SERIALIZE(outputs) + END_KV_SERIALIZE_MAP() + }; + + struct response_t + { + std::vector<amount_out> amount_outs; + std::string Error; + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(amount_outs) + KV_SERIALIZE(Error) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<response_t> response; + }; + //----------------------------------------------- } diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index 7c87e7114..0e0221c03 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -151,6 +151,7 @@ namespace tools if (m_wallet) { m_wallet->store(); + m_wallet->deinit(); delete m_wallet; m_wallet = NULL; } @@ -326,6 +327,7 @@ namespace tools entry.timestamp = pd.m_timestamp; entry.amount = pd.m_amount; entry.unlock_time = pd.m_unlock_time; + entry.locked = !m_wallet->is_transfer_unlocked(pd.m_unlock_time, pd.m_block_height); entry.fee = pd.m_fee; entry.note = m_wallet->get_tx_note(pd.m_tx_hash); entry.type = pd.m_coinbase ? "block" : "in"; @@ -344,6 +346,7 @@ namespace tools entry.height = pd.m_block_height; entry.timestamp = pd.m_timestamp; entry.unlock_time = pd.m_unlock_time; + entry.locked = !m_wallet->is_transfer_unlocked(pd.m_unlock_time, pd.m_block_height); entry.fee = pd.m_amount_in - pd.m_amount_out; uint64_t change = pd.m_change == (uint64_t)-1 ? 0 : pd.m_change; // change may not be known entry.amount = pd.m_amount_in - change - entry.fee; @@ -377,6 +380,7 @@ namespace tools entry.fee = pd.m_amount_in - pd.m_amount_out; entry.amount = pd.m_amount_in - pd.m_change - entry.fee; entry.unlock_time = pd.m_tx.unlock_time; + entry.locked = true; entry.note = m_wallet->get_tx_note(txid); for (const auto &d: pd.m_dests) { @@ -405,6 +409,7 @@ namespace tools entry.timestamp = pd.m_timestamp; entry.amount = pd.m_amount; entry.unlock_time = pd.m_unlock_time; + entry.locked = true; entry.fee = pd.m_fee; entry.note = m_wallet->get_tx_note(pd.m_tx_hash); entry.double_spend_seen = ppd.m_double_spend_seen; @@ -420,8 +425,8 @@ namespace tools if (!m_wallet) return not_open(er); try { - res.balance = req.all_accounts ? m_wallet->balance_all() : m_wallet->balance(req.account_index); - res.unlocked_balance = req.all_accounts ? m_wallet->unlocked_balance_all(&res.blocks_to_unlock) : m_wallet->unlocked_balance(req.account_index, &res.blocks_to_unlock); + res.balance = req.all_accounts ? m_wallet->balance_all(req.strict) : m_wallet->balance(req.account_index, req.strict); + res.unlocked_balance = req.all_accounts ? m_wallet->unlocked_balance_all(req.strict, &res.blocks_to_unlock) : m_wallet->unlocked_balance(req.account_index, req.strict, &res.blocks_to_unlock); res.multisig_import_needed = m_wallet->multisig() && m_wallet->has_multisig_partial_key_images(); std::map<uint32_t, std::map<uint32_t, uint64_t>> balance_per_subaddress_per_account; std::map<uint32_t, std::map<uint32_t, std::pair<uint64_t, uint64_t>>> unlocked_balance_per_subaddress_per_account; @@ -429,14 +434,14 @@ namespace tools { for (uint32_t account_index = 0; account_index < m_wallet->get_num_subaddress_accounts(); ++account_index) { - balance_per_subaddress_per_account[account_index] = m_wallet->balance_per_subaddress(account_index); - unlocked_balance_per_subaddress_per_account[account_index] = m_wallet->unlocked_balance_per_subaddress(account_index); + balance_per_subaddress_per_account[account_index] = m_wallet->balance_per_subaddress(account_index, req.strict); + unlocked_balance_per_subaddress_per_account[account_index] = m_wallet->unlocked_balance_per_subaddress(account_index, req.strict); } } else { - balance_per_subaddress_per_account[req.account_index] = m_wallet->balance_per_subaddress(req.account_index); - unlocked_balance_per_subaddress_per_account[req.account_index] = m_wallet->unlocked_balance_per_subaddress(req.account_index); + balance_per_subaddress_per_account[req.account_index] = m_wallet->balance_per_subaddress(req.account_index, req.strict); + unlocked_balance_per_subaddress_per_account[req.account_index] = m_wallet->unlocked_balance_per_subaddress(req.account_index, req.strict); } std::vector<tools::wallet2::transfer_details> transfers; m_wallet->get_transfers(transfers); @@ -594,8 +599,8 @@ namespace tools wallet_rpc::COMMAND_RPC_GET_ACCOUNTS::subaddress_account_info info; info.account_index = subaddr_index.major; info.base_address = m_wallet->get_subaddress_as_str(subaddr_index); - info.balance = m_wallet->balance(subaddr_index.major); - info.unlocked_balance = m_wallet->unlocked_balance(subaddr_index.major); + info.balance = m_wallet->balance(subaddr_index.major, req.strict_balances); + info.unlocked_balance = m_wallet->unlocked_balance(subaddr_index.major, req.strict_balances); info.label = m_wallet->get_subaddress_label(subaddr_index); info.tag = account_tags.second[subaddr_index.major]; res.subaddress_accounts.push_back(info); @@ -1698,6 +1703,7 @@ namespace tools rpc_payment.amount = payment.m_amount; rpc_payment.block_height = payment.m_block_height; rpc_payment.unlock_time = payment.m_unlock_time; + rpc_payment.locked = !m_wallet->is_transfer_unlocked(payment.m_unlock_time, payment.m_block_height); rpc_payment.subaddr_index = payment.m_subaddr_index; rpc_payment.address = m_wallet->get_subaddress_as_str(payment.m_subaddr_index); res.payments.push_back(rpc_payment); @@ -1727,6 +1733,7 @@ namespace tools rpc_payment.unlock_time = payment.second.m_unlock_time; rpc_payment.subaddr_index = payment.second.m_subaddr_index; rpc_payment.address = m_wallet->get_subaddress_as_str(payment.second.m_subaddr_index); + rpc_payment.locked = !m_wallet->is_transfer_unlocked(payment.second.m_unlock_time, payment.second.m_block_height); res.payments.push_back(std::move(rpc_payment)); } @@ -1781,6 +1788,7 @@ namespace tools rpc_payment.unlock_time = payment.m_unlock_time; rpc_payment.subaddr_index = payment.m_subaddr_index; rpc_payment.address = m_wallet->get_subaddress_as_str(payment.m_subaddr_index); + rpc_payment.locked = !m_wallet->is_transfer_unlocked(payment.m_unlock_time, payment.m_block_height); res.payments.push_back(std::move(rpc_payment)); } } @@ -2816,6 +2824,108 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_edit_address_book(const wallet_rpc::COMMAND_RPC_EDIT_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_EDIT_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er, const connection_context *ctx) + { + if (!m_wallet) return not_open(er); + if (m_restricted) + { + er.code = WALLET_RPC_ERROR_CODE_DENIED; + er.message = "Command unavailable in restricted mode."; + return false; + } + + const auto ab = m_wallet->get_address_book(); + if (req.index >= ab.size()) + { + er.code = WALLET_RPC_ERROR_CODE_WRONG_INDEX; + er.message = "Index out of range: " + std::to_string(req.index); + return false; + } + + tools::wallet2::address_book_row entry = ab[req.index]; + + cryptonote::address_parse_info info; + crypto::hash payment_id = crypto::null_hash; + if (req.set_address) + { + er.message = ""; + if(!get_account_address_from_str_or_url(info, m_wallet->nettype(), req.address, + [&er](const std::string &url, const std::vector<std::string> &addresses, bool dnssec_valid)->std::string { + if (!dnssec_valid) + { + er.message = std::string("Invalid DNSSEC for ") + url; + return {}; + } + if (addresses.empty()) + { + er.message = std::string("No Monero address found at ") + url; + return {}; + } + return addresses[0]; + })) + { + er.code = WALLET_RPC_ERROR_CODE_WRONG_ADDRESS; + if (er.message.empty()) + er.message = std::string("WALLET_RPC_ERROR_CODE_WRONG_ADDRESS: ") + req.address; + return false; + } + entry.m_address = info.address; + entry.m_is_subaddress = info.is_subaddress; + if (info.has_payment_id) + { + memcpy(entry.m_payment_id.data, info.payment_id.data, 8); + memset(entry.m_payment_id.data + 8, 0, 24); + } + } + + if (req.set_payment_id) + { + if (req.payment_id.empty()) + { + payment_id = crypto::null_hash; + } + else + { + if (req.set_address && info.has_payment_id) + { + er.code = WALLET_RPC_ERROR_CODE_WRONG_PAYMENT_ID; + er.message = "Separate payment ID given with integrated address"; + return false; + } + + if (!wallet2::parse_long_payment_id(req.payment_id, payment_id)) + { + crypto::hash8 spid; + if (!wallet2::parse_short_payment_id(req.payment_id, spid)) + { + er.code = WALLET_RPC_ERROR_CODE_WRONG_PAYMENT_ID; + er.message = "Payment id has invalid format: \"" + req.payment_id + "\", expected 64 character string"; + return false; + } + else + { + er.code = WALLET_RPC_ERROR_CODE_WRONG_PAYMENT_ID; + er.message = "Payment id has invalid format: standalone short payment IDs are forbidden, they must be part of an integrated address"; + return false; + } + } + } + + entry.m_payment_id = payment_id; + } + + if (req.set_description) + entry.m_description = req.description; + + if (!m_wallet->set_address_book_row(req.index, entry.m_address, entry.m_payment_id, entry.m_description, entry.m_is_subaddress)) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "Failed to edit address book entry"; + return false; + } + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_delete_address_book(const wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); diff --git a/src/wallet/wallet_rpc_server.h b/src/wallet/wallet_rpc_server.h index a327ed908..b2b5e7116 100644 --- a/src/wallet/wallet_rpc_server.h +++ b/src/wallet/wallet_rpc_server.h @@ -127,6 +127,7 @@ namespace tools MAP_JON_RPC_WE("parse_uri", on_parse_uri, wallet_rpc::COMMAND_RPC_PARSE_URI) MAP_JON_RPC_WE("get_address_book", on_get_address_book, wallet_rpc::COMMAND_RPC_GET_ADDRESS_BOOK_ENTRY) MAP_JON_RPC_WE("add_address_book", on_add_address_book, wallet_rpc::COMMAND_RPC_ADD_ADDRESS_BOOK_ENTRY) + MAP_JON_RPC_WE("edit_address_book", on_edit_address_book, wallet_rpc::COMMAND_RPC_EDIT_ADDRESS_BOOK_ENTRY) MAP_JON_RPC_WE("delete_address_book",on_delete_address_book,wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY) MAP_JON_RPC_WE("refresh", on_refresh, wallet_rpc::COMMAND_RPC_REFRESH) MAP_JON_RPC_WE("auto_refresh", on_auto_refresh, wallet_rpc::COMMAND_RPC_AUTO_REFRESH) @@ -212,6 +213,7 @@ namespace tools bool on_parse_uri(const wallet_rpc::COMMAND_RPC_PARSE_URI::request& req, wallet_rpc::COMMAND_RPC_PARSE_URI::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_get_address_book(const wallet_rpc::COMMAND_RPC_GET_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_GET_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_add_address_book(const wallet_rpc::COMMAND_RPC_ADD_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_ADD_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_edit_address_book(const wallet_rpc::COMMAND_RPC_EDIT_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_EDIT_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_delete_address_book(const wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_refresh(const wallet_rpc::COMMAND_RPC_REFRESH::request& req, wallet_rpc::COMMAND_RPC_REFRESH::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_auto_refresh(const wallet_rpc::COMMAND_RPC_AUTO_REFRESH::request& req, wallet_rpc::COMMAND_RPC_AUTO_REFRESH::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); diff --git a/src/wallet/wallet_rpc_server_commands_defs.h b/src/wallet/wallet_rpc_server_commands_defs.h index 2dfe6db85..0c86f404d 100644 --- a/src/wallet/wallet_rpc_server_commands_defs.h +++ b/src/wallet/wallet_rpc_server_commands_defs.h @@ -47,7 +47,7 @@ // advance which version they will stop working with // Don't go over 32767 for any of these #define WALLET_RPC_VERSION_MAJOR 1 -#define WALLET_RPC_VERSION_MINOR 14 +#define WALLET_RPC_VERSION_MINOR 16 #define MAKE_WALLET_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define WALLET_RPC_VERSION MAKE_WALLET_RPC_VERSION(WALLET_RPC_VERSION_MAJOR, WALLET_RPC_VERSION_MINOR) namespace tools @@ -64,10 +64,12 @@ namespace wallet_rpc uint32_t account_index; std::set<uint32_t> address_indices; bool all_accounts; + bool strict; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(account_index) KV_SERIALIZE(address_indices) KV_SERIALIZE_OPT(all_accounts, false); + KV_SERIALIZE_OPT(strict, false); END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<request_t> request; @@ -230,9 +232,11 @@ namespace wallet_rpc struct request_t { std::string tag; // all accounts if empty, otherwise those accounts with this tag + bool strict_balances; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tag) + KV_SERIALIZE_OPT(strict_balances, false) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<request_t> request; @@ -912,6 +916,7 @@ namespace wallet_rpc uint64_t amount; uint64_t block_height; uint64_t unlock_time; + bool locked; cryptonote::subaddress_index subaddr_index; std::string address; @@ -921,6 +926,7 @@ namespace wallet_rpc KV_SERIALIZE(amount) KV_SERIALIZE(block_height) KV_SERIALIZE(unlock_time) + KV_SERIALIZE(locked) KV_SERIALIZE(subaddr_index) KV_SERIALIZE(address) END_KV_SERIALIZE_MAP() @@ -1360,6 +1366,7 @@ namespace wallet_rpc std::list<transfer_destination> destinations; std::string type; uint64_t unlock_time; + bool locked; cryptonote::subaddress_index subaddr_index; std::vector<cryptonote::subaddress_index> subaddr_indices; std::string address; @@ -1378,6 +1385,7 @@ namespace wallet_rpc KV_SERIALIZE(destinations); KV_SERIALIZE(type); KV_SERIALIZE(unlock_time) + KV_SERIALIZE(locked) KV_SERIALIZE(subaddr_index); KV_SERIALIZE(subaddr_indices); KV_SERIALIZE(address); @@ -1837,6 +1845,38 @@ namespace wallet_rpc typedef epee::misc_utils::struct_init<response_t> response; }; + struct COMMAND_RPC_EDIT_ADDRESS_BOOK_ENTRY + { + struct request_t + { + uint64_t index; + bool set_address; + std::string address; + bool set_payment_id; + std::string payment_id; + bool set_description; + std::string description; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(index) + KV_SERIALIZE(set_address) + KV_SERIALIZE(address) + KV_SERIALIZE(set_payment_id) + KV_SERIALIZE(payment_id) + KV_SERIALIZE(set_description) + KV_SERIALIZE(description) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<request_t> request; + + struct response_t + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<response_t> response; + }; + struct COMMAND_RPC_GET_ADDRESS_BOOK_ENTRY { struct request_t diff --git a/tests/core_tests/block_validation.cpp b/tests/core_tests/block_validation.cpp index 566ec1440..55312bc15 100644 --- a/tests/core_tests/block_validation.cpp +++ b/tests/core_tests/block_validation.cpp @@ -640,3 +640,18 @@ bool gen_block_invalid_binary_format::check_all_blocks_purged(cryptonote::core& return true; } + +bool gen_block_late_v1_coinbase_tx::generate(std::vector<test_event_entry>& events) const +{ + BLOCK_VALIDATION_INIT_GENERATE(); + + block blk_1; + generator.construct_block_manually(blk_1, blk_0, miner_account, + test_generator::bf_major_ver | test_generator::bf_minor_ver, + HF_VERSION_MIN_V2_COINBASE_TX, HF_VERSION_MIN_V2_COINBASE_TX); + events.push_back(blk_1); + + DO_CALLBACK(events, "check_block_purged"); + + return true; +} diff --git a/tests/core_tests/block_validation.h b/tests/core_tests/block_validation.h index 4a65b029e..2393e1b01 100644 --- a/tests/core_tests/block_validation.h +++ b/tests/core_tests/block_validation.h @@ -206,3 +206,15 @@ struct gen_block_invalid_binary_format : public test_chain_unit_base private: size_t m_corrupt_blocks_begin_idx; }; + +struct gen_block_late_v1_coinbase_tx : public gen_block_verification_base<1> +{ + bool generate(std::vector<test_event_entry>& events) const; +}; +template<> +struct get_test_options<gen_block_late_v1_coinbase_tx> { + const std::pair<uint8_t, uint64_t> hard_forks[3] = {std::make_pair(1, 0), std::make_pair(HF_VERSION_MIN_V2_COINBASE_TX, 1), std::make_pair(0, 0)}; + const cryptonote::test_options test_options = { + hard_forks, 0 + }; +}; diff --git a/tests/core_tests/chaingen_main.cpp b/tests/core_tests/chaingen_main.cpp index cb35cfde2..8406f416f 100644 --- a/tests/core_tests/chaingen_main.cpp +++ b/tests/core_tests/chaingen_main.cpp @@ -133,6 +133,7 @@ int main(int argc, char* argv[]) GENERATE_AND_PLAY(gen_block_has_invalid_tx); GENERATE_AND_PLAY(gen_block_is_too_big); GENERATE_AND_PLAY(gen_block_invalid_binary_format); // Takes up to 3 hours, if CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW == 500, up to 30 minutes, if CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW == 10 + GENERATE_AND_PLAY(gen_block_late_v1_coinbase_tx); // Transaction verification tests GENERATE_AND_PLAY(gen_tx_big_version); @@ -207,6 +208,7 @@ int main(int argc, char* argv[]) GENERATE_AND_PLAY(gen_rct_tx_pre_rct_increase_vin_and_fee); GENERATE_AND_PLAY(gen_rct_tx_pre_rct_altered_extra); GENERATE_AND_PLAY(gen_rct_tx_rct_altered_extra); + GENERATE_AND_PLAY(gen_rct_tx_uses_output_too_early); GENERATE_AND_PLAY(gen_multisig_tx_valid_22_1_2); GENERATE_AND_PLAY(gen_multisig_tx_valid_22_1_2_many_inputs); diff --git a/tests/core_tests/rct.cpp b/tests/core_tests/rct.cpp index 248354177..dc77e408f 100644 --- a/tests/core_tests/rct.cpp +++ b/tests/core_tests/rct.cpp @@ -40,8 +40,8 @@ using namespace cryptonote; //---------------------------------------------------------------------------------------------------------------------- // Tests -bool gen_rct_tx_validation_base::generate_with(std::vector<test_event_entry>& events, - const int *out_idx, int mixin, uint64_t amount_paid, bool valid, +bool gen_rct_tx_validation_base::generate_with_full(std::vector<test_event_entry>& events, + const int *out_idx, int mixin, uint64_t amount_paid, size_t second_rewind, uint8_t last_version, const rct::RCTConfig &rct_config, bool valid, const std::function<void(std::vector<tx_source_entry> &sources, std::vector<tx_destination_entry> &destinations)> &pre_tx, const std::function<void(transaction &tx)> &post_tx) const { @@ -151,13 +151,13 @@ bool gen_rct_tx_validation_base::generate_with(std::vector<test_event_entry>& ev // rewind { - for (size_t i = 0; i < CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW; ++i) + for (size_t i = 0; i < second_rewind; ++i) { cryptonote::block blk; CHECK_AND_ASSERT_MES(generator.construct_block_manually(blk, blk_last, miner_account, test_generator::bf_major_ver | test_generator::bf_minor_ver | test_generator::bf_timestamp | test_generator::bf_hf_version | test_generator::bf_max_outs, - 4, 4, blk_last.timestamp + DIFFICULTY_BLOCKS_ESTIMATE_TIMESPAN * 2, // v2 has blocks twice as long - crypto::hash(), 0, transaction(), std::vector<crypto::hash>(), 0, 6, 4), + last_version, last_version, blk_last.timestamp + DIFFICULTY_BLOCKS_ESTIMATE_TIMESPAN * 2, // v2 has blocks twice as long + crypto::hash(), 0, transaction(), std::vector<crypto::hash>(), 0, 6, last_version), false, "Failed to generate block"); events.push_back(blk); blk_last = blk; @@ -213,6 +213,8 @@ bool gen_rct_tx_validation_base::generate_with(std::vector<test_event_entry>& ev td.addr = miner_account.get_keys().m_account_address; td.amount = amount_paid; std::vector<tx_destination_entry> destinations; + // from v12, we need two outputs at least + destinations.push_back(td); destinations.push_back(td); if (pre_tx) @@ -223,7 +225,7 @@ bool gen_rct_tx_validation_base::generate_with(std::vector<test_event_entry>& ev std::vector<crypto::secret_key> additional_tx_keys; std::unordered_map<crypto::public_key, cryptonote::subaddress_index> subaddresses; subaddresses[miner_accounts[0].get_keys().m_account_address.m_spend_public_key] = {0,0}; - bool r = construct_tx_and_get_tx_key(miner_accounts[0].get_keys(), subaddresses, sources, destinations, cryptonote::account_public_address{}, std::vector<uint8_t>(), tx, 0, tx_key, additional_tx_keys, true); + bool r = construct_tx_and_get_tx_key(miner_accounts[0].get_keys(), subaddresses, sources, destinations, cryptonote::account_public_address{}, std::vector<uint8_t>(), tx, 0, tx_key, additional_tx_keys, true, rct_config); CHECK_AND_ASSERT_MES(r, false, "failed to construct transaction"); if (post_tx) @@ -237,6 +239,15 @@ bool gen_rct_tx_validation_base::generate_with(std::vector<test_event_entry>& ev return true; } +bool gen_rct_tx_validation_base::generate_with(std::vector<test_event_entry>& events, + const int *out_idx, int mixin, uint64_t amount_paid, bool valid, + const std::function<void(std::vector<tx_source_entry> &sources, std::vector<tx_destination_entry> &destinations)> &pre_tx, + const std::function<void(transaction &tx)> &post_tx) const +{ + const rct::RCTConfig rct_config { rct::RangeProofBorromean, 0 }; + return generate_with_full(events, out_idx, mixin, amount_paid, CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE, 4, rct_config, valid, pre_tx, post_tx); +} + bool gen_rct_tx_valid_from_pre_rct::generate(std::vector<test_event_entry>& events) const { const int mixin = 2; @@ -507,3 +518,11 @@ bool gen_rct_tx_rct_altered_extra::generate(std::vector<test_event_entry>& event NULL, [&failed](transaction &tx) {std::string extra_nonce; crypto::hash pid = crypto::null_hash; set_payment_id_to_tx_extra_nonce(extra_nonce, pid); if (!add_extra_nonce_to_tx_extra(tx.extra, extra_nonce)) failed = true; }) && !failed; } +bool gen_rct_tx_uses_output_too_early::generate(std::vector<test_event_entry>& events) const +{ + const int mixin = 10; + const int out_idx[] = {1, -1}; + const uint64_t amount_paid = 10000; + const rct::RCTConfig rct_config { rct::RangeProofPaddedBulletproof, 2 }; + return generate_with_full(events, out_idx, mixin, amount_paid, CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE-3, HF_VERSION_ENFORCE_MIN_AGE, rct_config, false, NULL, NULL); +} diff --git a/tests/core_tests/rct.h b/tests/core_tests/rct.h index 00a2bd88c..9433d4b02 100644 --- a/tests/core_tests/rct.h +++ b/tests/core_tests/rct.h @@ -69,6 +69,10 @@ struct gen_rct_tx_validation_base : public test_chain_unit_base return true; } + bool generate_with_full(std::vector<test_event_entry>& events, const int *out_idx, int mixin, + uint64_t amount_paid, size_t second_rewind, uint8_t last_version, const rct::RCTConfig &rct_config, bool valid, + const std::function<void(std::vector<cryptonote::tx_source_entry> &sources, std::vector<cryptonote::tx_destination_entry> &destinations)> &pre_tx, + const std::function<void(cryptonote::transaction &tx)> &post_tx) const; bool generate_with(std::vector<test_event_entry>& events, const int *out_idx, int mixin, uint64_t amount_paid, bool valid, const std::function<void(std::vector<cryptonote::tx_source_entry> &sources, std::vector<cryptonote::tx_destination_entry> &destinations)> &pre_tx, @@ -262,3 +266,13 @@ struct gen_rct_tx_rct_altered_extra : public gen_rct_tx_validation_base }; template<> struct get_test_options<gen_rct_tx_rct_altered_extra>: public get_test_options<gen_rct_tx_validation_base> {}; +struct gen_rct_tx_uses_output_too_early : public gen_rct_tx_validation_base +{ + bool generate(std::vector<test_event_entry>& events) const; +}; +template<> struct get_test_options<gen_rct_tx_uses_output_too_early> { + const std::pair<uint8_t, uint64_t> hard_forks[5] = {std::make_pair(1, 0), std::make_pair(2, 1), std::make_pair(4, 65), std::make_pair(12, 69), std::make_pair(0, 0)}; + const cryptonote::test_options test_options = { + hard_forks, 0 + }; +}; diff --git a/tests/core_tests/tx_validation.cpp b/tests/core_tests/tx_validation.cpp index 232c86482..acdfdc41b 100644 --- a/tests/core_tests/tx_validation.cpp +++ b/tests/core_tests/tx_validation.cpp @@ -529,7 +529,7 @@ bool gen_tx_key_image_not_derive_from_tx_key::generate(std::vector<test_event_en // Tx with invalid key image can't be subscribed, so create empty signature builder.m_tx.signatures.resize(1); builder.m_tx.signatures[0].resize(1); - builder.m_tx.signatures[0][0] = boost::value_initialized<crypto::signature>(); + builder.m_tx.signatures[0][0] = crypto::signature{}; DO_CALLBACK(events, "mark_invalid_tx"); events.push_back(builder.m_tx); @@ -562,7 +562,7 @@ bool gen_tx_key_image_is_invalid::generate(std::vector<test_event_entry>& events // Tx with invalid key image can't be subscribed, so create empty signature builder.m_tx.signatures.resize(1); builder.m_tx.signatures[0].resize(1); - builder.m_tx.signatures[0][0] = boost::value_initialized<crypto::signature>(); + builder.m_tx.signatures[0][0] = crypto::signature{}; DO_CALLBACK(events, "mark_invalid_tx"); events.push_back(builder.m_tx); diff --git a/tests/functional_tests/CMakeLists.txt b/tests/functional_tests/CMakeLists.txt index fd49ba623..bc55da9e3 100644 --- a/tests/functional_tests/CMakeLists.txt +++ b/tests/functional_tests/CMakeLists.txt @@ -59,3 +59,7 @@ else() message(WARNING "functional_tests_rpc skipped, needs the 'requests' python module") set(CTEST_CUSTOM_TESTS_IGNORE ${CTEST_CUSTOM_TESTS_IGNORE} functional_tests_rpc) endif() + +add_test( + NAME check_missing_rpc_methods + COMMAND ${PYTHON_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/check_missing_rpc_methods.py" "${CMAKE_SOURCE_DIR}") diff --git a/tests/functional_tests/address_book.py b/tests/functional_tests/address_book.py new file mode 100755 index 000000000..8d8711ffc --- /dev/null +++ b/tests/functional_tests/address_book.py @@ -0,0 +1,304 @@ +#!/usr/bin/env python3 +#encoding=utf-8 + +# Copyright (c) 2019 The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Test wallet address book RPC +""" + +from __future__ import print_function +from framework.wallet import Wallet + +class AddressBookTest(): + def run_test(self): + self.create() + self.test_address_book() + + def create(self): + print('Creating wallet') + wallet = Wallet() + # close the wallet if any, will throw if none is loaded + try: wallet.close_wallet() + except: pass + seed = 'velvet lymph giddy number token physics poetry unquoted nibs useful sabotage limits benches lifestyle eden nitrogen anvil fewest avoid batch vials washing fences goat unquoted' + res = wallet.restore_deterministic_wallet(seed = seed) + assert res.address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm' + assert res.seed == seed + + def test_address_book(self): + print('Testing address book') + wallet = Wallet() + + # empty at start + res = wallet.get_address_book() + assert not 'entries' in res or (res.entries) == 0 + ok = False + try: wallet.get_address_book([0]) + except: ok = True + assert ok + ok = False + try: wallet.delete_address_book(0) + except: ok = True + assert ok + ok = False + try: wallet.edit_address_book(0, description = '') + except: ok = True + assert ok + + # add one + res = wallet.add_address_book('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', description = 'self') + assert res.index == 0 + for get_all in [True, False]: + res = wallet.get_address_book() if get_all else wallet.get_address_book([0]) + assert len(res.entries) == 1 + e = res.entries[0] + assert e.index == 0 + assert e.address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm' + assert e.payment_id == '' or e.payment_id == '0' * 16 or e.payment_id == '0' * 64 + assert e.description == 'self' + + # add a duplicate + res = wallet.add_address_book('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', description = 'self') + assert res.index == 1 + res = wallet.get_address_book() + assert len(res.entries) == 2 + assert res.entries[0].index == 0 + assert res.entries[1].index == 1 + assert res.entries[0].address == res.entries[1].address + assert res.entries[0].payment_id == res.entries[1].payment_id + assert res.entries[0].description == res.entries[1].description + e = res.entries[1] + res = wallet.get_address_book([1]) + assert len(res.entries) == 1 + assert e == res.entries[0] + + # request (partially) out of range + ok = False + try: res = wallet.get_address_book[4, 2] + except: ok = True + assert ok + ok = False + try: res = wallet.get_address_book[0, 2] + except: ok = True + assert ok + ok = False + try: res = wallet.get_address_book[2, 0] + except: ok = True + assert ok + + # delete first + res = wallet.delete_address_book(0) + res = wallet.get_address_book() + assert len(res.entries) == 1 + assert res.entries[0].index == 0 + assert res.entries[0].address == e.address + assert res.entries[0].payment_id == e.payment_id + assert res.entries[0].description == e.description + + # delete (new) first + res = wallet.delete_address_book(0) + res = wallet.get_address_book() + assert not 'entries' in res or (res.entries) == 0 + + # add non-addresses + errors = 0 + try: wallet.add_address_book('', description = 'bad') + except: errors += 1 + try: wallet.add_address_book('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm ', description = 'bad') + except: errors += 1 + try: wallet.add_address_book('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDn', description = 'bad') + except: errors += 1 + try: wallet.add_address_book('9ujeXrjzf7bfeK3KZdCqnYaMwZVFuXemPU8Ubw335rj2FN1CdMiWNyFV3ksEfMFvRp9L9qum5UxkP5rN9aLcPxbH1au4WAB', description = 'bad') + except: errors += 1 + try: wallet.add_address_book('donate@example.com', description = 'bad') + except: errors += 1 + assert errors == 5 + res = wallet.get_address_book() + assert not 'entries' in res or len(res.entries) == 0 + + # openalias + res = wallet.add_address_book('donate@getmonero.org', description = 'dev fund') + assert res.index == 0 + res = wallet.get_address_book() + assert len(res.entries) == 1 + e = res.entries[0] + assert e.address == '44AFFq5kSiGBoZ4NMDwYtN18obc8AemS33DBLWs3H7otXft3XjrpDtQGv7SqSsaBYBb98uNbr2VBBEt7f2wfn3RVGQBEP3A' + assert e.description == 'dev fund' + + # UTF-8 + res = wallet.add_address_book('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', description = u'ã‚ã¾ã‚„ã‹ã™') + assert res.index == 1 + res = wallet.get_address_book([1]) + assert len(res.entries) == 1 + assert res.entries[0].description == u'ã‚ã¾ã‚„ã‹ã™' + e = res.entries[0] + + # duplicate request + res = wallet.get_address_book([1, 1]) + assert len(res.entries) == 2 + assert res.entries[0] == e + assert res.entries[1] == e + + # payment IDs + res = wallet.add_address_book('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', payment_id = '0' * 64) + assert res.index == 2 + ok = False + try: res = wallet.add_address_book('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', payment_id = 'x' * 64) + except: ok = True + assert ok + ok = False + try: res = wallet.add_address_book('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', payment_id = '0' * 65) + except: ok = True + assert ok + ok = False + try: res = wallet.add_address_book('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', payment_id = '0' * 63) + except: ok = True + assert ok + ok = False + try: res = wallet.add_address_book('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', payment_id = '0' * 16) + except: ok = True + assert ok + + # various address types + res = wallet.make_integrated_address() + integrated_address = res.integrated_address + integrated_address_payment_id = res.payment_id + ok = False + try: res = wallet.add_address_book(integrated_address, payment_id = '0' * 64) + except: ok = True + assert ok + res = wallet.add_address_book(integrated_address) + assert res.index == 3 + res = wallet.add_address_book('87KfgTZ8ER5D3Frefqnrqif11TjVsTPaTcp37kqqKMrdDRUhpJRczeR7KiBmSHF32UJLP3HHhKUDmEQyJrv2mV8yFDCq8eB') + assert res.index == 4 + + # get them back + res = wallet.get_address_book([0]) + assert len(res.entries) == 1 + assert res.entries[0].address == '44AFFq5kSiGBoZ4NMDwYtN18obc8AemS33DBLWs3H7otXft3XjrpDtQGv7SqSsaBYBb98uNbr2VBBEt7f2wfn3RVGQBEP3A' + assert res.entries[0].description == 'dev fund' + res = wallet.get_address_book([1]) + assert len(res.entries) == 1 + assert res.entries[0].address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm' + assert res.entries[0].description == u'ã‚ã¾ã‚„ã‹ã™' + res = wallet.get_address_book([2]) + assert len(res.entries) == 1 + assert res.entries[0].address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm' + res = wallet.get_address_book([3]) + assert len(res.entries) == 1 + if False: # for now, the address book splits integrated addresses + assert res.entries[0].address == integrated_address + else: + assert res.entries[0].address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm' + assert res.entries[0].payment_id == integrated_address_payment_id + '0' * 48 + res = wallet.get_address_book([4]) + assert len(res.entries) == 1 + assert res.entries[0].address == '87KfgTZ8ER5D3Frefqnrqif11TjVsTPaTcp37kqqKMrdDRUhpJRczeR7KiBmSHF32UJLP3HHhKUDmEQyJrv2mV8yFDCq8eB' + + # edit + res = wallet.get_address_book([1]) + assert len(res.entries) == 1 + e = res.entries[0] + assert e.index == 1 + assert e.address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm' + assert e.payment_id == '0' * 64 + assert e.description == u'ã‚ã¾ã‚„ã‹ã™' + res = wallet.edit_address_book(1, payment_id = '1' * 64) + res = wallet.get_address_book([1]) + assert len(res.entries) == 1 + e = res.entries[0] + assert e.index == 1 + assert e.address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm' + assert e.payment_id == '1' * 64 + assert e.description == u'ã‚ã¾ã‚„ã‹ã™' + res = wallet.edit_address_book(1, description = '') + res = wallet.get_address_book([1]) + assert len(res.entries) == 1 + e = res.entries[0] + assert e.index == 1 + assert e.address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm' + assert e.payment_id == '1' * 64 + assert e.description == '' + res = wallet.edit_address_book(1, description = 'ãˆã‚“ã—ã‚…ã†') + res = wallet.get_address_book([1]) + assert len(res.entries) == 1 + e = res.entries[0] + assert e.index == 1 + assert e.address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm' + assert e.payment_id == '1' * 64 + assert e.description == u'ãˆã‚“ã—ã‚…ã†' + res = wallet.edit_address_book(1, address = '44AFFq5kSiGBoZ4NMDwYtN18obc8AemS33DBLWs3H7otXft3XjrpDtQGv7SqSsaBYBb98uNbr2VBBEt7f2wfn3RVGQBEP3A') + res = wallet.get_address_book([1]) + assert len(res.entries) == 1 + e = res.entries[0] + assert e.index == 1 + assert e.address == '44AFFq5kSiGBoZ4NMDwYtN18obc8AemS33DBLWs3H7otXft3XjrpDtQGv7SqSsaBYBb98uNbr2VBBEt7f2wfn3RVGQBEP3A' + assert e.payment_id == '1' * 64 + assert e.description == u'ãˆã‚“ã—ã‚…ã†' + res = wallet.edit_address_book(1, payment_id = '') + res = wallet.get_address_book([1]) + assert len(res.entries) == 1 + e = res.entries[0] + assert e.index == 1 + assert e.address == '44AFFq5kSiGBoZ4NMDwYtN18obc8AemS33DBLWs3H7otXft3XjrpDtQGv7SqSsaBYBb98uNbr2VBBEt7f2wfn3RVGQBEP3A' + assert e.payment_id == '0' * 64 + assert e.description == u'ãˆã‚“ã—ã‚…ã†' + ok = False + try: res = wallet.edit_address_book(1, address = '') + except: ok = True + assert ok + ok = False + try: res = wallet.edit_address_book(1, payment_id = 'asdnd') + except: ok = True + assert ok + ok = False + try: res = wallet.edit_address_book(1, address = 'address') + except: ok = True + assert ok + res = wallet.edit_address_book(1) + res = wallet.get_address_book([1]) + assert len(res.entries) == 1 + assert e == res.entries[0] + + # empty + wallet.delete_address_book(4) + wallet.delete_address_book(0) + res = wallet.get_address_book([0]) # entries above the deleted one collapse one slot up + assert len(res.entries) == 1 + assert res.entries[0].address == '44AFFq5kSiGBoZ4NMDwYtN18obc8AemS33DBLWs3H7otXft3XjrpDtQGv7SqSsaBYBb98uNbr2VBBEt7f2wfn3RVGQBEP3A' + assert res.entries[0].description == u'ãˆã‚“ã—ã‚…ã†' + wallet.delete_address_book(2) + wallet.delete_address_book(0) + wallet.delete_address_book(0) + res = wallet.get_address_book() + assert not 'entries' in res or len(res.entries) == 0 + + +if __name__ == '__main__': + AddressBookTest().run_test() diff --git a/tests/functional_tests/blockchain.py b/tests/functional_tests/blockchain.py index 2c3f34c35..324af624a 100755 --- a/tests/functional_tests/blockchain.py +++ b/tests/functional_tests/blockchain.py @@ -53,7 +53,8 @@ class BlockchainTest(): def reset(self): print('Resetting blockchain') daemon = Daemon() - daemon.pop_blocks(1000) + res = daemon.get_height() + daemon.pop_blocks(res.height - 1) daemon.flush_txpool() def _test_generateblocks(self, blocks): @@ -330,6 +331,9 @@ class BlockchainTest(): for txid in [alt_blocks[0], alt_blocks[2], alt_blocks[4]]: assert len([chain for chain in res.chains if chain.block_hash == txid]) == 1 + print('Saving blockchain explicitely') + daemon.save_bc() + if __name__ == '__main__': BlockchainTest().run_test() diff --git a/tests/functional_tests/check_missing_rpc_methods.py b/tests/functional_tests/check_missing_rpc_methods.py new file mode 100644 index 000000000..6fadebf9b --- /dev/null +++ b/tests/functional_tests/check_missing_rpc_methods.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python + +from __future__ import print_function +import sys +import re + +USAGE = 'usage: check_untested_methods.py <rootdir>' +try: + rootdir = sys.argv[1] +except: + print(USAGE) + sys.exit(1) + +sys.path.insert(0, rootdir + '/utils/python-rpc') + +from framework import daemon +from framework import wallet + +modules = [ + { + 'name': 'daemon', + 'object': daemon.Daemon(), + 'path': rootdir + '/src/rpc/core_rpc_server.h', + 'ignore': [] + }, + { + 'name': 'wallet', + 'object': wallet.Wallet(), + 'path': rootdir + '/src/wallet/wallet_rpc_server.h', + 'ignore': [] + } +] + +error = False +for module in modules: + for line in open(module['path']).readlines(): + if 'MAP_URI_AUTO_JON2' in line or 'MAP_JON_RPC' in line: + match = re.search('.*\"(.*)\".*', line) + name = match.group(1) + if name in module['ignore'] or name.endswith('.bin'): + continue + if 'MAP_URI_AUTO_JON2' in line: + if not name.startswith('/'): + print('Error: %s does not start with /' % name) + error = True + name = name[1:] + if not hasattr(module['object'], name): + print('Error: %s API method %s does not have a matching function' % (module['name'], name)) + +sys.exit(1 if error else 0) diff --git a/tests/functional_tests/cold_signing.py b/tests/functional_tests/cold_signing.py index a722d8927..f915df77a 100755 --- a/tests/functional_tests/cold_signing.py +++ b/tests/functional_tests/cold_signing.py @@ -45,7 +45,8 @@ class ColdSigningTest(): def reset(self): print('Resetting blockchain') daemon = Daemon() - daemon.pop_blocks(1000) + res = daemon.get_height() + daemon.pop_blocks(res.height - 1) daemon.flush_txpool() def create(self, idx): diff --git a/tests/functional_tests/functional_tests_rpc.py b/tests/functional_tests/functional_tests_rpc.py index 77d0e4c4d..9043565c7 100755 --- a/tests/functional_tests/functional_tests_rpc.py +++ b/tests/functional_tests/functional_tests_rpc.py @@ -10,7 +10,7 @@ import string import os USAGE = 'usage: functional_tests_rpc.py <python> <srcdir> <builddir> [<tests-to-run> | all]' -DEFAULT_TESTS = ['bans', 'daemon_info', 'blockchain', 'wallet_address', 'integrated_address', 'mining', 'transfer', 'txpool', 'multisig', 'cold_signing', 'sign_message', 'proofs', 'get_output_distribution'] +DEFAULT_TESTS = ['address_book', 'bans', 'blockchain', 'cold_signing', 'daemon_info', 'get_output_distribution', 'integrated_address', 'mining', 'multisig', 'proofs', 'sign_message', 'transfer', 'txpool', 'uri', 'validate_address', 'wallet'] try: python = sys.argv[1] srcdir = sys.argv[2] @@ -36,9 +36,10 @@ except: N_MONERODS = 1 N_WALLETS = 4 +WALLET_DIRECTORY = builddir + "/functional-tests-directory" monerod_base = [builddir + "/bin/monerod", "--regtest", "--fixed-difficulty", "1", "--offline", "--no-igd", "--p2p-bind-port", "monerod_p2p_port", "--rpc-bind-port", "monerod_rpc_port", "--zmq-rpc-bind-port", "monerod_zmq_port", "--non-interactive", "--disable-dns-checkpoints", "--check-updates", "disabled", "--rpc-ssl", "disabled", "--log-level", "1"] -wallet_base = [builddir + "/bin/monero-wallet-rpc", "--wallet-dir", builddir + "/functional-tests-directory", "--rpc-bind-port", "wallet_port", "--disable-rpc-login", "--rpc-ssl", "disabled", "--daemon-ssl", "disabled", "--daemon-port", "18180", "--log-level", "1"] +wallet_base = [builddir + "/bin/monero-wallet-rpc", "--wallet-dir", WALLET_DIRECTORY, "--rpc-bind-port", "wallet_port", "--disable-rpc-login", "--rpc-ssl", "disabled", "--daemon-ssl", "disabled", "--daemon-port", "18180", "--log-level", "1"] command_lines = [] processes = [] @@ -62,6 +63,8 @@ try: PYTHONPATH += ':' PYTHONPATH += srcdir + '/../../utils/python-rpc' os.environ['PYTHONPATH'] = PYTHONPATH + os.environ['WALLET_DIRECTORY'] = WALLET_DIRECTORY + os.environ['PYTHONIOENCODING'] = 'utf-8' for i in range(len(command_lines)): #print('Running: ' + str(command_lines[i])) processes.append(subprocess.Popen(command_lines[i], stdout = outputs[i])) @@ -133,6 +136,6 @@ else: if len(FAIL) == 0: print('Done, ' + str(len(PASS)) + '/' + str(len(tests)) + ' tests passed') else: - print('Done, ' + str(len(FAIL)) + '/' + str(len(tests)) + ' tests failed: ' + string.join(FAIL, ', ')) + print('Done, ' + str(len(FAIL)) + '/' + str(len(tests)) + ' tests failed: ' + ', '.join(FAIL)) sys.exit(0 if len(FAIL) == 0 else 1) diff --git a/tests/functional_tests/get_output_distribution.py b/tests/functional_tests/get_output_distribution.py index 93822e90a..077b094ba 100755 --- a/tests/functional_tests/get_output_distribution.py +++ b/tests/functional_tests/get_output_distribution.py @@ -44,7 +44,8 @@ class GetOutputDistributionTest(): def reset(self): print('Resetting blockchain') daemon = Daemon() - daemon.pop_blocks(1000) + res = daemon.get_height() + daemon.pop_blocks(res.height - 1) daemon.flush_txpool() def create(self): diff --git a/tests/functional_tests/mining.py b/tests/functional_tests/mining.py index 5c14d34fd..a08a45ad4 100755 --- a/tests/functional_tests/mining.py +++ b/tests/functional_tests/mining.py @@ -46,12 +46,15 @@ class MiningTest(): def run_test(self): self.reset() self.create() - self.mine() + self.mine(True) + self.mine(False) + self.submitblock() def reset(self): print('Resetting blockchain') daemon = Daemon() - daemon.pop_blocks(1000) + res = daemon.get_height() + daemon.pop_blocks(res.height - 1) daemon.flush_txpool() def create(self): @@ -62,8 +65,8 @@ class MiningTest(): except: pass res = wallet.restore_deterministic_wallet(seed = 'velvet lymph giddy number token physics poetry unquoted nibs useful sabotage limits benches lifestyle eden nitrogen anvil fewest avoid batch vials washing fences goat unquoted') - def mine(self): - print("Test mining") + def mine(self, via_daemon): + print("Test mining via " + ("daemon" if via_daemon else "wallet")) daemon = Daemon() wallet = Wallet() @@ -76,7 +79,10 @@ class MiningTest(): res_status = daemon.mining_status() - res = daemon.start_mining('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', threads_count = 1) + if via_daemon: + res = daemon.start_mining('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', threads_count = 1) + else: + res = wallet.start_mining(threads_count = 1) res_status = daemon.mining_status() assert res_status.active == True @@ -101,7 +107,11 @@ class MiningTest(): timeout -= 1 assert timeout >= 0 - res = daemon.stop_mining() + if via_daemon: + res = daemon.stop_mining() + else: + res = wallet.stop_mining() + res_status = daemon.mining_status() assert res_status.active == False @@ -113,7 +123,10 @@ class MiningTest(): balance = res_getbalance.balance assert balance >= prev_balance + (new_height - prev_height) * 600000000000 - res = daemon.start_mining('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', threads_count = 1, do_background_mining = True) + if via_daemon: + res = daemon.start_mining('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', threads_count = 1, do_background_mining = True) + else: + res = wallet.start_mining(threads_count = 1, do_background_mining = True) res_status = daemon.mining_status() assert res_status.active == True assert res_status.threads_count == 1 @@ -122,10 +135,40 @@ class MiningTest(): assert res_status.block_reward >= 600000000000 # don't wait, might be a while if the machine is busy, which it probably is - res = daemon.stop_mining() + if via_daemon: + res = daemon.stop_mining() + else: + res = wallet.stop_mining() res_status = daemon.mining_status() assert res_status.active == False + def submitblock(self): + print("Test submitblock") + + daemon = Daemon() + res = daemon.get_height() + height = res.height + res = daemon.generateblocks('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 5) + assert len(res.blocks) == 5 + hashes = res.blocks + blocks = [] + for block_hash in hashes: + res = daemon.getblock(hash = block_hash) + assert len(res.blob) > 0 and len(res.blob) % 2 == 0 + blocks.append(res.blob) + res = daemon.get_height() + assert res.height == height + 5 + res = daemon.pop_blocks(5) + res = daemon.get_height() + assert res.height == height + for i in range(len(hashes)): + block_hash = hashes[i] + assert len(block_hash) == 64 + res = daemon.submitblock(blocks[i]) + res = daemon.get_height() + assert res.height == height + i + 1 + assert res.hash == block_hash + if __name__ == '__main__': MiningTest().run_test() diff --git a/tests/functional_tests/multisig.py b/tests/functional_tests/multisig.py index b109acf91..e0d8b06a4 100755 --- a/tests/functional_tests/multisig.py +++ b/tests/functional_tests/multisig.py @@ -46,6 +46,8 @@ class MultisigTest(): self.mine('4ADHswEU3XBUee8pudBkZQd9beJainqNo1BQKkHJujAEPJyQrLj9U4dNm8HEMdHuWwKMFGzMUB3RCTvcTaW9kHpdRUDxgjW', 5) self.mine('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 60) + self.test_states() + self.create_multisig_wallets(2, 2, '493DsrfJPqiN3Suv9RcRDoZEbQtKZX1sNcGPA3GhkKYEEmivk8kjQrTdRdVc4ZbmzWJuE157z9NNUKmF2VDfdYDR3CziGMk') self.import_multisig_info([1, 0], 5) txid = self.transfer([1, 0]) @@ -79,7 +81,8 @@ class MultisigTest(): def reset(self): print('Resetting blockchain') daemon = Daemon() - daemon.pop_blocks(1000) + res = daemon.get_height() + daemon.pop_blocks(res.height - 1) daemon.flush_txpool() def mine(self, address, blocks): @@ -152,6 +155,72 @@ class MultisigTest(): assert res.threshold == M_threshold assert res.total == N_total + def test_states(self): + print('Testing multisig states') + seeds = [ + 'velvet lymph giddy number token physics poetry unquoted nibs useful sabotage limits benches lifestyle eden nitrogen anvil fewest avoid batch vials washing fences goat unquoted', + 'peeled mixture ionic radar utopia puddle buying illness nuns gadget river spout cavernous bounced paradise drunk looking cottage jump tequila melting went winter adjust spout', + 'dilute gutter certain antics pamphlet macro enjoy left slid guarded bogeys upload nineteen bomb jubilee enhanced irritate turnip eggs swung jukebox loudly reduce sedan slid', + ] + info = [] + wallet = [None, None, None] + for i in range(3): + wallet[i] = Wallet(idx = i) + try: wallet[i].close_wallet() + except: pass + res = wallet[i].restore_deterministic_wallet(seed = seeds[i]) + res = wallet[i].is_multisig() + assert not res.multisig + res = wallet[i].prepare_multisig() + assert len(res.multisig_info) > 0 + info.append(res.multisig_info) + + for i in range(3): + ok = False + try: res = wallet[i].finalize_multisig(info) + except: ok = True + assert ok + ok = False + try: res = wallet[i].exchange_multisig_keys(info) + except: ok = True + assert ok + res = wallet[i].is_multisig() + assert not res.multisig + + res = wallet[0].make_multisig(info[0:2], 2) + res = wallet[0].is_multisig() + assert res.multisig + assert res.ready + + ok = False + try: res = wallet[0].finalize_multisig(info) + except: ok = True + assert ok + + ok = False + try: res = wallet[0].prepare_multisig() + except: ok = True + assert ok + + ok = False + try: res = wallet[0].make_multisig(info[0:2], 2) + except: ok = True + assert ok + + res = wallet[1].make_multisig(info, 2) + res = wallet[1].is_multisig() + assert res.multisig + assert not res.ready + + ok = False + try: res = wallet[1].prepare_multisig() + except: ok = True + assert ok + + ok = False + try: res = wallet[1].make_multisig(info[0:2], 2) + except: ok = True + assert ok def import_multisig_info(self, signers, expected_outputs): assert len(signers) >= 2 diff --git a/tests/functional_tests/proofs.py b/tests/functional_tests/proofs.py index 243929dc3..7beb3ec6e 100755 --- a/tests/functional_tests/proofs.py +++ b/tests/functional_tests/proofs.py @@ -44,12 +44,14 @@ class ProofsTest(): txid, tx_key, amount = self.transfer() self.check_tx_key(txid, tx_key, amount) self.check_tx_proof(txid, amount) + self.check_spend_proof(txid) self.check_reserve_proof() def reset(self): print('Resetting blockchain') daemon = Daemon() - daemon.pop_blocks(1000) + res = daemon.get_height() + daemon.pop_blocks(res.height - 1) daemon.flush_txpool() def mine(self, address, blocks): @@ -217,6 +219,40 @@ class ProofsTest(): except: ok = True assert ok or not res.good + def check_spend_proof(self, txid): + daemon = Daemon() + + print('Checking spend proof') + + self.wallet[0].refresh() + self.wallet[1].refresh() + + res = self.wallet[0].get_spend_proof(txid, message = 'foo') + assert len(res.signature) > 0 + signature = res.signature + res = self.wallet[1].check_spend_proof(txid, message = 'foo', signature = signature) + assert res.good + + res = self.wallet[0].get_spend_proof(txid, message = 'foobar') + assert len(res.signature) > 0 + signature2 = res.signature + res = self.wallet[1].check_spend_proof(txid, message = 'foobar', signature = signature2) + assert res.good + + ok = False + try: res = self.wallet[1].check_spend_proof('0' * 64, message = 'foo', signature = signature) + except: ok = True + assert ok or not res.good + + ok = False + try: res = self.wallet[1].check_spend_proof(txid, message = 'bar', signature = signature) + except: ok = True + assert ok or not res.good + + ok = False + try: res = self.wallet[1].check_spend_proof(txid, message = 'foo', signature = signature2) + except: ok = True + assert ok or not res.good def check_reserve_proof(self): daemon = Daemon() diff --git a/tests/functional_tests/speed.py b/tests/functional_tests/speed.py index ed1e332e9..71be785b8 100755 --- a/tests/functional_tests/speed.py +++ b/tests/functional_tests/speed.py @@ -47,14 +47,27 @@ from framework.wallet import Wallet class SpeedTest(): - def set_test_params(self): - self.num_nodes = 1 + def reset(self): + print('Resetting blockchain') + daemon = Daemon() + res = daemon.get_height() + daemon.pop_blocks(res.height - 1) + daemon.flush_txpool() def run_test(self): + self.reset() + daemon = Daemon() wallet = Wallet() - destinations = wallet.make_uniform_destinations('44AFFq5kSiGBoZ4NMDwYtN18obc8AemS33DBLWs3H7otXft3XjrpDtQGv7SqSsaBYBb98uNbr2VBBEt7f2wfn3RVGQBEP3A',1,3) + # close the wallet if any, will throw if none is loaded + try: wallet.close_wallet() + except: pass + wallet.restore_deterministic_wallet('velvet lymph giddy number token physics poetry unquoted nibs useful sabotage limits benches lifestyle eden nitrogen anvil fewest avoid batch vials washing fences goat unquoted') + + destinations = [] + for i in range(3): + destinations.append({"amount":1,"address":'44AFFq5kSiGBoZ4NMDwYtN18obc8AemS33DBLWs3H7otXft3XjrpDtQGv7SqSsaBYBb98uNbr2VBBEt7f2wfn3RVGQBEP3A'}) self._test_speed_generateblocks(daemon=daemon, blocks=70) for i in range(1, 10): @@ -69,7 +82,6 @@ class SpeedTest(): start = time.time() res = daemon.generateblocks('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', blocks) - # wallet seed: velvet lymph giddy number token physics poetry unquoted nibs useful sabotage limits benches lifestyle eden nitrogen anvil fewest avoid batch vials washing fences goat unquoted print('generating ', blocks, 'blocks took: ', time.time() - start, 'seconds') @@ -77,7 +89,7 @@ class SpeedTest(): print('Test speed of transfer') start = time.time() - destinations = wallet.make_uniform_destinations('44AFFq5kSiGBoZ4NMDwYtN18obc8AemS33DBLWs3H7otXft3XjrpDtQGv7SqSsaBYBb98uNbr2VBBEt7f2wfn3RVGQBEP3A',1) + destinations = [{"amount":1,"address":'44AFFq5kSiGBoZ4NMDwYtN18obc8AemS33DBLWs3H7otXft3XjrpDtQGv7SqSsaBYBb98uNbr2VBBEt7f2wfn3RVGQBEP3A'}] res = wallet.transfer_split(destinations) print('generating tx took: ', time.time() - start, 'seconds') diff --git a/tests/functional_tests/transactions_flow_test.cpp b/tests/functional_tests/transactions_flow_test.cpp index 32b601d7a..218590a27 100644 --- a/tests/functional_tests/transactions_flow_test.cpp +++ b/tests/functional_tests/transactions_flow_test.cpp @@ -173,7 +173,7 @@ bool transactions_flow_test(std::string& working_folder, //wait for money, until balance will have enough money w1.refresh(true, blocks_fetched, received_money, ok); - while(w1.unlocked_balance(0) < amount_to_transfer) + while(w1.unlocked_balance(0, true) < amount_to_transfer) { misc_utils::sleep_no_w(1000); w1.refresh(true, blocks_fetched, received_money, ok); @@ -186,7 +186,7 @@ bool transactions_flow_test(std::string& working_folder, { tools::wallet2::transfer_container incoming_transfers; w1.get_transfers(incoming_transfers); - if(incoming_transfers.size() > FIRST_N_TRANSFERS && get_money_in_first_transfers(incoming_transfers, FIRST_N_TRANSFERS) < w1.unlocked_balance(0) ) + if(incoming_transfers.size() > FIRST_N_TRANSFERS && get_money_in_first_transfers(incoming_transfers, FIRST_N_TRANSFERS) < w1.unlocked_balance(0, true) ) { //lets go! size_t count = 0; @@ -221,7 +221,7 @@ bool transactions_flow_test(std::string& working_folder, for(i = 0; i != transactions_count; i++) { uint64_t amount_to_tx = (amount_to_transfer - transfered_money) > transfer_size ? transfer_size: (amount_to_transfer - transfered_money); - while(w1.unlocked_balance(0) < amount_to_tx + TEST_FEE) + while(w1.unlocked_balance(0, true) < amount_to_tx + TEST_FEE) { misc_utils::sleep_no_w(1000); LOG_PRINT_L0("not enough money, waiting for cashback or mining"); @@ -253,7 +253,7 @@ bool transactions_flow_test(std::string& working_folder, transfered_money += amount_to_tx; LOG_PRINT_L0("transferred " << amount_to_tx << ", i=" << i ); - tx_test_entry& ent = txs[get_transaction_hash(tx)] = boost::value_initialized<tx_test_entry>(); + tx_test_entry& ent = txs[get_transaction_hash(tx)] = tx_test_entry{}; ent.amount_transfered = amount_to_tx; ent.tx = tx; //if(i % transactions_per_second) @@ -270,7 +270,7 @@ bool transactions_flow_test(std::string& working_folder, misc_utils::sleep_no_w(DIFFICULTY_BLOCKS_ESTIMATE_TIMESPAN*1000);//wait two blocks before sync on another wallet on another daemon } - uint64_t money_2 = w2.balance(0); + uint64_t money_2 = w2.balance(0, true); if(money_2 == transfered_money) { MGINFO_GREEN("-----------------------FINISHING TRANSACTIONS FLOW TEST OK-----------------------"); diff --git a/tests/functional_tests/transfer.py b/tests/functional_tests/transfer.py index 7ebda6ebd..b4264f72d 100755 --- a/tests/functional_tests/transfer.py +++ b/tests/functional_tests/transfer.py @@ -44,14 +44,20 @@ class TransferTest(): self.mine() self.transfer() self.check_get_bulk_payments() + self.check_get_payments() self.check_double_spend_detection() + self.sweep_dust() self.sweep_single() self.check_destinations() + self.check_tx_notes() + self.check_rescan() + self.check_is_key_image_spent() def reset(self): print('Resetting blockchain') daemon = Daemon() - daemon.pop_blocks(1000) + res = daemon.get_height() + daemon.pop_blocks(res.height - 1) daemon.flush_txpool() def create(self): @@ -114,7 +120,7 @@ class TransferTest(): except: ok = True assert ok - res = self.wallet[0].transfer([dst], ring_size = 11, payment_id = payment_id, get_tx_key = False) + res = self.wallet[0].transfer([dst], ring_size = 11, payment_id = payment_id, get_tx_key = False, get_tx_hex = True) assert len(res.tx_hash) == 32*2 txid = res.tx_hash assert len(res.tx_key) == 0 @@ -122,12 +128,19 @@ class TransferTest(): amount = res.amount assert res.fee > 0 fee = res.fee - assert len(res.tx_blob) == 0 + assert len(res.tx_blob) > 0 + blob_size = len(res.tx_blob) // 2 assert len(res.tx_metadata) == 0 assert len(res.multisig_txset) == 0 assert len(res.unsigned_txset) == 0 unsigned_txset = res.unsigned_txset + res = daemon.get_fee_estimate(10) + assert res.fee > 0 + assert res.quantization_mask > 0 + expected_fee = (res.fee * 1 * blob_size + res.quantization_mask - 1) // res.quantization_mask * res.quantization_mask + assert abs(1 - fee / expected_fee) < 0.01 + self.wallet[0].refresh() res = daemon.get_info() @@ -523,6 +536,28 @@ class TransferTest(): res = self.wallet[1].get_bulk_payments(["1111111122222222"]) assert len(res.payments) >= 1 # we have one of these + def check_get_payments(self): + print('Checking get_payments') + + daemon = Daemon() + res = daemon.get_info() + height = res.height + + self.wallet[0].refresh() + self.wallet[1].refresh() + + res = self.wallet[0].get_payments('1234500000012345abcde00000abcdeff1234500000012345abcde00000abcde') + assert 'payments' not in res or len(res.payments) == 0 + + res = self.wallet[1].get_payments('1234500000012345abcde00000abcdeff1234500000012345abcde00000abcde') + assert len(res.payments) >= 2 + + res = self.wallet[1].get_payments('ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff') + assert 'payments' not in res or len(res.payments) == 0 + + res = self.wallet[1].get_payments(payment_id = '1111111122222222' + '0'*48) + assert len(res.payments) >= 1 # one tx to integrated address + def check_double_spend_detection(self): print('Checking double spend detection') txes = [[None, None], [None, None]] @@ -583,6 +618,13 @@ class TransferTest(): assert tx.in_pool assert tx.double_spend_seen + def sweep_dust(self): + print("Sweeping dust") + daemon = Daemon() + self.wallet[0].refresh() + res = self.wallet[0].sweep_dust() + assert not 'tx_hash_list' in res or len(res.tx_hash_list) == 0 # there's just one, but it cannot meet the fee + def sweep_single(self): daemon = Daemon() @@ -603,11 +645,19 @@ class TransferTest(): self.wallet[0].refresh() res = self.wallet[0].get_balance() balance = res.balance - res = self.wallet[0].incoming_transfers(transfer_type = 'all') + res = daemon.is_key_image_spent([ki]) + assert len(res.spent_status) == 1 + assert res.spent_status[0] == 0 res = self.wallet[0].sweep_single('44Kbx4sJ7JDRDV5aAhLJzQCjDz2ViLRduE3ijDZu3osWKBjMGkV1XPk4pfDUMqt1Aiezvephdqm6YD19GKFD9ZcXVUTp6BW', key_image = ki) assert len(res.tx_hash) == 64 tx_hash = res.tx_hash + res = daemon.is_key_image_spent([ki]) + assert len(res.spent_status) == 1 + assert res.spent_status[0] == 2 daemon.generateblocks('44Kbx4sJ7JDRDV5aAhLJzQCjDz2ViLRduE3ijDZu3osWKBjMGkV1XPk4pfDUMqt1Aiezvephdqm6YD19GKFD9ZcXVUTp6BW', 1) + res = daemon.is_key_image_spent([ki]) + assert len(res.spent_status) == 1 + assert res.spent_status[0] == 1 self.wallet[0].refresh() res = self.wallet[0].get_balance() new_balance = res.balance @@ -687,7 +737,97 @@ class TransferTest(): daemon.generateblocks('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 1) self.wallet[0].refresh() + def check_tx_notes(self): + daemon = Daemon() + + print('Testing tx notes') + res = self.wallet[0].get_transfers() + assert len(res['in']) > 0 + in_txid = res['in'][0].txid + assert len(res['out']) > 0 + out_txid = res['out'][0].txid + res = self.wallet[0].get_tx_notes([in_txid, out_txid]) + assert res.notes == ['', ''] + res = self.wallet[0].set_tx_notes([in_txid, out_txid], ['in txid', 'out txid']) + res = self.wallet[0].get_tx_notes([in_txid, out_txid]) + assert res.notes == ['in txid', 'out txid'] + res = self.wallet[0].get_tx_notes([out_txid, in_txid]) + assert res.notes == ['out txid', 'in txid'] + + def check_rescan(self): + daemon = Daemon() + + print('Testing rescan_spent') + res = self.wallet[0].incoming_transfers(transfer_type = 'all') + transfers = res.transfers + res = self.wallet[0].rescan_spent() + res = self.wallet[0].incoming_transfers(transfer_type = 'all') + assert transfers == res.transfers + + for hard in [False, True]: + print('Testing %s rescan_blockchain' % ('hard' if hard else 'soft')) + res = self.wallet[0].incoming_transfers(transfer_type = 'all') + transfers = res.transfers + res = self.wallet[0].get_transfers() + t_in = res['in'] + t_out = res.out + res = self.wallet[0].rescan_blockchain(hard = hard) + res = self.wallet[0].incoming_transfers(transfer_type = 'all') + assert transfers == res.transfers + res = self.wallet[0].get_transfers() + assert t_in == res['in'] + # some information can not be recovered for out txes + unrecoverable_fields = ['payment_id', 'destinations', 'note'] + old_t_out = [] + for x in t_out: + e = {} + for k in x.keys(): + if not k in unrecoverable_fields: + e[k] = x[k] + old_t_out.append(e) + new_t_out = [] + for x in res.out: + e = {} + for k in x.keys(): + if not k in unrecoverable_fields: + e[k] = x[k] + new_t_out.append(e) + assert sorted(old_t_out, key = lambda k: k['txid']) == sorted(new_t_out, key = lambda k: k['txid']) + + def check_is_key_image_spent(self): + daemon = Daemon() + print('Testing is_key_image_spent') + res = self.wallet[0].incoming_transfers(transfer_type = 'all') + transfers = res.transfers + ki = [x.key_image for x in transfers] + expected = [1 if x.spent else 0 for x in transfers] + res = daemon.is_key_image_spent(ki) + assert res.spent_status == expected + + res = self.wallet[0].incoming_transfers(transfer_type = 'available') + transfers = res.transfers + ki = [x.key_image for x in transfers] + expected = [0 for x in transfers] + res = daemon.is_key_image_spent(ki) + assert res.spent_status == expected + + res = self.wallet[0].incoming_transfers(transfer_type = 'unavailable') + transfers = res.transfers + ki = [x.key_image for x in transfers] + expected = [1 for x in transfers] + res = daemon.is_key_image_spent(ki) + assert res.spent_status == expected + + ki = [ki[-1]] * 5 + expected = [1] * len(ki) + res = daemon.is_key_image_spent(ki) + assert res.spent_status == expected + + ki = ['2'*64, '1'*64] + expected = [0, 0] + res = daemon.is_key_image_spent(ki) + assert res.spent_status == expected if __name__ == '__main__': diff --git a/tests/functional_tests/txpool.py b/tests/functional_tests/txpool.py index b6af4c84f..27ae89764 100755 --- a/tests/functional_tests/txpool.py +++ b/tests/functional_tests/txpool.py @@ -46,7 +46,8 @@ class TransferTest(): def reset(self): print('Resetting blockchain') daemon = Daemon() - daemon.pop_blocks(1000) + res = daemon.get_height() + daemon.pop_blocks(res.height - 1) daemon.flush_txpool() def create(self): @@ -81,6 +82,26 @@ class TransferTest(): return txes + def check_empty_pool(self): + daemon = Daemon() + + res = daemon.get_transaction_pool_hashes() + assert not 'tx_hashes' in res or len(res.tx_hashes) == 0 + res = daemon.get_transaction_pool_stats() + assert res.pool_stats.bytes_total == 0 + assert res.pool_stats.bytes_min == 0 + assert res.pool_stats.bytes_max == 0 + assert res.pool_stats.bytes_med == 0 + assert res.pool_stats.fee_total == 0 + assert res.pool_stats.oldest == 0 + assert res.pool_stats.txs_total == 0 + assert res.pool_stats.num_failing == 0 + assert res.pool_stats.num_10m == 0 + assert res.pool_stats.num_not_relayed == 0 + assert res.pool_stats.histo_98pc == 0 + assert not 'histo' in res.pool_stats or len(res.pool_stats.histo) == 0 + assert res.pool_stats.num_double_spends == 0 + def check_txpool(self): daemon = Daemon() wallet = Wallet() @@ -89,6 +110,8 @@ class TransferTest(): height = res.height txpool_size = res.tx_pool_size + self.check_empty_pool() + txes = self.create_txes('46r4nYSevkfBUMhuykdK3gQ98XDqDTYW1hNLaXNvjpsJaSbNtdXh1sKMsdVgqkaihChAzEy29zEDPMR3NHQvGoZCLGwTerK', 5) res = daemon.get_info() @@ -97,6 +120,10 @@ class TransferTest(): res = daemon.get_transaction_pool() assert len(res.transactions) == txpool_size + total_bytes = 0 + total_fee = 0 + min_bytes = 99999999999999 + max_bytes = 0 for txid in txes.keys(): x = [x for x in res.transactions if x.id_hash == txid] assert len(x) == 1 @@ -110,9 +137,26 @@ class TransferTest(): assert x.fee == txes[txid].fee assert x.tx_blob == txes[txid].tx_blob + total_bytes += x.blob_size + total_fee += x.fee + min_bytes = min(min_bytes, x.blob_size) + max_bytes = max(max_bytes, x.blob_size) + res = daemon.get_transaction_pool_hashes() assert sorted(res.tx_hashes) == sorted(txes.keys()) + res = daemon.get_transaction_pool_stats() + assert res.pool_stats.bytes_total == total_bytes + assert res.pool_stats.bytes_min == min_bytes + assert res.pool_stats.bytes_max == max_bytes + assert res.pool_stats.bytes_med >= min_bytes and res.pool_stats.bytes_med <= max_bytes + assert res.pool_stats.fee_total == total_fee + assert res.pool_stats.txs_total == len(txes) + assert res.pool_stats.num_failing == 0 + assert res.pool_stats.num_10m == 0 + assert res.pool_stats.num_not_relayed == 0 + assert res.pool_stats.num_double_spends == 0 + print('Flushing 2 transactions') txes_keys = list(txes.keys()) daemon.flush_txpool([txes_keys[1], txes_keys[3]]) @@ -127,6 +171,42 @@ class TransferTest(): res = daemon.get_transaction_pool_hashes() assert sorted(res.tx_hashes) == sorted(new_keys) + res = daemon.get_transaction_pool() + assert len(res.transactions) == len(new_keys) + total_bytes = 0 + total_fee = 0 + min_bytes = 99999999999999 + max_bytes = 0 + for txid in new_keys: + x = [x for x in res.transactions if x.id_hash == txid] + assert len(x) == 1 + x = x[0] + assert x.kept_by_block == False + assert x.last_failed_id_hash == '0'*64 + assert x.double_spend_seen == False + assert x.weight >= x.blob_size + + assert x.blob_size * 2 == len(txes[txid].tx_blob) + assert x.fee == txes[txid].fee + assert x.tx_blob == txes[txid].tx_blob + + total_bytes += x.blob_size + total_fee += x.fee + min_bytes = min(min_bytes, x.blob_size) + max_bytes = max(max_bytes, x.blob_size) + + res = daemon.get_transaction_pool_stats() + assert res.pool_stats.bytes_total == total_bytes + assert res.pool_stats.bytes_min == min_bytes + assert res.pool_stats.bytes_max == max_bytes + assert res.pool_stats.bytes_med >= min_bytes and res.pool_stats.bytes_med <= max_bytes + assert res.pool_stats.fee_total == total_fee + assert res.pool_stats.txs_total == len(new_keys) + assert res.pool_stats.num_failing == 0 + assert res.pool_stats.num_10m == 0 + assert res.pool_stats.num_not_relayed == 0 + assert res.pool_stats.num_double_spends == 0 + print('Flushing unknown transactions') unknown_txids = ['1'*64, '2'*64, '3'*64] daemon.flush_txpool(unknown_txids) @@ -140,6 +220,8 @@ class TransferTest(): res = daemon.get_transaction_pool_hashes() assert not 'tx_hashes' in res or len(res.tx_hashes) == 0 + self.check_empty_pool() + print('Popping block') daemon.pop_blocks(1) res = daemon.get_transaction_pool_hashes() @@ -159,6 +241,9 @@ class TransferTest(): assert x.fee == txes[txid].fee assert x.tx_blob == txes[txid].tx_blob + daemon.flush_txpool() + self.check_empty_pool() + if __name__ == '__main__': TransferTest().run_test() diff --git a/tests/functional_tests/uri.py b/tests/functional_tests/uri.py new file mode 100755 index 000000000..f759b316a --- /dev/null +++ b/tests/functional_tests/uri.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 +#encoding=utf-8 + +# Copyright (c) 2019 The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Test URI RPC +""" + +from __future__ import print_function +try: + from urllib import quote as urllib_quote +except: + from urllib.parse import quote as urllib_quote + +from framework.wallet import Wallet + +class URITest(): + def run_test(self): + self.create() + self.test_monero_uri() + + def create(self): + print('Creating wallet') + wallet = Wallet() + # close the wallet if any, will throw if none is loaded + try: wallet.close_wallet() + except: pass + seed = 'velvet lymph giddy number token physics poetry unquoted nibs useful sabotage limits benches lifestyle eden nitrogen anvil fewest avoid batch vials washing fences goat unquoted' + res = wallet.restore_deterministic_wallet(seed = seed) + assert res.address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm' + assert res.seed == seed + + def test_monero_uri(self): + print('Testing monero: URI') + wallet = Wallet() + + utf8string = [u'ãˆã‚“ã—ã‚…ã†', u'ã‚ã¾ã‚„ã‹ã™'] + quoted_utf8string = [urllib_quote(x.encode('utf8')) for x in utf8string] + + ok = False + try: res = wallet.make_uri() + except: ok = True + assert ok + ok = False + try: res = wallet.make_uri(address = '') + except: ok = True + assert ok + ok = False + try: res = wallet.make_uri(address = 'kjshdkj') + except: ok = True + assert ok + + for address in [ + '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', + '4BxSHvcgTwu25WooY4BVmgdcKwZu5EksVZSZkDd6ooxSVVqQ4ubxXkhLF6hEqtw96i9cf3cVfLw8UWe95bdDKfRQeYtPwLm1Jiw7AKt2LY', + '8AsN91rznfkBGTY8psSNkJBg9SZgxxGGRUhGwRptBhgr5XSQ1XzmA9m8QAnoxydecSh5aLJXdrgXwTDMMZ1AuXsN1EX5Mtm' + ]: + res = wallet.make_uri(address = address) + assert res.uri == 'monero:' + address + res = wallet.parse_uri(res.uri) + assert res.uri.address == address + assert res.uri.payment_id == '' + assert res.uri.amount == 0 + assert res.uri.tx_description == '' + assert res.uri.recipient_name == '' + assert not 'unknown_parameters' in res or len(res.unknown_parameters) == 0 + res = wallet.make_uri(address = address, amount = 11000000000) + assert res.uri == 'monero:' + address + '?tx_amount=0.011' or res.uri == 'monero:' + address + '?tx_amount=0.011000000000' + res = wallet.parse_uri(res.uri) + assert res.uri.address == address + assert res.uri.payment_id == '' + assert res.uri.amount == 11000000000 + assert res.uri.tx_description == '' + assert res.uri.recipient_name == '' + assert not 'unknown_parameters' in res or len(res.unknown_parameters) == 0 + + address = '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm' + + res = wallet.make_uri(address = address, tx_description = utf8string[0]) + assert res.uri == 'monero:' + address + '?tx_description=' + quoted_utf8string[0] + res = wallet.parse_uri(res.uri) + assert res.uri.address == address + assert res.uri.payment_id == '' + assert res.uri.amount == 0 + assert res.uri.tx_description == utf8string[0] + assert res.uri.recipient_name == '' + assert not 'unknown_parameters' in res or len(res.unknown_parameters) == 0 + + res = wallet.make_uri(address = address, recipient_name = utf8string[0]) + assert res.uri == 'monero:' + address + '?recipient_name=' + quoted_utf8string[0] + res = wallet.parse_uri(res.uri) + assert res.uri.address == address + assert res.uri.payment_id == '' + assert res.uri.amount == 0 + assert res.uri.tx_description == '' + assert res.uri.recipient_name == utf8string[0] + assert not 'unknown_parameters' in res or len(res.unknown_parameters) == 0 + + res = wallet.make_uri(address = address, recipient_name = utf8string[0], tx_description = utf8string[1]) + assert res.uri == 'monero:' + address + '?recipient_name=' + quoted_utf8string[0] + '&tx_description=' + quoted_utf8string[1] + res = wallet.parse_uri(res.uri) + assert res.uri.address == address + assert res.uri.payment_id == '' + assert res.uri.amount == 0 + assert res.uri.tx_description == utf8string[1] + assert res.uri.recipient_name == utf8string[0] + assert not 'unknown_parameters' in res or len(res.unknown_parameters) == 0 + + res = wallet.make_uri(address = address, recipient_name = utf8string[0], tx_description = utf8string[1], amount = 1000000000000) + assert res.uri == 'monero:' + address + '?tx_amount=1.000000000000&recipient_name=' + quoted_utf8string[0] + '&tx_description=' + quoted_utf8string[1] + res = wallet.parse_uri(res.uri) + assert res.uri.address == address + assert res.uri.payment_id == '' + assert res.uri.amount == 1000000000000 + assert res.uri.tx_description == utf8string[1] + assert res.uri.recipient_name == utf8string[0] + assert not 'unknown_parameters' in res or len(res.unknown_parameters) == 0 + + res = wallet.make_uri(address = address, recipient_name = utf8string[0], tx_description = utf8string[1], amount = 1000000000000, payment_id = '1' * 64) + assert res.uri == 'monero:' + address + '?tx_payment_id=' + '1' * 64 + '&tx_amount=1.000000000000&recipient_name=' + quoted_utf8string[0] + '&tx_description=' + quoted_utf8string[1] + res = wallet.parse_uri(res.uri) + assert res.uri.address == address + assert res.uri.payment_id == '1' * 64 + assert res.uri.amount == 1000000000000 + assert res.uri.tx_description == utf8string[1] + assert res.uri.recipient_name == utf8string[0] + assert not 'unknown_parameters' in res or len(res.unknown_parameters) == 0 + + # spaces must be encoded as %20 + res = wallet.make_uri(address = address, tx_description = ' ' + utf8string[1] + ' ' + utf8string[0] + ' ', amount = 1000000000000) + assert res.uri == 'monero:' + address + '?tx_amount=1.000000000000&tx_description=%20' + quoted_utf8string[1] + '%20' + quoted_utf8string[0] + '%20' + res = wallet.parse_uri(res.uri) + assert res.uri.address == address + assert res.uri.payment_id == '' + assert res.uri.amount == 1000000000000 + assert res.uri.tx_description == ' ' + utf8string[1] + ' ' + utf8string[0] + ' ' + assert res.uri.recipient_name == '' + assert not 'unknown_parameters' in res or len(res.unknown_parameters) == 0 + + # the example from the docs + res = wallet.parse_uri('monero:46BeWrHpwXmHDpDEUmZBWZfoQpdc6HaERCNmx1pEYL2rAcuwufPN9rXHHtyUA4QVy66qeFQkn6sfK8aHYjA3jk3o1Bv16em?tx_amount=239.39014&tx_description=donation') + assert res.uri.address == '46BeWrHpwXmHDpDEUmZBWZfoQpdc6HaERCNmx1pEYL2rAcuwufPN9rXHHtyUA4QVy66qeFQkn6sfK8aHYjA3jk3o1Bv16em' + assert res.uri.amount == 239390140000000 + assert res.uri.tx_description == 'donation' + assert res.uri.recipient_name == '' + assert res.uri.payment_id == '' + assert not 'unknown_parameters' in res or len(res.unknown_parameters) == 0 + + # malformed/invalid + for uri in [ + '', + ':', + 'monero', + 'notmonero:42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', + 'MONERO:42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', + 'MONERO::42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', + 'monero:', + 'monero:badaddress', + 'monero:tx_amount=10', + 'monero:?tx_amount=10', + 'monero:42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm?tx_amount=-1', + 'monero:42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm?tx_amount=1e12', + 'monero:42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm?tx_amount=+12', + 'monero:42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm?tx_amount=1+2', + 'monero:42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm?tx_amount=A', + 'monero:42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm?tx_amount=0x2', + 'monero:42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm?tx_amount=222222222222222222222', + 'monero:42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDn?tx_amount=10', + 'monero:42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm&', + 'monero:42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm&tx_amount', + 'monero:42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm&tx_amount=', + 'monero:42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm&tx_amount=10=', + 'monero:42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm&tx_amount=10=&', + 'monero:42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm&tx_amount=10=&foo=bar', + 'monero:42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm?tx_amount=10&tx_amount=20', + 'monero:42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm?tx_payment_id=1111111111111111', + 'monero:4BxSHvcgTwu25WooY4BVmgdcKwZu5EksVZSZkDd6ooxSVVqQ4ubxXkhLF6hEqtw96i9cf3cVfLw8UWe95bdDKfRQeYtPwLm1Jiw7AKt2LY?tx_payment_id=' + '1' * 64, + 'monero:9ujeXrjzf7bfeK3KZdCqnYaMwZVFuXemPU8Ubw335rj2FN1CdMiWNyFV3ksEfMFvRp9L9qum5UxkP5rN9aLcPxbH1au4WAB', + 'monero:5K8mwfjumVseCcQEjNbf59Um6R9NfVUNkHTLhhPCmNvgDLVS88YW5tScnm83rw9mfgYtchtDDTW5jEfMhygi27j1QYphX38hg6m4VMtN29', + 'monero:7A1Hr63MfgUa8pkWxueD5xBqhQczkusYiCMYMnJGcGmuQxa7aDBxN1G7iCuLCNB3VPeb2TW7U9FdxB27xKkWKfJ8VhUZthF', + ]: + ok = False + try: res = wallet.parse_uri(uri) + except: ok = True + assert ok, res + + # unknown parameters but otherwise valid + res = wallet.parse_uri('monero:' + address + '?tx_amount=239.39014&foo=bar') + assert res.uri.address == address + assert res.uri.amount == 239390140000000 + assert res.unknown_parameters == ['foo=bar'], res + res = wallet.parse_uri('monero:' + address + '?tx_amount=239.39014&foo=bar&baz=quux') + assert res.uri.address == address + assert res.uri.amount == 239390140000000 + assert res.unknown_parameters == ['foo=bar', 'baz=quux'], res + res = wallet.parse_uri('monero:' + address + '?tx_amount=239.39014&%20=%20') + assert res.uri.address == address + assert res.uri.amount == 239390140000000 + assert res.unknown_parameters == ['%20=%20'], res + res = wallet.parse_uri('monero:' + address + '?tx_amount=239.39014&unknown=' + quoted_utf8string[0]) + assert res.uri.address == address + assert res.uri.amount == 239390140000000 + assert res.unknown_parameters == [u'unknown=' + quoted_utf8string[0]], res + + + +if __name__ == '__main__': + URITest().run_test() diff --git a/tests/functional_tests/validate_address.py b/tests/functional_tests/validate_address.py index 58748b0a2..7c3d8abfa 100755 --- a/tests/functional_tests/validate_address.py +++ b/tests/functional_tests/validate_address.py @@ -28,11 +28,10 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import time - """Test address validation RPC calls """ +from __future__ import print_function from framework.wallet import Wallet class AddressValidationTest(): diff --git a/tests/functional_tests/wallet_address.py b/tests/functional_tests/wallet.py index eda52b432..5bb3ec80a 100755 --- a/tests/functional_tests/wallet_address.py +++ b/tests/functional_tests/wallet.py @@ -29,31 +29,54 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -"""Test transaction creation RPC calls - -Test the following RPCs: - - [TODO: many tests still need to be written] - +"""Test basic wallet functionality """ from __future__ import print_function +import sys +import os +import errno + from framework.wallet import Wallet from framework.daemon import Daemon -class WalletAddressTest(): +class WalletTest(): def run_test(self): self.reset() self.create() self.check_main_address() self.check_keys() self.create_subaddresses() + self.tags() + self.attributes() self.open_close() self.languages() + self.change_password() + self.store() + + def remove_file(self, name): + WALLET_DIRECTORY = os.environ['WALLET_DIRECTORY'] + assert WALLET_DIRECTORY != '' + try: + os.unlink(WALLET_DIRECTORY + '/' + name) + except OSError as e: + if e.errno != errno.ENOENT: + raise + + def remove_wallet_files(self, name): + for suffix in ['', '.keys']: + self.remove_file(name + suffix) + + def file_exists(self, name): + WALLET_DIRECTORY = os.environ['WALLET_DIRECTORY'] + assert WALLET_DIRECTORY != '' + return os.path.isfile(WALLET_DIRECTORY + '/' + name) def reset(self): print('Resetting blockchain') daemon = Daemon() - daemon.pop_blocks(1000) + res = daemon.get_height() + daemon.pop_blocks(res.height - 1) daemon.flush_txpool() def create(self): @@ -158,6 +181,103 @@ class WalletAddressTest(): res = wallet.get_address_index('82pP87g1Vkd3LUMssBCumk3MfyEsFqLAaGDf6oxddu61EgSFzt8gCwUD4tr3kp9TUfdPs2CnpD7xLZzyC1Ei9UsW3oyCWDf') assert res.index == {'major': 1, 'minor': 0} + res = wallet.label_account(0, "main") + + def tags(self): + print('Testing tags') + wallet = Wallet() + res = wallet.get_account_tags() + assert not 'account_tags' in res or len(res.account_tags) == 0 + ok = False + try: res = wallet.get_accounts('tag') + except: ok = True + assert ok or not 'subaddress_accounts' in res or res.subaddress_accounts == 0 + wallet.tag_accounts('tag0', [1]) + res = wallet.get_account_tags() + assert len(res.account_tags) == 1 + assert res.account_tags[0].tag == 'tag0' + assert res.account_tags[0].label == '' + assert res.account_tags[0].accounts == [1] + res = wallet.get_accounts('tag0') + assert len(res.subaddress_accounts) == 1 + assert res.subaddress_accounts[0].account_index == 1 + assert res.subaddress_accounts[0].base_address == '82pP87g1Vkd3LUMssBCumk3MfyEsFqLAaGDf6oxddu61EgSFzt8gCwUD4tr3kp9TUfdPs2CnpD7xLZzyC1Ei9UsW3oyCWDf' + assert res.subaddress_accounts[0].balance == 0 + assert res.subaddress_accounts[0].unlocked_balance == 0 + assert res.subaddress_accounts[0].label == 'idx1_new' + assert res.subaddress_accounts[0].tag == 'tag0' + wallet.untag_accounts([0]) + res = wallet.get_account_tags() + assert len(res.account_tags) == 1 + assert res.account_tags[0].tag == 'tag0' + assert res.account_tags[0].label == '' + assert res.account_tags[0].accounts == [1] + wallet.untag_accounts([1]) + res = wallet.get_account_tags() + assert not 'account_tags' in res or len(res.account_tags) == 0 + wallet.tag_accounts('tag0', [0]) + wallet.tag_accounts('tag1', [1]) + res = wallet.get_account_tags() + assert len(res.account_tags) == 2 + x = [x for x in res.account_tags if x.tag == 'tag0'] + assert len(x) == 1 + assert x[0].tag == 'tag0' + assert x[0].label == '' + assert x[0].accounts == [0] + x = [x for x in res.account_tags if x.tag == 'tag1'] + assert len(x) == 1 + assert x[0].tag == 'tag1' + assert x[0].label == '' + assert x[0].accounts == [1] + wallet.tag_accounts('tagA', [0, 1]) + res = wallet.get_account_tags() + assert len(res.account_tags) == 1 + assert res.account_tags[0].tag == 'tagA' + assert res.account_tags[0].label == '' + assert res.account_tags[0].accounts == [0, 1] + wallet.tag_accounts('tagB', [1, 0]) + res = wallet.get_account_tags() + assert len(res.account_tags) == 1 + assert res.account_tags[0].tag == 'tagB' + assert res.account_tags[0].label == '' + assert res.account_tags[0].accounts == [0, 1] + wallet.set_account_tag_description('tagB', 'tag B') + res = wallet.get_account_tags() + assert len(res.account_tags) == 1 + assert res.account_tags[0].tag == 'tagB' + assert res.account_tags[0].label == 'tag B' + assert res.account_tags[0].accounts == [0, 1] + res = wallet.get_accounts('tagB') + assert len(res.subaddress_accounts) == 2 + subaddress_accounts = [] + for x in res.subaddress_accounts: + assert x.balance == 0 + assert x.unlocked_balance == 0 + subaddress_accounts.append((x.account_index, x.base_address, x.label)) + assert sorted(subaddress_accounts) == [(0, '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 'main'), (1, '82pP87g1Vkd3LUMssBCumk3MfyEsFqLAaGDf6oxddu61EgSFzt8gCwUD4tr3kp9TUfdPs2CnpD7xLZzyC1Ei9UsW3oyCWDf', 'idx1_new')] + + def attributes(self): + print('Testing attributes') + wallet = Wallet() + + ok = False + try: res = wallet.get_attribute('foo') + except: ok = True + assert ok + res = wallet.set_attribute('foo', 'bar') + res = wallet.get_attribute('foo') + assert res.value == 'bar' + res = wallet.set_attribute('foo', 'ã„ã£ã—ã‚…ã‚“') + res = wallet.get_attribute('foo') + assert res.value == u'ã„ã£ã—ã‚…ã‚“' + ok = False + try: res = wallet.get_attribute('ã„ã¡ã‚Šã‚…ã†') + except: ok = True + assert ok + res = wallet.set_attribute('ã„ã¡ã‚Šã‚…ã†', 'ã„ã£ã½ã†') + res = wallet.get_attribute('ã„ã¡ã‚Šã‚…ã†') + assert res.value == u'ã„ã£ã½ã†' + def open_close(self): print('Testing open/close') wallet = Wallet() @@ -200,11 +320,72 @@ class WalletAddressTest(): languages = res.languages languages_local = res.languages_local for language in languages + languages_local: - print('Creating ' + language.encode('utf8') + ' wallet') + sys.stdout.write('Creating ' + language + ' wallet\n') wallet.create_wallet(filename = '', language = language) res = wallet.query_key('mnemonic') wallet.close_wallet() + def change_password(self): + print('Testing password change') + wallet = Wallet() + + # close the wallet if any, will throw if none is loaded + try: wallet.close_wallet() + except: pass + + self.remove_wallet_files('test1') + + seed = 'velvet lymph giddy number token physics poetry unquoted nibs useful sabotage limits benches lifestyle eden nitrogen anvil fewest avoid batch vials washing fences goat unquoted' + res = wallet.restore_deterministic_wallet(seed = seed, filename = 'test1') + assert res.address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm' + assert res.seed == seed + + wallet.close_wallet() + res = wallet.open_wallet('test1', password = '') + res = wallet.get_address() + assert res.address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm' + + res = wallet.change_wallet_password(old_password = '', new_password = 'foo') + wallet.close_wallet() + + ok = False + try: res = wallet.open_wallet('test1', password = '') + except: ok = True + assert ok + + res = wallet.open_wallet('test1', password = 'foo') + res = wallet.get_address() + assert res.address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm' + + wallet.close_wallet() + + self.remove_wallet_files('test1') + + def store(self): + print('Testing store') + wallet = Wallet() + + # close the wallet if any, will throw if none is loaded + try: wallet.close_wallet() + except: pass + + self.remove_wallet_files('test1') + + seed = 'velvet lymph giddy number token physics poetry unquoted nibs useful sabotage limits benches lifestyle eden nitrogen anvil fewest avoid batch vials washing fences goat unquoted' + res = wallet.restore_deterministic_wallet(seed = seed, filename = 'test1') + assert res.address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm' + assert res.seed == seed + + self.remove_file('test1') + assert self.file_exists('test1.keys') + assert not self.file_exists('test1') + wallet.store() + assert self.file_exists('test1.keys') + assert self.file_exists('test1') + + wallet.close_wallet() + self.remove_wallet_files('test1') + if __name__ == '__main__': - WalletAddressTest().run_test() + WalletTest().run_test() diff --git a/tests/performance_tests/check_hash.h b/tests/performance_tests/check_hash.h index 53746fec4..294654f37 100644 --- a/tests/performance_tests/check_hash.h +++ b/tests/performance_tests/check_hash.h @@ -29,6 +29,7 @@ #pragma once #include "string_tools.h" +#include "int-util.h" #include "cryptonote_basic/difficulty.h" template<uint64_t hash_target_high, uint64_t hash_target_low, uint64_t difficulty_high, uint64_t difficulty_low> @@ -44,13 +45,18 @@ public: difficulty = difficulty_high; difficulty = (difficulty << 64) | difficulty_low; boost::multiprecision::uint256_t hash_value = std::numeric_limits<boost::multiprecision::uint256_t>::max() / hash_target; - ((uint64_t*)&hash)[0] = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); + uint64_t val; + val = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); + ((uint64_t*)&hash)[0] = SWAP64LE(val); hash_value >>= 64; - ((uint64_t*)&hash)[1] = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); + val = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); + ((uint64_t*)&hash)[1] = SWAP64LE(val); hash_value >>= 64; - ((uint64_t*)&hash)[2] = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); + val = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); + ((uint64_t*)&hash)[2] = SWAP64LE(val); hash_value >>= 64; - ((uint64_t*)&hash)[3] = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); + val = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); + ((uint64_t*)&hash)[3] = SWAP64LE(val); return true; } diff --git a/tests/unit_tests/CMakeLists.txt b/tests/unit_tests/CMakeLists.txt index a5d179040..cac1fa943 100644 --- a/tests/unit_tests/CMakeLists.txt +++ b/tests/unit_tests/CMakeLists.txt @@ -117,7 +117,8 @@ target_link_libraries(unit_tests ${Boost_THREAD_LIBRARY} ${GTEST_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} - ${EXTRA_LIBRARIES}) + ${EXTRA_LIBRARIES} + ${ZMQ_LIB}) set_property(TARGET unit_tests PROPERTY FOLDER "tests") diff --git a/tests/unit_tests/blockchain_db.cpp b/tests/unit_tests/blockchain_db.cpp index d7c60cecb..6b569d113 100644 --- a/tests/unit_tests/blockchain_db.cpp +++ b/tests/unit_tests/blockchain_db.cpp @@ -38,9 +38,6 @@ #include "string_tools.h" #include "blockchain_db/blockchain_db.h" #include "blockchain_db/lmdb/db_lmdb.h" -#ifdef BERKELEY_DB -#include "blockchain_db/berkeleydb/db_bdb.h" -#endif #include "cryptonote_basic/cryptonote_format_utils.h" using namespace cryptonote; @@ -233,11 +230,7 @@ protected: using testing::Types; -typedef Types<BlockchainLMDB -#ifdef BERKELEY_DB - , BlockchainBDB -#endif -> implementations; +typedef Types<BlockchainLMDB> implementations; TYPED_TEST_CASE(BlockchainDBTest, implementations); diff --git a/tests/unit_tests/difficulty.cpp b/tests/unit_tests/difficulty.cpp index a732e6969..e9e3272f0 100644 --- a/tests/unit_tests/difficulty.cpp +++ b/tests/unit_tests/difficulty.cpp @@ -27,6 +27,7 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "gtest/gtest.h" +#include "int-util.h" #include "cryptonote_basic/difficulty.h" static cryptonote::difficulty_type MKDIFF(uint64_t high, uint64_t low) @@ -42,13 +43,18 @@ static crypto::hash MKHASH(uint64_t high, uint64_t low) hash_target = (hash_target << 64) | low; boost::multiprecision::uint256_t hash_value = std::numeric_limits<boost::multiprecision::uint256_t>::max() / hash_target; crypto::hash h; - ((uint64_t*)&h)[0] = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); + uint64_t val; + val = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); + ((uint64_t*)&h)[0] = SWAP64LE(val); hash_value >>= 64; - ((uint64_t*)&h)[1] = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); + val = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); + ((uint64_t*)&h)[1] = SWAP64LE(val); hash_value >>= 64; - ((uint64_t*)&h)[2] = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); + val = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); + ((uint64_t*)&h)[2] = SWAP64LE(val); hash_value >>= 64; - ((uint64_t*)&h)[3] = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); + val = (hash_value & 0xffffffffffffffff).convert_to<uint64_t>(); + ((uint64_t*)&h)[3] = SWAP64LE(val); return h; } diff --git a/tests/unit_tests/epee_levin_protocol_handler_async.cpp b/tests/unit_tests/epee_levin_protocol_handler_async.cpp index 7bdb4c43d..b27699a77 100644 --- a/tests/unit_tests/epee_levin_protocol_handler_async.cpp +++ b/tests/unit_tests/epee_levin_protocol_handler_async.cpp @@ -241,13 +241,13 @@ namespace m_in_data.assign(256, 't'); - m_req_head.m_signature = LEVIN_SIGNATURE; - m_req_head.m_cb = m_in_data.size(); + m_req_head.m_signature = SWAP64LE(LEVIN_SIGNATURE); + m_req_head.m_cb = SWAP64LE(m_in_data.size()); m_req_head.m_have_to_return_data = true; - m_req_head.m_command = expected_command; - m_req_head.m_return_code = LEVIN_OK; - m_req_head.m_flags = LEVIN_PACKET_REQUEST; - m_req_head.m_protocol_version = LEVIN_PROTOCOL_VER_1; + m_req_head.m_command = SWAP32LE(expected_command); + m_req_head.m_return_code = SWAP32LE(LEVIN_OK); + m_req_head.m_flags = SWAP32LE(LEVIN_PACKET_REQUEST); + m_req_head.m_protocol_version = SWAP32LE(LEVIN_PROTOCOL_VER_1); m_commands_handler.return_code(expected_return_code); m_commands_handler.invoke_out_buf(m_expected_invoke_out_buf); @@ -337,12 +337,12 @@ TEST_F(positive_test_connection_to_levin_protocol_handler_calls, handler_process std::string in_data(256, 'q'); epee::levin::bucket_head2 req_head; - req_head.m_signature = LEVIN_SIGNATURE; - req_head.m_cb = in_data.size(); + req_head.m_signature = SWAP64LE(LEVIN_SIGNATURE); + req_head.m_cb = SWAP64LE(in_data.size()); req_head.m_have_to_return_data = true; - req_head.m_command = expected_command; - req_head.m_flags = LEVIN_PACKET_REQUEST; - req_head.m_protocol_version = LEVIN_PROTOCOL_VER_1; + req_head.m_command = SWAP32LE(expected_command); + req_head.m_flags = SWAP32LE(LEVIN_PACKET_REQUEST); + req_head.m_protocol_version = SWAP32LE(LEVIN_PROTOCOL_VER_1); std::string buf(reinterpret_cast<const char*>(&req_head), sizeof(req_head)); buf += in_data; @@ -373,13 +373,13 @@ TEST_F(positive_test_connection_to_levin_protocol_handler_calls, handler_process // Check sent response ASSERT_EQ(expected_out_data, out_data); - ASSERT_EQ(LEVIN_SIGNATURE, resp_head.m_signature); - ASSERT_EQ(expected_command, resp_head.m_command); - ASSERT_EQ(expected_return_code, resp_head.m_return_code); - ASSERT_EQ(expected_out_data.size(), resp_head.m_cb); + ASSERT_EQ(LEVIN_SIGNATURE, SWAP64LE(resp_head.m_signature)); + ASSERT_EQ(expected_command, SWAP32LE(resp_head.m_command)); + ASSERT_EQ(expected_return_code, SWAP32LE(resp_head.m_return_code)); + ASSERT_EQ(expected_out_data.size(), SWAP64LE(resp_head.m_cb)); ASSERT_FALSE(resp_head.m_have_to_return_data); - ASSERT_EQ(LEVIN_PROTOCOL_VER_1, resp_head.m_protocol_version); - ASSERT_TRUE(0 != (resp_head.m_flags & LEVIN_PACKET_RESPONSE)); + ASSERT_EQ(SWAP32LE(LEVIN_PROTOCOL_VER_1), resp_head.m_protocol_version); + ASSERT_TRUE(0 != (SWAP32LE(resp_head.m_flags) & LEVIN_PACKET_RESPONSE)); } TEST_F(positive_test_connection_to_levin_protocol_handler_calls, handler_processes_handle_read_as_notify) @@ -392,12 +392,12 @@ TEST_F(positive_test_connection_to_levin_protocol_handler_calls, handler_process std::string in_data(256, 'e'); epee::levin::bucket_head2 req_head; - req_head.m_signature = LEVIN_SIGNATURE; - req_head.m_cb = in_data.size(); + req_head.m_signature = SWAP64LE(LEVIN_SIGNATURE); + req_head.m_cb = SWAP64LE(in_data.size()); req_head.m_have_to_return_data = false; - req_head.m_command = expected_command; - req_head.m_flags = LEVIN_PACKET_REQUEST; - req_head.m_protocol_version = LEVIN_PROTOCOL_VER_1; + req_head.m_command = SWAP32LE(expected_command); + req_head.m_flags = SWAP32LE(LEVIN_PACKET_REQUEST); + req_head.m_protocol_version = SWAP32LE(LEVIN_PROTOCOL_VER_1); std::string buf(reinterpret_cast<const char*>(&req_head), sizeof(req_head)); buf += in_data; @@ -618,7 +618,7 @@ TEST_F(test_levin_protocol_handler__hanle_recv_with_invalid_data, handles_two_re TEST_F(test_levin_protocol_handler__hanle_recv_with_invalid_data, handles_unexpected_response) { - m_req_head.m_flags = LEVIN_PACKET_RESPONSE; + m_req_head.m_flags = SWAP32LE(LEVIN_PACKET_RESPONSE); prepare_buf(); ASSERT_FALSE(m_conn->m_protocol_handler.handle_recv(m_buf.data(), m_buf.size())); diff --git a/tests/unit_tests/logging.cpp b/tests/unit_tests/logging.cpp index 056eae604..c8526abae 100644 --- a/tests/unit_tests/logging.cpp +++ b/tests/unit_tests/logging.cpp @@ -178,3 +178,20 @@ TEST(logging, last_precedence) cleanup(); } +TEST(logging, multiline) +{ + init(); + mlog_set_categories("global:INFO"); + MGINFO("first\nsecond\nthird"); + std::string str; + ASSERT_TRUE(load_log_to_string(log_filename, str)); + ASSERT_TRUE(nlines(str) == 3); + ASSERT_TRUE(str.find("global") != std::string::npos); + ASSERT_TRUE(str.find("first") != std::string::npos); + ASSERT_TRUE(str.find("second") != std::string::npos); + ASSERT_TRUE(str.find("third") != std::string::npos); + ASSERT_TRUE(str.find("first\nsecond") == std::string::npos); + ASSERT_TRUE(str.find("second\nthird") == std::string::npos); + cleanup(); +} + diff --git a/tests/unit_tests/net.cpp b/tests/unit_tests/net.cpp index f24ffb45c..253280d4d 100644 --- a/tests/unit_tests/net.cpp +++ b/tests/unit_tests/net.cpp @@ -40,6 +40,7 @@ #include <boost/range/adaptor/sliced.hpp> #include <boost/range/combine.hpp> #include <boost/system/error_code.hpp> +#include <boost/thread/scoped_thread.hpp> #include <boost/thread/thread.hpp> #include <boost/uuid/nil_generator.hpp> #include <boost/uuid/random_generator.hpp> @@ -59,6 +60,7 @@ #include "net/socks_connect.h" #include "net/parse.h" #include "net/tor_address.h" +#include "net/zmq.h" #include "p2p/net_peerlist_boost_serialization.h" #include "serialization/keyvalue_serialization.h" #include "storages/portable_storage.h" @@ -1040,7 +1042,8 @@ TEST(dandelionpp_map, dropped_connection_remapped) boost::uuids::random_generator random_uuid{}; std::vector<boost::uuids::uuid> connections{3}; - std::generate(connections.begin(), connections.end(), random_uuid); + for (auto &e: connections) + e = random_uuid(); std::sort(connections.begin(), connections.end()); // select 3 of 3 outgoing connections @@ -1070,7 +1073,8 @@ TEST(dandelionpp_map, dropped_connection_remapped) } std::map<boost::uuids::uuid, boost::uuids::uuid> mapping; std::vector<boost::uuids::uuid> in_connections{9}; - std::generate(in_connections.begin(), in_connections.end(), random_uuid); + for (auto &e: in_connections) + e = random_uuid(); { std::map<boost::uuids::uuid, std::size_t> used; std::multimap<boost::uuids::uuid, boost::uuids::uuid> inverse_mapping; @@ -1149,7 +1153,8 @@ TEST(dandelionpp_map, dropped_connection_remapped) } // map 8 new incoming connections across 3 outgoing links in_connections.resize(18); - std::generate(in_connections.begin() + 10, in_connections.end(), random_uuid); + for (size_t i = 10; i < in_connections.size(); ++i) + in_connections[i] = random_uuid(); { std::map<boost::uuids::uuid, std::size_t> used; for (const boost::uuids::uuid& connection : in_connections) @@ -1176,7 +1181,8 @@ TEST(dandelionpp_map, dropped_all_connections) boost::uuids::random_generator random_uuid{}; std::vector<boost::uuids::uuid> connections{8}; - std::generate(connections.begin(), connections.end(), random_uuid); + for (auto &e: connections) + e = random_uuid(); std::sort(connections.begin(), connections.end()); // select 3 of 8 outgoing connections @@ -1205,7 +1211,8 @@ TEST(dandelionpp_map, dropped_all_connections) } } std::vector<boost::uuids::uuid> in_connections{9}; - std::generate(in_connections.begin(), in_connections.end(), random_uuid); + for (auto &e: in_connections) + e = random_uuid(); { std::map<boost::uuids::uuid, std::size_t> used; std::map<boost::uuids::uuid, boost::uuids::uuid> mapping; @@ -1237,7 +1244,8 @@ TEST(dandelionpp_map, dropped_all_connections) // select 3 of 30 connections, only 7 should be remapped to new indexes (but all to new uuids) connections.resize(30); - std::generate(connections.begin(), connections.end(), random_uuid); + for (auto &e: connections) + e = random_uuid(); EXPECT_TRUE(mapper.update(connections)); { std::map<boost::uuids::uuid, std::size_t> used; @@ -1253,3 +1261,131 @@ TEST(dandelionpp_map, dropped_all_connections) EXPECT_EQ(3u, entry.second); } } + +TEST(zmq, error_codes) +{ + EXPECT_EQ( + std::addressof(net::zmq::error_category()), + std::addressof(net::zmq::make_error_code(0).category()) + ); + EXPECT_EQ( + std::make_error_condition(std::errc::not_a_socket), + net::zmq::make_error_code(ENOTSOCK) + ); + + EXPECT_TRUE( + []() -> expect<void> + { + MONERO_ZMQ_CHECK(zmq_msg_send(nullptr, nullptr, 0)); + return success(); + }().matches(std::errc::not_a_socket) + ); + + bool thrown = false; + try + { + MONERO_ZMQ_THROW("stuff"); + } + catch (const std::system_error& e) + { + thrown = true; + EXPECT_EQ(std::make_error_condition(std::errc::not_a_socket), e.code()); + } + EXPECT_TRUE(thrown); +} + +TEST(zmq, read_write) +{ + net::zmq::context context{zmq_init(1)}; + ASSERT_NE(nullptr, context); + + net::zmq::socket send_socket{zmq_socket(context.get(), ZMQ_REQ)}; + net::zmq::socket recv_socket{zmq_socket(context.get(), ZMQ_REP)}; + ASSERT_NE(nullptr, send_socket); + ASSERT_NE(nullptr, recv_socket); + + ASSERT_EQ(0u, zmq_bind(recv_socket.get(), "inproc://testing")); + ASSERT_EQ(0u, zmq_connect(send_socket.get(), "inproc://testing")); + + std::string message; + message.resize(1024); + crypto::rand(message.size(), reinterpret_cast<std::uint8_t*>(std::addressof(message[0]))); + + ASSERT_TRUE(bool(net::zmq::send(epee::strspan<std::uint8_t>(message), send_socket.get()))); + + const expect<std::string> received = net::zmq::receive(recv_socket.get()); + ASSERT_TRUE(bool(received)); + EXPECT_EQ(message, *received); +} + +TEST(zmq, read_write_multipart) +{ + net::zmq::context context{zmq_init(1)}; + ASSERT_NE(nullptr, context); + + net::zmq::socket send_socket{zmq_socket(context.get(), ZMQ_REQ)}; + net::zmq::socket recv_socket{zmq_socket(context.get(), ZMQ_REP)}; + ASSERT_NE(nullptr, send_socket); + ASSERT_NE(nullptr, recv_socket); + + ASSERT_EQ(0u, zmq_bind(recv_socket.get(), "inproc://testing")); + ASSERT_EQ(0u, zmq_connect(send_socket.get(), "inproc://testing")); + + std::string message; + message.resize(999); + crypto::rand(message.size(), reinterpret_cast<std::uint8_t*>(std::addressof(message[0]))); + + for (unsigned i = 0; i < 3; ++i) + { + const expect<std::string> received = net::zmq::receive(recv_socket.get(), ZMQ_DONTWAIT); + ASSERT_FALSE(bool(received)); + EXPECT_EQ(net::zmq::make_error_code(EAGAIN), received.error()); + + const epee::span<const std::uint8_t> bytes{ + reinterpret_cast<const std::uint8_t*>(std::addressof(message[0])) + (i * 333), 333 + }; + ASSERT_TRUE(bool(net::zmq::send(bytes, send_socket.get(), (i == 2 ? 0 : ZMQ_SNDMORE)))); + } + + const expect<std::string> received = net::zmq::receive(recv_socket.get(), ZMQ_DONTWAIT); + ASSERT_TRUE(bool(received)); + EXPECT_EQ(message, *received); +} + +TEST(zmq, read_write_termination) +{ + net::zmq::context context{zmq_init(1)}; + ASSERT_NE(nullptr, context); + + // must be declared before sockets and after context + boost::scoped_thread<> thread{}; + + net::zmq::socket send_socket{zmq_socket(context.get(), ZMQ_REQ)}; + net::zmq::socket recv_socket{zmq_socket(context.get(), ZMQ_REP)}; + ASSERT_NE(nullptr, send_socket); + ASSERT_NE(nullptr, recv_socket); + + ASSERT_EQ(0u, zmq_bind(recv_socket.get(), "inproc://testing")); + ASSERT_EQ(0u, zmq_connect(send_socket.get(), "inproc://testing")); + + std::string message; + message.resize(1024); + crypto::rand(message.size(), reinterpret_cast<std::uint8_t*>(std::addressof(message[0]))); + + ASSERT_TRUE(bool(net::zmq::send(epee::strspan<std::uint8_t>(message), send_socket.get(), ZMQ_SNDMORE))); + + expect<std::string> received = net::zmq::receive(recv_socket.get(), ZMQ_DONTWAIT); + ASSERT_FALSE(bool(received)); + EXPECT_EQ(net::zmq::make_error_code(EAGAIN), received.error()); + + thread = boost::scoped_thread<>{ + boost::thread{ + [&context] () { context.reset(); } + } + }; + + received = net::zmq::receive(recv_socket.get()); + ASSERT_FALSE(bool(received)); + EXPECT_EQ(net::zmq::make_error_code(ETERM), received.error()); +} + diff --git a/tests/unit_tests/output_distribution.cpp b/tests/unit_tests/output_distribution.cpp index 38f442c59..eec694d1e 100644 --- a/tests/unit_tests/output_distribution.cpp +++ b/tests/unit_tests/output_distribution.cpp @@ -93,7 +93,7 @@ bool get_output_distribution(uint64_t amount, uint64_t from, uint64_t to, uint64 crypto::hash get_block_hash(uint64_t height) { - crypto::hash hash; + crypto::hash hash = crypto::null_hash; *((uint64_t*)&hash) = height; return hash; } diff --git a/tests/unit_tests/test_protocol_pack.cpp b/tests/unit_tests/test_protocol_pack.cpp index 0ae2e9c68..59e46e332 100644 --- a/tests/unit_tests/test_protocol_pack.cpp +++ b/tests/unit_tests/test_protocol_pack.cpp @@ -42,7 +42,7 @@ TEST(protocol_pack, protocol_pack_command) r.total_height = 3; for(int i = 1; i < 10000; i += i*10) { - r.m_block_ids.resize(i, boost::value_initialized<crypto::hash>()); + r.m_block_ids.resize(i, crypto::hash{}); bool res = epee::serialization::store_t_to_binary(r, buff); ASSERT_TRUE(res); diff --git a/utils/python-rpc/framework/daemon.py b/utils/python-rpc/framework/daemon.py index 23d5ec0f0..1d6c57c56 100644 --- a/utils/python-rpc/framework/daemon.py +++ b/utils/python-rpc/framework/daemon.py @@ -49,6 +49,7 @@ class Daemon(object): 'id': '0' } return self.rpc.send_json_rpc_request(getblocktemplate) + get_block_template = getblocktemplate def send_raw_transaction(self, tx_as_hex, do_not_relay = False, do_sanity_checks = True): send_raw_transaction = { @@ -57,6 +58,7 @@ class Daemon(object): 'do_sanity_checks': do_sanity_checks, } return self.rpc.send_request("/send_raw_transaction", send_raw_transaction) + sendrawtransaction = send_raw_transaction def submitblock(self, block): submitblock = { @@ -66,6 +68,7 @@ class Daemon(object): 'id': '0' } return self.rpc.send_json_rpc_request(submitblock) + submit_block = submitblock def getblock(self, hash = '', height = 0, fill_pow_hash = False): getblock = { @@ -79,6 +82,7 @@ class Daemon(object): 'id': '0' } return self.rpc.send_json_rpc_request(getblock) + get_block = getblock def getlastblockheader(self): getlastblockheader = { @@ -89,6 +93,7 @@ class Daemon(object): 'id': '0' } return self.rpc.send_json_rpc_request(getlastblockheader) + get_last_block_header = getlastblockheader def getblockheaderbyhash(self, hash = "", hashes = []): getblockheaderbyhash = { @@ -101,6 +106,7 @@ class Daemon(object): 'id': '0' } return self.rpc.send_json_rpc_request(getblockheaderbyhash) + get_block_header_by_hash = getblockheaderbyhash def getblockheaderbyheight(self, height): getblockheaderbyheight = { @@ -112,6 +118,7 @@ class Daemon(object): 'id': '0' } return self.rpc.send_json_rpc_request(getblockheaderbyheight) + get_block_header_by_height = getblockheaderbyheight def getblockheadersrange(self, start_height, end_height, fill_pow_hash = False): getblockheadersrange = { @@ -125,6 +132,7 @@ class Daemon(object): 'id': '0' } return self.rpc.send_json_rpc_request(getblockheadersrange) + get_block_headers_range = getblockheadersrange def get_connections(self): get_connections = { @@ -141,6 +149,7 @@ class Daemon(object): 'id': '0' } return self.rpc.send_json_rpc_request(get_info) + getinfo = get_info def hard_fork_info(self): hard_fork_info = { @@ -172,6 +181,7 @@ class Daemon(object): 'id': '0' } return self.rpc.send_request("/get_height", get_height) + getheight = get_height def pop_blocks(self, nblocks = 1): pop_blocks = { @@ -208,6 +218,11 @@ class Daemon(object): } return self.rpc.send_request('/get_transaction_pool_hashes', get_transaction_pool_hashes) + def get_transaction_pool_stats(self): + get_transaction_pool_stats = { + } + return self.rpc.send_request('/get_transaction_pool_stats', get_transaction_pool_stats) + def flush_txpool(self, txids = []): flush_txpool = { 'method': 'flush_txpool', @@ -256,6 +271,7 @@ class Daemon(object): 'split': split, } return self.rpc.send_request('/get_transactions', get_transactions) + gettransactions = get_transactions def get_outs(self, outputs = [], get_txid = False): get_outs = { @@ -344,3 +360,141 @@ class Daemon(object): 'id': '0' } return self.rpc.send_json_rpc_request(get_fee_estimate) + + def is_key_image_spent(self, key_images = []): + is_key_image_spent = { + 'key_images': key_images, + } + return self.rpc.send_request('/is_key_image_spent', is_key_image_spent) + + def save_bc(self): + save_bc = { + } + return self.rpc.send_request('/save_bc', save_bc) + + def get_peer_list(self): + get_peer_list = { + } + return self.rpc.send_request('/get_peer_list', get_peer_list) + + def set_log_hash_rate(self, visible): + set_log_hash_rate = { + 'visible': visible, + } + return self.rpc.send_request('/set_log_hash_rate', set_log_hash_rate) + + def stop_daemon(self): + stop_daemon = { + } + return self.rpc.send_request('/stop_daemon', stop_daemon) + + def get_net_stats(self): + get_net_stats = { + } + return self.rpc.send_request('/get_net_stats', get_net_stats) + + def get_limit(self): + get_limit = { + } + return self.rpc.send_request('/get_limit', get_limit) + + def set_limit(self, limit_down, limit_up): + set_limit = { + 'limit_down': limit_down, + 'limit_up': limit_up, + } + return self.rpc.send_request('/set_limit', set_limit) + + def out_peers(self, out_peers): + out_peers = { + 'out_peers': out_peers, + } + return self.rpc.send_request('/out_peers', out_peers) + + def in_peers(self, in_peers): + in_peers = { + 'in_peers': in_peers, + } + return self.rpc.send_request('/in_peers', in_peers) + + def update(self, command, path = None): + update = { + 'command': command, + 'path': path, + } + return self.rpc.send_request('/update', update) + + def get_block_count(self): + get_block_count = { + 'method': 'get_block_count', + 'params': { + }, + 'jsonrpc': '2.0', + 'id': '0' + } + return self.rpc.send_json_rpc_request(get_block_count) + getblockcount = get_block_count + + def get_block_hash(self, height): + get_block_hash = { + 'method': 'get_block_hash', + 'params': [height], + 'jsonrpc': '2.0', + 'id': '0' + } + return self.rpc.send_json_rpc_request(get_block_hash) + on_get_block_hash = get_block_hash + on_getblockhash = get_block_hash + + def relay_tx(self, txids = []): + relay_tx = { + 'method': 'relay_tx', + 'params': { + 'txids': txids, + }, + 'jsonrpc': '2.0', + 'id': '0' + } + return self.rpc.send_json_rpc_request(relay_tx) + + def sync_info(self): + sync_info = { + 'method': 'sync_info', + 'params': { + }, + 'jsonrpc': '2.0', + 'id': '0' + } + return self.rpc.send_json_rpc_request(sync_info) + + def get_txpool_backlog(self): + get_txpool_backlog = { + 'method': 'get_txpool_backlog', + 'params': { + }, + 'jsonrpc': '2.0', + 'id': '0' + } + return self.rpc.send_json_rpc_request(get_txpool_backlog) + + def prune_blockchain(self, check = False): + prune_blockchain = { + 'method': 'prune_blockchain', + 'params': { + 'check': check, + }, + 'jsonrpc': '2.0', + 'id': '0' + } + return self.rpc.send_json_rpc_request(prune_blockchain) + + def get_block_rate(self, seconds = [3600]): + get_block_rate = { + 'method': 'get_block_rate', + 'params': { + 'seconds': seconds, + }, + 'jsonrpc': '2.0', + 'id': '0' + } + return self.rpc.send_json_rpc_request(get_block_rate) diff --git a/utils/python-rpc/framework/wallet.py b/utils/python-rpc/framework/wallet.py index 36ff3644f..6a3fabdc9 100644 --- a/utils/python-rpc/framework/wallet.py +++ b/utils/python-rpc/framework/wallet.py @@ -37,18 +37,6 @@ class Wallet(object): self.port = port self.rpc = JSONRPC('{protocol}://{host}:{port}'.format(protocol=protocol, host=host, port=port if port else 18090+idx)) - def make_uniform_destinations(self, address, transfer_amount, transfer_number_of_destinations=1): - destinations = [] - for i in range(transfer_number_of_destinations): - destinations.append({"amount":transfer_amount,"address":address}) - return destinations - - def make_destinations(self, addresses, transfer_amounts): - destinations = [] - for i in range(len(addresses)): - destinations.append({'amount':transfer_amounts[i],'address':addresses[i]}) - return destinations - def transfer(self, destinations, account_index = 0, subaddr_indices = [], priority = 0, ring_size = 0, unlock_time = 0, payment_id = '', get_tx_key = True, do_not_relay = False, get_tx_hex = False, get_tx_metadata = False): transfer = { 'method': 'transfer', @@ -103,6 +91,17 @@ class Wallet(object): } return self.rpc.send_json_rpc_request(get_transfer_by_txid) + def get_payments(self, payment_id): + get_payments = { + 'method': 'get_payments', + 'params': { + 'payment_id': payment_id, + }, + 'jsonrpc': '2.0', + 'id': '0' + } + return self.rpc.send_json_rpc_request(get_payments) + def get_bulk_payments(self, payment_ids = [], min_block_height = 0): get_bulk_payments = { 'method': 'get_bulk_payments', @@ -140,26 +139,35 @@ class Wallet(object): } return self.rpc.send_json_rpc_request(create_wallet) - def get_balance(self, account_index = 0, address_indices = [], all_accounts = False): + def get_balance(self, account_index = 0, address_indices = [], all_accounts = False, strict = False): get_balance = { 'method': 'get_balance', 'params': { 'account_index': account_index, 'address_indices': address_indices, 'all_accounts': all_accounts, + 'strict': strict, }, 'jsonrpc': '2.0', 'id': '0' } return self.rpc.send_json_rpc_request(get_balance) + getbalance = get_balance - def sweep_dust(self): + def sweep_dust(self, get_tx_keys = True, do_not_relay = False, get_tx_hex = False, get_tx_metadata = False): sweep_dust = { 'method': 'sweep_dust', + 'params': { + 'get_tx_keys': get_tx_keys, + 'do_not_relay': do_not_relay, + 'get_tx_hex': get_tx_hex, + 'get_tx_metadata': get_tx_metadata, + }, 'jsonrpc': '2.0', 'id': '0' } return self.rpc.send_json_rpc_request(sweep_dust) + sweep_unmixable = sweep_dust def sweep_all(self, address = '', account_index = 0, subaddr_indices = [], priority = 0, ring_size = 0, outputs = 1, unlock_time = 0, payment_id = '', get_tx_keys = False, below_amount = 0, do_not_relay = False, get_tx_hex = False, get_tx_metadata = False): sweep_all = { @@ -216,6 +224,7 @@ class Wallet(object): 'id': '0' } return self.rpc.send_json_rpc_request(get_address) + getaddress = get_address def create_account(self, label = ""): create_account = { @@ -344,6 +353,34 @@ class Wallet(object): } return self.rpc.send_json_rpc_request(close_wallet) + def change_wallet_password(self, old_password, new_password): + change_wallet_password = { + 'method': 'change_wallet_password', + 'params' : { + 'old_password': old_password, + 'new_password': new_password, + }, + 'jsonrpc': '2.0', + 'id': '0' + } + return self.rpc.send_json_rpc_request(change_wallet_password) + + def store(self): + store = { + 'method': 'store', + 'jsonrpc': '2.0', + 'id': '0' + } + return self.rpc.send_json_rpc_request(store) + + def stop_wallet(self): + stop_wallet = { + 'method': 'stop_wallet', + 'jsonrpc': '2.0', + 'id': '0' + } + return self.rpc.send_json_rpc_request(stop_wallet) + def refresh(self): refresh = { 'method': 'refresh', @@ -474,6 +511,18 @@ class Wallet(object): } return self.rpc.send_json_rpc_request(make_multisig) + def finalize_multisig(self, multisig_info, password = ''): + finalize_multisig = { + 'method': 'finalize_multisig', + 'params' : { + 'multisig_info': multisig_info, + 'password': password, + }, + 'jsonrpc': '2.0', + 'id': '0' + } + return self.rpc.send_json_rpc_request(finalize_multisig) + def exchange_multisig_keys(self, multisig_info, password = ''): exchange_multisig_keys = { 'method': 'exchange_multisig_keys', @@ -604,6 +653,31 @@ class Wallet(object): } return self.rpc.send_json_rpc_request(check_tx_proof) + def get_spend_proof(self, txid = '', message = ''): + get_spend_proof = { + 'method': 'get_spend_proof', + 'params' : { + 'txid': txid, + 'message': message, + }, + 'jsonrpc': '2.0', + 'id': '0' + } + return self.rpc.send_json_rpc_request(get_spend_proof) + + def check_spend_proof(self, txid = '', message = '', signature = ''): + check_spend_proof = { + 'method': 'check_spend_proof', + 'params' : { + 'txid': txid, + 'message': message, + 'signature': signature, + }, + 'jsonrpc': '2.0', + 'id': '0' + } + return self.rpc.send_json_rpc_request(check_spend_proof) + def get_reserve_proof(self, all_ = True, account_index = 0, amount = 0, message = ''): get_reserve_proof = { 'method': 'get_reserve_proof', @@ -662,6 +736,7 @@ class Wallet(object): 'id': '0' } return self.rpc.send_json_rpc_request(get_height) + getheight = get_height def relay_tx(self, hex_): relay_tx = { @@ -763,6 +838,230 @@ class Wallet(object): } return self.rpc.send_json_rpc_request(validate_address) + def get_accounts(self, tag): + get_accounts = { + 'method': 'get_accounts', + 'params': { + 'tag': tag, + }, + 'jsonrpc': '2.0', + 'id': '0' + } + return self.rpc.send_json_rpc_request(get_accounts) + + def get_account_tags(self): + get_account_tags = { + 'method': 'get_account_tags', + 'params': { + }, + 'jsonrpc': '2.0', + 'id': '0' + } + return self.rpc.send_json_rpc_request(get_account_tags) + + def tag_accounts(self, tag, accounts = []): + tag_accounts = { + 'method': 'tag_accounts', + 'params': { + 'tag': tag, + 'accounts': accounts, + }, + 'jsonrpc': '2.0', + 'id': '0' + } + return self.rpc.send_json_rpc_request(tag_accounts) + + def untag_accounts(self, accounts = []): + untag_accounts = { + 'method': 'untag_accounts', + 'params': { + 'accounts': accounts, + }, + 'jsonrpc': '2.0', + 'id': '0' + } + return self.rpc.send_json_rpc_request(untag_accounts) + + def set_account_tag_description(self, tag, description): + set_account_tag_description = { + 'method': 'set_account_tag_description', + 'params': { + 'tag': tag, + 'description': description, + }, + 'jsonrpc': '2.0', + 'id': '0' + } + return self.rpc.send_json_rpc_request(set_account_tag_description) + + def rescan_blockchain(self, hard = False): + rescan_blockchain = { + 'method': 'rescan_blockchain', + 'jsonrpc': '2.0', + 'params': { + 'hard': hard, + }, + 'id': '0' + } + return self.rpc.send_json_rpc_request(rescan_blockchain) + + def rescan_spent(self): + rescan_spent = { + 'method': 'rescan_spent', + 'jsonrpc': '2.0', + 'params': { + }, + 'id': '0' + } + return self.rpc.send_json_rpc_request(rescan_spent) + + def set_tx_notes(self, txids = [], notes = []): + set_tx_notes = { + 'method': 'set_tx_notes', + 'jsonrpc': '2.0', + 'params': { + 'txids': txids, + 'notes': notes, + }, + 'id': '0' + } + return self.rpc.send_json_rpc_request(set_tx_notes) + + def get_tx_notes(self, txids = []): + get_tx_notes = { + 'method': 'get_tx_notes', + 'jsonrpc': '2.0', + 'params': { + 'txids': txids, + }, + 'id': '0' + } + return self.rpc.send_json_rpc_request(get_tx_notes) + + def set_attribute(self, key, value): + set_attribute = { + 'method': 'set_attribute', + 'jsonrpc': '2.0', + 'params': { + 'key': key, + 'value': value, + }, + 'id': '0' + } + return self.rpc.send_json_rpc_request(set_attribute) + + def get_attribute(self, key): + get_attribute = { + 'method': 'get_attribute', + 'jsonrpc': '2.0', + 'params': { + 'key': key, + }, + 'id': '0' + } + return self.rpc.send_json_rpc_request(get_attribute) + + def make_uri(self, address = '', payment_id = '', amount = 0, tx_description = '', recipient_name = ''): + make_uri = { + 'method': 'make_uri', + 'jsonrpc': '2.0', + 'params': { + 'address': address, + 'payment_id': payment_id, + 'amount': amount, + 'tx_description': tx_description, + 'recipient_name': recipient_name, + }, + 'id': '0' + } + return self.rpc.send_json_rpc_request(make_uri) + + def parse_uri(self, uri): + parse_uri = { + 'method': 'parse_uri', + 'jsonrpc': '2.0', + 'params': { + 'uri': uri, + }, + 'id': '0' + } + return self.rpc.send_json_rpc_request(parse_uri) + + def add_address_book(self, address = '', payment_id = '', description = ''): + add_address_book = { + 'method': 'add_address_book', + 'jsonrpc': '2.0', + 'params': { + 'address': address, + 'payment_id': payment_id, + 'description': description, + }, + 'id': '0' + } + return self.rpc.send_json_rpc_request(add_address_book) + + def edit_address_book(self, index, address = None, payment_id = None, description = None): + edit_address_book = { + 'method': 'edit_address_book', + 'jsonrpc': '2.0', + 'params': { + 'index': index, + 'set_address': address != None, + 'address': address or '', + 'set_payment_id': payment_id != None, + 'payment_id': payment_id or '', + 'set_description': description != None, + 'description': description or '', + }, + 'id': '0' + } + return self.rpc.send_json_rpc_request(edit_address_book) + + def get_address_book(self, entries = []): + get_address_book = { + 'method': 'get_address_book', + 'jsonrpc': '2.0', + 'params': { + 'entries': entries, + }, + 'id': '0' + } + return self.rpc.send_json_rpc_request(get_address_book) + + def delete_address_book(self, index): + delete_address_book = { + 'method': 'delete_address_book', + 'jsonrpc': '2.0', + 'params': { + 'index': index, + }, + 'id': '0' + } + return self.rpc.send_json_rpc_request(delete_address_book) + + def start_mining(self, threads_count, do_background_mining = False, ignore_battery = False): + start_mining = { + 'method': 'start_mining', + 'jsonrpc': '2.0', + 'params': { + 'threads_count': threads_count, + 'do_background_mining': do_background_mining, + 'ignore_battery': ignore_battery, + }, + 'id': '0' + } + return self.rpc.send_json_rpc_request(start_mining) + + def stop_mining(self): + stop_mining = { + 'method': 'stop_mining', + 'jsonrpc': '2.0', + 'params': { + }, + 'id': '0' + } + return self.rpc.send_json_rpc_request(stop_mining) + def get_version(self): get_version = { 'method': 'get_version', |