diff options
148 files changed, 4028 insertions, 1672 deletions
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7aae84977..bf7b24f61 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,6 +23,7 @@ env: CCACHE_SETTINGS: | ccache --max-size=150M ccache --set-config=compression=true + USE_DEVICE_TREZOR_MANDATORY: ON jobs: build-macos: @@ -39,7 +40,9 @@ jobs: key: ccache-${{ runner.os }}-build-${{ github.sha }} restore-keys: ccache-${{ runner.os }}-build- - name: install dependencies - run: HOMEBREW_NO_AUTO_UPDATE=1 brew install boost hidapi openssl zmq libpgm miniupnpc expat libunwind-headers protobuf ccache + run: | + HOMEBREW_NO_AUTO_UPDATE=1 brew install boost hidapi openssl zmq libpgm miniupnpc expat libunwind-headers protobuf@21 ccache + brew link protobuf@21 - name: build run: | ${{env.CCACHE_SETTINGS}} @@ -65,7 +68,12 @@ jobs: - uses: msys2/setup-msys2@v2 with: update: true - install: mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-ccache mingw-w64-x86_64-boost mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq mingw-w64-x86_64-libsodium mingw-w64-x86_64-hidapi mingw-w64-x86_64-protobuf-c mingw-w64-x86_64-libusb mingw-w64-x86_64-unbound git + install: mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-ccache mingw-w64-x86_64-boost mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq mingw-w64-x86_64-libsodium mingw-w64-x86_64-hidapi mingw-w64-x86_64-libusb mingw-w64-x86_64-unbound git + - shell: msys2 {0} + run: | + curl -O https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-protobuf-c-1.4.1-1-any.pkg.tar.zst + curl -O https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-protobuf-21.9-1-any.pkg.tar.zst + pacman --noconfirm -U mingw-w64-x86_64-protobuf-c-1.4.1-1-any.pkg.tar.zst mingw-w64-x86_64-protobuf-21.9-1-any.pkg.tar.zst - name: build run: | ${{env.CCACHE_SETTINGS}} diff --git a/.github/workflows/copyright.yml b/.github/workflows/copyright.yml index 16bd43ea7..27e0bb457 100644 --- a/.github/workflows/copyright.yml +++ b/.github/workflows/copyright.yml @@ -1,5 +1,6 @@ name: ci/gh-actions/copyright on: + workflow_dispatch: schedule: - cron: '0 0 1 1 *' jobs: @@ -45,7 +46,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} commit-message: "Copyright: Update to ${{ env.YEAR }}" committer: GitHub <noreply@github.com> - author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> + author: copyCat <copy@Cat.com> signoff: false branch: bump-copyright-year delete-branch: true diff --git a/.github/workflows/depends.yml b/.github/workflows/depends.yml index 27b294503..c534965e8 100644 --- a/.github/workflows/depends.yml +++ b/.github/workflows/depends.yml @@ -18,6 +18,7 @@ env: CCACHE_SETTINGS: | ccache --max-size=150M ccache --set-config=compression=true + USE_DEVICE_TREZOR_MANDATORY: ON jobs: build-cross: @@ -50,10 +51,10 @@ jobs: host: "x86_64-unknown-linux-gnu" packages: "gperf cmake python3-zmq libdbus-1-dev libharfbuzz-dev" - name: "Cross-Mac x86_64" - host: "x86_64-apple-darwin11" + host: "x86_64-apple-darwin" packages: "cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev python3-setuptools-git" - name: "Cross-Mac aarch64" - host: "aarch64-apple-darwin11" + host: "aarch64-apple-darwin" packages: "cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev python3-setuptools-git" - name: "x86_64 Freebsd" host: "x86_64-unknown-freebsd" @@ -104,7 +105,7 @@ jobs: ${{env.CCACHE_SETTINGS}} make depends target=${{ matrix.toolchain.host }} -j2 - uses: actions/upload-artifact@v3 - if: ${{ matrix.toolchain.host == 'x86_64-w64-mingw32' || matrix.toolchain.host == 'x86_64-apple-darwin11' || matrix.toolchain.host == 'x86_64-unknown-linux-gnu' }} + if: ${{ matrix.toolchain.host == 'x86_64-w64-mingw32' || matrix.toolchain.host == 'x86_64-apple-darwin' || matrix.toolchain.host == 'x86_64-unknown-linux-gnu' }} with: name: ${{ matrix.toolchain.name }} path: | diff --git a/CMakeLists.txt b/CMakeLists.txt index 1f634c114..83d760c66 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,6 +28,9 @@ # # Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers +cmake_minimum_required(VERSION 3.5) +message(STATUS "CMake version ${CMAKE_VERSION}") + set(CMAKE_EXPORT_COMPILE_COMMANDS ON) list(INSERT CMAKE_MODULE_PATH 0 @@ -37,15 +40,15 @@ include(CheckCXXCompilerFlag) include(CheckLinkerFlag) include(CheckLibraryExists) include(CheckFunctionExists) +if (POLICY CMP0148) + cmake_policy(SET CMP0148 OLD) # https://cmake.org/cmake/help/latest/policy/CMP0148.html +endif() include(FindPythonInterp) if (IOS) INCLUDE(CmakeLists_IOS.txt) endif() -cmake_minimum_required(VERSION 3.5) -message(STATUS "CMake version ${CMAKE_VERSION}") - project(monero) option (USE_CCACHE "Use ccache if a usable instance is found" ON) @@ -96,7 +99,7 @@ enable_language(C ASM) set(CMAKE_C_STANDARD 11) set(CMAKE_C_STANDARD_REQUIRED ON) set(CMAKE_C_EXTENSIONS OFF) -set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) @@ -219,7 +222,7 @@ function(forbid_undefined_symbols) file(MAKE_DIRECTORY "${TEST_PROJECT}") file(WRITE "${TEST_PROJECT}/CMakeLists.txt" [=[ -cmake_minimum_required(VERSION 3.1) +cmake_minimum_required(VERSION 3.5) project(test) option(EXPECT_SUCCESS "" ON) file(WRITE "${CMAKE_SOURCE_DIR}/incorrect_source.cpp" "void undefined_symbol(); void symbol() { undefined_symbol(); }") @@ -999,6 +1002,9 @@ else() if(APPLE) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=default -DGTEST_HAS_TR1_TUPLE=0") + if(ARM) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-aligned-allocation") + endif() endif() set(DEBUG_FLAGS "-g3") @@ -1070,8 +1076,14 @@ if(STATIC) set(Boost_USE_STATIC_LIBS ON) set(Boost_USE_STATIC_RUNTIME ON) endif() -find_package(Boost 1.58 QUIET REQUIRED COMPONENTS system filesystem thread date_time chrono regex serialization program_options locale) + +set(BOOST_COMPONENTS system filesystem thread date_time chrono regex serialization program_options) +if (WIN32) + list(APPEND BOOST_COMPONENTS locale) +endif() +find_package(Boost 1.58 QUIET REQUIRED COMPONENTS ${BOOST_COMPONENTS}) add_definitions(-DBOOST_ASIO_ENABLE_SEQUENTIAL_STRAND_ALLOCATION) +add_definitions(-DBOOST_NO_AUTO_PTR) set(CMAKE_FIND_LIBRARY_SUFFIXES ${OLD_LIB_SUFFIXES}) if(NOT Boost_FOUND) @@ -1095,9 +1107,7 @@ include_directories(SYSTEM ${Boost_INCLUDE_DIRS}) if(MINGW) set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wa,-mbig-obj") set(EXTRA_LIBRARIES mswsock;ws2_32;iphlpapi;crypt32;bcrypt) - if(DEPENDS) - set(ICU_LIBRARIES iconv) - else() + if(NOT DEPENDS) set(ICU_LIBRARIES icuio icuin icuuc icudt icutu iconv) endif() elseif(APPLE OR OPENBSD OR ANDROID) @@ -1120,7 +1130,7 @@ list(APPEND EXTRA_LIBRARIES ${CMAKE_DL_LIBS}) if (HIDAPI_FOUND OR LibUSB_COMPILE_TEST_PASSED) if (APPLE) if(DEPENDS) - list(APPEND EXTRA_LIBRARIES "-framework Foundation -framework AppKit -framework IOKit") + list(APPEND EXTRA_LIBRARIES "-framework Foundation -framework AppKit -framework IOKit -framework Security") else() find_library(COREFOUNDATION CoreFoundation) find_library(APPKIT AppKit) @@ -104,7 +104,7 @@ release-all: release-static: mkdir -p $(builddir)/release - cd $(builddir)/release && cmake -D STATIC=ON -D ARCH="x86-64" -D BUILD_64=ON -D CMAKE_BUILD_TYPE=Release $(topdir) && $(MAKE) + cd $(builddir)/release && cmake -D STATIC=ON -D BUILD_64=ON -D CMAKE_BUILD_TYPE=Release $(topdir) && $(MAKE) coverage: mkdir -p $(builddir)/debug @@ -30,7 +30,6 @@ Portions Copyright (c) 2012-2013 The Cryptonote developers. ## Development resources - Web: [getmonero.org](https://getmonero.org) -- Forum: [forum.getmonero.org](https://forum.getmonero.org) - Mail: [dev@getmonero.org](mailto:dev@getmonero.org) - GitHub: [https://github.com/monero-project/monero](https://github.com/monero-project/monero) - IRC: [#monero-dev on Libera](https://web.libera.chat/#monero-dev) @@ -167,7 +166,7 @@ library archives (`.a`). | Dep | Min. version | Vendored | Debian/Ubuntu pkg | Arch pkg | Void pkg | Fedora pkg | Optional | Purpose | | ------------ | ------------- | -------- | -------------------- | ------------ | ------------------ | ------------------- | -------- | --------------- | -| GCC | 5 | NO | `build-essential` | `base-devel` | `base-devel` | `gcc` | NO | | +| GCC | 7 | NO | `build-essential` | `base-devel` | `base-devel` | `gcc` | NO | | | CMake | 3.5 | NO | `cmake` | `cmake` | `cmake` | `cmake` | NO | | | pkg-config | any | NO | `pkg-config` | `base-devel` | `base-devel` | `pkgconf` | NO | | | Boost | 1.58 | NO | `libboost-all-dev` | `boost` | `boost-devel` | `boost-devel` | NO | C++ libraries | @@ -566,7 +565,7 @@ You can also cross-compile static binaries on Linux for Windows and macOS with t * Requires: `python3 g++-mingw-w64-x86-64 wine1.6 bc` * You also need to run: ```update-alternatives --set x86_64-w64-mingw32-g++ x86_64-w64-mingw32-g++-posix && update-alternatives --set x86_64-w64-mingw32-gcc x86_64-w64-mingw32-gcc-posix``` -* ```make depends target=x86_64-apple-darwin11``` for macOS binaries. +* ```make depends target=x86_64-apple-darwin``` for macOS binaries. * Requires: `cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev` * ```make depends target=i686-linux-gnu``` for 32-bit linux binaries. * Requires: `g++-multilib bc` @@ -590,6 +589,16 @@ Using `depends` might also be easier to compile Monero on Windows than using MSY The produced binaries still link libc dynamically. If the binary is compiled on a current distribution, it might not run on an older distribution with an older installation of libc. Passing `-DBACKCOMPAT=ON` to cmake will make sure that the binary will run on systems having at least libc version 2.17. +### Trezor hardware wallet support + +If you have an issue with building Monero with Trezor support, you can disable it by setting `USE_DEVICE_TREZOR=OFF`, e.g., + +```bash +USE_DEVICE_TREZOR=OFF make release +``` + +For more information, please check out Trezor [src/device_trezor/README.md](src/device_trezor/README.md). + ### Gitian builds See [contrib/gitian/README.md](contrib/gitian/README.md). diff --git a/cmake/CheckTrezor.cmake b/cmake/CheckTrezor.cmake index f600cc0bb..05b8ebd93 100644 --- a/cmake/CheckTrezor.cmake +++ b/cmake/CheckTrezor.cmake @@ -1,8 +1,27 @@ -OPTION(USE_DEVICE_TREZOR "Trezor support compilation" ON) -OPTION(USE_DEVICE_TREZOR_LIBUSB "Trezor LibUSB compilation" ON) -OPTION(USE_DEVICE_TREZOR_UDP_RELEASE "Trezor UdpTransport in release mode" OFF) -OPTION(USE_DEVICE_TREZOR_DEBUG "Trezor Debugging enabled" OFF) -OPTION(TREZOR_DEBUG "Main trezor debugging switch" OFF) +# Function for setting default options default values via env vars +function(_trezor_default_val val_name val_default) + if(NOT DEFINED ENV{${val_name}}) + set(ENV{${val_name}} ${val_default}) + endif() +endfunction() + +# Define default options via env vars +_trezor_default_val(USE_DEVICE_TREZOR ON) +_trezor_default_val(USE_DEVICE_TREZOR_MANDATORY OFF) +_trezor_default_val(USE_DEVICE_TREZOR_PROTOBUF_TEST ON) +_trezor_default_val(USE_DEVICE_TREZOR_LIBUSB ON) +_trezor_default_val(USE_DEVICE_TREZOR_UDP_RELEASE OFF) +_trezor_default_val(USE_DEVICE_TREZOR_DEBUG OFF) +_trezor_default_val(TREZOR_DEBUG OFF) + +# Main options +OPTION(USE_DEVICE_TREZOR "Trezor support compilation" $ENV{USE_DEVICE_TREZOR}) +OPTION(USE_DEVICE_TREZOR_MANDATORY "Trezor compilation is mandatory, fail build if Trezor support cannot be compiled" $ENV{USE_DEVICE_TREZOR_MANDATORY}) +OPTION(USE_DEVICE_TREZOR_PROTOBUF_TEST "Trezor Protobuf test" $ENV{USE_DEVICE_TREZOR_PROTOBUF_TEST}) +OPTION(USE_DEVICE_TREZOR_LIBUSB "Trezor LibUSB compilation" $ENV{USE_DEVICE_TREZOR_LIBUSB}) +OPTION(USE_DEVICE_TREZOR_UDP_RELEASE "Trezor UdpTransport in release mode" $ENV{USE_DEVICE_TREZOR_UDP_RELEASE}) +OPTION(USE_DEVICE_TREZOR_DEBUG "Trezor Debugging enabled" $ENV{USE_DEVICE_TREZOR_DEBUG}) +OPTION(TREZOR_DEBUG "Main Trezor debugging switch" $ENV{TREZOR_DEBUG}) # Helper function to fix cmake < 3.6.0 FindProtobuf variables function(_trezor_protobuf_fix_vars) @@ -30,33 +49,62 @@ function(_trezor_protobuf_fix_vars) endif() endfunction() +macro(trezor_fatal_msg msg) + if ($ENV{USE_DEVICE_TREZOR_MANDATORY}) + message(FATAL_ERROR + "${msg}\n" + "==========================================================================\n" + "[ERROR] To compile without Trezor support, set USE_DEVICE_TREZOR=OFF. " + "It is possible both via cmake variable and environment variable, e.g., " + "`USE_DEVICE_TREZOR=OFF make release`\n" + "For more information, please check src/device_trezor/README.md\n" + ) + else() + message(WARNING + "${msg}\n" + "==========================================================================\n" + "[WARNING] Trezor support cannot be compiled! Skipping Trezor compilation. \n" + "For more information, please check src/device_trezor/README.md\n") + set(USE_DEVICE_TREZOR OFF) + return() # finish this cmake file processing (as this is macro). + endif() +endmacro() + # Use Trezor master switch if (USE_DEVICE_TREZOR) # Protobuf is required to build protobuf messages for Trezor include(FindProtobuf OPTIONAL) - find_package(Protobuf) + + FIND_PACKAGE(Protobuf CONFIG) + if (NOT Protobuf_FOUND) + FIND_PACKAGE(Protobuf) + endif() + _trezor_protobuf_fix_vars() - # Protobuf handling the cache variables set in docker. - if(NOT Protobuf_FOUND AND NOT Protobuf_LIBRARY AND NOT Protobuf_PROTOC_EXECUTABLE AND NOT Protobuf_INCLUDE_DIR) - message(STATUS "Could not find Protobuf") + # Early fail for optional Trezor support + if(${Protobuf_VERSION} GREATER 21) + trezor_fatal_msg("Trezor: Unsupported Protobuf version ${Protobuf_VERSION}. Please, use Protobuf v21.") + elseif(NOT Protobuf_FOUND AND NOT Protobuf_LIBRARY AND NOT Protobuf_PROTOC_EXECUTABLE AND NOT Protobuf_INCLUDE_DIR) + trezor_fatal_msg("Trezor: Could not find Protobuf") elseif(NOT Protobuf_LIBRARY OR NOT EXISTS "${Protobuf_LIBRARY}") - message(STATUS "Protobuf library not found: ${Protobuf_LIBRARY}") + trezor_fatal_msg("Trezor: Protobuf library not found: ${Protobuf_LIBRARY}") unset(Protobuf_FOUND) elseif(NOT Protobuf_PROTOC_EXECUTABLE OR NOT EXISTS "${Protobuf_PROTOC_EXECUTABLE}") - message(STATUS "Protobuf executable not found: ${Protobuf_PROTOC_EXECUTABLE}") + trezor_fatal_msg("Trezor: Protobuf executable not found: ${Protobuf_PROTOC_EXECUTABLE}") unset(Protobuf_FOUND) elseif(NOT Protobuf_INCLUDE_DIR OR NOT EXISTS "${Protobuf_INCLUDE_DIR}") - message(STATUS "Protobuf include dir not found: ${Protobuf_INCLUDE_DIR}") + trezor_fatal_msg("Trezor: Protobuf include dir not found: ${Protobuf_INCLUDE_DIR}") unset(Protobuf_FOUND) else() - message(STATUS "Protobuf lib: ${Protobuf_LIBRARY}, inc: ${Protobuf_INCLUDE_DIR}, protoc: ${Protobuf_PROTOC_EXECUTABLE}") + message(STATUS "Trezor: Protobuf lib: ${Protobuf_LIBRARY}, inc: ${Protobuf_INCLUDE_DIR}, protoc: ${Protobuf_PROTOC_EXECUTABLE}") set(Protobuf_INCLUDE_DIRS ${Protobuf_INCLUDE_DIR}) - set(Protobuf_FOUND 1) # override found if all rquired info was provided by variables + set(Protobuf_FOUND 1) # override found if all required info was provided by variables endif() if(TREZOR_DEBUG) set(USE_DEVICE_TREZOR_DEBUG 1) + message(STATUS "Trezor: debug build enabled") endif() # Compile debugging support (for tests) @@ -64,7 +112,7 @@ if (USE_DEVICE_TREZOR) add_definitions(-DWITH_TREZOR_DEBUGGING=1) endif() else() - message(STATUS "Trezor support disabled by USE_DEVICE_TREZOR") + message(STATUS "Trezor: support disabled by USE_DEVICE_TREZOR") endif() if(Protobuf_FOUND AND USE_DEVICE_TREZOR) @@ -85,7 +133,7 @@ if(Protobuf_FOUND AND USE_DEVICE_TREZOR) endif() if(NOT TREZOR_PYTHON) - message(STATUS "Trezor: Python not found") + trezor_fatal_msg("Trezor: Python not found") endif() endif() @@ -93,27 +141,42 @@ endif() if(Protobuf_FOUND AND USE_DEVICE_TREZOR AND TREZOR_PYTHON) execute_process(COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I "${CMAKE_CURRENT_LIST_DIR}" -I "${Protobuf_INCLUDE_DIR}" "${CMAKE_CURRENT_LIST_DIR}/test-protobuf.proto" --cpp_out ${CMAKE_BINARY_DIR} RESULT_VARIABLE RET OUTPUT_VARIABLE OUT ERROR_VARIABLE ERR) if(RET) - message(STATUS "Protobuf test generation failed: ${OUT} ${ERR}") - endif() - - try_compile(Protobuf_COMPILE_TEST_PASSED - "${CMAKE_BINARY_DIR}" - SOURCES - "${CMAKE_BINARY_DIR}/test-protobuf.pb.cc" - "${CMAKE_CURRENT_LIST_DIR}/test-protobuf.cpp" - CMAKE_FLAGS - "-DINCLUDE_DIRECTORIES=${Protobuf_INCLUDE_DIR};${CMAKE_BINARY_DIR}" - "-DCMAKE_CXX_STANDARD=11" - LINK_LIBRARIES ${Protobuf_LIBRARY} - OUTPUT_VARIABLE OUTPUT - ) - if(NOT Protobuf_COMPILE_TEST_PASSED) - message(STATUS "Protobuf Compilation test failed: ${OUTPUT}.") + trezor_fatal_msg("Trezor: Protobuf test generation failed: ${OUT} ${ERR}") + endif() + + if(ANDROID) + set(CMAKE_TRY_COMPILE_LINKER_FLAGS "${CMAKE_TRY_COMPILE_LINKER_FLAGS} -llog") + set(CMAKE_TRY_COMPILE_LINK_LIBRARIES "${CMAKE_TRY_COMPILE_LINK_LIBRARIES} log") + endif() + + if(USE_DEVICE_TREZOR_PROTOBUF_TEST) + # For now, Protobuf v21 is the maximum supported version as v23 requires C++17. TODO: Remove once we move to C++17 + if(${Protobuf_VERSION} GREATER 21) + trezor_fatal_msg("Trezor: Unsupported Protobuf version ${Protobuf_VERSION}. Please, use Protobuf v21.") + endif() + + try_compile(Protobuf_COMPILE_TEST_PASSED + "${CMAKE_BINARY_DIR}" + SOURCES + "${CMAKE_BINARY_DIR}/test-protobuf.pb.cc" + "${CMAKE_CURRENT_LIST_DIR}/test-protobuf.cpp" + CMAKE_FLAGS + CMAKE_EXE_LINKER_FLAGS ${CMAKE_TRY_COMPILE_LINKER_FLAGS} + "-DINCLUDE_DIRECTORIES=${Protobuf_INCLUDE_DIR};${CMAKE_BINARY_DIR}" + "-DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}" + LINK_LIBRARIES ${Protobuf_LIBRARY} ${CMAKE_TRY_COMPILE_LINK_LIBRARIES} + OUTPUT_VARIABLE OUTPUT + ) + if(NOT Protobuf_COMPILE_TEST_PASSED) + trezor_fatal_msg("Trezor: Protobuf Compilation test failed: ${OUTPUT}.") + endif() + else () + message(STATUS "Trezor: Protobuf Compilation test skipped, build may fail later") endif() endif() # Try to build protobuf messages -if(Protobuf_FOUND AND USE_DEVICE_TREZOR AND TREZOR_PYTHON AND Protobuf_COMPILE_TEST_PASSED) +if(Protobuf_FOUND AND USE_DEVICE_TREZOR AND TREZOR_PYTHON) set(ENV{PROTOBUF_INCLUDE_DIRS} "${Protobuf_INCLUDE_DIR}") set(ENV{PROTOBUF_PROTOC_EXECUTABLE} "${Protobuf_PROTOC_EXECUTABLE}") set(TREZOR_PROTOBUF_PARAMS "") @@ -123,59 +186,62 @@ if(Protobuf_FOUND AND USE_DEVICE_TREZOR AND TREZOR_PYTHON AND Protobuf_COMPILE_T execute_process(COMMAND ${TREZOR_PYTHON} tools/build_protob.py ${TREZOR_PROTOBUF_PARAMS} WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/../src/device_trezor/trezor RESULT_VARIABLE RET OUTPUT_VARIABLE OUT ERROR_VARIABLE ERR) if(RET) - message(WARNING "Trezor protobuf messages could not be regenerated (err=${RET}, python ${PYTHON})." + trezor_fatal_msg("Trezor: protobuf messages could not be regenerated (err=${RET}, python ${PYTHON})." "OUT: ${OUT}, ERR: ${ERR}." "Please read src/device_trezor/trezor/tools/README.md") - else() - message(STATUS "Trezor protobuf messages regenerated out: \"${OUT}.\"") - set(DEVICE_TREZOR_READY 1) - add_definitions(-DDEVICE_TREZOR_READY=1) - add_definitions(-DPROTOBUF_INLINE_NOT_IN_HEADERS=0) + endif() - if(CMAKE_BUILD_TYPE STREQUAL "Debug") - add_definitions(-DTREZOR_DEBUG=1) - endif() + message(STATUS "Trezor: protobuf messages regenerated out: \"${OUT}.\"") + set(DEVICE_TREZOR_READY 1) + add_definitions(-DDEVICE_TREZOR_READY=1) + add_definitions(-DPROTOBUF_INLINE_NOT_IN_HEADERS=0) - if(USE_DEVICE_TREZOR_UDP_RELEASE) - add_definitions(-DUSE_DEVICE_TREZOR_UDP_RELEASE=1) - endif() + if(CMAKE_BUILD_TYPE STREQUAL "Debug") + add_definitions(-DTREZOR_DEBUG=1) + endif() - if (Protobuf_INCLUDE_DIR) - include_directories(${Protobuf_INCLUDE_DIR}) - endif() + if(USE_DEVICE_TREZOR_UDP_RELEASE) + message(STATUS "Trezor: UDP transport enabled (emulator)") + add_definitions(-DUSE_DEVICE_TREZOR_UDP_RELEASE=1) + endif() - # LibUSB support, check for particular version - # Include support only if compilation test passes - if (USE_DEVICE_TREZOR_LIBUSB) - find_package(LibUSB) - endif() + if (Protobuf_INCLUDE_DIR) + include_directories(${Protobuf_INCLUDE_DIR}) + endif() - if (LibUSB_COMPILE_TEST_PASSED) - add_definitions(-DHAVE_TREZOR_LIBUSB=1) - if(LibUSB_INCLUDE_DIRS) - include_directories(${LibUSB_INCLUDE_DIRS}) - endif() - endif() + # LibUSB support, check for particular version + # Include support only if compilation test passes + if (USE_DEVICE_TREZOR_LIBUSB) + find_package(LibUSB) + endif() - set(TREZOR_LIBUSB_LIBRARIES "") - if(LibUSB_COMPILE_TEST_PASSED) - list(APPEND TREZOR_LIBUSB_LIBRARIES ${LibUSB_LIBRARIES} ${LIBUSB_DEP_LINKER}) - message(STATUS "Trezor compatible LibUSB found at: ${LibUSB_INCLUDE_DIRS}") + if (LibUSB_COMPILE_TEST_PASSED) + add_definitions(-DHAVE_TREZOR_LIBUSB=1) + if(LibUSB_INCLUDE_DIRS) + include_directories(${LibUSB_INCLUDE_DIRS}) endif() + endif() + + set(TREZOR_LIBUSB_LIBRARIES "") + if(LibUSB_COMPILE_TEST_PASSED) + list(APPEND TREZOR_LIBUSB_LIBRARIES ${LibUSB_LIBRARIES} ${LIBUSB_DEP_LINKER}) + message(STATUS "Trezor: compatible LibUSB found at: ${LibUSB_INCLUDE_DIRS}") + elseif(USE_DEVICE_TREZOR_LIBUSB AND NOT ANDROID) + trezor_fatal_msg("Trezor: LibUSB not found or test failed, please install libusb-1.0.26") + endif() - if (BUILD_GUI_DEPS) - set(TREZOR_DEP_LIBS "") - set(TREZOR_DEP_LINKER "") + if (BUILD_GUI_DEPS) + set(TREZOR_DEP_LIBS "") + set(TREZOR_DEP_LINKER "") - if (Protobuf_LIBRARY) - list(APPEND TREZOR_DEP_LIBS ${Protobuf_LIBRARY}) - string(APPEND TREZOR_DEP_LINKER " -lprotobuf") - endif() + if (Protobuf_LIBRARY) + list(APPEND TREZOR_DEP_LIBS ${Protobuf_LIBRARY}) + string(APPEND TREZOR_DEP_LINKER " -lprotobuf") + endif() - if (TREZOR_LIBUSB_LIBRARIES) - list(APPEND TREZOR_DEP_LIBS ${TREZOR_LIBUSB_LIBRARIES}) - string(APPEND TREZOR_DEP_LINKER " -lusb-1.0 ${LIBUSB_DEP_LINKER}") - endif() + if (TREZOR_LIBUSB_LIBRARIES) + list(APPEND TREZOR_DEP_LIBS ${TREZOR_LIBUSB_LIBRARIES}) + string(APPEND TREZOR_DEP_LINKER " -lusb-1.0 ${LIBUSB_DEP_LINKER}") endif() endif() endif() diff --git a/cmake/FindLibUSB.cmake b/cmake/FindLibUSB.cmake index f780628f8..f701b6398 100644 --- a/cmake/FindLibUSB.cmake +++ b/cmake/FindLibUSB.cmake @@ -95,11 +95,35 @@ if ( LibUSB_FOUND ) endif ( LibUSB_FOUND ) if ( LibUSB_FOUND ) + if (APPLE) + if(DEPENDS) + list(APPEND TEST_COMPILE_EXTRA_LIBRARIES "-framework Foundation -framework IOKit -framework Security") + else() + find_library(COREFOUNDATION CoreFoundation) + find_library(IOKIT IOKit) + find_library(SECURITY_FRAMEWORK Security) + list(APPEND TEST_COMPILE_EXTRA_LIBRARIES ${IOKIT}) + list(APPEND TEST_COMPILE_EXTRA_LIBRARIES ${COREFOUNDATION}) + list(APPEND TEST_COMPILE_EXTRA_LIBRARIES ${SECURITY_FRAMEWORK}) + + if(STATIC) + find_library(OBJC objc.a) + set(LIBUSB_DEP_LINKER ${OBJC}) + list(APPEND TEST_COMPILE_EXTRA_LIBRARIES ${LIBUSB_DEP_LINKER}) + endif() + endif() + endif() + if (WIN32) + list(APPEND TEST_COMPILE_EXTRA_LIBRARIES setupapi) + endif() + list(APPEND TEST_COMPILE_EXTRA_LIBRARIES ${LibUSB_LIBRARIES}) + set(CMAKE_REQUIRED_LIBRARIES ${TEST_COMPILE_EXTRA_LIBRARIES}) + check_library_exists ( "${LibUSB_LIBRARIES}" usb_open "" LibUSB_FOUND ) check_library_exists ( "${LibUSB_LIBRARIES}" libusb_get_device_list "" LibUSB_VERSION_1.0 ) check_library_exists ( "${LibUSB_LIBRARIES}" libusb_get_port_numbers "" LibUSB_VERSION_1.0.16 ) - if((STATIC AND UNIX AND NOT APPLE) OR (DEPENDS AND CMAKE_SYSTEM_NAME STREQUAL "Linux") OR ANDROID) + if((STATIC AND UNIX AND NOT APPLE AND NOT FREEBSD) OR (DEPENDS AND CMAKE_SYSTEM_NAME STREQUAL "Linux") OR ANDROID) find_library(LIBUDEV_LIBRARY udev) if(LIBUDEV_LIBRARY) set(LibUSB_LIBRARIES "${LibUSB_LIBRARIES};${LIBUDEV_LIBRARY}") @@ -111,27 +135,6 @@ if ( LibUSB_FOUND ) # Library 1.0.16+ compilation test. # The check_library_exists does not work well on Apple with shared libs. if (APPLE OR LibUSB_VERSION_1.0.16 OR STATIC) - if (APPLE) - if(DEPENDS) - list(APPEND TEST_COMPILE_EXTRA_LIBRARIES "-framework Foundation -framework IOKit -framework Security") - else() - find_library(COREFOUNDATION CoreFoundation) - find_library(IOKIT IOKit) - list(APPEND TEST_COMPILE_EXTRA_LIBRARIES ${IOKIT}) - list(APPEND TEST_COMPILE_EXTRA_LIBRARIES ${COREFOUNDATION}) - - if(STATIC) - find_library(OBJC objc.a) - set(LIBUSB_DEP_LINKER ${OBJC}) - list(APPEND TEST_COMPILE_EXTRA_LIBRARIES ${LIBUSB_DEP_LINKER}) - endif() - endif() - endif() - if (WIN32) - list(APPEND TEST_COMPILE_EXTRA_LIBRARIES setupapi) - endif() - list(APPEND TEST_COMPILE_EXTRA_LIBRARIES ${LibUSB_LIBRARIES}) - try_compile(LibUSB_COMPILE_TEST_PASSED ${CMAKE_BINARY_DIR} "${CMAKE_CURRENT_LIST_DIR}/test-libusb-version.c" diff --git a/contrib/brew/Brewfile b/contrib/brew/Brewfile index c74e7b2a2..c2fae0ffc 100644 --- a/contrib/brew/Brewfile +++ b/contrib/brew/Brewfile @@ -31,4 +31,5 @@ brew "doxygen" brew "graphviz" brew "libunwind-headers" brew "xz" -brew "protobuf" +brew "protobuf@21", link: true +brew "libusb" diff --git a/contrib/depends/Makefile b/contrib/depends/Makefile index 4b60ee054..ce74de184 100644 --- a/contrib/depends/Makefile +++ b/contrib/depends/Makefile @@ -68,21 +68,6 @@ $(host_arch)_$(host_os)_prefix=$(BASEDIR)/$(host) $(host_arch)_$(host_os)_host=$(host) host_prefix=$($(host_arch)_$(host_os)_prefix) build_prefix=$(host_prefix)/native -ifeq ($(host_os),mingw32) -host_cmake=Windows -endif -ifeq ($(host_os),linux) -host_cmake=Linux -endif -ifeq ($(host_os),freebsd) -host_cmake=FreeBSD -endif -ifeq ($(host_os),darwin) -host_cmake=Darwin -endif -ifeq ($(host_os),android) -host_cmake=Android -endif AT_$(V):= AT_:=@ @@ -151,7 +136,7 @@ $(host_prefix)/share/toolchain.cmake : toolchain.cmake.in $(host_prefix)/.stamp_ -e 's|@debug@|$(DEBUG)|' \ -e 's|@release_type@|$(release_type)|' \ -e 's|@build_tests@|$(build_tests)|' \ - -e 's|@depends@|$(host_cmake)|' \ + -e 's|@cmake_system_name@|$($(host_os)_cmake_system)|' \ -e 's|@prefix@|$($(host_arch)_$(host_os)_prefix)|'\ -e 's|@arch@|$(host_arch)|'\ $< > $@ @@ -185,7 +170,7 @@ install: check-packages $(host_prefix)/share/toolchain.cmake download-one: check-sources $(all_sources) download-osx: - @$(MAKE) -s HOST=x86_64-apple-darwin11 download-one + @$(MAKE) -s HOST=x86_64-apple-darwin download-one download-linux: @$(MAKE) -s HOST=x86_64-unknown-linux-gnu download-one download-win: diff --git a/contrib/depends/README.md b/contrib/depends/README.md index 2f1d8424a..d19baaf8a 100644 --- a/contrib/depends/README.md +++ b/contrib/depends/README.md @@ -32,7 +32,7 @@ Common `host-platform-triplets` for cross compilation are: - `i686-w64-mingw32` for Win32 - `x86_64-w64-mingw32` for Win64 -- `x86_64-apple-darwin11` for MacOSX x86_64 +- `x86_64-apple-darwin` for MacOSX x86_64 - `arm-linux-gnueabihf` for Linux ARM 32 bit - `aarch64-linux-gnu` for Linux ARM 64 bit - `riscv64-linux-gnu` for Linux RISCV 64 bit diff --git a/contrib/depends/hosts/android.mk b/contrib/depends/hosts/android.mk index d6f8b99dd..b08126e86 100644 --- a/contrib/depends/hosts/android.mk +++ b/contrib/depends/hosts/android.mk @@ -20,3 +20,4 @@ android_debug_CXXFLAGS=$(android_debug_CFLAGS) android_native_toolchain=android_ndk +android_cmake_system=Android diff --git a/contrib/depends/hosts/darwin.mk b/contrib/depends/hosts/darwin.mk index 79d449054..9a96e2601 100644 --- a/contrib/depends/hosts/darwin.mk +++ b/contrib/depends/hosts/darwin.mk @@ -1,12 +1,12 @@ -OSX_MIN_VERSION=10.8 +OSX_MIN_VERSION=10.13 LD64_VERSION=609 ifeq (aarch64, $(host_arch)) CC_target=arm64-apple-$(host_os) else CC_target=$(host) endif -darwin_CC=clang -target $(CC_target) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(host_prefix)/native/SDK/ -mlinker-version=$(LD64_VERSION) -B$(host_prefix)/native/bin/$(host)- -darwin_CXX=clang++ -target $(CC_target) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(host_prefix)/native/SDK/ -mlinker-version=$(LD64_VERSION) -stdlib=libc++ -B$(host_prefix)/native/bin/$(host)- +darwin_CC=clang -target $(CC_target) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(host_prefix)/native/SDK/ -iwithsysroot/usr/include -iframeworkwithsysroot/System/Library/Frameworks -mlinker-version=$(LD64_VERSION) -B$(host_prefix)/native/bin/$(host)- +darwin_CXX=clang++ -target $(CC_target) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(host_prefix)/native/SDK/ -iwithsysroot/usr/include/c++/v1 -iwithsysroot/usr/include -iframeworkwithsysroot/System/Library/Frameworks -mlinker-version=$(LD64_VERSION) -stdlib=libc++ -B$(host_prefix)/native/bin/$(host)- darwin_CFLAGS=-pipe darwin_CXXFLAGS=$(darwin_CFLAGS) @@ -19,3 +19,5 @@ darwin_debug_CFLAGS=-O1 darwin_debug_CXXFLAGS=$(darwin_debug_CFLAGS) darwin_native_toolchain=native_cctools darwin_sdk + +darwin_cmake_system=Darwin diff --git a/contrib/depends/hosts/freebsd.mk b/contrib/depends/hosts/freebsd.mk index 2e3b5933e..d3c6c6149 100644 --- a/contrib/depends/hosts/freebsd.mk +++ b/contrib/depends/hosts/freebsd.mk @@ -16,3 +16,4 @@ freebsd_debug_CXXFLAGS=$(freebsd_debug_CFLAGS) freebsd_native_toolchain=freebsd_base +freebsd_cmake_system=FreeBSD diff --git a/contrib/depends/hosts/linux.mk b/contrib/depends/hosts/linux.mk index 912fdb03c..0f9379f4a 100644 --- a/contrib/depends/hosts/linux.mk +++ b/contrib/depends/hosts/linux.mk @@ -30,3 +30,5 @@ i686_linux_CXX=$(default_host_CXX) -m32 x86_64_linux_CC=$(default_host_CC) -m64 x86_64_linux_CXX=$(default_host_CXX) -m64 endif + +linux_cmake_system=Linux diff --git a/contrib/depends/hosts/mingw32.mk b/contrib/depends/hosts/mingw32.mk index ccc4c5082..58a9a929d 100644 --- a/contrib/depends/hosts/mingw32.mk +++ b/contrib/depends/hosts/mingw32.mk @@ -9,3 +9,5 @@ mingw32_debug_CFLAGS=-O1 mingw32_debug_CXXFLAGS=$(mingw32_debug_CFLAGS) mingw32_debug_CPPFLAGS=-D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC + +mingw32_cmake_system=Windows diff --git a/contrib/depends/packages/boost.mk b/contrib/depends/packages/boost.mk index fd06c5393..d81bca973 100644 --- a/contrib/depends/packages/boost.mk +++ b/contrib/depends/packages/boost.mk @@ -3,7 +3,6 @@ $(package)_version=1_64_0 $(package)_download_path=https://downloads.sourceforge.net/project/boost/boost/1.64.0/ $(package)_file_name=$(package)_$($(package)_version).tar.bz2 $(package)_sha256_hash=7bcc5caace97baa948931d712ea5f37038dbb1c5d89b43ad4def4ed7cb683332 -$(package)_dependencies=libiconv $(package)_patches=fix_aroptions.patch fix_arm_arch.patch define $(package)_set_vars @@ -22,7 +21,8 @@ $(package)_toolset_$(host_os)=gcc $(package)_archiver_$(host_os)=$($(package)_ar) $(package)_toolset_darwin=darwin $(package)_archiver_darwin=$($(package)_libtool) -$(package)_config_libraries=chrono,filesystem,program_options,system,thread,test,date_time,regex,serialization,locale +$(package)_config_libraries_$(host_os)="chrono,filesystem,program_options,system,thread,test,date_time,regex,serialization" +$(package)_config_libraries_mingw32="chrono,filesystem,program_options,system,thread,test,date_time,regex,serialization,locale" $(package)_cxxflags=-std=c++11 $(package)_cxxflags_linux=-fPIC $(package)_cxxflags_freebsd=-fPIC @@ -35,7 +35,7 @@ define $(package)_preprocess_cmds endef define $(package)_config_cmds - ./bootstrap.sh --without-icu --with-libraries=$(boost_config_libraries) + ./bootstrap.sh --without-icu --with-libraries=$(boost_config_libraries_$(host_os)) endef define $(package)_build_cmds diff --git a/contrib/depends/packages/darwin_sdk.mk b/contrib/depends/packages/darwin_sdk.mk index d639c422e..3355dcf3a 100644 --- a/contrib/depends/packages/darwin_sdk.mk +++ b/contrib/depends/packages/darwin_sdk.mk @@ -1,8 +1,8 @@ package=darwin_sdk -$(package)_version=11.1 -$(package)_download_path=https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/ -$(package)_file_name=MacOSX$($(package)_version).sdk.tar.xz -$(package)_sha256_hash=68797baaacb52f56f713400de306a58a7ca00b05c3dc6d58f0a8283bcac721f8 +$(package)_version=12.2 +$(package)_download_path=https://bitcoincore.org/depends-sources/sdks +$(package)_file_name=Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz +$(package)_sha256_hash=df75d30ecafc429e905134333aeae56ac65fac67cb4182622398fd717df77619 define $(package)_stage_cmds mkdir -p $($(package)_staging_dir)/$(host_prefix)/native/SDK &&\ diff --git a/contrib/depends/packages/libiconv.mk b/contrib/depends/packages/libiconv.mk deleted file mode 100644 index eac8b4331..000000000 --- a/contrib/depends/packages/libiconv.mk +++ /dev/null @@ -1,35 +0,0 @@ -package=libiconv -$(package)_version=1.15 -$(package)_download_path=https://ftp.gnu.org/gnu/libiconv -$(package)_file_name=libiconv-$($(package)_version).tar.gz -$(package)_sha256_hash=ccf536620a45458d26ba83887a983b96827001e92a13847b45e4925cc8913178 -$(package)_patches=fix-whitespace.patch - -define $(package)_set_vars - $(package)_config_opts=--disable-nls - $(package)_config_opts=--enable-static - $(package)_config_opts=--disable-shared - $(package)_config_opts_linux=--with-pic - $(package)_config_opts_freebsd=--with-pic -endef - -define $(package)_preprocess_cmds - cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub build-aux/ &&\ - patch -p1 < $($(package)_patch_dir)/fix-whitespace.patch -endef - -define $(package)_config_cmds - $($(package)_autoconf) AR_FLAGS=$($(package)_arflags) -endef - -define $(package)_build_cmds - $(MAKE) -endef - -define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) install -endef - -define $(package)_postprocess_cmds - rm lib/*.la -endef diff --git a/contrib/depends/packages/libusb.mk b/contrib/depends/packages/libusb.mk index c1d9fe6a9..fb6646685 100644 --- a/contrib/depends/packages/libusb.mk +++ b/contrib/depends/packages/libusb.mk @@ -3,8 +3,10 @@ $(package)_version=1.0.26 $(package)_download_path=https://github.com/libusb/libusb/releases/download/v$($(package)_version) $(package)_file_name=$(package)-$($(package)_version).tar.bz2 $(package)_sha256_hash=12ce7a61fc9854d1d2a1ffe095f7b5fac19ddba095c259e6067a46500381b5a5 +$(package)_patches=fix_osx_avail.patch define $(package)_preprocess_cmds + patch -p1 < $($(package)_patch_dir)/fix_osx_avail.patch &&\ autoreconf -i endef @@ -13,6 +15,7 @@ define $(package)_set_vars $(package)_config_opts_linux=--with-pic --disable-udev $(package)_config_opts_mingw32=--disable-udev $(package)_config_opts_darwin=--disable-udev + $(package)_config_opts_freebsd=--with-pic --disable-udev endef ifneq ($(host_os),darwin) diff --git a/contrib/depends/packages/native_clang.mk b/contrib/depends/packages/native_clang.mk index 115f8f389..4ff21ada0 100644 --- a/contrib/depends/packages/native_clang.mk +++ b/contrib/depends/packages/native_clang.mk @@ -24,6 +24,5 @@ define $(package)_stage_cmds cp lib/libLTO.so $($(package)_staging_prefix_dir)/lib/ && \ cp -rf lib/clang/$($(package)_version)/include/* $($(package)_staging_prefix_dir)/lib/clang/$($(package)_version)/include/ && \ cp bin/dsymutil $($(package)_staging_prefix_dir)/bin/$(host)-dsymutil && \ - if `test -d include/c++/`; then cp -rf include/c++/ $($(package)_staging_prefix_dir)/include/; fi && \ if `test -d lib/c++/`; then cp -rf lib/c++/ $($(package)_staging_prefix_dir)/lib/; fi endef diff --git a/contrib/depends/packages/native_protobuf.mk b/contrib/depends/packages/native_protobuf.mk index 35f648b9a..2dc11b23c 100644 --- a/contrib/depends/packages/native_protobuf.mk +++ b/contrib/depends/packages/native_protobuf.mk @@ -1,8 +1,9 @@ package=protobuf3 -$(package)_version=3.6.1 +$(package)_version=21.12 +$(package)_version_protobuf_cpp=3.21.12 $(package)_download_path=https://github.com/protocolbuffers/protobuf/releases/download/v$($(package)_version)/ -$(package)_file_name=protobuf-cpp-$($(package)_version).tar.gz -$(package)_sha256_hash=b3732e471a9bb7950f090fd0457ebd2536a9ba0891b7f3785919c654fe2a2529 +$(package)_file_name=protobuf-cpp-$($(package)_version_protobuf_cpp).tar.gz +$(package)_sha256_hash=4eab9b524aa5913c6fffb20b2a8abf5ef7f95a80bc0701f3a6dbb4c607f73460 $(package)_cxxflags=-std=c++11 define $(package)_set_vars diff --git a/contrib/depends/packages/openssl.mk b/contrib/depends/packages/openssl.mk index bdfb031ed..b35cf5e97 100644 --- a/contrib/depends/packages/openssl.mk +++ b/contrib/depends/packages/openssl.mk @@ -1,8 +1,8 @@ package=openssl -$(package)_version=3.0.9 +$(package)_version=3.0.11 $(package)_download_path=https://www.openssl.org/source $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=eb1ab04781474360f77c318ab89d8c5a03abc38e63d65a603cabbf1b00a1dc90 +$(package)_sha256_hash=b3425d3bb4a2218d0697eb41f7fc0cdede016ed19ca49d168b78e8d947887f55 define $(package)_set_vars $(package)_config_env=AR="$($(package)_ar)" ARFLAGS=$($(package)_arflags) RANLIB="$($(package)_ranlib)" CC="$($(package)_cc)" diff --git a/contrib/depends/packages/packages.mk b/contrib/depends/packages/packages.mk index ea4a0effd..6bc5a6fa1 100644 --- a/contrib/depends/packages/packages.mk +++ b/contrib/depends/packages/packages.mk @@ -1,4 +1,4 @@ -packages:=boost openssl zeromq libiconv expat unbound +packages:=boost openssl zeromq expat unbound # ccache is useless in gitian builds ifneq ($(GITIAN),1) @@ -8,15 +8,15 @@ endif hardware_packages := hidapi protobuf libusb hardware_native_packages := native_protobuf -android_native_packages = android_ndk -android_packages = ncurses readline sodium +android_native_packages = android_ndk $(hardware_native_packages) +android_packages = ncurses readline sodium protobuf darwin_native_packages = $(hardware_native_packages) darwin_packages = ncurses readline sodium $(hardware_packages) # not really native... -freebsd_native_packages = freebsd_base -freebsd_packages = ncurses readline sodium +freebsd_native_packages = freebsd_base $(hardware_native_packages) +freebsd_packages = ncurses readline sodium protobuf libusb linux_packages = eudev ncurses readline sodium $(hardware_packages) linux_native_packages = $(hardware_native_packages) @@ -25,10 +25,6 @@ ifeq ($(build_tests),ON) packages += gtest endif -ifneq ($(host_arch),riscv64) -linux_packages += unwind -endif - mingw32_packages = sodium $(hardware_packages) mingw32_native_packages = $(hardware_native_packages) diff --git a/contrib/depends/packages/protobuf.mk b/contrib/depends/packages/protobuf.mk index ddec1eb59..780357c90 100644 --- a/contrib/depends/packages/protobuf.mk +++ b/contrib/depends/packages/protobuf.mk @@ -1,21 +1,17 @@ package=protobuf $(package)_version=$(native_$(package)_version) +$(package)_version_protobuf_cpp=$(native_$(package)_version_protobuf_cpp) $(package)_download_path=$(native_$(package)_download_path) $(package)_file_name=$(native_$(package)_file_name) $(package)_sha256_hash=$(native_$(package)_sha256_hash) $(package)_dependencies=native_$(package) $(package)_cxxflags=-std=c++11 -$(package)_patches=visibility.patch define $(package)_set_vars $(package)_config_opts=--disable-shared --with-protoc=$(build_prefix)/bin/protoc $(package)_config_opts_linux=--with-pic endef -define $(package)_preprocess_cmds - patch -p0 < $($(package)_patch_dir)/visibility.patch -endef - define $(package)_config_cmds $($(package)_autoconf) AR_FLAGS=$($(package)_arflags) endef diff --git a/contrib/depends/packages/zeromq.mk b/contrib/depends/packages/zeromq.mk index 20f2d4bd9..669d06d83 100644 --- a/contrib/depends/packages/zeromq.mk +++ b/contrib/depends/packages/zeromq.mk @@ -1,9 +1,8 @@ package=zeromq -$(package)_version=4.3.4 +$(package)_version=4.3.5 $(package)_download_path=https://github.com/zeromq/libzmq/releases/download/v$($(package)_version)/ $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=c593001a89f5a85dd2ddf564805deb860e02471171b3f204944857336295c3e5 -$(package)_patches=06aba27b04c5822cb88a69677382a0f053367143.patch +$(package)_sha256_hash=6653ef5910f17954861fe72332e68b03ca6e4d9c7160eb3a8de5a5a913bfab43 define $(package)_set_vars $(package)_config_opts=--without-documentation --disable-shared --without-libsodium --disable-curve @@ -12,10 +11,6 @@ define $(package)_set_vars $(package)_cxxflags=-std=c++11 endef -define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/06aba27b04c5822cb88a69677382a0f053367143.patch -endef - define $(package)_config_cmds $($(package)_autoconf) AR_FLAGS=$($(package)_arflags) endef diff --git a/contrib/depends/patches/libiconv/fix-whitespace.patch b/contrib/depends/patches/libiconv/fix-whitespace.patch deleted file mode 100644 index 531364b45..000000000 --- a/contrib/depends/patches/libiconv/fix-whitespace.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/preload/configure b/preload/configure -index aab5c77..e20b8f0 100755 ---- a/preload/configure -+++ b/preload/configure -@@ -588,7 +588,7 @@ MAKEFLAGS= - PACKAGE_NAME='libiconv' - PACKAGE_TARNAME='libiconv' - PACKAGE_VERSION='0' --PACKAGE_STRING='libiconv 0' -+PACKAGE_STRING='libiconv0' - PACKAGE_BUGREPORT='' - PACKAGE_URL='' - diff --git a/contrib/depends/patches/libusb/fix_osx_avail.patch b/contrib/depends/patches/libusb/fix_osx_avail.patch new file mode 100644 index 000000000..2474993f5 --- /dev/null +++ b/contrib/depends/patches/libusb/fix_osx_avail.patch @@ -0,0 +1,12 @@ +--- a/libusb/os/darwin_usb.h 2023-03-19 12:07:53 ++++ b/libusb/os/darwin_usb.h 2023-03-19 12:07:47 +@@ -165,7 +165,8 @@ + #define __has_builtin(x) 0 // Compatibility with non-clang compilers. + #endif + #if __has_builtin(__builtin_available) +- #define HAS_CAPTURE_DEVICE() __builtin_available(macOS 10.10, *) ++// #define HAS_CAPTURE_DEVICE() __builtin_available(macOS 10.10, *) ++ #define HAS_CAPTURE_DEVICE() 0 + #else + #define HAS_CAPTURE_DEVICE() 0 + #endif diff --git a/contrib/depends/patches/protobuf/visibility.patch b/contrib/depends/patches/protobuf/visibility.patch deleted file mode 100644 index e66d5961f..000000000 --- a/contrib/depends/patches/protobuf/visibility.patch +++ /dev/null @@ -1,159 +0,0 @@ ---- src/google/protobuf/descriptor.cc.O 2018-07-30 22:16:10.000000000 +0000 -+++ src/google/protobuf/descriptor.cc 2022-05-06 13:38:14.827309092 +0000 -@@ -32,6 +32,9 @@ - // Based on original Protocol Buffers design by - // Sanjay Ghemawat, Jeff Dean, and others. - -+#if defined(__APPLE__) && defined(__arm64__) -+#pragma GCC visibility push(hidden) -+#endif - #include <algorithm> - #include <functional> - #include <google/protobuf/stubs/hash.h> -@@ -7274,3 +7277,6 @@ - - } // namespace protobuf - } // namespace google -+#if defined(__APPLE__) && defined(__arm64__) -+#pragma GCC visibility pop -+#endif ---- src/google/protobuf/extension_set.cc.O 2018-07-23 20:56:42.000000000 +0000 -+++ src/google/protobuf/extension_set.cc 2022-05-06 14:48:55.369877050 +0000 -@@ -32,6 +32,9 @@ - // Based on original Protocol Buffers design by - // Sanjay Ghemawat, Jeff Dean, and others. - -+#if defined(__APPLE__) && defined(__arm64__) -+#pragma GCC visibility push(hidden) -+#endif - #include <google/protobuf/stubs/hash.h> - #include <tuple> - #include <utility> -@@ -1914,3 +1917,6 @@ - } // namespace internal - } // namespace protobuf - } // namespace google -+#if defined(__APPLE__) && defined(__arm64__) -+#pragma GCC visibility pop -+#endif ---- src/google/protobuf/extension_set_heavy.cc.O 2018-07-30 22:16:10.000000000 +0000 -+++ src/google/protobuf/extension_set_heavy.cc 2022-05-06 14:14:27.847320946 +0000 -@@ -35,6 +35,10 @@ - // Contains methods defined in extension_set.h which cannot be part of the - // lite library because they use descriptors or reflection. - -+#if defined(__APPLE__) && defined(__arm64__) -+#pragma GCC visibility push(hidden) -+#endif -+ - #include <google/protobuf/stubs/casts.h> - #include <google/protobuf/descriptor.pb.h> - #include <google/protobuf/io/coded_stream.h> -@@ -814,3 +818,6 @@ - } // namespace internal - } // namespace protobuf - } // namespace google -+#if defined(__APPLE__) && defined(__arm64__) -+#pragma GCC visibility pop -+#endif ---- src/google/protobuf/generated_message_reflection.cc.O 2018-07-23 20:56:42.000000000 +0000 -+++ src/google/protobuf/generated_message_reflection.cc 2022-05-06 13:38:49.655540772 +0000 -@@ -32,6 +32,9 @@ - // Based on original Protocol Buffers design by - // Sanjay Ghemawat, Jeff Dean, and others. - -+#if defined(__APPLE__) && defined(__arm64__) -+#pragma GCC visibility push(hidden) -+#endif - #include <algorithm> - #include <set> - -@@ -2420,3 +2423,6 @@ - } // namespace internal - } // namespace protobuf - } // namespace google -+#if defined(__APPLE__) && defined(__arm64__) -+#pragma GCC visibility pop -+#endif ---- src/google/protobuf/map_field.cc.O 2018-07-23 20:56:42.000000000 +0000 -+++ src/google/protobuf/map_field.cc 2022-05-06 13:34:44.913905697 +0000 -@@ -28,6 +28,10 @@ - // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -+#if defined(__APPLE__) && defined(__arm64__) -+#pragma GCC visibility push(hidden) -+#endif -+ - #include <google/protobuf/map_field.h> - #include <google/protobuf/map_field_inl.h> - -@@ -462,3 +466,6 @@ - } // namespace internal - } // namespace protobuf - } // namespace google -+#if defined(__APPLE__) && defined(__arm64__) -+#pragma GCC visibility pop -+#endif ---- src/google/protobuf/text_format.cc.O 2018-07-30 22:16:11.000000000 +0000 -+++ src/google/protobuf/text_format.cc 2022-05-06 13:34:58.881999517 +0000 -@@ -32,6 +32,10 @@ - // Based on original Protocol Buffers design by - // Sanjay Ghemawat, Jeff Dean, and others. - -+#if defined(__APPLE__) && defined(__arm64__) -+#pragma GCC visibility push(hidden) -+#endif -+ - #include <algorithm> - #include <float.h> - #include <math.h> -@@ -2258,3 +2262,6 @@ - - } // namespace protobuf - } // namespace google -+#if defined(__APPLE__) && defined(__arm64__) -+#pragma GCC visibility pop -+#endif ---- src/google/protobuf/wire_format.cc.O 2018-07-23 20:56:42.000000000 +0000 -+++ src/google/protobuf/wire_format.cc 2022-05-06 13:06:23.294219228 +0000 -@@ -32,6 +32,10 @@ - // Based on original Protocol Buffers design by - // Sanjay Ghemawat, Jeff Dean, and others. - -+#if defined(__APPLE__) && defined(__arm64__) -+#pragma GCC visibility push(hidden) -+#endif -+ - #include <stack> - #include <string> - #include <vector> -@@ -1445,3 +1449,7 @@ - } // namespace internal - } // namespace protobuf - } // namespace google -+ -+#if defined(__APPLE__) && defined(__arm64__) -+#pragma GCC visibility pop -+#endif ---- src/google/protobuf/stubs/status.cc.O 2018-07-23 20:56:42.000000000 +0000 -+++ src/google/protobuf/stubs/status.cc 2022-05-06 15:18:53.393208814 +0000 -@@ -27,6 +27,11 @@ - // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ -+#if defined(__APPLE__) && defined(__arm64__) -+#pragma GCC visibility push(hidden) -+#endif -+ - #include <google/protobuf/stubs/status.h> - - #include <ostream> -@@ -132,3 +137,6 @@ - } // namespace util - } // namespace protobuf - } // namespace google -+#if defined(__APPLE__) && defined(__arm64__) -+#pragma GCC visibility pop -+#endif diff --git a/contrib/depends/patches/zeromq/06aba27b04c5822cb88a69677382a0f053367143.patch b/contrib/depends/patches/zeromq/06aba27b04c5822cb88a69677382a0f053367143.patch deleted file mode 100644 index 53e18a452..000000000 --- a/contrib/depends/patches/zeromq/06aba27b04c5822cb88a69677382a0f053367143.patch +++ /dev/null @@ -1,22 +0,0 @@ -From 06aba27b04c5822cb88a69677382a0f053367143 Mon Sep 17 00:00:00 2001 -From: sabotagebeats <27985126+sabotagebeats@users.noreply.github.com> -Date: Thu, 22 Jul 2021 21:53:19 -0700 -Subject: [PATCH] fix: building libzmq fails with error src/clock.cpp:131:16: - error: unused variable 'nsecs_per_usec' - ---- - src/clock.cpp | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/src/clock.cpp b/src/clock.cpp -index 93da90a8e..63c0100a5 100644 ---- a/src/clock.cpp -+++ b/src/clock.cpp -@@ -195,6 +195,7 @@ uint64_t zmq::clock_t::now_us () - - #else - -+ LIBZMQ_UNUSED (nsecs_per_usec); - // Use POSIX gettimeofday function to get precise time. - struct timeval tv; - int rc = gettimeofday (&tv, NULL); diff --git a/contrib/depends/toolchain.cmake.in b/contrib/depends/toolchain.cmake.in index 570065560..e44882f4c 100644 --- a/contrib/depends/toolchain.cmake.in +++ b/contrib/depends/toolchain.cmake.in @@ -1,5 +1,5 @@ # Set the system name to one of Android, Darwin, FreeBSD, Linux, or Windows -SET(CMAKE_SYSTEM_NAME @depends@) +SET(CMAKE_SYSTEM_NAME @cmake_system_name@) SET(CMAKE_SYSTEM_PROCESSOR @arch@) SET(CMAKE_BUILD_TYPE @release_type@) @@ -27,12 +27,6 @@ SET(Terminfo_LIBRARY @prefix@/lib/libtinfo.a) SET(UNBOUND_INCLUDE_DIR @prefix@/include) SET(UNBOUND_LIBRARIES @prefix@/lib/libunbound.a) -if(NOT CMAKE_SYSTEM_NAME STREQUAL "Android") -SET(LIBUNWIND_INCLUDE_DIR @prefix@/include) -SET(LIBUNWIND_LIBRARIES @prefix@/lib/libunwind.a) -SET(LIBUNWIND_LIBRARY_DIRS @prefix@/lib) - -if(NOT CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") SET(LIBUSB-1.0_LIBRARY @prefix@/lib/libusb-1.0.a) SET(LIBUDEV_LIBRARY @prefix@/lib/libudev.a) @@ -41,8 +35,11 @@ SET(Protobuf_PROTOC_EXECUTABLE @prefix@/native/bin/protoc CACHE FILEPATH "Path t SET(Protobuf_INCLUDE_DIR @prefix@/include CACHE PATH "Protobuf include dir") SET(Protobuf_INCLUDE_DIRS @prefix@/include CACHE PATH "Protobuf include dir") SET(Protobuf_LIBRARY @prefix@/lib/libprotobuf.a CACHE FILEPATH "Protobuf library") -endif() +if(NOT CMAKE_SYSTEM_NAME STREQUAL "Android") +SET(LIBUNWIND_INCLUDE_DIR @prefix@/include) +SET(LIBUNWIND_LIBRARIES @prefix@/lib/libunwind.a) +SET(LIBUNWIND_LIBRARY_DIRS @prefix@/lib) endif() SET(ZMQ_INCLUDE_PATH @prefix@/include) @@ -68,15 +65,15 @@ set (CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) # Find includes in target # specify the cross compiler to be used. Darwin uses clang provided by the SDK. if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") if(ARCHITECTURE STREQUAL "aarch64") - SET(CLANG_TARGET "arm64-apple-darwin11") - SET(CONF_TRIPLE "aarch64-apple-darwin11") + SET(CLANG_TARGET "arm64-apple-darwin") + SET(CONF_TRIPLE "aarch64-apple-darwin") SET(BUILD_TAG "mac-armv8") SET(CMAKE_OSX_ARCHITECTURES "arm64") set(ARM ON) set(ARM_ID "armv8-a") else() - SET(CLANG_TARGET "x86_64-apple-darwin11") - SET(CONF_TRIPLE "x86_64-apple-darwin11") + SET(CLANG_TARGET "x86_64-apple-darwin") + SET(CONF_TRIPLE "x86_64-apple-darwin") SET(BUILD_TAG "mac-x64") SET(CMAKE_OSX_ARCHITECTURES "x86_64") endif() @@ -94,8 +91,8 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") SET(BREW OFF) SET(PORT OFF) SET(CMAKE_OSX_SYSROOT "@prefix@/native/SDK/") - SET(CMAKE_OSX_DEPLOYMENT_TARGET "10.08") - SET(CMAKE_CXX_STANDARD 14) + SET(CMAKE_OSX_DEPLOYMENT_TARGET "10.13") + SET(CMAKE_CXX_STANDARD 17) SET(LLVM_ENABLE_PIC OFF) SET(LLVM_ENABLE_PIE OFF) elseif(CMAKE_SYSTEM_NAME STREQUAL "Android") @@ -144,8 +141,8 @@ elseif(ARCHITECTURE STREQUAL "aarch64") endif() if(ARCHITECTURE STREQUAL "riscv64") - set(NO_AES ON) - set(ARCH "rv64imafdc") + set(ARCH_ID "riscv64") + set(ARCH "rv64gc") endif() if(ARCHITECTURE STREQUAL "i686") diff --git a/contrib/epee/include/math_helper.h b/contrib/epee/include/math_helper.h index 6a759b515..00d695179 100644 --- a/contrib/epee/include/math_helper.h +++ b/contrib/epee/include/math_helper.h @@ -24,213 +24,20 @@ // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // - - - #pragma once +#include <cstdint> // uint64_t -#include <list> -#include <numeric> -#include <random> -#include <boost/timer/timer.hpp> -#include <boost/uuid/uuid.hpp> -#include <boost/uuid/random_generator.hpp> - -#include "syncobj.h" -#include "time_helper.h" +#ifdef _WIN32 +#include <sysinfoapi.h> // GetSystemTimeAsFileTime +#else +#include <sys/time.h> // gettimeofday +#endif namespace epee { namespace math_helper { - - template<typename val, int default_base> - class average - { - public: - - average() - { - m_base = default_base; - m_last_avg_val = 0; - } - - bool set_base() - { - CRITICAL_REGION_LOCAL(m_lock); - - m_base = default_base; - if(m_list.size() > m_base) - m_list.resize(m_base); - - return true; - } - - typedef val value_type; - - void push(const value_type& vl) - { - CRITICAL_REGION_LOCAL(m_lock); - -//#ifndef DEBUG_STUB - m_list.push_back(vl); - if(m_list.size() > m_base ) - m_list.pop_front(); -//#endif - } - - double update(const value_type& vl) - { - CRITICAL_REGION_LOCAL(m_lock); -//#ifndef DEBUG_STUB - push(vl); -//#endif - - return get_avg(); - } - - double get_avg() - { - CRITICAL_REGION_LOCAL(m_lock); - - value_type vl = std::accumulate(m_list.begin(), m_list.end(), value_type(0)); - if(m_list.size()) - return m_last_avg_val = (double)(vl/m_list.size()); - - return m_last_avg_val = (double)vl; - } - - value_type get_last_val() - { - CRITICAL_REGION_LOCAL(m_lock); - if(m_list.size()) - return m_list.back(); - - return 0; - } - - private: - unsigned int m_base; - double m_last_avg_val; - std::list<value_type> m_list; - critical_section m_lock; - }; - - -#ifdef WINDOWS_PLATFORM - - /************************************************************************/ - /* */ - /************************************************************************/ - class timing_guard_base - { - public: - virtual ~timing_guard_base(){}; - }; - - template<class T> - class timing_guard: public timing_guard_base - { - public: - timing_guard(T& avrg):m_avrg(avrg) - { - m_start_ticks = ::GetTickCount(); - } - - ~timing_guard() - { - m_avrg.push(::GetTickCount()-m_start_ticks); - } - - private: - T& m_avrg; - DWORD m_start_ticks; - }; - - template<class t_timing> - timing_guard_base* create_timing_guard(t_timing& timing){return new timing_guard<t_timing>(timing);} - -#define BEGIN_TIMING_ZONE(timing_var) { boost::shared_ptr<math_helper::timing_guard_base> local_timing_guard_ptr(math_helper::create_timing_guard(timing_var)); -#define END_TIMING_ZONE() } -#endif - -//#ifdef WINDOWS_PLATFORM_EX - template<uint64_t default_time_window> - class speed - { - public: - - speed() - { - m_time_window = default_time_window; - m_last_speed_value = 0; - } - bool chick() - { -#ifndef DEBUG_STUB - uint64_t ticks = misc_utils::get_tick_count(); - CRITICAL_REGION_BEGIN(m_lock); - m_chicks.push_back(ticks); - CRITICAL_REGION_END(); - //flush(ticks); -#endif - return true; - } - - bool chick(size_t count) - { - for(size_t s = 0; s != count; s++) - chick(); - - return true; - } - - - size_t get_speed() - { - flush(misc_utils::get_tick_count()); - return m_last_speed_value = m_chicks.size(); - } - private: - - bool flush(uint64_t ticks) - { - CRITICAL_REGION_BEGIN(m_lock); - std::list<uint64_t>::iterator it = m_chicks.begin(); - while(it != m_chicks.end()) - { - if(*it + m_time_window < ticks) - m_chicks.erase(it++); - else - break; - } - CRITICAL_REGION_END(); - return true; - } - - std::list<uint64_t> m_chicks; - uint64_t m_time_window; - size_t m_last_speed_value; - critical_section m_lock; - }; -//#endif - - template<class tlist> - void randomize_list(tlist& t_list) - { - for(typename tlist::iterator it = t_list.begin();it!=t_list.end();it++) - { - size_t offset = rand()%t_list.size(); - typename tlist::iterator it_2 = t_list.begin(); - for(size_t local_offset = 0;local_offset!=offset;local_offset++) - it_2++; - if(it_2 == it) - continue; - std::swap(*it_2, *it); - } - - } template<typename get_interval, bool start_immediate = true> class once_a_time { diff --git a/contrib/epee/include/net/http_base.h b/contrib/epee/include/net/http_base.h index 15fd30bf3..b53766780 100644 --- a/contrib/epee/include/net/http_base.h +++ b/contrib/epee/include/net/http_base.h @@ -34,6 +34,7 @@ #include <string> #include <utility> #include <list> +#include <stdint.h> #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net.http" diff --git a/contrib/epee/include/serialization/keyvalue_serialization.h b/contrib/epee/include/serialization/keyvalue_serialization.h index 06d74329f..ea767865c 100644 --- a/contrib/epee/include/serialization/keyvalue_serialization.h +++ b/contrib/epee/include/serialization/keyvalue_serialization.h @@ -105,7 +105,7 @@ public: \ #define KV_SERIALIZE_VAL_POD_AS_BLOB_OPT_N(varialble, val_name, default_value) \ do { \ static_assert(std::is_pod<decltype(this_ref.varialble)>::value, "t_type must be a POD type."); \ - bool ret = KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE_N(varialble, val_name); \ + bool ret = KV_SERIALIZE_VAL_POD_AS_BLOB_FORCE_N(varialble, val_name) \ if (!ret) \ epee::serialize_default(this_ref.varialble, default_value); \ } while(0); diff --git a/contrib/epee/include/serialization/wire/write.h b/contrib/epee/include/serialization/wire/write.h index c18f7dbcc..c2359918c 100644 --- a/contrib/epee/include/serialization/wire/write.h +++ b/contrib/epee/include/serialization/wire/write.h @@ -30,6 +30,7 @@ #include <boost/utility/string_ref.hpp> #include <boost/range/size.hpp> #include <cstdint> +#include <iterator> #include <system_error> #include <type_traits> @@ -188,7 +189,13 @@ namespace wire_write template<typename T> inline std::size_t array_size(std::true_type, const T& source) - { return boost::size(source); } + { + static_assert( + !std::is_same<typename std::iterator_traits<typename T::const_iterator>::iterator_category, std::input_iterator_tag>{}, + "Input iterators must use json (or similar) derived classes directly" + ); + return boost::size(source); + } template<typename T> inline constexpr std::size_t array_size(std::false_type, const T&) noexcept diff --git a/contrib/epee/include/storages/portable_storage_from_bin.h b/contrib/epee/include/storages/portable_storage_from_bin.h index d8a8a4a49..9fcaf5d01 100644 --- a/contrib/epee/include/storages/portable_storage_from_bin.h +++ b/contrib/epee/include/storages/portable_storage_from_bin.h @@ -33,6 +33,9 @@ #include "portable_storage_base.h" #include "portable_storage_bin_utils.h" +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "serialization" + #ifdef EPEE_PORTABLE_STORAGE_RECURSION_LIMIT #define EPEE_PORTABLE_STORAGE_RECURSION_LIMIT_INTERNAL EPEE_PORTABLE_STORAGE_RECURSION_LIMIT #else @@ -228,6 +231,7 @@ namespace epee default: CHECK_AND_ASSERT_THROW_MES(false, "unknown entry_type code = " << type); } + return read_ae<int8_t>(); // unreachable, dummy return to avoid compiler warning } inline @@ -319,6 +323,7 @@ namespace epee default: CHECK_AND_ASSERT_THROW_MES(false, "unknown entry_type code = " << ent_type); } + return read_se<int8_t>(); // unreachable, dummy return to avoid compiler warning } inline void throwable_buffer_reader::read(section& sec) diff --git a/contrib/epee/include/storages/portable_storage_from_json.h b/contrib/epee/include/storages/portable_storage_from_json.h index 95507328d..60acfccb8 100644 --- a/contrib/epee/include/storages/portable_storage_from_json.h +++ b/contrib/epee/include/storages/portable_storage_from_json.h @@ -31,6 +31,9 @@ #include "parserse_base_utils.h" #include "file_io_utils.h" +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "serialization" + #define EPEE_JSON_RECURSION_LIMIT_INTERNAL 100 namespace epee diff --git a/contrib/epee/include/string_tools.h b/contrib/epee/include/string_tools.h index dbbe1906e..7fcd02726 100644 --- a/contrib/epee/include/string_tools.h +++ b/contrib/epee/include/string_tools.h @@ -127,7 +127,6 @@ namespace string_tools return s; } - bool validate_hex(uint64_t length, const std::string& str); std::string get_extension(const std::string& str); std::string cut_off_extension(const std::string& str); diff --git a/contrib/epee/src/http_auth.cpp b/contrib/epee/src/http_auth.cpp index 5071d2685..d6de6a0e1 100644 --- a/contrib/epee/src/http_auth.cpp +++ b/contrib/epee/src/http_auth.cpp @@ -217,15 +217,13 @@ namespace //// Digest Authentication template<typename Digest> - typename std::result_of<Digest()>::type generate_a1( - Digest digest, const http::login& creds, const boost::string_ref realm) + auto generate_a1(Digest digest, const http::login& creds, const boost::string_ref realm) { return digest(creds.username, u8":", realm, u8":", creds.password); } template<typename Digest> - typename std::result_of<Digest()>::type generate_a1( - Digest digest, const http::http_client_auth::session& user) + auto generate_a1(Digest digest, const http::http_client_auth::session& user) { return generate_a1(std::move(digest), user.credentials, user.server.realm); } diff --git a/contrib/epee/src/string_tools.cpp b/contrib/epee/src/string_tools.cpp index 984a151b5..3abb83c74 100644 --- a/contrib/epee/src/string_tools.cpp +++ b/contrib/epee/src/string_tools.cpp @@ -82,16 +82,6 @@ namespace string_tools return true; } //---------------------------------------------------------------------------- - bool validate_hex(uint64_t length, const std::string& str) - { - if (str.size() != length) - return false; - for (char c: str) - if (!isxdigit(c)) - return false; - return true; - } - //---------------------------------------------------------------------------- bool parse_peer_from_string(uint32_t& ip, uint16_t& port, const std::string& addres) { //parse ip and address diff --git a/contrib/gitian/gitian-linux.yml b/contrib/gitian/gitian-linux.yml index 63d2bc5d2..41915deb9 100644 --- a/contrib/gitian/gitian-linux.yml +++ b/contrib/gitian/gitian-linux.yml @@ -21,6 +21,7 @@ packages: - "g++-7-arm-linux-gnueabihf" - "gcc-arm-linux-gnueabihf" - "g++-arm-linux-gnueabihf" +- "g++-riscv64-linux-gnu" - "g++-7-multilib" - "gcc-7-multilib" - "binutils-arm-linux-gnueabihf" @@ -43,7 +44,7 @@ files: [] script: | WRAP_DIR=$HOME/wrapped - HOSTS="x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu i686-linux-gnu" + HOSTS="x86_64-linux-gnu arm-linux-gnueabihf aarch64-linux-gnu i686-linux-gnu riscv64-linux-gnu" FAKETIME_HOST_PROGS="" FAKETIME_PROGS="date" HOST_CFLAGS="-O2 -g" @@ -159,7 +160,13 @@ script: | fi export C_INCLUDE_PATH="$EXTRA_INCLUDES" export CPLUS_INCLUDE_PATH="$EXTRA_INCLUDES" - cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake -DBACKCOMPAT=ON -DCMAKE_SKIP_RPATH=ON + # glibc only added riscv support in 2.27, disable backwards compatibility + if [ "$i" == "riscv64-linux-gnu" ]; then + BACKCOMPAT_OPTION=OFF + else + BACKCOMPAT_OPTION=ON + fi + cmake .. -DCMAKE_TOOLCHAIN_FILE=${BASEPREFIX}/${i}/share/toolchain.cmake -DBACKCOMPAT=${BACKCOMPAT_OPTION} -DCMAKE_SKIP_RPATH=ON make ${MAKEOPTS} chmod 755 bin/* cp ../LICENSE ../README.md ../docs/ANONYMITY_NETWORKS.md bin diff --git a/contrib/gitian/gitian-osx.yml b/contrib/gitian/gitian-osx.yml index 648688bcd..74ad21ea3 100644 --- a/contrib/gitian/gitian-osx.yml +++ b/contrib/gitian/gitian-osx.yml @@ -29,7 +29,7 @@ remotes: files: [] script: | WRAP_DIR=$HOME/wrapped - HOSTS="x86_64-apple-darwin11 aarch64-apple-darwin11" + HOSTS="x86_64-apple-darwin aarch64-apple-darwin" FAKETIME_HOST_PROGS="" FAKETIME_PROGS="ar ranlib date dmg genisoimage python" diff --git a/external/easylogging++/ea_config.h b/external/easylogging++/ea_config.h index 4ba0cd611..54db506e6 100644 --- a/external/easylogging++/ea_config.h +++ b/external/easylogging++/ea_config.h @@ -11,7 +11,7 @@ #define ELPP_UTC_DATETIME #ifdef EASYLOGGING_CC -#if !(!defined __GLIBC__ || !defined __GNUC__ || defined __MINGW32__ || defined __MINGW64__ || defined __ANDROID__ || defined __NetBSD__) +#if !(!defined __GLIBC__ || !defined __GNUC__ || defined __MINGW32__ || defined __MINGW64__ || defined __ANDROID__ || defined __NetBSD__) || (defined __FreeBSD__) #define ELPP_FEATURE_CRASH_LOG #endif #endif diff --git a/external/randomx b/external/randomx -Subproject 261d58c77fc5547c0aa7fdfeb58421ba7e0e6e1 +Subproject 102f8acf90a7649ada410de5499a7ec62e49e1d diff --git a/external/trezor-common b/external/trezor-common -Subproject bff7fdfe436c727982cc553bdfb29a9021b423b +Subproject bc28c316d05bf1e9ebfe3d7df1ab25831d98d16 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index fed042ce2..93643af24 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -96,6 +96,7 @@ add_subdirectory(hardforks) add_subdirectory(blockchain_db) add_subdirectory(mnemonics) add_subdirectory(rpc) +add_subdirectory(seraphis_crypto) if(NOT IOS) add_subdirectory(serialization) endif() diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index b38ec9e05..9628a5c4d 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -1306,6 +1306,21 @@ public: virtual bool get_pruned_tx_blobs_from(const crypto::hash& h, size_t count, std::vector<cryptonote::blobdata> &bd) const = 0; /** + * @brief Get all txids in the database (chain and pool) that match a certain nbits txid template + * + * To be more specific, for all `dbtxid` txids in the database, return `dbtxid` if + * `0 == cryptonote::compare_hash32_reversed_nbits(txid_template, dbtxid, nbits)`. + * + * @param txid_template the transaction id template + * @param nbits number of bits to compare against in the template + * @param max_num_txs The maximum number of txids to match, if we hit this limit, throw early + * @return std::vector<crypto::hash> the list of all matching txids + * + * @throw TX_EXISTS if the number of txids that match exceed `max_num_txs` + */ + virtual std::vector<crypto::hash> get_txids_loose(const crypto::hash& txid_template, std::uint32_t nbits, uint64_t max_num_txs = 0) = 0; + + /** * @brief fetches a variable number of blocks and transactions from the given height, in canonical blockchain order * * The subclass should return the blocks and transactions stored from the one with the given diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 4178c862b..2c015faee 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -3144,6 +3144,58 @@ bool BlockchainLMDB::get_pruned_tx_blobs_from(const crypto::hash& h, size_t coun return true; } +std::vector<crypto::hash> BlockchainLMDB::get_txids_loose(const crypto::hash& txid_template, std::uint32_t bits, uint64_t max_num_txs) +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + std::vector<crypto::hash> matching_hashes; + + TXN_PREFIX_RDONLY(); // Start a read-only transaction + RCURSOR(tx_indices); // Open cursors to the tx_indices and txpool_meta databases + RCURSOR(txpool_meta); + + // Search on-chain and pool transactions together, starting with on-chain txs + MDB_cursor* cursor = m_cur_tx_indices; + MDB_val k = zerokval; // tx_indicies DB uses a dummy key + MDB_val_set(v, txid_template); // tx_indicies DB indexes data values by crypto::hash value on front + MDB_cursor_op op = MDB_GET_BOTH_RANGE; // Set the cursor to the first key/value pair >= the given key + bool doing_chain = true; // this variable tells us whether we are processing chain or pool txs + while (1) + { + const int get_result = mdb_cursor_get(cursor, &k, &v, op); + op = doing_chain ? MDB_NEXT_DUP : MDB_NEXT; // Set the cursor to the next key/value pair + if (get_result && get_result != MDB_NOTFOUND) + throw0(DB_ERROR(lmdb_error("DB error attempting to fetch txid range", get_result).c_str())); + + // In tx_indicies, the hash is stored at the data, in txpool_meta at the key + const crypto::hash* const p_dbtxid = (const crypto::hash*)(doing_chain ? v.mv_data : k.mv_data); + + // Check if we reached the end of a DB or the hashes no longer match the template + if (get_result == MDB_NOTFOUND || compare_hash32_reversed_nbits(txid_template, *p_dbtxid, bits)) + { + if (doing_chain) // done with chain processing, switch to pool processing + { + k.mv_size = sizeof(crypto::hash); // txpool_meta DB is indexed using crypto::hash as keys + k.mv_data = (void*) txid_template.data; + cursor = m_cur_txpool_meta; // switch databases + op = MDB_SET_RANGE; // Set the cursor to the first key >= the given key + doing_chain = false; + continue; + } + break; // if we get to this point, then we finished pool processing and we are done + } + else if (matching_hashes.size() >= max_num_txs && max_num_txs != 0) + throw0(TX_EXISTS("number of tx hashes in template range exceeds maximum")); + + matching_hashes.push_back(*p_dbtxid); + } + + TXN_POSTFIX_RDONLY(); // End the read-only transaction + + return matching_hashes; +} + bool BlockchainLMDB::get_blocks_from(uint64_t start_height, size_t min_block_count, size_t max_block_count, size_t max_tx_count, size_t max_size, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata>>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index c352458b4..95e7b2aa4 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -262,6 +262,8 @@ public: virtual bool get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const; virtual bool get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const; + virtual std::vector<crypto::hash> get_txids_loose(const crypto::hash& h, std::uint32_t bits, uint64_t max_num_txs = 0); + virtual uint64_t get_tx_count() const; virtual std::vector<transaction> get_tx_list(const std::vector<crypto::hash>& hlist) const; diff --git a/src/blockchain_db/testdb.h b/src/blockchain_db/testdb.h index 946f26270..a27183b2c 100644 --- a/src/blockchain_db/testdb.h +++ b/src/blockchain_db/testdb.h @@ -71,6 +71,7 @@ public: virtual bool get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const override { return false; } virtual bool get_pruned_tx_blobs_from(const crypto::hash& h, size_t count, std::vector<cryptonote::blobdata> &bd) const override { return false; } virtual bool get_blocks_from(uint64_t start_height, size_t min_block_count, size_t max_block_count, size_t max_tx_count, size_t max_size, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata>>>>& blocks, bool pruned, bool skip_coinbase, bool get_miner_tx_hash) const override { return false; } + virtual std::vector<crypto::hash> get_txids_loose(const crypto::hash& h, std::uint32_t bits, uint64_t max_num_txs = 0) override { return {}; } virtual bool get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const override { return false; } virtual bool get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const override { return false; } virtual uint64_t get_block_height(const crypto::hash& h) const override { return 0; } diff --git a/src/blockchain_utilities/blockchain_stats.cpp b/src/blockchain_utilities/blockchain_stats.cpp index f65054fc5..f0a8e5adc 100644 --- a/src/blockchain_utilities/blockchain_stats.cpp +++ b/src/blockchain_utilities/blockchain_stats.cpp @@ -33,6 +33,7 @@ #include "cryptonote_basic/cryptonote_boost_serialization.h" #include "cryptonote_core/cryptonote_core.h" #include "blockchain_db/blockchain_db.h" +#include "time_helper.h" #include "version.h" #undef MONERO_DEFAULT_LOG_CATEGORY diff --git a/src/common/dns_utils.cpp b/src/common/dns_utils.cpp index 9c10c420c..6fc784fb2 100644 --- a/src/common/dns_utils.cpp +++ b/src/common/dns_utils.cpp @@ -28,6 +28,7 @@ #include "common/dns_utils.h" // check local first (in the event of static or in-source compilation of libunbound) +#include "misc_language.h" #include "unbound.h" #include <deque> @@ -195,36 +196,7 @@ boost::optional<std::string> tlsa_to_string(const char* src, size_t len) return boost::none; return std::string(src, len); } - -// custom smart pointer. -// TODO: see if std::auto_ptr and the like support custom destructors -template<typename type, void (*freefunc)(type*)> -class scoped_ptr -{ -public: - scoped_ptr(): - ptr(nullptr) - { - } - scoped_ptr(type *p): - ptr(p) - { - } - ~scoped_ptr() - { - freefunc(ptr); - } - operator type *() { return ptr; } - type **operator &() { return &ptr; } - type *operator->() { return ptr; } - operator const type*() const { return &ptr; } - -private: - type* ptr; -}; - -typedef class scoped_ptr<ub_result,ub_resolve_free> ub_result_ptr; - + struct DNSResolverData { ub_ctx* m_ub_context; @@ -327,10 +299,14 @@ std::vector<std::string> DNSResolver::get_record(const std::string& url, int rec std::vector<std::string> addresses; dnssec_available = false; dnssec_valid = false; - - // destructor takes care of cleanup - ub_result_ptr result; - + + ub_result *result; + // Make sure we are cleaning after result. + epee::misc_utils::auto_scope_leave_caller scope_exit_handler = + epee::misc_utils::create_scope_leave_handler([&](){ + ub_resolve_free(result); + }); + MDEBUG("Performing DNSSEC " << get_record_name(record_type) << " record query for " << url); // call DNS resolver, blocking. if return value not zero, something went wrong diff --git a/src/crypto/jh.c b/src/crypto/jh.c index 12d536375..738c681f8 100644 --- a/src/crypto/jh.c +++ b/src/crypto/jh.c @@ -34,7 +34,7 @@ typedef struct { unsigned long long databitlen; /*the message size in bits*/ unsigned long long datasize_in_buffer; /*the size of the message remained in buffer; assumed to be multiple of 8bits except for the last partial block at the end of the message*/ DATA_ALIGN16(uint64 x[8][2]); /*the 1024-bit state, ( x[i][0] || x[i][1] ) is the ith row of the state in the pseudocode*/ - unsigned char buffer[64]; /*the 512-bit message block to be hashed;*/ + DATA_ALIGN16(unsigned char buffer[64]); /*the 512-bit message block to be hashed;*/ } hashState; @@ -213,16 +213,24 @@ static void E8(hashState *state) /*The compression function F8 */ static void F8(hashState *state) { - uint64 i; + uint64_t* x = (uint64_t*)state->x; /*xor the 512-bit message with the fist half of the 1024-bit hash state*/ - for (i = 0; i < 8; i++) state->x[i >> 1][i & 1] ^= ((uint64*)state->buffer)[i]; + for (int i = 0; i < 8; ++i) { + uint64 b; + memcpy(&b, &state->buffer[i << 3], sizeof(b)); + x[i] ^= b; + } /*the bijective function E8 */ E8(state); /*xor the 512-bit message with the second half of the 1024-bit hash state*/ - for (i = 0; i < 8; i++) state->x[(8+i) >> 1][(8+i) & 1] ^= ((uint64*)state->buffer)[i]; + for (int i = 0; i < 8; ++i) { + uint64 b; + memcpy(&b, &state->buffer[i << 3], sizeof(b)); + x[i + 8] ^= b; + } } /*before hashing a message, initialize the hash state as H0 */ @@ -240,6 +248,7 @@ static HashReturn Init(hashState *state, int hashbitlen) case 224: memcpy(state->x,JH224_H0,128); break; case 256: memcpy(state->x,JH256_H0,128); break; case 384: memcpy(state->x,JH384_H0,128); break; + default: case 512: memcpy(state->x,JH512_H0,128); break; } diff --git a/src/cryptonote_basic/account_generators.h b/src/cryptonote_basic/account_generators.h new file mode 100644 index 000000000..c1102db60 --- /dev/null +++ b/src/cryptonote_basic/account_generators.h @@ -0,0 +1,71 @@ +// Copyright (c) 2021, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include "crypto/crypto.h" +#include "crypto/generators.h" + + +namespace cryptonote +{ + +enum class account_generator_era : unsigned char +{ + unknown = 0, + cryptonote = 1 //and ringct +}; + +struct account_generators +{ + crypto::public_key m_primary; //e.g. for spend key + crypto::public_key m_secondary; //e.g. for view key +}; + +inline crypto::public_key get_primary_generator(const account_generator_era era) +{ + if (era == account_generator_era::cryptonote) + return crypto::get_G(); + else + return crypto::null_pkey; //error +} + +inline crypto::public_key get_secondary_generator(const account_generator_era era) +{ + if (era == account_generator_era::cryptonote) + return crypto::get_G(); + else + return crypto::null_pkey; //error +} + +inline account_generators get_account_generators(const account_generator_era era) +{ + return account_generators{get_primary_generator(era), get_secondary_generator(era)}; +} + +} //namespace cryptonote diff --git a/src/cryptonote_basic/cryptonote_basic.h b/src/cryptonote_basic/cryptonote_basic.h index ae112c31f..0531439d6 100644 --- a/src/cryptonote_basic/cryptonote_basic.h +++ b/src/cryptonote_basic/cryptonote_basic.h @@ -77,7 +77,7 @@ namespace cryptonote // outputs <= HF_VERSION_VIEW_TAGS struct txout_to_key { - txout_to_key() { } + txout_to_key(): key() { } txout_to_key(const crypto::public_key &_key) : key(_key) { } crypto::public_key key; }; @@ -85,7 +85,7 @@ namespace cryptonote // outputs >= HF_VERSION_VIEW_TAGS struct txout_to_tagged_key { - txout_to_tagged_key() { } + txout_to_tagged_key(): key(), view_tag() { } txout_to_tagged_key(const crypto::public_key &_key, const crypto::view_tag &_view_tag) : key(_key), view_tag(_view_tag) { } crypto::public_key key; crypto::view_tag view_tag; // optimization to reduce scanning time diff --git a/src/cryptonote_basic/cryptonote_basic_impl.cpp b/src/cryptonote_basic/cryptonote_basic_impl.cpp index 9bde20609..7fe398283 100644 --- a/src/cryptonote_basic/cryptonote_basic_impl.cpp +++ b/src/cryptonote_basic/cryptonote_basic_impl.cpp @@ -310,6 +310,53 @@ namespace cryptonote { bool operator ==(const cryptonote::block& a, const cryptonote::block& b) { return cryptonote::get_block_hash(a) == cryptonote::get_block_hash(b); } + //-------------------------------------------------------------------------------- + int compare_hash32_reversed_nbits(const crypto::hash& ha, const crypto::hash& hb, unsigned int nbits) + { + static_assert(sizeof(uint64_t) * 4 == sizeof(crypto::hash), "hash is wrong size"); + + // We have to copy these buffers b/c of the strict aliasing rule + uint64_t va[4]; + memcpy(va, &ha, sizeof(crypto::hash)); + uint64_t vb[4]; + memcpy(vb, &hb, sizeof(crypto::hash)); + + for (int n = 3; n >= 0 && nbits; --n) + { + const unsigned int msb_nbits = std::min<unsigned int>(64, nbits); + const uint64_t lsb_nbits_dropped = static_cast<uint64_t>(64 - msb_nbits); + const uint64_t van = SWAP64LE(va[n]) >> lsb_nbits_dropped; + const uint64_t vbn = SWAP64LE(vb[n]) >> lsb_nbits_dropped; + nbits -= msb_nbits; + + if (van < vbn) return -1; else if (van > vbn) return 1; + } + + return 0; + } + + crypto::hash make_hash32_loose_template(unsigned int nbits, const crypto::hash& h) + { + static_assert(sizeof(uint64_t) * 4 == sizeof(crypto::hash), "hash is wrong size"); + + // We have to copy this buffer b/c of the strict aliasing rule + uint64_t vh[4]; + memcpy(vh, &h, sizeof(crypto::hash)); + + for (int n = 3; n >= 0; --n) + { + const unsigned int msb_nbits = std::min<unsigned int>(64, nbits); + const uint64_t mask = msb_nbits ? (~((std::uint64_t(1) << (64 - msb_nbits)) - 1)) : 0; + nbits -= msb_nbits; + + vh[n] &= SWAP64LE(mask); + } + + crypto::hash res; + memcpy(&res, vh, sizeof(crypto::hash)); + return res; + } + //-------------------------------------------------------------------------------- } //-------------------------------------------------------------------------------- diff --git a/src/cryptonote_basic/cryptonote_basic_impl.h b/src/cryptonote_basic/cryptonote_basic_impl.h index 984bee19f..53dbc47d7 100644 --- a/src/cryptonote_basic/cryptonote_basic_impl.h +++ b/src/cryptonote_basic/cryptonote_basic_impl.h @@ -112,6 +112,41 @@ namespace cryptonote { bool operator ==(const cryptonote::transaction& a, const cryptonote::transaction& b); bool operator ==(const cryptonote::block& a, const cryptonote::block& b); + + /************************************************************************/ + /* K-anonymity helper functions */ + /************************************************************************/ + + /** + * @brief Compares two hashes up to `nbits` bits in reverse byte order ("LMDB key order") + * + * The comparison essentially goes from the 31th, 30th, 29th, ..., 0th byte and compares the MSBs + * to the LSBs in each byte, up to `nbits` bits. If we use up `nbits` bits before finding a + * difference in the bits between the two hashes, we return 0. If we encounter a zero bit in `ha` + * where `hb` has a one in that bit place, then we reutrn -1. If the converse scenario happens, + * we return a 1. When `nbits` == 256 (there are 256 bits in `crypto::hash`), calling this is + * functionally identical to `BlockchainLMDB::compare_hash32`. + * + * @param ha left hash + * @param hb right hash + * @param nbits the number of bits to consider, a higher value means a finer comparison + * @return int 0 if ha == hb, -1 if ha < hb, 1 if ha > hb + */ + int compare_hash32_reversed_nbits(const crypto::hash& ha, const crypto::hash& hb, unsigned int nbits); + + /** + * @brief Make a template which matches `h` in LMDB order up to `nbits` bits, safe for k-anonymous fetching + * + * To be more technical, this function creates a hash which satifies the following property: + * For all `H_prime` s.t. `0 == compare_hash32_reversed_nbits(real_hash, H_prime, nbits)`, + * `1 > compare_hash32_reversed_nbits(real_hash, H_prime, 256)`. + * In other words, we return the "least" hash nbit-equal to `real_hash`. + * + * @param nbits The number of "MSB" bits to include in the template + * @param real_hash The original hash which contains more information than we want to disclose + * @return crypto::hash hash template that contains `nbits` bits matching real_hash and no more + */ + crypto::hash make_hash32_loose_template(unsigned int nbits, const crypto::hash& real_hash); } bool parse_hash256(const std::string &str_hash, crypto::hash& hash); diff --git a/src/cryptonote_basic/cryptonote_format_utils.cpp b/src/cryptonote_basic/cryptonote_format_utils.cpp index 9aeee3d55..c86e7d848 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.cpp +++ b/src/cryptonote_basic/cryptonote_format_utils.cpp @@ -739,22 +739,28 @@ namespace cryptonote return true; } //--------------------------------------------------------------- - bool add_mm_merkle_root_to_tx_extra(std::vector<uint8_t>& tx_extra, const crypto::hash& mm_merkle_root, size_t mm_merkle_tree_depth) + bool add_mm_merkle_root_to_tx_extra(std::vector<uint8_t>& tx_extra, const crypto::hash& mm_merkle_root, uint64_t mm_merkle_tree_depth) { - CHECK_AND_ASSERT_MES(mm_merkle_tree_depth < 32, false, "merge mining merkle tree depth should be less than 32"); size_t start_pos = tx_extra.size(); - tx_extra.resize(tx_extra.size() + 3 + 32); + static const size_t max_varint_size = 16; + tx_extra.resize(tx_extra.size() + 2 + 32 + max_varint_size); //write tag tx_extra[start_pos] = TX_EXTRA_MERGE_MINING_TAG; //write data size ++start_pos; - tx_extra[start_pos] = 33; - //write depth varint (always one byte here) + const off_t len_bytes = start_pos; + // one byte placeholder for length since we'll only know the size later after writing a varint + tx_extra[start_pos] = 0; + //write depth varint ++start_pos; - tx_extra[start_pos] = mm_merkle_tree_depth; + uint8_t *ptr = &tx_extra[start_pos], *start = ptr; + tools::write_varint(ptr, mm_merkle_tree_depth); //write data - ++start_pos; + const size_t varint_size = ptr - start; + start_pos += varint_size; memcpy(&tx_extra[start_pos], &mm_merkle_root, 32); + tx_extra.resize(tx_extra.size() - (max_varint_size - varint_size)); + tx_extra[len_bytes] = 32 + varint_size; return true; } //--------------------------------------------------------------- diff --git a/src/cryptonote_basic/cryptonote_format_utils.h b/src/cryptonote_basic/cryptonote_format_utils.h index 33fec238e..bd4dcd83d 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.h +++ b/src/cryptonote_basic/cryptonote_format_utils.h @@ -83,7 +83,7 @@ namespace cryptonote std::vector<crypto::public_key> get_additional_tx_pub_keys_from_extra(const transaction_prefix& tx); bool add_additional_tx_pub_keys_to_extra(std::vector<uint8_t>& tx_extra, const std::vector<crypto::public_key>& additional_pub_keys); bool add_extra_nonce_to_tx_extra(std::vector<uint8_t>& tx_extra, const blobdata& extra_nonce); - bool add_mm_merkle_root_to_tx_extra(std::vector<uint8_t>& tx_extra, const crypto::hash& mm_merkle_root, size_t mm_merkle_tree_depth); + bool add_mm_merkle_root_to_tx_extra(std::vector<uint8_t>& tx_extra, const crypto::hash& mm_merkle_root, uint64_t mm_merkle_tree_depth); bool remove_field_from_tx_extra(std::vector<uint8_t>& tx_extra, const std::type_info &type); void set_payment_id_to_tx_extra_nonce(blobdata& extra_nonce, const crypto::hash& payment_id); void set_encrypted_payment_id_to_tx_extra_nonce(blobdata& extra_nonce, const crypto::hash8& payment_id); diff --git a/src/cryptonote_basic/merge_mining.cpp b/src/cryptonote_basic/merge_mining.cpp index 0e649e095..3e26c00e3 100644 --- a/src/cryptonote_basic/merge_mining.cpp +++ b/src/cryptonote_basic/merge_mining.cpp @@ -71,21 +71,21 @@ uint32_t get_path_from_aux_slot(uint32_t slot, uint32_t n_aux_chains) return path; } //--------------------------------------------------------------- -uint32_t encode_mm_depth(uint32_t n_aux_chains, uint32_t nonce) +uint64_t encode_mm_depth(uint32_t n_aux_chains, uint32_t nonce) { CHECK_AND_ASSERT_THROW_MES(n_aux_chains > 0, "n_aux_chains is 0"); + CHECK_AND_ASSERT_THROW_MES(n_aux_chains <= 256, "n_aux_chains is too large"); // how many bits to we need to representing n_aux_chains - 1 uint32_t n_bits = 1; - while ((1u << n_bits) < n_aux_chains && n_bits < 16) + while ((1u << n_bits) < n_aux_chains) ++n_bits; - CHECK_AND_ASSERT_THROW_MES(n_bits <= 16, "Way too many bits required"); - const uint32_t depth = (n_bits - 1) | ((n_aux_chains - 1) << 3) | (nonce << (3 + n_bits)); + const uint64_t depth = (n_bits - 1) | ((n_aux_chains - 1) << 3) | (((uint64_t)nonce) << (3 + n_bits)); return depth; } //--------------------------------------------------------------- -bool decode_mm_depth(uint32_t depth, uint32_t &n_aux_chains, uint32_t &nonce) +bool decode_mm_depth(uint64_t depth, uint32_t &n_aux_chains, uint32_t &nonce) { const uint32_t n_bits = 1 + (depth & 7); n_aux_chains = 1 + (depth >> 3 & ((1 << n_bits) - 1)); diff --git a/src/cryptonote_basic/merge_mining.h b/src/cryptonote_basic/merge_mining.h index acb70ebf0..ef4c92d67 100644 --- a/src/cryptonote_basic/merge_mining.h +++ b/src/cryptonote_basic/merge_mining.h @@ -36,6 +36,6 @@ namespace cryptonote { uint32_t get_aux_slot(const crypto::hash &id, uint32_t nonce, uint32_t n_aux_chains); uint32_t get_path_from_aux_slot(uint32_t slot, uint32_t n_aux_chains); - uint32_t encode_mm_depth(uint32_t n_aux_chains, uint32_t nonce); - bool decode_mm_depth(uint32_t depth, uint32_t &n_aux_chains, uint32_t &nonce); + uint64_t encode_mm_depth(uint32_t n_aux_chains, uint32_t nonce); + bool decode_mm_depth(uint64_t depth, uint32_t &n_aux_chains, uint32_t &nonce); } diff --git a/src/cryptonote_basic/miner.cpp b/src/cryptonote_basic/miner.cpp index 91ee86d60..4b0e43213 100644 --- a/src/cryptonote_basic/miner.cpp +++ b/src/cryptonote_basic/miner.cpp @@ -42,6 +42,7 @@ #include "string_coding.h" #include "string_tools.h" #include "storages/portable_storage_template_helper.h" +#include "time_helper.h" #include "boost/logic/tribool.hpp" #include <boost/filesystem.hpp> diff --git a/src/cryptonote_basic/tx_extra.h b/src/cryptonote_basic/tx_extra.h index af6ee5197..cbf942d4c 100644 --- a/src/cryptonote_basic/tx_extra.h +++ b/src/cryptonote_basic/tx_extra.h @@ -52,7 +52,7 @@ namespace cryptonote // load template <template <bool> class Archive> - bool do_serialize(Archive<false>& ar) + bool member_do_serialize(Archive<false>& ar) { // size - 1 - because of variant tag for (size = 1; size <= TX_EXTRA_PADDING_MAX_COUNT; ++size) @@ -73,7 +73,7 @@ namespace cryptonote // store template <template <bool> class Archive> - bool do_serialize(Archive<true>& ar) + bool member_do_serialize(Archive<true>& ar) { if(TX_EXTRA_PADDING_MAX_COUNT < size) return false; @@ -124,12 +124,12 @@ namespace cryptonote END_SERIALIZE() }; - size_t depth; + uint64_t depth; crypto::hash merkle_root; // load template <template <bool> class Archive> - bool do_serialize(Archive<false>& ar) + bool member_do_serialize(Archive<false>& ar) { std::string field; if(!::do_serialize(ar, field)) @@ -142,7 +142,7 @@ namespace cryptonote // store template <template <bool> class Archive> - bool do_serialize(Archive<true>& ar) + bool member_do_serialize(Archive<true>& ar) { std::ostringstream oss; binary_archive<true> oar(oss); diff --git a/src/cryptonote_config.h b/src/cryptonote_config.h index 45da75b66..fec905928 100644 --- a/src/cryptonote_config.h +++ b/src/cryptonote_config.h @@ -30,6 +30,7 @@ #pragma once +#include <cstdint> #include <stdexcept> #include <string> #include <boost/uuid/uuid.hpp> diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 8036c84cd..7c9bd9163 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -4615,40 +4615,9 @@ bool Blockchain::update_next_cumulative_weight_limit(uint64_t *long_term_effecti } else { - const uint64_t block_weight = m_db->get_block_weight(db_height - 1); + const uint64_t nblocks = std::min<uint64_t>(m_long_term_block_weights_window, db_height); + const uint64_t long_term_median = get_long_term_block_weight_median(db_height - nblocks, nblocks); - uint64_t long_term_median; - if (db_height == 1) - { - long_term_median = CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5; - } - else - { - uint64_t nblocks = std::min<uint64_t>(m_long_term_block_weights_window, db_height); - if (nblocks == db_height) - --nblocks; - long_term_median = get_long_term_block_weight_median(db_height - nblocks - 1, nblocks); - } - - m_long_term_effective_median_block_weight = std::max<uint64_t>(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, long_term_median); - - uint64_t short_term_constraint = m_long_term_effective_median_block_weight; - if (hf_version >= HF_VERSION_2021_SCALING) - short_term_constraint += m_long_term_effective_median_block_weight * 7 / 10; - else - short_term_constraint += m_long_term_effective_median_block_weight * 2 / 5; - uint64_t long_term_block_weight = std::min<uint64_t>(block_weight, short_term_constraint); - - if (db_height == 1) - { - long_term_median = long_term_block_weight; - } - else - { - m_long_term_block_weights_cache_tip_hash = m_db->get_block_hash_from_height(db_height - 1); - m_long_term_block_weights_cache_rolling_median.insert(long_term_block_weight); - long_term_median = m_long_term_block_weights_cache_rolling_median.median(); - } m_long_term_effective_median_block_weight = std::max<uint64_t>(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, long_term_median); std::vector<uint64_t> weights; diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index 3ad051fc3..be2848d09 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -1609,6 +1609,6 @@ namespace cryptonote */ void send_miner_notifications(uint64_t height, const crypto::hash &seed_hash, const crypto::hash &prev_id, uint64_t already_generated_coins); - friend class BlockchainAndPool; + friend struct BlockchainAndPool; }; } // namespace cryptonote diff --git a/src/cryptonote_core/blockchain_and_pool.h b/src/cryptonote_core/blockchain_and_pool.h index c0f607f64..497342aea 100644 --- a/src/cryptonote_core/blockchain_and_pool.h +++ b/src/cryptonote_core/blockchain_and_pool.h @@ -37,6 +37,7 @@ #include "blockchain.h" #include "tx_pool.h" +#include "warnings.h" namespace cryptonote { @@ -52,7 +53,10 @@ struct BlockchainAndPool { Blockchain blockchain; tx_memory_pool tx_pool; - + +PUSH_WARNINGS +DISABLE_GCC_WARNING(uninitialized) BlockchainAndPool(): blockchain(tx_pool), tx_pool(blockchain) {} +POP_WARNINGS }; } diff --git a/src/cryptonote_core/tx_pool.h b/src/cryptonote_core/tx_pool.h index 47268efb6..3bb96d3a8 100644 --- a/src/cryptonote_core/tx_pool.h +++ b/src/cryptonote_core/tx_pool.h @@ -676,7 +676,7 @@ private: //! Next timestamp that a DB check for relayable txes is allowed std::atomic<time_t> m_next_check; - friend class BlockchainAndPool; + friend struct BlockchainAndPool; }; } diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.h b/src/cryptonote_protocol/cryptonote_protocol_handler.h index a29ee8287..80582fad2 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.h @@ -39,6 +39,7 @@ #include "byte_slice.h" #include "math_helper.h" +#include "syncobj.h" #include "storages/levin_abstract_invoke2.h" #include "warnings.h" #include "cryptonote_protocol_defs.h" diff --git a/src/device/device_ledger.cpp b/src/device/device_ledger.cpp index 27d080a1c..009ee16ca 100644 --- a/src/device/device_ledger.cpp +++ b/src/device/device_ledger.cpp @@ -34,9 +34,6 @@ #include "cryptonote_basic/subaddress_index.h" #include "cryptonote_core/cryptonote_tx_utils.h" -#include <boost/thread/locks.hpp> -#include <boost/thread/lock_guard.hpp> - namespace hw { namespace ledger { @@ -322,10 +319,10 @@ namespace hw { //automatic lock one more level on device ensuring the current thread is allowed to use it #define AUTO_LOCK_CMD() \ /* lock both mutexes without deadlock*/ \ - boost::lock(device_locker, command_locker); \ + std::lock(device_locker, command_locker); \ /* make sure both already-locked mutexes are unlocked at the end of scope */ \ - boost::lock_guard<boost::recursive_mutex> lock1(device_locker, boost::adopt_lock); \ - boost::lock_guard<boost::mutex> lock2(command_locker, boost::adopt_lock) + std::lock_guard<std::recursive_mutex> lock1(device_locker, std::adopt_lock); \ + std::lock_guard<std::mutex> lock2(command_locker, std::adopt_lock) //lock the device for a long sequence void device_ledger::lock(void) { diff --git a/src/device/device_ledger.hpp b/src/device/device_ledger.hpp index 071274160..b65943f0c 100644 --- a/src/device/device_ledger.hpp +++ b/src/device/device_ledger.hpp @@ -35,8 +35,7 @@ #include "device.hpp" #include "log.hpp" #include "device_io_hid.hpp" -#include <boost/thread/mutex.hpp> -#include <boost/thread/recursive_mutex.hpp> +#include <mutex> namespace hw { @@ -140,8 +139,8 @@ namespace hw { class device_ledger : public hw::device { private: // Locker for concurrent access - mutable boost::recursive_mutex device_locker; - mutable boost::mutex command_locker; + mutable std::recursive_mutex device_locker; + mutable std::mutex command_locker; //IO hw::io::device_io_hid hw_device; diff --git a/src/device_trezor/CMakeLists.txt b/src/device_trezor/CMakeLists.txt index c30fb2b16..2d5614507 100644 --- a/src/device_trezor/CMakeLists.txt +++ b/src/device_trezor/CMakeLists.txt @@ -65,12 +65,16 @@ set(trezor_private_headers) # Protobuf and LibUSB processed by CheckTrezor if(DEVICE_TREZOR_READY) - message(STATUS "Trezor support enabled") + message(STATUS "Trezor: support enabled") if(USE_DEVICE_TREZOR_DEBUG) list(APPEND trezor_headers trezor/debug_link.hpp trezor/messages/messages-debug.pb.h) list(APPEND trezor_sources trezor/debug_link.cpp trezor/messages/messages-debug.pb.cc) - message(STATUS "Trezor debugging enabled") + message(STATUS "Trezor: debugging enabled") + endif() + + if(ANDROID) + set(TREZOR_EXTRA_LIBRARIES "log") endif() monero_private_headers(device_trezor @@ -93,10 +97,11 @@ if(DEVICE_TREZOR_READY) ${Protobuf_LIBRARY} ${TREZOR_LIBUSB_LIBRARIES} PRIVATE - ${EXTRA_LIBRARIES}) + ${EXTRA_LIBRARIES} + ${TREZOR_EXTRA_LIBRARIES}) else() - message(STATUS "Trezor support disabled") + message(STATUS "Trezor: support disabled") monero_private_headers(device_trezor) monero_add_library(device_trezor device_trezor.cpp) target_link_libraries(device_trezor PUBLIC cncrypto) diff --git a/src/device_trezor/README.md b/src/device_trezor/README.md new file mode 100644 index 000000000..ce08c0009 --- /dev/null +++ b/src/device_trezor/README.md @@ -0,0 +1,74 @@ +# Trezor hardware wallet support + +This module adds [Trezor] hardware support to Monero. + + +## Basic information + +Trezor integration is based on the following original proposal: https://github.com/ph4r05/monero-trezor-doc + +A custom high-level transaction signing protocol uses Trezor in a similar way a cold wallet is used. +Transaction is build incrementally on the device. + +Trezor implements the signing protocol in [trezor-firmware] repository, in the [monero](https://github.com/trezor/trezor-firmware/tree/master/core/src/apps/monero) application. +Please, refer to [monero readme](https://github.com/trezor/trezor-firmware/blob/master/core/src/apps/monero/README.md) for more information. + +## Dependencies + +Trezor uses [Protobuf](https://protobuf.dev/) library. As Monero is compiled with C++14, the newest Protobuf library version cannot be compiled because it requires C++17 (through its dependency Abseil library). +This can result in a compilation failure. + +Protobuf v21 is the latest compatible protobuf version. + +If you want to compile Monero with Trezor support, please make sure the Protobuf v21 is installed. + +More about this limitation: [PR #8752](https://github.com/monero-project/monero/pull/8752), +[1](https://github.com/monero-project/monero/pull/8752#discussion_r1246174755), [2](https://github.com/monero-project/monero/pull/8752#discussion_r1246480393) + +### OSX + +To build with installed, but not linked protobuf: + +```bash +CMAKE_PREFIX_PATH=$(find /opt/homebrew/Cellar/protobuf@21 -maxdepth 1 -type d -name "21.*" -print -quit) \ +make release +``` + +or to install and link as a default protobuf version: +```bash +# Either install all requirements as +brew update && brew bundle --file=contrib/brew/Brewfile + +# or install protobufv21 specifically +brew install protobuf@21 && brew link protobuf@21 +``` + +### MSYS32 + +```bash +curl -O https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-protobuf-c-1.4.1-1-any.pkg.tar.zst +curl -O https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-protobuf-21.9-1-any.pkg.tar.zst +pacman --noconfirm -U mingw-w64-x86_64-protobuf-c-1.4.1-1-any.pkg.tar.zst mingw-w64-x86_64-protobuf-21.9-1-any.pkg.tar.zst +``` + +### Other systems + +- install protobufv21 +- point `CMAKE_PREFIX_PATH` environment variable to Protobuf v21 installation. + +## Troubleshooting + +To disable Trezor support, set `USE_DEVICE_TREZOR=OFF`, e.g.: + +```shell +USE_DEVICE_TREZOR=OFF make release +``` + +## Resources: + +- First pull request https://github.com/monero-project/monero/pull/4241 +- Integration proposal https://github.com/ph4r05/monero-trezor-doc +- Integration readme in trezor-firmware https://github.com/trezor/trezor-firmware/blob/master/core/src/apps/monero/README.md + +[Trezor]: https://trezor.io/ +[trezor-firmware]: https://github.com/trezor/trezor-firmware/
\ No newline at end of file diff --git a/src/device_trezor/device_trezor.cpp b/src/device_trezor/device_trezor.cpp index 9c8148ed6..fa1e7c088 100644 --- a/src/device_trezor/device_trezor.cpp +++ b/src/device_trezor/device_trezor.cpp @@ -165,7 +165,7 @@ namespace trezor { auto res = get_address(); cryptonote::address_parse_info info{}; - bool r = cryptonote::get_account_address_from_str(info, this->network_type, res->address()); + bool r = cryptonote::get_account_address_from_str(info, this->m_network_type, res->address()); CHECK_AND_ASSERT_MES(r, false, "Could not parse returned address. Address parse failed: " + res->address()); CHECK_AND_ASSERT_MES(!info.is_subaddress, false, "Trezor returned a sub address"); @@ -693,14 +693,11 @@ namespace trezor { unsigned device_trezor::client_version() { auto trezor_version = get_version(); - if (trezor_version < pack_version(2, 4, 3)){ - throw exc::TrezorException("Minimal Trezor firmware version is 2.4.3. Please update."); + if (trezor_version < pack_version(2, 5, 2)){ + throw exc::TrezorException("Minimal Trezor firmware version is 2.5.2. Please update."); } - unsigned client_version = 3; - if (trezor_version >= pack_version(2, 5, 2)){ - client_version = 4; - } + unsigned client_version = 4; // since 2.5.2 #ifdef WITH_TREZOR_DEBUGGING // Override client version for tests diff --git a/src/device_trezor/device_trezor.hpp b/src/device_trezor/device_trezor.hpp index 35bb78927..804ed62c8 100644 --- a/src/device_trezor/device_trezor.hpp +++ b/src/device_trezor/device_trezor.hpp @@ -102,7 +102,7 @@ namespace trezor { bool has_ki_cold_sync() const override { return true; } bool has_tx_cold_sign() const override { return true; } - void set_network_type(cryptonote::network_type network_type) override { this->network_type = network_type; } + void set_network_type(cryptonote::network_type network_type) override { this->m_network_type = network_type; } void set_live_refresh_enabled(bool enabled) { m_live_refresh_enabled = enabled; } bool live_refresh_enabled() const { return m_live_refresh_enabled; } diff --git a/src/device_trezor/device_trezor_base.cpp b/src/device_trezor/device_trezor_base.cpp index a8a3d9f67..f65870be5 100644 --- a/src/device_trezor/device_trezor_base.cpp +++ b/src/device_trezor/device_trezor_base.cpp @@ -300,9 +300,6 @@ namespace trezor { case messages::MessageType_PassphraseRequest: on_passphrase_request(input, dynamic_cast<const messages::common::PassphraseRequest*>(input.m_msg.get())); return true; - case messages::MessageType_Deprecated_PassphraseStateRequest: - on_passphrase_state_request(input, dynamic_cast<const messages::common::Deprecated_PassphraseStateRequest*>(input.m_msg.get())); - return true; case messages::MessageType_PinMatrixRequest: on_pin_request(input, dynamic_cast<const messages::common::PinMatrixRequest*>(input.m_msg.get())); return true; @@ -475,21 +472,9 @@ namespace trezor { CHECK_AND_ASSERT_THROW_MES(msg, "Empty message"); MDEBUG("on_passhprase_request"); - // Backward compatibility, migration clause. - if (msg->has__on_device() && msg->_on_device()){ - messages::common::PassphraseAck m; - resp = call_raw(&m); - return; - } - m_seen_passphrase_entry_message = true; - bool on_device = true; - if (msg->has__on_device() && !msg->_on_device()){ - on_device = false; // do not enter on device, old devices. - } - - if (on_device && m_features && m_features->capabilities_size() > 0){ - on_device = false; + bool on_device = false; + if (m_features){ for (auto it = m_features->capabilities().begin(); it != m_features->capabilities().end(); it++) { if (*it == messages::management::Features::Capability_PassphraseEntry){ on_device = true; @@ -526,18 +511,6 @@ namespace trezor { resp = call_raw(&m); } - void device_trezor_base::on_passphrase_state_request(GenericMessage & resp, const messages::common::Deprecated_PassphraseStateRequest * msg) - { - MDEBUG("on_passhprase_state_request"); - CHECK_AND_ASSERT_THROW_MES(msg, "Empty message"); - - if (msg->has_state()) { - m_device_session_id = msg->state(); - } - messages::common::Deprecated_PassphraseStateAck m; - resp = call_raw(&m); - } - #ifdef WITH_TREZOR_DEBUGGING void device_trezor_base::wipe_device() { diff --git a/src/device_trezor/device_trezor_base.hpp b/src/device_trezor/device_trezor_base.hpp index 3ec21e157..e0fc2aafb 100644 --- a/src/device_trezor/device_trezor_base.hpp +++ b/src/device_trezor/device_trezor_base.hpp @@ -32,13 +32,12 @@ #include <cstddef> +#include <mutex> #include <string> #include "device/device.hpp" #include "device/device_default.hpp" #include "device/device_cold.hpp" #include <boost/scope_exit.hpp> -#include <boost/thread/mutex.hpp> -#include <boost/thread/recursive_mutex.hpp> #include "cryptonote_config.h" #include "trezor.hpp" @@ -49,12 +48,12 @@ //automatic lock one more level on device ensuring the current thread is allowed to use it #define TREZOR_AUTO_LOCK_CMD() \ /* lock both mutexes without deadlock*/ \ - boost::lock(device_locker, command_locker); \ + std::lock(device_locker, command_locker); \ /* make sure both already-locked mutexes are unlocked at the end of scope */ \ - boost::lock_guard<boost::recursive_mutex> lock1(device_locker, boost::adopt_lock); \ - boost::lock_guard<boost::mutex> lock2(command_locker, boost::adopt_lock) + std::lock_guard<std::recursive_mutex> lock1(device_locker, std::adopt_lock); \ + std::lock_guard<std::mutex> lock2(command_locker, std::adopt_lock) -#define TREZOR_AUTO_LOCK_DEVICE() boost::lock_guard<boost::recursive_mutex> lock1_device(device_locker) +#define TREZOR_AUTO_LOCK_DEVICE() std::lock_guard<std::recursive_mutex> lock1_device(device_locker) namespace hw { namespace trezor { @@ -86,8 +85,8 @@ namespace trezor { protected: // Locker for concurrent access - mutable boost::recursive_mutex device_locker; - mutable boost::mutex command_locker; + mutable std::recursive_mutex device_locker; + mutable std::mutex command_locker; std::shared_ptr<Transport> m_transport; i_device_callback * m_callback; @@ -100,7 +99,7 @@ namespace trezor { boost::optional<epee::wipeable_string> m_passphrase; messages::MessageType m_last_msg_type; - cryptonote::network_type network_type; + cryptonote::network_type m_network_type; bool m_reply_with_empty_passphrase; bool m_always_use_empty_passphrase; bool m_seen_passphrase_entry_message; @@ -227,9 +226,9 @@ namespace trezor { } if (network_type){ - msg->set_network_type(static_cast<uint32_t>(network_type.get())); + msg->set_network_type(static_cast<messages::monero::MoneroNetworkType>(network_type.get())); } else { - msg->set_network_type(static_cast<uint32_t>(this->network_type)); + msg->set_network_type(static_cast<messages::monero::MoneroNetworkType>(this->m_network_type)); } } @@ -318,7 +317,6 @@ namespace trezor { void on_button_pressed(); void on_pin_request(GenericMessage & resp, const messages::common::PinMatrixRequest * msg); void on_passphrase_request(GenericMessage & resp, const messages::common::PassphraseRequest * msg); - void on_passphrase_state_request(GenericMessage & resp, const messages::common::Deprecated_PassphraseStateRequest * msg); #ifdef WITH_TREZOR_DEBUGGING void set_debug(bool debug){ diff --git a/src/device_trezor/trezor/debug_link.cpp b/src/device_trezor/trezor/debug_link.cpp index ff7cf0ad6..76adcb164 100644 --- a/src/device_trezor/trezor/debug_link.cpp +++ b/src/device_trezor/trezor/debug_link.cpp @@ -67,7 +67,7 @@ namespace trezor{ void DebugLink::input_button(bool button){ messages::debug::DebugLinkDecision decision; - decision.set_yes_no(button); + decision.set_button(button ? messages::debug::DebugLinkDecision_DebugButton_YES : messages::debug::DebugLinkDecision_DebugButton_NO); call(decision, boost::none, true); } diff --git a/src/device_trezor/trezor/transport.cpp b/src/device_trezor/trezor/transport.cpp index 8c6b30787..23a04f9c7 100644 --- a/src/device_trezor/trezor/transport.cpp +++ b/src/device_trezor/trezor/transport.cpp @@ -154,6 +154,7 @@ namespace trezor{ // Helpers // +#define PROTO_MAGIC_SIZE 3 #define PROTO_HEADER_SIZE 6 static size_t message_size(const google::protobuf::Message &req){ @@ -193,7 +194,7 @@ namespace trezor{ } serialize_message_header(buff, msg_wire_num, msg_size); - if (!req.SerializeToArray(buff + 6, msg_size)){ + if (!req.SerializeToArray(buff + PROTO_HEADER_SIZE, msg_size)){ throw exc::EncodingException("Message serialization error"); } } @@ -252,16 +253,16 @@ namespace trezor{ throw exc::CommunicationException("Read chunk has invalid size"); } - if (memcmp(chunk_buff_raw, "?##", 3) != 0){ + if (memcmp(chunk_buff_raw, "?##", PROTO_MAGIC_SIZE) != 0){ throw exc::CommunicationException("Malformed chunk"); } uint16_t tag; uint32_t len; - nread -= 3 + 6; - deserialize_message_header(chunk_buff_raw + 3, tag, len); + nread -= PROTO_MAGIC_SIZE + PROTO_HEADER_SIZE; + deserialize_message_header(chunk_buff_raw + PROTO_MAGIC_SIZE, tag, len); - epee::wipeable_string data_acc(chunk_buff_raw + 3 + 6, nread); + epee::wipeable_string data_acc(chunk_buff_raw + PROTO_MAGIC_SIZE + PROTO_HEADER_SIZE, nread); data_acc.reserve(len); while(nread < len){ @@ -482,7 +483,7 @@ namespace trezor{ uint16_t msg_tag; uint32_t msg_len; deserialize_message_header(bin_data->data(), msg_tag, msg_len); - if (bin_data->size() != msg_len + 6){ + if (bin_data->size() != msg_len + PROTO_HEADER_SIZE){ throw exc::CommunicationException("Response is not well hexcoded"); } @@ -491,7 +492,7 @@ namespace trezor{ } std::shared_ptr<google::protobuf::Message> msg_wrap(MessageMapper::get_message(msg_tag)); - if (!msg_wrap->ParseFromArray(bin_data->data() + 6, msg_len)){ + if (!msg_wrap->ParseFromArray(bin_data->data() + PROTO_HEADER_SIZE, msg_len)){ throw exc::EncodingException("Response is not well hexcoded"); } msg = msg_wrap; diff --git a/src/multisig/multisig_kex_msg.cpp b/src/multisig/multisig_kex_msg.cpp index 49350948a..321305283 100644 --- a/src/multisig/multisig_kex_msg.cpp +++ b/src/multisig/multisig_kex_msg.cpp @@ -206,8 +206,13 @@ namespace multisig //---------------------------------------------------------------------------------------------------------------------- void multisig_kex_msg::parse_and_validate_msg() { + CHECK_AND_ASSERT_THROW_MES(MULTISIG_KEX_MSG_V2_MAGIC_1.size() == MULTISIG_KEX_MSG_V2_MAGIC_N.size(), + "Multisig kex msg magic inconsistency."); + CHECK_AND_ASSERT_THROW_MES(MULTISIG_KEX_MSG_V2_MAGIC_1.size() >= MULTISIG_KEX_V1_MAGIC.size(), + "Multisig kex msg magic inconsistency."); + // check message type - CHECK_AND_ASSERT_THROW_MES(m_msg.size() > 0, "Kex message unexpectedly empty."); + CHECK_AND_ASSERT_THROW_MES(m_msg.size() >= MULTISIG_KEX_MSG_V2_MAGIC_1.size(), "Kex message unexpectedly small."); CHECK_AND_ASSERT_THROW_MES(m_msg.substr(0, MULTISIG_KEX_V1_MAGIC.size()) != MULTISIG_KEX_V1_MAGIC, "V1 multisig kex messages are deprecated (unsafe)."); CHECK_AND_ASSERT_THROW_MES(m_msg.substr(0, MULTISIG_KEX_MSG_V1_MAGIC.size()) != MULTISIG_KEX_MSG_V1_MAGIC, @@ -215,8 +220,6 @@ namespace multisig // deserialize the message std::string msg_no_magic; - CHECK_AND_ASSERT_THROW_MES(MULTISIG_KEX_MSG_V2_MAGIC_1.size() == MULTISIG_KEX_MSG_V2_MAGIC_N.size(), - "Multisig kex msg magic inconsistency."); CHECK_AND_ASSERT_THROW_MES(tools::base58::decode(m_msg.substr(MULTISIG_KEX_MSG_V2_MAGIC_1.size()), msg_no_magic), "Multisig kex msg decoding error."); binary_archive<false> b_archive{epee::strspan<std::uint8_t>(msg_no_magic)}; diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index 4f77ce834..815c1b354 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -51,7 +51,6 @@ #include "common/dns_utils.h" #include "common/pruning.h" #include "net/error.h" -#include "math_helper.h" #include "misc_log_ex.h" #include "p2p_protocol_defs.h" #include "crypto/crypto.h" diff --git a/src/ringct/rctTypes.h b/src/ringct/rctTypes.h index 380ca2f53..585d5fb49 100644 --- a/src/ringct/rctTypes.h +++ b/src/ringct/rctTypes.h @@ -218,7 +218,7 @@ namespace rct { rct::key a, b, t; Bulletproof(): - A({}), S({}), T1({}), T2({}), taux({}), mu({}), a({}), b({}), t({}) {} + A({}), S({}), T1({}), T2({}), taux({}), mu({}), a({}), b({}), t({}), V({}), L({}), R({}) {} Bulletproof(const rct::key &V, const rct::key &A, const rct::key &S, const rct::key &T1, const rct::key &T2, const rct::key &taux, const rct::key &mu, const rct::keyV &L, const rct::keyV &R, const rct::key &a, const rct::key &b, const rct::key &t): V({V}), A(A), S(S), T1(T1), T2(T2), taux(taux), mu(mu), L(L), R(R), a(a), b(b), t(t) {} Bulletproof(const rct::keyV &V, const rct::key &A, const rct::key &S, const rct::key &T1, const rct::key &T2, const rct::key &taux, const rct::key &mu, const rct::keyV &L, const rct::keyV &R, const rct::key &a, const rct::key &b, const rct::key &t): @@ -253,7 +253,7 @@ namespace rct { rct::key r1, s1, d1; rct::keyV L, R; - BulletproofPlus() {} + BulletproofPlus(): V(), A(), A1(), B(), r1(), s1(), d1(), L(), R() {} BulletproofPlus(const rct::key &V, const rct::key &A, const rct::key &A1, const rct::key &B, const rct::key &r1, const rct::key &s1, const rct::key &d1, const rct::keyV &L, const rct::keyV &R): V({V}), A(A), A1(A1), B(B), r1(r1), s1(s1), d1(d1), L(L), R(R) {} BulletproofPlus(const rct::keyV &V, const rct::key &A, const rct::key &A1, const rct::key &B, const rct::key &r1, const rct::key &s1, const rct::key &d1, const rct::keyV &L, const rct::keyV &R): @@ -362,11 +362,17 @@ namespace rct { { if (type == RCTTypeBulletproof2 || type == RCTTypeCLSAG || type == RCTTypeBulletproofPlus) { + // Since RCTTypeBulletproof2 enote types, we don't serialize the blinding factor, and only serialize the + // first 8 bytes of ecdhInfo[i].amount ar.begin_object(); - if (!typename Archive<W>::is_saving()) + crypto::hash8 trunc_amount; // placeholder variable needed to maintain "strict aliasing" + if (!typename Archive<W>::is_saving()) // loading memset(ecdhInfo[i].amount.bytes, 0, sizeof(ecdhInfo[i].amount.bytes)); - crypto::hash8 &amount = (crypto::hash8&)ecdhInfo[i].amount; - FIELD(amount); + else // saving + memcpy(trunc_amount.data, ecdhInfo[i].amount.bytes, sizeof(trunc_amount)); + FIELD(trunc_amount); + if (!typename Archive<W>::is_saving()) // loading + memcpy(ecdhInfo[i].amount.bytes, trunc_amount.data, sizeof(trunc_amount)); ar.end_object(); } else diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index 03e9ec494..9a0b02f70 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -2082,11 +2082,12 @@ namespace cryptonote } crypto::hash merkle_root; - size_t merkle_tree_depth = 0; std::vector<std::pair<crypto::hash, crypto::hash>> aux_pow; std::vector<crypto::hash> aux_pow_raw; + std::vector<crypto::hash> aux_pow_id_raw; aux_pow.reserve(req.aux_pow.size()); - aux_pow_raw.reserve(req.aux_pow.size()); + aux_pow_raw.resize(req.aux_pow.size()); + aux_pow_id_raw.resize(req.aux_pow.size()); for (const auto &s: req.aux_pow) { aux_pow.push_back({}); @@ -2102,7 +2103,6 @@ namespace cryptonote error_resp.message = "Invalid aux pow hash"; return false; } - aux_pow_raw.push_back(aux_pow.back().second); } size_t path_domain = 1; @@ -2111,11 +2111,14 @@ namespace cryptonote uint32_t nonce; const uint32_t max_nonce = 65535; bool collision = true; + std::vector<uint32_t> slots(aux_pow.size()); for (nonce = 0; nonce <= max_nonce; ++nonce) { - std::vector<bool> slots(aux_pow.size(), false); + std::vector<bool> slot_seen(aux_pow.size(), false); collision = false; for (size_t idx = 0; idx < aux_pow.size(); ++idx) + slots[idx] = 0xffffffff; + for (size_t idx = 0; idx < aux_pow.size(); ++idx) { const uint32_t slot = cryptonote::get_aux_slot(aux_pow[idx].first, nonce, aux_pow.size()); if (slot >= aux_pow.size()) @@ -2124,12 +2127,13 @@ namespace cryptonote error_resp.message = "Computed slot is out of range"; return false; } - if (slots[slot]) + if (slot_seen[slot]) { collision = true; break; } - slots[slot] = true; + slot_seen[slot] = true; + slots[idx] = slot; } if (!collision) break; @@ -2141,6 +2145,19 @@ namespace cryptonote return false; } + // set the order determined above + for (size_t i = 0; i < aux_pow.size(); ++i) + { + if (slots[i] >= aux_pow.size()) + { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = "Slot value out of range"; + return false; + } + aux_pow_raw[slots[i]] = aux_pow[i].second; + aux_pow_id_raw[slots[i]] = aux_pow[i].first; + } + crypto::tree_hash((const char(*)[crypto::HASH_SIZE])aux_pow_raw.data(), aux_pow_raw.size(), merkle_root.data); res.merkle_root = epee::string_tools::pod_to_hex(merkle_root); res.merkle_tree_depth = cryptonote::encode_mm_depth(aux_pow.size(), nonce); @@ -2167,7 +2184,7 @@ namespace cryptonote error_resp.message = "Error removing existing merkle root"; return false; } - if (!add_mm_merkle_root_to_tx_extra(b.miner_tx.extra, merkle_root, merkle_tree_depth)) + if (!add_mm_merkle_root_to_tx_extra(b.miner_tx.extra, merkle_root, res.merkle_tree_depth)) { error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; error_resp.message = "Error adding merkle root"; @@ -2181,7 +2198,8 @@ namespace cryptonote res.blocktemplate_blob = string_tools::buff_to_hex_nodelimer(block_blob); res.blockhashing_blob = string_tools::buff_to_hex_nodelimer(hashing_blob); - res.aux_pow = req.aux_pow; + for (size_t i = 0; i < aux_pow_raw.size(); ++i) + res.aux_pow.push_back({epee::string_tools::pod_to_hex(aux_pow_id_raw[i]), epee::string_tools::pod_to_hex(aux_pow_raw[i])}); res.status = CORE_RPC_STATUS_OK; return true; } @@ -2358,6 +2376,7 @@ namespace cryptonote bool core_rpc_server::use_bootstrap_daemon_if_necessary(const invoke_http_mode &mode, const std::string &command_name, const typename COMMAND_TYPE::request& req, typename COMMAND_TYPE::response& res, bool &r) { res.untrusted = false; + r = false; boost::upgrade_lock<boost::shared_mutex> upgrade_lock(m_bootstrap_daemon_mutex); @@ -3535,6 +3554,82 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_get_txids_loose(const COMMAND_RPC_GET_TXIDS_LOOSE::request& req, COMMAND_RPC_GET_TXIDS_LOOSE::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) + { + RPC_TRACKER(get_txids_loose); + + // Maybe don't use bootstrap since this endpoint is meant to retreive TXIDs w/ k-anonymity, + // so shunting this request to a random node seems counterproductive. + +#if BYTE_ORDER == LITTLE_ENDIAN + const uint64_t max_num_txids = RESTRICTED_SPENT_KEY_IMAGES_COUNT * (m_restricted ? 1 : 10); + + // Sanity check parameters + if (req.num_matching_bits > 256) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_PARAM; + error_resp.message = "There are only 256 bits in a hash, you gave too many"; + return false; + } + + // Attempt to guess when bit count is too low before fetching, within a certain margin of error + const uint64_t num_txs_ever = m_core.get_blockchain_storage().get_db().get_tx_count(); + const uint64_t num_expected_fetch = (num_txs_ever >> std::min((int) req.num_matching_bits, 63)); + const uint64_t max_num_txids_with_margin = 2 * max_num_txids; + if (num_expected_fetch > max_num_txids_with_margin) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_PARAM; + error_resp.message = "Trying to search with too few matching bits, detected before fetching"; + return false; + } + + // Convert txid template to a crypto::hash + crypto::hash search_hash; + if (!epee::string_tools::hex_to_pod(req.txid_template, search_hash)) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_PARAM; + error_resp.message = "Could not decode hex txid"; + return false; + } + // Check that txid template is zeroed correctly for number of given matchign bits + else if (search_hash != make_hash32_loose_template(req.num_matching_bits, search_hash)) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_PARAM; + error_resp.message = "Txid template is not zeroed correctly for number of bits. You could be leaking true txid!"; + return false; + } + + try + { + // Do the DB fetch + const auto txids = m_core.get_blockchain_storage().get_db().get_txids_loose(search_hash, req.num_matching_bits, max_num_txids); + // Fill out response form + for (const auto& txid : txids) + res.txids.emplace_back(epee::string_tools::pod_to_hex(txid)); + } + catch (const TX_EXISTS&) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_PARAM; + error_resp.message = "Trying to search with too few matching bits"; + return false; + } + catch (const std::exception& e) + { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = std::string("Error during get_txids_loose: ") + e.what(); + return false; + } + + res.status = CORE_RPC_STATUS_OK; + return true; +#else // BYTE_ORDER == BIG_ENDIAN + // BlockchainLMDB::compare_hash32 has different key ordering (thus different txid templates) on BE systems + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = "Due to implementation details, this feature is not available on big-endian daemons"; + return false; +#endif + } + //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_rpc_access_submit_nonce(const COMMAND_RPC_ACCESS_SUBMIT_NONCE::request& req, COMMAND_RPC_ACCESS_SUBMIT_NONCE::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { RPC_TRACKER(rpc_access_submit_nonce); diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index 790d5eb23..7c31d2539 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -178,6 +178,7 @@ namespace cryptonote MAP_JON_RPC_WE("get_output_distribution", on_get_output_distribution, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION) MAP_JON_RPC_WE_IF("prune_blockchain", on_prune_blockchain, COMMAND_RPC_PRUNE_BLOCKCHAIN, !m_restricted) MAP_JON_RPC_WE_IF("flush_cache", on_flush_cache, COMMAND_RPC_FLUSH_CACHE, !m_restricted) + MAP_JON_RPC_WE("get_txids_loose", on_get_txids_loose, COMMAND_RPC_GET_TXIDS_LOOSE) MAP_JON_RPC_WE("rpc_access_info", on_rpc_access_info, COMMAND_RPC_ACCESS_INFO) MAP_JON_RPC_WE("rpc_access_submit_nonce",on_rpc_access_submit_nonce, COMMAND_RPC_ACCESS_SUBMIT_NONCE) MAP_JON_RPC_WE("rpc_access_pay", on_rpc_access_pay, COMMAND_RPC_ACCESS_PAY) @@ -255,6 +256,7 @@ namespace cryptonote bool on_get_output_distribution(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_prune_blockchain(const COMMAND_RPC_PRUNE_BLOCKCHAIN::request& req, COMMAND_RPC_PRUNE_BLOCKCHAIN::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_flush_cache(const COMMAND_RPC_FLUSH_CACHE::request& req, COMMAND_RPC_FLUSH_CACHE::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_get_txids_loose(const COMMAND_RPC_GET_TXIDS_LOOSE::request& req, COMMAND_RPC_GET_TXIDS_LOOSE::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_rpc_access_info(const COMMAND_RPC_ACCESS_INFO::request& req, COMMAND_RPC_ACCESS_INFO::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_rpc_access_submit_nonce(const COMMAND_RPC_ACCESS_SUBMIT_NONCE::request& req, COMMAND_RPC_ACCESS_SUBMIT_NONCE::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_rpc_access_pay(const COMMAND_RPC_ACCESS_PAY::request& req, COMMAND_RPC_ACCESS_PAY::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index 819d77c1f..2a0a6201d 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -617,7 +617,7 @@ namespace cryptonote bool do_sanity_checks; BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE_PARENT(rpc_access_request_base); + KV_SERIALIZE_PARENT(rpc_access_request_base) KV_SERIALIZE(tx_as_hex) KV_SERIALIZE_OPT(do_not_relay, false) KV_SERIALIZE_OPT(do_sanity_checks, true) @@ -693,7 +693,7 @@ namespace cryptonote struct request_t: public rpc_access_request_base { BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE_PARENT(rpc_access_request_base); + KV_SERIALIZE_PARENT(rpc_access_request_base) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<request_t> request; @@ -1094,7 +1094,7 @@ namespace cryptonote blobdata blocktemplate_blob; blobdata blockhashing_blob; std::string merkle_root; - uint32_t merkle_tree_depth; + uint64_t merkle_tree_depth; std::vector<aux_pow_t> aux_pow; BEGIN_KV_SERIALIZE_MAP() @@ -1217,7 +1217,7 @@ namespace cryptonote BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE_PARENT(rpc_access_request_base) - KV_SERIALIZE_OPT(fill_pow_hash, false); + KV_SERIALIZE_OPT(fill_pow_hash, false) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<request_t> request; @@ -1247,7 +1247,7 @@ namespace cryptonote KV_SERIALIZE_PARENT(rpc_access_request_base) KV_SERIALIZE(hash) KV_SERIALIZE(hashes) - KV_SERIALIZE_OPT(fill_pow_hash, false); + KV_SERIALIZE_OPT(fill_pow_hash, false) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<request_t> request; @@ -1276,7 +1276,7 @@ namespace cryptonote BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE_PARENT(rpc_access_request_base) KV_SERIALIZE(height) - KV_SERIALIZE_OPT(fill_pow_hash, false); + KV_SERIALIZE_OPT(fill_pow_hash, false) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<request_t> request; @@ -1305,7 +1305,7 @@ namespace cryptonote KV_SERIALIZE_PARENT(rpc_access_request_base) KV_SERIALIZE(hash) KV_SERIALIZE(height) - KV_SERIALIZE_OPT(fill_pow_hash, false); + KV_SERIALIZE_OPT(fill_pow_hash, false) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<request_t> request; @@ -1763,7 +1763,7 @@ namespace cryptonote KV_SERIALIZE_PARENT(rpc_access_request_base) KV_SERIALIZE(start_height) KV_SERIALIZE(end_height) - KV_SERIALIZE_OPT(fill_pow_hash, false); + KV_SERIALIZE_OPT(fill_pow_hash, false) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<request_t> request; @@ -2124,12 +2124,12 @@ namespace cryptonote uint64_t recent_cutoff; BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE_PARENT(rpc_access_request_base); - KV_SERIALIZE(amounts); - KV_SERIALIZE(min_count); - KV_SERIALIZE(max_count); - KV_SERIALIZE(unlocked); - KV_SERIALIZE(recent_cutoff); + KV_SERIALIZE_PARENT(rpc_access_request_base) + KV_SERIALIZE(amounts) + KV_SERIALIZE(min_count) + KV_SERIALIZE(max_count) + KV_SERIALIZE(unlocked) + KV_SERIALIZE(recent_cutoff) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<request_t> request; @@ -2142,10 +2142,10 @@ namespace cryptonote uint64_t recent_instances; BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(amount); - KV_SERIALIZE(total_instances); - KV_SERIALIZE(unlocked_instances); - KV_SERIALIZE(recent_instances); + KV_SERIALIZE(amount) + KV_SERIALIZE(total_instances) + KV_SERIALIZE(unlocked_instances) + KV_SERIALIZE(recent_instances) END_KV_SERIALIZE_MAP() entry(uint64_t amount, uint64_t total_instances, uint64_t unlocked_instances, uint64_t recent_instances): @@ -2216,9 +2216,9 @@ namespace cryptonote uint64_t count; BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE_PARENT(rpc_access_request_base); - KV_SERIALIZE(height); - KV_SERIALIZE(count); + KV_SERIALIZE_PARENT(rpc_access_request_base) + KV_SERIALIZE(height) + KV_SERIALIZE(count) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<request_t> request; @@ -2793,4 +2793,31 @@ namespace cryptonote typedef epee::misc_utils::struct_init<response_t> response; }; + struct COMMAND_RPC_GET_TXIDS_LOOSE + { + struct request_t: public rpc_request_base + { + std::string txid_template; + std::uint32_t num_matching_bits; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_PARENT(rpc_request_base) + KV_SERIALIZE(txid_template) + KV_SERIALIZE(num_matching_bits) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<request_t> request; + + struct response_t: public rpc_response_base + { + std::vector<std::string> txids; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_PARENT(rpc_response_base) + KV_SERIALIZE(txids) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<response_t> response; + }; + } diff --git a/src/rpc/daemon_messages.cpp b/src/rpc/daemon_messages.cpp index 0da22f15f..36528fcea 100644 --- a/src/rpc/daemon_messages.cpp +++ b/src/rpc/daemon_messages.cpp @@ -48,6 +48,11 @@ void GetHeight::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) c void GetHeight::Response::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, height, height); } @@ -61,6 +66,11 @@ void GetBlocksFast::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest void GetBlocksFast::Request::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, block_ids, block_ids); GET_FROM_JSON_OBJECT(val, start_height, start_height); GET_FROM_JSON_OBJECT(val, prune, prune); @@ -76,6 +86,11 @@ void GetBlocksFast::Response::doToJson(rapidjson::Writer<epee::byte_stream>& des void GetBlocksFast::Response::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, blocks, blocks); GET_FROM_JSON_OBJECT(val, start_height, start_height); GET_FROM_JSON_OBJECT(val, current_height, current_height); @@ -91,6 +106,11 @@ void GetHashesFast::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest void GetHashesFast::Request::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, known_hashes, known_hashes); GET_FROM_JSON_OBJECT(val, start_height, start_height); } @@ -100,10 +120,16 @@ void GetHashesFast::Response::doToJson(rapidjson::Writer<epee::byte_stream>& des INSERT_INTO_JSON_OBJECT(dest, hashes, hashes); INSERT_INTO_JSON_OBJECT(dest, start_height, start_height); INSERT_INTO_JSON_OBJECT(dest, current_height, current_height); + } void GetHashesFast::Response::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, hashes, hashes); GET_FROM_JSON_OBJECT(val, start_height, start_height); GET_FROM_JSON_OBJECT(val, current_height, current_height); @@ -117,6 +143,11 @@ void GetTransactions::Request::doToJson(rapidjson::Writer<epee::byte_stream>& de void GetTransactions::Request::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, tx_hashes, tx_hashes); } @@ -128,6 +159,11 @@ void GetTransactions::Response::doToJson(rapidjson::Writer<epee::byte_stream>& d void GetTransactions::Response::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, txs, txs); GET_FROM_JSON_OBJECT(val, missed_hashes, missed_hashes); } @@ -140,6 +176,11 @@ void KeyImagesSpent::Request::doToJson(rapidjson::Writer<epee::byte_stream>& des void KeyImagesSpent::Request::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, key_images, key_images); } @@ -150,6 +191,11 @@ void KeyImagesSpent::Response::doToJson(rapidjson::Writer<epee::byte_stream>& de void KeyImagesSpent::Response::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, spent_status, spent_status); } @@ -161,6 +207,11 @@ void GetTxGlobalOutputIndices::Request::doToJson(rapidjson::Writer<epee::byte_st void GetTxGlobalOutputIndices::Request::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, tx_hash, tx_hash); } @@ -171,6 +222,11 @@ void GetTxGlobalOutputIndices::Response::doToJson(rapidjson::Writer<epee::byte_s void GetTxGlobalOutputIndices::Response::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, output_indices, output_indices); } @@ -182,6 +238,11 @@ void SendRawTx::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) co void SendRawTx::Request::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, tx, tx); GET_FROM_JSON_OBJECT(val, relay, relay); } @@ -194,6 +255,11 @@ void SendRawTx::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) c void SendRawTx::Response::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, relayed, relayed); } @@ -205,6 +271,11 @@ void SendRawTxHex::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) void SendRawTxHex::Request::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, tx_as_hex, tx_as_hex); GET_FROM_JSON_OBJECT(val, relay, relay); } @@ -219,6 +290,11 @@ void StartMining::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) void StartMining::Request::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, miner_address, miner_address); GET_FROM_JSON_OBJECT(val, threads_count, threads_count); GET_FROM_JSON_OBJECT(val, do_background_mining, do_background_mining); @@ -266,6 +342,11 @@ void MiningStatus::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest void MiningStatus::Response::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, active, active); GET_FROM_JSON_OBJECT(val, speed, speed); GET_FROM_JSON_OBJECT(val, threads_count, threads_count); @@ -288,6 +369,11 @@ void GetInfo::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) con void GetInfo::Response::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, info, info); } @@ -314,6 +400,11 @@ void GetBlockHash::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) void GetBlockHash::Request::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, height, height); } @@ -324,6 +415,11 @@ void GetBlockHash::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest void GetBlockHash::Response::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, hash, hash); } @@ -342,6 +438,11 @@ void GetLastBlockHeader::Response::doToJson(rapidjson::Writer<epee::byte_stream> void GetLastBlockHeader::Response::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, header, header); } @@ -353,6 +454,11 @@ void GetBlockHeaderByHash::Request::doToJson(rapidjson::Writer<epee::byte_stream void GetBlockHeaderByHash::Request::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, hash, hash); } @@ -363,6 +469,11 @@ void GetBlockHeaderByHash::Response::doToJson(rapidjson::Writer<epee::byte_strea void GetBlockHeaderByHash::Response::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, header, header); } @@ -374,6 +485,11 @@ void GetBlockHeaderByHeight::Request::doToJson(rapidjson::Writer<epee::byte_stre void GetBlockHeaderByHeight::Request::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, height, height); } @@ -384,6 +500,11 @@ void GetBlockHeaderByHeight::Response::doToJson(rapidjson::Writer<epee::byte_str void GetBlockHeaderByHeight::Response::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, header, header); } @@ -395,6 +516,11 @@ void GetBlockHeadersByHeight::Request::doToJson(rapidjson::Writer<epee::byte_str void GetBlockHeadersByHeight::Request::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, heights, heights); } @@ -405,6 +531,11 @@ void GetBlockHeadersByHeight::Response::doToJson(rapidjson::Writer<epee::byte_st void GetBlockHeadersByHeight::Response::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, headers, headers); } @@ -424,6 +555,11 @@ void GetPeerList::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest) void GetPeerList::Response::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, white_list, white_list); GET_FROM_JSON_OBJECT(val, gray_list, gray_list); } @@ -436,6 +572,11 @@ void SetLogLevel::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) void SetLogLevel::Request::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, level, level); } @@ -462,6 +603,11 @@ void GetTransactionPool::Response::doToJson(rapidjson::Writer<epee::byte_stream> void GetTransactionPool::Response::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, transactions, transactions); GET_FROM_JSON_OBJECT(val, key_images, key_images); } @@ -474,6 +620,11 @@ void HardForkInfo::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest) void HardForkInfo::Request::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, version, version); } @@ -484,6 +635,11 @@ void HardForkInfo::Response::doToJson(rapidjson::Writer<epee::byte_stream>& dest void HardForkInfo::Response::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, info, info); } @@ -499,6 +655,11 @@ void GetOutputHistogram::Request::doToJson(rapidjson::Writer<epee::byte_stream>& void GetOutputHistogram::Request::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, amounts, amounts); GET_FROM_JSON_OBJECT(val, min_count, min_count); GET_FROM_JSON_OBJECT(val, max_count, max_count); @@ -513,6 +674,11 @@ void GetOutputHistogram::Response::doToJson(rapidjson::Writer<epee::byte_stream> void GetOutputHistogram::Response::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, histogram, histogram); } @@ -524,6 +690,11 @@ void GetOutputKeys::Request::doToJson(rapidjson::Writer<epee::byte_stream>& dest void GetOutputKeys::Request::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, outputs, outputs); } @@ -534,6 +705,11 @@ void GetOutputKeys::Response::doToJson(rapidjson::Writer<epee::byte_stream>& des void GetOutputKeys::Response::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, keys, keys); } @@ -552,6 +728,11 @@ void GetRPCVersion::Response::doToJson(rapidjson::Writer<epee::byte_stream>& des void GetRPCVersion::Response::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, version, version); } @@ -562,6 +743,11 @@ void GetFeeEstimate::Request::doToJson(rapidjson::Writer<epee::byte_stream>& des void GetFeeEstimate::Request::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, num_grace_blocks, num_grace_blocks); } @@ -575,6 +761,11 @@ void GetFeeEstimate::Response::doToJson(rapidjson::Writer<epee::byte_stream>& de void GetFeeEstimate::Response::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, estimated_base_fee, estimated_base_fee); GET_FROM_JSON_OBJECT(val, fee_mask, fee_mask); GET_FROM_JSON_OBJECT(val, size_scale, size_scale); @@ -591,6 +782,11 @@ void GetOutputDistribution::Request::doToJson(rapidjson::Writer<epee::byte_strea void GetOutputDistribution::Request::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, amounts, amounts); GET_FROM_JSON_OBJECT(val, from_height, from_height); GET_FROM_JSON_OBJECT(val, to_height, to_height); @@ -605,6 +801,11 @@ void GetOutputDistribution::Response::doToJson(rapidjson::Writer<epee::byte_stre void GetOutputDistribution::Response::fromJson(const rapidjson::Value& val) { + if (!val.IsObject()) + { + throw json::WRONG_TYPE("json object"); + } + GET_FROM_JSON_OBJECT(val, status, status); GET_FROM_JSON_OBJECT(val, distributions, distributions); } diff --git a/src/rpc/rpc_payment.h b/src/rpc/rpc_payment.h index 4ead5a344..a4cc6db57 100644 --- a/src/rpc/rpc_payment.h +++ b/src/rpc/rpc_payment.h @@ -148,8 +148,8 @@ namespace cryptonote template <class t_archive> inline void serialize(t_archive &a, const unsigned int ver) { - a & m_client_info.parent(); - a & m_hashrate.parent(); + a & m_client_info; + a & m_hashrate; a & m_credits_total; a & m_credits_used; a & m_nonces_good; @@ -177,9 +177,9 @@ namespace cryptonote cryptonote::account_public_address m_address; uint64_t m_diff; uint64_t m_credits_per_hash_found; - serializable_unordered_map<crypto::public_key, client_info> m_client_info; + std::unordered_map<crypto::public_key, client_info> m_client_info; std::string m_directory; - serializable_map<uint64_t, uint64_t> m_hashrate; + std::map<uint64_t, uint64_t> m_hashrate; uint64_t m_credits_total; uint64_t m_credits_used; uint64_t m_nonces_good; diff --git a/src/rpc/zmq_server.cpp b/src/rpc/zmq_server.cpp index cb8a8bea4..d73ea3bc9 100644 --- a/src/rpc/zmq_server.cpp +++ b/src/rpc/zmq_server.cpp @@ -158,13 +158,22 @@ void ZmqServer::serve() if (!pub || sockets[2].revents) { - std::string message = MONERO_UNWRAP(net::zmq::receive(rep.get(), read_flags)); - MDEBUG("Received RPC request: \"" << message << "\""); - epee::byte_slice response = handler.handle(std::move(message)); - - const boost::string_ref response_view{reinterpret_cast<const char*>(response.data()), response.size()}; - MDEBUG("Sending RPC reply: \"" << response_view << "\""); - MONERO_UNWRAP(net::zmq::send(std::move(response), rep.get())); + expect<std::string> message = net::zmq::receive(rep.get(), read_flags); + if (!message) + { + // EAGAIN can occur when using `zmq_poll`, which doesn't inspect for message validity + if (message != net::zmq::make_error_code(EAGAIN)) + MONERO_THROW(message.error(), "Read failure on ZMQ-RPC"); + } + else // no errors + { + MDEBUG("Received RPC request: \"" << *message << "\""); + epee::byte_slice response = handler.handle(std::move(*message)); + + const boost::string_ref response_view{reinterpret_cast<const char*>(response.data()), response.size()}; + MDEBUG("Sending RPC reply: \"" << response_view << "\""); + MONERO_UNWRAP(net::zmq::send(std::move(response), rep.get())); + } } } } diff --git a/src/seraphis_crypto/CMakeLists.txt b/src/seraphis_crypto/CMakeLists.txt new file mode 100644 index 000000000..aeae61ad1 --- /dev/null +++ b/src/seraphis_crypto/CMakeLists.txt @@ -0,0 +1,51 @@ +# Copyright (c) 2021, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set(seraphis_crypto_sources + dummy.cpp) + +monero_find_all_headers(seraphis_crypto_headers, "${CMAKE_CURRENT_SOURCE_DIR}") + +monero_add_library(seraphis_crypto + ${seraphis_crypto_sources} + ${seraphis_crypto_headers}) + +target_link_libraries(seraphis_crypto + PUBLIC + cncrypto + common + epee + ringct + PRIVATE + ${EXTRA_LIBRARIES}) + +target_include_directories(seraphis_crypto + PUBLIC + "${CMAKE_CURRENT_SOURCE_DIR}" + PRIVATE + ${Boost_INCLUDE_DIRS}) diff --git a/src/seraphis_crypto/dummy.cpp b/src/seraphis_crypto/dummy.cpp new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/seraphis_crypto/dummy.cpp diff --git a/src/seraphis_crypto/sp_transcript.h b/src/seraphis_crypto/sp_transcript.h new file mode 100644 index 000000000..c2fbd88c2 --- /dev/null +++ b/src/seraphis_crypto/sp_transcript.h @@ -0,0 +1,397 @@ +// Copyright (c) 2022, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Transcript class for assembling data that needs to be hashed. + +#pragma once + +//local headers +#include "crypto/crypto.h" +#include "cryptonote_config.h" +#include "ringct/rctTypes.h" +#include "wipeable_string.h" + +//third party headers +#include <boost/utility/string_ref.hpp> + +//standard headers +#include <list> +#include <string> +#include <type_traits> +#include <vector> + +//forward declarations + + +namespace sp +{ + +//// +// SpTranscriptBuilder +// - build a transcript +// - the user must provide a label when trying to append something to the transcript; labels are prepended to objects in +// the transcript +// - data types +// - unsigned int: uint_flag || varint(uint_variable) +// - signed int: int_flag || uchar{int_variable < 0 ? 1 : 0} || varint(abs(int_variable)) +// - byte buffer (assumed little-endian): buffer_flag || buffer_length || buffer +// - all labels are treated as byte buffers +// - named container: container_flag || container_name || data_member1 || ... || container_terminator_flag +// - list-type container (same-type elements only): list_flag || list_length || element1 || element2 || ... +// - the transcript can be used like a string via the .data() and .size() member functions +// - simple mode: exclude all labels, flags, and lengths +/// +class SpTranscriptBuilder final +{ +public: +//public member types + /// transcript builder mode + enum class Mode + { + FULL, + SIMPLE + }; + +//constructors + /// normal constructor + SpTranscriptBuilder(const std::size_t estimated_data_size, const Mode mode) : + m_mode{mode} + { + m_transcript.reserve(2 * estimated_data_size + 20); + } + +//overloaded operators + /// disable copy/move (this is a scoped manager [of the 'transcript' concept]) + SpTranscriptBuilder& operator=(SpTranscriptBuilder&&) = delete; + +//member functions + /// append a value to the transcript + template <typename T> + void append(const boost::string_ref label, const T &value) + { + this->append_impl(label, value); + } + + /// access the transcript data + const void* data() const { return m_transcript.data(); } + std::size_t size() const { return m_transcript.size(); } + +private: +//member variables + /// in simple mode, exclude labels, flags, and lengths + Mode m_mode; + /// the transcript buffer (wipeable in case it contains sensitive data) + epee::wipeable_string m_transcript; + +//private member types + /// flags for separating items added to the transcript + enum SpTranscriptBuilderFlag : unsigned char + { + UNSIGNED_INTEGER = 0, + SIGNED_INTEGER = 1, + BYTE_BUFFER = 2, + NAMED_CONTAINER = 3, + NAMED_CONTAINER_TERMINATOR = 4, + LIST_TYPE_CONTAINER = 5 + }; + +//transcript builders + void append_character(const unsigned char character) + { + m_transcript += static_cast<char>(character); + } + void append_uint(const std::uint64_t unsigned_integer) + { + unsigned char v_variable[(sizeof(std::uint64_t) * 8 + 6) / 7]; + unsigned char *v_variable_end = v_variable; + + // append uint to string as a varint + tools::write_varint(v_variable_end, unsigned_integer); + assert(v_variable_end <= v_variable + sizeof(v_variable)); + m_transcript.append(reinterpret_cast<const char*>(v_variable), v_variable_end - v_variable); + } + void append_flag(const SpTranscriptBuilderFlag flag) + { + if (m_mode == Mode::SIMPLE) + return; + + static_assert(sizeof(SpTranscriptBuilderFlag) == sizeof(unsigned char), ""); + this->append_character(static_cast<unsigned char>(flag)); + } + void append_length(const std::size_t length) + { + if (m_mode == Mode::SIMPLE) + return; + + static_assert(sizeof(std::size_t) <= sizeof(std::uint64_t), ""); + this->append_uint(static_cast<std::uint64_t>(length)); + } + void append_buffer(const void *data, const std::size_t length) + { + this->append_flag(SpTranscriptBuilderFlag::BYTE_BUFFER); + this->append_length(length); + m_transcript.append(reinterpret_cast<const char*>(data), length); + } + void append_label(const boost::string_ref label) + { + if (m_mode == Mode::SIMPLE || + label.size() == 0) + return; + + this->append_buffer(label.data(), label.size()); + } + void append_labeled_buffer(const boost::string_ref label, const void *data, const std::size_t length) + { + this->append_label(label); + this->append_buffer(data, length); + } + void begin_named_container(const boost::string_ref container_name) + { + this->append_flag(SpTranscriptBuilderFlag::NAMED_CONTAINER); + this->append_label(container_name); + } + void end_named_container() + { + this->append_flag(SpTranscriptBuilderFlag::NAMED_CONTAINER_TERMINATOR); + } + void begin_list_type_container(const std::size_t list_length) + { + this->append_flag(SpTranscriptBuilderFlag::LIST_TYPE_CONTAINER); + this->append_length(list_length); + } + +//append overloads + void append_impl(const boost::string_ref label, const rct::key &key_buffer) + { + this->append_labeled_buffer(label, key_buffer.bytes, sizeof(key_buffer)); + } + void append_impl(const boost::string_ref label, const rct::ctkey &ctkey) + { + this->append_label(label); + this->append_impl("ctmask", ctkey.mask); + this->append_impl("ctdest", ctkey.dest); + } + void append_impl(const boost::string_ref label, const crypto::secret_key &point_buffer) + { + this->append_labeled_buffer(label, point_buffer.data, sizeof(point_buffer)); + } + void append_impl(const boost::string_ref label, const crypto::public_key &scalar_buffer) + { + this->append_labeled_buffer(label, scalar_buffer.data, sizeof(scalar_buffer)); + } + void append_impl(const boost::string_ref label, const crypto::key_derivation &derivation_buffer) + { + this->append_labeled_buffer(label, derivation_buffer.data, sizeof(derivation_buffer)); + } + void append_impl(const boost::string_ref label, const crypto::key_image &key_image_buffer) + { + this->append_labeled_buffer(label, key_image_buffer.data, sizeof(key_image_buffer)); + } + void append_impl(const boost::string_ref label, const std::string &string_buffer) + { + this->append_labeled_buffer(label, string_buffer.data(), string_buffer.size()); + } + void append_impl(const boost::string_ref label, const epee::wipeable_string &string_buffer) + { + this->append_labeled_buffer(label, string_buffer.data(), string_buffer.size()); + } + void append_impl(const boost::string_ref label, const boost::string_ref string_buffer) + { + this->append_labeled_buffer(label, string_buffer.data(), string_buffer.size()); + } + template<std::size_t Sz> + void append_impl(const boost::string_ref label, const unsigned char(&uchar_buffer)[Sz]) + { + this->append_labeled_buffer(label, uchar_buffer, Sz); + } + template<std::size_t Sz> + void append_impl(const boost::string_ref label, const char(&char_buffer)[Sz]) + { + this->append_labeled_buffer(label, char_buffer, Sz); + } + void append_impl(const boost::string_ref label, const std::vector<unsigned char> &vector_buffer) + { + this->append_labeled_buffer(label, vector_buffer.data(), vector_buffer.size()); + } + void append_impl(const boost::string_ref label, const std::vector<char> &vector_buffer) + { + this->append_labeled_buffer(label, vector_buffer.data(), vector_buffer.size()); + } + void append_impl(const boost::string_ref label, const char single_character) + { + this->append_label(label); + this->append_character(static_cast<unsigned char>(single_character)); + } + void append_impl(const boost::string_ref label, const unsigned char single_character) + { + this->append_label(label); + this->append_character(single_character); + } + template<typename T, + std::enable_if_t<std::is_unsigned<T>::value, bool> = true> + void append_impl(const boost::string_ref label, const T unsigned_integer) + { + static_assert(sizeof(T) <= sizeof(std::uint64_t), "SpTranscriptBuilder: unsupported unsigned integer type."); + this->append_label(label); + this->append_flag(SpTranscriptBuilderFlag::UNSIGNED_INTEGER); + this->append_uint(unsigned_integer); + } + template<typename T, + std::enable_if_t<std::is_integral<T>::value, bool> = true, + std::enable_if_t<!std::is_unsigned<T>::value, bool> = true> + void append_impl(const boost::string_ref label, const T signed_integer) + { + using unsigned_type = std::make_unsigned<T>::type; + static_assert(sizeof(unsigned_type) <= sizeof(std::uint64_t), + "SpTranscriptBuilder: unsupported signed integer type."); + this->append_label(label); + this->append_flag(SpTranscriptBuilderFlag::SIGNED_INTEGER); + this->append_uint(static_cast<std::uint64_t>(static_cast<unsigned_type>(signed_integer))); + } + template<typename T, + std::enable_if_t<!std::is_integral<T>::value, bool> = true> + void append_impl(const boost::string_ref label, const T &named_container) + { + // named containers must satisfy two concepts: + // const boost::string_ref container_name(const T &container); + // void append_to_transcript(const T &container, SpTranscriptBuilder &transcript_inout); + //todo: container_name() and append_to_transcript() must be defined in the sp namespace, but that is not generic + this->append_label(label); + this->begin_named_container(container_name(named_container)); + append_to_transcript(named_container, *this); //non-member function assumed to be implemented elsewhere + this->end_named_container(); + } + template<typename T> + void append_impl(const boost::string_ref label, const std::vector<T> &list_container) + { + this->append_label(label); + this->begin_list_type_container(list_container.size()); + for (const T &element : list_container) + this->append_impl("", element); + } + template<typename T> + void append_impl(const boost::string_ref label, const std::list<T> &list_container) + { + this->append_label(label); + this->begin_list_type_container(list_container.size()); + for (const T &element : list_container) + this->append_impl("", element); + } +}; + +//// +// SpFSTranscript +// - build a Fiat-Shamir transcript +// - main format: prefix || domain_separator || object1_label || object1 || object2_label || object2 || ... +// note: prefix defaults to "monero" +/// +class SpFSTranscript final +{ +public: +//constructors + /// normal constructor: start building a transcript with the domain separator + SpFSTranscript(const boost::string_ref domain_separator, + const std::size_t estimated_data_size, + const boost::string_ref prefix = config::TRANSCRIPT_PREFIX) : + m_transcript_builder{15 + domain_separator.size() + estimated_data_size + prefix.size(), + SpTranscriptBuilder::Mode::FULL} + { + // transcript = prefix || domain_separator + m_transcript_builder.append("FSp", prefix); + m_transcript_builder.append("ds", domain_separator); + } + +//overloaded operators + /// disable copy/move (this is a scoped manager [of the 'transcript' concept]) + SpFSTranscript& operator=(SpFSTranscript&&) = delete; + +//member functions + /// build the transcript + template<typename T> + void append(const boost::string_ref label, const T &value) + { + m_transcript_builder.append(label, value); + } + + /// access the transcript data + const void* data() const { return m_transcript_builder.data(); } + std::size_t size() const { return m_transcript_builder.size(); } + +//member variables +private: + /// underlying transcript builder + SpTranscriptBuilder m_transcript_builder; +}; + +//// +// SpKDFTranscript +// - build a data string for a key-derivation function +// - mainly intended for short transcripts (~128 bytes or less) with fixed-length and statically ordered components +// - main format: prefix || domain_separator || object1 || object2 || ... +// - simple transcript mode: no labels, flags, or lengths +// note: prefix defaults to "monero" +/// +class SpKDFTranscript final +{ +public: +//constructors + /// normal constructor: start building a transcript with the domain separator + SpKDFTranscript(const boost::string_ref domain_separator, + const std::size_t estimated_data_size, + const boost::string_ref prefix = config::TRANSCRIPT_PREFIX) : + m_transcript_builder{10 + domain_separator.size() + estimated_data_size + prefix.size(), + SpTranscriptBuilder::Mode::SIMPLE} + { + // transcript = prefix || domain_separator + m_transcript_builder.append("", prefix); + m_transcript_builder.append("", domain_separator); + } + +//overloaded operators + /// disable copy/move (this is a scoped manager [of the 'transcript' concept]) + SpKDFTranscript& operator=(SpKDFTranscript&&) = delete; + +//member functions + /// build the transcript + template<typename T> + void append(const boost::string_ref, const T &value) + { + m_transcript_builder.append("", value); + } + + /// access the transcript data + const void* data() const { return m_transcript_builder.data(); } + std::size_t size() const { return m_transcript_builder.size(); } + +//member variables +private: + /// underlying transcript builder + SpTranscriptBuilder m_transcript_builder; +}; + +} //namespace sp diff --git a/src/serialization/container.h b/src/serialization/container.h index 80e1892f2..6f0c48069 100644 --- a/src/serialization/container.h +++ b/src/serialization/container.h @@ -42,7 +42,7 @@ namespace serialization typename std::enable_if<!use_container_varint<T>(), bool>::type serialize_container_element(Archive& ar, T& e) { - return ::do_serialize(ar, e); + return do_serialize(ar, e); } template<typename Archive, typename T> @@ -52,13 +52,33 @@ namespace serialization static constexpr const bool previously_varint = std::is_same<uint64_t, T>() || std::is_same<uint32_t, T>(); if (!previously_varint && ar.varint_bug_backward_compatibility_enabled() && !typename Archive::is_saving()) - return ::do_serialize(ar, e); + return do_serialize(ar, e); ar.serialize_varint(e); return true; } - template <typename C> - void do_reserve(C &c, size_t N) {} + //! @brief Add an element to a container, inserting at the back if applicable. + template <class Container> + auto do_add(Container &c, typename Container::value_type &&e) -> decltype(c.emplace_back(e)) + { return c.emplace_back(e); } + template <class Container> + auto do_add(Container &c, typename Container::value_type &&e) -> decltype(c.emplace(e)) + { return c.emplace(e); } + + //! @brief Reserve space for N elements if applicable for container. + template<typename... C> + void do_reserve(const C&...) {} + template<typename C> + auto do_reserve(C &c, std::size_t N) -> decltype(c.reserve(N)) { return c.reserve(N); } + + // The value_type of STL map-like containers come in the form std::pair<const K, V>. + // Since we can't {de}serialize const types in this lib, we must convert this to std::pair<K, V> + template <class Container, typename = void> + struct serializable_value_type + { using type = typename Container::value_type; }; + template <class Container> + struct serializable_value_type<Container, std::conditional_t<false, typename Container::mapped_type, void>> + { using type = std::pair<typename Container::key_type, typename Container::mapped_type>; }; } } @@ -82,7 +102,7 @@ bool do_serialize_container(Archive<false> &ar, C &v) for (size_t i = 0; i < cnt; i++) { if (i > 0) ar.delimit_array(); - typename C::value_type e; + typename ::serialization::detail::serializable_value_type<C>::type e; if (!::serialization::detail::serialize_container_element(ar, e)) return false; ::serialization::detail::do_add(v, std::move(e)); @@ -104,7 +124,8 @@ bool do_serialize_container(Archive<true> &ar, C &v) return false; if (i != v.begin()) ar.delimit_array(); - if(!::serialization::detail::serialize_container_element(ar, (typename C::value_type&)*i)) + using serializable_value_type = typename ::serialization::detail::serializable_value_type<C>::type; + if(!::serialization::detail::serialize_container_element(ar, (serializable_value_type&)*i)) return false; if (!ar.good()) return false; diff --git a/src/serialization/containers.h b/src/serialization/containers.h index 65e6359b2..28eaa50fb 100644 --- a/src/serialization/containers.h +++ b/src/serialization/containers.h @@ -38,91 +38,26 @@ #include <set> #include "serialization.h" -template <template <bool> class Archive, class T> bool do_serialize(Archive<false> &ar, std::vector<T> &v); -template <template <bool> class Archive, class T> bool do_serialize(Archive<true> &ar, std::vector<T> &v); - -template <template <bool> class Archive, class T> bool do_serialize(Archive<false> &ar, std::deque<T> &v); -template <template <bool> class Archive, class T> bool do_serialize(Archive<true> &ar, std::deque<T> &v); - -template<typename K, typename V> -class serializable_unordered_map: public std::unordered_map<K, V> -{ -public: - typedef typename std::pair<K, V> value_type; - typename std::unordered_map<K, V> &parent() { return *this; } -}; - -template <template <bool> class Archive, typename K, typename V> bool do_serialize(Archive<false> &ar, serializable_unordered_map<K, V> &v); -template <template <bool> class Archive, typename K, typename V> bool do_serialize(Archive<true> &ar, serializable_unordered_map<K, V> &v); - -template<typename K, typename V> -class serializable_map: public std::map<K, V> -{ -public: - typedef typename std::pair<K, V> value_type; - typename std::map<K, V> &parent() { return *this; } -}; - -template <template <bool> class Archive, typename K, typename V> bool do_serialize(Archive<false> &ar, serializable_map<K, V> &v); -template <template <bool> class Archive, typename K, typename V> bool do_serialize(Archive<true> &ar, serializable_map<K, V> &v); - -template<typename K, typename V> -class serializable_unordered_multimap: public std::unordered_multimap<K, V> -{ -public: - typedef typename std::pair<K, V> value_type; - typename std::unordered_multimap<K, V> &parent() { return *this; } -}; - -template <template <bool> class Archive, typename K, typename V> bool do_serialize(Archive<false> &ar, serializable_unordered_multimap<K, V> &v); -template <template <bool> class Archive, typename K, typename V> bool do_serialize(Archive<true> &ar, serializable_unordered_multimap<K, V> &v); - -template <template <bool> class Archive, class T> bool do_serialize(Archive<false> &ar, std::unordered_set<T> &v); -template <template <bool> class Archive, class T> bool do_serialize(Archive<true> &ar, std::unordered_set<T> &v); - -template <template <bool> class Archive, class T> bool do_serialize(Archive<false> &ar, std::set<T> &v); -template <template <bool> class Archive, class T> bool do_serialize(Archive<true> &ar, std::set<T> &v); - namespace serialization { - namespace detail - { - template <typename T> void do_reserve(std::vector<T> &c, size_t N) { c.reserve(N); } - template <typename T> void do_add(std::vector<T> &c, T &&e) { c.emplace_back(std::forward<T>(e)); } - - template <typename T> void do_add(std::deque<T> &c, T &&e) { c.emplace_back(std::forward<T>(e)); } - - template <typename K, typename V> void do_add(serializable_unordered_map<K, V> &c, std::pair<K, V> &&e) { c.insert(std::forward<std::pair<K, V>>(e)); } - - template <typename K, typename V> void do_add(serializable_map<K, V> &c, std::pair<K, V> &&e) { c.insert(std::forward<std::pair<K, V>>(e)); } - - template <typename K, typename V> void do_add(serializable_unordered_multimap<K, V> &c, std::pair<K, V> &&e) { c.insert(std::forward<std::pair<K, V>>(e)); } - - template <typename T> void do_add(std::unordered_set<T> &c, T &&e) { c.insert(std::forward<T>(e)); } - - template <typename T> void do_add(std::set<T> &c, T &&e) { c.insert(std::forward<T>(e)); } - } + //! @brief Is this type a STL-like container? + //! To add a new container to be serialized, partially specialize the template is_container like so: + template <typename T> struct is_container: std::false_type {}; + template <typename... TA> struct is_container<std::deque<TA...>>: std::true_type {}; + template <typename... TA> struct is_container<std::map<TA...>>: std::true_type {}; + template <typename... TA> struct is_container<std::multimap<TA...>>: std::true_type {}; + template <typename... TA> struct is_container<std::set<TA...>>: std::true_type {}; + template <typename... TA> struct is_container<std::unordered_map<TA...>>: std::true_type {}; + template <typename... TA> struct is_container<std::unordered_multimap<TA...>>: std::true_type {}; + template <typename... TA> struct is_container<std::unordered_set<TA...>>: std::true_type {}; + template <typename... TA> struct is_container<std::vector<TA...>>: std::true_type {}; } #include "container.h" -template <template <bool> class Archive, class T> bool do_serialize(Archive<false> &ar, std::vector<T> &v) { return do_serialize_container(ar, v); } -template <template <bool> class Archive, class T> bool do_serialize(Archive<true> &ar, std::vector<T> &v) { return do_serialize_container(ar, v); } - -template <template <bool> class Archive, class T> bool do_serialize(Archive<false> &ar, std::deque<T> &v) { return do_serialize_container(ar, v); } -template <template <bool> class Archive, class T> bool do_serialize(Archive<true> &ar, std::deque<T> &v) { return do_serialize_container(ar, v); } - -template <template <bool> class Archive, typename K, typename V> bool do_serialize(Archive<false> &ar, serializable_unordered_map<K, V> &v) { return do_serialize_container(ar, v); } -template <template <bool> class Archive, typename K, typename V> bool do_serialize(Archive<true> &ar, serializable_unordered_map<K, V> &v) { return do_serialize_container(ar, v); } - -template <template <bool> class Archive, typename K, typename V> bool do_serialize(Archive<false> &ar, serializable_map<K, V> &v) { return do_serialize_container(ar, v); } -template <template <bool> class Archive, typename K, typename V> bool do_serialize(Archive<true> &ar, serializable_map<K, V> &v) { return do_serialize_container(ar, v); } - -template <template <bool> class Archive, typename K, typename V> bool do_serialize(Archive<false> &ar, serializable_unordered_multimap<K, V> &v) { return do_serialize_container(ar, v); } -template <template <bool> class Archive, typename K, typename V> bool do_serialize(Archive<true> &ar, serializable_unordered_multimap<K, V> &v) { return do_serialize_container(ar, v); } - -template <template <bool> class Archive, class T> bool do_serialize(Archive<false> &ar, std::unordered_set<T> &v) { return do_serialize_container(ar, v); } -template <template <bool> class Archive, class T> bool do_serialize(Archive<true> &ar, std::unordered_set<T> &v) { return do_serialize_container(ar, v); } - -template <template <bool> class Archive, class T> bool do_serialize(Archive<false> &ar, std::set<T> &v) { return do_serialize_container(ar, v); } -template <template <bool> class Archive, class T> bool do_serialize(Archive<true> &ar, std::set<T> &v) { return do_serialize_container(ar, v); } +template <class Archive, class Container> +std::enable_if_t<::serialization::is_container<Container>::value, bool> +do_serialize(Archive &ar, Container &c) +{ + return ::do_serialize_container(ar, c); +} diff --git a/src/serialization/debug_archive.h b/src/serialization/debug_archive.h index ef3bb4345..f4df963db 100644 --- a/src/serialization/debug_archive.h +++ b/src/serialization/debug_archive.h @@ -42,14 +42,12 @@ struct debug_archive : public json_archive<W> { }; template <class T> -struct serializer<debug_archive<true>, T> +static inline bool do_serialize(debug_archive<true> &ar, T &v) { - static void serialize(debug_archive<true> &ar, T &v) - { ar.begin_object(); ar.tag(variant_serialization_traits<debug_archive<true>, T>::get_tag()); - serializer<json_archive<true>, T>::serialize(ar, v); + do_serialize(static_cast<json_archive<true>&>(ar), v); ar.end_object(); ar.stream() << std::endl; - } -}; + return true; +} diff --git a/src/serialization/difficulty_type.h b/src/serialization/difficulty_type.h index 37761839b..c071f287b 100644 --- a/src/serialization/difficulty_type.h +++ b/src/serialization/difficulty_type.h @@ -31,8 +31,6 @@ #include "cryptonote_basic/difficulty.h" #include "serialization.h" -template<> struct is_basic_type<cryptonote::difficulty_type> { typedef boost::true_type type; }; - template <template <bool> class Archive> inline bool do_serialize(Archive<false>& ar, cryptonote::difficulty_type &diff) { diff --git a/src/serialization/pair.h b/src/serialization/pair.h index f21041fe8..888033fb9 100644 --- a/src/serialization/pair.h +++ b/src/serialization/pair.h @@ -47,7 +47,7 @@ namespace serialization typename std::enable_if<!use_pair_varint<T>(), bool>::type serialize_pair_element(Archive& ar, T& e) { - return ::do_serialize(ar, e); + return do_serialize(ar, e); } template<typename Archive, typename T> @@ -57,7 +57,7 @@ namespace serialization static constexpr const bool previously_varint = std::is_same<uint64_t, T>(); if (!previously_varint && ar.varint_bug_backward_compatibility_enabled() && !typename Archive::is_saving()) - return ::do_serialize(ar, e); + return do_serialize(ar, e); ar.serialize_varint(e); return true; } diff --git a/src/serialization/serialization.h b/src/serialization/serialization.h index efe1270a4..2911aecb5 100644 --- a/src/serialization/serialization.h +++ b/src/serialization/serialization.h @@ -57,73 +57,30 @@ template <class T> struct is_blob_type { typedef boost::false_type type; }; -/*! \struct has_free_serializer - * - * \brief a descriptor for dispatching serialize - */ -template <class T> -struct has_free_serializer { typedef boost::true_type type; }; - -/*! \struct is_basic_type - * - * \brief a descriptor for dispatching serialize - */ -template <class T> -struct is_basic_type { typedef boost::false_type type; }; - -template<typename F, typename S> -struct is_basic_type<std::pair<F,S>> { typedef boost::true_type type; }; -template<> -struct is_basic_type<std::string> { typedef boost::true_type type; }; - -/*! \struct serializer +/*! \fn do_serialize(Archive &ar, T &v) * - * \brief ... wouldn't a class be better? + * \brief main function for dispatching serialization for a given pair of archive and value types * - * \detailed The logic behind serializing data. Places the archive - * data into the supplied parameter. This dispatches based on the - * supplied \a T template parameter's traits of is_blob_type or it is - * an integral (as defined by the is_integral trait). Depends on the - * \a Archive parameter to have overloaded the serialize_blob(T v, - * size_t size) and serialize_int(T v) base on which trait it - * applied. When the class has neither types, it falls to the - * overloaded method do_serialize(Archive ar) in T to do the work. + * Types marked true with is_blob_type<T> will be serialized as a blob, integral types will be + * serialized as integers, and types who have a `member_do_serialize` method will be serialized + * using that method. Booleans are serialized like blobs. */ template <class Archive, class T> -struct serializer{ - static bool serialize(Archive &ar, T &v) { - return serialize(ar, v, typename boost::is_integral<T>::type(), typename is_blob_type<T>::type(), typename is_basic_type<T>::type()); - } - template<typename A> - static bool serialize(Archive &ar, T &v, boost::false_type, boost::true_type, A a) { - ar.serialize_blob(&v, sizeof(v)); - return true; - } - template<typename A> - static bool serialize(Archive &ar, T &v, boost::true_type, boost::false_type, A a) { - ar.serialize_int(v); - return true; - } - static bool serialize(Archive &ar, T &v, boost::false_type, boost::false_type, boost::false_type) { - //serialize_custom(ar, v, typename has_free_serializer<T>::type()); - return v.do_serialize(ar); - } - static bool serialize(Archive &ar, T &v, boost::false_type, boost::false_type, boost::true_type) { - //serialize_custom(ar, v, typename has_free_serializer<T>::type()); - return do_serialize(ar, v); - } - static void serialize_custom(Archive &ar, T &v, boost::true_type) { - } -}; - -/*! \fn do_serialize(Archive &ar, T &v) - * - * \brief just calls the serialize function defined for ar and v... - */ +inline std::enable_if_t<is_blob_type<T>::type::value, bool> do_serialize(Archive &ar, T &v) +{ + ar.serialize_blob(&v, sizeof(v)); + return true; +} template <class Archive, class T> -inline bool do_serialize(Archive &ar, T &v) +inline std::enable_if_t<boost::is_integral<T>::value, bool> do_serialize(Archive &ar, T &v) { - return ::serializer<Archive, T>::serialize(ar, v); + ar.serialize_int(v); + return true; +} +template <class Archive, class T> +inline auto do_serialize(Archive &ar, T &v) -> decltype(v.member_do_serialize(ar), true) +{ + return v.member_do_serialize(ar); } template <class Archive> inline bool do_serialize(Archive &ar, bool &v) @@ -144,16 +101,6 @@ inline bool do_serialize(Archive &ar, bool &v) typedef boost::true_type type; \ } -/*! \macro FREE_SERIALIZER - * - * \brief adds the has_free_serializer to the type - */ -#define FREE_SERIALIZER(T) \ - template<> \ - struct has_free_serializer<T> { \ - typedef boost::true_type type; \ - } - /*! \macro VARIANT_TAG * * \brief Adds the tag \tag to the \a Archive of \a Type @@ -174,7 +121,7 @@ inline bool do_serialize(Archive &ar, bool &v) */ #define BEGIN_SERIALIZE() \ template <bool W, template <bool> class Archive> \ - bool do_serialize(Archive<W> &ar) { + bool member_do_serialize(Archive<W> &ar) { /*! \macro BEGIN_SERIALIZE_OBJECT * @@ -183,7 +130,7 @@ inline bool do_serialize(Archive &ar, bool &v) */ #define BEGIN_SERIALIZE_OBJECT() \ template <bool W, template <bool> class Archive> \ - bool do_serialize(Archive<W> &ar) { \ + bool member_do_serialize(Archive<W> &ar) { \ ar.begin_object(); \ bool r = do_serialize_object(ar); \ ar.end_object(); \ @@ -197,11 +144,6 @@ inline bool do_serialize(Archive &ar, bool &v) #define PREPARE_CUSTOM_VECTOR_SERIALIZATION(size, vec) \ ::serialization::detail::prepare_custom_vector_serialization(size, vec, typename Archive<W>::is_saving()) -/*! \macro PREPARE_CUSTOM_DEQUE_SERIALIZATION - */ -#define PREPARE_CUSTOM_DEQUE_SERIALIZATION(size, vec) \ - ::serialization::detail::prepare_custom_deque_serialization(size, vec, typename Archive<W>::is_saving()) - /*! \macro END_SERIALIZE * \brief self-explanatory */ @@ -209,16 +151,6 @@ inline bool do_serialize(Archive &ar, bool &v) return ar.good(); \ } -/*! \macro VALUE(f) - * \brief the same as FIELD(f) - */ -#define VALUE(f) \ - do { \ - ar.tag(#f); \ - bool r = ::do_serialize(ar, f); \ - if (!r || !ar.good()) return false; \ - } while(0); - /*! \macro FIELD_N(t,f) * * \brief serializes a field \a f tagged \a t @@ -226,7 +158,7 @@ inline bool do_serialize(Archive &ar, bool &v) #define FIELD_N(t, f) \ do { \ ar.tag(t); \ - bool r = ::do_serialize(ar, f); \ + bool r = do_serialize(ar, f); \ if (!r || !ar.good()) return false; \ } while(0); @@ -237,7 +169,7 @@ inline bool do_serialize(Archive &ar, bool &v) #define FIELD(f) \ do { \ ar.tag(#f); \ - bool r = ::do_serialize(ar, f); \ + bool r = do_serialize(ar, f); \ if (!r || !ar.good()) return false; \ } while(0); @@ -247,7 +179,7 @@ inline bool do_serialize(Archive &ar, bool &v) */ #define FIELDS(f) \ do { \ - bool r = ::do_serialize(ar, f); \ + bool r = do_serialize(ar, f); \ if (!r || !ar.good()) return false; \ } while(0); @@ -317,17 +249,6 @@ namespace serialization { vec.resize(size); } - template <typename T> - void prepare_custom_deque_serialization(size_t size, std::deque<T>& vec, const boost::mpl::bool_<true>& /*is_saving*/) - { - } - - template <typename T> - void prepare_custom_deque_serialization(size_t size, std::deque<T>& vec, const boost::mpl::bool_<false>& /*is_saving*/) - { - vec.resize(size); - } - /*! \fn do_check_stream_state * * \brief self explanatory diff --git a/src/serialization/tuple.h b/src/serialization/tuple.h index e76ca0b99..d9592bc96 100644 --- a/src/serialization/tuple.h +++ b/src/serialization/tuple.h @@ -39,7 +39,7 @@ namespace serialization template <typename Archive, class T> bool serialize_tuple_element(Archive& ar, T& e) { - return ::do_serialize(ar, e); + return do_serialize(ar, e); } template <typename Archive> diff --git a/src/serialization/variant.h b/src/serialization/variant.h index 77d767a1c..1cf93c8cf 100644 --- a/src/serialization/variant.h +++ b/src/serialization/variant.h @@ -72,7 +72,7 @@ struct variant_reader { if(variant_serialization_traits<Archive, current_type>::get_tag() == t) { current_type x; - if(!::do_serialize(ar, x)) + if(!do_serialize(ar, x)) { ar.set_fail(); return false; @@ -100,19 +100,13 @@ struct variant_reader<Archive, Variant, TBegin, TBegin> } }; - -template <template <bool> class Archive, BOOST_VARIANT_ENUM_PARAMS(typename T)> -struct serializer<Archive<false>, boost::variant<BOOST_VARIANT_ENUM_PARAMS(T)>> -{ - typedef boost::variant<BOOST_VARIANT_ENUM_PARAMS(T)> variant_type; - typedef typename Archive<false>::variant_tag_type variant_tag_type; - typedef typename variant_type::types types; - - static bool serialize(Archive<false> &ar, variant_type &v) { - variant_tag_type t; +template <template <bool> class Archive, typename... T> +static bool do_serialize(Archive<false> &ar, boost::variant<T...> &v) { + using types = typename boost::variant<T...>::types; + typename Archive<false>::variant_tag_type t; ar.begin_variant(); ar.read_variant_tag(t); - if(!variant_reader<Archive<false>, variant_type, + if(!variant_reader<Archive<false>, boost::variant<T...>, typename boost::mpl::begin<types>::type, typename boost::mpl::end<types>::type>::read(ar, v, t)) { @@ -121,27 +115,21 @@ struct serializer<Archive<false>, boost::variant<BOOST_VARIANT_ENUM_PARAMS(T)>> } ar.end_variant(); return true; - } -}; +} -template <template <bool> class Archive, BOOST_VARIANT_ENUM_PARAMS(typename T)> -struct serializer<Archive<true>, boost::variant<BOOST_VARIANT_ENUM_PARAMS(T)>> +template <template <bool> class Archive> +struct variant_write_visitor : public boost::static_visitor<bool> { - typedef boost::variant<BOOST_VARIANT_ENUM_PARAMS(T)> variant_type; - //typedef typename Archive<true>::variant_tag_type variant_tag_type; - - struct visitor : public boost::static_visitor<bool> - { Archive<true> &ar; - visitor(Archive<true> &a) : ar(a) { } + variant_write_visitor(Archive<true> &a) : ar(a) { } template <class T> bool operator ()(T &rv) const { ar.begin_variant(); ar.write_variant_tag(variant_serialization_traits<Archive<true>, T>::get_tag()); - if(!::do_serialize(ar, rv)) + if(!do_serialize(ar, rv)) { ar.set_fail(); return false; @@ -149,9 +137,10 @@ struct serializer<Archive<true>, boost::variant<BOOST_VARIANT_ENUM_PARAMS(T)>> ar.end_variant(); return true; } - }; - - static bool serialize(Archive<true> &ar, variant_type &v) { - return boost::apply_visitor(visitor(ar), v); - } }; + +template <template <bool> class Archive, typename... T> +static bool do_serialize(Archive<true> &ar, boost::variant<T...> &v) +{ + return boost::apply_visitor(variant_write_visitor<Archive>(ar), v); +} diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index 6e5d0b1ec..bff62f1a8 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -188,7 +188,7 @@ namespace const char* USAGE_INCOMING_TRANSFERS("incoming_transfers [available|unavailable] [verbose] [uses] [index=<N1>[,<N2>[,...]]]"); const char* USAGE_PAYMENTS("payments <PID_1> [<PID_2> ... <PID_N>]"); const char* USAGE_PAYMENT_ID("payment_id"); - const char* USAGE_TRANSFER("transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] (<URI> | <address> <amount>) [<payment_id>]"); + const char* USAGE_TRANSFER("transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] (<URI> | <address> <amount>) [subtractfeefrom=<D0>[,<D1>,all,...]] [<payment_id>]"); const char* USAGE_LOCKED_TRANSFER("locked_transfer [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] (<URI> | <addr> <amount>) <lockblocks> [<payment_id (obsolete)>]"); const char* USAGE_LOCKED_SWEEP_ALL("locked_sweep_all [index=<N1>[,<N2>,...] | index=all] [<priority>] [<ring_size>] <address> <lockblocks> [<payment_id (obsolete)>]"); const char* USAGE_SWEEP_ALL("sweep_all [index=<N1>[,<N2>,...] | index=all] [<priority>] [<ring_size>] [outputs=<N>] <address> [<payment_id (obsolete)>]"); @@ -520,7 +520,52 @@ namespace fail_msg_writer() << sw::tr("invalid format for subaddress lookahead; must be <major>:<minor>"); return r; } -} + + static constexpr std::string_view SFFD_ARG_NAME{"subtractfeefrom="}; + + bool parse_subtract_fee_from_outputs + ( + const std::string& arg, + tools::wallet2::unique_index_container& subtract_fee_from_outputs, + bool& subtract_fee_from_all, + bool& matches + ) + { + matches = false; + if (!boost::string_ref{arg}.starts_with(SFFD_ARG_NAME.data())) // if arg doesn't match + return true; + matches = true; + + const char* arg_end = arg.c_str() + arg.size(); + for (const char* p = arg.c_str() + SFFD_ARG_NAME.size(); p < arg_end;) + { + const char* new_p = nullptr; + const unsigned long dest_index = strtoul(p, const_cast<char**>(&new_p), 10); + if (dest_index == 0 && new_p == p) // numerical conversion failed + { + if (0 != strncmp(p, "all", 3)) + { + fail_msg_writer() << tr("Failed to parse subtractfeefrom list"); + return false; + } + subtract_fee_from_all = true; + break; + } + else if (dest_index > std::numeric_limits<uint32_t>::max()) + { + fail_msg_writer() << tr("Destination index is too large") << ": " << dest_index; + return false; + } + else + { + subtract_fee_from_outputs.insert(dest_index); + p = new_p + 1; // skip the comma + } + } + + return true; + } +} // anonymous namespace void simple_wallet::handle_transfer_exception(const std::exception_ptr &e, bool trusted_daemon) { @@ -3080,7 +3125,7 @@ simple_wallet::simple_wallet() tr("Show the blockchain height.")); m_cmd_binder.set_handler("transfer", boost::bind(&simple_wallet::on_command, this, &simple_wallet::transfer, _1), tr(USAGE_TRANSFER), - tr("Transfer <amount> to <address>. If the parameter \"index=<N1>[,<N2>,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. <priority> is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability. Multiple payments can be made at once by adding URI_2 or <address_2> <amount_2> etcetera (before the payment ID, if it's included)")); + tr("Transfer <amount> to <address>. If the parameter \"index=<N1>[,<N2>,...]\" is specified, the wallet uses outputs received by addresses of those indices. If omitted, the wallet randomly chooses address indices to be used. In any case, it tries its best not to combine outputs across multiple addresses. <priority> is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. <ring_size> is the number of inputs to include for untraceability. Multiple payments can be made at once by adding URI_2 or <address_2> <amount_2> etcetera (before the payment ID, if it's included). The \"subtractfeefrom=\" list allows you to choose which destinations to fund the tx fee from instead of the change output. The fee will be split across the chosen destinations proportionally equally. For example, to make 3 transfers where the fee is taken from the first and third destinations, one could do: \"transfer <addr1> 3 <addr2> 0.5 <addr3> 1 subtractfeefrom=0,2\". Let's say the tx fee is 0.1. The balance would drop by exactly 4.5 XMR including fees, and addr1 & addr3 would receive 2.925 & 0.975 XMR, respectively. Use \"subtractfeefrom=all\" to spread the fee across all destinations.")); m_cmd_binder.set_handler("locked_transfer", boost::bind(&simple_wallet::on_command, this, &simple_wallet::locked_transfer,_1), tr(USAGE_LOCKED_TRANSFER), @@ -6349,6 +6394,27 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri local_args.pop_back(); } + // Parse subtractfeefrom destination list + tools::wallet2::unique_index_container subtract_fee_from_outputs; + bool subtract_fee_from_all = false; + for (auto it = local_args.begin(); it < local_args.end();) + { + bool matches = false; + if (!parse_subtract_fee_from_outputs(*it, subtract_fee_from_outputs, subtract_fee_from_all, matches)) + { + return false; + } + else if (matches) + { + it = local_args.erase(it); + break; + } + else + { + ++it; + } + } + vector<cryptonote::address_parse_info> dsts_info; vector<cryptonote::tx_destination_entry> dsts; for (size_t i = 0; i < local_args.size(); ) @@ -6445,6 +6511,13 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri dsts.push_back(de); } + if (subtract_fee_from_all) + { + subtract_fee_from_outputs.clear(); + for (decltype(subtract_fee_from_outputs)::value_type i = 0; i < dsts.size(); ++i) + subtract_fee_from_outputs.insert(i); + } + SCOPED_WALLET_UNLOCK_ON_BAD_PASSWORD(return false;); try @@ -6463,13 +6536,13 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector<std::stri return false; } unlock_block = bc_height + locked_blocks; - ptx_vector = m_wallet->create_transactions_2(dsts, fake_outs_count, unlock_block /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices); + ptx_vector = m_wallet->create_transactions_2(dsts, fake_outs_count, unlock_block /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices, subtract_fee_from_outputs); break; default: LOG_ERROR("Unknown transfer method, using default"); /* FALLTHRU */ case Transfer: - ptx_vector = m_wallet->create_transactions_2(dsts, fake_outs_count, 0 /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices); + ptx_vector = m_wallet->create_transactions_2(dsts, fake_outs_count, 0 /* unlock_time */, priority, extra, m_current_subaddress_account, subaddr_indices, subtract_fee_from_outputs); break; } diff --git a/src/wallet/api/wallet.cpp b/src/wallet/api/wallet.cpp index 0c3aaf853..7f4dbbc79 100644 --- a/src/wallet/api/wallet.cpp +++ b/src/wallet/api/wallet.cpp @@ -45,7 +45,10 @@ #include <sstream> #include <unordered_map> +#ifdef WIN32 #include <boost/locale.hpp> +#endif + #include <boost/filesystem.hpp> using namespace std; diff --git a/src/wallet/message_transporter.cpp b/src/wallet/message_transporter.cpp index 7aa765c87..e6c26cba0 100644 --- a/src/wallet/message_transporter.cpp +++ b/src/wallet/message_transporter.cpp @@ -61,7 +61,7 @@ namespace bitmessage_rpc KV_SERIALIZE(toAddress) KV_SERIALIZE(read) KV_SERIALIZE(msgid) - KV_SERIALIZE(message); + KV_SERIALIZE(message) KV_SERIALIZE(fromAddress) KV_SERIALIZE(receivedTime) KV_SERIALIZE(subject) @@ -273,16 +273,29 @@ bool message_transporter::post_request(const std::string &request, std::string & { if ((string_value.find("API Error 0021") == 0) && (request.find("joinChan") != std::string::npos)) { + // "API Error 0021: Unexpected API Failure" // Error that occurs if one tries to join an already joined chan, which can happen // if several auto-config participants share one PyBitmessage instance: As a little // hack simply ignore the error. (A clean solution would be to check for the chan // with 'listAddresses2', but parsing the returned array is much more complicated.) } + else if ((string_value.find("API Error 0024") == 0) && (request.find("joinChan") != std::string::npos)) + { + // "API Error 0024: Chan address is already present." + // Maybe a result of creating the chan in a slightly different way i.e. not with + // 'createChan'; everything works by just ignoring this error + } else if ((string_value.find("API Error 0013") == 0) && (request.find("leaveChan") != std::string::npos)) { + // "API Error 0013: Could not find your fromAddress in the keys.dat file." // Error that occurs if one tries to leave an already left / deleted chan, which can happen // if several auto-config participants share one PyBitmessage instance: Also ignore. } + else if ((string_value.find("API Error 0025") == 0) && (request.find("leaveChan") != std::string::npos)) + { + // "API Error 0025: Specified address is not a chan address. Use deleteAddress API call instead." + // Error does not really make sense, but everything works by just ignoring + } else { THROW_WALLET_EXCEPTION(tools::error::bitmessage_api_error, string_value); diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 49f95b1a1..e9c48f9a8 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -1821,7 +1821,7 @@ void reattach_blockchain(hashchain &blockchain, wallet2::detached_blockchain_dat } //---------------------------------------------------------------------------------------------------- bool has_nonrequested_tx_at_height_or_above_requested(uint64_t height, const std::unordered_set<crypto::hash> &requested_txids, const wallet2::transfer_container &transfers, - const wallet2::payment_container &payments, const serializable_unordered_map<crypto::hash, wallet2::confirmed_transfer_details> &confirmed_txs) + const wallet2::payment_container &payments, const std::unordered_map<crypto::hash, wallet2::confirmed_transfer_details> &confirmed_txs) { for (const auto &td : transfers) if (td.m_block_height >= height && requested_txids.find(td.m_txid) == requested_txids.end()) @@ -3317,7 +3317,7 @@ void check_block_hard_fork_version(cryptonote::network_type nettype, uint8_t hf_ if (wallet_hard_forks[fork_index].version == hf_version) break; THROW_WALLET_EXCEPTION_IF(fork_index == wallet_num_hard_forks, error::wallet_internal_error, "Fork not found in table"); - uint64_t start_height = wallet_hard_forks[fork_index].height; + uint64_t start_height = hf_version == 1 ? 0 : wallet_hard_forks[fork_index].height; uint64_t end_height = fork_index == wallet_num_hard_forks - 1 ? std::numeric_limits<uint64_t>::max() : wallet_hard_forks[fork_index + 1].height; @@ -4402,7 +4402,7 @@ boost::optional<wallet2::keys_file_data> wallet2::get_keys_file_data(const epee: value2.SetInt(m_key_device_type); json.AddMember("key_on_device", value2, json.GetAllocator()); - value2.SetInt(watch_only ? 1 :0); // WTF ? JSON has different true and false types, and not boolean ?? + value2.SetInt((watch_only || m_watch_only) ? 1 :0); // WTF ? JSON has different true and false types, and not boolean ?? json.AddMember("watch_only", value2, json.GetAllocator()); value2.SetInt(m_multisig ? 1 :0); @@ -6163,6 +6163,20 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass error::wallet_files_doesnt_correspond, m_keys_file, m_wallet_file); } + // Wallets used to wipe, but not erase, old unused multisig key info, which lead to huge memory leaks. + // Here we erase these multisig keys if they're zero'd out to free up space. + for (auto &td : m_transfers) + { + auto mk_it = td.m_multisig_k.begin(); + while (mk_it != td.m_multisig_k.end()) + { + if (*mk_it == rct::zero()) + mk_it = td.m_multisig_k.erase(mk_it); + else + ++mk_it; + } + } + cryptonote::block genesis; generate_genesis(genesis); crypto::hash genesis_hash = get_block_hash(genesis); @@ -6314,7 +6328,7 @@ void wallet2::store_to(const std::string &path, const epee::wipeable_string &pas if (!same_file || force_rewrite_keys) { - bool r = store_keys(m_keys_file, password, false); + bool r = store_keys(m_keys_file, password, m_watch_only); THROW_WALLET_EXCEPTION_IF(!r, error::file_save_error, m_keys_file); } @@ -7036,7 +7050,10 @@ void wallet2::commit_tx(pending_tx& ptx) // tx generated, get rid of used k values for (size_t idx: ptx.selected_transfers) + { memwipe(m_transfers[idx].m_multisig_k.data(), m_transfers[idx].m_multisig_k.size() * sizeof(m_transfers[idx].m_multisig_k[0])); + m_transfers[idx].m_multisig_k.clear(); + } //fee includes dust if dust policy specified it. LOG_PRINT_L1("Transaction successfully sent. <" << txid << ">" << ENDL @@ -7287,9 +7304,7 @@ bool wallet2::sign_tx(unsigned_tx_set &exported_txs, std::vector<wallet2::pendin crypto::key_derivation derivation; std::vector<crypto::key_derivation> additional_derivations; - // compute public keys from out secret keys - crypto::public_key tx_pub_key; - crypto::secret_key_to_public_key(txs[n].tx_key, tx_pub_key); + crypto::public_key tx_pub_key = get_tx_pub_key_from_extra(tx); std::vector<crypto::public_key> additional_tx_pub_keys; for (const crypto::secret_key &skey: txs[n].additional_tx_keys) { @@ -7540,7 +7555,10 @@ std::string wallet2::save_multisig_tx(multisig_tx_set txs) // txes generated, get rid of used k values for (size_t n = 0; n < txs.m_ptx.size(); ++n) for (size_t idx: txs.m_ptx[n].construction_data.selected_transfers) + { memwipe(m_transfers[idx].m_multisig_k.data(), m_transfers[idx].m_multisig_k.size() * sizeof(m_transfers[idx].m_multisig_k[0])); + m_transfers[idx].m_multisig_k.clear(); + } // zero out some data we don't want to share for (auto &ptx: txs.m_ptx) @@ -7864,7 +7882,10 @@ bool wallet2::sign_multisig_tx(multisig_tx_set &exported_txs, std::vector<crypto // inputs in the transactions worked on here) for (size_t n = 0; n < exported_txs.m_ptx.size(); ++n) for (size_t idx: exported_txs.m_ptx[n].construction_data.selected_transfers) + { memwipe(m_transfers[idx].m_multisig_k.data(), m_transfers[idx].m_multisig_k.size() * sizeof(m_transfers[idx].m_multisig_k[0])); + m_transfers[idx].m_multisig_k.clear(); + } exported_txs.m_signers.insert(get_multisig_signer_public_key()); @@ -8662,6 +8683,26 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> COMMAND_RPC_GET_OUTPUTS_BIN::request req = AUTO_VAL_INIT(req); COMMAND_RPC_GET_OUTPUTS_BIN::response daemon_resp = AUTO_VAL_INIT(daemon_resp); + // The secret picking order contains outputs in the order that we selected them. + // + // We will later sort the output request entries in a pre-determined order so that the daemon + // that we're requesting information from doesn't learn any information about the true spend + // for each ring. However, internally, we want to prefer to construct our rings using the + // outputs that we picked first versus outputs picked later. + // + // The reason why is because each consecutive output pick within a ring becomes increasing less + // statistically independent from other picks, since we pick outputs from a finite set + // *without replacement*, due to the protocol not allowing duplicate ring members. This effect + // is exacerbated by the fact that we pick 1.5x + 75 as many outputs as we need per RPC + // request to account for unusable outputs. This effect is small, but non-neglibile and gets + // worse with larger ring sizes. + std::vector<get_outputs_out> secret_picking_order; + + // Convenience/safety lambda to make sure that both output lists req.outputs and secret_picking_order are updated together + // Each ring section of req.outputs gets sorted later after selecting all outputs for that ring + const auto add_output_to_lists = [&req, &secret_picking_order](const get_outputs_out &goo) + { req.outputs.push_back(goo); secret_picking_order.push_back(goo); }; + std::unique_ptr<gamma_picker> gamma; if (has_rct) gamma.reset(new gamma_picker(rct_offsets)); @@ -8796,7 +8837,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> if (out < num_outs) { MINFO("Using it"); - req.outputs.push_back({amount, out}); + add_output_to_lists({amount, out}); ++num_found; seen_indices.emplace(out); if (out == td.m_global_output_index) @@ -8818,12 +8859,12 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> if (num_outs <= requested_outputs_count) { for (uint64_t i = 0; i < num_outs; i++) - req.outputs.push_back({amount, i}); + add_output_to_lists({amount, i}); // duplicate to make up shortfall: this will be caught after the RPC call, // so we can also output the amounts for which we can't reach the required // mixin after checking the actual unlockedness for (uint64_t i = num_outs; i < requested_outputs_count; ++i) - req.outputs.push_back({amount, num_outs - 1}); + add_output_to_lists({amount, num_outs - 1}); } else { @@ -8832,7 +8873,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> { num_found = 1; seen_indices.emplace(td.m_global_output_index); - req.outputs.push_back({amount, td.m_global_output_index}); + add_output_to_lists({amount, td.m_global_output_index}); LOG_PRINT_L1("Selecting real output: " << td.m_global_output_index << " for " << print_money(amount)); } @@ -8940,7 +8981,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> seen_indices.emplace(i); picks[type].insert(i); - req.outputs.push_back({amount, i}); + add_output_to_lists({amount, i}); ++num_found; MDEBUG("picked " << i << ", " << num_found << " now picked"); } @@ -8954,7 +8995,7 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> // we'll error out later while (num_found < requested_outputs_count) { - req.outputs.push_back({amount, 0}); + add_output_to_lists({amount, 0}); ++num_found; } } @@ -8964,6 +9005,10 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> [](const get_outputs_out &a, const get_outputs_out &b) { return a.index < b.index; }); } + THROW_WALLET_EXCEPTION_IF(req.outputs.size() != secret_picking_order.size(), error::wallet_internal_error, + "bug: we did not update req.outputs/secret_picking_order in tandem"); + + // List all requested outputs to debug log if (ELPP->vRegistry()->allowed(el::Level::Debug, MONERO_DEFAULT_LOG_CATEGORY)) { std::map<uint64_t, std::set<uint64_t>> outs; @@ -9081,18 +9126,21 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> } } - // then pick others in random order till we reach the required number - // since we use an equiprobable pick here, we don't upset the triangular distribution - std::vector<size_t> order; - order.resize(requested_outputs_count); - for (size_t n = 0; n < order.size(); ++n) - order[n] = n; - std::shuffle(order.begin(), order.end(), crypto::random_device{}); - + // While we are still lacking outputs in this result ring, in our secret pick order... LOG_PRINT_L2("Looking for " << (fake_outputs_count+1) << " outputs of size " << print_money(td.is_rct() ? 0 : td.amount())); - for (size_t o = 0; o < requested_outputs_count && outs.back().size() < fake_outputs_count + 1; ++o) + for (size_t ring_pick_idx = base; ring_pick_idx < base + requested_outputs_count && outs.back().size() < fake_outputs_count + 1; ++ring_pick_idx) { - size_t i = base + order[o]; + const get_outputs_out attempted_output = secret_picking_order[ring_pick_idx]; + + // Find the index i of our pick in the request/response arrays + size_t i; + for (i = base; i < base + requested_outputs_count; ++i) + if (req.outputs[i].index == attempted_output.index) + break; + THROW_WALLET_EXCEPTION_IF(i == base + requested_outputs_count, error::wallet_internal_error, + "Could not find index of picked output in requested outputs"); + + // Try adding this output's information to result ring if output isn't invalid LOG_PRINT_L2("Index " << i << "/" << requested_outputs_count << ": idx " << req.outputs[i].index << " (real " << td.m_global_output_index << "), unlocked " << daemon_resp.outs[i].unlocked << ", key " << daemon_resp.outs[i].key); tx_add_fake_output(outs, req.outputs[i].index, daemon_resp.outs[i].key, daemon_resp.outs[i].mask, td.m_global_output_index, daemon_resp.outs[i].unlocked, valid_public_keys_cache); } @@ -9803,7 +9851,7 @@ static uint32_t get_count_above(const std::vector<wallet2::transfer_details> &tr // This system allows for sending (almost) the entire balance, since it does // not generate spurious change in all txes, thus decreasing the instantaneous // usable balance. -std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices) +std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices, const unique_index_container& subtract_fee_from_outputs) { //ensure device is let in NONE mode in any case hw::device &hwdev = m_account.get_device(); @@ -9814,11 +9862,12 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp std::vector<std::pair<uint32_t, std::vector<size_t>>> unused_transfers_indices_per_subaddr; std::vector<std::pair<uint32_t, std::vector<size_t>>> unused_dust_indices_per_subaddr; - uint64_t needed_money; + uint64_t needed_money, total_needed_money; // 'needed_money' is the sum of the destination amounts, while 'total_needed_money' includes 'needed_money' plus the fee if not 'subtract_fee_from_outputs' uint64_t accumulated_fee, accumulated_change; struct TX { std::vector<size_t> selected_transfers; std::vector<cryptonote::tx_destination_entry> dsts; + std::vector<bool> dsts_are_fee_subtractable; cryptonote::transaction tx; pending_tx ptx; size_t weight; @@ -9828,9 +9877,11 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp TX() : weight(0), needed_fee(0) {} /* Add an output to the transaction. + * If merge_destinations is true, when adding a destination with an existing address, to increment the amount of the existing tx output instead of creating a new one + * If subtracting_fee is true, when we generate a final list of destinations for transfer_selected[_rct], this destination will be used to fund the tx fee * Returns True if the output was added, False if there are no more available output slots. */ - bool add(const cryptonote::tx_destination_entry &de, uint64_t amount, unsigned int original_output_index, bool merge_destinations, size_t max_dsts) { + bool add(const cryptonote::tx_destination_entry &de, uint64_t amount, unsigned int original_output_index, bool merge_destinations, size_t max_dsts, bool subtracting_fee) { if (merge_destinations) { std::vector<cryptonote::tx_destination_entry>::iterator i; @@ -9840,6 +9891,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp if (dsts.size() >= max_dsts) return false; dsts.push_back(de); + dsts_are_fee_subtractable.push_back(subtracting_fee); i = dsts.end() - 1; i->amount = 0; } @@ -9855,13 +9907,67 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp return false; dsts.push_back(de); dsts.back().amount = 0; + dsts_are_fee_subtractable.push_back(subtracting_fee); } THROW_WALLET_EXCEPTION_IF(memcmp(&dsts[original_output_index].addr, &de.addr, sizeof(de.addr)), error::wallet_internal_error, "Mismatched destination address"); dsts[original_output_index].amount += amount; } return true; } + + // Returns destinations adjusted for given fee if subtract_fee_from_outputs is enabled + std::vector<cryptonote::tx_destination_entry> get_adjusted_dsts(uint64_t needed_fee) const + { + uint64_t dest_total = 0; + uint64_t subtractable_dest_total = 0; + std::vector<size_t> subtractable_indices; + subtractable_indices.reserve(dsts.size()); + for (size_t i = 0; i < dsts.size(); ++i) + { + dest_total += dsts[i].amount; + if (dsts_are_fee_subtractable[i]) + { + subtractable_dest_total += dsts[i].amount; + subtractable_indices.push_back(i); + } + } + + if (subtractable_indices.empty()) // if subtract_fee_from_outputs is not enabled for this tx + return dsts; + + THROW_WALLET_EXCEPTION_IF(subtractable_dest_total < needed_fee, error::tx_not_possible, + subtractable_dest_total, dest_total, needed_fee); + + std::vector<cryptonote::tx_destination_entry> res = dsts; + + // subtract fees from destinations equally, rounded down, until dust is left where we subtract 1 + uint64_t subtractable_remaining = needed_fee; + auto si_it = subtractable_indices.cbegin(); + uint64_t amount_to_subtract = 0; + while (subtractable_remaining) + { + // Set the amount to subtract iterating at the beginning of the list so equal amounts are + // subtracted throughout the list of destinations. We use max(x, 1) so that we we still step + // forwards even when the amount remaining is less than the number of subtractable indices + if (si_it == subtractable_indices.cbegin()) + amount_to_subtract = std::max<uint64_t>(subtractable_remaining / subtractable_indices.size(), 1); + + cryptonote::tx_destination_entry& d = res[*si_it]; + THROW_WALLET_EXCEPTION_IF(d.amount <= amount_to_subtract, error::zero_amount); + + subtractable_remaining -= amount_to_subtract; + d.amount -= amount_to_subtract; + ++si_it; + + // Wrap around to first subtractable index once we hit the end of the list + if (si_it == subtractable_indices.cend()) + si_it = subtractable_indices.cbegin(); + } + + return res; + } }; + std::vector<TX> txes; bool adding_fee; // true if new outputs go towards fee, rather than destinations uint64_t needed_fee, available_for_fee = 0; @@ -9884,6 +9990,14 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp // throw if attempting a transaction with no destinations THROW_WALLET_EXCEPTION_IF(dsts.empty(), error::zero_destination); + // throw if subtract_fee_from_outputs has a bad index + THROW_WALLET_EXCEPTION_IF(subtract_fee_from_outputs.size() && *subtract_fee_from_outputs.crbegin() >= dsts.size(), + error::subtract_fee_from_bad_index, *subtract_fee_from_outputs.crbegin()); + + // throw if subtract_fee_from_outputs is enabled and we have too many outputs to fit into one tx + THROW_WALLET_EXCEPTION_IF(subtract_fee_from_outputs.size() && dsts.size() > BULLETPROOF_MAX_OUTPUTS - 1, + error::wallet_internal_error, "subtractfeefrom transfers cannot be split over multiple transactions yet"); + // calculate total amount being sent to all destinations // throw if total amount overflows uint64_t needed_money = 0; @@ -9911,6 +10025,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp // we could also check for being within FEE_PER_KB, but if the fee calculation // ever changes, this might be missed, so let this go through const uint64_t min_fee = (base_fee * estimate_tx_size(use_rct, 1, fake_outs_count, 2, extra.size(), bulletproof, clsag, bulletproof_plus, use_view_tags)); + total_needed_money = needed_money + (subtract_fee_from_outputs.size() ? 0 : min_fee); uint64_t balance_subtotal = 0; uint64_t unlocked_balance_subtotal = 0; for (uint32_t index_minor : subaddr_indices) @@ -9918,10 +10033,10 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp balance_subtotal += balance_per_subaddr[index_minor]; unlocked_balance_subtotal += unlocked_balance_per_subaddr[index_minor].first; } - THROW_WALLET_EXCEPTION_IF(needed_money + min_fee > balance_subtotal, error::not_enough_money, + THROW_WALLET_EXCEPTION_IF(total_needed_money > balance_subtotal || min_fee > balance_subtotal, error::not_enough_money, balance_subtotal, needed_money, 0); // first check overall balance is enough, then unlocked one, so we throw distinct exceptions - THROW_WALLET_EXCEPTION_IF(needed_money + min_fee > unlocked_balance_subtotal, error::not_enough_unlocked_money, + THROW_WALLET_EXCEPTION_IF(total_needed_money > unlocked_balance_subtotal || min_fee > unlocked_balance_subtotal, error::not_enough_unlocked_money, unlocked_balance_subtotal, needed_money, 0); for (uint32_t i : subaddr_indices) @@ -10024,7 +10139,8 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp // this is used to build a tx that's 1 or 2 inputs, and 2 outputs, which // will get us a known fee. uint64_t estimated_fee = estimate_fee(use_per_byte_fee, use_rct, 2, fake_outs_count, 2, extra.size(), bulletproof, clsag, bulletproof_plus, use_view_tags, base_fee, fee_quantization_mask); - preferred_inputs = pick_preferred_rct_inputs(needed_money + estimated_fee, subaddr_account, subaddr_indices); + total_needed_money = needed_money + (subtract_fee_from_outputs.size() ? 0 : estimated_fee); + preferred_inputs = pick_preferred_rct_inputs(total_needed_money, subaddr_account, subaddr_indices); if (!preferred_inputs.empty()) { string s; @@ -10057,7 +10173,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp // - we have something to send // - or we need to gather more fee // - or we have just one input in that tx, which is rct (to try and make all/most rct txes 2/2) - unsigned int original_output_index = 0; + unsigned int original_output_index = 0, destination_index = 0; std::vector<size_t>* unused_transfers_indices = &unused_transfers_indices_per_subaddr[0].second; std::vector<size_t>* unused_dust_indices = &unused_dust_indices_per_subaddr[0].second; @@ -10140,7 +10256,8 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp // we can fully pay that destination LOG_PRINT_L2("We can fully pay " << get_account_address_as_str(m_nettype, dsts[0].is_subaddress, dsts[0].addr) << " for " << print_money(dsts[0].amount)); - if (!tx.add(dsts[0], dsts[0].amount, original_output_index, m_merge_destinations, BULLETPROOF_MAX_OUTPUTS-1)) + const bool subtract_fee_from_this_dest = subtract_fee_from_outputs.count(destination_index); + if (!tx.add(dsts[0], dsts[0].amount, original_output_index, m_merge_destinations, BULLETPROOF_MAX_OUTPUTS-1, subtract_fee_from_this_dest)) { LOG_PRINT_L2("Didn't pay: ran out of output slots"); out_slots_exhausted = true; @@ -10150,6 +10267,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp dsts[0].amount = 0; pop_index(dsts, 0); ++original_output_index; + ++destination_index; } if (!out_slots_exhausted && available_amount > 0 && !dsts.empty() && @@ -10157,7 +10275,8 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp // we can partially fill that destination LOG_PRINT_L2("We can partially pay " << get_account_address_as_str(m_nettype, dsts[0].is_subaddress, dsts[0].addr) << " for " << print_money(available_amount) << "/" << print_money(dsts[0].amount)); - if (tx.add(dsts[0], available_amount, original_output_index, m_merge_destinations, BULLETPROOF_MAX_OUTPUTS-1)) + const bool subtract_fee_from_this_dest = subtract_fee_from_outputs.count(destination_index); + if (tx.add(dsts[0], available_amount, original_output_index, m_merge_destinations, BULLETPROOF_MAX_OUTPUTS-1, subtract_fee_from_this_dest)) { dsts[0].amount -= available_amount; available_amount = 0; @@ -10236,9 +10355,13 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp // Try to carve the estimated fee from the partial payment (if there is one) available_for_fee = try_carving_from_partial_payment(needed_fee, available_for_fee); - uint64_t inputs = 0, outputs = needed_fee; + uint64_t inputs = 0, outputs = 0; for (size_t idx: tx.selected_transfers) inputs += m_transfers[idx].amount(); for (const auto &o: tx.dsts) outputs += o.amount; + if (subtract_fee_from_outputs.empty()) // if normal tx that doesn't subtract fees + { + outputs += needed_fee; + } if (inputs < outputs) { @@ -10249,15 +10372,32 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp LOG_PRINT_L2("Trying to create a tx now, with " << tx.dsts.size() << " outputs and " << tx.selected_transfers.size() << " inputs"); + auto tx_dsts = tx.get_adjusted_dsts(needed_fee); if (use_rct) - transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, outs, valid_public_keys_cache, unlock_time, needed_fee, extra, + transfer_selected_rct(tx_dsts, tx.selected_transfers, fake_outs_count, outs, valid_public_keys_cache, unlock_time, needed_fee, extra, test_tx, test_ptx, rct_config, use_view_tags); else - transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, outs, valid_public_keys_cache, unlock_time, needed_fee, extra, + transfer_selected(tx_dsts, tx.selected_transfers, fake_outs_count, outs, valid_public_keys_cache, unlock_time, needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx, use_view_tags); auto txBlob = t_serializable_object_to_blob(test_ptx.tx); needed_fee = calculate_fee(use_per_byte_fee, test_ptx.tx, txBlob.size(), base_fee, fee_quantization_mask); - available_for_fee = test_ptx.fee + test_ptx.change_dts.amount + (!test_ptx.dust_added_to_fee ? test_ptx.dust : 0); + + // Depending on the mode, we take extra fees from either our change output or the destination outputs for which subtract_fee_from_outputs is true + uint64_t output_available_for_fee = 0; + bool tx_has_subtractable_output = false; + for (size_t di = 0; di < tx.dsts.size(); ++di) + { + if (tx.dsts_are_fee_subtractable[di]) + { + output_available_for_fee += tx.dsts[di].amount; + tx_has_subtractable_output = true; + } + } + if (!tx_has_subtractable_output) + { + output_available_for_fee = test_ptx.change_dts.amount; + } + available_for_fee = test_ptx.fee + output_available_for_fee + (!test_ptx.dust_added_to_fee ? test_ptx.dust : 0); LOG_PRINT_L2("Made a " << get_weight_string(test_ptx.tx, txBlob.size()) << " tx, with " << print_money(available_for_fee) << " available for fee (" << print_money(needed_fee) << " needed)"); @@ -10273,18 +10413,24 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp else { LOG_PRINT_L2("We made a tx, adjusting fee and saving it, we need " << print_money(needed_fee) << " and we have " << print_money(test_ptx.fee)); - do { + size_t fee_tries; + for (fee_tries = 0; fee_tries < 10 && needed_fee > test_ptx.fee; ++fee_tries) { + tx_dsts = tx.get_adjusted_dsts(needed_fee); + if (use_rct) - transfer_selected_rct(tx.dsts, tx.selected_transfers, fake_outs_count, outs, valid_public_keys_cache, unlock_time, needed_fee, extra, + transfer_selected_rct(tx_dsts, tx.selected_transfers, fake_outs_count, outs, valid_public_keys_cache, unlock_time, needed_fee, extra, test_tx, test_ptx, rct_config, use_view_tags); else - transfer_selected(tx.dsts, tx.selected_transfers, fake_outs_count, outs, valid_public_keys_cache, unlock_time, needed_fee, extra, + transfer_selected(tx_dsts, tx.selected_transfers, fake_outs_count, outs, valid_public_keys_cache, unlock_time, needed_fee, extra, detail::digit_split_strategy, tx_dust_policy(::config::DEFAULT_DUST_THRESHOLD), test_tx, test_ptx, use_view_tags); txBlob = t_serializable_object_to_blob(test_ptx.tx); needed_fee = calculate_fee(use_per_byte_fee, test_ptx.tx, txBlob.size(), base_fee, fee_quantization_mask); LOG_PRINT_L2("Made an attempt at a final " << get_weight_string(test_ptx.tx, txBlob.size()) << " tx, with " << print_money(test_ptx.fee) << " fee and " << print_money(test_ptx.change_dts.amount) << " change"); - } while (needed_fee > test_ptx.fee); + }; + + THROW_WALLET_EXCEPTION_IF(fee_tries == 10, error::wallet_internal_error, + "Too many attempts to raise pending tx fee to level of needed fee"); LOG_PRINT_L2("Made a final " << get_weight_string(test_ptx.tx, txBlob.size()) << " tx, with " << print_money(test_ptx.fee) << " fee and " << print_money(test_ptx.change_dts.amount) << " change"); @@ -10337,10 +10483,13 @@ skip_tx: for (std::vector<TX>::iterator i = txes.begin(); i != txes.end(); ++i) { TX &tx = *i; + + const auto tx_dsts = tx.get_adjusted_dsts(tx.needed_fee); + cryptonote::transaction test_tx; pending_tx test_ptx; if (use_rct) { - transfer_selected_rct(tx.dsts, /* NOMOD std::vector<cryptonote::tx_destination_entry> dsts,*/ + transfer_selected_rct(tx_dsts, /* NOMOD std::vector<cryptonote::tx_destination_entry> dsts,*/ tx.selected_transfers, /* const std::list<size_t> selected_transfers */ fake_outs_count, /* CONST size_t fake_outputs_count, */ tx.outs, /* MOD std::vector<std::vector<tools::wallet2::get_outs_entry>> &outs, */ @@ -10353,7 +10502,7 @@ skip_tx: rct_config, use_view_tags); /* const bool use_view_tags */ } else { - transfer_selected(tx.dsts, + transfer_selected(tx_dsts, tx.selected_transfers, fake_outs_count, tx.outs, @@ -10387,23 +10536,38 @@ skip_tx: ptx_vector.push_back(tx.ptx); } - THROW_WALLET_EXCEPTION_IF(!sanity_check(ptx_vector, original_dsts), error::wallet_internal_error, "Created transaction(s) failed sanity check"); + THROW_WALLET_EXCEPTION_IF(!sanity_check(ptx_vector, original_dsts, subtract_fee_from_outputs), error::wallet_internal_error, "Created transaction(s) failed sanity check"); // if we made it this far, we're OK to actually send the transactions return ptx_vector; } -bool wallet2::sanity_check(const std::vector<wallet2::pending_tx> &ptx_vector, std::vector<cryptonote::tx_destination_entry> dsts) const +bool wallet2::sanity_check(const std::vector<wallet2::pending_tx> &ptx_vector, const std::vector<cryptonote::tx_destination_entry>& dsts, const unique_index_container& subtract_fee_from_outputs) const { - MDEBUG("sanity_check: " << ptx_vector.size() << " txes, " << dsts.size() << " destinations"); + MDEBUG("sanity_check: " << ptx_vector.size() << " txes, " << dsts.size() << " destinations, subtract_fee_from_outputs " << + (subtract_fee_from_outputs.size() ? "enabled" : "disabled")); THROW_WALLET_EXCEPTION_IF(ptx_vector.empty(), error::wallet_internal_error, "No transactions"); + THROW_WALLET_EXCEPTION_IF(!subtract_fee_from_outputs.empty() && ptx_vector.size() != 1, + error::wallet_internal_error, "feature subtractfeefrom not supported for split transactions"); + + // For destinations from where the fee is subtracted, the required amount has to be at least + // target amount - (tx fee / num_subtractable + 1). +1 since fee might not be evenly divisble by + // the number of subtractble destinations. For non-subtractable destinations, we need at least + // the target amount. + const size_t num_subtractable_dests = subtract_fee_from_outputs.size(); + const uint64_t fee0 = ptx_vector[0].fee; + const uint64_t subtractable_fee_deduction = fee0 / std::max<size_t>(num_subtractable_dests, 1) + 1; // check every party in there does receive at least the required amount std::unordered_map<account_public_address, std::pair<uint64_t, bool>> required; - for (const auto &d: dsts) + for (size_t i = 0; i < dsts.size(); ++i) { - required[d.addr].first += d.amount; + const cryptonote::tx_destination_entry& d = dsts[i]; + const bool dest_is_subtractable = subtract_fee_from_outputs.count(i); + const uint64_t fee_deduction = dest_is_subtractable ? subtractable_fee_deduction : 0; + const uint64_t required_amount = d.amount - std::min(fee_deduction, d.amount); + required[d.addr].first += required_amount; required[d.addr].second = d.is_subaddress; } @@ -11985,7 +12149,7 @@ std::string wallet2::get_reserve_proof(const boost::optional<std::pair<uint32_t, } // collect all subaddress spend keys that received those outputs and generate their signatures - serializable_unordered_map<crypto::public_key, crypto::signature> subaddr_spendkeys; + std::unordered_map<crypto::public_key, crypto::signature> subaddr_spendkeys; for (const cryptonote::subaddress_index &index : subaddr_indices) { crypto::secret_key subaddr_spend_skey = m_account.get_keys().m_spend_secret_key; @@ -12030,7 +12194,7 @@ bool wallet2::check_reserve_proof(const cryptonote::account_public_address &addr bool loaded = false; std::vector<reserve_proof_entry> proofs; - serializable_unordered_map<crypto::public_key, crypto::signature> subaddr_spendkeys; + std::unordered_map<crypto::public_key, crypto::signature> subaddr_spendkeys; try { binary_archive<false> ar{epee::strspan<std::uint8_t>(sig_decoded)}; @@ -12044,7 +12208,7 @@ bool wallet2::check_reserve_proof(const cryptonote::account_public_address &addr { std::istringstream iss(sig_decoded); boost::archive::portable_binary_iarchive ar(iss); - ar >> proofs >> subaddr_spendkeys.parent(); + ar >> proofs >> subaddr_spendkeys; } THROW_WALLET_EXCEPTION_IF(subaddr_spendkeys.count(address.m_spend_public_key) == 0, error::wallet_internal_error, @@ -12288,7 +12452,7 @@ std::string wallet2::get_description() const return ""; } -const std::pair<serializable_map<std::string, std::string>, std::vector<std::string>>& wallet2::get_account_tags() +const std::pair<std::map<std::string, std::string>, std::vector<std::string>>& wallet2::get_account_tags() { // ensure consistency if (m_account_tags.second.size() != get_num_subaddress_accounts()) @@ -13511,7 +13675,10 @@ cryptonote::blobdata wallet2::export_multisig() transfer_details &td = m_transfers[n]; crypto::key_image ki; if (td.m_multisig_k.size()) + { memwipe(td.m_multisig_k.data(), td.m_multisig_k.size() * sizeof(td.m_multisig_k[0])); + td.m_multisig_k.clear(); + } info[n].m_LR.clear(); info[n].m_partial_key_images.clear(); diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index d93b9b9fb..c5d2dc90c 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -590,7 +590,8 @@ private: }; typedef std::vector<transfer_details> transfer_container; - typedef serializable_unordered_multimap<crypto::hash, payment_details> payment_container; + typedef std::unordered_multimap<crypto::hash, payment_details> payment_container; + typedef std::set<uint32_t> unique_index_container; struct multisig_sig { @@ -701,7 +702,7 @@ private: { std::vector<pending_tx> ptx; std::vector<crypto::key_image> key_images; - serializable_unordered_map<crypto::public_key, crypto::key_image> tx_key_images; + std::unordered_map<crypto::public_key, crypto::key_image> tx_key_images; BEGIN_SERIALIZE_OBJECT() VERSION_FIELD(0) @@ -1096,11 +1097,11 @@ private: bool parse_unsigned_tx_from_str(const std::string &unsigned_tx_st, unsigned_tx_set &exported_txs) const; bool load_tx(const std::string &signed_filename, std::vector<tools::wallet2::pending_tx> &ptx, std::function<bool(const signed_tx_set&)> accept_func = NULL); bool parse_tx_from_str(const std::string &signed_tx_st, std::vector<tools::wallet2::pending_tx> &ptx, std::function<bool(const signed_tx_set &)> accept_func); - std::vector<wallet2::pending_tx> create_transactions_2(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices); // pass subaddr_indices by value on purpose + std::vector<wallet2::pending_tx> create_transactions_2(std::vector<cryptonote::tx_destination_entry> dsts, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices, const unique_index_container& subtract_fee_from_outputs = {}); // pass subaddr_indices by value on purpose std::vector<wallet2::pending_tx> create_transactions_all(uint64_t below, const cryptonote::account_public_address &address, bool is_subaddress, const size_t outputs, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra, uint32_t subaddr_account, std::set<uint32_t> subaddr_indices); std::vector<wallet2::pending_tx> create_transactions_single(const crypto::key_image &ki, const cryptonote::account_public_address &address, bool is_subaddress, const size_t outputs, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra); std::vector<wallet2::pending_tx> create_transactions_from(const cryptonote::account_public_address &address, bool is_subaddress, const size_t outputs, std::vector<size_t> unused_transfers_indices, std::vector<size_t> unused_dust_indices, const size_t fake_outs_count, const uint64_t unlock_time, uint32_t priority, const std::vector<uint8_t>& extra); - bool sanity_check(const std::vector<wallet2::pending_tx> &ptx_vector, std::vector<cryptonote::tx_destination_entry> dsts) const; + bool sanity_check(const std::vector<wallet2::pending_tx> &ptx_vector, const std::vector<cryptonote::tx_destination_entry>& dsts, const unique_index_container& subtract_fee_from_outputs = {}) const; void cold_tx_aux_import(const std::vector<pending_tx>& ptx, const std::vector<std::string>& tx_device_aux); void cold_sign_tx(const std::vector<pending_tx>& ptx_vector, signed_tx_set &exported_txs, std::vector<cryptonote::address_parse_info> &dsts_info, std::vector<std::string> & tx_device_aux); uint64_t cold_key_image_sync(uint64_t &spent, uint64_t &unspent); @@ -1157,25 +1158,25 @@ private: } a & m_transfers; a & m_account_public_address; - a & m_key_images.parent(); + a & m_key_images; if(ver < 6) return; - a & m_unconfirmed_txs.parent(); + a & m_unconfirmed_txs; if(ver < 7) return; - a & m_payments.parent(); + a & m_payments; if(ver < 8) return; - a & m_tx_keys.parent(); + a & m_tx_keys; if(ver < 9) return; - a & m_confirmed_txs.parent(); + a & m_confirmed_txs; if(ver < 11) return; a & dummy_refresh_height; if(ver < 12) return; - a & m_tx_notes.parent(); + a & m_tx_notes; if(ver < 13) return; if (ver < 17) @@ -1200,7 +1201,7 @@ private: } return; } - a & m_pub_keys.parent(); + a & m_pub_keys; if(ver < 16) return; a & m_address_book; @@ -1221,17 +1222,17 @@ private: a & m_scanned_pool_txs[1]; if (ver < 20) return; - a & m_subaddresses.parent(); + a & m_subaddresses; std::unordered_map<cryptonote::subaddress_index, crypto::public_key> dummy_subaddresses_inv; a & dummy_subaddresses_inv; a & m_subaddress_labels; - a & m_additional_tx_keys.parent(); + a & m_additional_tx_keys; if(ver < 21) return; - a & m_attributes.parent(); + a & m_attributes; if(ver < 22) return; - a & m_unconfirmed_payments.parent(); + a & m_unconfirmed_payments; if(ver < 23) return; a & (std::pair<std::map<std::string, std::string>, std::vector<std::string>>&)m_account_tags; @@ -1243,13 +1244,13 @@ private: a & m_last_block_reward; if(ver < 26) return; - a & m_tx_device.parent(); + a & m_tx_device; if(ver < 27) return; a & m_device_last_key_image_sync; if(ver < 28) return; - a & m_cold_key_images.parent(); + a & m_cold_key_images; if(ver < 29) return; crypto::secret_key dummy_rpc_client_secret_key; // Compatibility for old RPC payment system @@ -1464,7 +1465,7 @@ private: * \brief Get the list of registered account tags. * \return first.Key=(tag's name), first.Value=(tag's label), second[i]=(i-th account's tag) */ - const std::pair<serializable_map<std::string, std::string>, std::vector<std::string>>& get_account_tags(); + const std::pair<std::map<std::string, std::string>, std::vector<std::string>>& get_account_tags(); /*! * \brief Set a tag to the given accounts. * \param account_indices Indices of accounts. @@ -1776,28 +1777,28 @@ private: std::string m_mms_file; const std::unique_ptr<epee::net_utils::http::abstract_http_client> m_http_client; hashchain m_blockchain; - serializable_unordered_map<crypto::hash, unconfirmed_transfer_details> m_unconfirmed_txs; - serializable_unordered_map<crypto::hash, confirmed_transfer_details> m_confirmed_txs; - serializable_unordered_multimap<crypto::hash, pool_payment_details> m_unconfirmed_payments; - serializable_unordered_map<crypto::hash, crypto::secret_key> m_tx_keys; + std::unordered_map<crypto::hash, unconfirmed_transfer_details> m_unconfirmed_txs; + std::unordered_map<crypto::hash, confirmed_transfer_details> m_confirmed_txs; + std::unordered_multimap<crypto::hash, pool_payment_details> m_unconfirmed_payments; + std::unordered_map<crypto::hash, crypto::secret_key> m_tx_keys; cryptonote::checkpoints m_checkpoints; - serializable_unordered_map<crypto::hash, std::vector<crypto::secret_key>> m_additional_tx_keys; + std::unordered_map<crypto::hash, std::vector<crypto::secret_key>> m_additional_tx_keys; transfer_container m_transfers; payment_container m_payments; - serializable_unordered_map<crypto::key_image, size_t> m_key_images; - serializable_unordered_map<crypto::public_key, size_t> m_pub_keys; + std::unordered_map<crypto::key_image, size_t> m_key_images; + std::unordered_map<crypto::public_key, size_t> m_pub_keys; cryptonote::account_public_address m_account_public_address; - serializable_unordered_map<crypto::public_key, cryptonote::subaddress_index> m_subaddresses; + std::unordered_map<crypto::public_key, cryptonote::subaddress_index> m_subaddresses; std::vector<std::vector<std::string>> m_subaddress_labels; - serializable_unordered_map<crypto::hash, std::string> m_tx_notes; - serializable_unordered_map<std::string, std::string> m_attributes; + std::unordered_map<crypto::hash, std::string> m_tx_notes; + std::unordered_map<std::string, std::string> m_attributes; std::vector<tools::wallet2::address_book_row> m_address_book; - std::pair<serializable_map<std::string, std::string>, std::vector<std::string>> m_account_tags; + std::pair<std::map<std::string, std::string>, std::vector<std::string>> m_account_tags; uint64_t m_upper_transaction_weight_limit; //TODO: auto-calc this value or request from daemon, now use some fixed value const std::vector<std::vector<tools::wallet2::multisig_info>> *m_multisig_rescan_info; const std::vector<std::vector<rct::key>> *m_multisig_rescan_k; - serializable_unordered_map<crypto::public_key, crypto::key_image> m_cold_key_images; + std::unordered_map<crypto::public_key, crypto::key_image> m_cold_key_images; std::atomic<bool> m_run; @@ -1868,7 +1869,7 @@ private: bool m_allow_mismatched_daemon_version; // Aux transaction data from device - serializable_unordered_map<crypto::hash, std::string> m_tx_device; + std::unordered_map<crypto::hash, std::string> m_tx_device; std::string m_ring_database; bool m_ring_history_saved; @@ -2313,7 +2314,7 @@ namespace boost a & x.key_images; if (ver < 1) return; - a & x.tx_key_images.parent(); + a & x.tx_key_images; } template <class Archive> diff --git a/src/wallet/wallet_errors.h b/src/wallet/wallet_errors.h index 6706e77ff..5166f868f 100644 --- a/src/wallet/wallet_errors.h +++ b/src/wallet/wallet_errors.h @@ -85,6 +85,7 @@ namespace tools // tx_too_big // zero_amount // zero_destination + // subtract_fee_from_bad_index // wallet_rpc_error * // daemon_busy // no_connection_to_daemon @@ -779,6 +780,15 @@ namespace tools } }; //---------------------------------------------------------------------------------------------------- + struct subtract_fee_from_bad_index : public transfer_error + { + explicit subtract_fee_from_bad_index(std::string&& loc, long bad_index) + : transfer_error(std::move(loc), + "subtractfeefrom: bad index: " + std::to_string(bad_index) + " (indexes are 0-based)") + { + } + }; + //---------------------------------------------------------------------------------------------------- struct wallet_rpc_error : public wallet_logic_error { const std::string& request() const { return m_request; } diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index 0cc42ae3f..c43745dc6 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -987,9 +987,9 @@ namespace tools return amount; } //------------------------------------------------------------------------------------------------------------------------------ - template<typename Ts, typename Tu, typename Tk> + template<typename Ts, typename Tu, typename Tk, typename Ta> bool wallet_rpc_server::fill_response(std::vector<tools::wallet2::pending_tx> &ptx_vector, - bool get_tx_key, Ts& tx_key, Tu &amount, Tu &fee, Tu &weight, std::string &multisig_txset, std::string &unsigned_txset, bool do_not_relay, + bool get_tx_key, Ts& tx_key, Tu &amount, Ta &amounts_by_dest, Tu &fee, Tu &weight, std::string &multisig_txset, std::string &unsigned_txset, bool do_not_relay, Ts &tx_hash, bool get_tx_hex, Ts &tx_blob, bool get_tx_metadata, Ts &tx_metadata, Tk &spent_key_images, epee::json_rpc::error &er) { for (const auto & ptx : ptx_vector) @@ -1006,6 +1006,12 @@ namespace tools fill(fee, ptx.fee); fill(weight, cryptonote::get_transaction_weight(ptx.tx)); + // add amounts by destination + tools::wallet_rpc::amounts_list abd; + for (const auto& dst : ptx.dests) + abd.amounts.push_back(dst.amount); + fill(amounts_by_dest, abd); + // add spent key images tools::wallet_rpc::key_image_list key_image_list; bool all_are_txin_to_key = std::all_of(ptx.tx.vin.begin(), ptx.tx.vin.end(), [&](const cryptonote::txin_v& s_e) -> bool @@ -1086,7 +1092,7 @@ namespace tools { uint64_t mixin = m_wallet->adjust_mixin(req.ring_size ? req.ring_size - 1 : 0); uint32_t priority = m_wallet->adjust_priority(req.priority); - std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_2(dsts, mixin, req.unlock_time, priority, extra, req.account_index, req.subaddr_indices); + std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_2(dsts, mixin, req.unlock_time, priority, extra, req.account_index, req.subaddr_indices, req.subtract_fee_from_outputs); if (ptx_vector.empty()) { @@ -1103,7 +1109,7 @@ namespace tools return false; } - return fill_response(ptx_vector, req.get_tx_key, res.tx_key, res.amount, res.fee, res.weight, res.multisig_txset, res.unsigned_txset, req.do_not_relay, + return fill_response(ptx_vector, req.get_tx_key, res.tx_key, res.amount, res.amounts_by_dest, res.fee, res.weight, res.multisig_txset, res.unsigned_txset, req.do_not_relay, res.tx_hash, req.get_tx_hex, res.tx_blob, req.get_tx_metadata, res.tx_metadata, res.spent_key_images, er); } catch (const std::exception& e) @@ -1151,7 +1157,7 @@ namespace tools return false; } - return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.weight_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay, + return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.amounts_by_dest_list, res.fee_list, res.weight_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay, res.tx_hash_list, req.get_tx_hex, res.tx_blob_list, req.get_tx_metadata, res.tx_metadata_list, res.spent_key_images_list, er); } catch (const std::exception& e) @@ -1540,7 +1546,7 @@ namespace tools { std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_unmixable_sweep_transactions(); - return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.weight_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay, + return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.amounts_by_dest_list, res.fee_list, res.weight_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay, res.tx_hash_list, req.get_tx_hex, res.tx_blob_list, req.get_tx_metadata, res.tx_metadata_list, res.spent_key_images_list, er); } catch (const std::exception& e) @@ -1600,7 +1606,7 @@ namespace tools uint32_t priority = m_wallet->adjust_priority(req.priority); std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_all(req.below_amount, dsts[0].addr, dsts[0].is_subaddress, req.outputs, mixin, req.unlock_time, priority, extra, req.account_index, subaddr_indices); - return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.weight_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay, + return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.amounts_by_dest_list, res.fee_list, res.weight_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay, res.tx_hash_list, req.get_tx_hex, res.tx_blob_list, req.get_tx_metadata, res.tx_metadata_list, res.spent_key_images_list, er); } catch (const std::exception& e) @@ -1677,7 +1683,7 @@ namespace tools return false; } - return fill_response(ptx_vector, req.get_tx_key, res.tx_key, res.amount, res.fee, res.weight, res.multisig_txset, res.unsigned_txset, req.do_not_relay, + return fill_response(ptx_vector, req.get_tx_key, res.tx_key, res.amount, res.amounts_by_dest, res.fee, res.weight, res.multisig_txset, res.unsigned_txset, req.do_not_relay, res.tx_hash, req.get_tx_hex, res.tx_blob, req.get_tx_metadata, res.tx_metadata, res.spent_key_images, er); } catch (const std::exception& e) diff --git a/src/wallet/wallet_rpc_server.h b/src/wallet/wallet_rpc_server.h index 282035052..bfb7013e2 100644 --- a/src/wallet/wallet_rpc_server.h +++ b/src/wallet/wallet_rpc_server.h @@ -35,7 +35,6 @@ #include <string> #include "common/util.h" #include "net/http_server_impl_base.h" -#include "math_helper.h" #include "wallet_rpc_server_commands_defs.h" #include "wallet2.h" @@ -263,9 +262,9 @@ namespace tools bool not_open(epee::json_rpc::error& er); void handle_rpc_exception(const std::exception_ptr& e, epee::json_rpc::error& er, int default_error_code); - template<typename Ts, typename Tu, typename Tk> + template<typename Ts, typename Tu, typename Tk, typename Ta> bool fill_response(std::vector<tools::wallet2::pending_tx> &ptx_vector, - bool get_tx_key, Ts& tx_key, Tu &amount, Tu &fee, Tu &weight, std::string &multisig_txset, std::string &unsigned_txset, bool do_not_relay, + bool get_tx_key, Ts& tx_key, Tu &amount, Ta &amounts_by_dest, Tu &fee, Tu &weight, std::string &multisig_txset, std::string &unsigned_txset, bool do_not_relay, Ts &tx_hash, bool get_tx_hex, Ts &tx_blob, bool get_tx_metadata, Ts &tx_metadata, Tk &spent_key_images, epee::json_rpc::error &er); bool validate_transfer(const std::list<wallet_rpc::transfer_destination>& destinations, const std::string& payment_id, std::vector<cryptonote::tx_destination_entry>& dsts, std::vector<uint8_t>& extra, bool at_least_one_destination, epee::json_rpc::error& er); diff --git a/src/wallet/wallet_rpc_server_commands_defs.h b/src/wallet/wallet_rpc_server_commands_defs.h index c00271030..5d5579ced 100644 --- a/src/wallet/wallet_rpc_server_commands_defs.h +++ b/src/wallet/wallet_rpc_server_commands_defs.h @@ -47,7 +47,7 @@ // advance which version they will stop working with // Don't go over 32767 for any of these #define WALLET_RPC_VERSION_MAJOR 1 -#define WALLET_RPC_VERSION_MINOR 26 +#define WALLET_RPC_VERSION_MINOR 27 #define MAKE_WALLET_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define WALLET_RPC_VERSION MAKE_WALLET_RPC_VERSION(WALLET_RPC_VERSION_MAJOR, WALLET_RPC_VERSION_MINOR) namespace tools @@ -68,8 +68,8 @@ namespace wallet_rpc BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(account_index) KV_SERIALIZE(address_indices) - KV_SERIALIZE_OPT(all_accounts, false); - KV_SERIALIZE_OPT(strict, false); + KV_SERIALIZE_OPT(all_accounts, false) + KV_SERIALIZE_OPT(strict, false) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<request_t> request; @@ -348,9 +348,9 @@ namespace wallet_rpc std::vector<uint32_t> accounts; BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(tag); - KV_SERIALIZE(label); - KV_SERIALIZE(accounts); + KV_SERIALIZE(tag) + KV_SERIALIZE(label) + KV_SERIALIZE(accounts) END_KV_SERIALIZE_MAP() }; @@ -530,11 +530,23 @@ namespace wallet_rpc END_KV_SERIALIZE_MAP() }; + struct amounts_list + { + std::list<uint64_t> amounts; + + bool operator==(const amounts_list& other) const { return amounts == other.amounts; } + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(amounts) + END_KV_SERIALIZE_MAP() + }; + struct single_transfer_response { std::string tx_hash; std::string tx_key; uint64_t amount; + amounts_list amounts_by_dest; uint64_t fee; uint64_t weight; std::string tx_blob; @@ -547,6 +559,7 @@ namespace wallet_rpc KV_SERIALIZE(tx_hash) KV_SERIALIZE(tx_key) KV_SERIALIZE(amount) + KV_SERIALIZE_OPT(amounts_by_dest, decltype(amounts_by_dest)()) KV_SERIALIZE(fee) KV_SERIALIZE(weight) KV_SERIALIZE(tx_blob) @@ -564,6 +577,7 @@ namespace wallet_rpc std::list<transfer_destination> destinations; uint32_t account_index; std::set<uint32_t> subaddr_indices; + std::set<uint32_t> subtract_fee_from_outputs; uint32_t priority; uint64_t ring_size; uint64_t unlock_time; @@ -577,6 +591,7 @@ namespace wallet_rpc KV_SERIALIZE(destinations) KV_SERIALIZE(account_index) KV_SERIALIZE(subaddr_indices) + KV_SERIALIZE_OPT(subtract_fee_from_outputs, decltype(subtract_fee_from_outputs)()) KV_SERIALIZE(priority) KV_SERIALIZE_OPT(ring_size, (uint64_t)0) KV_SERIALIZE(unlock_time) @@ -598,6 +613,7 @@ namespace wallet_rpc std::list<std::string> tx_hash_list; std::list<std::string> tx_key_list; std::list<uint64_t> amount_list; + std::list<amounts_list> amounts_by_dest_list; std::list<uint64_t> fee_list; std::list<uint64_t> weight_list; std::list<std::string> tx_blob_list; @@ -610,6 +626,7 @@ namespace wallet_rpc KV_SERIALIZE(tx_hash_list) KV_SERIALIZE(tx_key_list) KV_SERIALIZE(amount_list) + KV_SERIALIZE_OPT(amounts_by_dest_list, decltype(amounts_by_dest_list)()) KV_SERIALIZE(fee_list) KV_SERIALIZE(weight_list) KV_SERIALIZE(tx_blob_list) @@ -1029,7 +1046,7 @@ namespace wallet_rpc KV_SERIALIZE(tx_hash) KV_SERIALIZE(subaddr_index) KV_SERIALIZE(key_image) - KV_SERIALIZE(pubkey); + KV_SERIALIZE(pubkey) KV_SERIALIZE(block_height) KV_SERIALIZE(frozen) KV_SERIALIZE(unlocked) @@ -1165,7 +1182,7 @@ namespace wallet_rpc bool hard; BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE_OPT(hard, false); + KV_SERIALIZE_OPT(hard, false) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<request_t> request; @@ -1408,21 +1425,21 @@ namespace wallet_rpc uint64_t suggested_confirmations_threshold; BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(txid); - KV_SERIALIZE(payment_id); - KV_SERIALIZE(height); - KV_SERIALIZE(timestamp); - KV_SERIALIZE(amount); - KV_SERIALIZE(amounts); - KV_SERIALIZE(fee); - KV_SERIALIZE(note); - KV_SERIALIZE(destinations); - KV_SERIALIZE(type); + KV_SERIALIZE(txid) + KV_SERIALIZE(payment_id) + KV_SERIALIZE(height) + KV_SERIALIZE(timestamp) + KV_SERIALIZE(amount) + KV_SERIALIZE(amounts) + KV_SERIALIZE(fee) + KV_SERIALIZE(note) + KV_SERIALIZE(destinations) + KV_SERIALIZE(type) KV_SERIALIZE(unlock_time) KV_SERIALIZE(locked) - KV_SERIALIZE(subaddr_index); - KV_SERIALIZE(subaddr_indices); - KV_SERIALIZE(address); + KV_SERIALIZE(subaddr_index) + KV_SERIALIZE(subaddr_indices) + KV_SERIALIZE(address) KV_SERIALIZE(double_spend_seen) KV_SERIALIZE_OPT(confirmations, (uint64_t)0) KV_SERIALIZE_OPT(suggested_confirmations_threshold, (uint64_t)0) @@ -1559,17 +1576,17 @@ namespace wallet_rpc bool all_accounts; BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(in); - KV_SERIALIZE(out); - KV_SERIALIZE(pending); - KV_SERIALIZE(failed); - KV_SERIALIZE(pool); - KV_SERIALIZE(filter_by_height); - KV_SERIALIZE(min_height); - KV_SERIALIZE_OPT(max_height, (uint64_t)CRYPTONOTE_MAX_BLOCK_NUMBER); - KV_SERIALIZE(account_index); - KV_SERIALIZE(subaddr_indices); - KV_SERIALIZE_OPT(all_accounts, false); + KV_SERIALIZE(in) + KV_SERIALIZE(out) + KV_SERIALIZE(pending) + KV_SERIALIZE(failed) + KV_SERIALIZE(pool) + KV_SERIALIZE(filter_by_height) + KV_SERIALIZE(min_height) + KV_SERIALIZE_OPT(max_height, (uint64_t)CRYPTONOTE_MAX_BLOCK_NUMBER) + KV_SERIALIZE(account_index) + KV_SERIALIZE(subaddr_indices) + KV_SERIALIZE_OPT(all_accounts, false) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<request_t> request; @@ -1583,11 +1600,11 @@ namespace wallet_rpc std::list<transfer_entry> pool; BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(in); - KV_SERIALIZE(out); - KV_SERIALIZE(pending); - KV_SERIALIZE(failed); - KV_SERIALIZE(pool); + KV_SERIALIZE(in) + KV_SERIALIZE(out) + KV_SERIALIZE(pending) + KV_SERIALIZE(failed) + KV_SERIALIZE(pool) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<response_t> response; @@ -1601,7 +1618,7 @@ namespace wallet_rpc uint32_t account_index; BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(txid); + KV_SERIALIZE(txid) KV_SERIALIZE_OPT(account_index, (uint32_t)0) END_KV_SERIALIZE_MAP() }; @@ -1613,8 +1630,8 @@ namespace wallet_rpc std::list<transfer_entry> transfers; BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(transfer); - KV_SERIALIZE(transfers); + KV_SERIALIZE(transfer) + KV_SERIALIZE(transfers) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<response_t> response; @@ -1643,7 +1660,7 @@ namespace wallet_rpc std::string signature; BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(signature); + KV_SERIALIZE(signature) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<response_t> response; @@ -1658,9 +1675,9 @@ namespace wallet_rpc std::string signature; BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(data); - KV_SERIALIZE(address); - KV_SERIALIZE(signature); + KV_SERIALIZE(data) + KV_SERIALIZE(address) + KV_SERIALIZE(signature) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<request_t> request; @@ -1673,10 +1690,10 @@ namespace wallet_rpc std::string signature_type; BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(good); - KV_SERIALIZE(version); - KV_SERIALIZE(old); - KV_SERIALIZE(signature_type); + KV_SERIALIZE(good) + KV_SERIALIZE(version) + KV_SERIALIZE(old) + KV_SERIALIZE(signature_type) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<response_t> response; @@ -1703,7 +1720,7 @@ namespace wallet_rpc std::string outputs_data_hex; BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(outputs_data_hex); + KV_SERIALIZE(outputs_data_hex) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<response_t> response; @@ -1716,7 +1733,7 @@ namespace wallet_rpc std::string outputs_data_hex; BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(outputs_data_hex); + KV_SERIALIZE(outputs_data_hex) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<request_t> request; @@ -1726,7 +1743,7 @@ namespace wallet_rpc uint64_t num_imported; BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(num_imported); + KV_SERIALIZE(num_imported) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<response_t> response; @@ -1739,7 +1756,7 @@ namespace wallet_rpc bool all; BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE_OPT(all, false); + KV_SERIALIZE_OPT(all, false) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<request_t> request; @@ -1750,8 +1767,8 @@ namespace wallet_rpc std::string signature; BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(key_image); - KV_SERIALIZE(signature); + KV_SERIALIZE(key_image) + KV_SERIALIZE(signature) END_KV_SERIALIZE_MAP() }; @@ -1761,8 +1778,8 @@ namespace wallet_rpc std::vector<signed_key_image> signed_key_images; BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(offset); - KV_SERIALIZE(signed_key_images); + KV_SERIALIZE(offset) + KV_SERIALIZE(signed_key_images) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<response_t> response; @@ -1776,8 +1793,8 @@ namespace wallet_rpc std::string signature; BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(key_image); - KV_SERIALIZE(signature); + KV_SERIALIZE(key_image) + KV_SERIALIZE(signature) END_KV_SERIALIZE_MAP() }; @@ -1787,8 +1804,8 @@ namespace wallet_rpc std::vector<signed_key_image> signed_key_images; BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE_OPT(offset, (uint32_t)0); - KV_SERIALIZE(signed_key_images); + KV_SERIALIZE_OPT(offset, (uint32_t)0) + KV_SERIALIZE(signed_key_images) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<request_t> request; @@ -1817,11 +1834,11 @@ namespace wallet_rpc std::string recipient_name; BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(address); - KV_SERIALIZE(payment_id); - KV_SERIALIZE(amount); - KV_SERIALIZE(tx_description); - KV_SERIALIZE(recipient_name); + KV_SERIALIZE(address) + KV_SERIALIZE(payment_id) + KV_SERIALIZE(amount) + KV_SERIALIZE(tx_description) + KV_SERIALIZE(recipient_name) END_KV_SERIALIZE_MAP() }; @@ -1861,8 +1878,8 @@ namespace wallet_rpc std::vector<std::string> unknown_parameters; BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(uri); - KV_SERIALIZE(unknown_parameters); + KV_SERIALIZE(uri) + KV_SERIALIZE(unknown_parameters) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<response_t> response; @@ -1887,7 +1904,7 @@ namespace wallet_rpc uint64_t index; BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(index); + KV_SERIALIZE(index) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<response_t> response; @@ -1964,7 +1981,7 @@ namespace wallet_rpc uint64_t index; BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(index); + KV_SERIALIZE(index) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<request_t> request; @@ -2012,8 +2029,8 @@ namespace wallet_rpc bool received_money; BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(blocks_fetched); - KV_SERIALIZE(received_money); + KV_SERIALIZE(blocks_fetched) + KV_SERIALIZE(received_money) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<response_t> response; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 00896818c..736872b04 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -32,7 +32,7 @@ set(MONERO_WALLET_CRYPTO_BENCH "auto" CACHE STRING "Select wallet crypto librari # The docs say this only affects grouping in IDEs set(folder "tests") -set(TEST_DATA_DIR "${CMAKE_CURRENT_LIST_DIR}/data") +set(TEST_DATA_DIR "${CMAKE_CURRENT_BINARY_DIR}/data") if (WIN32 AND STATIC) add_definitions(-DSTATICLIB) diff --git a/tests/core_tests/block_reward.cpp b/tests/core_tests/block_reward.cpp index bd2fc2a9b..7f75aea5b 100644 --- a/tests/core_tests/block_reward.cpp +++ b/tests/core_tests/block_reward.cpp @@ -172,21 +172,21 @@ bool gen_block_reward::generate(std::vector<test_event_entry>& events) const return false; // Test: fee increases block reward - transaction tx_0(construct_tx_with_fee(events, blk_5, miner_account, bob_account, MK_COINS(1), 3 * TESTS_DEFAULT_FEE)); + transaction tx_0(construct_tx_with_fee(events, blk_5r, miner_account, bob_account, MK_COINS(1), 3 * TESTS_DEFAULT_FEE)); MAKE_NEXT_BLOCK_TX1(events, blk_6, blk_5r, miner_account, tx_0); DO_CALLBACK(events, "mark_checked_block"); // Test: fee from all block transactions increase block reward std::list<transaction> txs_0; - txs_0.push_back(construct_tx_with_fee(events, blk_5, miner_account, bob_account, MK_COINS(1), 5 * TESTS_DEFAULT_FEE)); - txs_0.push_back(construct_tx_with_fee(events, blk_5, miner_account, bob_account, MK_COINS(1), 7 * TESTS_DEFAULT_FEE)); + txs_0.push_back(construct_tx_with_fee(events, blk_5r, miner_account, bob_account, MK_COINS(1), 5 * TESTS_DEFAULT_FEE)); + txs_0.push_back(construct_tx_with_fee(events, blk_5r, miner_account, bob_account, MK_COINS(1), 7 * TESTS_DEFAULT_FEE)); MAKE_NEXT_BLOCK_TX_LIST(events, blk_7, blk_6, miner_account, txs_0); DO_CALLBACK(events, "mark_checked_block"); // Test: block reward == transactions fee { - transaction tx_1 = construct_tx_with_fee(events, blk_5, miner_account, bob_account, MK_COINS(1), 11 * TESTS_DEFAULT_FEE); - transaction tx_2 = construct_tx_with_fee(events, blk_5, miner_account, bob_account, MK_COINS(1), 13 * TESTS_DEFAULT_FEE); + transaction tx_1 = construct_tx_with_fee(events, blk_5r, miner_account, bob_account, MK_COINS(1), 11 * TESTS_DEFAULT_FEE); + transaction tx_2 = construct_tx_with_fee(events, blk_5r, miner_account, bob_account, MK_COINS(1), 13 * TESTS_DEFAULT_FEE); size_t txs_1_weight = get_transaction_weight(tx_1) + get_transaction_weight(tx_2); uint64_t txs_fee = get_tx_fee(tx_1) + get_tx_fee(tx_2); diff --git a/tests/core_tests/block_validation.cpp b/tests/core_tests/block_validation.cpp index 37f19d94b..071d2198e 100644 --- a/tests/core_tests/block_validation.cpp +++ b/tests/core_tests/block_validation.cpp @@ -580,7 +580,7 @@ bool gen_block_invalid_binary_format::generate(std::vector<test_event_entry>& ev while (diffic < 1500); blk_last = boost::get<block>(events.back()); - MAKE_TX(events, tx_0, miner_account, miner_account, MK_COINS(30), boost::get<block>(events[1])); + MAKE_TX(events, tx_0, miner_account, miner_account, MK_COINS(30), blk_last); DO_CALLBACK(events, "corrupt_blocks_boundary"); block blk_test; diff --git a/tests/core_tests/chain_split_1.cpp b/tests/core_tests/chain_split_1.cpp index 8f49b1af8..cffabb9c1 100644 --- a/tests/core_tests/chain_split_1.cpp +++ b/tests/core_tests/chain_split_1.cpp @@ -114,9 +114,9 @@ bool gen_simple_chain_split_1::generate(std::vector<test_event_entry> &events) c REWIND_BLOCKS(events, blk_23r, blk_23, first_miner_account); // 30...N1 GENERATE_ACCOUNT(alice); - MAKE_TX(events, tx_0, first_miner_account, alice, MK_COINS(10), blk_23); // N1+1 - MAKE_TX(events, tx_1, first_miner_account, alice, MK_COINS(20), blk_23); // N1+2 - MAKE_TX(events, tx_2, first_miner_account, alice, MK_COINS(30), blk_23); // N1+3 + MAKE_TX(events, tx_0, first_miner_account, alice, MK_COINS(10), blk_23r); // N1+1 + MAKE_TX(events, tx_1, first_miner_account, alice, MK_COINS(20), blk_23r); // N1+2 + MAKE_TX(events, tx_2, first_miner_account, alice, MK_COINS(30), blk_23r); // N1+3 DO_CALLBACK(events, "check_mempool_1"); // N1+4 MAKE_NEXT_BLOCK_TX1(events, blk_24, blk_23r, first_miner_account, tx_0); // N1+5 DO_CALLBACK(events, "check_mempool_2"); // N1+6 diff --git a/tests/core_tests/chain_switch_1.cpp b/tests/core_tests/chain_switch_1.cpp index 8271ab783..2d9cdcbd8 100644 --- a/tests/core_tests/chain_switch_1.cpp +++ b/tests/core_tests/chain_switch_1.cpp @@ -72,37 +72,37 @@ bool gen_chain_switch_1::generate(std::vector<test_event_entry>& events) const MAKE_ACCOUNT(events, recipient_account_3); // 3 MAKE_ACCOUNT(events, recipient_account_4); // 4 REWIND_BLOCKS(events, blk_0r, blk_0, miner_account) // <N blocks> - MAKE_TX(events, tx_00, miner_account, recipient_account_1, MK_COINS(5), blk_0); // 5 + N + MAKE_TX(events, tx_00, miner_account, recipient_account_1, MK_COINS(5), blk_0r); // 5 + N MAKE_NEXT_BLOCK_TX1(events, blk_1, blk_0r, miner_account, tx_00); // 6 + N MAKE_NEXT_BLOCK(events, blk_2, blk_1, miner_account); // 7 + N REWIND_BLOCKS(events, blk_2r, blk_2, miner_account) // <N blocks> // Transactions to test account balances after switch - MAKE_TX_LIST_START(events, txs_blk_3, miner_account, recipient_account_2, MK_COINS(7), blk_2); // 8 + 2N - MAKE_TX_LIST_START(events, txs_blk_4, miner_account, recipient_account_3, MK_COINS(11), blk_2); // 9 + 2N - MAKE_TX_LIST_START(events, txs_blk_5, miner_account, recipient_account_4, MK_COINS(13), blk_2); // 10 + 2N + MAKE_TX_LIST_START(events, txs_blk_3, miner_account, recipient_account_2, MK_COINS(7), blk_2r); // 8 + 2N + MAKE_TX_LIST_START(events, txs_blk_4, miner_account, recipient_account_3, MK_COINS(11), blk_2r); // 9 + 2N + MAKE_TX_LIST_START(events, txs_blk_5, miner_account, recipient_account_4, MK_COINS(13), blk_2r); // 10 + 2N std::list<transaction> txs_blk_6; txs_blk_6.push_back(txs_blk_4.front()); // Transactions, that has different order in alt block chains - MAKE_TX_LIST(events, txs_blk_3, miner_account, recipient_account_1, MK_COINS(1), blk_2); // 11 + 2N + MAKE_TX_LIST(events, txs_blk_3, miner_account, recipient_account_1, MK_COINS(1), blk_2r); // 11 + 2N txs_blk_5.push_back(txs_blk_3.back()); - MAKE_TX_LIST(events, txs_blk_3, miner_account, recipient_account_1, MK_COINS(2), blk_2); // 12 + 2N + MAKE_TX_LIST(events, txs_blk_3, miner_account, recipient_account_1, MK_COINS(2), blk_2r); // 12 + 2N txs_blk_6.push_back(txs_blk_3.back()); - MAKE_TX_LIST(events, txs_blk_3, miner_account, recipient_account_2, MK_COINS(1), blk_2); // 13 + 2N + MAKE_TX_LIST(events, txs_blk_3, miner_account, recipient_account_2, MK_COINS(1), blk_2r); // 13 + 2N txs_blk_5.push_back(txs_blk_3.back()); - MAKE_TX_LIST(events, txs_blk_4, miner_account, recipient_account_2, MK_COINS(2), blk_2); // 14 + 2N + MAKE_TX_LIST(events, txs_blk_4, miner_account, recipient_account_2, MK_COINS(2), blk_2r); // 14 + 2N txs_blk_5.push_back(txs_blk_4.back()); - MAKE_TX_LIST(events, txs_blk_3, miner_account, recipient_account_3, MK_COINS(1), blk_2); // 15 + 2N + MAKE_TX_LIST(events, txs_blk_3, miner_account, recipient_account_3, MK_COINS(1), blk_2r); // 15 + 2N txs_blk_6.push_back(txs_blk_3.back()); - MAKE_TX_LIST(events, txs_blk_4, miner_account, recipient_account_3, MK_COINS(2), blk_2); // 16 + 2N + MAKE_TX_LIST(events, txs_blk_4, miner_account, recipient_account_3, MK_COINS(2), blk_2r); // 16 + 2N txs_blk_5.push_back(txs_blk_4.back()); - MAKE_TX_LIST(events, txs_blk_4, miner_account, recipient_account_4, MK_COINS(1), blk_2); // 17 + 2N + MAKE_TX_LIST(events, txs_blk_4, miner_account, recipient_account_4, MK_COINS(1), blk_2r); // 17 + 2N txs_blk_5.push_back(txs_blk_4.back()); - MAKE_TX_LIST(events, txs_blk_3, miner_account, recipient_account_4, MK_COINS(2), blk_2); // 18 + 2N + MAKE_TX_LIST(events, txs_blk_3, miner_account, recipient_account_4, MK_COINS(2), blk_2r); // 18 + 2N txs_blk_6.push_back(txs_blk_3.back()); MAKE_NEXT_BLOCK_TX_LIST(events, blk_3, blk_2r, miner_account, txs_blk_3); // 19 + 2N diff --git a/tests/core_tests/chaingen.cpp b/tests/core_tests/chaingen.cpp index b66aa7924..9aa0de8e5 100644 --- a/tests/core_tests/chaingen.cpp +++ b/tests/core_tests/chaingen.cpp @@ -504,7 +504,7 @@ bool init_spent_output_indices(map_output_idx_t& outs, map_output_t& outs_mine, return true; } -bool fill_output_entries(std::vector<output_index>& out_indices, size_t sender_out, size_t nmix, size_t& real_entry_idx, std::vector<tx_source_entry::output_entry>& output_entries) +bool fill_output_entries(std::vector<output_index>& out_indices, size_t sender_out, size_t nmix, size_t& real_entry_idx, std::vector<tx_source_entry::output_entry>& output_entries, uint64_t cur_height, const boost::optional<fnc_accept_output_t>& fnc_accept = boost::none) { if (out_indices.size() <= nmix) return false; @@ -516,6 +516,10 @@ bool fill_output_entries(std::vector<output_index>& out_indices, size_t sender_o const output_index& oi = out_indices[i]; if (oi.spent) continue; + if (oi.unlock_time < CRYPTONOTE_MAX_BLOCK_NUMBER && oi.unlock_time > cur_height + 1) + continue; + if (fnc_accept && !(fnc_accept.get())({.oi=oi, .cur_height=cur_height})) + continue; bool append = false; if (i == sender_out) @@ -542,7 +546,8 @@ bool fill_output_entries(std::vector<output_index>& out_indices, size_t sender_o } bool fill_tx_sources(std::vector<tx_source_entry>& sources, const std::vector<test_event_entry>& events, - const block& blk_head, const cryptonote::account_base& from, uint64_t amount, size_t nmix) + const block& blk_head, const cryptonote::account_base& from, uint64_t amount, size_t nmix, + bool check_unlock_time, const boost::optional<fnc_accept_output_t>& fnc_accept) { map_output_idx_t outs; map_output_t outs_mine; @@ -559,6 +564,7 @@ bool fill_tx_sources(std::vector<tx_source_entry>& sources, const std::vector<te return false; // Iterate in reverse is more efficiency + uint64_t head_height = check_unlock_time ? get_block_height(blk_head) : std::numeric_limits<uint64_t>::max() - 1; uint64_t sources_amount = 0; bool sources_found = false; BOOST_REVERSE_FOREACH(const map_output_t::value_type o, outs_mine) @@ -571,13 +577,17 @@ bool fill_tx_sources(std::vector<tx_source_entry>& sources, const std::vector<te continue; if (oi.rct) continue; + if (oi.unlock_time < CRYPTONOTE_MAX_BLOCK_NUMBER && oi.unlock_time > head_height + 1) + continue; + if (fnc_accept && !(fnc_accept.get())({.oi=oi, .cur_height=head_height})) + continue; cryptonote::tx_source_entry ts; ts.amount = oi.amount; ts.real_output_in_tx_index = oi.out_no; ts.real_out_tx_key = get_tx_pub_key_from_extra(*oi.p_tx); // incoming tx public key size_t realOutput; - if (!fill_output_entries(outs[o.first], sender_out, nmix, realOutput, ts.outputs)) + if (!fill_output_entries(outs[o.first], sender_out, nmix, realOutput, ts.outputs, head_height, fnc_accept)) continue; ts.real_output = realOutput; @@ -692,7 +702,7 @@ void block_tracker::global_indices(const cryptonote::transaction *tx, std::vecto } } -void block_tracker::get_fake_outs(size_t num_outs, uint64_t amount, uint64_t global_index, uint64_t cur_height, std::vector<get_outs_entry> &outs){ +void block_tracker::get_fake_outs(size_t num_outs, uint64_t amount, uint64_t global_index, uint64_t cur_height, std::vector<get_outs_entry> &outs, const boost::optional<fnc_accept_output_t>& fnc_accept){ auto & vct = m_outs[amount]; const size_t n_outs = vct.size(); CHECK_AND_ASSERT_THROW_MES(n_outs > 0, "n_outs is 0"); @@ -717,15 +727,16 @@ void block_tracker::get_fake_outs(size_t num_outs, uint64_t amount, uint64_t glo continue; if (oi.out.type() != typeid(cryptonote::txout_to_key)) continue; - if (oi.unlock_time > cur_height) + if (oi.unlock_time < CRYPTONOTE_MAX_BLOCK_NUMBER && oi.unlock_time > cur_height + 1) continue; if (used.find(oi_idx) != used.end()) continue; + if (fnc_accept && !(fnc_accept.get())({.oi=oi, .cur_height=cur_height})) + continue; rct::key comm = oi.commitment(); auto out = boost::get<txout_to_key>(oi.out); - auto item = std::make_tuple(oi.idx, out.key, comm); - outs.push_back(item); + outs.emplace_back(oi.idx, out.key, comm); used.insert(oi_idx); } } @@ -924,13 +935,14 @@ void fill_tx_destinations(const var_addr_t& from, const cryptonote::account_publ void fill_tx_sources_and_destinations(const std::vector<test_event_entry>& events, const block& blk_head, const cryptonote::account_base& from, const cryptonote::account_public_address& to, uint64_t amount, uint64_t fee, size_t nmix, std::vector<tx_source_entry>& sources, - std::vector<tx_destination_entry>& destinations) + std::vector<tx_destination_entry>& destinations, bool check_unlock_time, + const boost::optional<fnc_accept_output_t>& fnc_tx_in_accept) { sources.clear(); destinations.clear(); - if (!fill_tx_sources(sources, events, blk_head, from, amount + fee, nmix)) - throw std::runtime_error("couldn't fill transaction sources"); + if (!fill_tx_sources(sources, events, blk_head, from, amount + fee, nmix, check_unlock_time, fnc_tx_in_accept)) + throw tx_construct_tx_fill_error(); fill_tx_destinations(from, to, amount, fee, sources, destinations, false); } @@ -938,9 +950,10 @@ void fill_tx_sources_and_destinations(const std::vector<test_event_entry>& event void fill_tx_sources_and_destinations(const std::vector<test_event_entry>& events, const block& blk_head, const cryptonote::account_base& from, const cryptonote::account_base& to, uint64_t amount, uint64_t fee, size_t nmix, std::vector<tx_source_entry>& sources, - std::vector<tx_destination_entry>& destinations) + std::vector<tx_destination_entry>& destinations, bool check_unlock_time, + const boost::optional<fnc_accept_output_t>& fnc_tx_in_accept) { - fill_tx_sources_and_destinations(events, blk_head, from, to.get_keys().m_account_address, amount, fee, nmix, sources, destinations); + fill_tx_sources_and_destinations(events, blk_head, from, to.get_keys().m_account_address, amount, fee, nmix, sources, destinations, check_unlock_time, fnc_tx_in_accept); } cryptonote::tx_destination_entry build_dst(const var_addr_t& to, bool is_subaddr, uint64_t amount) @@ -952,10 +965,14 @@ cryptonote::tx_destination_entry build_dst(const var_addr_t& to, bool is_subaddr return de; } -std::vector<cryptonote::tx_destination_entry> build_dsts(const var_addr_t& to1, bool sub1, uint64_t am1) +std::vector<cryptonote::tx_destination_entry> build_dsts(const var_addr_t& to1, bool sub1, uint64_t am1, size_t repeat) { std::vector<cryptonote::tx_destination_entry> res; - res.push_back(build_dst(to1, sub1, am1)); + res.reserve(repeat); + for(size_t i = 0; i < repeat; ++i) + { + res.emplace_back(build_dst(to1, sub1, am1)); + } return res; } @@ -1019,25 +1036,27 @@ bool construct_miner_tx_manually(size_t height, uint64_t already_generated_coins bool construct_tx_to_key(const std::vector<test_event_entry>& events, cryptonote::transaction& tx, const cryptonote::block& blk_head, const cryptonote::account_base& from, const var_addr_t& to, uint64_t amount, - uint64_t fee, size_t nmix, bool rct, rct::RangeProofType range_proof_type, int bp_version) + uint64_t fee, size_t nmix, bool rct, rct::RangeProofType range_proof_type, int bp_version, + bool check_unlock_time, const boost::optional<fnc_accept_output_t>& fnc_tx_in_accept) { vector<tx_source_entry> sources; vector<tx_destination_entry> destinations; - fill_tx_sources_and_destinations(events, blk_head, from, get_address(to), amount, fee, nmix, sources, destinations); + fill_tx_sources_and_destinations(events, blk_head, from, get_address(to), amount, fee, nmix, sources, destinations, check_unlock_time, fnc_tx_in_accept); return construct_tx_rct(from.get_keys(), sources, destinations, from.get_keys().m_account_address, std::vector<uint8_t>(), tx, 0, rct, range_proof_type, bp_version); } bool construct_tx_to_key(const std::vector<test_event_entry>& events, cryptonote::transaction& tx, const cryptonote::block& blk_head, - const cryptonote::account_base& from, std::vector<cryptonote::tx_destination_entry> destinations, - uint64_t fee, size_t nmix, bool rct, rct::RangeProofType range_proof_type, int bp_version) + const cryptonote::account_base& from, const std::vector<cryptonote::tx_destination_entry>& destinations, + uint64_t fee, size_t nmix, bool rct, rct::RangeProofType range_proof_type, int bp_version, + bool check_unlock_time, const boost::optional<fnc_accept_output_t>& fnc_tx_in_accept) { vector<tx_source_entry> sources; vector<tx_destination_entry> destinations_all; uint64_t amount = sum_amount(destinations); - if (!fill_tx_sources(sources, events, blk_head, from, amount + fee, nmix)) - throw std::runtime_error("couldn't fill transaction sources"); + if (!fill_tx_sources(sources, events, blk_head, from, amount + fee, nmix, check_unlock_time, fnc_tx_in_accept)) + throw tx_construct_tx_fill_error(); fill_tx_destinations(from, destinations, fee, sources, destinations_all, false); diff --git a/tests/core_tests/chaingen.h b/tests/core_tests/chaingen.h index 31580502d..41f7275d3 100644 --- a/tests/core_tests/chaingen.h +++ b/tests/core_tests/chaingen.h @@ -372,6 +372,13 @@ typedef struct { uint64_t amount; } dest_wrapper_t; +typedef struct { + const output_index &oi; + uint64_t cur_height; +} fnc_accept_output_crate_t; + +typedef std::function<bool(const fnc_accept_output_crate_t &info)> fnc_accept_output_t; + // Daemon functionality class block_tracker { @@ -388,7 +395,7 @@ public: void process(const std::vector<const cryptonote::block*>& blockchain, const map_hash2tx_t& mtx); void process(const cryptonote::block* blk, const cryptonote::transaction * tx, size_t i); void global_indices(const cryptonote::transaction *tx, std::vector<uint64_t> &indices); - void get_fake_outs(size_t num_outs, uint64_t amount, uint64_t global_index, uint64_t cur_height, std::vector<get_outs_entry> &outs); + void get_fake_outs(size_t num_outs, uint64_t amount, uint64_t global_index, uint64_t cur_height, std::vector<get_outs_entry> &outs, const boost::optional<fnc_accept_output_t>& fnc_accept = boost::none); std::string dump_data(); void dump_data(const std::string & fname); @@ -405,6 +412,19 @@ private: } }; +class tx_construct_error : public std::runtime_error +{ +public: + tx_construct_error(const char *s) : runtime_error(s) { } +}; + +class tx_construct_tx_fill_error : public tx_construct_error +{ +public: + tx_construct_tx_fill_error() : tx_construct_error("Couldn't fill transaction sources") { } + tx_construct_tx_fill_error(const char *s) : tx_construct_error(s) { } +}; + std::string dump_data(const cryptonote::transaction &tx); cryptonote::account_public_address get_address(const var_addr_t& inp); cryptonote::account_public_address get_address(const cryptonote::account_public_address& inp); @@ -416,7 +436,7 @@ inline cryptonote::difficulty_type get_test_difficulty(const boost::optional<uin inline uint64_t current_difficulty_window(const boost::optional<uint8_t>& hf_ver=boost::none){ return !hf_ver || hf_ver.get() <= 1 ? DIFFICULTY_TARGET_V1 : DIFFICULTY_TARGET_V2; } cryptonote::tx_destination_entry build_dst(const var_addr_t& to, bool is_subaddr=false, uint64_t amount=0); -std::vector<cryptonote::tx_destination_entry> build_dsts(const var_addr_t& to1, bool sub1=false, uint64_t am1=0); +std::vector<cryptonote::tx_destination_entry> build_dsts(const var_addr_t& to1, bool sub1=false, uint64_t am1=0, size_t repeat=1); std::vector<cryptonote::tx_destination_entry> build_dsts(std::initializer_list<dest_wrapper_t> inps); uint64_t sum_amount(const std::vector<cryptonote::tx_destination_entry>& destinations); uint64_t sum_amount(const std::vector<cryptonote::tx_source_entry>& sources); @@ -428,11 +448,13 @@ bool construct_miner_tx_manually(size_t height, uint64_t already_generated_coins bool construct_tx_to_key(const std::vector<test_event_entry>& events, cryptonote::transaction& tx, const cryptonote::block& blk_head, const cryptonote::account_base& from, const var_addr_t& to, uint64_t amount, - uint64_t fee, size_t nmix, bool rct=false, rct::RangeProofType range_proof_type=rct::RangeProofBorromean, int bp_version = 0); + uint64_t fee, size_t nmix, bool rct=false, rct::RangeProofType range_proof_type=rct::RangeProofBorromean, int bp_version = 0, + bool check_unlock_time = true, const boost::optional<fnc_accept_output_t>& fnc_tx_in_accept = boost::none); bool construct_tx_to_key(const std::vector<test_event_entry>& events, cryptonote::transaction& tx, const cryptonote::block& blk_head, - const cryptonote::account_base& from, std::vector<cryptonote::tx_destination_entry> destinations, - uint64_t fee, size_t nmix, bool rct=false, rct::RangeProofType range_proof_type=rct::RangeProofBorromean, int bp_version = 0); + const cryptonote::account_base& from, const std::vector<cryptonote::tx_destination_entry>& destinations, + uint64_t fee, size_t nmix, bool rct=false, rct::RangeProofType range_proof_type=rct::RangeProofBorromean, int bp_version = 0, + bool check_unlock_time = true, const boost::optional<fnc_accept_output_t>& fnc_tx_in_accept = boost::none); bool construct_tx_to_key(cryptonote::transaction& tx, const cryptonote::account_base& from, const var_addr_t& to, uint64_t amount, std::vector<cryptonote::tx_source_entry> &sources, @@ -463,6 +485,10 @@ bool trim_block_chain(std::vector<const cryptonote::block*>& blockchain, const c bool find_block_chain(const std::vector<test_event_entry>& events, std::vector<cryptonote::block>& blockchain, map_hash2tx_t& mtx, const crypto::hash& head); bool find_block_chain(const std::vector<test_event_entry>& events, std::vector<const cryptonote::block*>& blockchain, map_hash2tx_t& mtx, const crypto::hash& head); +bool fill_tx_sources(std::vector<cryptonote::tx_source_entry>& sources, const std::vector<test_event_entry>& events, + const cryptonote::block& blk_head, const cryptonote::account_base& from, uint64_t amount, size_t nmix, + bool check_unlock_time = true, const boost::optional<fnc_accept_output_t>& fnc_accept = boost::none); + void fill_tx_destinations(const var_addr_t& from, const cryptonote::account_public_address& to, uint64_t amount, uint64_t fee, const std::vector<cryptonote::tx_source_entry> &sources, @@ -486,13 +512,17 @@ void fill_tx_sources_and_destinations(const std::vector<test_event_entry>& event const cryptonote::account_base& from, const cryptonote::account_public_address& to, uint64_t amount, uint64_t fee, size_t nmix, std::vector<cryptonote::tx_source_entry>& sources, - std::vector<cryptonote::tx_destination_entry>& destinations); + std::vector<cryptonote::tx_destination_entry>& destinations, + bool check_unlock_time = true, + const boost::optional<fnc_accept_output_t>& fnc_tx_in_accept = boost::none); void fill_tx_sources_and_destinations(const std::vector<test_event_entry>& events, const cryptonote::block& blk_head, const cryptonote::account_base& from, const cryptonote::account_base& to, uint64_t amount, uint64_t fee, size_t nmix, std::vector<cryptonote::tx_source_entry>& sources, - std::vector<cryptonote::tx_destination_entry>& destinations); + std::vector<cryptonote::tx_destination_entry>& destinations, + bool check_unlock_time = true, + const boost::optional<fnc_accept_output_t>& fnc_tx_in_accept = boost::none); uint64_t get_balance(const cryptonote::account_base& addr, const std::vector<cryptonote::block>& blockchain, const map_hash2tx_t& mtx); @@ -883,15 +913,6 @@ inline bool do_replay_file(const std::string& filename) } \ VEC_EVENTS.push_back(BLK_NAME); -#define MAKE_NEXT_BLOCK_TX1_HF(VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC, TX1, HF) \ - cryptonote::block BLK_NAME; \ - { \ - std::list<cryptonote::transaction> tx_list; \ - tx_list.push_back(TX1); \ - generator.construct_block(BLK_NAME, PREV_BLOCK, MINER_ACC, tx_list, HF); \ - } \ - VEC_EVENTS.push_back(BLK_NAME); - #define MAKE_NEXT_BLOCK_TX_LIST(VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC, TXLIST) \ cryptonote::block BLK_NAME; \ generator.construct_block(BLK_NAME, PREV_BLOCK, MINER_ACC, TXLIST); \ @@ -916,18 +937,12 @@ inline bool do_replay_file(const std::string& filename) #define REWIND_BLOCKS_N(VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC, COUNT) REWIND_BLOCKS_N_HF(VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC, COUNT, boost::none) #define REWIND_BLOCKS(VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC) REWIND_BLOCKS_N(VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC, CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW) -#define REWIND_BLOCKS_HF(VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC, HF) REWIND_BLOCKS_N_HF(VEC_EVENTS, BLK_NAME, PREV_BLOCK, MINER_ACC, CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW, HF) #define MAKE_TX_MIX(VEC_EVENTS, TX_NAME, FROM, TO, AMOUNT, NMIX, HEAD) \ cryptonote::transaction TX_NAME; \ construct_tx_to_key(VEC_EVENTS, TX_NAME, HEAD, FROM, TO, AMOUNT, TESTS_DEFAULT_FEE, NMIX); \ VEC_EVENTS.push_back(TX_NAME); -#define MAKE_TX_MIX_RCT(VEC_EVENTS, TX_NAME, FROM, TO, AMOUNT, NMIX, HEAD) \ - cryptonote::transaction TX_NAME; \ - construct_tx_to_key(VEC_EVENTS, TX_NAME, HEAD, FROM, TO, AMOUNT, TESTS_DEFAULT_FEE, NMIX, true, rct::RangeProofPaddedBulletproof); \ - VEC_EVENTS.push_back(TX_NAME); - #define MAKE_TX(VEC_EVENTS, TX_NAME, FROM, TO, AMOUNT, HEAD) MAKE_TX_MIX(VEC_EVENTS, TX_NAME, FROM, TO, AMOUNT, 0, HEAD) #define MAKE_TX_MIX_LIST(VEC_EVENTS, SET_NAME, FROM, TO, AMOUNT, NMIX, HEAD) \ @@ -938,36 +953,12 @@ inline bool do_replay_file(const std::string& filename) VEC_EVENTS.push_back(t); \ } -#define MAKE_TX_MIX_LIST_RCT(VEC_EVENTS, SET_NAME, FROM, TO, AMOUNT, NMIX, HEAD) \ - MAKE_TX_MIX_LIST_RCT_EX(VEC_EVENTS, SET_NAME, FROM, TO, AMOUNT, NMIX, HEAD, rct::RangeProofPaddedBulletproof, 1) -#define MAKE_TX_MIX_LIST_RCT_EX(VEC_EVENTS, SET_NAME, FROM, TO, AMOUNT, NMIX, HEAD, RCT_TYPE, BP_VER) \ - { \ - cryptonote::transaction t; \ - construct_tx_to_key(VEC_EVENTS, t, HEAD, FROM, TO, AMOUNT, TESTS_DEFAULT_FEE, NMIX, true, RCT_TYPE, BP_VER); \ - SET_NAME.push_back(t); \ - VEC_EVENTS.push_back(t); \ - } - -#define MAKE_TX_MIX_DEST_LIST_RCT(VEC_EVENTS, SET_NAME, FROM, TO, NMIX, HEAD) \ - MAKE_TX_MIX_DEST_LIST_RCT_EX(VEC_EVENTS, SET_NAME, FROM, TO, NMIX, HEAD, rct::RangeProofPaddedBulletproof, 1) -#define MAKE_TX_MIX_DEST_LIST_RCT_EX(VEC_EVENTS, SET_NAME, FROM, TO, NMIX, HEAD, RCT_TYPE, BP_VER) \ - { \ - cryptonote::transaction t; \ - construct_tx_to_key(VEC_EVENTS, t, HEAD, FROM, TO, TESTS_DEFAULT_FEE, NMIX, true, RCT_TYPE, BP_VER); \ - SET_NAME.push_back(t); \ - VEC_EVENTS.push_back(t); \ - } - #define MAKE_TX_LIST(VEC_EVENTS, SET_NAME, FROM, TO, AMOUNT, HEAD) MAKE_TX_MIX_LIST(VEC_EVENTS, SET_NAME, FROM, TO, AMOUNT, 0, HEAD) #define MAKE_TX_LIST_START(VEC_EVENTS, SET_NAME, FROM, TO, AMOUNT, HEAD) \ std::list<cryptonote::transaction> SET_NAME; \ MAKE_TX_LIST(VEC_EVENTS, SET_NAME, FROM, TO, AMOUNT, HEAD); -#define MAKE_TX_LIST_START_RCT(VEC_EVENTS, SET_NAME, FROM, TO, AMOUNT, NMIX, HEAD) \ - std::list<cryptonote::transaction> SET_NAME; \ - MAKE_TX_MIX_LIST_RCT(VEC_EVENTS, SET_NAME, FROM, TO, AMOUNT, NMIX, HEAD); - #define MAKE_MINER_TX_AND_KEY_AT_HF_MANUALLY(TX, BLK, HF_VERSION, KEY) \ transaction TX; \ if (!construct_miner_tx_manually(get_block_height(BLK) + 1, generator.get_already_generated_coins(BLK), \ diff --git a/tests/core_tests/chaingen001.cpp b/tests/core_tests/chaingen001.cpp index 8738585e0..1dde16ff5 100644 --- a/tests/core_tests/chaingen001.cpp +++ b/tests/core_tests/chaingen001.cpp @@ -120,18 +120,18 @@ bool gen_simple_chain_001::generate(std::vector<test_event_entry> &events) std::cout << "BALANCE = " << get_balance(miner, chain, mtx) << std::endl; REWIND_BLOCKS(events, blk_2r, blk_2, miner); - MAKE_TX_LIST_START(events, txlist_0, miner, alice, MK_COINS(1), blk_2); - MAKE_TX_LIST(events, txlist_0, miner, alice, MK_COINS(2), blk_2); - MAKE_TX_LIST(events, txlist_0, miner, alice, MK_COINS(4), blk_2); + MAKE_TX_LIST_START(events, txlist_0, miner, alice, MK_COINS(1), blk_2r); + MAKE_TX_LIST(events, txlist_0, miner, alice, MK_COINS(2), blk_2r); + MAKE_TX_LIST(events, txlist_0, miner, alice, MK_COINS(4), blk_2r); MAKE_NEXT_BLOCK_TX_LIST(events, blk_3, blk_2r, miner, txlist_0); REWIND_BLOCKS(events, blk_3r, blk_3, miner); - MAKE_TX(events, tx_1, miner, alice, MK_COINS(50), blk_3); + MAKE_TX(events, tx_1, miner, alice, MK_COINS(50), blk_3r); MAKE_NEXT_BLOCK_TX1(events, blk_4, blk_3r, miner, tx_1); REWIND_BLOCKS(events, blk_4r, blk_4, miner); - MAKE_TX(events, tx_2, miner, alice, MK_COINS(50), blk_4); + MAKE_TX(events, tx_2, miner, alice, MK_COINS(50), blk_4r); MAKE_NEXT_BLOCK_TX1(events, blk_5, blk_4r, miner, tx_2); REWIND_BLOCKS(events, blk_5r, blk_5, miner); - MAKE_TX(events, tx_3, miner, alice, MK_COINS(50), blk_5); + MAKE_TX(events, tx_3, miner, alice, MK_COINS(50), blk_5r); MAKE_NEXT_BLOCK_TX1(events, blk_6, blk_5r, miner, tx_3); DO_CALLBACK(events, "verify_callback_1"); diff --git a/tests/core_tests/double_spend.h b/tests/core_tests/double_spend.h index b1f071f38..85c855bcf 100644 --- a/tests/core_tests/double_spend.h +++ b/tests/core_tests/double_spend.h @@ -144,7 +144,7 @@ public: MAKE_ACCOUNT(events, bob_account); \ MAKE_ACCOUNT(events, alice_account); \ REWIND_BLOCKS(events, blk_0r, blk_0, miner_account); \ - MAKE_TX(events, tx_0, miner_account, bob_account, send_amount, blk_0); \ + MAKE_TX(events, tx_0, miner_account, bob_account, send_amount, blk_0r); \ MAKE_NEXT_BLOCK_TX1(events, blk_1, blk_0r, miner_account, tx_0); \ REWIND_BLOCKS(events, blk_1r, blk_1, miner_account); diff --git a/tests/core_tests/double_spend.inl b/tests/core_tests/double_spend.inl index 4b60da3a3..b6e3726cc 100644 --- a/tests/core_tests/double_spend.inl +++ b/tests/core_tests/double_spend.inl @@ -126,20 +126,14 @@ bool gen_double_spend_in_tx<txs_keeped_by_block>::generate(std::vector<test_even DO_CALLBACK(events, "mark_last_valid_block"); std::vector<cryptonote::tx_source_entry> sources; - cryptonote::tx_source_entry se; - se.amount = tx_0.vout[0].amount; - se.push_output(0, boost::get<cryptonote::txout_to_key>(tx_0.vout[0].target).key, se.amount); - se.real_output = 0; - se.rct = false; - se.real_out_tx_key = get_tx_pub_key_from_extra(tx_0); - se.real_output_in_tx_index = 0; - sources.push_back(se); + CHECK_AND_ASSERT_THROW_MES(fill_tx_sources(sources, events, blk_1r, bob_account, send_amount, 0), "Source find error"); + // Double spend! - sources.push_back(se); + sources.push_back(sources[0]); cryptonote::tx_destination_entry de; de.addr = alice_account.get_keys().m_account_address; - de.amount = 2 * se.amount - TESTS_DEFAULT_FEE; + de.amount = 2 * send_amount - TESTS_DEFAULT_FEE; std::vector<cryptonote::tx_destination_entry> destinations; destinations.push_back(de); diff --git a/tests/core_tests/integer_overflow.cpp b/tests/core_tests/integer_overflow.cpp index 42ad0d26a..6ffc3786b 100644 --- a/tests/core_tests/integer_overflow.cpp +++ b/tests/core_tests/integer_overflow.cpp @@ -123,8 +123,8 @@ bool gen_uint_overflow_1::generate(std::vector<test_event_entry>& events) const events.push_back(blk_2); REWIND_BLOCKS(events, blk_2r, blk_2, miner_account); - MAKE_TX_LIST_START(events, txs_0, miner_account, bob_account, MONEY_SUPPLY, blk_2); - MAKE_TX_LIST(events, txs_0, miner_account, bob_account, MONEY_SUPPLY, blk_2); + MAKE_TX_LIST_START(events, txs_0, miner_account, bob_account, MONEY_SUPPLY, blk_2r); + MAKE_TX_LIST(events, txs_0, miner_account, bob_account, MONEY_SUPPLY, blk_2r); MAKE_NEXT_BLOCK_TX_LIST(events, blk_3, blk_2r, miner_account, txs_0); REWIND_BLOCKS(events, blk_3r, blk_3, miner_account); diff --git a/tests/core_tests/ring_signature_1.cpp b/tests/core_tests/ring_signature_1.cpp index f3f8109c3..218747f21 100644 --- a/tests/core_tests/ring_signature_1.cpp +++ b/tests/core_tests/ring_signature_1.cpp @@ -70,19 +70,19 @@ bool gen_ring_signature_1::generate(std::vector<test_event_entry>& events) const MAKE_NEXT_BLOCK(events, blk_4, blk_3, miner_account); // 8 REWIND_BLOCKS(events, blk_5, blk_4, miner_account); // <N blocks> REWIND_BLOCKS(events, blk_5r, blk_5, miner_account); // <N blocks> - MAKE_TX_LIST_START(events, txs_blk_6, miner_account, bob_account, MK_COINS(1), blk_5); // 9 + 2N - MAKE_TX_LIST(events, txs_blk_6, miner_account, bob_account, MK_COINS(11) + rnd_11, blk_5); // 10 + 2N - MAKE_TX_LIST(events, txs_blk_6, miner_account, bob_account, MK_COINS(11) + rnd_11, blk_5); // 11 + 2N - MAKE_TX_LIST(events, txs_blk_6, miner_account, bob_account, MK_COINS(20) + rnd_20, blk_5); // 12 + 2N - MAKE_TX_LIST(events, txs_blk_6, miner_account, bob_account, MK_COINS(29) + rnd_29, blk_5); // 13 + 2N - MAKE_TX_LIST(events, txs_blk_6, miner_account, bob_account, MK_COINS(29) + rnd_29, blk_5); // 14 + 2N - MAKE_TX_LIST(events, txs_blk_6, miner_account, bob_account, MK_COINS(29) + rnd_29, blk_5); // 15 + 2N - MAKE_TX_LIST(events, txs_blk_6, miner_account, some_account_1, MK_COINS(11) + rnd_11, blk_5); // 16 + 2N - MAKE_TX_LIST(events, txs_blk_6, miner_account, some_account_1, MK_COINS(11) + rnd_11, blk_5); // 17 + 2N - MAKE_TX_LIST(events, txs_blk_6, miner_account, some_account_1, MK_COINS(11) + rnd_11, blk_5); // 18 + 2N - MAKE_TX_LIST(events, txs_blk_6, miner_account, some_account_1, MK_COINS(11) + rnd_11, blk_5); // 19 + 2N - MAKE_TX_LIST(events, txs_blk_6, miner_account, some_account_1, MK_COINS(20) + rnd_20, blk_5); // 20 + 2N - MAKE_TX_LIST(events, txs_blk_6, miner_account, some_account_2, MK_COINS(20) + rnd_20, blk_5); // 21 + 2N + MAKE_TX_LIST_START(events, txs_blk_6, miner_account, bob_account, MK_COINS(1), blk_5r); // 9 + 2N + MAKE_TX_LIST(events, txs_blk_6, miner_account, bob_account, MK_COINS(11) + rnd_11, blk_5r); // 10 + 2N + MAKE_TX_LIST(events, txs_blk_6, miner_account, bob_account, MK_COINS(11) + rnd_11, blk_5r); // 11 + 2N + MAKE_TX_LIST(events, txs_blk_6, miner_account, bob_account, MK_COINS(20) + rnd_20, blk_5r); // 12 + 2N + MAKE_TX_LIST(events, txs_blk_6, miner_account, bob_account, MK_COINS(29) + rnd_29, blk_5r); // 13 + 2N + MAKE_TX_LIST(events, txs_blk_6, miner_account, bob_account, MK_COINS(29) + rnd_29, blk_5r); // 14 + 2N + MAKE_TX_LIST(events, txs_blk_6, miner_account, bob_account, MK_COINS(29) + rnd_29, blk_5r); // 15 + 2N + MAKE_TX_LIST(events, txs_blk_6, miner_account, some_account_1, MK_COINS(11) + rnd_11, blk_5r); // 16 + 2N + MAKE_TX_LIST(events, txs_blk_6, miner_account, some_account_1, MK_COINS(11) + rnd_11, blk_5r); // 17 + 2N + MAKE_TX_LIST(events, txs_blk_6, miner_account, some_account_1, MK_COINS(11) + rnd_11, blk_5r); // 18 + 2N + MAKE_TX_LIST(events, txs_blk_6, miner_account, some_account_1, MK_COINS(11) + rnd_11, blk_5r); // 19 + 2N + MAKE_TX_LIST(events, txs_blk_6, miner_account, some_account_1, MK_COINS(20) + rnd_20, blk_5r); // 20 + 2N + MAKE_TX_LIST(events, txs_blk_6, miner_account, some_account_2, MK_COINS(20) + rnd_20, blk_5r); // 21 + 2N MAKE_NEXT_BLOCK_TX_LIST(events, blk_6, blk_5r, miner_account, txs_blk_6); // 22 + 2N DO_CALLBACK(events, "check_balances_1"); // 23 + 2N REWIND_BLOCKS(events, blk_6r, blk_6, miner_account); // <N blocks> @@ -161,14 +161,14 @@ bool gen_ring_signature_2::generate(std::vector<test_event_entry>& events) const MAKE_NEXT_BLOCK(events, blk_2, blk_1, miner_account); // 4 MAKE_NEXT_BLOCK(events, blk_3, blk_2, miner_account); // 5 REWIND_BLOCKS(events, blk_3r, blk_3, miner_account); // <N blocks> - MAKE_TX_LIST_START(events, txs_blk_4, miner_account, bob_account, MK_COINS(13), blk_3); // 6 + N - MAKE_TX_LIST(events, txs_blk_4, miner_account, bob_account, MK_COINS(13), blk_3); // 7 + N - MAKE_TX_LIST(events, txs_blk_4, miner_account, bob_account, MK_COINS(13), blk_3); // 8 + N - MAKE_TX_LIST(events, txs_blk_4, miner_account, bob_account, MK_COINS(13), blk_3); // 9 + N + MAKE_TX_LIST_START(events, txs_blk_4, miner_account, bob_account, MK_COINS(13), blk_3r); // 6 + N + MAKE_TX_LIST(events, txs_blk_4, miner_account, bob_account, MK_COINS(13), blk_3r); // 7 + N + MAKE_TX_LIST(events, txs_blk_4, miner_account, bob_account, MK_COINS(13), blk_3r); // 8 + N + MAKE_TX_LIST(events, txs_blk_4, miner_account, bob_account, MK_COINS(13), blk_3r); // 9 + N MAKE_NEXT_BLOCK_TX_LIST(events, blk_4, blk_3r, miner_account, txs_blk_4); // 10 + N DO_CALLBACK(events, "check_balances_1"); // 11 + N REWIND_BLOCKS(events, blk_4r, blk_4, miner_account); // <N blocks> - MAKE_TX_MIX(events, tx_0, bob_account, alice_account, MK_COINS(52) - TESTS_DEFAULT_FEE, 3, blk_4); // 12 + 2N + MAKE_TX_MIX(events, tx_0, bob_account, alice_account, MK_COINS(52) - TESTS_DEFAULT_FEE, 3, blk_4r); // 12 + 2N MAKE_NEXT_BLOCK_TX1(events, blk_5, blk_4r, miner_account, tx_0); // 13 + 2N DO_CALLBACK(events, "check_balances_2"); // 14 + 2N diff --git a/tests/core_tests/tx_pool.cpp b/tests/core_tests/tx_pool.cpp index d3dc5d9bb..1b8480526 100644 --- a/tests/core_tests/tx_pool.cpp +++ b/tests/core_tests/tx_pool.cpp @@ -98,7 +98,7 @@ bool txpool_spend_key_public::generate(std::vector<test_event_entry>& events) co INIT_MEMPOOL_TEST(); DO_CALLBACK(events, "check_txpool_spent_keys"); - MAKE_TX(events, tx_0, miner_account, bob_account, send_amount, blk_0); + MAKE_TX(events, tx_0, miner_account, bob_account, send_amount, blk_0r); DO_CALLBACK(events, "increase_broadcasted_tx_count"); DO_CALLBACK(events, "increase_all_tx_count"); DO_CALLBACK(events, "check_txpool_spent_keys"); @@ -112,7 +112,7 @@ bool txpool_spend_key_all::generate(std::vector<test_event_entry>& events) SET_EVENT_VISITOR_SETT(events, event_visitor_settings::set_txs_do_not_relay); DO_CALLBACK(events, "check_txpool_spent_keys"); - MAKE_TX(events, tx_0, miner_account, bob_account, send_amount, blk_0); + MAKE_TX(events, tx_0, miner_account, bob_account, send_amount, blk_0r); DO_CALLBACK(events, "increase_all_tx_count"); DO_CALLBACK(events, "check_txpool_spent_keys"); @@ -502,7 +502,7 @@ bool txpool_double_spend_norelay::generate(std::vector<test_event_entry>& events SET_EVENT_VISITOR_SETT(events, event_visitor_settings::set_txs_do_not_relay); DO_CALLBACK(events, "mark_no_new"); - MAKE_TX(events, tx_0, miner_account, bob_account, send_amount, blk_0); + MAKE_TX(events, tx_0, miner_account, bob_account, send_amount, blk_0r); DO_CALLBACK(events, "increase_all_tx_count"); DO_CALLBACK(events, "check_txpool_spent_keys"); @@ -539,7 +539,7 @@ bool txpool_double_spend_local::generate(std::vector<test_event_entry>& events) SET_EVENT_VISITOR_SETT(events, event_visitor_settings::set_local_relay); DO_CALLBACK(events, "mark_no_new"); - MAKE_TX(events, tx_0, miner_account, bob_account, send_amount, blk_0); + MAKE_TX(events, tx_0, miner_account, bob_account, send_amount, blk_0r); DO_CALLBACK(events, "increase_all_tx_count"); DO_CALLBACK(events, "check_txpool_spent_keys"); @@ -574,7 +574,7 @@ bool txpool_double_spend_keyimage::generate(std::vector<test_event_entry>& event DO_CALLBACK(events, "mark_no_new"); const std::size_t tx_index1 = events.size(); - MAKE_TX(events, tx_0, miner_account, bob_account, send_amount, blk_0); + MAKE_TX(events, tx_0, miner_account, bob_account, send_amount, blk_0r); SET_EVENT_VISITOR_SETT(events, event_visitor_settings::set_txs_stem); DO_CALLBACK(events, "increase_all_tx_count"); @@ -595,7 +595,7 @@ bool txpool_double_spend_keyimage::generate(std::vector<test_event_entry>& event auto events_copy = events; events_copy.erase(events_copy.begin() + tx_index1); events_copy.erase(events_copy.begin() + tx_index2 - 1); - MAKE_TX(events_copy, tx_temp, miner_account, bob_account, send_amount, blk_0); + MAKE_TX(events_copy, tx_temp, miner_account, bob_account, send_amount, blk_0r); tx_1 = tx_temp; } @@ -616,7 +616,7 @@ bool txpool_stem_loop::generate(std::vector<test_event_entry>& events) const SET_EVENT_VISITOR_SETT(events, event_visitor_settings::set_txs_stem); DO_CALLBACK(events, "mark_no_new"); - MAKE_TX(events, tx_0, miner_account, bob_account, send_amount, blk_0); + MAKE_TX(events, tx_0, miner_account, bob_account, send_amount, blk_0r); DO_CALLBACK(events, "increase_all_tx_count"); DO_CALLBACK(events, "check_txpool_spent_keys"); diff --git a/tests/core_tests/tx_validation.cpp b/tests/core_tests/tx_validation.cpp index cfc8f089d..bc8622539 100644 --- a/tests/core_tests/tx_validation.cpp +++ b/tests/core_tests/tx_validation.cpp @@ -141,7 +141,7 @@ namespace { std::vector<tx_source_entry> sources; std::vector<tx_destination_entry> destinations; - fill_tx_sources_and_destinations(events, blk_head, from, to, amount, TESTS_DEFAULT_FEE, 0, sources, destinations); + fill_tx_sources_and_destinations(events, blk_head, from, to, amount, TESTS_DEFAULT_FEE, 0, sources, destinations, false); tx_builder builder; builder.step1_init(1, unlock_time); @@ -191,7 +191,7 @@ bool gen_tx_big_version::generate(std::vector<test_event_entry>& events) const std::vector<tx_source_entry> sources; std::vector<tx_destination_entry> destinations; - fill_tx_sources_and_destinations(events, blk_0, miner_account, miner_account, MK_COINS(1), TESTS_DEFAULT_FEE, 0, sources, destinations); + fill_tx_sources_and_destinations(events, blk_0r, miner_account, miner_account, MK_COINS(1), TESTS_DEFAULT_FEE, 0, sources, destinations); tx_builder builder; builder.step1_init(1 + 1, 0); @@ -217,7 +217,7 @@ bool gen_tx_unlock_time::generate(std::vector<test_event_entry>& events) const auto make_tx_with_unlock_time = [&](uint64_t unlock_time) -> transaction { - return make_simple_tx_with_unlock_time(events, blk_1, miner_account, miner_account, MK_COINS(1), unlock_time); + return make_simple_tx_with_unlock_time(events, blk_1r, miner_account, miner_account, MK_COINS(1), unlock_time); }; std::list<transaction> txs_0; @@ -266,7 +266,7 @@ bool gen_tx_input_is_not_txin_to_key::generate(std::vector<test_event_entry>& ev { std::vector<tx_source_entry> sources; std::vector<tx_destination_entry> destinations; - fill_tx_sources_and_destinations(events, blk_0, miner_account, miner_account, MK_COINS(1), TESTS_DEFAULT_FEE, 0, sources, destinations); + fill_tx_sources_and_destinations(events, blk_0r, miner_account, miner_account, MK_COINS(1), TESTS_DEFAULT_FEE, 0, sources, destinations); tx_builder builder; builder.step1_init(); @@ -309,7 +309,7 @@ bool gen_tx_no_inputs_has_outputs::generate(std::vector<test_event_entry>& event std::vector<tx_source_entry> sources; std::vector<tx_destination_entry> destinations; - fill_tx_sources_and_destinations(events, blk_0, miner_account, miner_account, MK_COINS(1), TESTS_DEFAULT_FEE, 0, sources, destinations); + fill_tx_destinations(miner_account, miner_account.get_keys().m_account_address, MK_COINS(1), TESTS_DEFAULT_FEE, sources, destinations, false); tx_builder builder; builder.step1_init(); @@ -331,7 +331,7 @@ bool gen_tx_has_inputs_no_outputs::generate(std::vector<test_event_entry>& event std::vector<tx_source_entry> sources; std::vector<tx_destination_entry> destinations; - fill_tx_sources_and_destinations(events, blk_0, miner_account, miner_account, MK_COINS(1), TESTS_DEFAULT_FEE, 0, sources, destinations); + fill_tx_sources_and_destinations(events, blk_0r, miner_account, miner_account, MK_COINS(1), TESTS_DEFAULT_FEE, 0, sources, destinations); destinations.clear(); tx_builder builder; @@ -357,7 +357,7 @@ bool gen_tx_invalid_input_amount::generate(std::vector<test_event_entry>& events std::vector<tx_source_entry> sources; std::vector<tx_destination_entry> destinations; - fill_tx_sources_and_destinations(events, blk_0, miner_account, miner_account, MK_COINS(1), TESTS_DEFAULT_FEE, 0, sources, destinations); + fill_tx_sources_and_destinations(events, blk_0r, miner_account, miner_account, MK_COINS(1), TESTS_DEFAULT_FEE, 0, sources, destinations); sources.front().amount++; tx_builder builder; @@ -383,7 +383,7 @@ bool gen_tx_input_wo_key_offsets::generate(std::vector<test_event_entry>& events std::vector<tx_source_entry> sources; std::vector<tx_destination_entry> destinations; - fill_tx_sources_and_destinations(events, blk_0, miner_account, miner_account, MK_COINS(1), TESTS_DEFAULT_FEE, 0, sources, destinations); + fill_tx_sources_and_destinations(events, blk_0r, miner_account, miner_account, MK_COINS(1), TESTS_DEFAULT_FEE, 0, sources, destinations); tx_builder builder; builder.step1_init(); @@ -414,8 +414,8 @@ bool gen_tx_key_offest_points_to_foreign_key::generate(std::vector<test_event_en REWIND_BLOCKS(events, blk_1r, blk_1, miner_account); MAKE_ACCOUNT(events, alice_account); MAKE_ACCOUNT(events, bob_account); - MAKE_TX_LIST_START(events, txs_0, miner_account, bob_account, MK_COINS(15) + 1, blk_1); - MAKE_TX_LIST(events, txs_0, miner_account, alice_account, MK_COINS(15) + 1, blk_1); + MAKE_TX_LIST_START(events, txs_0, miner_account, bob_account, MK_COINS(15) + 1, blk_1r); + MAKE_TX_LIST(events, txs_0, miner_account, alice_account, MK_COINS(15) + 1, blk_1r); MAKE_NEXT_BLOCK_TX_LIST(events, blk_2, blk_1r, miner_account, txs_0); std::vector<tx_source_entry> sources_bob; @@ -451,7 +451,7 @@ bool gen_tx_sender_key_offest_not_exist::generate(std::vector<test_event_entry>& std::vector<tx_source_entry> sources; std::vector<tx_destination_entry> destinations; - fill_tx_sources_and_destinations(events, blk_0, miner_account, miner_account, MK_COINS(1), TESTS_DEFAULT_FEE, 0, sources, destinations); + fill_tx_sources_and_destinations(events, blk_0r, miner_account, miner_account, MK_COINS(1), TESTS_DEFAULT_FEE, 0, sources, destinations); tx_builder builder; builder.step1_init(); @@ -478,8 +478,8 @@ bool gen_tx_mixed_key_offest_not_exist::generate(std::vector<test_event_entry>& REWIND_BLOCKS(events, blk_1r, blk_1, miner_account); MAKE_ACCOUNT(events, alice_account); MAKE_ACCOUNT(events, bob_account); - MAKE_TX_LIST_START(events, txs_0, miner_account, bob_account, MK_COINS(1) + TESTS_DEFAULT_FEE, blk_1); - MAKE_TX_LIST(events, txs_0, miner_account, alice_account, MK_COINS(1) + TESTS_DEFAULT_FEE, blk_1); + MAKE_TX_LIST_START(events, txs_0, miner_account, bob_account, MK_COINS(1) + TESTS_DEFAULT_FEE, blk_1r); + MAKE_TX_LIST(events, txs_0, miner_account, alice_account, MK_COINS(1) + TESTS_DEFAULT_FEE, blk_1r); MAKE_NEXT_BLOCK_TX_LIST(events, blk_2, blk_1r, miner_account, txs_0); std::vector<tx_source_entry> sources; @@ -511,7 +511,7 @@ bool gen_tx_key_image_not_derive_from_tx_key::generate(std::vector<test_event_en std::vector<tx_source_entry> sources; std::vector<tx_destination_entry> destinations; - fill_tx_sources_and_destinations(events, blk_0, miner_account, miner_account, MK_COINS(1), TESTS_DEFAULT_FEE, 0, sources, destinations); + fill_tx_sources_and_destinations(events, blk_0r, miner_account, miner_account, MK_COINS(1), TESTS_DEFAULT_FEE, 0, sources, destinations); tx_builder builder; builder.step1_init(); @@ -547,7 +547,7 @@ bool gen_tx_key_image_is_invalid::generate(std::vector<test_event_entry>& events std::vector<tx_source_entry> sources; std::vector<tx_destination_entry> destinations; - fill_tx_sources_and_destinations(events, blk_0, miner_account, miner_account, MK_COINS(1), TESTS_DEFAULT_FEE, 0, sources, destinations); + fill_tx_sources_and_destinations(events, blk_0r, miner_account, miner_account, MK_COINS(1), TESTS_DEFAULT_FEE, 0, sources, destinations); tx_builder builder; builder.step1_init(); @@ -591,7 +591,7 @@ bool gen_tx_check_input_unlock_time::generate(std::vector<test_event_entry>& eve std::list<transaction> txs_0; auto make_tx_to_acc = [&](size_t acc_idx, uint64_t unlock_time) { - txs_0.push_back(make_simple_tx_with_unlock_time(events, blk_1, miner_account, accounts[acc_idx], + txs_0.push_back(make_simple_tx_with_unlock_time(events, blk_1r, miner_account, accounts[acc_idx], MK_COINS(1) + TESTS_DEFAULT_FEE, unlock_time)); events.push_back(txs_0.back()); }; @@ -641,7 +641,7 @@ bool gen_tx_txout_to_key_has_invalid_key::generate(std::vector<test_event_entry> std::vector<tx_source_entry> sources; std::vector<tx_destination_entry> destinations; - fill_tx_sources_and_destinations(events, blk_0, miner_account, miner_account, MK_COINS(1), TESTS_DEFAULT_FEE, 0, sources, destinations); + fill_tx_sources_and_destinations(events, blk_0r, miner_account, miner_account, MK_COINS(1), TESTS_DEFAULT_FEE, 0, sources, destinations); tx_builder builder; builder.step1_init(); @@ -670,7 +670,7 @@ bool gen_tx_output_with_zero_amount::generate(std::vector<test_event_entry>& eve std::vector<tx_source_entry> sources; std::vector<tx_destination_entry> destinations; - fill_tx_sources_and_destinations(events, blk_0, miner_account, miner_account, MK_COINS(1), TESTS_DEFAULT_FEE, 0, sources, destinations); + fill_tx_sources_and_destinations(events, blk_0r, miner_account, miner_account, MK_COINS(1), TESTS_DEFAULT_FEE, 0, sources, destinations); tx_builder builder; builder.step1_init(); @@ -698,7 +698,7 @@ bool gen_tx_output_is_not_txout_to_key::generate(std::vector<test_event_entry>& std::vector<tx_source_entry> sources; std::vector<tx_destination_entry> destinations; - fill_tx_sources_and_destinations(events, blk_0, miner_account, miner_account, MK_COINS(1), TESTS_DEFAULT_FEE, 0, sources, destinations); + fill_tx_sources_and_destinations(events, blk_0r, miner_account, miner_account, MK_COINS(1), TESTS_DEFAULT_FEE, 0, sources, destinations); tx_builder builder; builder.step1_init(); @@ -740,8 +740,8 @@ bool gen_tx_signatures_are_invalid::generate(std::vector<test_event_entry>& even REWIND_BLOCKS(events, blk_1r, blk_1, miner_account); MAKE_ACCOUNT(events, alice_account); MAKE_ACCOUNT(events, bob_account); - MAKE_TX_LIST_START(events, txs_0, miner_account, bob_account, MK_COINS(1) + TESTS_DEFAULT_FEE, blk_1); - MAKE_TX_LIST(events, txs_0, miner_account, alice_account, MK_COINS(1) + TESTS_DEFAULT_FEE, blk_1); + MAKE_TX_LIST_START(events, txs_0, miner_account, bob_account, MK_COINS(1) + TESTS_DEFAULT_FEE, blk_1r); + MAKE_TX_LIST(events, txs_0, miner_account, alice_account, MK_COINS(1) + TESTS_DEFAULT_FEE, blk_1r); MAKE_NEXT_BLOCK_TX_LIST(events, blk_2, blk_1r, miner_account, txs_0); MAKE_TX(events, tx_0, miner_account, miner_account, MK_COINS(60), blk_2); diff --git a/tests/core_tests/wallet_tools.cpp b/tests/core_tests/wallet_tools.cpp index a3b66e835..ee29fdad1 100644 --- a/tests/core_tests/wallet_tools.cpp +++ b/tests/core_tests/wallet_tools.cpp @@ -31,11 +31,19 @@ void wallet_accessor_test::set_account(tools::wallet2 * wallet, cryptonote::acco void wallet_accessor_test::process_parsed_blocks(tools::wallet2 * wallet, uint64_t start_height, const std::vector<cryptonote::block_complete_entry> &blocks, const std::vector<tools::wallet2::parsed_block> &parsed_blocks, uint64_t& blocks_added) { - wallet->process_parsed_blocks(start_height, blocks, parsed_blocks, blocks_added); + if (wallet != nullptr) { + wallet->process_parsed_blocks(start_height, blocks, parsed_blocks, blocks_added); + } } void wallet_tools::process_transactions(tools::wallet2 * wallet, const std::vector<test_event_entry>& events, const cryptonote::block& blk_head, block_tracker &bt, const boost::optional<crypto::hash>& blk_tail) { + std::vector<tools::wallet2*> wallet_vector = { wallet }; + process_transactions(wallet_vector, events, blk_head, bt, blk_tail); +} + +void wallet_tools::process_transactions(const std::vector<tools::wallet2*>& wallets, const std::vector<test_event_entry>& events, const cryptonote::block& blk_head, block_tracker &bt, const boost::optional<crypto::hash>& blk_tail) +{ map_hash2tx_t mtx; std::vector<const cryptonote::block*> blockchain; find_block_chain(events, blockchain, mtx, get_block_hash(blk_head)); @@ -44,10 +52,15 @@ void wallet_tools::process_transactions(tools::wallet2 * wallet, const std::vect trim_block_chain(blockchain, blk_tail.get()); } - process_transactions(wallet, blockchain, mtx, bt); + process_transactions(wallets, blockchain, mtx, bt); +} + +void wallet_tools::process_transactions(tools::wallet2 * wallet, const std::vector<const cryptonote::block*>& blockchain, const map_hash2tx_t & mtx, block_tracker &bt){ + std::vector<tools::wallet2*> wallet_vector = { wallet }; + process_transactions(wallet_vector, blockchain, mtx, bt); } -void wallet_tools::process_transactions(tools::wallet2 * wallet, const std::vector<const cryptonote::block*>& blockchain, const map_hash2tx_t & mtx, block_tracker &bt) +void wallet_tools::process_transactions(const std::vector<tools::wallet2*>& wallets, const std::vector<const cryptonote::block*>& blockchain, const map_hash2tx_t & mtx, block_tracker &bt) { uint64_t start_height=0, blocks_added=0; std::vector<cryptonote::block_complete_entry> v_bche; @@ -67,11 +80,12 @@ void wallet_tools::process_transactions(tools::wallet2 * wallet, const std::vect wallet_tools::gen_block_data(bt, bl, mtx, v_bche.back(), v_parsed_block.back(), idx == 1 ? start_height : height); } - if (wallet) + for(auto wallet: wallets) { wallet_accessor_test::process_parsed_blocks(wallet, start_height, v_bche, v_parsed_block, blocks_added); + } } -bool wallet_tools::fill_tx_sources(tools::wallet2 * wallet, std::vector<cryptonote::tx_source_entry>& sources, size_t mixin, const boost::optional<size_t>& num_utxo, const boost::optional<uint64_t>& min_amount, block_tracker &bt, std::vector<size_t> &selected, uint64_t cur_height, ssize_t offset, int step, const boost::optional<fnc_accept_tx_source_t>& fnc_accept) +bool wallet_tools::fill_tx_sources(tools::wallet2 * wallet, std::vector<cryptonote::tx_source_entry>& sources, size_t mixin, const boost::optional<size_t>& num_utxo, const boost::optional<uint64_t>& min_amount, block_tracker &bt, std::vector<size_t> &selected, uint64_t cur_height, ssize_t offset, int step, const boost::optional<fnc_accept_tx_source_t>& fnc_accept, const boost::optional<fnc_accept_output_t>& fnc_in_accept) { CHECK_AND_ASSERT_THROW_MES(step != 0, "Step is zero"); sources.clear(); @@ -84,7 +98,7 @@ bool wallet_tools::fill_tx_sources(tools::wallet2 * wallet, std::vector<cryptono size_t iters = 0; uint64_t sum = 0; size_t cur_utxo = 0; - bool abort = false; + bool should_abort_search = false; unsigned brk_cond = 0; unsigned brk_thresh = num_utxo && min_amount ? 2 : (num_utxo || min_amount ? 1 : 0); @@ -96,7 +110,7 @@ bool wallet_tools::fill_tx_sources(tools::wallet2 * wallet, std::vector<cryptono brk_cond += 1; \ } while(0) - for(ssize_t i = roffset; iters < ntrans && !abort; i += step, ++iters) + for(ssize_t i = roffset; iters < ntrans && !should_abort_search; i += step, ++iters) { EVAL_BRK_COND(); if (brk_cond >= brk_thresh) @@ -106,7 +120,7 @@ bool wallet_tools::fill_tx_sources(tools::wallet2 * wallet, std::vector<cryptono auto & td = transfers[i]; if (td.m_spent) continue; - if (td.m_block_height + CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW > cur_height) + if (td.m_tx.unlock_time < CRYPTONOTE_MAX_BLOCK_NUMBER && td.m_tx.unlock_time > cur_height + 1) continue; if (selected_idx.find((size_t)i) != selected_idx.end()){ MERROR("Should not happen (selected_idx not found): " << i); @@ -119,18 +133,12 @@ bool wallet_tools::fill_tx_sources(tools::wallet2 * wallet, std::vector<cryptono try { cryptonote::tx_source_entry src; - wallet_tools::gen_tx_src(mixin, cur_height, td, src, bt); + wallet_tools::gen_tx_src(mixin, cur_height, td, src, bt, fnc_in_accept); // Acceptor function - if (fnc_accept){ - tx_source_info_crate_t c_info{.td=&td, .src=&src, .selected_idx=&selected_idx, .selected_kis=&selected_kis, - .ntrans=ntrans, .iters=iters, .sum=sum, .cur_utxo=cur_utxo}; - - bool take_it = (fnc_accept.get())(c_info, abort); - if (!take_it){ + if (fnc_accept && !(fnc_accept.get())({.td=&td, .src=&src, .selected_idx=&selected_idx, .selected_kis=&selected_kis, + .ntrans=ntrans, .iters=iters, .sum=sum, .cur_utxo=cur_utxo}, should_abort_search)) continue; - } - } MDEBUG("Selected " << i << " from tx: " << dump_keys(td.m_txid.data) << " ki: " << dump_keys(td.m_key_image.data) @@ -154,18 +162,22 @@ bool wallet_tools::fill_tx_sources(tools::wallet2 * wallet, std::vector<cryptono } EVAL_BRK_COND(); - return brk_cond >= brk_thresh; + const auto res = brk_cond >= brk_thresh; + if (!res) { + MDEBUG("fill_tx_sources fails, brk_cond: " << brk_cond << ", brk_thresh: " << brk_thresh << ", utxos: " << cur_utxo << ", sum: " << sum); + } + return res; #undef EVAL_BRK_COND } -void wallet_tools::gen_tx_src(size_t mixin, uint64_t cur_height, const tools::wallet2::transfer_details & td, cryptonote::tx_source_entry & src, block_tracker &bt) +void wallet_tools::gen_tx_src(size_t mixin, uint64_t cur_height, const tools::wallet2::transfer_details & td, cryptonote::tx_source_entry & src, block_tracker &bt, const boost::optional<fnc_accept_output_t>& fnc_accept) { CHECK_AND_ASSERT_THROW_MES(mixin != 0, "mixin is zero"); src.amount = td.amount(); src.rct = td.is_rct(); std::vector<tools::wallet2::get_outs_entry> outs; - bt.get_fake_outs(mixin, td.is_rct() ? 0 : td.amount(), td.m_global_output_index, cur_height, outs); + bt.get_fake_outs(mixin, td.is_rct() ? 0 : td.amount(), td.m_global_output_index, cur_height, outs, fnc_accept); for (size_t n = 0; n < mixin; ++n) { diff --git a/tests/core_tests/wallet_tools.h b/tests/core_tests/wallet_tools.h index 1fb8017bf..2201e11f4 100644 --- a/tests/core_tests/wallet_tools.h +++ b/tests/core_tests/wallet_tools.h @@ -68,12 +68,14 @@ public: class wallet_tools { public: - static void gen_tx_src(size_t mixin, uint64_t cur_height, const tools::wallet2::transfer_details & td, cryptonote::tx_source_entry & src, block_tracker &bt); + static void gen_tx_src(size_t mixin, uint64_t cur_height, const tools::wallet2::transfer_details & td, cryptonote::tx_source_entry & src, block_tracker &bt, const boost::optional<fnc_accept_output_t>& fnc_accept = boost::none); static void gen_block_data(block_tracker &bt, const cryptonote::block *bl, const map_hash2tx_t & mtx, cryptonote::block_complete_entry &bche, tools::wallet2::parsed_block &parsed_block, uint64_t &height); static void compute_subaddresses(std::unordered_map<crypto::public_key, cryptonote::subaddress_index> &subaddresses, cryptonote::account_base & creds, size_t account, size_t minors); static void process_transactions(tools::wallet2 * wallet, const std::vector<test_event_entry>& events, const cryptonote::block& blk_head, block_tracker &bt, const boost::optional<crypto::hash>& blk_tail=boost::none); + static void process_transactions(const std::vector<tools::wallet2*>& wallets, const std::vector<test_event_entry>& events, const cryptonote::block& blk_head, block_tracker &bt, const boost::optional<crypto::hash>& blk_tail=boost::none); static void process_transactions(tools::wallet2 * wallet, const std::vector<const cryptonote::block*>& blockchain, const map_hash2tx_t & mtx, block_tracker &bt); - static bool fill_tx_sources(tools::wallet2 * wallet, std::vector<cryptonote::tx_source_entry>& sources, size_t mixin, const boost::optional<size_t>& num_utxo, const boost::optional<uint64_t>& min_amount, block_tracker &bt, std::vector<size_t> &selected, uint64_t cur_height, ssize_t offset=0, int step=1, const boost::optional<fnc_accept_tx_source_t>& fnc_accept=boost::none); + static void process_transactions(const std::vector<tools::wallet2*>& wallets, const std::vector<const cryptonote::block*>& blockchain, const map_hash2tx_t & mtx, block_tracker &bt); + static bool fill_tx_sources(tools::wallet2 * wallet, std::vector<cryptonote::tx_source_entry>& sources, size_t mixin, const boost::optional<size_t>& num_utxo, const boost::optional<uint64_t>& min_amount, block_tracker &bt, std::vector<size_t> &selected, uint64_t cur_height, ssize_t offset=0, int step=1, const boost::optional<fnc_accept_tx_source_t>& fnc_accept=boost::none, const boost::optional<fnc_accept_output_t>& fnc_in_accept = boost::none); }; cryptonote::account_public_address get_address(const tools::wallet2*); diff --git a/tests/functional_tests/functional_tests_rpc.py b/tests/functional_tests/functional_tests_rpc.py index 9975cdfa2..7e4d49ffa 100755 --- a/tests/functional_tests/functional_tests_rpc.py +++ b/tests/functional_tests/functional_tests_rpc.py @@ -10,7 +10,11 @@ import string import os USAGE = 'usage: functional_tests_rpc.py <python> <srcdir> <builddir> [<tests-to-run> | all]' -DEFAULT_TESTS = ['address_book', 'bans', 'blockchain', 'cold_signing', 'daemon_info', 'get_output_distribution', 'integrated_address', 'mining', 'multisig', 'p2p', 'proofs', 'rpc_payment', 'sign_message', 'transfer', 'txpool', 'uri', 'validate_address', 'wallet'] +DEFAULT_TESTS = [ + 'address_book', 'bans', 'blockchain', 'cold_signing', 'daemon_info', 'get_output_distribution', + 'integrated_address', 'k_anonymity', 'mining', 'multisig', 'p2p', 'proofs', 'rpc_payment', + 'sign_message', 'transfer', 'txpool', 'uri', 'validate_address', 'wallet' +] try: python = sys.argv[1] srcdir = sys.argv[2] diff --git a/tests/functional_tests/k_anonymity.py b/tests/functional_tests/k_anonymity.py new file mode 100755 index 000000000..ffa670b4c --- /dev/null +++ b/tests/functional_tests/k_anonymity.py @@ -0,0 +1,314 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2023, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import print_function +import math +import random + +""" +Test the k-anonymity daemon RPC features: +* txid fetching by prefix +""" + +from framework.daemon import Daemon +from framework.wallet import Wallet + +seeds = [ + 'velvet lymph giddy number token physics poetry unquoted nibs useful sabotage limits benches lifestyle eden nitrogen anvil fewest avoid batch vials washing fences goat unquoted', + 'peeled mixture ionic radar utopia puddle buying illness nuns gadget river spout cavernous bounced paradise drunk looking cottage jump tequila melting went winter adjust spout', + 'tadpoles shrugged ritual exquisite deepest rest people musical farming otherwise shelter fabrics altitude seventh request tidy ivory diet vapidly syllabus logic espionage oozed opened people', + 'ocio charla pomelo humilde maduro geranio bruto moño admitir mil difícil diva lucir cuatro odisea riego bebida mueble cáncer puchero carbón poeta flor fruta fruta' +] + +pub_addrs = [ + '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', + '44Kbx4sJ7JDRDV5aAhLJzQCjDz2ViLRduE3ijDZu3osWKBjMGkV1XPk4pfDUMqt1Aiezvephdqm6YD19GKFD9ZcXVUTp6BW', + '45uQD4jzWwPazqr9QJx8CmFPN7a9RaEE8T4kULg6r8GzfcrcgKXshfYf8cezLWwmENHC9pDN2fGAUFmmdFxjeZSs3n671rz', + '48hKTTTMfuiW2gDkmsibERHCjTCpqyCCh57WcU4KBeqDSAw7dG7Ad1h7v8iJF4q59aDqBATg315MuZqVmkF89E3cLPrBWsi' +] + +CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW = 60 +CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE = 10 +RESTRICTED_SPENT_KEY_IMAGES_COUNT = 5000 + +def make_hash32_loose_template(txid, nbits): + txid_bytes = list(bytes.fromhex(txid)) + for i in reversed(range(32)): + mask_nbits = min(8, nbits) + mask = 256 - (1 << (8 - mask_nbits)) + nbits -= mask_nbits + txid_bytes[i] &= mask + return bytes(txid_bytes).hex() + +def txid_list_is_sorted_in_template_order(txids): + reversed_txid_bytes = [bytes(reversed(bytes.fromhex(txid))) for txid in txids] + return sorted(reversed_txid_bytes) == reversed_txid_bytes + +def txid_matches_template(txid, template, nbits): + txid_bytes = bytes.fromhex(txid) + template_bytes = bytes.fromhex(template) + for i in reversed(range(32)): + mask_nbits = min(8, nbits) + mask = 256 - (1 << (8 - mask_nbits)) + nbits -= mask_nbits + if 0 != ((txid_bytes[i] ^ template_bytes[i]) & mask): + return False + return True + +class KAnonymityTest: + def run_test(self): + self.reset() + self.create_wallets() + + # If each of the N wallets is making N-1 transfers the first round, each N wallets needs + # N-1 unlocked coinbase outputs + N = len(seeds) + self.mine_and_refresh(2 * N * (N - 1)) + self.mine_and_refresh(CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW) + + # Generate a bunch of transactions + NUM_ROUNDS = 10 + intermediate_mining_period = int(math.ceil(CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE / N)) * N + for i in range(NUM_ROUNDS): + self.transfer_around() + self.mine_and_refresh(intermediate_mining_period) + print("Wallets created {} transactions in {} rounds".format(len(self.wallet_txids), NUM_ROUNDS)) + + self.test_all_chain_txids() # Also gathers miner_txids + + self.test_get_txids_loose_chain_suite() + + self.test_get_txids_loose_pool_suite() + + self.test_bad_txid_templates() + + def reset(self): + print('Resetting blockchain') + daemon = Daemon() + res = daemon.get_height() + daemon.pop_blocks(res.height - 1) + daemon.flush_txpool() + self.wallet_txids = set() + self.total_blocks_mined = 0 + self.miner_txids = set() + self.pool_txids = set() + + def create_wallets(self): + print('Creating wallets') + assert len(seeds) == len(pub_addrs) + self.wallet = [None] * len(seeds) + for i in range(len(seeds)): + self.wallet[i] = Wallet(idx = i) + # close the wallet if any, will throw if none is loaded + try: self.wallet[i].close_wallet() + except: pass + res = self.wallet[i].restore_deterministic_wallet(seed = seeds[i]) + + def mine_and_refresh(self, num_blocks): + print("Mining {} blocks".format(num_blocks)) + daemon = Daemon() + + res = daemon.get_info() + old_height = res.height + + assert num_blocks % len(self.wallet) == 0 + assert len(self.wallet) == len(pub_addrs) + + for i in range(len(self.wallet)): + daemon.generateblocks(pub_addrs[i], num_blocks // len(self.wallet)) + + res = daemon.get_info() + new_height = res.height + assert new_height == old_height + num_blocks, "height {} -> {}".format(old_height, new_height) + + for i in range(len(self.wallet)): + self.wallet[i].refresh() + res = self.wallet[i].get_height() + assert res.height == new_height, "{} vs {}".format(res.height, new_height) + + self.wallet_txids.update(self.pool_txids) + self.pool_txids.clear() + self.total_blocks_mined += num_blocks + + def transfer_around(self): + N = len(self.wallet) + assert N == len(pub_addrs) + + print("Creating transfers b/t wallets") + + num_successful_transfers = 0 + fee_margin = 0.05 # 5% + for sender in range(N): + receivers = list((r for r in range(N) if r != sender)) + random.shuffle(receivers) + assert len(receivers) == N - 1 + for j, receiver in enumerate(receivers): + unlocked_balance = self.wallet[sender].get_balance().unlocked_balance + if 0 == unlocked_balance: + assert j != 0 # we want all wallets to start out with at least some funds + break + imperfect_starting_balance = unlocked_balance * (N - 1) / (N - 1 - j) * (1 - fee_margin) + transfer_amount = int(imperfect_starting_balance / (N - 1)) + assert transfer_amount < unlocked_balance + dst = {'address': pub_addrs[receiver], 'amount': transfer_amount} + res = self.wallet[sender].transfer([dst], get_tx_metadata = True) + tx_hex = res.tx_metadata + self.pool_txids.add(res.tx_hash) + res = self.wallet[sender].relay_tx(tx_hex) + self.wallet[sender].refresh() + num_successful_transfers += 1 + + print("Transferred {} times".format(num_successful_transfers)) + + def test_all_chain_txids(self): + daemon = Daemon() + + print("Grabbing all txids from the daemon and testing against known txids") + + # If assert stmt below fails, this test case needs to be rewritten to chunk the requests; + # there are simply too many txids on-chain to gather at once + expected_total_num_txids = len(self.wallet_txids) + self.total_blocks_mined + 1 # +1 for genesis coinbase tx + assert expected_total_num_txids <= RESTRICTED_SPENT_KEY_IMAGES_COUNT + + res = daemon.get_txids_loose('0' * 64, 0) + all_txids = res.txids + assert 'c88ce9783b4f11190d7b9c17a69c1c52200f9faaee8e98dd07e6811175177139' in all_txids # genesis coinbase tx + assert len(all_txids) == expected_total_num_txids, "{} {}".format(len(all_txids), expected_total_num_txids) + + assert txid_list_is_sorted_in_template_order(all_txids) + + for txid in self.wallet_txids: + assert txid in all_txids + + self.miner_txids = set(all_txids) - self.wallet_txids + + def test_get_txids_loose_success(self, txid, num_matching_bits): + daemon = Daemon() + + txid_template = make_hash32_loose_template(txid, num_matching_bits) + + res = daemon.get_txids_loose(txid_template, num_matching_bits) + assert 'txids' in res + txids = res.txids + + first_pool_index = 0 + while first_pool_index < len(txids): + if txids[first_pool_index] in self.pool_txids: + break + else: + first_pool_index += 1 + + chain_txids = txids[:first_pool_index] + pool_txids = txids[first_pool_index:] + + assert txid_list_is_sorted_in_template_order(chain_txids) + assert txid_list_is_sorted_in_template_order(pool_txids) + + # Assert we know where txids came from + for txid in chain_txids: + assert (txid in self.wallet_txids) or (txid in self.miner_txids) + for txid in pool_txids: + assert txid in self.pool_txids + + # Assert that all known txids were matched as they should've been + for txid in self.wallet_txids: + assert txid_matches_template(txid, txid_template, num_matching_bits) == (txid in chain_txids) + for txid in self.miner_txids: + assert txid_matches_template(txid, txid_template, num_matching_bits) == (txid in chain_txids) + for txid in self.pool_txids: + assert txid_matches_template(txid, txid_template, num_matching_bits) == (txid in pool_txids) + + def test_get_txids_loose_chain_suite(self): + daemon = Daemon() + + print("Testing grabbing on-chain txids loosely with all different bit sizes") + + # Assert pool empty + assert len(self.pool_txids) == 0 + res = daemon.get_transaction_pool_hashes() + assert not 'tx_hashes' in res or len(res.tx_hashes) == 0 + + assert len(self.wallet_txids) + + current_chain_txids = list(self.wallet_txids.union(self.miner_txids)) + for nbits in range(0, 256): + random_txid = random.choice(current_chain_txids) + self.test_get_txids_loose_success(random_txid, nbits) + + def test_get_txids_loose_pool_suite(self): + daemon = Daemon() + + print("Testing grabbing pool txids loosely with all different bit sizes") + + # Create transactions to pool + self.transfer_around() + + # Assert pool not empty + assert len(self.pool_txids) != 0 + res = daemon.get_transaction_pool_hashes() + assert 'tx_hashes' in res and set(res.tx_hashes) == self.pool_txids + + current_pool_txids = list(self.pool_txids) + for nbits in range(0, 256): + random_txid = random.choice(current_pool_txids) + self.test_get_txids_loose_success(random_txid, nbits) + + def test_bad_txid_templates(self): + daemon = Daemon() + + print("Making sure the daemon catches bad txid templates") + + test_cases = [ + ['q', 256], + ['a', 128], + ['69' * 32, 257], + ['0abcdef1234567890abcdef1234567890abcdef1234567890abcdef123456789', 0], + ['0abcdef1234567890abcdef1234567890abcdef1234567890abcdef123456789', 1], + ['0abcdef1234567890abcdef1234567890abcdef1234567890abcdef123456789', 2], + ['0abcdef1234567890abcdef1234567890abcdef1234567890abcdef123456789', 4], + ['0abcdef1234567890abcdef1234567890abcdef1234567890abcdef123456789', 8], + ['0abcdef1234567890abcdef1234567890abcdef1234567890abcdef123456789', 16], + ['0abcdef1234567890abcdef1234567890abcdef1234567890abcdef123456789', 32], + ['0abcdef1234567890abcdef1234567890abcdef1234567890abcdef123456789', 64], + ['0abcdef1234567890abcdef1234567890abcdef1234567890abcdef123456789', 128], + ['0abcdef1234567890abcdef1234567890abcdef1234567890abcdef123456789', 193], + ['0000000000000000000000000000000000000000000000000000000000000080', 0], + ['0000000000000000000000000000000000000000000000000000000000000007', 5], + ['00000000000000000000000000000000000000000000000000000000000000f7', 5], + ] + + for txid_template, num_matching_bits in test_cases: + ok = False + try: res = daemon.get_txids_loose(txid_template, num_matching_bits) + except: ok = True + assert ok, 'bad template didnt error: {} {}'.format(txid_template, num_matching_bits) + +if __name__ == '__main__': + KAnonymityTest().run_test() diff --git a/tests/functional_tests/transfer.py b/tests/functional_tests/transfer.py index ddc930eb0..56a2514d9 100755 --- a/tests/functional_tests/transfer.py +++ b/tests/functional_tests/transfer.py @@ -63,6 +63,7 @@ class TransferTest(): self.check_is_key_image_spent() self.check_multiple_submissions() self.check_scan_tx() + self.check_subtract_fee_from_outputs() def reset(self): print('Resetting blockchain') @@ -1081,5 +1082,85 @@ class TransferTest(): diff_transfers(receiver_wallet.get_transfers(), res) assert receiver_wallet.get_balance().balance == expected_receiver_balance + def check_subtract_fee_from_outputs(self): + daemon = Daemon() + + print('Testing fee-included transfers') + + def inner_test_external_transfer(dsts, subtract_fee_from_outputs): + # refresh wallet and get balance + self.wallet[0].refresh() + balance1 = self.wallet[0].get_balance().balance + + # Check that this transaction is possible with our current balance + other preconditions + dst_sum = sum(map(lambda x: x['amount'], dsts)) + assert balance1 >= dst_sum + if subtract_fee_from_outputs: + assert max(subtract_fee_from_outputs) < len(dsts) + + # transfer with subtractfeefrom=all + transfer_res = self.wallet[0].transfer(dsts, subtract_fee_from_outputs = subtract_fee_from_outputs, get_tx_metadata = True) + tx_hex = transfer_res.tx_metadata + tx_fee = transfer_res.fee + amount_spent = transfer_res.amount + amounts_by_dest = transfer_res.amounts_by_dest.amounts + + # Assert that fee and amount spent to outputs adds up + assert tx_fee != 0 + if subtract_fee_from_outputs: + assert tx_fee + amount_spent == dst_sum + else: + assert amount_spent == dst_sum + + # Check the amounts by each destination that only the destinations set as subtractable + # got subtracted and that the subtracted dests are approximately correct + assert len(amounts_by_dest) == len(dsts) # if this fails... idk + for i in range(len(dsts)): + if i in subtract_fee_from_outputs: # dest is subtractable + approx_subtraction = tx_fee // len(subtract_fee_from_outputs) + assert amounts_by_dest[i] < dsts[i]['amount'] + assert dsts[i]['amount'] - amounts_by_dest[i] - approx_subtraction <= 1 + else: + assert amounts_by_dest[i] == dsts[i]['amount'] + + # relay tx and generate block (not to us, to simplify balance change calculations) + relay_res = self.wallet[0].relay_tx(tx_hex) + daemon.generateblocks('44Kbx4sJ7JDRDV5aAhLJzQCjDz2ViLRduE3ijDZu3osWKBjMGkV1XPk4pfDUMqt1Aiezvephdqm6YD19GKFD9ZcXVUTp6BW', 1) + + # refresh and get balance again + self.wallet[0].refresh() + balance2 = self.wallet[0].get_balance().balance + + # Check that the wallet balance dropped by the correct amount + balance_drop = balance1 - balance2 + if subtract_fee_from_outputs: + assert balance_drop == dst_sum + else: + assert balance_drop == dst_sum + tx_fee + + dst1 = {'address': '44Kbx4sJ7JDRDV5aAhLJzQCjDz2ViLRduE3ijDZu3osWKBjMGkV1XPk4pfDUMqt1Aiezvephdqm6YD19GKFD9ZcXVUTp6BW', 'amount': 1100000000001} + dst2 = {'address': '46r4nYSevkfBUMhuykdK3gQ98XDqDTYW1hNLaXNvjpsJaSbNtdXh1sKMsdVgqkaihChAzEy29zEDPMR3NHQvGoZCLGwTerK', 'amount': 1200000000000} + dst3 = {'address': '46r4nYSevkfBUMhuykdK3gQ98XDqDTYW1hNLaXNvjpsJaSbNtdXh1sKMsdVgqkaihChAzEy29zEDPMR3NHQvGoZCLGwTerK', 'amount': 1} + + inner_test_external_transfer([dst1, dst2], [0, 1]) + inner_test_external_transfer([dst1, dst2], [0]) + inner_test_external_transfer([dst1, dst2], [1]) + inner_test_external_transfer([dst1, dst2], []) + inner_test_external_transfer([dst1], [0]) + inner_test_external_transfer([dst1], []) + inner_test_external_transfer([dst3], []) + try: + inner_test_external_transfer([dst1, dst3], [0, 1]) # Test subtractfeefrom if one of the outputs would underflow w/o good checks + raise ValueError('transfer request with tiny subtractable destination should have thrown') + except: + pass + + # Check for JSONRPC error on bad index + try: + transfer_res = self.wallet[0].transfer([dst1], subtract_fee_from_outputs = [1]) + raise ValueError('transfer request with index should have thrown') + except AssertionError: + pass + if __name__ == '__main__': TransferTest().run_test() diff --git a/tests/performance_tests/CMakeLists.txt b/tests/performance_tests/CMakeLists.txt index 7462f7119..5dd278c6f 100644 --- a/tests/performance_tests/CMakeLists.txt +++ b/tests/performance_tests/CMakeLists.txt @@ -67,6 +67,7 @@ target_link_libraries(performance_tests common cncrypto epee + seraphis_crypto ${Boost_CHRONO_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} ${EXTRA_LIBRARIES}) diff --git a/tests/trezor/daemon.cpp b/tests/trezor/daemon.cpp index 47c0cde9b..de4f9bc51 100644 --- a/tests/trezor/daemon.cpp +++ b/tests/trezor/daemon.cpp @@ -309,11 +309,10 @@ void mock_daemon::stop_p2p() m_server.send_stop_signal(); } -void mock_daemon::mine_blocks(size_t num_blocks, const std::string &miner_address) +void mock_daemon::mine_blocks(size_t num_blocks, const std::string &miner_address, std::chrono::seconds timeout) { bool blocks_mined = false; const uint64_t start_height = get_height(); - const auto mining_timeout = std::chrono::seconds(360); MDEBUG("Current height before mining: " << start_height); start_mining(miner_address); @@ -331,14 +330,14 @@ void mock_daemon::mine_blocks(size_t num_blocks, const std::string &miner_addres } auto current_time = std::chrono::system_clock::now(); - if (mining_timeout < current_time - mining_started) + if (timeout < current_time - mining_started) { break; } } stop_mining(); - CHECK_AND_ASSERT_THROW_MES(blocks_mined, "Mining failed in the time limit"); + CHECK_AND_ASSERT_THROW_MES(blocks_mined, "Mining failed in the time limit: " << timeout.count()); } constexpr const std::chrono::seconds mock_daemon::rpc_timeout; diff --git a/tests/trezor/daemon.h b/tests/trezor/daemon.h index b87fd5036..3c0b6b78b 100644 --- a/tests/trezor/daemon.h +++ b/tests/trezor/daemon.h @@ -139,7 +139,7 @@ public: void stop_and_deinit(); void try_init_and_run(boost::optional<unsigned> initial_port=boost::none); - void mine_blocks(size_t num_blocks, const std::string &miner_address); + void mine_blocks(size_t num_blocks, const std::string &miner_address, std::chrono::seconds timeout = std::chrono::seconds(360)); void start_mining(const std::string &miner_address, uint64_t threads_count=1, bool do_background_mining=false, bool ignore_battery=true); void stop_mining(); uint64_t get_height(); diff --git a/tests/trezor/trezor_tests.cpp b/tests/trezor/trezor_tests.cpp index 10c52ff29..b21da95c0 100644 --- a/tests/trezor/trezor_tests.cpp +++ b/tests/trezor/trezor_tests.cpp @@ -37,6 +37,7 @@ using namespace cryptonote; +#include <cmath> #include <boost/regex.hpp> #include <common/apply_permutation.h> #include "common/util.h" @@ -51,18 +52,20 @@ namespace po = boost::program_options; namespace { - const command_line::arg_descriptor<std::string> arg_filter = { "filter", "Regular expression filter for which tests to run" }; - const command_line::arg_descriptor<std::string> arg_trezor_path = {"trezor_path", "Path to the trezor device to use, has to support debug link", ""}; - const command_line::arg_descriptor<bool> arg_heavy_tests = {"heavy_tests", "Runs expensive tests (volume tests with real device)", false}; - const command_line::arg_descriptor<std::string> arg_chain_path = {"chain_path", "Path to the serialized blockchain, speeds up testing", ""}; - const command_line::arg_descriptor<bool> arg_fix_chain = {"fix_chain", "If chain_patch is given and file cannot be used, it is ignored and overwriten", false}; + const command_line::arg_descriptor<std::string> arg_filter = {"filter", "Regular expression filter for which tests to run" }; + const command_line::arg_descriptor<std::string> arg_trezor_path = {"trezor-path", "Path to the trezor device to use, has to support debug link", ""}; + const command_line::arg_descriptor<bool> arg_heavy_tests = {"heavy-tests", "Runs expensive tests (volume tests)", false}; + const command_line::arg_descriptor<std::string> arg_chain_path = {"chain-path", "Path to the serialized blockchain, speeds up testing", ""}; + const command_line::arg_descriptor<bool> arg_fix_chain = {"fix-chain", "If chain-patch is given and file cannot be used, it is ignored and overwriten", false}; } #define HW_TREZOR_NAME "Trezor" -#define TREZOR_ACCOUNT_ORDERING &m_miner_account, &m_alice_account, &m_bob_account, &m_eve_account -#define TREZOR_COMMON_TEST_CASE(genclass, CORE, BASE) do { \ - rollback_chain(CORE, BASE.head_block()); \ - { \ +#define TEST_DECOY_MINING_COUNTS 3 +#define TREZOR_ACCOUNT_ORDERING &m_miner_account, &m_alice_account, &m_alice2_account, &m_bob_account, &m_eve_account +#define TREZOR_ALL_WALLET_VCT vct_wallets(m_wl_alice.get(), m_wl_alice2.get(), m_wl_bob.get(), m_wl_eve.get()) +#define TREZOR_COMMON_TEST_CASE(genclass, CORE, BASE) do { \ + if (filter.empty() || boost::regex_match(std::string(#genclass), match, boost::regex(filter))) { \ + rollback_chain(CORE, BASE.head_block()); \ genclass ctest; \ BASE.fork(ctest); \ GENERATE_AND_PLAY_INSTANCE(genclass, ctest, *(CORE)); \ @@ -79,17 +82,27 @@ namespace } \ } while(0) +typedef struct { + bool heavy_tests; +} chain_file_opts_t; +static const std::string CUR_CHAIN_MAGIC = "MoneroTrezorTestsEventFile"; +static const unsigned long CUR_CHAIN_VERSION = 2; static device_trezor_test *trezor_device = nullptr; static device_trezor_test *ensure_trezor_test_device(); static void rollback_chain(cryptonote::core * core, const cryptonote::block & head); static void setup_chain(cryptonote::core * core, gen_trezor_base & trezor_base, std::string chain_path, bool fix_chain, const po::variables_map & vm_core); -static long get_env_long(const char * flag_name, boost::optional<long> def = boost::none){ - const char *env_data = getenv(flag_name); +static long get_env_long(const char * env_name, boost::optional<long> def = boost::none){ + const char *env_data = getenv(env_name); return env_data ? atol(env_data) : (def ? def.get() : 0); } +static std::string get_env_string(const char * env_name, boost::optional<std::string> def = boost::none){ + const char *env_data = getenv(env_name); + return env_data ? std::string(env_data) : (def ? def.get() : std::string()); +} + int main(int argc, char* argv[]) { TRY_ENTRY(); @@ -137,9 +150,13 @@ int main(int argc, char* argv[]) hw::register_device(HW_TREZOR_NAME, ensure_trezor_test_device()); // shim device for call tracking // Bootstrapping common chain & accounts - const uint8_t initial_hf = (uint8_t)get_env_long("TEST_MIN_HF", HF_VERSION_BULLETPROOF_PLUS); - const uint8_t max_hf = (uint8_t)get_env_long("TEST_MAX_HF", HF_VERSION_BULLETPROOF_PLUS); - auto sync_test = get_env_long("TEST_KI_SYNC", 1); + const std::string trezor_path_env = get_env_string("TREZOR_PATH"); + const uint8_t initial_hf = (uint8_t)get_env_long("TEST_MIN_HF", TREZOR_TEST_MIN_HF_DEFAULT); + const uint8_t max_hf = (uint8_t)get_env_long("TEST_MAX_HF", TREZOR_TEST_MAX_HF_DEFAULT); + const bool mining_enabled = get_env_long("TEST_MINING_ENABLED", TREZOR_TEST_MINING_ENABLED_DEFAULT || heavy_tests) > 0; + const long mining_timeout = get_env_long("TEST_MINING_TIMEOUT", TREZOR_TEST_MINING_TIMEOUT_DEFAULT); + const auto sync_test = get_env_long("TEST_KI_SYNC", TREZOR_TEST_KI_SYNC_DEFAULT); + const bool env_gen_heavy = get_env_long("TEST_GEN_HEAVY", 0) > 0 || heavy_tests; MINFO("Test versions " << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")"); MINFO("Testing hardforks [" << (int)initial_hf << ", " << (int)max_hf << "], sync-test: " << sync_test); @@ -148,7 +165,8 @@ int main(int argc, char* argv[]) std::shared_ptr<mock_daemon> daemon = nullptr; gen_trezor_base trezor_base; - trezor_base.setup_args(trezor_path, heavy_tests); + trezor_base.setup_args(!trezor_path.empty() ? trezor_path : trezor_path_env, heavy_tests, mining_enabled, mining_timeout); + trezor_base.heavy_test_set(env_gen_heavy || heavy_tests); trezor_base.set_hard_fork(initial_hf); // Arguments for core & daemon @@ -194,6 +212,10 @@ int main(int argc, char* argv[]) TREZOR_COMMON_TEST_CASE(gen_trezor_ki_sync_with_refresh, core, trezor_base); } + TREZOR_COMMON_TEST_CASE(gen_trezor_no_passphrase, core, trezor_base); + TREZOR_COMMON_TEST_CASE(gen_trezor_wallet_passphrase, core, trezor_base); + TREZOR_COMMON_TEST_CASE(gen_trezor_passphrase, core, trezor_base); + TREZOR_COMMON_TEST_CASE(gen_trezor_pin, core, trezor_base); TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo, core, trezor_base); TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo_paymentid_short, core, trezor_base); TREZOR_COMMON_TEST_CASE(gen_trezor_1utxo_paymentid_short_integrated, core, trezor_base); @@ -205,6 +227,7 @@ int main(int argc, char* argv[]) TREZOR_COMMON_TEST_CASE(gen_trezor_2utxo_sub_acc_to_1norm_2sub, core, trezor_base); TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo_to_7outs, core, trezor_base); TREZOR_COMMON_TEST_CASE(gen_trezor_4utxo_to_15outs, core, trezor_base); + TREZOR_COMMON_TEST_CASE(gen_trezor_16utxo_to_sub, core, trezor_base); TREZOR_COMMON_TEST_CASE(wallet_api_tests, core, trezor_base); } @@ -242,7 +265,7 @@ static void rollback_chain(cryptonote::core * core, const cryptonote::block & he crypto::hash head_hash = get_block_hash(head), cur_hash{}; uint64_t height = get_block_height(head), cur_height=0; - MDEBUG("Rollbacking to " << height << " to hash " << head_hash); + MDEBUG("Rolling back to " << height << " to hash " << head_hash); do { core->get_blockchain_top(cur_height, cur_hash); @@ -255,11 +278,12 @@ static void rollback_chain(cryptonote::core * core, const cryptonote::block & he } while(true); } -static bool unserialize_chain_from_file(std::vector<test_event_entry>& events, gen_trezor_base &test_base, const std::string& file_path) +static bool deserialize_chain_from_file(std::vector<test_event_entry>& events, gen_trezor_base &test_base, unsigned long &version, const std::string& file_path, chain_file_opts_t& opts) { TRY_ENTRY(); std::ifstream data_file; data_file.open( file_path, std::ios_base::binary | std::ios_base::in); + std::string magic; if(data_file.fail()) return false; try @@ -267,8 +291,20 @@ static bool unserialize_chain_from_file(std::vector<test_event_entry>& events, g boost::archive::portable_binary_iarchive a(data_file); test_base.clear(); + a >> magic; + CHECK_AND_ASSERT_THROW_MES(magic == CUR_CHAIN_MAGIC, "Chain file load error - magic differs"); + + a >> version; + a >> opts.heavy_tests; + a >> magic; + CHECK_AND_ASSERT_THROW_MES(magic == CUR_CHAIN_MAGIC, "Chain file load error - magic differs"); + a >> events; a >> test_base; + a >> magic; + CHECK_AND_ASSERT_THROW_MES(magic == CUR_CHAIN_MAGIC, "Chain file load error - magic differs"); + CHECK_AND_ASSERT_THROW_MES(!data_file.fail(), "Chain file load error - file.fail()"); + CHECK_AND_ASSERT_THROW_MES(!data_file.bad(), "Chain file load error - bad read"); return true; } catch(...) @@ -276,10 +312,10 @@ static bool unserialize_chain_from_file(std::vector<test_event_entry>& events, g MWARNING("Chain deserialization failed"); return false; } - CATCH_ENTRY_L0("unserialize_chain_from_file", false); + CATCH_ENTRY_L0("deserialize_chain_from_file", false); } -static bool serialize_chain_to_file(std::vector<test_event_entry>& events, gen_trezor_base &test_base, const std::string& file_path) +static bool serialize_chain_to_file(std::vector<test_event_entry>& events, gen_trezor_base &test_base, const std::string& file_path, const chain_file_opts_t& opts) { TRY_ENTRY(); std::ofstream data_file; @@ -290,13 +326,19 @@ static bool serialize_chain_to_file(std::vector<test_event_entry>& events, gen_t { boost::archive::portable_binary_oarchive a(data_file); + a << CUR_CHAIN_MAGIC; + a << CUR_CHAIN_VERSION; + a << opts.heavy_tests; + a << CUR_CHAIN_MAGIC; + a << events; a << test_base; + a << CUR_CHAIN_MAGIC; return !data_file.fail(); } catch(...) { - MWARNING("Chain deserialization failed"); + MWARNING("Chain serialization failed"); return false; } return false; @@ -339,19 +381,39 @@ static void setup_chain(cryptonote::core * core, gen_trezor_base & trezor_base, const bool chain_file_exists = do_serialize && boost::filesystem::exists(chain_path); bool loaded = false; bool generated = false; + unsigned long chain_version = 0; + chain_file_opts_t opts = {.heavy_tests=trezor_base.heavy_tests()}; if (chain_file_exists) { - if (!unserialize_chain_from_file(events, trezor_base, chain_path)) - { - MERROR("Failed to deserialize data from file: " << chain_path); - CHECK_AND_ASSERT_THROW_MES(fix_chain, "Chain load error"); - } else + chain_file_opts_t loaded_opts; + const auto fix_suggestion = fix_chain ? "Chain file will be regenerated." : "Use --fix_chain to regenerate chain file"; + bool deserialize_ok = deserialize_chain_from_file(events, trezor_base, chain_version, chain_path, loaded_opts); + if (!deserialize_ok) { + MERROR("Failed to deserialize data from file: " << chain_path << ". " << fix_suggestion); + CHECK_AND_ASSERT_THROW_MES(fix_chain, "Chain load error. " << fix_suggestion); + } + else if (chain_version != CUR_CHAIN_VERSION) { + MERROR("Loaded chain version " << chain_version << " differs from current target" << CUR_CHAIN_VERSION << " in the chain file: " << chain_path << ". " << fix_suggestion); + CHECK_AND_ASSERT_THROW_MES(fix_chain, "Chain load error - versions differ. " << fix_suggestion); + deserialize_ok = false; + } + else if (opts.heavy_tests && !loaded_opts.heavy_tests) { + MERROR("Loaded chain does not include transactions for heavy test, the chain file: " << chain_path << ". " << fix_suggestion); + CHECK_AND_ASSERT_THROW_MES(fix_chain, "Chain does not heavy tx set - versions differ. " << fix_suggestion); + deserialize_ok = false; + } + + if (deserialize_ok) { trezor_base.load(events); generated = true; loaded = true; } + else { + events.clear(); + trezor_base.clear(); + } } if (!generated) @@ -360,11 +422,12 @@ static void setup_chain(cryptonote::core * core, gen_trezor_base & trezor_base, { trezor_base.clear(); generated = trezor_base.generate(events); + trezor_base.fix_hf(events); if (generated && !loaded && do_serialize) { trezor_base.update_trackers(events); - if (!serialize_chain_to_file(events, trezor_base, chain_path)) + if (!serialize_chain_to_file(events, trezor_base, chain_path, opts)) { MERROR("Failed to serialize data to file: " << chain_path); } @@ -385,6 +448,10 @@ static void setup_chain(cryptonote::core * core, gen_trezor_base & trezor_base, } } +static fnc_accept_output_t fnc_trezor_default_acceptor_tx_in = [] (const fnc_accept_output_crate_t &info) -> bool { + return info.oi.is_coin_base || info.oi.blk_height + CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE <= info.cur_height + 1; +}; + static device_trezor_test *ensure_trezor_test_device(){ if (!trezor_device) { trezor_device = new device_trezor_test(); @@ -393,6 +460,20 @@ static device_trezor_test *ensure_trezor_test_device(){ return trezor_device; } +static size_t min_mixin_for_hf(uint8_t hf) +{ + if (hf >= HF_VERSION_MIN_MIXIN_15) + return 15; + if (hf >= HF_VERSION_MIN_MIXIN_10) + return 10; + return 0; +} + +static int bp_version_from_hf(uint8_t hf) +{ + return hf >= HF_VERSION_BULLETPROOF_PLUS ? 4 : hf >= HF_VERSION_CLSAG ? 3 : hf >= HF_VERSION_SMALLER_BP ? 2 : 1; +} + static void add_hforks(std::vector<test_event_entry>& events, const v_hardforks_t& hard_forks) { event_replay_settings repl_set; @@ -426,6 +507,52 @@ static crypto::public_key get_tx_pub_key_from_received_outs(const tools::wallet2 } } +static void generate_tx_block(std::vector<test_event_entry>& events, std::list<cryptonote::transaction> &txs_lst, cryptonote::block & head, + const cryptonote::account_base& from, const cryptonote::account_base& to, uint64_t amount, size_t repeat, + size_t mixin, bool rct, rct::RangeProofType range_proof_type, int bp_version) +{ + const size_t per_tx_limit = 14; + const size_t iters = repeat ? std::ceil(repeat / (double)per_tx_limit) : 1; + + for(size_t j = 0; j < iters; ++j) + { + const auto dests = build_dsts(to, false, amount, j < iters - 1 ? per_tx_limit : repeat - j * per_tx_limit); + + cryptonote::transaction t; + construct_tx_to_key(events, t, head, from, dests, TESTS_DEFAULT_FEE, mixin, rct, range_proof_type, bp_version, true, fnc_trezor_default_acceptor_tx_in); + txs_lst.push_back(t); + events.push_back(t); + } +} + +static void generate_tx_block(std::vector<test_event_entry>& events, std::list<cryptonote::transaction> &txs_lst, cryptonote::block & head, + const cryptonote::account_base& from, const std::vector<cryptonote::tx_destination_entry>& destinations, size_t repeat, + size_t mixin, bool rct, rct::RangeProofType range_proof_type, int bp_version) +{ + CHECK_AND_ASSERT_THROW_MES(destinations.size() < 16, "Keep destinations below 16, BP+ limit"); + for(size_t j = 0; j < repeat; ++j) + { + cryptonote::transaction t; + construct_tx_to_key(events, t, head, from, destinations, TESTS_DEFAULT_FEE, mixin, rct, range_proof_type, bp_version, true, fnc_trezor_default_acceptor_tx_in); + txs_lst.push_back(t); + events.push_back(t); + } +} + +static void generate_tx_block(std::vector<test_event_entry>& events, std::list<cryptonote::transaction> &txs_lst, cryptonote::block & head, + const cryptonote::account_base& from, const cryptonote::account_base& to, uint64_t amount, size_t repeat = 1, + uint8_t hf = 1) +{ + generate_tx_block(events, txs_lst, head, from, to, amount, repeat, min_mixin_for_hf(hf), hf >= 8, rct::RangeProofPaddedBulletproof, bp_version_from_hf(hf)); +} + +static void generate_tx_block(std::vector<test_event_entry>& events, std::list<cryptonote::transaction> &txs_lst, cryptonote::block & head, + const cryptonote::account_base& from, const std::vector<cryptonote::tx_destination_entry>& destinations, size_t repeat = 1, + uint8_t hf = 1) +{ + generate_tx_block(events, txs_lst, head, from, destinations, repeat, min_mixin_for_hf(hf), hf >= 8, rct::RangeProofPaddedBulletproof, bp_version_from_hf(hf)); +} + static void setup_shim(hw::wallet_shim * shim) { shim->get_tx_pub_key_from_received_outs = &get_tx_pub_key_from_received_outs; @@ -593,29 +720,57 @@ static std::vector<tools::wallet2*> vct_wallets(tools::wallet2* w1=nullptr, tool return res; } -static uint64_t get_available_funds(tools::wallet2* wallet, uint32_t account=0) +#define TREZOR_ALL_ACCOUNTS 0xffffff +static uint64_t get_available_funds(tools::wallet2* wallet, uint32_t account=0, uint64_t * nutxos = nullptr, uint64_t * nutxos_rct = nullptr) { tools::wallet2::transfer_container transfers; wallet->get_transfers(transfers); uint64_t sum = 0; for(const auto & cur : transfers) { - sum += !cur.m_spent && cur.m_subaddr_index.major == account ? cur.amount() : 0; + const bool utxo = !cur.m_spent && (account == TREZOR_ALL_ACCOUNTS || cur.m_subaddr_index.major == account); + sum += utxo ? cur.amount() : 0; + if (nutxos) { + *nutxos += utxo; + } + if (nutxos_rct && cur.is_rct()) { + *nutxos_rct += utxo; + } } return sum; } +static std::string wallet_desc_str(tools::wallet2* wallet, uint32_t account=0) +{ + std::stringstream ss; + uint64_t nutxos = 0, nutxos_rct = 0, nutxos_all = 0, nutxos_rct_all = 0; + uint64_t funds = get_available_funds(wallet, account, &nutxos, &nutxos_rct); + uint64_t funds_all = get_available_funds(wallet, TREZOR_ALL_ACCOUNTS, &nutxos_all, &nutxos_rct_all); + ss << "funds: " << std::setfill(' ') << std::setw(12) << ((double)funds / COIN) + << ", utxos: " << std::setfill(' ') << std::setw(3) << nutxos + << ", rct: " << std::setfill(' ') << std::setw(3) << nutxos_rct + << "; all accounts: " << std::setfill(' ') << std::setw(12) << ((double)funds_all / COIN) + << ", utxos: " << std::setfill(' ') << std::setw(3) << nutxos_all + << ", rct: " << std::setfill(' ') << std::setw(3) << nutxos_rct_all; + return ss.str(); +} + // gen_trezor_base const uint64_t gen_trezor_base::m_ts_start = 1397862000; // As default wallet timestamp is 1397516400 const uint64_t gen_trezor_base::m_wallet_ts = m_ts_start - 60*60*24*4; const std::string gen_trezor_base::m_device_name = "Trezor:udp"; +const std::string gen_trezor_base::m_miner_master_seed_str = "8b133a3868993176b613738816247a7f4d357cae555996519cf5b543e9b3554b"; // sha256(miner) const std::string gen_trezor_base::m_master_seed_str = "14821d0bc5659b24cafbc889dc4fc60785ee08b65d71c525f81eeaba4f3a570f"; const std::string gen_trezor_base::m_device_seed = "permit universe parent weapon amused modify essay borrow tobacco budget walnut lunch consider gallery ride amazing frog forget treat market chapter velvet useless topple"; const std::string gen_trezor_base::m_alice_spend_private = m_master_seed_str; const std::string gen_trezor_base::m_alice_view_private = "a6ccd4ac344a295d1387f8d18c81bdd394f1845de84188e204514ef9370fd403"; +const std::string gen_trezor_base::m_alice2_passphrase = "a"; +const std::string gen_trezor_base::m_alice2_master_seed = "0b86cf0e71204ca3cdc389daf0bb1cf654ac7d54edfed68a9faf921ba140a708"; +const std::string gen_trezor_base::m_bob_master_seed = "81b637d8fcd2c6da6359e6963113a1170de795e4b725b84d1e0b4cfd9ec58ce9"; // sha256(bob) +const std::string gen_trezor_base::m_eve_master_seed = "85262adf74518bbb70c7cb94cd6159d91669e5a81edf1efebd543eadbda9fa2b"; // sha256(eve) gen_trezor_base::gen_trezor_base(){ - m_rct_config = {rct::RangeProofPaddedBulletproof, 1}; + m_rct_config = {rct::RangeProofPaddedBulletproof, 4}; m_test_get_tx_key = true; m_network_type = cryptonote::TESTNET; } @@ -623,26 +778,37 @@ gen_trezor_base::gen_trezor_base(){ gen_trezor_base::gen_trezor_base(const gen_trezor_base &other): m_generator(other.m_generator), m_bt(other.m_bt), m_miner_account(other.m_miner_account), m_bob_account(other.m_bob_account), m_alice_account(other.m_alice_account), m_eve_account(other.m_eve_account), - m_hard_forks(other.m_hard_forks), m_trezor(other.m_trezor), m_rct_config(other.m_rct_config), m_top_hard_fork(other.m_top_hard_fork), + m_hard_forks(other.m_hard_forks), m_head(other.m_head), m_events(other.m_events), + m_trezor(other.m_trezor), m_rct_config(other.m_rct_config), m_top_hard_fork(other.m_top_hard_fork), m_heavy_tests(other.m_heavy_tests), m_test_get_tx_key(other.m_test_get_tx_key), m_live_refresh_enabled(other.m_live_refresh_enabled), - m_network_type(other.m_network_type), m_daemon(other.m_daemon) + m_network_type(other.m_network_type), m_daemon(other.m_daemon), m_alice2_account(other.m_alice2_account), + m_trezor_path(other.m_trezor_path), m_trezor_pin(other.m_trezor_pin), m_trezor_passphrase(other.m_trezor_passphrase), + m_trezor_use_passphrase(other.m_trezor_use_passphrase), m_trezor_use_alice2(other.m_trezor_use_alice2), + m_gen_heavy_test_set(other.m_gen_heavy_test_set), + m_mining_enabled(other.m_mining_enabled), + m_mining_timeout(other.m_mining_timeout), + m_no_change_in_tested_tx(other.m_no_change_in_tested_tx) { } -void gen_trezor_base::setup_args(const std::string & trezor_path, bool heavy_tests) +void gen_trezor_base::setup_args(const std::string & trezor_path, bool heavy_tests, bool mining_enabled, long mining_timeout) { m_trezor_path = trezor_path.empty() ? m_device_name : std::string("Trezor:") + trezor_path; m_heavy_tests = heavy_tests; + m_gen_heavy_test_set |= heavy_tests; + m_mining_enabled = mining_enabled; + m_mining_timeout = mining_timeout; } -void gen_trezor_base::setup_trezor() +void gen_trezor_base::setup_trezor(bool use_passphrase, const std::string & pin) { hw::device &hwdev = hw::get_device(m_trezor_path); auto trezor = dynamic_cast<device_trezor_test *>(&hwdev); CHECK_AND_ASSERT_THROW_MES(trezor, "Dynamic cast failed"); - trezor->setup_for_tests(m_trezor_path, m_device_seed, m_network_type); + trezor->set_callback(this); + trezor->setup_for_tests(m_trezor_path, m_device_seed, m_network_type, use_passphrase, pin); m_trezor = trezor; } @@ -658,15 +824,25 @@ void gen_trezor_base::fork(gen_trezor_base & other) other.m_top_hard_fork = m_top_hard_fork; other.m_trezor_path = m_trezor_path; other.m_heavy_tests = m_heavy_tests; + other.m_gen_heavy_test_set = m_gen_heavy_test_set; other.m_rct_config = m_rct_config; other.m_test_get_tx_key = m_test_get_tx_key; other.m_live_refresh_enabled = m_live_refresh_enabled; + other.m_trezor_pin = m_trezor_pin; + other.m_trezor_passphrase = m_trezor_passphrase; + other.m_trezor_use_passphrase = m_trezor_use_passphrase; + other.m_trezor_use_alice2 = m_trezor_use_alice2; + other.m_mining_enabled = m_mining_enabled; + other.m_mining_timeout = m_mining_timeout; + other.m_miner_account = m_miner_account; other.m_bob_account = m_bob_account; other.m_alice_account = m_alice_account; + other.m_alice2_account = m_alice2_account; other.m_eve_account = m_eve_account; other.m_trezor = m_trezor; + other.m_no_change_in_tested_tx = m_no_change_in_tested_tx; other.m_generator.set_events(&other.m_events); other.m_generator.set_network_type(m_network_type); } @@ -688,16 +864,55 @@ void gen_trezor_base::add_shared_events(std::vector<test_event_entry>& events) } } -void gen_trezor_base::init_fields() +void gen_trezor_base::init_accounts() { - m_miner_account.generate(); - DEFAULT_HARDFORKS(m_hard_forks); - crypto::secret_key master_seed{}; - CHECK_AND_ASSERT_THROW_MES(epee::string_tools::hex_to_pod(m_master_seed_str, master_seed), "Hexdecode fails"); + CHECK_AND_ASSERT_THROW_MES(epee::string_tools::hex_to_pod(m_miner_master_seed_str, master_seed), "Hexdecode fails: m_miner_master_seed_str"); + + m_miner_account.generate(master_seed, true); + DEFAULT_HARDFORKS(m_hard_forks); + CHECK_AND_ASSERT_THROW_MES(epee::string_tools::hex_to_pod(m_master_seed_str, master_seed), "Hexdecode fails: m_master_seed_str"); m_alice_account.generate(master_seed, true); m_alice_account.set_createtime(m_wallet_ts); + + CHECK_AND_ASSERT_THROW_MES(epee::string_tools::hex_to_pod(m_alice2_master_seed, master_seed), "Hexdecode fails: m_alice2_master_seed"); + m_alice2_account.generate(master_seed, true); + m_alice2_account.set_createtime(m_wallet_ts); + + CHECK_AND_ASSERT_THROW_MES(epee::string_tools::hex_to_pod(m_bob_master_seed, master_seed), "Hexdecode fails: m_bob_master_seed"); + m_bob_account.generate(master_seed, true); + m_bob_account.set_createtime(m_wallet_ts); + + CHECK_AND_ASSERT_THROW_MES(epee::string_tools::hex_to_pod(m_eve_master_seed, master_seed), "Hexdecode fails: m_eve_master_seed"); + m_eve_account.generate(master_seed, true); + m_eve_account.set_createtime(m_wallet_ts); +} + +void gen_trezor_base::init_wallets() +{ + m_wl_alice.reset(new tools::wallet2(m_network_type, 1, true)); + m_wl_alice2.reset(new tools::wallet2(m_network_type, 1, true)); + m_wl_bob.reset(new tools::wallet2(m_network_type, 1, true)); + m_wl_eve.reset(new tools::wallet2(m_network_type, 1, true)); + + wallet_accessor_test::set_account(m_wl_alice.get(), m_alice_account); + wallet_accessor_test::set_account(m_wl_alice2.get(), m_alice2_account); + wallet_accessor_test::set_account(m_wl_bob.get(), m_bob_account); + wallet_accessor_test::set_account(m_wl_eve.get(), m_eve_account); + + m_wl_alice->expand_subaddresses({5, 20}); + m_wl_alice2->expand_subaddresses({5, 20}); + m_wl_bob->expand_subaddresses({5, 20}); + m_wl_eve->expand_subaddresses({5, 20}); +} + +void gen_trezor_base::log_wallets_desc() +{ + MDEBUG("Wallet for Alice: " << wallet_desc_str(m_wl_alice.get())); + MDEBUG("Wallet for Alice2: " << wallet_desc_str(m_wl_alice2.get())); + MDEBUG("Wallet for Bob: " << wallet_desc_str(m_wl_bob.get())); + MDEBUG("Wallet for Eve: " << wallet_desc_str(m_wl_eve.get())); } void gen_trezor_base::update_client_settings() @@ -710,14 +925,8 @@ void gen_trezor_base::update_client_settings() bool gen_trezor_base::generate(std::vector<test_event_entry>& events) { - init_fields(); - setup_trezor(); - - m_live_refresh_enabled = false; - update_client_settings(); - - m_alice_account.create_from_device(*m_trezor); - m_alice_account.set_createtime(m_wallet_ts); + init_accounts(); + setup_trezor(m_trezor_use_passphrase, m_trezor_pin); // Events, custom genesis so it matches wallet genesis auto & generator = m_generator; // macro shortcut @@ -733,10 +942,6 @@ bool gen_trezor_base::generate(std::vector<test_event_entry>& events) generator.add_block(blk_gen, 0, block_weights, 0, rew); // First event has to be the genesis block - m_bob_account.generate(); - m_eve_account.generate(); - m_bob_account.set_createtime(m_wallet_ts); - m_eve_account.set_createtime(m_wallet_ts); cryptonote::account_base * accounts[] = {TREZOR_ACCOUNT_ORDERING}; for(cryptonote::account_base * ac : accounts){ events.push_back(*ac); @@ -745,6 +950,7 @@ bool gen_trezor_base::generate(std::vector<test_event_entry>& events) // Another block with predefined timestamp. // Carefully set reward and already generated coins so it passes miner_tx check. cryptonote::block blk_0; + cryptonote::block & last_block = blk_0; { std::list<cryptonote::transaction> tx_list; const crypto::hash prev_id = get_block_hash(blk_gen); @@ -754,40 +960,37 @@ bool gen_trezor_base::generate(std::vector<test_event_entry>& events) generator.construct_block(blk_0, 1, prev_id, m_miner_account, m_ts_start, already_generated_coins, block_weights, tx_list); } - events.push_back(blk_0); - MDEBUG("Gen+1 block has time: " << blk_0.timestamp << " blid: " << get_block_hash(blk_0)); + events.push_back(last_block); + MDEBUG("Gen+1 block has time: " << last_block.timestamp << " blid: " << get_block_hash(last_block)); // Generate some spendable funds on the Miner account - REWIND_BLOCKS_N(events, blk_3, blk_0, m_miner_account, 40); - - // Rewind so the miners funds are unlocked for initial transactions. - REWIND_BLOCKS(events, blk_3r, blk_3, m_miner_account); - - // Non-rct transactions Miner -> Bob - MAKE_TX_LIST_START(events, txs_blk_4, m_miner_account, m_alice_account, MK_COINS(10), blk_3); - MAKE_TX_LIST(events, txs_blk_4, m_miner_account, m_alice_account, MK_COINS(7), blk_3); - MAKE_TX_LIST(events, txs_blk_4, m_miner_account, m_alice_account, MK_COINS(7), blk_3); - MAKE_TX_LIST(events, txs_blk_4, m_miner_account, m_alice_account, MK_COINS(14), blk_3); - MAKE_TX_LIST(events, txs_blk_4, m_miner_account, m_alice_account, MK_COINS(20), blk_3); - MAKE_TX_LIST(events, txs_blk_4, m_miner_account, m_alice_account, MK_COINS(2), blk_3); - MAKE_TX_LIST(events, txs_blk_4, m_miner_account, m_alice_account, MK_COINS(2), blk_3); - MAKE_TX_LIST(events, txs_blk_4, m_miner_account, m_alice_account, MK_COINS(5), blk_3); - MAKE_NEXT_BLOCK_TX_LIST(events, blk_4, blk_3r, m_miner_account, txs_blk_4); - REWIND_BLOCKS(events, blk_4r, blk_4, m_miner_account); // rewind to unlock + REWIND_BLOCKS_N(events, blk_3, last_block, m_miner_account, 20); + last_block = blk_3; + + // Rewind so the miners funds are unlocked for initial transactions, CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW blocks + REWIND_BLOCKS_N(events, blk_3r1, last_block, m_miner_account, CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW - 10); + last_block = blk_3r1; + last_block = rewind_blocks_with_decoys(events, last_block, 10, 1, TEST_DECOY_MINING_COUNTS); + + MDEBUG("Mining chain generated, height: " << get_block_height(last_block) << ", #events: " << events.size() << ", last block: " << get_block_hash(last_block)); + + // Non-rct transactions Miner -> Alice + // Note that change transaction is spendable after CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE so we batch it to a tx + std::list<cryptonote::transaction> txs_blk_4; + generate_tx_block(events, txs_blk_4, last_block, m_miner_account, m_alice_account, MK_TCOINS(0.1), 20, 1); + MAKE_NEXT_BLOCK_TX_LIST(events, blk_4, last_block, m_miner_account, txs_blk_4); + last_block = blk_4; + MDEBUG("Non-rct transactions generated, height: " << get_block_height(last_block) << ", #events: " << events.size() << ", last block: " << get_block_hash(last_block)); // Hard fork to bulletproofs version, v9. const uint8_t CUR_HF = 9; auto hardfork_height = num_blocks(events); // next block is v9 ADD_HARDFORK(m_hard_forks, CUR_HF, hardfork_height); add_hforks(events, m_hard_forks); - MDEBUG("Hardfork height: " << hardfork_height << " at block: " << get_block_hash(blk_4r)); + MDEBUG("Hardfork " << (int)CUR_HF << " height: " << hardfork_height << ", #events: " << events.size() << " at block: " << get_block_hash(last_block)); // RCT transactions, wallets have to be used, wallet init - m_wl_alice.reset(new tools::wallet2(m_network_type, 1, true)); - m_wl_bob.reset(new tools::wallet2(m_network_type, 1, true)); - wallet_accessor_test::set_account(m_wl_alice.get(), m_alice_account); - wallet_accessor_test::set_account(m_wl_bob.get(), m_bob_account); - + init_wallets(); auto addr_alice_sub_0_1 = m_wl_alice->get_subaddress({0, 1}); auto addr_alice_sub_0_2 = m_wl_alice->get_subaddress({0, 2}); auto addr_alice_sub_0_3 = m_wl_alice->get_subaddress({0, 3}); @@ -797,40 +1000,50 @@ bool gen_trezor_base::generate(std::vector<test_event_entry>& events) auto addr_alice_sub_1_1 = m_wl_alice->get_subaddress({1, 1}); auto addr_alice_sub_1_2 = m_wl_alice->get_subaddress({1, 2}); - // Miner -> Bob, RCT funds - MAKE_TX_LIST_START_RCT(events, txs_blk_5, m_miner_account, m_alice_account, MK_COINS(50), 10, blk_4); + // Miner -> Alice, RCT funds + std::list<cryptonote::transaction> txs_blk_5; + generate_tx_block(events, txs_blk_5, last_block, m_miner_account, m_alice_account, MK_TCOINS(1), 1, CUR_HF); + generate_tx_block(events, txs_blk_5, last_block, m_miner_account, m_alice_account, MK_TCOINS(0.1), heavy_tests() || heavy_test_set() ? 95 : 0, CUR_HF); - const size_t target_rct = m_heavy_tests ? 105 : 15; - for(size_t i = 0; i < target_rct; ++i) - { - MAKE_TX_MIX_LIST_RCT(events, txs_blk_5, m_miner_account, m_alice_account, MK_COINS(1) >> 2, 10, blk_4); - } - - // Sub-address destinations - MAKE_TX_MIX_DEST_LIST_RCT(events, txs_blk_5, m_miner_account, build_dsts(addr_alice_sub_0_1, true, MK_COINS(1) >> 1), 10, blk_4); - MAKE_TX_MIX_DEST_LIST_RCT(events, txs_blk_5, m_miner_account, build_dsts(addr_alice_sub_0_2, true, MK_COINS(1) >> 1), 10, blk_4); - MAKE_TX_MIX_DEST_LIST_RCT(events, txs_blk_5, m_miner_account, build_dsts(addr_alice_sub_0_3, true, MK_COINS(1) >> 1), 10, blk_4); - MAKE_TX_MIX_DEST_LIST_RCT(events, txs_blk_5, m_miner_account, build_dsts(addr_alice_sub_0_4, true, MK_COINS(1) >> 1), 10, blk_4); + // Simple RCT transactions + MDEBUG("blk_5 construction - RCT set"); + generate_tx_block(events, txs_blk_5, last_block, m_miner_account, m_alice_account, MK_TCOINS(1), 1, CUR_HF); + generate_tx_block(events, txs_blk_5, last_block, m_miner_account, build_dsts({ + { m_alice_account, false, MK_TCOINS(1) }, + { m_alice_account, false, MK_TCOINS(1) }, + { m_alice_account, false, MK_TCOINS(1) }, + { m_alice2_account, false, MK_TCOINS(1) } + }), 1, CUR_HF); // Sub-address destinations + multi out to force use of additional keys - MAKE_TX_MIX_DEST_LIST_RCT(events, txs_blk_5, m_miner_account, build_dsts({{addr_alice_sub_0_1, true, MK_COINS(1) >> 1}, {addr_alice_sub_0_2, true, MK_COINS(1) >> 1}}), 10, blk_4); - MAKE_TX_MIX_DEST_LIST_RCT(events, txs_blk_5, m_miner_account, build_dsts({{addr_alice_sub_0_1, true, MK_COINS(1) >> 1}, {addr_alice_sub_0_2, true, MK_COINS(1) >> 1}, {addr_alice_sub_0_3, true, MK_COINS(1) >> 1}}), 10, blk_4); - MAKE_TX_MIX_DEST_LIST_RCT(events, txs_blk_5, m_miner_account, build_dsts({{m_miner_account, false, MK_COINS(1) >> 1}, {addr_alice_sub_0_2, true, MK_COINS(1) >> 1}, {addr_alice_sub_0_3, true, MK_COINS(1) >> 1}}), 10, blk_4); - MAKE_TX_MIX_DEST_LIST_RCT(events, txs_blk_5, m_miner_account, build_dsts({{m_miner_account, false, MK_COINS(1) >> 1}, {addr_alice_sub_0_2, true, MK_COINS(1) >> 1}, {addr_alice_sub_0_3, true, MK_COINS(1) >> 1}}), 10, blk_4); + generate_tx_block(events, txs_blk_5, last_block, m_miner_account, build_dsts({ + {addr_alice_sub_0_1, true, MK_TCOINS(0.1)}, + {addr_alice_sub_0_2, true, MK_TCOINS(0.1)}, + {addr_alice_sub_0_1, true, MK_TCOINS(0.1)}, + {addr_alice_sub_0_2, true, MK_TCOINS(0.1)}, + {addr_alice_sub_0_3, true, MK_TCOINS(0.1)}, + {m_miner_account, false, MK_TCOINS(0.1)}, + {addr_alice_sub_0_2, true, MK_TCOINS(0.1)}, + {addr_alice_sub_0_3, true, MK_TCOINS(0.1)}, + {m_miner_account, false, MK_TCOINS(0.1)}, + {addr_alice_sub_0_2, true, MK_TCOINS(0.1)}, + {addr_alice_sub_0_4, true, MK_TCOINS(0.1)}}), 1, CUR_HF); // Transfer to other accounts - MAKE_TX_MIX_DEST_LIST_RCT(events, txs_blk_5, m_miner_account, build_dsts(addr_alice_sub_1_0, true, MK_COINS(1) >> 1), 10, blk_4); - MAKE_TX_MIX_DEST_LIST_RCT(events, txs_blk_5, m_miner_account, build_dsts(addr_alice_sub_1_1, true, MK_COINS(1) >> 1), 10, blk_4); - MAKE_TX_MIX_DEST_LIST_RCT(events, txs_blk_5, m_miner_account, build_dsts({{addr_alice_sub_1_0, true, MK_COINS(1) >> 1}, {addr_alice_sub_1_1, true, MK_COINS(1) >> 1}, {addr_alice_sub_0_3, true, MK_COINS(1) >> 1}}), 10, blk_4); - MAKE_TX_MIX_DEST_LIST_RCT(events, txs_blk_5, m_miner_account, build_dsts({{addr_alice_sub_1_1, true, MK_COINS(1) >> 1}, {addr_alice_sub_1_1, true, MK_COINS(1) >> 1}, {addr_alice_sub_0_2, true, MK_COINS(1) >> 1}}), 10, blk_4); - MAKE_TX_MIX_DEST_LIST_RCT(events, txs_blk_5, m_miner_account, build_dsts({{addr_alice_sub_1_2, true, MK_COINS(1) >> 1}, {addr_alice_sub_1_1, true, MK_COINS(1) >> 1}, {addr_alice_sub_0_5, true, MK_COINS(1) >> 1}}), 10, blk_4); - - // Simple RCT transactions - MAKE_TX_MIX_LIST_RCT(events, txs_blk_5, m_miner_account, m_alice_account, MK_COINS(7), 10, blk_4); - MAKE_TX_MIX_LIST_RCT(events, txs_blk_5, m_miner_account, m_alice_account, MK_COINS(10), 10, blk_4); - MAKE_TX_MIX_LIST_RCT(events, txs_blk_5, m_miner_account, m_alice_account, MK_COINS(30), 10, blk_4); - MAKE_TX_MIX_LIST_RCT(events, txs_blk_5, m_miner_account, m_alice_account, MK_COINS(40), 10, blk_4); - MAKE_NEXT_BLOCK_TX_LIST_HF(events, blk_5, blk_4r, m_miner_account, txs_blk_5, CUR_HF); + generate_tx_block(events, txs_blk_5, last_block, m_miner_account, build_dsts({ + {addr_alice_sub_1_0, true, MK_TCOINS(0.1)}, + {addr_alice_sub_1_1, true, MK_TCOINS(0.1)}, + {addr_alice_sub_1_0, true, MK_TCOINS(0.1)}, + {addr_alice_sub_1_1, true, MK_TCOINS(0.1)}, + {addr_alice_sub_0_3, true, MK_TCOINS(0.1)}, + {addr_alice_sub_1_1, true, MK_TCOINS(0.1)}, + {addr_alice_sub_1_1, true, MK_TCOINS(0.1)}, + {addr_alice_sub_0_2, true, MK_TCOINS(0.1)}, + {addr_alice_sub_1_2, true, MK_TCOINS(0.1)}, + {addr_alice_sub_1_1, true, MK_TCOINS(0.1)}, + {addr_alice_sub_0_5, true, MK_TCOINS(0.1)}}), 1, CUR_HF); + MAKE_NEXT_BLOCK_TX_LIST_HF(events, blk_5, last_block, m_miner_account, txs_blk_5, CUR_HF); + last_block = blk_5; // Simple transaction check bool resx = rct::verRctSemanticsSimple(txs_blk_5.begin()->rct_signatures); @@ -838,23 +1051,34 @@ bool gen_trezor_base::generate(std::vector<test_event_entry>& events) CHECK_AND_ASSERT_THROW_MES(resx, "Tsx5[0] semantics failed"); CHECK_AND_ASSERT_THROW_MES(resy, "Tsx5[0] non-semantics failed"); - REWIND_BLOCKS_HF(events, blk_5r, blk_5, m_miner_account, CUR_HF); // rewind to unlock + // Rewind blocks so there are more spendable coins + last_block = rewind_blocks_with_decoys(events, last_block, CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE + 2, CUR_HF, TEST_DECOY_MINING_COUNTS); // RCT transactions, wallets have to be used - wallet_tools::process_transactions(m_wl_alice.get(), events, blk_5r, m_bt); - wallet_tools::process_transactions(m_wl_bob.get(), events, blk_5r, m_bt); + MDEBUG("Post blk_5 construction"); + wallet_tools::process_transactions(vct_wallets(m_wl_alice.get(), m_wl_alice2.get(), m_wl_bob.get()), events, last_block, m_bt); + log_wallets_desc(); // Send Alice -> Bob, manually constructed. Simple TX test, precondition. cryptonote::transaction tx_1; + uint64_t tx_1_amount = MK_TCOINS(0.1); std::vector<size_t> selected_transfers; std::vector<tx_source_entry> sources; - bool res = wallet_tools::fill_tx_sources(m_wl_alice.get(), sources, TREZOR_TEST_MIXIN, boost::none, MK_COINS(2), m_bt, selected_transfers, num_blocks(events) - 1, 0, 1); - CHECK_AND_ASSERT_THROW_MES(res, "TX Fill sources failed"); + bool res = wallet_tools::fill_tx_sources(m_wl_alice.get(), sources, TREZOR_TEST_MIXIN, boost::none, tx_1_amount + TREZOR_TEST_FEE, m_bt, selected_transfers, get_block_height(last_block), 0, 1); + CHECK_AND_ASSERT_THROW_MES(res, "Tx01: tx fill sources failed"); + + res = construct_tx_to_key(tx_1, m_wl_alice.get(), m_bob_account, tx_1_amount, sources, TREZOR_TEST_FEE, true, rct::RangeProofPaddedBulletproof, 1); + CHECK_AND_ASSERT_THROW_MES(res, "Tx01: tx construction failed"); - construct_tx_to_key(tx_1, m_wl_alice.get(), m_bob_account, MK_COINS(1), sources, TREZOR_TEST_FEE, true, rct::RangeProofPaddedBulletproof, 1); + std::list<cryptonote::transaction> txs_blk_6; + txs_blk_6.push_back(tx_1); events.push_back(tx_1); - MAKE_NEXT_BLOCK_TX1_HF(events, blk_6, blk_5r, m_miner_account, tx_1, CUR_HF); - MDEBUG("Post 1st tsx: " << (num_blocks(events) - 1) << " at block: " << get_block_hash(blk_6)); + + generate_tx_block(events, txs_blk_6, last_block, m_miner_account, m_alice_account, MK_TCOINS(1), 1, CUR_HF); + MAKE_NEXT_BLOCK_TX_LIST_HF(events, blk_6, last_block, m_miner_account, txs_blk_6, CUR_HF); + + last_block = blk_6; + MDEBUG("Post 1st tsx: " << (num_blocks(events) - 1) << " at block: " << get_block_hash(last_block) << ", event: " << events.size()); // Simple transaction check resx = rct::verRctSemanticsSimple(tx_1.rct_signatures); @@ -862,20 +1086,20 @@ bool gen_trezor_base::generate(std::vector<test_event_entry>& events) CHECK_AND_ASSERT_THROW_MES(resx, "tx_1 semantics failed"); CHECK_AND_ASSERT_THROW_MES(resy, "tx_1 non-semantics failed"); - REWIND_BLOCKS_N_HF(events, blk_6r, blk_6, m_miner_account, 10, CUR_HF); - wallet_tools::process_transactions(m_wl_alice.get(), events, blk_6, m_bt); - wallet_tools::process_transactions(m_wl_bob.get(), events, blk_6, m_bt); - MDEBUG("Available funds on Alice: " << get_available_funds(m_wl_alice.get())); - MDEBUG("Available funds on Bob: " << get_available_funds(m_wl_bob.get())); + last_block = rewind_blocks_with_decoys(events, last_block, CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE, CUR_HF, TEST_DECOY_MINING_COUNTS); + MDEBUG("Blockchain generation up to " << (int)CUR_HF << " finished at height " << num_blocks(events) << ", #events: " << events.size()); + + wallet_tools::process_transactions(TREZOR_ALL_WALLET_VCT, events, last_block, m_bt); + log_wallets_desc(); - m_head = blk_6r; + m_head = last_block; m_events = events; return true; } void gen_trezor_base::load(std::vector<test_event_entry>& events) { - init_fields(); + init_accounts(); m_events = events; m_generator.set_events(&m_events); m_generator.set_network_type(m_network_type); @@ -910,37 +1134,31 @@ void gen_trezor_base::load(std::vector<test_event_entry>& events) } // Setup wallets, synchronize blocks - m_bob_account.set_createtime(m_wallet_ts); - m_eve_account.set_createtime(m_wallet_ts); - - setup_trezor(); - update_client_settings(); - m_alice_account.create_from_device(*m_trezor); - m_alice_account.set_createtime(m_wallet_ts); - - m_wl_alice.reset(new tools::wallet2(m_network_type, 1, true)); - m_wl_bob.reset(new tools::wallet2(m_network_type, 1, true)); - m_wl_eve.reset(new tools::wallet2(m_network_type, 1, true)); - wallet_accessor_test::set_account(m_wl_alice.get(), m_alice_account); - wallet_accessor_test::set_account(m_wl_bob.get(), m_bob_account); - wallet_accessor_test::set_account(m_wl_eve.get(), m_eve_account); - - wallet_tools::process_transactions(m_wl_alice.get(), events, m_head, m_bt); - wallet_tools::process_transactions(m_wl_bob.get(), events, m_head, m_bt); - MDEBUG("Available funds on Alice: " << get_available_funds(m_wl_alice.get())); - MDEBUG("Available funds on Bob: " << get_available_funds(m_wl_bob.get())); + init_wallets(); + wallet_tools::process_transactions(TREZOR_ALL_WALLET_VCT, events, m_head, m_bt); + log_wallets_desc(); } void gen_trezor_base::rewind_blocks(std::vector<test_event_entry>& events, size_t rewind_n, uint8_t hf) { - auto & generator = m_generator; // macro shortcut - REWIND_BLOCKS_N_HF(events, blk_new, m_head, m_miner_account, rewind_n, hf); - m_head = blk_new; + m_head = rewind_blocks_with_decoys(events, m_head, rewind_n, hf, TEST_DECOY_MINING_COUNTS); m_events = events; - MDEBUG("Blocks rewound: " << rewind_n << ", #blocks: " << num_blocks(events) << ", #events: " << events.size()); +} + +cryptonote::block gen_trezor_base::rewind_blocks_with_decoys(std::vector<test_event_entry>& events, cryptonote::block & head, size_t rewind_n, uint8_t hf, size_t num_decoys_per_block) +{ + auto & generator = m_generator; // shortcut required by MAKE_NEXT_BLOCK_TX_LIST_HF + cryptonote::block blk_last = head; + for (size_t i = 0; i < rewind_n; ++i) + { + std::list<cryptonote::transaction> txs_lst; + generate_tx_block(events, txs_lst, blk_last, m_miner_account, m_bob_account, MK_TCOINS(0.0001), num_decoys_per_block, hf); + MAKE_NEXT_BLOCK_TX_LIST_HF(events, blk, blk_last, m_miner_account, txs_lst, hf); + blk_last = blk; + } - wallet_tools::process_transactions(m_wl_alice.get(), events, m_head, m_bt); - wallet_tools::process_transactions(m_wl_bob.get(), events, m_head, m_bt); + MDEBUG("Blocks rewound: " << rewind_n << ", #blocks: " << num_blocks(events) << ", #events: " << events.size()); + return blk_last; } void gen_trezor_base::fix_hf(std::vector<test_event_entry>& events) @@ -958,8 +1176,15 @@ void gen_trezor_base::fix_hf(std::vector<test_event_entry>& events) ADD_HARDFORK(m_hard_forks, hf_to_add, hardfork_height); add_top_hfork(events, m_hard_forks); MDEBUG("Hardfork added at height: " << hardfork_height << ", from " << (int)current_hf << " to " << (int)hf_to_add); - rewind_blocks(events, 10, hf_to_add); + + // Wallet2 requires 10 blocks extra to activate HF + rewind_blocks(events, hf_to_add == m_top_hard_fork ? 10 : 1, hf_to_add); + + if (hf_to_add == m_top_hard_fork) { + MDEBUG("Blockchain generation up to " << (int)hf_to_add << " finished at height " << num_blocks(events) << ", #events: " << events.size()); + } } + wallet_tools::process_transactions(TREZOR_ALL_WALLET_VCT, events, m_head, m_bt); } void gen_trezor_base::update_trackers(std::vector<test_event_entry>& events) @@ -967,25 +1192,25 @@ void gen_trezor_base::update_trackers(std::vector<test_event_entry>& events) wallet_tools::process_transactions(nullptr, events, m_head, m_bt); } -void gen_trezor_base::test_setup(std::vector<test_event_entry>& events) -{ - add_shared_events(events); - - setup_trezor(); +void gen_trezor_base::init_trezor_account() { + setup_trezor(m_trezor_use_passphrase, m_trezor_pin); update_client_settings(); - m_alice_account.create_from_device(*m_trezor); - m_alice_account.set_createtime(m_wallet_ts); + if (m_trezor_use_alice2) { + m_alice2_account.create_from_device(*m_trezor); + m_alice2_account.set_createtime(m_wallet_ts); + } else { + m_alice_account.create_from_device(*m_trezor); + m_alice_account.set_createtime(m_wallet_ts); + } +} - m_wl_alice.reset(new tools::wallet2(m_network_type, 1, true)); - m_wl_bob.reset(new tools::wallet2(m_network_type, 1, true)); - m_wl_eve.reset(new tools::wallet2(m_network_type, 1, true)); - wallet_accessor_test::set_account(m_wl_alice.get(), m_alice_account); - wallet_accessor_test::set_account(m_wl_bob.get(), m_bob_account); - wallet_accessor_test::set_account(m_wl_eve.get(), m_eve_account); - wallet_tools::process_transactions(m_wl_alice.get(), events, m_head, m_bt); - wallet_tools::process_transactions(m_wl_bob.get(), events, m_head, m_bt); - wallet_tools::process_transactions(m_wl_eve.get(), events, m_head, m_bt); +void gen_trezor_base::test_setup(std::vector<test_event_entry>& events) +{ + add_shared_events(events); + init_trezor_account(); + init_wallets(); + wallet_tools::process_transactions(TREZOR_ALL_WALLET_VCT, events, m_head, m_bt); } void gen_trezor_base::add_transactions_to_events( @@ -1012,8 +1237,10 @@ void gen_trezor_base::add_transactions_to_events( m_head = blk_new; } -void gen_trezor_base::test_trezor_tx(std::vector<test_event_entry>& events, std::vector<tools::wallet2::pending_tx>& ptxs, std::vector<cryptonote::address_parse_info>& dsts_info, test_generator &generator, std::vector<tools::wallet2*> wallets, bool is_sweep) +void gen_trezor_base::test_trezor_tx(std::vector<test_event_entry>& events, std::vector<tools::wallet2::pending_tx>& ptxs, std::vector<cryptonote::address_parse_info>& dsts_info, test_generator &generator, std::vector<tools::wallet2*> wallets, bool is_sweep, size_t sender_idx) { + CHECK_AND_ASSERT_THROW_MES(sender_idx < wallets.size(), "Sender index out of bounds"); + // Construct pending transaction for signature in the Trezor. const uint64_t height_pre = num_blocks(events) - 1; cryptonote::block head_block = get_head_block(events); @@ -1067,6 +1294,7 @@ void gen_trezor_base::test_trezor_tx(std::vector<test_event_entry>& events, std: // TX receive test uint64_t sum_in = 0; uint64_t sum_out = 0; + const cryptonote::account_base * sender_account = &wallets[sender_idx]->get_account(); for(size_t txid = 0; txid < exported_txs.ptx.size(); ++txid) { auto &c_ptx = exported_txs.ptx[txid]; @@ -1097,7 +1325,7 @@ void gen_trezor_base::test_trezor_tx(std::vector<test_event_entry>& events, std: CHECK_AND_ASSERT_THROW_MES(c_tx.rct_signatures.txnFee + cur_sum_out == cur_sum_in, "Tx Input Output amount mismatch"); for (size_t widx = 0; widx < wallets.size(); ++widx) { - const bool sender = widx == 0; + const bool sender = widx == sender_idx; tools::wallet2 *wl = wallets[widx]; wallet_tools::process_transactions(wl, events, m_head, m_bt, boost::make_optional(head_hash)); @@ -1151,8 +1379,8 @@ void gen_trezor_base::test_trezor_tx(std::vector<test_event_entry>& events, std: CHECK_AND_ASSERT_THROW_MES(exp_payment_id.empty() || num_payment_id_checks_done > 0, "No Payment ID checks"); if(!is_sweep){ - CHECK_AND_ASSERT_THROW_MES(num_received == num_outs, "Number of received outputs do not match number of outgoing"); - CHECK_AND_ASSERT_THROW_MES(recv_out_idx.size() == num_outs, "Num of outs received do not match"); + CHECK_AND_ASSERT_THROW_MES((num_received + m_no_change_in_tested_tx) == num_outs, "Number of received outputs do not match number of outgoing"); + CHECK_AND_ASSERT_THROW_MES((recv_out_idx.size() + m_no_change_in_tested_tx) == num_outs, "Num of outs received do not match"); } else { CHECK_AND_ASSERT_THROW_MES(num_received + 1 >= num_outs, "Number of received outputs do not match number of outgoing"); CHECK_AND_ASSERT_THROW_MES(recv_out_idx.size() + 1 >= num_outs, "Num of outs received do not match"); // can have dummy out @@ -1165,7 +1393,8 @@ void gen_trezor_base::test_trezor_tx(std::vector<test_event_entry>& events, std: CHECK_AND_ASSERT_THROW_MES(sum_in == sum_out, "Tx amount mismatch"); // Test get_tx_key feature for stored private tx keys - test_get_tx(events, wallets, exported_txs.ptx, aux_data.tx_device_aux); + CHECK_AND_ASSERT_THROW_MES(sender_account != nullptr, "Sender account is null"); + test_get_tx(events, wallets, sender_account, exported_txs.ptx, aux_data.tx_device_aux); } bool gen_trezor_base::verify_tx_key(const ::crypto::secret_key & tx_priv, const ::crypto::public_key & tx_pub, const subaddresses_t & subs) @@ -1187,6 +1416,7 @@ bool gen_trezor_base::verify_tx_key(const ::crypto::secret_key & tx_priv, const void gen_trezor_base::test_get_tx( std::vector<test_event_entry>& events, std::vector<tools::wallet2*> wallets, + const cryptonote::account_base * account, const std::vector<tools::wallet2::pending_tx> &ptxs, const std::vector<std::string> &aux_tx_info) { @@ -1228,7 +1458,7 @@ void gen_trezor_base::test_get_tx( dev_cold->load_tx_key_data(tx_key_data, aux_tx_info[txid]); CHECK_AND_ASSERT_THROW_MES(std::string(tx_prefix_hash.data, 32) == tx_key_data.tx_prefix_hash, "TX prefix mismatch"); - dev_cold->get_tx_key(tx_keys, tx_key_data, m_alice_account.get_keys().m_view_secret_key); + dev_cold->get_tx_key(tx_keys, tx_key_data, account->get_keys().m_view_secret_key); // alice CHECK_AND_ASSERT_THROW_MES(!tx_keys.empty(), "Empty TX keys"); CHECK_AND_ASSERT_THROW_MES(verify_tx_key(tx_keys[0], tx_pub, all_subs), "Tx pub mismatch"); CHECK_AND_ASSERT_THROW_MES(additional_pub_keys.size() == tx_keys.size() - 1, "Invalid additional keys count"); @@ -1242,11 +1472,16 @@ void gen_trezor_base::test_get_tx( void gen_trezor_base::mine_and_test(std::vector<test_event_entry>& events) { + if (!m_mining_enabled) { + MDEBUG("Mining test disabled"); + return; + } + cryptonote::core * core = daemon()->core(); const uint64_t height_before_mining = daemon()->get_height(); const auto miner_address = cryptonote::get_account_address_as_str(FAKECHAIN, false, get_address(m_miner_account)); - daemon()->mine_blocks(1, miner_address); + daemon()->mine_blocks(1, miner_address, std::chrono::seconds(m_mining_timeout)); const uint64_t cur_height = daemon()->get_height(); CHECK_AND_ASSERT_THROW_MES(height_before_mining < cur_height, "Mining fail"); @@ -1285,6 +1520,15 @@ void gen_trezor_base::set_hard_fork(uint8_t hf) } } +boost::optional<epee::wipeable_string> gen_trezor_base::on_pin_request() { + return boost::make_optional(epee::wipeable_string((m_trezor_pin).data(), (m_trezor_pin).size())); +} + +boost::optional<epee::wipeable_string> gen_trezor_base::on_passphrase_request(bool &on_device) { + on_device = false; + return boost::make_optional(epee::wipeable_string((m_trezor_passphrase).data(), (m_trezor_passphrase).size())); +} + #define TREZOR_TEST_PREFIX() \ test_generator generator(m_generator); \ test_setup(events); \ @@ -1294,7 +1538,7 @@ void gen_trezor_base::set_hard_fork(uint8_t hf) #define TREZOR_TEST_SUFFIX() \ auto _dsts = t_builder->build(); \ auto _dsts_info = t_builder->dest_info(); \ - test_trezor_tx(events, _dsts, _dsts_info, generator, vct_wallets(m_wl_alice.get(), m_wl_bob.get(), m_wl_eve.get())); \ + test_trezor_tx(events, _dsts, _dsts_info, generator, TREZOR_ALL_WALLET_VCT); \ return true #define TREZOR_SKIP_IF_VERSION_LEQ(x) if (m_trezor->get_version() <= x) { MDEBUG("Test skipped"); return true; } @@ -1307,7 +1551,7 @@ tsx_builder * tsx_builder::sources(std::vector<cryptonote::tx_source_entry> & so return this; } -tsx_builder * tsx_builder::compute_sources(boost::optional<size_t> num_utxo, boost::optional<uint64_t> min_amount, ssize_t offset, int step, boost::optional<fnc_accept_tx_source_t> fnc_accept) +tsx_builder * tsx_builder::compute_sources(boost::optional<size_t> num_utxo, uint64_t extra_amount, ssize_t offset, int step, boost::optional<fnc_accept_tx_source_t> fnc_accept) { CHECK_AND_ASSERT_THROW_MES(m_tester, "m_tester wallet empty"); CHECK_AND_ASSERT_THROW_MES(m_from, "m_from wallet empty"); @@ -1326,13 +1570,21 @@ tsx_builder * tsx_builder::compute_sources(boost::optional<size_t> num_utxo, boo return true; }; - fnc_accept_to_use = fnc_acc; - bool res = wallet_tools::fill_tx_sources(m_from, m_sources, m_mixin, num_utxo, min_amount, m_tester->m_bt, m_selected_transfers, m_cur_height, offset, step, fnc_accept_to_use); + const uint64_t needed_amount = m_fee + std::accumulate(m_destinations_orig.begin(), m_destinations_orig.end(), 0, [](int a, const cryptonote::tx_destination_entry& b){return a + b.amount;}); + const uint64_t real_amount = needed_amount + extra_amount; + MDEBUG("Computing sources for tx, |utxos| = " << (num_utxo ? std::to_string(num_utxo.get()) : "*") + << ", sum: " << std::setfill(' ') << std::setw(12) << ((double)real_amount / COIN) + << ", need: " << std::setfill(' ') << std::setw(12) << ((double)needed_amount / COIN) + << ", extra: " << std::setfill(' ') << std::setw(12) << ((double)extra_amount / COIN) + << ", fee: " << std::setfill(' ') << std::setw(12) << ((double)m_fee / COIN)); + + bool res = wallet_tools::fill_tx_sources(m_from, m_sources, m_mixin, num_utxo, real_amount, m_tester->m_bt, m_selected_transfers, m_cur_height, offset, step, fnc_acc, fnc_trezor_default_acceptor_tx_in); + CHECK_AND_ASSERT_THROW_MES(!num_utxo || num_utxo == m_sources.size(), "!!Test error, Number of computed inputs " << m_sources.size() << " does not match desired number " << num_utxo.get()); CHECK_AND_ASSERT_THROW_MES(res, "Tx source fill error"); return this; } -tsx_builder * tsx_builder::compute_sources_to_sub(boost::optional<size_t> num_utxo, boost::optional<uint64_t> min_amount, ssize_t offset, int step, boost::optional<fnc_accept_tx_source_t> fnc_accept) +tsx_builder * tsx_builder::compute_sources_to_sub(boost::optional<size_t> num_utxo, uint64_t extra_amount, ssize_t offset, int step, boost::optional<fnc_accept_tx_source_t> fnc_accept) { fnc_accept_tx_source_t fnc = [&fnc_accept] (const tx_source_info_crate_t &info, bool &abort) -> bool { if (info.td->m_subaddr_index.minor == 0){ @@ -1344,10 +1596,10 @@ tsx_builder * tsx_builder::compute_sources_to_sub(boost::optional<size_t> num_ut return true; }; - return compute_sources(num_utxo, min_amount, offset, step, fnc); + return compute_sources(num_utxo, extra_amount, offset, step, fnc); } -tsx_builder * tsx_builder::compute_sources_to_sub_acc(boost::optional<size_t> num_utxo, boost::optional<uint64_t> min_amount, ssize_t offset, int step, boost::optional<fnc_accept_tx_source_t> fnc_accept) +tsx_builder * tsx_builder::compute_sources_to_sub_acc(boost::optional<size_t> num_utxo, uint64_t extra_amount, ssize_t offset, int step, boost::optional<fnc_accept_tx_source_t> fnc_accept) { fnc_accept_tx_source_t fnc = [&fnc_accept] (const tx_source_info_crate_t &info, bool &abort) -> bool { if (info.td->m_subaddr_index.minor == 0 || info.src->real_out_additional_tx_keys.size() == 0){ @@ -1359,7 +1611,7 @@ tsx_builder * tsx_builder::compute_sources_to_sub_acc(boost::optional<size_t> nu return true; }; - return compute_sources(num_utxo, min_amount, offset, step, fnc); + return compute_sources(num_utxo, extra_amount, offset, step, fnc); } tsx_builder * tsx_builder::destinations(std::vector<cryptonote::tx_destination_entry> &dsts) @@ -1513,11 +1765,11 @@ device_trezor_test::device_trezor_test(): m_tx_sign_ctr(0), m_compute_key_image_ void device_trezor_test::clear_test_counters(){ m_tx_sign_ctr = 0; m_compute_key_image_ctr = 0; + m_live_synced_ki.clear(); } -void device_trezor_test::setup_for_tests(const std::string & trezor_path, const std::string & seed, cryptonote::network_type network_type){ +void device_trezor_test::setup_for_tests(const std::string & trezor_path, const std::string & seed, cryptonote::network_type network_type, bool use_passphrase, const std::string & pin){ this->clear_test_counters(); - this->set_callback(nullptr); this->set_debug(true); // debugging commands on Trezor (auto-confirm transactions) CHECK_AND_ASSERT_THROW_MES(this->set_name(trezor_path), "Could not set device name " << trezor_path); @@ -1527,7 +1779,7 @@ void device_trezor_test::setup_for_tests(const std::string & trezor_path, const CHECK_AND_ASSERT_THROW_MES(this->init(), "Could not initialize the device " << trezor_path); CHECK_AND_ASSERT_THROW_MES(this->connect(), "Could not connect to the device " << trezor_path); this->wipe_device(); - this->load_device(seed); + this->load_device(seed, pin, use_passphrase); this->release(); this->disconnect(); } @@ -1540,6 +1792,7 @@ bool device_trezor_test::compute_key_image(const ::cryptonote::account_keys &ack bool res = device_trezor::compute_key_image(ack, out_key, recv_derivation, real_output_index, received_index, in_ephemeral, ki); m_compute_key_image_ctr += res; + m_live_synced_ki.insert(ki); return res; } @@ -1580,8 +1833,13 @@ bool gen_trezor_ki_sync::generate(std::vector<test_event_entry>& events) CHECK_AND_ASSERT_THROW_MES(dev_cold, "Device does not implement test interface"); if (!m_live_refresh_enabled) CHECK_AND_ASSERT_THROW_MES(dev_trezor_test->m_compute_key_image_ctr == 0, "Live refresh should not happen: " << dev_trezor_test->m_compute_key_image_ctr); - else - CHECK_AND_ASSERT_THROW_MES(dev_trezor_test->m_compute_key_image_ctr == ski.size(), "Live refresh counts invalid"); + else { + CHECK_AND_ASSERT_THROW_MES(dev_trezor_test->m_compute_key_image_ctr >= ski.size(), "Live refresh counts invalid"); + for(size_t i = 0; i < transfers.size(); ++i) + { + CHECK_AND_ASSERT_THROW_MES(dev_trezor_test->m_live_synced_ki.count(ski[i].first) > 0, "Key image not foind in live sync: " << ski[i].first); + } + } return true; } @@ -1667,13 +1925,16 @@ bool gen_trezor_live_refresh::generate(std::vector<test_event_entry>& events) bool gen_trezor_1utxo::generate(std::vector<test_event_entry>& events) { TREZOR_TEST_PREFIX(); + m_no_change_in_tested_tx = true; + const uint64_t amount = MK_TCOINS(1); + const fnc_accept_tx_source_t acceptor = [] (const tx_source_info_crate_t &info, bool &abort) -> bool { return info.td->amount() == amount && info.td->m_subaddr_index.minor == 0; }; t_builder->cur_height(num_blocks(events) - 1) ->mixin(num_mixin()) ->fee(TREZOR_TEST_FEE) ->from(m_wl_alice.get(), 0) - ->compute_sources(boost::none, MK_COINS(1), -1, -1) - ->add_destination(m_eve_account, false, 1000) + ->add_destination(m_eve_account, false, MK_TCOINS(1) - TREZOR_TEST_FEE) // no change ->rct_config(m_rct_config) + ->compute_sources(boost::none, 0, -1, -1, acceptor) ->build_tx(); TREZOR_TEST_SUFFIX(); @@ -1682,15 +1943,14 @@ bool gen_trezor_1utxo::generate(std::vector<test_event_entry>& events) bool gen_trezor_1utxo_paymentid_short::generate(std::vector<test_event_entry>& events) { TREZOR_TEST_PREFIX(); - TREZOR_SKIP_IF_VERSION_LEQ(hw::trezor::pack_version(2, 0, 9)); t_builder->cur_height(num_blocks(events) - 1) ->mixin(num_mixin()) ->fee(TREZOR_TEST_FEE) ->from(m_wl_alice.get(), 0) - ->compute_sources(boost::none, MK_COINS(1), -1, -1) ->add_destination(m_eve_account, false, 1000) ->payment_id(TREZOR_TEST_PAYMENT_ID) ->rct_config(m_rct_config) + ->compute_sources(1, MK_TCOINS(0.0001), -1, -1) ->build_tx(); TREZOR_TEST_SUFFIX(); @@ -1699,16 +1959,15 @@ bool gen_trezor_1utxo_paymentid_short::generate(std::vector<test_event_entry>& e bool gen_trezor_1utxo_paymentid_short_integrated::generate(std::vector<test_event_entry>& events) { TREZOR_TEST_PREFIX(); - TREZOR_SKIP_IF_VERSION_LEQ(hw::trezor::pack_version(2, 0, 9)); t_builder->cur_height(num_blocks(events) - 1) ->mixin(num_mixin()) ->fee(TREZOR_TEST_FEE) ->from(m_wl_alice.get(), 0) - ->compute_sources(boost::none, MK_COINS(1), -1, -1) ->add_destination(m_eve_account, false, 1000) ->payment_id(TREZOR_TEST_PAYMENT_ID) ->set_integrated(0) ->rct_config(m_rct_config) + ->compute_sources(1, MK_TCOINS(0.0001), -1, -1) ->build_tx(); TREZOR_TEST_SUFFIX(); @@ -1721,9 +1980,9 @@ bool gen_trezor_4utxo::generate(std::vector<test_event_entry>& events) ->mixin(num_mixin()) ->fee(TREZOR_TEST_FEE) ->from(m_wl_alice.get(), 0) - ->compute_sources(4, MK_COINS(1), -1, -1) ->add_destination(m_eve_account, false, 1000) ->rct_config(m_rct_config) + ->compute_sources(4, MK_TCOINS(0.0001), -1, -1) ->build_tx(); TREZOR_TEST_SUFFIX(); @@ -1736,9 +1995,9 @@ bool gen_trezor_4utxo_acc1::generate(std::vector<test_event_entry>& events) ->mixin(num_mixin()) ->fee(TREZOR_TEST_FEE) ->from(m_wl_alice.get(), 1) - ->compute_sources(4, MK_COINS(1), -1, -1) ->add_destination(m_wl_eve->get_subaddress({0, 1}), true, 1000) ->rct_config(m_rct_config) + ->compute_sources(4, MK_TCOINS(0.0001), -1, -1) ->build_tx(); TREZOR_TEST_SUFFIX(); @@ -1751,9 +2010,9 @@ bool gen_trezor_4utxo_to_sub::generate(std::vector<test_event_entry>& events) ->mixin(num_mixin()) ->fee(TREZOR_TEST_FEE) ->from(m_wl_alice.get(), 0) - ->compute_sources(4, MK_COINS(1), -1, -1) ->add_destination(m_wl_eve->get_subaddress({0, 1}), true, 1000) ->rct_config(m_rct_config) + ->compute_sources(4, MK_TCOINS(0.0001), -1, -1) ->build_tx(); TREZOR_TEST_SUFFIX(); @@ -1766,10 +2025,10 @@ bool gen_trezor_4utxo_to_2sub::generate(std::vector<test_event_entry>& events) ->mixin(num_mixin()) ->fee(TREZOR_TEST_FEE) ->from(m_wl_alice.get(), 0) - ->compute_sources(4, MK_COINS(1), -1, -1) ->add_destination(m_wl_eve->get_subaddress({0, 1}), true, 1000) ->add_destination(m_wl_eve->get_subaddress({1, 3}), true, 1000) ->rct_config(m_rct_config) + ->compute_sources(4, MK_TCOINS(0.0001), -1, -1) ->build_tx(); TREZOR_TEST_SUFFIX(); @@ -1782,10 +2041,10 @@ bool gen_trezor_4utxo_to_1norm_2sub::generate(std::vector<test_event_entry>& eve ->mixin(num_mixin()) ->fee(TREZOR_TEST_FEE) ->from(m_wl_alice.get(), 0) - ->compute_sources(4, MK_COINS(1), -1, -1) ->add_destination(m_wl_eve->get_subaddress({1, 1}), true, 1000) ->add_destination(m_wl_eve->get_subaddress({2, 1}), true, 1000) ->add_destination(m_wl_eve.get(), false, 1000) + ->compute_sources(4, MK_TCOINS(0.0001), -1, -1) ->rct_config(m_rct_config) ->build_tx(); @@ -1799,11 +2058,11 @@ bool gen_trezor_2utxo_sub_acc_to_1norm_2sub::generate(std::vector<test_event_ent ->mixin(num_mixin()) ->fee(TREZOR_TEST_FEE) ->from(m_wl_alice.get(), 0) - ->compute_sources_to_sub_acc(2, MK_COINS(1) >> 2, -1, -1) ->add_destination(m_wl_eve->get_subaddress({1, 1}), true, 1000) ->add_destination(m_wl_eve->get_subaddress({2, 1}), true, 1000) ->add_destination(m_wl_eve.get(), false, 1000) ->rct_config(m_rct_config) + ->compute_sources_to_sub_acc(2, MK_TCOINS(0.0001), -1, -1) ->build_tx(); TREZOR_TEST_SUFFIX(); @@ -1816,7 +2075,6 @@ bool gen_trezor_4utxo_to_7outs::generate(std::vector<test_event_entry>& events) ->mixin(num_mixin()) ->fee(TREZOR_TEST_FEE) ->from(m_wl_alice.get(), 0) - ->compute_sources(4, MK_COINS(1), -1, -1) ->add_destination(m_wl_eve->get_subaddress({1, 1}), true, 1000) ->add_destination(m_wl_eve->get_subaddress({2, 1}), true, 1000) ->add_destination(m_wl_eve->get_subaddress({0, 1}), true, 1000) @@ -1825,6 +2083,7 @@ bool gen_trezor_4utxo_to_7outs::generate(std::vector<test_event_entry>& events) ->add_destination(m_wl_eve->get_subaddress({0, 4}), true, 1000) ->add_destination(m_wl_eve.get(), false, 1000) ->rct_config(m_rct_config) + ->compute_sources(4, MK_TCOINS(0.0001), -1, -1) ->build_tx(); TREZOR_TEST_SUFFIX(); @@ -1837,7 +2096,6 @@ bool gen_trezor_4utxo_to_15outs::generate(std::vector<test_event_entry>& events) ->mixin(num_mixin()) ->fee(TREZOR_TEST_FEE) ->from(m_wl_alice.get(), 0) - ->compute_sources(4, MK_COINS(1), -1, -1) ->add_destination(m_wl_eve->get_subaddress({1, 1}), true, 1000) ->add_destination(m_wl_eve->get_subaddress({2, 1}), true, 1000) ->add_destination(m_wl_eve->get_subaddress({0, 1}), true, 1000) @@ -1853,6 +2111,22 @@ bool gen_trezor_4utxo_to_15outs::generate(std::vector<test_event_entry>& events) ->add_destination(m_wl_eve->get_subaddress({0, 4}), true, 1000) ->add_destination(m_wl_eve.get(), false, 1000) ->rct_config(m_rct_config) + ->compute_sources(4, MK_TCOINS(0.0001), -1, -1) + ->build_tx(); + + TREZOR_TEST_SUFFIX(); +} + +bool gen_trezor_16utxo_to_sub::generate(std::vector<test_event_entry>& events) +{ + TREZOR_TEST_PREFIX(); + t_builder->cur_height(num_blocks(events) - 1) + ->mixin(num_mixin()) + ->fee(TREZOR_TEST_FEE) + ->from(m_wl_alice.get(), 0) + ->add_destination(m_wl_eve->get_subaddress({0, 1}), true, 1000) + ->rct_config(m_rct_config) + ->compute_sources(16, MK_TCOINS(0.0001), -1, -1) ->build_tx(); TREZOR_TEST_SUFFIX(); @@ -1865,9 +2139,9 @@ bool gen_trezor_many_utxo::generate(std::vector<test_event_entry>& events) ->mixin(num_mixin()) ->fee(TREZOR_TEST_FEE) ->from(m_wl_alice.get(), 0) - ->compute_sources(110, MK_COINS(1), -1, -1) ->add_destination(m_eve_account, false, 1000) ->rct_config(m_rct_config) + ->compute_sources(110, MK_TCOINS(0.0001), -1, -1) ->build_tx(); TREZOR_TEST_SUFFIX(); @@ -1880,7 +2154,6 @@ bool gen_trezor_many_utxo_many_txo::generate(std::vector<test_event_entry>& even ->mixin(num_mixin()) ->fee(TREZOR_TEST_FEE) ->from(m_wl_alice.get(), 0) - ->compute_sources(40, MK_COINS(1), -1, -1) ->add_destination(m_eve_account, false, 1000) ->add_destination(m_wl_eve->get_subaddress({1, 1}), true, 1000) ->add_destination(m_wl_eve->get_subaddress({2, 1}), true, 1000) @@ -1897,18 +2170,67 @@ bool gen_trezor_many_utxo_many_txo::generate(std::vector<test_event_entry>& even ->add_destination(m_wl_eve->get_subaddress({2, 4}), true, 1000) ->add_destination(m_wl_eve->get_subaddress({3, 4}), true, 1000) ->rct_config(m_rct_config) + ->compute_sources(40, MK_TCOINS(0.0001), -1, -1) ->build_tx(); TREZOR_TEST_SUFFIX(); } -void wallet_api_tests::init() +bool gen_trezor_no_passphrase::generate(std::vector<test_event_entry>& events) +{ + m_trezor_use_passphrase = false; + TREZOR_TEST_PREFIX(); + t_builder->cur_height(num_blocks(events) - 1) + ->mixin(num_mixin()) + ->fee(TREZOR_TEST_FEE) + ->from(m_wl_alice.get(), 0) + ->add_destination(m_eve_account, false, 1000) + ->rct_config(m_rct_config) + ->compute_sources(boost::none, MK_TCOINS(0.0001), -1, -1) + ->build_tx(); + + TREZOR_TEST_SUFFIX(); + + return true; +} + +bool gen_trezor_wallet_passphrase::generate(std::vector<test_event_entry>& events) { m_wallet_dir = boost::filesystem::unique_path(); boost::filesystem::create_directories(m_wallet_dir); + + m_trezor_use_alice2 = true; + m_trezor_use_passphrase = true; + m_trezor_passphrase = m_alice2_passphrase; + test_setup(events); + + const auto wallet_path = (m_wallet_dir / "alice2").string(); + const epee::wipeable_string& password = epee::wipeable_string("test-pass"); + m_wl_alice2->store_to(wallet_path, password); + + // Positive load + MDEBUG("Loading wallet with correct passphrase"); + m_wl_alice2->callback(this); + m_wl_alice2->load(wallet_path, password); + + // Negative load + m_trezor_passphrase = "bad-passphrase-0112312312312"; + setup_trezor(m_trezor_use_passphrase, m_trezor_pin); + update_client_settings(); + m_wl_alice2->callback(this); + + MDEBUG("Loading wallet with different passphrase - should fail"); + try { + m_wl_alice2->load(wallet_path, password); + CHECK_AND_ASSERT_THROW_MES(false, "Wallet load should not pass with different passphrase"); + } catch (const tools::error::wallet_internal_error & ex) { + CHECK_AND_ASSERT_THROW_MES(ex.to_string().find("does not match wallet") != std::string::npos, "Unexpected wallet exception"); + } + + return true; } -wallet_api_tests::~wallet_api_tests() +gen_trezor_wallet_passphrase::~gen_trezor_wallet_passphrase() { try { @@ -1917,10 +2239,72 @@ wallet_api_tests::~wallet_api_tests() boost::filesystem::remove_all(m_wallet_dir); } } - catch(...) + catch(...) {} +} + +boost::optional<epee::wipeable_string> gen_trezor_wallet_passphrase::on_device_pin_request() { + return on_pin_request(); +} + +boost::optional<epee::wipeable_string> gen_trezor_wallet_passphrase::on_device_passphrase_request(bool &on_device) { + return on_passphrase_request(on_device); +} + +bool gen_trezor_passphrase::generate(std::vector<test_event_entry>& events) +{ + m_trezor_use_passphrase = true; + m_trezor_use_alice2 = true; + m_trezor_passphrase = m_alice2_passphrase; + TREZOR_TEST_PREFIX(); + t_builder->cur_height(num_blocks(events) - 1) + ->mixin(num_mixin()) + ->fee(TREZOR_TEST_FEE) + ->from(m_wl_alice2.get(), 0) + ->add_destination(m_eve_account, false, 1000) + ->rct_config(m_rct_config) + ->compute_sources(boost::none, MK_TCOINS(0.0001), -1, -1) + ->build_tx(); + + auto _dsts = t_builder->build(); + auto _dsts_info = t_builder->dest_info(); + test_trezor_tx(events, _dsts, _dsts_info, generator, TREZOR_ALL_WALLET_VCT, false, 1); + return true; +} + +bool gen_trezor_pin::generate(std::vector<test_event_entry>& events) +{ + m_trezor_pin = "0000"; + TREZOR_TEST_PREFIX(); + t_builder->cur_height(num_blocks(events) - 1) + ->mixin(num_mixin()) + ->fee(TREZOR_TEST_FEE) + ->from(m_wl_alice.get(), 0) + ->add_destination(m_eve_account, false, 1000) + ->rct_config(m_rct_config) + ->compute_sources(boost::none, MK_TCOINS(0.0001), -1, -1) + ->build_tx(); + + TREZOR_TEST_SUFFIX(); + + return true; +} + +void wallet_api_tests::init() +{ + m_wallet_dir = boost::filesystem::unique_path(); + boost::filesystem::create_directories(m_wallet_dir); +} + +wallet_api_tests::~wallet_api_tests() +{ + try { - MERROR("Could not remove wallet directory"); + if (!m_wallet_dir.empty() && boost::filesystem::exists(m_wallet_dir)) + { + boost::filesystem::remove_all(m_wallet_dir); + } } + catch(...) {} } bool wallet_api_tests::generate(std::vector<test_event_entry>& events) @@ -1931,23 +2315,30 @@ bool wallet_api_tests::generate(std::vector<test_event_entry>& events) const auto api_net_type = m_network_type == TESTNET ? Monero::TESTNET : Monero::MAINNET; Monero::WalletManager *wmgr = Monero::WalletManagerFactory::getWalletManager(); - std::unique_ptr<Monero::Wallet> w{wmgr->createWalletFromDevice(wallet_path, "", api_net_type, m_trezor_path, 1)}; + std::unique_ptr<Monero::Wallet> w{wmgr->createWalletFromDevice(wallet_path, "", api_net_type, m_trezor_path, 1, "", 1, this)}; CHECK_AND_ASSERT_THROW_MES(w->init(daemon()->rpc_addr(), 0), "Wallet init fail"); auto walletImpl = dynamic_cast<Monero::WalletImpl *>(w.get()); CHECK_AND_ASSERT_THROW_MES(walletImpl, "Dynamic wallet cast failed"); WalletApiAccessorTest::allow_mismatched_daemon_version(walletImpl, true); + walletImpl->setTrustedDaemon(true); CHECK_AND_ASSERT_THROW_MES(w->refresh(), "Refresh fail"); + + uint64_t spent, unspent; + walletImpl->coldKeyImageSync(spent, unspent); + CHECK_AND_ASSERT_THROW_MES(walletImpl->rescanSpent(), "Rescan spent fail"); + uint64_t balance = w->balance(0); MDEBUG("Balance: " << balance); CHECK_AND_ASSERT_THROW_MES(w->status() == Monero::PendingTransaction::Status_Ok, "Status nok, " << w->errorString()); + const uint64_t tx_amount = MK_TCOINS(0.5); auto addr = get_address(m_eve_account); auto recepient_address = cryptonote::get_account_address_as_str(m_network_type, false, addr); Monero::PendingTransaction * transaction = w->createTransaction(recepient_address, "", - MK_COINS(10), + tx_amount, num_mixin(), Monero::PendingTransaction::Priority_Medium, 0, @@ -1955,14 +2346,23 @@ bool wallet_api_tests::generate(std::vector<test_event_entry>& events) CHECK_AND_ASSERT_THROW_MES(transaction->status() == Monero::PendingTransaction::Status_Ok, "Status nok: " << transaction->status() << ", msg: " << transaction->errorString()); w->refresh(); - CHECK_AND_ASSERT_THROW_MES(w->balance(0) == balance, "Err"); - CHECK_AND_ASSERT_THROW_MES(transaction->amount() == MK_COINS(10), "Err"); - CHECK_AND_ASSERT_THROW_MES(transaction->commit(), "Err"); - CHECK_AND_ASSERT_THROW_MES(w->balance(0) != balance, "Err"); - CHECK_AND_ASSERT_THROW_MES(wmgr->closeWallet(w.get()), "Err"); + CHECK_AND_ASSERT_THROW_MES(w->balance(0) == balance, "Err balance"); + CHECK_AND_ASSERT_THROW_MES(transaction->amount() == tx_amount, "Err amount"); + CHECK_AND_ASSERT_THROW_MES(transaction->commit(), "Err commit"); + CHECK_AND_ASSERT_THROW_MES(w->balance(0) != balance, "Err balance2"); + CHECK_AND_ASSERT_THROW_MES(wmgr->closeWallet(w.get()), "Err close"); (void)w.release(); mine_and_test(events); return true; } +Monero::optional<std::string> wallet_api_tests::onDevicePinRequest() { + return Monero::optional<std::string>(m_trezor_pin); +} + +Monero::optional<std::string> wallet_api_tests::onDevicePassphraseRequest(bool &on_device) { + on_device = false; + return Monero::optional<std::string>(m_trezor_passphrase); +} + diff --git a/tests/trezor/trezor_tests.h b/tests/trezor/trezor_tests.h index 0f9ee30ca..16d7641db 100644 --- a/tests/trezor/trezor_tests.h +++ b/tests/trezor/trezor_tests.h @@ -36,16 +36,24 @@ #include "../core_tests/chaingen.h" #include "../core_tests/wallet_tools.h" -#define TREZOR_TEST_FEE 90000000000 #define TREZOR_TEST_CLSAG_MIXIN 11 #define TREZOR_TEST_HF15_MIXIN 16 #define TREZOR_TEST_MIXIN TREZOR_TEST_CLSAG_MIXIN +#define TREZOR_TEST_MIN_HF_DEFAULT HF_VERSION_BULLETPROOF_PLUS +#define TREZOR_TEST_MAX_HF_DEFAULT HF_VERSION_BULLETPROOF_PLUS +#define TREZOR_TEST_KI_SYNC_DEFAULT 1 +#define TREZOR_TEST_MINING_ENABLED_DEFAULT false +#define TREZOR_TEST_MINING_TIMEOUT_DEFAULT 360 + +#define MK_TCOINS(amount) ((unsigned long long) ((amount) * (COIN))) +#define TREZOR_TEST_FEE_FRAC 0.1 +#define TREZOR_TEST_FEE MK_TCOINS(TREZOR_TEST_FEE_FRAC) /************************************************************************/ /* */ /************************************************************************/ class tsx_builder; -class gen_trezor_base : public test_chain_unit_base +class gen_trezor_base : public test_chain_unit_base, public hw::i_device_callback { public: friend class tsx_builder; @@ -54,7 +62,7 @@ public: gen_trezor_base(const gen_trezor_base &other); virtual ~gen_trezor_base() {}; - virtual void setup_args(const std::string & trezor_path, bool heavy_tests=false); + virtual void setup_args(const std::string & trezor_path, bool heavy_tests, bool mining_enabled, long mining_timeout); virtual bool generate(std::vector<test_event_entry>& events); virtual void load(std::vector<test_event_entry>& events); // load events, init test obj virtual void fix_hf(std::vector<test_event_entry>& events); @@ -76,42 +84,58 @@ public: std::vector<cryptonote::address_parse_info>& dsts_info, test_generator &generator, std::vector<tools::wallet2*> wallets, - bool is_sweep=false); + bool is_sweep=false, + size_t sender_idx=0); virtual void test_get_tx( std::vector<test_event_entry>& events, std::vector<tools::wallet2*> wallets, + const cryptonote::account_base * account, const std::vector<tools::wallet2::pending_tx> &ptxs, const std::vector<std::string> &aux_tx_info); virtual void mine_and_test(std::vector<test_event_entry>& events); virtual void rewind_blocks(std::vector<test_event_entry>& events, size_t rewind_n, uint8_t hf); + virtual cryptonote::block rewind_blocks_with_decoys(std::vector<test_event_entry>& events, cryptonote::block & head, size_t rewind_n, uint8_t hf, size_t num_decoys_per_block); virtual void set_hard_fork(uint8_t hf); crypto::hash head_hash() const { return get_block_hash(m_head); } cryptonote::block head_block() const { return m_head; } bool heavy_tests() const { return m_heavy_tests; } + bool heavy_test_set() const { return m_gen_heavy_test_set; } + void heavy_test_set(bool set) { m_gen_heavy_test_set = set; } void rct_config(rct::RCTConfig rct_config) { m_rct_config = rct_config; } - uint8_t cur_hf() const { return m_hard_forks.size() > 0 ? m_hard_forks.back().first : 0; } + uint8_t cur_hf() const { return !m_hard_forks.empty() ? m_hard_forks.back().first : 0; } size_t num_mixin() const { return m_top_hard_fork >= HF_VERSION_BULLETPROOF_PLUS ? TREZOR_TEST_HF15_MIXIN : TREZOR_TEST_CLSAG_MIXIN; } cryptonote::network_type nettype() const { return m_network_type; } std::shared_ptr<mock_daemon> daemon() const { return m_daemon; } void daemon(std::shared_ptr<mock_daemon> daemon){ m_daemon = std::move(daemon); } + boost::optional<epee::wipeable_string> on_pin_request() override; + boost::optional<epee::wipeable_string> on_passphrase_request(bool &on_device) override; + // Static configuration static const uint64_t m_ts_start; static const uint64_t m_wallet_ts; static const std::string m_device_name; + static const std::string m_miner_master_seed_str; static const std::string m_master_seed_str; static const std::string m_device_seed; static const std::string m_alice_spend_private; static const std::string m_alice_view_private; + static const std::string m_alice2_passphrase; + static const std::string m_alice2_master_seed; + static const std::string m_bob_master_seed; + static const std::string m_eve_master_seed; protected: - virtual void setup_trezor(); - virtual void init_fields(); + virtual void setup_trezor(bool use_passphrase, const std::string & pin); + virtual void init_accounts(); + virtual void init_wallets(); + virtual void init_trezor_account(); + virtual void log_wallets_desc(); virtual void update_client_settings(); virtual bool verify_tx_key(const ::crypto::secret_key & tx_priv, const ::crypto::public_key & tx_pub, const subaddresses_t & subs); @@ -126,17 +150,27 @@ protected: std::vector<test_event_entry> m_events; std::string m_trezor_path; + std::string m_trezor_pin; + std::string m_trezor_passphrase; + bool m_trezor_use_passphrase = true; + bool m_trezor_use_alice2 = false; bool m_heavy_tests; + bool m_gen_heavy_test_set; bool m_test_get_tx_key; rct::RCTConfig m_rct_config; bool m_live_refresh_enabled; + bool m_mining_enabled = TREZOR_TEST_MINING_ENABLED_DEFAULT; + long m_mining_timeout = TREZOR_TEST_MINING_TIMEOUT_DEFAULT; + bool m_no_change_in_tested_tx = false; cryptonote::account_base m_miner_account; cryptonote::account_base m_bob_account; cryptonote::account_base m_alice_account; cryptonote::account_base m_eve_account; + cryptonote::account_base m_alice2_account; hw::trezor::device_trezor * m_trezor; std::unique_ptr<tools::wallet2> m_wl_alice; + std::unique_ptr<tools::wallet2> m_wl_alice2; std::unique_ptr<tools::wallet2> m_wl_bob; std::unique_ptr<tools::wallet2> m_wl_eve; @@ -165,9 +199,9 @@ public: tsx_builder * payment_id(const std::string & payment_id) { m_payment_id = payment_id; return this; } tsx_builder * from(tools::wallet2 *from, uint32_t account=0) { m_from = from; m_account=account; return this; } tsx_builder * sources(std::vector<cryptonote::tx_source_entry> & sources, std::vector<size_t> & selected_transfers); - tsx_builder * compute_sources(boost::optional<size_t> num_utxo=boost::none, boost::optional<uint64_t> min_amount=boost::none, ssize_t offset=-1, int step=1, boost::optional<fnc_accept_tx_source_t> fnc_accept=boost::none); - tsx_builder * compute_sources_to_sub(boost::optional<size_t> num_utxo=boost::none, boost::optional<uint64_t> min_amount=boost::none, ssize_t offset=-1, int step=1, boost::optional<fnc_accept_tx_source_t> fnc_accept=boost::none); - tsx_builder * compute_sources_to_sub_acc(boost::optional<size_t> num_utxo=boost::none, boost::optional<uint64_t> min_amount=boost::none, ssize_t offset=-1, int step=1, boost::optional<fnc_accept_tx_source_t> fnc_accept=boost::none); + tsx_builder * compute_sources(boost::optional<size_t> num_utxo=boost::none, uint64_t extra_amount=0, ssize_t offset=-1, int step=1, boost::optional<fnc_accept_tx_source_t> fnc_accept=boost::none); + tsx_builder * compute_sources_to_sub(boost::optional<size_t> num_utxo=boost::none, uint64_t extra_amount=0, ssize_t offset=-1, int step=1, boost::optional<fnc_accept_tx_source_t> fnc_accept=boost::none); + tsx_builder * compute_sources_to_sub_acc(boost::optional<size_t> num_utxo=boost::none, uint64_t extra_amount=0, ssize_t offset=-1, int step=1, boost::optional<fnc_accept_tx_source_t> fnc_accept=boost::none); tsx_builder * destinations(std::vector<cryptonote::tx_destination_entry> &dsts); tsx_builder * add_destination(const cryptonote::tx_destination_entry &dst); @@ -208,11 +242,12 @@ class device_trezor_test : public hw::trezor::device_trezor { public: size_t m_tx_sign_ctr; size_t m_compute_key_image_ctr; + std::unordered_set<crypto::key_image> m_live_synced_ki; device_trezor_test(); void clear_test_counters(); - void setup_for_tests(const std::string & trezor_path, const std::string & seed, cryptonote::network_type network_type); + void setup_for_tests(const std::string & trezor_path, const std::string & seed, cryptonote::network_type network_type, bool use_passphrase=false, const std::string & pin = ""); bool compute_key_image(const ::cryptonote::account_keys &ack, const ::crypto::public_key &out_key, const ::crypto::key_derivation &recv_derivation, size_t real_output_index, @@ -315,6 +350,12 @@ public: bool generate(std::vector<test_event_entry>& events) override; }; +class gen_trezor_16utxo_to_sub : public gen_trezor_base +{ +public: + bool generate(std::vector<test_event_entry>& events) override; +}; + class gen_trezor_many_utxo : public gen_trezor_base { public: @@ -327,14 +368,52 @@ public: bool generate(std::vector<test_event_entry>& events) override; }; +class gen_trezor_no_passphrase : public gen_trezor_base +{ +public: + bool generate(std::vector<test_event_entry>& events) override; +}; + +class gen_trezor_wallet_passphrase : public gen_trezor_base, public tools::i_wallet2_callback +{ +public: + ~gen_trezor_wallet_passphrase() override; + bool generate(std::vector<test_event_entry>& events) override; + boost::optional<epee::wipeable_string> on_device_pin_request() override; + boost::optional<epee::wipeable_string> on_device_passphrase_request(bool &on_device) override; +protected: + boost::filesystem::path m_wallet_dir; +}; + +class gen_trezor_passphrase : public gen_trezor_base +{ +public: + bool generate(std::vector<test_event_entry>& events) override; +}; + +class gen_trezor_pin : public gen_trezor_base +{ +public: + bool generate(std::vector<test_event_entry>& events) override; +}; + // Wallet::API tests -class wallet_api_tests : public gen_trezor_base +class wallet_api_tests : public gen_trezor_base, public Monero::WalletListener { public: - virtual ~wallet_api_tests(); + ~wallet_api_tests() override; void init(); bool generate(std::vector<test_event_entry>& events) override; + Monero::optional<std::string> onDevicePinRequest() override; + Monero::optional<std::string> onDevicePassphraseRequest(bool &on_device) override; + void moneySpent(const std::string &txId, uint64_t amount) override {}; + void moneyReceived(const std::string &txId, uint64_t amount) override {}; + void unconfirmedMoneyReceived(const std::string &txId, uint64_t amount) override {}; + void newBlock(uint64_t height) override {}; + void updated() override {}; + void refreshed() override {}; + protected: boost::filesystem::path m_wallet_dir; };
\ No newline at end of file diff --git a/tests/unit_tests/CMakeLists.txt b/tests/unit_tests/CMakeLists.txt index a9f0944c8..212f834d2 100644 --- a/tests/unit_tests/CMakeLists.txt +++ b/tests/unit_tests/CMakeLists.txt @@ -123,6 +123,7 @@ target_link_libraries(unit_tests rpc net serialization + seraphis_crypto wallet p2p version @@ -143,8 +144,6 @@ if (NOT MSVC) COMPILE_FLAGS " -Wno-undef -Wno-sign-compare") endif () -SET_PROPERTY(SOURCE main.cpp PROPERTY COMPILE_FLAGS -DDEFAULT_DATA_DIR="\\"${CMAKE_SOURCE_DIR}/tests/data\\"") - SET_PROPERTY(SOURCE memwipe.cpp PROPERTY COMPILE_FLAGS -Ofast) add_test( diff --git a/tests/unit_tests/crypto.cpp b/tests/unit_tests/crypto.cpp index e41f3955f..88ecb5853 100644 --- a/tests/unit_tests/crypto.cpp +++ b/tests/unit_tests/crypto.cpp @@ -307,17 +307,21 @@ TEST(Crypto, tree_branch) ASSERT_FALSE(crypto::tree_branch((const char(*)[32])inputs, 5, crypto::null_hash.data, (char(*)[32])branch, &depth, &path)); // depth encoding roundtrip - for (uint32_t n_chains = 1; n_chains <= 65; ++n_chains) + for (uint32_t n_chains = 1; n_chains <= 256; ++n_chains) { - for (uint32_t nonce = 0; nonce < 1024; ++nonce) + for (uint32_t nonce = 0xffffffff - 512; nonce != 1025; ++nonce) { - const uint32_t depth = cryptonote::encode_mm_depth(n_chains, nonce); + const uint64_t depth = cryptonote::encode_mm_depth(n_chains, nonce); uint32_t n_chains_2, nonce_2; ASSERT_TRUE(cryptonote::decode_mm_depth(depth, n_chains_2, nonce_2)); ASSERT_EQ(n_chains, n_chains_2); ASSERT_EQ(nonce, nonce_2); } } + + // 257 chains is too much + try { cryptonote::encode_mm_depth(257, 0); ASSERT_TRUE(false); } + catch (...) {} } TEST(Crypto, generator_consistency) diff --git a/tests/unit_tests/epee_serialization.cpp b/tests/unit_tests/epee_serialization.cpp index 035f5e206..ecc26e971 100644 --- a/tests/unit_tests/epee_serialization.cpp +++ b/tests/unit_tests/epee_serialization.cpp @@ -29,8 +29,11 @@ #include <cstdint> #include <gtest/gtest.h> +#include <vector> +#include "serialization/keyvalue_serialization.h" #include "storages/portable_storage.h" +#include "storages/portable_storage_template_helper.h" #include "span.h" TEST(epee_binary, two_keys) @@ -54,3 +57,87 @@ TEST(epee_binary, duplicate_key) epee::serialization::portable_storage storage{}; EXPECT_FALSE(storage.load_from_binary(data)); } + +namespace +{ +struct ObjOfObjs +{ + std::vector<ObjOfObjs> x; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(x) + END_KV_SERIALIZE_MAP() +}; + +struct ObjOfInts +{ + std::list<int> x; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(x) + END_KV_SERIALIZE_MAP() +}; +} + +TEST(epee_binary, any_empty_seq) +{ + // Test that any c++ sequence type (std::vector, std::list, etc) can deserialize without error + // from an *empty* epee binary array of any type. This property is useful for other projects who + // maintain code which serializes epee binary but don't know the type of arrays until runtime. + // Without any elements to actually serialize, they don't know what type code to use for arrays + // and should be able to choose a default typecode. + + static constexpr const std::uint8_t data_empty_bool[] = { + 0x01, 0x11, 0x01, 0x1, 0x01, 0x01, 0x02, 0x1, 0x1, 0x04, 0x01, 'x', 0x8B /*array of bools*/, 0x00 /*length 0*/ + }; + + static constexpr const std::uint8_t data_empty_double[] = { + 0x01, 0x11, 0x01, 0x1, 0x01, 0x01, 0x02, 0x1, 0x1, 0x04, 0x01, 'x', 0x89 /*array of doubles*/, 0x00 /*length 0*/ + }; + + static constexpr const std::uint8_t data_empty_string[] = { + 0x01, 0x11, 0x01, 0x1, 0x01, 0x01, 0x02, 0x1, 0x1, 0x04, 0x01, 'x', 0x8A /*array of strings*/, 0x00 /*length 0*/ + }; + + static constexpr const std::uint8_t data_empty_int64[] = { + 0x01, 0x11, 0x01, 0x1, 0x01, 0x01, 0x02, 0x1, 0x1, 0x04, 0x01, 'x', 0x81 /*array of int64s*/, 0x00 /*length 0*/ + }; + + static constexpr const std::uint8_t data_empty_object[] = { + 0x01, 0x11, 0x01, 0x1, 0x01, 0x01, 0x02, 0x1, 0x1, 0x04, 0x01, 'x', 0x8C /*array of objects*/, 0x00 /*length 0*/ + }; + + ObjOfObjs o; + + EXPECT_TRUE(epee::serialization::load_t_from_binary(o, epee::span<const std::uint8_t>(data_empty_bool))); + EXPECT_EQ(0, o.x.size()); + + EXPECT_TRUE(epee::serialization::load_t_from_binary(o, epee::span<const std::uint8_t>(data_empty_double))); + EXPECT_EQ(0, o.x.size()); + + EXPECT_TRUE(epee::serialization::load_t_from_binary(o, epee::span<const std::uint8_t>(data_empty_string))); + EXPECT_EQ(0, o.x.size()); + + EXPECT_TRUE(epee::serialization::load_t_from_binary(o, epee::span<const std::uint8_t>(data_empty_int64))); + EXPECT_EQ(0, o.x.size()); + + EXPECT_TRUE(epee::serialization::load_t_from_binary(o, epee::span<const std::uint8_t>(data_empty_object))); + EXPECT_EQ(0, o.x.size()); + + ObjOfInts i; + + EXPECT_TRUE(epee::serialization::load_t_from_binary(i, epee::span<const std::uint8_t>(data_empty_bool))); + EXPECT_EQ(0, i.x.size()); + + EXPECT_TRUE(epee::serialization::load_t_from_binary(i, epee::span<const std::uint8_t>(data_empty_double))); + EXPECT_EQ(0, i.x.size()); + + EXPECT_TRUE(epee::serialization::load_t_from_binary(i, epee::span<const std::uint8_t>(data_empty_string))); + EXPECT_EQ(0, i.x.size()); + + EXPECT_TRUE(epee::serialization::load_t_from_binary(i, epee::span<const std::uint8_t>(data_empty_int64))); + EXPECT_EQ(0, i.x.size()); + + EXPECT_TRUE(epee::serialization::load_t_from_binary(i, epee::span<const std::uint8_t>(data_empty_object))); + EXPECT_EQ(0, i.x.size()); +} diff --git a/tests/unit_tests/epee_utils.cpp b/tests/unit_tests/epee_utils.cpp index c776fbcc9..dd3aafaa6 100644 --- a/tests/unit_tests/epee_utils.cpp +++ b/tests/unit_tests/epee_utils.cpp @@ -1852,3 +1852,60 @@ TEST(parsing, unicode) epee::misc_utils::parse::match_string2(si, s.end(), bs); EXPECT_EQ(bs, "あまやかす"); } + +TEST(parsing, strtoul) +{ + long ul; + const char* p; + const char* endp; + + errno = 0; // Some libc's only set errno on failure, some set it to 0 on success + + p = "0"; + endp = nullptr; + ul = std::strtoul(p, const_cast<char**>(&endp), 10); + EXPECT_EQ(0, errno); + EXPECT_EQ(0, ul); + EXPECT_EQ(p + 1, endp); + + p = "000000"; + endp = nullptr; + ul = std::strtoul(p, const_cast<char**>(&endp), 10); + EXPECT_EQ(0, errno); + EXPECT_EQ(0, ul); + EXPECT_EQ(p + 6, endp); + + p = "1"; + endp = nullptr; + ul = std::strtoul(p, const_cast<char**>(&endp), 10); + EXPECT_EQ(0, errno); + EXPECT_EQ(1, ul); + EXPECT_EQ(p + 1, endp); + + p = "0q"; + endp = nullptr; + ul = std::strtoul(p, const_cast<char**>(&endp), 10); + EXPECT_EQ(0, errno); + EXPECT_EQ(0, ul); + EXPECT_EQ(p + 1, endp); + + p = " \t 0"; + endp = nullptr; + ul = std::strtoul(p, const_cast<char**>(&endp), 10); + EXPECT_EQ(0, errno); + EXPECT_EQ(0, ul); + EXPECT_EQ(p + 9, endp); + + p = "q"; + endp = nullptr; + ul = std::strtoul(p, const_cast<char**>(&endp), 10); + EXPECT_EQ(0, errno); + EXPECT_EQ(0, ul); + EXPECT_EQ(p, endp); + + p = "999999999999999999999999999999999999999"; + endp = nullptr; + ul = std::strtoul(p, const_cast<char**>(&endp), 10); + EXPECT_EQ(ERANGE, errno); + EXPECT_EQ(ULLONG_MAX, ul); +} diff --git a/tests/unit_tests/json_serialization.cpp b/tests/unit_tests/json_serialization.cpp index 9fa589139..aa46b68dc 100644 --- a/tests/unit_tests/json_serialization.cpp +++ b/tests/unit_tests/json_serialization.cpp @@ -13,6 +13,7 @@ #include "cryptonote_basic/cryptonote_format_utils.h" #include "cryptonote_core/cryptonote_tx_utils.h" #include "serialization/json_object.h" +#include "rpc/daemon_messages.h" namespace test @@ -240,3 +241,9 @@ TEST(JsonSerialization, BulletproofTransaction) EXPECT_EQ(tx_bytes, tx_copy_bytes); } +TEST(JsonRpcSerialization, HandlerFromJson) +{ + cryptonote::rpc::FullMessage req_full("{\"jsonrpc\":\"2.0\",\"method\":\"get_hashes_fast\",\"params\":[1]}", true); + cryptonote::rpc::GetHashesFast::Request request{}; + EXPECT_THROW(request.fromJson(req_full.getMessage()), cryptonote::json::WRONG_TYPE); +} diff --git a/tests/unit_tests/lmdb.cpp b/tests/unit_tests/lmdb.cpp index c859439eb..c213577fb 100644 --- a/tests/unit_tests/lmdb.cpp +++ b/tests/unit_tests/lmdb.cpp @@ -30,10 +30,14 @@ #include <boost/range/algorithm/equal.hpp> #include <gtest/gtest.h> +#include "blockchain_db/lmdb/db_lmdb.h" +#include "cryptonote_basic/cryptonote_basic_impl.h" +#include "hex.h" #include "lmdb/database.h" #include "lmdb/table.h" #include "lmdb/transaction.h" #include "lmdb/util.h" +#include "string_tools.h" namespace { @@ -53,6 +57,24 @@ namespace MDB_val right_val = lmdb::to_val(right); return (*cmp)(&left_val, &right_val); } + + crypto::hash postfix_hex_to_hash(const std::string& hex) + { + if (hex.size() > 64) throw std::logic_error("postfix_hex_to_hash"); + std::string decoded_bytes; + if (!epee::from_hex::to_string(decoded_bytes, hex)) throw std::logic_error("postfix_hex_to_hash"); + crypto::hash res = crypto::null_hash; + memcpy(res.data + 32 - decoded_bytes.size(), decoded_bytes.data(), decoded_bytes.size()); + return res; + } + + void test_make_template(const std::string& input_hex, unsigned int nbits, const std::string& expected_hex) + { + const crypto::hash input = postfix_hex_to_hash(input_hex); + const crypto::hash expected = postfix_hex_to_hash(expected_hex); + const crypto::hash actual = cryptonote::make_hash32_loose_template(nbits, input); + ASSERT_EQ(expected, actual); + } } TEST(LMDB, Traits) @@ -401,4 +423,164 @@ TEST(LMDB, InvalidKeyIterator) EXPECT_FALSE(test2 != test1); } +TEST(LMDB_kanonymity, compare_hash32_reversed_nbits) +{ + static constexpr size_t NUM_RANDOM_HASHES = 128; + std::vector<crypto::hash> random_hashes; + random_hashes.reserve(500); + for (size_t i = 0; i < NUM_RANDOM_HASHES; ++i) + random_hashes.push_back(crypto::rand<crypto::hash>()); + + bool r = true; + + // Compare behavior of compare_hash32_reversed_nbits(nbits=256) to BlockchainLMDB::compare_hash32 + for (size_t i = 0; i < NUM_RANDOM_HASHES; ++i) + { + for (size_t j = 0; j < NUM_RANDOM_HASHES; ++j) + { + const crypto::hash& ha = random_hashes[i]; + const crypto::hash& hb = random_hashes[j]; + const MDB_val mva = {sizeof(crypto::hash), (void*)(&ha)}; + const MDB_val mvb = {sizeof(crypto::hash), (void*)(&hb)}; + const int expected = cryptonote::BlockchainLMDB::compare_hash32(&mva, &mvb); + const int actual = cryptonote::compare_hash32_reversed_nbits(ha, hb, 256); + if (actual != expected) + { + std::cerr << "Failed compare_hash32_reversed_nbits test case with hashes:" << std::endl; + std::cerr << " " << epee::string_tools::pod_to_hex(ha) << std::endl; + std::cerr << " " << epee::string_tools::pod_to_hex(hb) << std::endl; + r = false; + } + EXPECT_EQ(expected, actual); + } + } + + ASSERT_TRUE(r); + + const auto cmp_byte_rev = [](const crypto::hash& ha, const crypto::hash& hb, unsigned int nbytes) -> int + { + if (nbytes > sizeof(crypto::hash)) throw std::logic_error("can't compare with nbytes too big"); + const uint8_t* va = (const uint8_t*)ha.data; + const uint8_t* vb = (const uint8_t*)hb.data; + for (size_t i = 31; nbytes; --i, --nbytes) + { + if (va[i] < vb[i]) return -1; + else if (va[i] > vb[i]) return 1; + } + return 0; + }; + + // Test partial hash compares w/o partial bytes + for (size_t i = 0; i < NUM_RANDOM_HASHES; ++i) + { + for (size_t j = 0; j < NUM_RANDOM_HASHES; ++j) + { + for (unsigned int nbytes = 0; nbytes <= 32; ++nbytes) + { + const crypto::hash& ha = random_hashes[i]; + const crypto::hash& hb = random_hashes[j]; + const int expected = cmp_byte_rev(ha, hb, nbytes); + const int actual = cryptonote::compare_hash32_reversed_nbits(ha, hb, nbytes * 8); + if (actual != expected) + { + std::cerr << "Failed compare_hash32_reversed_nbits test case with hashes and args:" << std::endl; + std::cerr << " " << epee::string_tools::pod_to_hex(ha) << std::endl; + std::cerr << " " << epee::string_tools::pod_to_hex(hb) << std::endl; + std::cerr << " nbytes=" << nbytes << std::endl; + r = false; + } + EXPECT_EQ(expected, actual); + } + } + } + + ASSERT_TRUE(r); + + // Test partial hash compares w/ partial bytes + for (size_t i = 0; i < NUM_RANDOM_HASHES; ++i) + { + const crypto::hash& ha = random_hashes[i]; + for (size_t modnbytes = 0; modnbytes < 32; ++modnbytes) + { + for (size_t modbitpos = 0; modbitpos < 8; ++modbitpos) + { + const size_t modbytepos = 31 - modnbytes; + const uint8_t mask = 1 << modbitpos; + const bool bit_was_zero = 0 == (static_cast<uint8_t>(ha.data[modbytepos]) & mask); + const unsigned int modnbits = modnbytes * 8 + (7 - modbitpos); + + // Create modified random hash by flipping one bit + crypto::hash hb = ha; + hb.data[modbytepos] = static_cast<uint8_t>(hb.data[modbytepos]) ^ mask; + + for (unsigned int cmpnbits = 0; cmpnbits <= 256; ++cmpnbits) + { + const int expected = cmpnbits <= modnbits ? 0 : bit_was_zero ? -1 : 1; + const int actual = cryptonote::compare_hash32_reversed_nbits(ha, hb, cmpnbits); + if (actual != expected) + { + std::cerr << "Failed compare_hash32_reversed_nbits test case with hashes and args:" << std::endl; + std::cerr << " " << epee::string_tools::pod_to_hex(ha) << std::endl; + std::cerr << " " << epee::string_tools::pod_to_hex(hb) << std::endl; + std::cerr << " modnbytes=" << modnbytes << std::endl; + std::cerr << " modbitpos=" << modbitpos << std::endl; + std::cerr << " cmpnbits=" << cmpnbits << std::endl; + r = false; + } + EXPECT_EQ(expected, actual); + } + } + } + } + + ASSERT_TRUE(r); + // Test equality + for (size_t i = 0; i < NUM_RANDOM_HASHES; ++i) + { + const crypto::hash& ha = random_hashes[i]; + for (unsigned int nbits = 0; nbits <= 256; ++nbits) + { + const int actual = cryptonote::compare_hash32_reversed_nbits(ha, ha, nbits); + if (actual) + { + std::cerr << "Failed compare_hash32_reversed_nbits test case with hash and args:" << std::endl; + std::cerr << " " << epee::string_tools::pod_to_hex(ha) << std::endl; + std::cerr << " nbits=" << nbits << std::endl; + r = false; + } + EXPECT_EQ(0, actual); + } + } + +} + +TEST(LMDB_kanonymity, make_hash32_loose_template) +{ + const std::string example_1 = "0abcdef1234567890abcdef1234567890abcdef1234567890abcdef123456789"; + + test_make_template(example_1, 0, ""); + + test_make_template(example_1, 1, "80"); + test_make_template(example_1, 2, "80"); + test_make_template(example_1, 3, "80"); + test_make_template(example_1, 4, "80"); + test_make_template(example_1, 5, "88"); + test_make_template(example_1, 6, "88"); + test_make_template(example_1, 7, "88"); + test_make_template(example_1, 8, "89"); + + test_make_template(example_1, 9, "0089"); + test_make_template(example_1, 10, "4089"); + test_make_template(example_1, 11, "6089"); + test_make_template(example_1, 12, "6089"); + test_make_template(example_1, 13, "6089"); + test_make_template(example_1, 14, "6489"); + test_make_template(example_1, 15, "6689"); + test_make_template(example_1, 16, "6789"); + + test_make_template(example_1, 32, "23456789"); + test_make_template(example_1, 64, "0abcdef123456789"); + test_make_template(example_1, 128, "0abcdef1234567890abcdef123456789"); + test_make_template(example_1, 256, example_1); +} diff --git a/tests/unit_tests/long_term_block_weight.cpp b/tests/unit_tests/long_term_block_weight.cpp index e52bd1955..5d954bf0c 100644 --- a/tests/unit_tests/long_term_block_weight.cpp +++ b/tests/unit_tests/long_term_block_weight.cpp @@ -400,3 +400,38 @@ TEST(long_term_block_weight, long_growth_spike_and_drop) ASSERT_GT(long_term_effective_median_block_weight, 300000 * 1.07); ASSERT_LT(long_term_effective_median_block_weight, 300000 * 1.09); } + +TEST(long_term_block_weight, cache_matches_true_value) +{ + PREFIX(16); + + // Add big blocks to increase the block weight limit + for (uint64_t h = 0; h <= 2000; ++h) + { + size_t w = bc->get_current_cumulative_block_weight_limit(); + uint64_t ltw = bc->get_next_long_term_block_weight(w); + bc->get_db().add_block(std::make_pair(cryptonote::block(), ""), w, ltw, h, h, {}); + bc->update_next_cumulative_weight_limit(); + } + + ASSERT_GT(bc->get_current_cumulative_block_weight_limit() * 10/17 , 300000); + + // Add small blocks to the top of the chain + for (uint64_t h = 2000; h <= 5001; ++h) + { + size_t w = (bc->get_current_cumulative_block_weight_median() * 10/17) - 1000; + uint64_t ltw = bc->get_next_long_term_block_weight(w); + bc->get_db().add_block(std::make_pair(cryptonote::block(), ""), w, ltw, h, h, {}); + bc->update_next_cumulative_weight_limit(); + } + + // get the weight limit + uint64_t weight_limit = bc->get_current_cumulative_block_weight_limit(); + // refresh the cache + bc->m_long_term_block_weights_cache_rolling_median.clear(); + bc->get_long_term_block_weight_median(bc->get_db().height() - TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW, TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW); + bc->update_next_cumulative_weight_limit(); + + // make sure the weight limit is the same + ASSERT_EQ(weight_limit, bc->get_current_cumulative_block_weight_limit()); +} diff --git a/tests/unit_tests/main.cpp b/tests/unit_tests/main.cpp index f690f3581..4094d79b7 100644 --- a/tests/unit_tests/main.cpp +++ b/tests/unit_tests/main.cpp @@ -62,9 +62,14 @@ int main(int argc, char** argv) ::testing::InitGoogleTest(&argc, argv); + // the default test data directory is ../data (relative to the executable's directory) + const auto default_test_data_dir = boost::filesystem::canonical(argv[0]).parent_path().parent_path() / "data"; + po::options_description desc_options("Command line options"); - const command_line::arg_descriptor<std::string> arg_data_dir = { "data-dir", "Data files directory", DEFAULT_DATA_DIR }; + const command_line::arg_descriptor<std::string> arg_data_dir = { "data-dir", "Data files directory", default_test_data_dir.string() }; + const command_line::arg_descriptor<std::string> arg_log_level = { "log-level", "0-4 or categories", "" }; command_line::add_arg(desc_options, arg_data_dir); + command_line::add_arg(desc_options, arg_log_level); po::variables_map vm; bool r = command_line::handle_error_helper(desc_options, [&]() @@ -76,7 +81,12 @@ int main(int argc, char** argv) if (! r) return 1; - unit_test::data_dir = command_line::get_arg(vm, arg_data_dir); + // set the test data directory + unit_test::data_dir = command_line::get_arg(vm, arg_data_dir).c_str(); + + // set the log level + if (!command_line::is_arg_defaulted(vm, arg_log_level)) + mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); CATCH_ENTRY_L0("main", 1); diff --git a/tests/unit_tests/net.cpp b/tests/unit_tests/net.cpp index b9555b213..2b565f7d0 100644 --- a/tests/unit_tests/net.cpp +++ b/tests/unit_tests/net.cpp @@ -239,7 +239,7 @@ namespace net::tor_address tor; BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(tor); + KV_SERIALIZE(tor) END_KV_SERIALIZE_MAP() }; } @@ -611,7 +611,7 @@ namespace net::i2p_address i2p; BEGIN_KV_SERIALIZE_MAP() - KV_SERIALIZE(i2p); + KV_SERIALIZE(i2p) END_KV_SERIALIZE_MAP() }; } diff --git a/tests/unit_tests/node_server.cpp b/tests/unit_tests/node_server.cpp index 747778c69..6f490d7b9 100644 --- a/tests/unit_tests/node_server.cpp +++ b/tests/unit_tests/node_server.cpp @@ -306,8 +306,8 @@ TEST(node_server, bind_same_p2p_port) Relevant part about REUSEADDR from man: https://www.man7.org/linux/man-pages/man7/ip.7.html - For Mac OSX, set the following alias, before running the test, or else it will fail: - sudo ifconfig lo0 alias 127.0.0.2 + For Mac OSX and OpenBSD, set the following alias (by running the command as root), before running the test, or else it will fail: + ifconfig lo0 alias 127.0.0.2 */ vm.find(nodetool::arg_p2p_bind_ip.name)->second = boost::program_options::variable_value(std::string("127.0.0.2"), false); vm.find(nodetool::arg_p2p_bind_port.name)->second = boost::program_options::variable_value(std::string(port), false); diff --git a/tests/unit_tests/serialization.cpp b/tests/unit_tests/serialization.cpp index 49f1dc310..ab81acefc 100644 --- a/tests/unit_tests/serialization.cpp +++ b/tests/unit_tests/serialization.cpp @@ -59,9 +59,7 @@ struct Struct }; template <class Archive> -struct serializer<Archive, Struct> -{ - static bool serialize(Archive &ar, Struct &s) { +static bool do_serialize(Archive &ar, Struct &s) { ar.begin_object(); ar.tag("a"); ar.serialize_int(s.a); @@ -71,8 +69,7 @@ struct serializer<Archive, Struct> ar.serialize_blob(s.blob, sizeof(s.blob)); ar.end_object(); return true; - } -}; +} struct Struct1 { @@ -122,6 +119,23 @@ bool try_parse(const string &blob) return serialization::parse_binary(blob, s1); } +namespace example_namespace +{ + struct ADLExampleStruct + { + std::string msg; + }; + + template <class Archive> + static bool do_serialize(Archive &ar, ADLExampleStruct &aes) + { + ar.begin_object(); + FIELD_N("custom_fieldname", aes.msg); + ar.end_object(); + return ar.good(); + } +} + TEST(Serialization, BinaryArchiveInts) { uint64_t x = 0xff00000000, x1; @@ -1178,3 +1192,18 @@ TEST(Serialization, difficulty_type) ASSERT_EQ(v_original, v_unserialized); } + +TEST(Serialization, adl_free_function) +{ + std::stringstream ss; + json_archive<true> ar(ss); + + const std::string msg = "Howdy, World!"; + example_namespace::ADLExampleStruct aes{msg}; + + ASSERT_TRUE(serialization::serialize(ar, aes)); + + // VVVVVVVVVVVVVVVVVVVVVVVVVV weird string serialization artifact + const std::string expected = "{\"custom_fieldname\": " + std::to_string(msg.size()) + '"' + epee::string_tools::buff_to_hex_nodelimer(msg) + "\"}"; + EXPECT_EQ(expected, ss.str()); +} diff --git a/tests/unit_tests/wallet_storage.cpp b/tests/unit_tests/wallet_storage.cpp index dacaff960..ff01e452c 100644 --- a/tests/unit_tests/wallet_storage.cpp +++ b/tests/unit_tests/wallet_storage.cpp @@ -29,6 +29,8 @@ #include "unit_tests_utils.h" #include "gtest/gtest.h" +#include <cctype> + #include "file_io_utils.h" #include "wallet/wallet2.h" @@ -38,6 +40,9 @@ using namespace epee::file_io_utils; static constexpr const char WALLET_00fd416a_PRIMARY_ADDRESS[] = "45p2SngJAPSJbqSiUvYfS3BfhEdxZmv8pDt25oW1LzxrZv9Uq6ARagiFViMGUE3gJk5VPWingCXVf1p2tyAy6SUeSHPhbve"; +// https://github.com/monero-project/monero/blob/67d190ce7c33602b6a3b804f633ee1ddb7fbb4a1/src/wallet/wallet2.cpp#L156 +static constexpr const char WALLET2_ASCII_OUTPUT_MAGIC[] = "MoneroAsciiDataV1"; + TEST(wallet_storage, store_to_file2file) { const path source_wallet_file = unit_test::data_dir / "wallet_00fd416a"; @@ -264,3 +269,115 @@ TEST(wallet_storage, change_password_mem2file) EXPECT_EQ(primary_address_1, primary_address_2); } + +TEST(wallet_storage, gen_ascii_format) +{ + const path target_wallet_file = unit_test::data_dir / "wallet_gen_ascii_format"; + + if (is_file_exist(target_wallet_file.string())) + remove(target_wallet_file); + if (is_file_exist(target_wallet_file.string() + ".keys")) + remove(target_wallet_file.string() + ".keys"); + ASSERT_FALSE(is_file_exist(target_wallet_file.string())); + ASSERT_FALSE(is_file_exist(target_wallet_file.string() + ".keys")); + + const epee::wipeable_string password("https://safecurves.cr.yp.to/rigid.html"); + + std::string primary_address_1, primary_address_2; + { + tools::wallet2 w; + w.set_export_format(tools::wallet2::Ascii); + ASSERT_EQ(tools::wallet2::Ascii, w.export_format()); + w.generate(target_wallet_file.string(), password); + primary_address_1 = w.get_address_as_str(); + } + + ASSERT_TRUE(is_file_exist(target_wallet_file.string())); + ASSERT_TRUE(is_file_exist(target_wallet_file.string() + ".keys")); + + // Assert that we store keys in ascii format + { + std::string key_file_contents; + ASSERT_TRUE(epee::file_io_utils::load_file_to_string(target_wallet_file.string() + ".keys", key_file_contents)); + EXPECT_NE(std::string::npos, key_file_contents.find(WALLET2_ASCII_OUTPUT_MAGIC)); + for (const char c : key_file_contents) + ASSERT_TRUE(std::isprint(c) || c == '\n' || c == '\r'); + } + + { + tools::wallet2 w; + w.set_export_format(tools::wallet2::Ascii); + ASSERT_EQ(tools::wallet2::Ascii, w.export_format()); + w.load(target_wallet_file.string(), password); + primary_address_2 = w.get_address_as_str(); + } + + EXPECT_EQ(primary_address_1, primary_address_2); +} + +TEST(wallet_storage, change_export_format) +{ + const path target_wallet_file = unit_test::data_dir / "wallet_change_export_format"; + + if (is_file_exist(target_wallet_file.string())) + remove(target_wallet_file); + if (is_file_exist(target_wallet_file.string() + ".keys")) + remove(target_wallet_file.string() + ".keys"); + ASSERT_FALSE(is_file_exist(target_wallet_file.string())); + ASSERT_FALSE(is_file_exist(target_wallet_file.string() + ".keys")); + + const epee::wipeable_string password("https://safecurves.cr.yp.to/rigid.html"); + + std::string primary_address_1, primary_address_2; + { + tools::wallet2 w; + ASSERT_EQ(tools::wallet2::Binary, w.export_format()); + w.generate(target_wallet_file.string(), password); + primary_address_1 = w.get_address_as_str(); + w.store(); + + // Assert that we initially store keys in binary format + { + std::string key_file_contents; + ASSERT_TRUE(epee::file_io_utils::load_file_to_string(target_wallet_file.string() + ".keys", key_file_contents)); + EXPECT_EQ(std::string::npos, key_file_contents.find(WALLET2_ASCII_OUTPUT_MAGIC)); + bool only_printable = true; + for (const char c : key_file_contents) + { + if (!std::isprint(c) && c != '\n' && c != '\r') + { + only_printable = false; + break; + } + } + EXPECT_FALSE(only_printable); + } + + // switch formats and store + w.set_export_format(tools::wallet2::Ascii); + ASSERT_EQ(tools::wallet2::Ascii, w.export_format()); + w.store_to("", password, /*force_rewrite_keys=*/ true); + } + + ASSERT_TRUE(is_file_exist(target_wallet_file.string())); + ASSERT_TRUE(is_file_exist(target_wallet_file.string() + ".keys")); + + // Assert that we store keys in ascii format + { + std::string key_file_contents; + ASSERT_TRUE(epee::file_io_utils::load_file_to_string(target_wallet_file.string() + ".keys", key_file_contents)); + EXPECT_NE(std::string::npos, key_file_contents.find(WALLET2_ASCII_OUTPUT_MAGIC)); + for (const char c : key_file_contents) + ASSERT_TRUE(std::isprint(c) || c == '\n' || c == '\r'); + } + + { + tools::wallet2 w; + w.set_export_format(tools::wallet2::Ascii); + ASSERT_EQ(tools::wallet2::Ascii, w.export_format()); + w.load(target_wallet_file.string(), password); + primary_address_2 = w.get_address_as_str(); + } + + EXPECT_EQ(primary_address_1, primary_address_2); +} diff --git a/utils/gpg_keys/0xfffc0000.asc b/utils/gpg_keys/0xfffc0000.asc new file mode 100644 index 000000000..6da5391a3 --- /dev/null +++ b/utils/gpg_keys/0xfffc0000.asc @@ -0,0 +1,13 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mDMEZZx2rhYJKwYBBAHaRw8BAQdADWrpnGYQMJB7m7OYO5QhLPbNZckNDp5XXX5C +NRqQSv60ITB4ZmZmYzAwMDAgPDB4ZmZmYzAwMDBAcHJvdG9uLm1lPoiTBBMWCgA7 +FiEESleXwQVNMqPAAcfxZQ98K3vaOBkFAmWcdq4CGwMFCwkIBwICIgIGFQoJCAsC +BBYCAwECHgcCF4AACgkQZQ98K3vaOBmm6QEA1l5FuxR7MtER+pl4XiL+UEH4rGjQ +BHTD/Tv6Ar6uq2oA/34XecxRCKmj4ximF1aMucJSvS8nIjRbRQ9E5llcLOYMuDgE +ZZx2rhIKKwYBBAGXVQEFAQEHQIxFNA2STqB1WQNDtPHalCrKOBxheB6Lg1U+ceOF +rTFcAwEIB4h4BBgWCgAgFiEESleXwQVNMqPAAcfxZQ98K3vaOBkFAmWcdq4CGwwA +CgkQZQ98K3vaOBnZ2gEAsrARrt+Mm+zfqVFZ7iAQHD5xmdj9Q0Qf3zl5dVbkFlQB +APOdwpn/iY0zA6McjZ1H+Uiv+bw8uTVZadgjtJGb474O +=J9WU +-----END PGP PUBLIC KEY BLOCK----- diff --git a/utils/gpg_keys/tobtoht.asc b/utils/gpg_keys/tobtoht.asc new file mode 100644 index 000000000..16311b2d8 --- /dev/null +++ b/utils/gpg_keys/tobtoht.asc @@ -0,0 +1,51 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBGLhQcoBEADiEB9S7FRpHpnI8TpFhBoYgeugroHJgupRmSvMt+0myD9AkKbL +6FG3F98HRyFA3ljq3UWd5kUxZIeljdwWn/sHd3eu+BXJ2gXU7jo8MzOFDQrW+bcE +/4PFEdbLhsvmEvXkURFFZTyVA6CgAfsd86l2okG0X4dbDLwGRcVx30bYThH5a/lK +buTKOANbP9Rw6hRkZAIZof2DDZgU4G5UEFvxf80lhbXQgPtQoCAad9b/cRGUnau8 +STtA6qKiVch1IS0Y2+EE91aN2MLmGxjTr5N6kQm/oYeIKqLiZuVBkGmNVVP5CyDw +k4YjmrCyckewAWwsjDNZ6X/eAV06inblOxgX9g2VJKVHck4WtN9c+5SmTkWz5YZ0 +hXxj7h0ASukvud0uf9UwLOAIXRnXRUyxAAw0LhTmHIlsDHw1XJoqIjKnJNpCGIqE +Hn/AQT5n8W5HYF7GJQAav1Qso7yclkUdXP1mfSpbC1Q+R92WAlgLrIwEeUG8iEeO +6KZazSSzn8VwG7Z5VnE32LpIdGDDJOpqABd7CD7b9dtKP/QH8wbdZxVzcMsdD2Zk +VARUSMsMRNKXpQuURN02zPPyu//jYwqrMXHbUE2Mt8quhJQya4xVRbjBW0LHwZQO +/vpUm6reOO5vIJdkgqH0YP5WMyj4/BnhkkOGPjDgffzxMihue34fO+V05wARAQAB +tB90b2J0b2h0IDx0b2JAZmVhdGhlcndhbGxldC5vcmc+iQJOBBMBCAA4FiEE6HvZ +Ic3YhcnXijjF5FsQ3QJ9JHIFAmLhQcoCGwMFCwkIBwIGFQoJCAsCBBYCAwECHgEC +F4AACgkQ5FsQ3QJ9JHLw/hAA28cJgy2KOAFNaX3lSeOGC6ZdQUv4dtTkg1wZjmDq +AKE8I1ANCaVHtB0Df7CJTnBnCvkLAhf8gA3ldNBD3PKySBmP1rnjRYjYmNOZ+9io +EHwlgL7WyknXUtbitrpe/wM7xWN/cDdEmlRvOb9pf4GTPw2MaZLM+E43KnKoEWJ/ +Ms8vOLi2UIZNmpYhWzjS178EtC9AfvsCwWexVwYhIxlmqF++W2QYMVUEqDaVMKsJ +BohhPrOb/SQJVd5D6vrcIGs/aZZOEuthvBp1ycFdCEGajwBNzmrmhLfl9CuP0NqM +4KkYYHrvJio8QbsMxU27KJfDaGloNfSkk3V+MhZGvMqFUkTuabeyz6Uh/CREV6s/ +JApTPHvAmMeD9p6PNfxZvSq/jcbwbxPdqvWyy+dqEaaPIGbR1sOSJOOzxcJ84/1F +VwA2FQENmlob6VRroqHWTZzKPmwOi2qFAjEleEHZrjfaaHlXrGbi1ucFOqn+2/Qh +F4oi1gzFG0jQYajXxRC1CrM3ykKWgGt6cTeMDXQxEdQOkWdgP7pkVbpc9j4OW0aF +WeUlr1i+BU6UwY6eibAwT57pBiU4A9TtJjHBBPUG+XGIB9JPAWD43mfXAONfmll3 +zyV0Y6Tq8NiQUs0yuWNWqo6fDn/EW5ldNsDz+PwDGGusEMJYrGc8emWqi+CsSg83 +imu5Ag0EYuFBygEQAMXwWjBPxiWN1wDrHNOwoVf4aX37mYjsgtQ+223x5uFD0ULl +6pH2hUCg/FSVqsm+xvVqX6DV+WmzkGhhC1B2bnpEkzxQr79dXEq0V+hsKypa7t66 +8/e+4pNxeSz+4UMJApEc/lIGtEcWPgx2xNChagB2V/074hN3eZ6jpq4KJS4GG7ly +ts+t2jNs8EArrWKWKzIp2vc+Upad5LD4SSWR5qQQUgXtVbJO3EoVfeK3+g0iiVOv +o29yEStCUBrkbuK1Kwe1wjw6ePIDWKXkvo0vSRKxolCGALFhToLM0QNLjp1WNXJG +TuZtcUYIY58b7WAcEoqM7uRc3YcFMjl1tW/e3UF8Ii6CBr69bxzeygfXEdXjM+XC +H9WvxtHFFgelnloA2OHRqe1ehtsLtmlVHoJ1RArKEFYrMiApAiGSf9ztm/eQ3yX4 +6E28k3OOZSRUyyZHC++NxmNDKjQpcVQ2k5yCGje5xB4844n5fTMafWNGYnnveFTx +ZlXwz+ZlwvGHKwwI6u+n+ZA5eV+R24mo9JaPQVlQ9MLpzbWhpUvGy8ugn4rhiaz2 +4gw6hdd1BaTzkMgrCh1uT4GxVSQxanX4ubHK06QiH5yiopAIcrLXQCtIaNwmQPD2 +1K9WvZ0xagrXDOInnsGAUM8V7ce5VhwWxwIjjP7QCh886P2HHg5KM7twSG6rABEB +AAGJAjYEGAEIACAWIQToe9khzdiFydeKOMXkWxDdAn0kcgUCYuFBygIbDAAKCRDk +WxDdAn0kcuc8EACCJWI0o5AYxDQ1tkbPcnfwzFQQ50vH6pfEmk3/YQXbaJxtNLVS +ba13zC7rw5pBfVLXPDu7oOcf0x8/3HoeKfAFP/ChlU/PA2yojigZJV9zCpqCC0dm +vV7N+Mgd/GD6FGwnSM3kHnkYPgbu1/uMDbm7OTq3fbNK5BTKikkmW8ZxDyNwGYlR +5Smt2dymlxcEIYZSw27R5yyJhw6ri5trwJwJYay7LgKnWb+ePNDPUR0+ph0bXYl8 +DKa1c3vkNfmN17AUtzl0DByMZ+C0mJiu1cjWbbgg5PuQQXiNkV2xcDcMrLjVE686 +YkYI9KKLJQVLO8d57OEv0irE1bD7BlyILwemrdjuyHsPRBwHSnw8s1+Sy4qWg099 +rCGcK7wu+L9r/+fgTxONSyqb74pcO4JQDXZKKjsDfR1R2qhcvCbU4VpOvIdoiXh9 +wUBY4pBrMGDbtah3DOqvCEb7Jol754KWOaTkjDNbGrBZ5UBGdVa5flKOzIN+YOVQ +ce9Nz8XeSVy8PzrUpf5P/DFBe4Gr/E4PZkuLtJGfzNHbzhBMVMv/vfbL24uACQM0 +DKQ3vplo+++Zcyey8Nz1MNJDYqcCD/o1SInMPWbtlytNNWLj+5i75K37TIOl3vkq +Aa6OeR9y7rn8bJ88tKgnXiEalMy7HUyBDZbMCeE6IVGZvbUSdLz5VGpKmQ== +=KI4D +-----END PGP PUBLIC KEY BLOCK----- diff --git a/utils/python-rpc/framework/daemon.py b/utils/python-rpc/framework/daemon.py index 43a1aa469..c7831d1ee 100644 --- a/utils/python-rpc/framework/daemon.py +++ b/utils/python-rpc/framework/daemon.py @@ -590,6 +590,18 @@ class Daemon(object): } return self.rpc.send_json_rpc_request(flush_cache) + def get_txids_loose(self, txid_template, num_matching_bits): + get_txids_loose = { + 'method': 'get_txids_loose', + 'params': { + 'txid_template': txid_template, + 'num_matching_bits': num_matching_bits + }, + 'jsonrpc': '2.0', + 'id': '0' + } + return self.rpc.send_json_rpc_request(get_txids_loose) + def sync_txpool(self): sync_txpool = { 'method': 'sync_txpool', diff --git a/utils/python-rpc/framework/wallet.py b/utils/python-rpc/framework/wallet.py index 1c42f7f04..18af4edc4 100644 --- a/utils/python-rpc/framework/wallet.py +++ b/utils/python-rpc/framework/wallet.py @@ -38,13 +38,14 @@ class Wallet(object): self.port = port self.rpc = JSONRPC('{protocol}://{host}:{port}'.format(protocol=protocol, host=host, port=port if port else 18090+idx)) - def transfer(self, destinations, account_index = 0, subaddr_indices = [], priority = 0, ring_size = 0, unlock_time = 0, payment_id = '', get_tx_key = True, do_not_relay = False, get_tx_hex = False, get_tx_metadata = False): + def transfer(self, destinations, account_index = 0, subaddr_indices = [], priority = 0, ring_size = 0, unlock_time = 0, payment_id = '', get_tx_key = True, do_not_relay = False, get_tx_hex = False, get_tx_metadata = False, subtract_fee_from_outputs = []): transfer = { 'method': 'transfer', 'params': { 'destinations': destinations, 'account_index': account_index, 'subaddr_indices': subaddr_indices, + 'subtract_fee_from_outputs': subtract_fee_from_outputs, 'priority': priority, 'ring_size' : ring_size, 'unlock_time' : unlock_time, |