diff options
270 files changed, 6800 insertions, 2952 deletions
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c6ed13e62..7c28a71ce 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,6 +2,18 @@ name: ci/gh-actions/cli on: [push, pull_request] +# The below variables reduce repetitions across similar targets +env: + REMOVE_BUNDLED_BOOST : rm -rf /usr/local/share/boost + BUILD_DEFAULT_LINUX: | + ccache --max-size=150M + cmake -S . -B build -D ARCH="default" -D BUILD_TESTS=ON -D CMAKE_BUILD_TYPE=release && cmake --build build -j3 + APT_INSTALL_LINUX: 'sudo apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libnorm-dev libusb-1.0-0-dev libpgm-dev libprotobuf-dev protobuf-compiler ccache' + APT_SET_CONF: | + echo "Acquire::Retries \"3\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + echo "Acquire::http::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + echo "Acquire::ftp::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + jobs: build-macos: runs-on: macOS-latest @@ -14,7 +26,7 @@ jobs: submodules: recursive - uses: actions/cache@v2 with: - path: ~/.ccache + path: /Users/runner/Library/Caches/ccache key: ccache-macos-build-${{ github.sha }} restore-keys: ccache-macos-build- - name: install dependencies @@ -26,6 +38,10 @@ jobs: build-windows: runs-on: windows-latest + env: + CCACHE_COMPRESS: 1 + CCACHE_TEMPDIR: C:\Users\runneradmin\.ccache-temp + CCACHE_DIR: C:\Users\runneradmin\.ccache defaults: run: shell: msys2 {0} @@ -33,18 +49,31 @@ jobs: - uses: actions/checkout@v1 with: submodules: recursive + - uses: actions/cache@v2 + with: + path: C:\Users\runneradmin\.ccache + key: ccache-windows-build-${{ github.sha }} + restore-keys: ccache-windows-build- - uses: eine/setup-msys2@v2 with: update: true - install: mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-boost mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq mingw-w64-x86_64-libsodium mingw-w64-x86_64-hidapi mingw-w64-x86_64-protobuf-c mingw-w64-x86_64-libusb git + install: mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-ccache mingw-w64-x86_64-boost mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq mingw-w64-x86_64-libsodium mingw-w64-x86_64-hidapi mingw-w64-x86_64-protobuf-c mingw-w64-x86_64-libusb git - name: build - run: make release-static-win64 -j2 + run: | + ccache --max-size=150M + make release-static-win64 -j2 + +# See the OS labels and monitor deprecations here: +# https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources build-ubuntu: - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} env: CCACHE_COMPRESS: 1 CCACHE_TEMPDIR: /tmp/.ccache-temp + strategy: + matrix: + os: [ubuntu-latest, ubuntu-18.04] steps: - uses: actions/checkout@v1 with: @@ -52,23 +81,18 @@ jobs: - uses: actions/cache@v2 with: path: ~/.ccache - key: ccache-ubuntu-build-${{ github.sha }} - restore-keys: ccache-ubuntu-build- + key: ccache-ubuntu-build-${{ matrix.os }}-${{ github.sha }} + restore-keys: ccache-ubuntu-build-${{ matrix.os }} - name: remove bundled boost - run: sudo rm -rf /usr/local/share/boost + run: ${{env.REMOVE_BUNDLED_BOOST}} - name: set apt conf - run: | - echo "Acquire::Retries \"3\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom - echo "Acquire::http::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom - echo "Acquire::ftp::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + run: ${{env.APT_SET_CONF}} - name: update apt run: sudo apt update - name: install monero dependencies - run: sudo apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libnorm-dev libusb-1.0-0-dev libpgm-dev libprotobuf-dev protobuf-compiler ccache + run: ${{env.APT_INSTALL_LINUX}} - name: build - run: | - ccache --max-size=150M - make -j3 + run: ${{env.BUILD_DEFAULT_LINUX}} libwallet-ubuntu: runs-on: ubuntu-latest @@ -85,43 +109,69 @@ jobs: key: ccache-ubuntu-libwallet-${{ github.sha }} restore-keys: ccache-ubuntu-libwallet- - name: remove bundled boost - run: sudo rm -rf /usr/local/share/boost + run: ${{env.REMOVE_BUNDLED_BOOST}} - name: set apt conf - run: | - echo "Acquire::Retries \"3\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom - echo "Acquire::http::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom - echo "Acquire::ftp::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + run: ${{env.APT_SET_CONF}} - name: update apt run: sudo apt update - name: install monero dependencies - run: sudo apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libnorm-dev libusb-1.0-0-dev libpgm-dev libprotobuf-dev protobuf-compiler ccache + run: ${{env.APT_INSTALL_LINUX}} - name: build run: | ccache --max-size=150M - cmake -DBUILD_GUI_DEPS=ON - make -j3 + cmake . + make wallet_api -j3 test-ubuntu: needs: build-ubuntu runs-on: ubuntu-latest + env: + CCACHE_COMPRESS: 1 + CCACHE_TEMPDIR: /tmp/.ccache-temp steps: - uses: actions/checkout@v1 with: submodules: recursive + - name: ccache + uses: actions/cache@v2 + with: + path: ~/.ccache + key: ccache-ubuntu-build-ubuntu-latest-${{ github.sha }} + restore-keys: ccache-ubuntu-build-ubuntu-latest - name: remove bundled boost - run: sudo rm -rf /usr/local/share/boost + run: ${{env.REMOVE_BUNDLED_BOOST}} - name: set apt conf - run: | - echo "Acquire::Retries \"3\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom - echo "Acquire::http::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom - echo "Acquire::ftp::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom + run: ${{env.APT_SET_CONF}} - name: update apt run: sudo apt update - name: install monero dependencies - run: sudo apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libnorm-dev libusb-1.0-0-dev libpgm-dev libprotobuf-dev protobuf-compiler - - name: install requests - run: pip install requests + run: ${{env.APT_INSTALL_LINUX}} + - name: install Python dependencies + run: pip install requests psutil monotonic - name: tests env: CTEST_OUTPUT_ON_FAILURE: ON - run: make release-test -j3 + run: | + ${{env.BUILD_DEFAULT_LINUX}} + cmake --build build --target test + +# ARCH="default" (not "native") ensures, that a different execution host can execute binaries compiled elsewhere. +# BUILD_SHARED_LIBS=ON speeds up the linkage part a bit, reduces size, and is the only place where the dynamic linkage is tested. + + source-archive: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v1 + with: + submodules: recursive + - name: archive + run: | + pip install git-archive-all + export VERSION="monero-$(git describe)" + export OUTPUT="$VERSION.tar" + echo "OUTPUT=$OUTPUT" >> $GITHUB_ENV + /home/runner/.local/bin/git-archive-all --prefix "$VERSION/" --force-submodules "$OUTPUT" + - uses: actions/upload-artifact@v2 + with: + name: ${{ env.OUTPUT }} + path: /home/runner/work/monero/monero/${{ env.OUTPUT }} diff --git a/.github/workflows/depends.yml b/.github/workflows/depends.yml index 3e0f048c7..8e4eaf177 100644 --- a/.github/workflows/depends.yml +++ b/.github/workflows/depends.yml @@ -45,15 +45,29 @@ jobs: - uses: actions/checkout@v1 with: submodules: recursive +# Most volatile cache - name: ccache uses: actions/cache@v2 with: - path: | - ~/.ccache - contrib/depends/built - contrib/depends/sdk-sources + path: ~/.ccache key: ccache-${{ matrix.toolchain.host }}-${{ github.sha }} restore-keys: ccache-${{ matrix.toolchain.host }}- +# Less volatile cache + - name: depends cache + uses: actions/cache@v2 + with: + path: contrib/depends/built + key: depends-${{ matrix.toolchain.host }}-${{ hashFiles('contrib/depends/packages/*') }} + restore-keys: | + depends-${{ matrix.toolchain.host }}-${{ hashFiles('contrib/depends/packages/*') }} + depends-${{ matrix.toolchain.host }}- +# Static cache + - name: OSX SDK cache + uses: actions/cache@v2 + with: + path: contrib/depends/sdk-sources + key: sdk-${{ matrix.toolchain.host }}-${{ matrix.toolchain.osx_sdk }} + restore-keys: sdk-${{ matrix.toolchain.host }}-${{ matrix.toolchain.osx_sdk }} - name: set apt conf run: | echo "Acquire::Retries \"3\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom diff --git a/.gitmodules b/.gitmodules index 9dacf534f..3cf831bcd 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,8 +4,7 @@ branch = monero [submodule "external/miniupnp"] path = external/miniupnp - url = https://github.com/monero-project/miniupnp - branch = monero + url = https://github.com/miniupnp/miniupnp [submodule "external/rapidjson"] path = external/rapidjson url = https://github.com/Tencent/rapidjson diff --git a/CMakeLists.txt b/CMakeLists.txt index c6d102ba7..d06729931 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -43,7 +43,7 @@ if (IOS) INCLUDE(CmakeLists_IOS.txt) endif() -cmake_minimum_required(VERSION 2.8.7) +cmake_minimum_required(VERSION 3.5) message(STATUS "CMake version ${CMAKE_VERSION}") project(monero) @@ -106,16 +106,20 @@ function (die msg) endfunction () function (add_c_flag_if_supported flag var) - string(REPLACE "-" "_" supported ${flag}_c) - check_c_compiler_flag(${flag} ${supported}) + # Prepending the flag with -Werror will only add the flag, + # if it doesn't result in generation of a warning of using a flag unknown to the compiler. + set(TMP "-Werror ${flag}") + string(REGEX REPLACE "[- ]" "_" supported ${TMP}_c) + check_c_compiler_flag(${TMP} ${supported}) if(${${supported}}) set(${var} "${${var}} ${flag}" PARENT_SCOPE) endif() endfunction() function (add_cxx_flag_if_supported flag var) - string(REPLACE "-" "_" supported ${flag}_cxx) - check_cxx_compiler_flag(${flag} ${supported}) + set(TMP "-Werror ${flag}") + string(REGEX REPLACE "[- ]" "_" supported ${TMP}_cxx) + check_cxx_compiler_flag(${TMP} ${supported}) if(${${supported}}) set(${var} "${${var}} ${flag}" PARENT_SCOPE) endif() @@ -154,6 +158,15 @@ function (monero_set_target_no_relink target) endif() endfunction() +option(STRIP_TARGETS "Strip symbols from targets?" OFF) +function (monero_set_target_strip target) + if (STRIP_TARGETS) + set_target_properties("${target}" PROPERTIES LINK_FLAGS_RELEASE -s) + set_target_properties("${target}" PROPERTIES LINK_FLAGS_DEBUG -s) + # Stripping from Debug might make sense if you're low on disk space, but want to test if debug version builds properly. + endif() +endfunction() + function (monero_add_minimal_executable name) source_group("${name}" FILES @@ -161,8 +174,83 @@ function (monero_add_minimal_executable name) add_executable("${name}" ${ARGN}) - monero_set_target_no_relink( ${name} ) + monero_set_target_no_relink("${name}") + monero_set_target_strip ("${name}") +endfunction() + +# Finds all headers in a directory and its subdirs, to be able to search for them and autosave in IDEs. +# +# Parameters: +# - headers_found: Output variable, which will hold the found headers +# - module_root_dir: The search path for the headers. Typically it will be the module's root dir, so "${CMAKE_CURRENT_SOURCE_DIR}" or a derivative of it. +macro (monero_find_all_headers headers_found module_root_dir) + file(GLOB ${headers_found} + "${module_root_dir}/*.h*" # h* will include hpps as well. + "${module_root_dir}/**/*.h*" # Any number of subdirs will be included. + "${module_root_dir}/*.inl" # .inl is typically template code and is being treated as headers (it's being included). + "${module_root_dir}/**/*.inl" +) +endmacro() + +# Function to forbid undefined symbols and also verify +# 1) Test project with all types of libraries and without undefined symbols can compile successfully +# 2) Test project with all types of libraries and undefined symbols can not compile successfully +function(forbid_undefined_symbols) + unset(TMP) + # https://www.unix.com/man-page/linux/1/ld, --no-undefined, Report unresolved symbol references from regular object files. + add_linker_flag_if_supported(-Wl,--no-undefined TMP) + # https://www.unix.com/man-page/osx/1/ld/, -undefined, Specifies how undefined symbols are to be treated. + add_linker_flag_if_supported(-Wl,-undefined,error TMP) + string(APPEND CMAKE_SHARED_LINKER_FLAGS ${TMP}) + string(APPEND CMAKE_MODULE_LINKER_FLAGS ${TMP}) + set(CMAKE_SHARED_LINKER_FLAGS ${CMAKE_SHARED_LINKER_FLAGS} PARENT_SCOPE) + set(CMAKE_MODULE_LINKER_FLAGS ${CMAKE_MODULE_LINKER_FLAGS} PARENT_SCOPE) + set(TEST_PROJECT "${CMAKE_BINARY_DIR}/${CMAKE_FILES_DIRECTORY}/CMakeTmp/test_project") + foreach(EXPECT IN ITEMS TRUE FALSE) + file(REMOVE_RECURSE "${TEST_PROJECT}") + file(MAKE_DIRECTORY "${TEST_PROJECT}") + file(WRITE "${TEST_PROJECT}/CMakeLists.txt" + [=[ +cmake_minimum_required(VERSION 3.1) +project(test) +option(EXPECT_SUCCESS "" ON) +file(WRITE "${CMAKE_SOURCE_DIR}/incorrect_source.cpp" "void undefined_symbol(); void symbol() { undefined_symbol(); }") +if (EXPECT_SUCCESS) + file(APPEND "${CMAKE_SOURCE_DIR}/incorrect_source.cpp" " void undefined_symbol() {}; ") +endif() +add_library(l0 SHARED incorrect_source.cpp) +add_library(l1 MODULE incorrect_source.cpp) +add_library(l2 STATIC incorrect_source.cpp) +add_library(l3 OBJECT incorrect_source.cpp) +]=] + ) + try_compile(SUCCESS "${TEST_PROJECT}/build" "${TEST_PROJECT}" test + CMAKE_FLAGS + "-DCMAKE_SHARED_LINKER_FLAGS=${CMAKE_SHARED_LINKER_FLAGS}" + "-DCMAKE_MODULE_LINKER_FLAGS=${CMAKE_MODULE_LINKER_FLAGS}" + "-DEXPECT_SUCCESS=${EXPECT}" + ) + if (NOT ${SUCCESS} STREQUAL ${EXPECT}) + message(FATAL_ERROR "Undefined symbols test failure: expect(${EXPECT}), success(${SUCCESS})") + endif() + file(REMOVE_RECURSE "${TEST_PROJECT}") + endforeach() endfunction() +if (NOT (CMAKE_SYSTEM_NAME MATCHES "kOpenBSD.*|OpenBSD.*") AND NOT OSSFUZZ) + forbid_undefined_symbols() +endif() + +if (MINGW) + function(export_all_symbols) + unset(TMP) + add_linker_flag_if_supported(-Wl,--export-all-symbols TMP) + string(APPEND CMAKE_SHARED_LINKER_FLAGS ${TMP}) + string(APPEND CMAKE_MODULE_LINKER_FLAGS ${TMP}) + set(CMAKE_SHARED_LINKER_FLAGS ${CMAKE_SHARED_LINKER_FLAGS} PARENT_SCOPE) + set(CMAKE_MODULE_LINKER_FLAGS ${CMAKE_MODULE_LINKER_FLAGS} PARENT_SCOPE) + endfunction() + export_all_symbols() +endif() if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE) @@ -484,7 +572,7 @@ if (CMAKE_SYSTEM_NAME MATCHES "(SunOS|Solaris)") endif () if (APPLE AND NOT IOS) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=x86-64 -fvisibility=default -std=c++11") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=default -std=c++11") if (NOT OPENSSL_ROOT_DIR) EXECUTE_PROCESS(COMMAND brew --prefix openssl OUTPUT_VARIABLE OPENSSL_ROOT_DIR @@ -502,6 +590,10 @@ if(STATIC AND NOT IOS) endif() endif() +if (WIN32) + list(APPEND OPENSSL_LIBRARIES ws2_32 crypt32) +endif() + find_package(HIDAPI) add_definition_if_library_exists(c memset_s "string.h" HAVE_MEMSET_S) @@ -513,6 +605,61 @@ add_definitions(-DAUTO_INITIALIZE_EASYLOGGINGPP) set(MONERO_GENERATED_HEADERS_DIR "${CMAKE_CURRENT_BINARY_DIR}/generated_include") include_directories(${MONERO_GENERATED_HEADERS_DIR}) +# As of OpenBSD 6.8, -march=<anything> breaks the build +function(set_default_arch) + if (OPENBSD) + set(ARCH default) + else() + set(ARCH native) + endif() + + set(ARCH ${ARCH} CACHE STRING "CPU to build for: -march value or 'default' to not pass -march at all") +endfunction() + +if (NOT (MSVC OR ARCH)) + set_default_arch() +endif() + +CHECK_C_COMPILER_FLAG(-std=c11 HAVE_C11) + +option(COVERAGE "Enable profiling for test coverage report" OFF) +if(COVERAGE) + message(STATUS "Building with profiling for test coverage report") +endif() +macro (monero_enable_coverage) + if(COVERAGE) + foreach(COV_FLAG -fprofile-arcs -ftest-coverage --coverage) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COV_FLAG}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COV_FLAG}") + endforeach() + endif() +endmacro() + +function (monero_add_library name) + monero_add_library_with_deps(NAME "${name}" SOURCES ${ARGN}) +endfunction() + +function (monero_add_library_with_deps) + cmake_parse_arguments(MONERO_ADD_LIBRARY "" "NAME" "DEPENDS;SOURCES" ${ARGN}) + source_group("${MONERO_ADD_LIBRARY_NAME}" FILES ${MONERO_ADD_LIBRARY_SOURCES}) + + # Define a ("virtual") object library and an actual library that links those + # objects together. The virtual libraries can be arbitrarily combined to link + # any subset of objects into one library archive. This is used for releasing + # libwallet, which combines multiple components. + set(objlib obj_${MONERO_ADD_LIBRARY_NAME}) + add_library(${objlib} OBJECT ${MONERO_ADD_LIBRARY_SOURCES}) + add_library("${MONERO_ADD_LIBRARY_NAME}" $<TARGET_OBJECTS:${objlib}>) + monero_set_target_no_relink("${MONERO_ADD_LIBRARY_NAME}") + monero_set_target_strip ("${MONERO_ADD_LIBRARY_NAME}") + if (MONERO_ADD_LIBRARY_DEPENDS) + add_dependencies(${objlib} ${MONERO_ADD_LIBRARY_DEPENDS}) + endif() + set_property(TARGET "${MONERO_ADD_LIBRARY_NAME}" PROPERTY FOLDER "libs") + target_compile_definitions(${objlib} + PRIVATE $<TARGET_PROPERTY:${MONERO_ADD_LIBRARY_NAME},INTERFACE_COMPILE_DEFINITIONS>) +endfunction () + # Generate header for embedded translations # Generate header for embedded translations, use target toolchain if depends, otherwise use the # lrelease and lupdate binaries from the host @@ -522,7 +669,7 @@ ExternalProject_Add(generate_translations_header BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/translations" STAMP_DIR ${LRELEASE_PATH} CMAKE_ARGS -DLRELEASE_PATH=${LRELEASE_PATH} - INSTALL_COMMAND cmake -E echo "") + INSTALL_COMMAND ${CMAKE_COMMAND} -E echo "") include_directories("${CMAKE_CURRENT_BINARY_DIR}/translations") add_subdirectory(external) @@ -566,9 +713,6 @@ if(MSVC) include_directories(SYSTEM src/platform/msc) else() include(TestCXXAcceptsFlag) - if (NOT ARCH) - set(ARCH native CACHE STRING "CPU to build for: -march value or 'default' to not pass -march at all") - endif() message(STATUS "Building on ${CMAKE_SYSTEM_PROCESSOR} for ${ARCH}") if(ARCH STREQUAL "default") set(ARCH_FLAG "") @@ -629,8 +773,8 @@ else() set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${ARCH_FLAG}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${ARCH_FLAG}") - set(WARNINGS "-Wall -Wextra -Wpointer-arith -Wundef -Wvla -Wwrite-strings -Wno-error=extra -Wno-error=deprecated-declarations -Wno-unused-parameter -Wno-unused-variable -Wno-error=unused-variable -Wno-error=undef -Wno-error=uninitialized") - if(CMAKE_C_COMPILER_ID STREQUAL "Clang") + set(WARNINGS "-Wall -Wextra -Wpointer-arith -Wundef -Wvla -Wwrite-strings -Wno-error=extra -Wno-error=deprecated-declarations -Wno-unused-parameter -Wno-error=unused-variable -Wno-error=undef -Wno-error=uninitialized") + if(CMAKE_C_COMPILER_ID MATCHES "Clang") if(ARM) set(WARNINGS "${WARNINGS} -Wno-error=inline-asm") endif() @@ -665,13 +809,7 @@ else() set(STATIC_ASSERT_CPP_FLAG "-Dstatic_assert=_Static_assert") endif() - option(COVERAGE "Enable profiling for test coverage report" 0) - - if(COVERAGE) - message(STATUS "Building with profiling for test coverage report") - set(COVERAGE_FLAGS "-fprofile-arcs -ftest-coverage --coverage") - endif() - + monero_enable_coverage() # With GCC 6.1.1 the compiled binary malfunctions due to aliasing. Until that # is fixed in the code (Issue #847), force compiler to be conservative. set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-strict-aliasing") @@ -714,10 +852,16 @@ else() endif() # linker - if (NOT SANITIZE AND NOT OSSFUZZ AND NOT (WIN32 AND (CMAKE_C_COMPILER_ID STREQUAL "GNU" AND CMAKE_C_COMPILER_VERSION VERSION_LESS 9.1))) + if (NOT SANITIZE AND NOT OSSFUZZ AND NOT (WIN32 AND (CMAKE_C_COMPILER_ID STREQUAL "GNU" AND (CMAKE_C_COMPILER_VERSION VERSION_LESS 9.1 OR NOT STATIC)))) # PIE executables randomly crash at startup with ASAN # Windows binaries die on startup with PIE when compiled with GCC <9.x - add_linker_flag_if_supported(-pie LD_SECURITY_FLAGS) + # Windows dynamically-linked binaries die on startup with PIE regardless of GCC version + if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") + # Clang does not support -pie flag + add_linker_flag_if_supported("-Wl,-pie" LD_SECURITY_FLAGS) + else() + add_linker_flag_if_supported("-pie" LD_SECURITY_FLAGS) + endif() endif() add_linker_flag_if_supported(-Wl,-z,relro LD_SECURITY_FLAGS) add_linker_flag_if_supported(-Wl,-z,now LD_SECURITY_FLAGS) @@ -743,12 +887,19 @@ else() add_linker_flag_if_supported(-Wl,--high-entropy-va LD_SECURITY_FLAGS) endif() + # Warnings, that when ignored are so severe, that they can segfault or even UB any application. + # Treat them as errors. + add_c_flag_if_supported( -Werror=switch C_SECURITY_FLAGS) + add_cxx_flag_if_supported(-Werror=switch CXX_SECURITY_FLAGS) + add_c_flag_if_supported( -Werror=return-type C_SECURITY_FLAGS) + add_cxx_flag_if_supported(-Werror=return-type CXX_SECURITY_FLAGS) + message(STATUS "Using C security hardening flags: ${C_SECURITY_FLAGS}") message(STATUS "Using C++ security hardening flags: ${CXX_SECURITY_FLAGS}") message(STATUS "Using linker security hardening flags: ${LD_SECURITY_FLAGS}") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c11 -D_GNU_SOURCE ${MINGW_FLAG} ${STATIC_ASSERT_FLAG} ${WARNINGS} ${C_WARNINGS} ${COVERAGE_FLAGS} ${PIC_FLAG} ${C_SECURITY_FLAGS}") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -D_GNU_SOURCE ${MINGW_FLAG} ${STATIC_ASSERT_CPP_FLAG} ${WARNINGS} ${CXX_WARNINGS} ${COVERAGE_FLAGS} ${PIC_FLAG} ${CXX_SECURITY_FLAGS}") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c11 -D_GNU_SOURCE ${MINGW_FLAG} ${STATIC_ASSERT_FLAG} ${WARNINGS} ${C_WARNINGS} ${PIC_FLAG} ${C_SECURITY_FLAGS}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -D_GNU_SOURCE ${MINGW_FLAG} ${STATIC_ASSERT_CPP_FLAG} ${WARNINGS} ${CXX_WARNINGS} ${PIC_FLAG} ${CXX_SECURITY_FLAGS}") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${LD_SECURITY_FLAGS} ${LD_BACKCOMPAT_FLAGS}") # With GCC 6.1.1 the compiled binary malfunctions due to aliasing. Until that @@ -866,7 +1017,7 @@ else() endif() set(USE_LTO ${USE_LTO_DEFAULT} CACHE BOOL "Use Link-Time Optimization (Release mode only)") - if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") # There is a clang bug that does not allow to compile code that uses AES-NI intrinsics if -flto is enabled, so explicitly disable set(USE_LTO false) endif() @@ -1013,7 +1164,7 @@ if(ANDROID) set(ATOMIC libatomic.a) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=user-defined-warnings") endif() -if(CMAKE_C_COMPILER_ID STREQUAL "Clang" AND ARCH_WIDTH EQUAL "32" AND NOT IOS AND NOT FREEBSD) +if(CMAKE_C_COMPILER_ID MATCHES "Clang" AND ARCH_WIDTH EQUAL "32" AND NOT IOS AND NOT FREEBSD) find_library(ATOMIC atomic) if (ATOMIC_FOUND) list(APPEND EXTRA_LIBRARIES ${ATOMIC}) @@ -1097,9 +1248,6 @@ option(BUILD_GUI_DEPS "Build GUI dependencies." OFF) # on libunbound shipped with their distribution instead option(INSTALL_VENDORED_LIBUNBOUND "Install libunbound binary built from source vendored with this repo." OFF) - -CHECK_C_COMPILER_FLAG(-std=c11 HAVE_C11) - find_package(PythonInterp) find_program(iwyu_tool_path NAMES iwyu_tool.py iwyu_tool) if (iwyu_tool_path AND PYTHONINTERP_FOUND) diff --git a/Dockerfile b/Dockerfile index 61bbd76f2..f21f74ac3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,7 +46,7 @@ ARG BOOST_VERSION=1_70_0 ARG BOOST_VERSION_DOT=1.70.0 ARG BOOST_HASH=430ae8354789de4fd19ee52f3b1f739e1fba576f0aded0897c3c2bc00fb38778 RUN set -ex \ - && curl -s -L -o boost_${BOOST_VERSION}.tar.bz2 https://dl.bintray.com/boostorg/release/${BOOST_VERSION_DOT}/source/boost_${BOOST_VERSION}.tar.bz2 \ + && curl -s -L -o boost_${BOOST_VERSION}.tar.bz2 https://downloads.sourceforge.net/project/boost/boost/${BOOST_VERSION_DOT}/boost_${BOOST_VERSION}.tar.bz2 \ && echo "${BOOST_HASH} boost_${BOOST_VERSION}.tar.bz2" | sha256sum -c \ && tar -xvf boost_${BOOST_VERSION}.tar.bz2 \ && cd boost_${BOOST_VERSION} \ @@ -55,8 +55,8 @@ RUN set -ex \ ENV BOOST_ROOT /usr/local/boost_${BOOST_VERSION} # OpenSSL -ARG OPENSSL_VERSION=1.1.1g -ARG OPENSSL_HASH=ddb04774f1e32f0c49751e21b67216ac87852ceb056b75209af2443400636d46 +ARG OPENSSL_VERSION=1.1.1i +ARG OPENSSL_HASH=e8be6a35fe41d10603c3cc635e93289ed00bf34b79671a3a4de64fcee00d5242 RUN set -ex \ && curl -s -O https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz \ && echo "${OPENSSL_HASH} openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c \ @@ -68,7 +68,7 @@ OUTPUT_DIRECTORY = doc # performance problems for the file system. # The default value is: NO. -CREATE_SUBDIRS = NO +CREATE_SUBDIRS = YES # If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII # characters to appear in the names of generated files. If set to NO, non-ASCII @@ -754,7 +754,7 @@ WARN_LOGFILE = # spaces. # Note: If this tag is empty the current directory is searched. -INPUT = src +INPUT = . # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses @@ -1552,7 +1552,7 @@ EXTRA_SEARCH_MAPPINGS = # If the GENERATE_LATEX tag is set to YES doxygen will generate LaTeX output. # The default value is: YES. -GENERATE_LATEX = YES +GENERATE_LATEX = NO # The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. If a # relative path is entered the value of OUTPUT_DIRECTORY will be put in front of @@ -1,6 +1,6 @@ # Monero -Copyright (c) 2014-2020 The Monero Project. +Copyright (c) 2014-2021 The Monero Project. Portions Copyright (c) 2012-2013 The Cryptonote developers. ## Table of Contents @@ -32,7 +32,7 @@ Portions Copyright (c) 2012-2013 The Cryptonote developers. - Forum: [forum.getmonero.org](https://forum.getmonero.org) - Mail: [dev@getmonero.org](mailto:dev@getmonero.org) - GitHub: [https://github.com/monero-project/monero](https://github.com/monero-project/monero) -- IRC: [#monero-dev on Freenode](https://webchat.freenode.net/?randomnick=1&channels=%23monero-dev&prompt=1&uio=d4) +- IRC: [#monero-dev on Libera](https://web.libera.chat/#monero-dev) - It is HIGHLY recommended that you join the #monero-dev IRC channel if you are developing software that uses Monero. Due to the nature of this open source software project, joining this channel and idling is the best way to stay updated on best practices and new developments in the Monero ecosystem. All you need to do is join the IRC channel and idle to stay updated with the latest in Monero development. If you do not, you risk wasting resources on developing integrations that are not compatible with the Monero network. The Monero core team and community continuously make efforts to communicate updates, developments, and documentation via other platforms – but for the best information, you need to talk to other Monero developers, and they are on IRC. #monero-dev is about Monero development, not getting help about using Monero, or help about development of other software, including yours, unless it also pertains to Monero code itself. For these cases, checkout #monero. ## Vulnerability response @@ -44,7 +44,7 @@ Portions Copyright (c) 2012-2013 The Cryptonote developers. The [Monero Research Lab](https://src.getmonero.org/resources/research-lab/) is an open forum where the community coordinates research into Monero cryptography, protocols, fungibility, analysis, and more. We welcome collaboration and contributions from outside researchers! Because not all Lab work and publications are distributed as traditional preprints or articles, they may be easy to miss if you are conducting literature reviews for your own Monero research. You are encouraged to get in touch with our researchers if you have questions, wish to collaborate, or would like guidance to help avoid unnecessarily duplicating earlier or known work. -Our researchers are available on IRC in [#monero-research-lab on Freenode](https://webchat.freenode.net/?randomnick=1&channels=%23monero-research-lab&prompt=1&uio=d4) or by email: +Our researchers are available on IRC in [#monero-research-lab on Libera](https://web.libera.chat/#monero-research-lab) or by email: - Sarang Noether, Ph.D.: [sarang@getmonero.org](mailto:sarang@getmonero.org) or [sarang.noether@protonmail.com](mailto:sarang.noether@protonmail.com); [research repository](https://github.com/SarangNoether/research-lab) - Surae Noether (Brandon Goodell), Ph.D.: [surae@getmonero.org](mailto:surae@getmonero.org) or [surae.noether@protonmail.com](mailto:surae.noether@protonmail.com); [research repository](https://github.com/b-g-goodell/research-lab) @@ -92,7 +92,7 @@ As with many development projects, the repository on Github is considered to be Monero is a 100% community-sponsored endeavor. If you want to join our efforts, the easiest thing you can do is support the project financially. Both Monero and Bitcoin donations can be made to **donate.getmonero.org** if using a client that supports the [OpenAlias](https://openalias.org) standard. Alternatively, you can send XMR to the Monero donation address via the `donate` command (type `help` in the command-line wallet for details). -The Monero donation address is: `888tNkZrPN6JsEgekjMnABU4TBzc2Dt29EPAvkRxbANsAnjyPbb3iQ1YBRk1UXcdRsiKc9dhwMVgN5S9cQUiyoogDavup3H` (viewkey: `f359631075708155cc3d92a32b75a7d02a5dcf27756707b47a2b31b21c389501`) +The Monero donation address is: `888tNkZrPN6JsEgekjMnABU4TBzc2Dt29EPAvkRxbANsAnjyPbb3iQ1YBRk1UXcdRsiKc9dhwMVgN5S9cQUiyoogDavup3H` (viewkey: `f359631075708155cc3d92a32b75a7d02a5dcf27756707b47a2b31b21c389501`; base address for restoring with address and viewkey: `44AFFq5kSiGBoZ4NMDwYtN18obc8AemS33DBLWs3H7otXft3XjrpDtQGv7SqSsaBYBb98uNbr2VBBEt7f2wfn3RVGQBEP3A`) The Bitcoin donation address is: `1KTexdemPdxSBcG55heUuTjDRYqbC5ZL8H` @@ -165,7 +165,7 @@ library archives (`.a`). | pkg-config | any | NO | `pkg-config` | `base-devel` | `base-devel` | `pkgconf` | NO | | | Boost | 1.58 | NO | `libboost-all-dev` | `boost` | `boost-devel` | `boost-devel` | NO | C++ libraries | | OpenSSL | basically any | NO | `libssl-dev` | `openssl` | `libressl-devel` | `openssl-devel` | NO | sha256 sum | -| libzmq | 3.0.0 | NO | `libzmq3-dev` | `zeromq` | `zeromq-devel` | `zeromq-devel` | NO | ZeroMQ library | +| libzmq | 4.2.0 | NO | `libzmq3-dev` | `zeromq` | `zeromq-devel` | `zeromq-devel` | NO | ZeroMQ library | | OpenPGM | ? | NO | `libpgm-dev` | `libpgm` | | `openpgm-devel` | NO | For ZeroMQ | | libnorm[2] | ? | NO | `libnorm-dev` | | | | YES | For ZeroMQ | | libunbound | 1.4.16 | YES | `libunbound-dev` | `unbound` | `unbound-devel` | `unbound-devel` | NO | DNS resolver | @@ -176,6 +176,7 @@ library archives (`.a`). | ldns | 1.6.17 | NO | `libldns-dev` | `ldns` | `libldns-devel` | `ldns-devel` | YES | SSL toolkit | | expat | 1.1 | NO | `libexpat1-dev` | `expat` | `expat-devel` | `expat-devel` | YES | XML parsing | | GTest | 1.5 | YES | `libgtest-dev`[1] | `gtest` | `gtest-devel` | `gtest-devel` | YES | Test suite | +| ccache | any | NO | `ccache` | `ccache` | `ccache` | `ccache` | YES | Compil. cache | | Doxygen | any | NO | `doxygen` | `doxygen` | `doxygen` | `doxygen` | YES | Documentation | | Graphviz | any | NO | `graphviz` | `graphviz` | `graphviz` | `graphviz` | YES | Documentation | | lrelease | ? | NO | `qttools5-dev-tools` | `qt5-tools` | `qt5-tools` | `qt5-linguist` | YES | Translations | @@ -183,7 +184,7 @@ library archives (`.a`). | libusb | ? | NO | `libusb-1.0-0-dev` | `libusb` | `libusb-devel` | `libusbx-devel` | YES | Hardware wallet | | libprotobuf | ? | NO | `libprotobuf-dev` | `protobuf` | `protobuf-devel` | `protobuf-devel` | YES | Hardware wallet | | protoc | ? | NO | `protobuf-compiler` | `protobuf` | `protobuf` | `protobuf-compiler` | YES | Hardware wallet | -| libudev | ? | No | `libudev-dev` | `systemd` | `eudev-libudev-devel` | `systemd-devel` | YES | Hardware wallet | +| libudev | ? | No | `libudev-dev` | `systemd` | `eudev-libudev-devel` | `systemd-devel` | YES | Hardware wallet | [1] On Debian/Ubuntu `libgtest-dev` only includes sources and headers. You must build the library binary manually. This can be done with the following command ```sudo apt-get install libgtest-dev && cd /usr/src/gtest && sudo cmake . && sudo make && sudo mv libg* /usr/lib/ ``` @@ -191,7 +192,11 @@ build the library binary manually. This can be done with the following command ` Install all dependencies at once on Debian/Ubuntu: -``` sudo apt update && sudo apt install build-essential cmake pkg-config libboost-all-dev libssl-dev libzmq3-dev libunbound-dev libsodium-dev libunwind8-dev liblzma-dev libreadline6-dev libldns-dev libexpat1-dev doxygen graphviz libpgm-dev qttools5-dev-tools libhidapi-dev libusb-1.0-0-dev libprotobuf-dev protobuf-compiler libudev-dev``` +``` sudo apt update && sudo apt install build-essential cmake pkg-config libssl-dev libzmq3-dev libunbound-dev libsodium-dev libunwind8-dev liblzma-dev libreadline6-dev libldns-dev libexpat1-dev libpgm-dev qttools5-dev-tools libhidapi-dev libusb-1.0-0-dev libprotobuf-dev protobuf-compiler libudev-dev libboost-chrono-dev libboost-date-time-dev libboost-filesystem-dev libboost-locale-dev libboost-program-options-dev libboost-regex-dev libboost-serialization-dev libboost-system-dev libboost-thread-dev ccache doxygen graphviz ``` + +Install all dependencies at once on openSUSE: + +``` sudo zypper ref && sudo zypper in cppzmq-devel ldns-devel libboost_chrono-devel libboost_date_time-devel libboost_filesystem-devel libboost_locale-devel libboost_program_options-devel libboost_regex-devel libboost_serialization-devel libboost_system-devel libboost_thread-devel libexpat-devel libminiupnpc-devel libsodium-devel libunwind-devel unbound-devel cmake doxygen ccache fdupes gcc-c++ libevent-devel libopenssl-devel pkgconf-pkg-config readline-devel xz-devel libqt5-qttools-devel patterns-devel-C-C++-devel_C_C++``` Install all dependencies at once on macOS with the provided Brewfile: ``` brew update && brew bundle --file=contrib/brew/Brewfile ``` @@ -209,6 +214,10 @@ If you already have a repo cloned, initialize and update: `$ cd monero && git submodule init && git submodule update` +*Note*: If there are submodule differences between branches, you may need +to use ```git submodule sync && git submodule update``` after changing branches +to build successfully. + ### Build instructions Monero uses the CMake build system and a top-level [Makefile](Makefile) that @@ -270,6 +279,12 @@ Dependencies need to be built with -fPIC. Static libraries usually aren't, so yo HAVE_DOT=YES doxygen Doxyfile ``` +* **Optional**: use ccache not to rebuild translation units, that haven't really changed. Monero's CMakeLists.txt file automatically handles it + + ```bash + sudo apt install ccache + ``` + #### On the Raspberry Pi Tested on a Raspberry Pi Zero with a clean install of minimal Raspbian Stretch (2017-09-07 or later) from https://www.raspberrypi.org/downloads/raspbian/. If you are using Raspian Jessie, [please see note in the following section](#note-for-raspbian-jessie-users). @@ -300,14 +315,16 @@ Tested on a Raspberry Pi Zero with a clean install of minimal Raspbian Stretch ( * Build: ```bash - make release + USE_SINGLE_BUILDDIR=1 make release ``` * Wait 4-6 hours * The resulting executables can be found in `build/release/bin` -* Add `PATH="$PATH:$HOME/monero/build/release/bin"` to `.profile` +* Add `export PATH="$PATH:$HOME/monero/build/release/bin"` to `$HOME/.profile` + +* Run `source $HOME/.profile` * Run Monero with `monerod --detach` @@ -491,19 +508,6 @@ cd ../.. Then you can run make as usual. -### On Linux for Android (using docker): - -```bash -# Build image (for ARM 32-bit) -docker build -f utils/build_scripts/android32.Dockerfile -t monero-android . -# Build image (for ARM 64-bit) -docker build -f utils/build_scripts/android64.Dockerfile -t monero-android . -# Create container -docker create -it --name monero-android monero-android bash -# Get binaries -docker cp monero-android:/src/build/release/bin . -``` - ### Building portable statically linked binaries By default, in either dynamically or statically linked builds, binaries target the specific host processor on which the build happens and are not portable to other processors. Portable binaries can be built using the following targets: @@ -704,14 +708,16 @@ DNS_PUBLIC=tcp torsocks ./monerod --p2p-bind-ip 127.0.0.1 --no-igd --rpc-bind-ip ## Pruning -As of May 2020, the full Monero blockchain file is about 80 GB. One can store a pruned blockchain, which is about 28 GB. +As of May 2020, the full Monero blockchain file is about 100 GB. One can store a pruned blockchain, which is about 30 GB. A pruned blockchain can only serve part of the historical chain data to other peers, but is otherwise identical in functionality to the full blockchain. -To use a pruned blockchain, it is best to start the initial sync with --prune-blockchain. However, it is also possible -to prune an existing blockchain using the monero-blockchain-prune tool or using the --prune-blockchain monerod option +To use a pruned blockchain, it is best to start the initial sync with `--prune-blockchain`. However, it is also possible +to prune an existing blockchain using the `monero-blockchain-prune` tool or using the `--prune-blockchain` `monerod` option with an existing chain. If an existing chain exists, pruning will temporarily require disk space to store both the full and pruned blockchains. +For more detailed information see the ['Pruning' entry in the Moneropedia](https://www.getmonero.org/resources/moneropedia/pruning.html) + ## Debugging This section contains general instructions for debugging failed installs or problems encountered with Monero. First, ensure you are running the latest version built from the Github repo. diff --git a/cmake/CheckLinkerFlag.cmake b/cmake/CheckLinkerFlag.cmake index 2b507ab71..7ecf5f610 100644 --- a/cmake/CheckLinkerFlag.cmake +++ b/cmake/CheckLinkerFlag.cmake @@ -15,6 +15,7 @@ macro(CHECK_LINKER_FLAG flag VARIABLE) ${_cle_source} COMPILE_DEFINITIONS ${CMAKE_REQUIRED_DEFINITIONS} ${flag} CMAKE_FLAGS + "-DCMAKE_EXE_LINKER_FLAGS=${flag}" OUTPUT_VARIABLE OUTPUT) unset(_cle_source) set(CMAKE_C_FLAGS ${saved_CMAKE_C_FLAGS}) diff --git a/cmake/FindCcache.cmake b/cmake/FindCcache.cmake index a0734a312..d98518134 100644 --- a/cmake/FindCcache.cmake +++ b/cmake/FindCcache.cmake @@ -42,12 +42,19 @@ find_program(CCACHE_FOUND ccache) if (CCACHE_FOUND) # Try to compile a test program with ccache, in order to verify if it really works. (needed on exotic setups) - # Create a temporary file with a simple program. - set(TEMP_CPP_FILE "${CMAKE_BINARY_DIR}/${CMAKE_FILES_DIRECTORY}/CMakeTmp/test-program.cpp") - file(WRITE "${TEMP_CPP_FILE}" "int main() { return 0; }") - # And run the found ccache on it. - execute_process(COMMAND "${CCACHE_FOUND}" "${CMAKE_CXX_COMPILER}" "${TEMP_CPP_FILE}" RESULT_VARIABLE RET) - if (${RET} EQUAL 0) + set(TEST_PROJECT "${CMAKE_BINARY_DIR}/${CMAKE_FILES_DIRECTORY}/CMakeTmp") + file(WRITE "${TEST_PROJECT}/CMakeLists.txt" [=[ +cmake_minimum_required(VERSION 3.5) +project(test) +option (CCACHE "") +file(WRITE "${CMAKE_SOURCE_DIR}/test.cpp" "int main() { return 0; }") +set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${CCACHE}") +set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK "${CCACHE}") +add_executable(main test.cpp) +]=]) + try_compile(RET "${TEST_PROJECT}/build" "${TEST_PROJECT}" "test" CMAKE_FLAGS -DCCACHE="${CCACHE_FOUND}") + unset(TEST_PROJECT) + if (${RET}) # Success message(STATUS "Found usable ccache: ${CCACHE_FOUND}") set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${CCACHE_FOUND}") diff --git a/cmake/FindHIDAPI.cmake b/cmake/FindHIDAPI.cmake index 9b45dcc4c..f56bab482 100644 --- a/cmake/FindHIDAPI.cmake +++ b/cmake/FindHIDAPI.cmake @@ -39,19 +39,28 @@ find_package_handle_standard_args(HIDAPI if(HIDAPI_FOUND) set(HIDAPI_LIBRARIES "${HIDAPI_LIBRARY}") - if((STATIC AND UNIX AND NOT APPLE) OR (DEPENDS AND CMAKE_SYSTEM_NAME STREQUAL "Linux")) + if((STATIC AND UNIX AND NOT APPLE) OR (DEPENDS AND CMAKE_SYSTEM_NAME STREQUAL "Linux") OR ANDROID) find_library(LIBUSB-1.0_LIBRARY usb-1.0) find_library(LIBUDEV_LIBRARY udev) if(LIBUSB-1.0_LIBRARY) set(HIDAPI_LIBRARIES "${HIDAPI_LIBRARIES};${LIBUSB-1.0_LIBRARY}") if(LIBUDEV_LIBRARY) set(HIDAPI_LIBRARIES "${HIDAPI_LIBRARIES};${LIBUDEV_LIBRARY}") - else() + elseif(NOT ANDROID) message(WARNING "libudev library not found, binaries may fail to link.") endif() else() message(WARNING "libusb-1.0 library not found, binaries may fail to link.") endif() + if(ANDROID) + # libusb uses android log library + find_library(ANDROID_LOG_LIBRARY log) + if(ANDROID_LOG_LIBRARY) + set(HIDAPI_LIBRARIES "${HIDAPI_LIBRARIES};${ANDROID_LOG_LIBRARY}") + else() + message(WARNING "Android log library not found, binaries may fail to link.") + endif() + endif() endif() set(HIDAPI_INCLUDE_DIRS "${HIDAPI_INCLUDE_DIR}") diff --git a/cmake/FindLibUSB.cmake b/cmake/FindLibUSB.cmake index 79d8fba75..7f8a11460 100644 --- a/cmake/FindLibUSB.cmake +++ b/cmake/FindLibUSB.cmake @@ -99,7 +99,7 @@ if ( LibUSB_FOUND ) check_library_exists ( "${LibUSB_LIBRARIES}" libusb_get_device_list "" LibUSB_VERSION_1.0 ) check_library_exists ( "${LibUSB_LIBRARIES}" libusb_get_port_numbers "" LibUSB_VERSION_1.0.16 ) - if((STATIC AND UNIX AND NOT APPLE) OR (DEPENDS AND CMAKE_SYSTEM_NAME STREQUAL "Linux")) + if((STATIC AND UNIX AND NOT APPLE) OR (DEPENDS AND CMAKE_SYSTEM_NAME STREQUAL "Linux") OR ANDROID) find_library(LIBUDEV_LIBRARY udev) if(LIBUDEV_LIBRARY) set(LibUSB_LIBRARIES "${LibUSB_LIBRARIES};${LIBUDEV_LIBRARY}") @@ -134,7 +134,7 @@ if ( LibUSB_FOUND ) try_compile(LibUSB_COMPILE_TEST_PASSED ${CMAKE_BINARY_DIR} - "${CMAKE_SOURCE_DIR}/cmake/test-libusb-version.c" + "${CMAKE_CURRENT_LIST_DIR}/test-libusb-version.c" CMAKE_FLAGS "-DINCLUDE_DIRECTORIES=${LibUSB_INCLUDE_DIRS}" "-DLINK_DIRECTORIES=${LibUSB_LIBRARIES}" diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 511f0416e..046115bd3 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -26,5 +26,6 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +monero_enable_coverage() add_subdirectory(epee) diff --git a/contrib/brew/Brewfile b/contrib/brew/Brewfile index 1fdf45cfe..36b073b25 100644 --- a/contrib/brew/Brewfile +++ b/contrib/brew/Brewfile @@ -27,6 +27,7 @@ brew "miniupnpc" brew "readline" brew "ldns" brew "expat" +brew "ccache" brew "doxygen" brew "graphviz" brew "libunwind-headers" diff --git a/contrib/depends/Makefile b/contrib/depends/Makefile index 28ec972e4..9ffe8a106 100644 --- a/contrib/depends/Makefile +++ b/contrib/depends/Makefile @@ -3,15 +3,15 @@ SOURCES_PATH ?= $(BASEDIR)/sources BASE_CACHE ?= $(BASEDIR)/built SDK_PATH ?= $(BASEDIR)/SDKs -FALLBACK_DOWNLOAD_PATH ?= https://bitcoincore.org/depends-sources +FALLBACK_DOWNLOAD_PATH ?= https://downloads.getmonero.org/depends-sources BUILD = $(shell ./config.guess) HOST ?= $(BUILD) PATCHES_PATH = $(BASEDIR)/patches BASEDIR = $(CURDIR) HASH_LENGTH:=11 -DOWNLOAD_CONNECT_TIMEOUT:=10 -DOWNLOAD_RETRIES:=3 +DOWNLOAD_CONNECT_TIMEOUT:=30 +DOWNLOAD_RETRIES:=5 HOST_ID_SALT ?= salt BUILD_ID_SALT ?= salt diff --git a/contrib/depends/funcs.mk b/contrib/depends/funcs.mk index 2d6b37190..ffda2b5b2 100644 --- a/contrib/depends/funcs.mk +++ b/contrib/depends/funcs.mk @@ -32,7 +32,7 @@ endef define fetch_file ( test -f $$($(1)_source_dir)/$(4) || \ ( $(call fetch_file_inner,$(1),$(2),$(3),$(4),$(5)) || \ - $(call fetch_file_inner,$(1),$(FALLBACK_DOWNLOAD_PATH),$(3),$(4),$(5)))) + $(call fetch_file_inner,$(1),$(FALLBACK_DOWNLOAD_PATH),$(4),$(4),$(5)))) endef define int_get_build_recipe_hash diff --git a/contrib/depends/packages/boost.mk b/contrib/depends/packages/boost.mk index 0d241928e..4571d4232 100644 --- a/contrib/depends/packages/boost.mk +++ b/contrib/depends/packages/boost.mk @@ -1,6 +1,6 @@ package=boost $(package)_version=1_64_0 -$(package)_download_path=https://dl.bintray.com/boostorg/release/1.64.0/source/ +$(package)_download_path=https://downloads.sourceforge.net/project/boost/boost/1.64.0/ $(package)_file_name=$(package)_$($(package)_version).tar.bz2 $(package)_sha256_hash=7bcc5caace97baa948931d712ea5f37038dbb1c5d89b43ad4def4ed7cb683332 $(package)_dependencies=libiconv diff --git a/contrib/depends/packages/eudev.mk b/contrib/depends/packages/eudev.mk index 0e930df93..6b6887e98 100644 --- a/contrib/depends/packages/eudev.mk +++ b/contrib/depends/packages/eudev.mk @@ -1,7 +1,8 @@ package=eudev $(package)_version=v3.2.6 $(package)_download_path=https://github.com/gentoo/eudev/archive/ -$(package)_file_name=$($(package)_version).tar.gz +$(package)_download_file=$($(package)_version).tar.gz +$(package)_file_name=$(package)-$($(package)_version).tar.gz $(package)_sha256_hash=a96ecb8637667897b8bd4dee4c22c7c5f08b327be45186e912ce6bc768385852 define $(package)_set_vars diff --git a/contrib/depends/packages/freebsd_base.mk b/contrib/depends/packages/freebsd_base.mk index c6a209dcd..ad9975f8d 100644 --- a/contrib/depends/packages/freebsd_base.mk +++ b/contrib/depends/packages/freebsd_base.mk @@ -12,8 +12,8 @@ endef define $(package)_build_cmds mkdir bin &&\ - echo "exec /usr/bin/clang-8 -target x86_64-unknown-freebsd$($(package)_version) --sysroot=$(host_prefix)/native $$$$""@" > bin/clang-8 &&\ - echo "exec /usr/bin/clang++-8 -target x86_64-unknown-freebsd$($(package)_version) --sysroot=$(host_prefix)/native $$$$""@" > bin/clang++-8 &&\ + echo "#!/bin/sh\n\nexec /usr/bin/clang-8 -target x86_64-unknown-freebsd$($(package)_version) --sysroot=$(host_prefix)/native $$$$""@" > bin/clang-8 &&\ + echo "#!/bin/sh\n\nexec /usr/bin/clang++-8 -target x86_64-unknown-freebsd$($(package)_version) --sysroot=$(host_prefix)/native $$$$""@" > bin/clang++-8 &&\ chmod 755 bin/* endef diff --git a/contrib/depends/packages/gtest.mk b/contrib/depends/packages/gtest.mk index 5df07a32e..1208d7c03 100644 --- a/contrib/depends/packages/gtest.mk +++ b/contrib/depends/packages/gtest.mk @@ -1,7 +1,8 @@ package=gtest $(package)_version=1.8.1 $(package)_download_path=https://github.com/google/googletest/archive/ -$(package)_file_name=release-$($(package)_version).tar.gz +$(package)_download_file=release-$($(package)_version).tar.gz +$(package)_file_name=$(package)-$($(package)_version).tar.gz $(package)_sha256_hash=9bf1fe5182a604b4135edc1a425ae356c9ad15e9b23f9f12a02e80184c3a249c $(package)_cxxflags=-std=c++11 $(package)_cxxflags_linux=-fPIC diff --git a/contrib/depends/packages/native_cctools.mk b/contrib/depends/packages/native_cctools.mk index 8bf71254f..64a72e484 100644 --- a/contrib/depends/packages/native_cctools.mk +++ b/contrib/depends/packages/native_cctools.mk @@ -1,7 +1,8 @@ package=native_cctools $(package)_version=807d6fd1be5d2224872e381870c0a75387fe05e6 $(package)_download_path=https://github.com/theuni/cctools-port/archive -$(package)_file_name=$($(package)_version).tar.gz +$(package)_download_file=$($(package)_version).tar.gz +$(package)_file_name=$(package)-$($(package)_version).tar.gz $(package)_sha256_hash=a09c9ba4684670a0375e42d9d67e7f12c1f62581a27f28f7c825d6d7032ccc6a $(package)_build_subdir=cctools $(package)_clang_version=3.7.1 diff --git a/contrib/depends/packages/openssl.mk b/contrib/depends/packages/openssl.mk index 62e975e50..d80775b22 100644 --- a/contrib/depends/packages/openssl.mk +++ b/contrib/depends/packages/openssl.mk @@ -1,8 +1,8 @@ package=openssl -$(package)_version=1.1.1i +$(package)_version=1.1.1k $(package)_download_path=https://www.openssl.org/source $(package)_file_name=$(package)-$($(package)_version).tar.gz -$(package)_sha256_hash=e8be6a35fe41d10603c3cc635e93289ed00bf34b79671a3a4de64fcee00d5242 +$(package)_sha256_hash=892a0875b9872acd04a9fde79b1f943075d5ea162415de3047c327df33fbaee5 define $(package)_set_vars $(package)_config_env=AR="$($(package)_ar)" ARFLAGS=$($(package)_arflags) RANLIB="$($(package)_ranlib)" CC="$($(package)_cc)" diff --git a/contrib/epee/demo/CMakeLists.txt b/contrib/epee/demo/CMakeLists.txt index b4ac2cc8b..d2ae0ed55 100644 --- a/contrib/epee/demo/CMakeLists.txt +++ b/contrib/epee/demo/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.8) +cmake_minimum_required(VERSION 3.5) set(Boost_USE_MULTITHREADED ON) #set(Boost_DEBUG 1) find_package(Boost COMPONENTS system filesystem thread date_time chrono regex ) diff --git a/contrib/epee/include/byte_slice.h b/contrib/epee/include/byte_slice.h index 6b79f6d92..18d60e088 100644 --- a/contrib/epee/include/byte_slice.h +++ b/contrib/epee/include/byte_slice.h @@ -112,7 +112,7 @@ namespace epee explicit byte_slice(std::string&& buffer); //! Convert `stream` into a slice with zero allocations. - explicit byte_slice(byte_stream&& stream) noexcept; + explicit byte_slice(byte_stream&& stream, bool shrink = true); byte_slice(byte_slice&& source) noexcept; ~byte_slice() noexcept = default; diff --git a/contrib/epee/include/file_io_utils.h b/contrib/epee/include/file_io_utils.h index 25f8c648b..84dc79266 100644 --- a/contrib/epee/include/file_io_utils.h +++ b/contrib/epee/include/file_io_utils.h @@ -24,211 +24,23 @@ // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // - #ifndef _FILE_IO_UTILS_H_ #define _FILE_IO_UTILS_H_ -#include <fstream> -#include <boost/filesystem/path.hpp> -#include <boost/filesystem/operations.hpp> -#ifdef WIN32 -#include <windows.h> -#include "string_tools.h" -#endif - -// On Windows there is a problem with non-ASCII characters in path and file names -// as far as support by the standard components used is concerned: - -// The various file stream classes, e.g. std::ifstream and std::ofstream, are -// part of the GNU C++ Library / libstdc++. On the most basic level they use the -// fopen() call as defined / made accessible to programs compiled within MSYS2 -// by the stdio.h header file maintained by the MinGW project. - -// The critical point: The implementation of fopen() is part of MSVCRT, the -// Microsoft Visual C/C++ Runtime Library, and this method does NOT offer any -// Unicode support. - -// Monero code that would want to continue to use the normal file stream classes -// but WITH Unicode support could therefore not solve this problem on its own, -// but 2 different projects from 2 different maintaining groups would need changes -// in this particular direction - something probably difficult to achieve and -// with a long time to wait until all new versions / releases arrive. - -// Implemented solution approach: Circumvent the problem by stopping to use std -// file stream classes on Windows and directly use Unicode-capable WIN32 API -// calls. Most of the code doing so is concentrated in this header file here. +#include <string> +#include <ctime> namespace epee { namespace file_io_utils { - inline - bool is_file_exist(const std::string& path) - { - boost::filesystem::path p(path); - return boost::filesystem::exists(p); - } - - inline - bool save_string_to_file(const std::string& path_to_file, const std::string& str) - { -#ifdef WIN32 - std::wstring wide_path; - try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; } - HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); - if (file_handle == INVALID_HANDLE_VALUE) - return false; - DWORD bytes_written; - DWORD bytes_to_write = (DWORD)str.size(); - BOOL result = WriteFile(file_handle, str.data(), bytes_to_write, &bytes_written, NULL); - CloseHandle(file_handle); - if (bytes_written != bytes_to_write) - result = FALSE; - return result; -#else - try - { - std::ofstream fstream; - fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); - fstream.open(path_to_file, std::ios_base::binary | std::ios_base::out | std::ios_base::trunc); - fstream << str; - fstream.close(); - return true; - } - - catch(...) - { - return false; - } -#endif - } - - inline - bool get_file_time(const std::string& path_to_file, time_t& ft) - { - boost::system::error_code ec; - ft = boost::filesystem::last_write_time(boost::filesystem::path(path_to_file), ec); - if(!ec) - return true; - else - return false; - } - - inline - bool set_file_time(const std::string& path_to_file, const time_t& ft) - { - boost::system::error_code ec; - boost::filesystem::last_write_time(boost::filesystem::path(path_to_file), ft, ec); - if(!ec) - return true; - else - return false; - } - - - inline - bool load_file_to_string(const std::string& path_to_file, std::string& target_str, size_t max_size = 1000000000) - { -#ifdef WIN32 - std::wstring wide_path; - try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; } - HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); - if (file_handle == INVALID_HANDLE_VALUE) - return false; - DWORD file_size = GetFileSize(file_handle, NULL); - if ((file_size == INVALID_FILE_SIZE) || (uint64_t)file_size > (uint64_t)max_size) { - CloseHandle(file_handle); - return false; - } - target_str.resize(file_size); - DWORD bytes_read; - BOOL result = ReadFile(file_handle, &target_str[0], file_size, &bytes_read, NULL); - CloseHandle(file_handle); - if (bytes_read != file_size) - result = FALSE; - return result; -#else - try - { - std::ifstream fstream; - fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); - fstream.open(path_to_file, std::ios_base::binary | std::ios_base::in | std::ios::ate); - - std::ifstream::pos_type file_size = fstream.tellg(); - - if((uint64_t)file_size > (uint64_t)max_size) // ensure a large domain for comparison, and negative -> too large - return false;//don't go crazy - size_t file_size_t = static_cast<size_t>(file_size); - - target_str.resize(file_size_t); - - fstream.seekg (0, std::ios::beg); - fstream.read((char*)target_str.data(), target_str.size()); - fstream.close(); - return true; - } - - catch(...) - { - return false; - } -#endif - } - - inline - bool append_string_to_file(const std::string& path_to_file, const std::string& str) - { - // No special Windows implementation because so far not used in Monero code - try - { - std::ofstream fstream; - fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); - fstream.open(path_to_file.c_str(), std::ios_base::binary | std::ios_base::out | std::ios_base::app); - fstream << str; - fstream.close(); - return true; - } - - catch(...) - { - return false; - } - } - - inline - bool get_file_size(const std::string& path_to_file, uint64_t &size) - { -#ifdef WIN32 - std::wstring wide_path; - try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; } - HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); - if (file_handle == INVALID_HANDLE_VALUE) - return false; - LARGE_INTEGER file_size; - BOOL result = GetFileSizeEx(file_handle, &file_size); - CloseHandle(file_handle); - if (result) { - size = file_size.QuadPart; - } - return size; -#else - try - { - std::ifstream fstream; - fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); - fstream.open(path_to_file, std::ios_base::binary | std::ios_base::in | std::ios::ate); - size = fstream.tellg(); - fstream.close(); - return true; - } - - catch(...) - { - return false; - } -#endif - } - + bool is_file_exist(const std::string& path); + bool save_string_to_file(const std::string& path_to_file, const std::string& str); + bool get_file_time(const std::string& path_to_file, time_t& ft); + bool set_file_time(const std::string& path_to_file, const time_t& ft); + bool load_file_to_string(const std::string& path_to_file, std::string& target_str, size_t max_size = 1000000000); + bool append_string_to_file(const std::string& path_to_file, const std::string& str); + bool get_file_size(const std::string& path_to_file, uint64_t &size); } } diff --git a/contrib/epee/include/memwipe.h b/contrib/epee/include/memwipe.h index c6e8f072c..c6f247913 100644 --- a/contrib/epee/include/memwipe.h +++ b/contrib/epee/include/memwipe.h @@ -32,6 +32,7 @@ #ifdef __cplusplus #include <array> +#include <cstddef> extern "C" { #endif diff --git a/contrib/epee/include/misc_language.h b/contrib/epee/include/misc_language.h index a04c63231..5ccfe6fcc 100644 --- a/contrib/epee/include/misc_language.h +++ b/contrib/epee/include/misc_language.h @@ -28,9 +28,11 @@ #pragma once -#include <limits> -#include <boost/thread.hpp> #include <boost/utility/value_init.hpp> +#include <boost/shared_ptr.hpp> +#include <limits> +#include <functional> +#include <vector> namespace epee { #define STD_TRY_BEGIN() try { @@ -95,16 +97,7 @@ namespace misc_utils return memcmp(&_Left, &_Right, sizeof(_Left)) < 0; } - - inline - bool sleep_no_w(long ms ) - { - boost::this_thread::sleep( - boost::get_system_time() + - boost::posix_time::milliseconds( std::max<long>(ms,0) ) ); - - return true; - } + bool sleep_no_w(long ms ); template <typename T> T get_mid(const T &a, const T &b) diff --git a/contrib/epee/include/misc_log_ex.h b/contrib/epee/include/misc_log_ex.h index 3be335e85..392ec11eb 100644 --- a/contrib/epee/include/misc_log_ex.h +++ b/contrib/epee/include/misc_log_ex.h @@ -41,7 +41,7 @@ #define MAX_LOG_FILES 50 #define MCLOG_TYPE(level, cat, color, type, x) do { \ - if (ELPP->vRegistry()->allowed(level, cat)) { \ + if (el::Loggers::allowed(level, cat)) { \ el::base::Writer(level, color, __FILE__, __LINE__, ELPP_FUNC, type).construct(cat) << x; \ } \ } while (0) @@ -89,7 +89,7 @@ #define IFLOG(level, cat, color, type, init, x) \ do { \ - if (ELPP->vRegistry()->allowed(level, cat)) { \ + if (el::Loggers::allowed(level, cat)) { \ init; \ el::base::Writer(level, color, __FILE__, __LINE__, ELPP_FUNC, type).construct(cat) << x; \ } \ diff --git a/contrib/epee/include/misc_os_dependent.h b/contrib/epee/include/misc_os_dependent.h index 5fffde8d5..522cdf263 100644 --- a/contrib/epee/include/misc_os_dependent.h +++ b/contrib/epee/include/misc_os_dependent.h @@ -47,7 +47,7 @@ #endif #include <iostream> -#include <boost/lexical_cast.hpp> +#include <ctime> #pragma once namespace epee @@ -115,15 +115,7 @@ namespace misc_utils } - - inline std::string get_thread_string_id() - { -#if defined(_WIN32) - return boost::lexical_cast<std::string>(GetCurrentThreadId()); -#elif defined(__GNUC__) - return boost::lexical_cast<std::string>(pthread_self()); -#endif - } + std::string get_thread_string_id(); inline bool get_gmt_time(time_t t, struct tm &tm) { diff --git a/contrib/epee/include/net/abstract_tcp_server2.h b/contrib/epee/include/net/abstract_tcp_server2.h index 3c31cf22b..f40cd108a 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.h +++ b/contrib/epee/include/net/abstract_tcp_server2.h @@ -265,6 +265,12 @@ namespace net_utils template<class t_callback> bool connect_async(const std::string& adr, const std::string& port, uint32_t conn_timeot, const t_callback &cb, const std::string& bind_ip = "0.0.0.0", epee::net_utils::ssl_support_t ssl_support = epee::net_utils::ssl_support_t::e_ssl_support_autodetect); + boost::asio::ssl::context& get_ssl_context() noexcept + { + assert(m_state != nullptr); + return m_state->ssl_context; + } + typename t_protocol_handler::config_type& get_config_object() { assert(m_state != nullptr); // always set in constructor diff --git a/contrib/epee/include/net/abstract_tcp_server2.inl b/contrib/epee/include/net/abstract_tcp_server2.inl index cb1388f3b..89971bea2 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.inl +++ b/contrib/epee/include/net/abstract_tcp_server2.inl @@ -40,8 +40,9 @@ #include <boost/date_time/posix_time/posix_time.hpp> // TODO #include <boost/thread/condition_variable.hpp> // TODO #include <boost/make_shared.hpp> +#include <boost/thread.hpp> #include "warnings.h" -#include "string_tools.h" +#include "string_tools_lexical.h" #include "misc_language.h" #include "net/local_ip.h" #include "pragma_comp_defs.h" @@ -269,8 +270,6 @@ PRAGMA_WARNING_DISABLE_VS(4355) //_dbg3("[sock " << socket().native_handle() << "] add_ref, m_peer_number=" << mI->m_peer_number); CRITICAL_REGION_LOCAL(self->m_self_refs_lock); //_dbg3("[sock " << socket().native_handle() << "] add_ref 2, m_peer_number=" << mI->m_peer_number); - if(m_was_shutdown) - return false; ++m_reference_count; m_self_ref = std::move(self); return true; @@ -562,7 +561,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) { // LOCK: chunking epee::critical_region_t<decltype(m_chunking_lock)> send_guard(m_chunking_lock); // *** critical *** - MDEBUG("do_send() will SPLIT into small chunks, from packet="<<message_size<<" B for ptr="<<message_data); + MDEBUG("do_send() will SPLIT into small chunks, from packet="<<message_size<<" B for ptr="<<(const void*)message_data); // 01234567890 // ^^^^ (pos=0, len=4) ; pos:=pos+len, pos=4 // ^^^^ (pos=4, len=4) ; pos:=pos+len, pos=8 @@ -575,14 +574,14 @@ PRAGMA_WARNING_DISABLE_VS(4355) while (!message.empty()) { byte_slice chunk = message.take_slice(chunksize_good); - MDEBUG("chunk_start="<<(void*)chunk.data()<<" ptr="<<message_data<<" pos="<<(chunk.data() - message_data)); + MDEBUG("chunk_start="<<(void*)chunk.data()<<" ptr="<<(const void*)message_data<<" pos="<<(chunk.data() - message_data)); MDEBUG("part of " << message.size() << ": pos="<<(chunk.data() - message_data) << " len="<<chunk.size()); bool ok = do_send_chunk(std::move(chunk)); // <====== *** all_ok = all_ok && ok; if (!all_ok) { - MDEBUG("do_send() DONE ***FAILED*** from packet="<<message_size<<" B for ptr="<<message_data); + MDEBUG("do_send() DONE ***FAILED*** from packet="<<message_size<<" B for ptr="<<(const void*)message_data); MDEBUG("do_send() SEND was aborted in middle of big package - this is mostly harmless " << " (e.g. peer closed connection) but if it causes trouble tell us at #monero-dev. " << message_size); return false; // partial failure in sending @@ -590,7 +589,7 @@ PRAGMA_WARNING_DISABLE_VS(4355) // (in catch block, or uniq pointer) delete buf; } // each chunk - MDEBUG("do_send() DONE SPLIT from packet="<<message_size<<" B for ptr="<<message_data); + MDEBUG("do_send() DONE SPLIT from packet="<<message_size<<" B for ptr="<<(const void*)message_data); MDEBUG("do_send() m_connection_type = " << m_connection_type); diff --git a/contrib/epee/include/net/http_base.h b/contrib/epee/include/net/http_base.h index bf6589c92..4af4da790 100644 --- a/contrib/epee/include/net/http_base.h +++ b/contrib/epee/include/net/http_base.h @@ -27,14 +27,13 @@ #pragma once -#include <boost/lexical_cast.hpp> -#include <boost/regex.hpp> +#include "memwipe.h" + #include <boost/utility/string_ref.hpp> + #include <string> #include <utility> - -#include "memwipe.h" -#include "string_tools.h" +#include <list> #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net.http" @@ -66,34 +65,9 @@ namespace net_utils typedef std::list<std::pair<std::string, std::string> > fields_list; - inline - std::string get_value_from_fields_list(const std::string& param_name, const net_utils::http::fields_list& fields) - { - fields_list::const_iterator it = fields.begin(); - for(; it != fields.end(); it++) - if(!string_tools::compare_no_case(param_name, it->first)) - break; - - if(it==fields.end()) - return std::string(); + std::string get_value_from_fields_list(const std::string& param_name, const net_utils::http::fields_list& fields); - return it->second; - } - - - inline - std::string get_value_from_uri_line(const std::string& param_name, const std::string& uri) - { - std::string buff = "([\\?|&])"; - buff += param_name + "=([^&]*)"; - boost::regex match_param(buff.c_str(), boost::regex::icase | boost::regex::normal); - boost::smatch result; - if(boost::regex_search(uri, result, match_param, boost::match_default) && result[0].matched) - { - return result[2]; - } - return std::string(); - } + std::string get_value_from_uri_line(const std::string& param_name, const std::string& uri); static inline void add_field(std::string& out, const boost::string_ref name, const boost::string_ref value) { diff --git a/contrib/epee/include/net/http_client.h b/contrib/epee/include/net/http_client.h index 3725ef079..29ef82fb1 100644 --- a/contrib/epee/include/net/http_client.h +++ b/contrib/epee/include/net/http_client.h @@ -30,7 +30,6 @@ #include <ctype.h> #include <boost/shared_ptr.hpp> #include <boost/regex.hpp> -#include <boost/lexical_cast.hpp> #include <boost/optional/optional.hpp> #include <boost/utility/string_ref.hpp> //#include <mbstring.h> @@ -46,6 +45,7 @@ #endif #include "string_tools.h" +#include "string_tools_lexical.h" #include "reg_exp_definer.h" #include "abstract_http_client.h" #include "http_base.h" diff --git a/contrib/epee/include/net/http_server_handlers_map2.h b/contrib/epee/include/net/http_server_handlers_map2.h index 1665fdac7..ffb3f3b7e 100644 --- a/contrib/epee/include/net/http_server_handlers_map2.h +++ b/contrib/epee/include/net/http_server_handlers_map2.h @@ -74,7 +74,13 @@ uint64_t ticks = misc_utils::get_tick_count(); \ boost::value_initialized<command_type::request> req; \ bool parse_res = epee::serialization::load_t_from_json(static_cast<command_type::request&>(req), query_info.m_body); \ - CHECK_AND_ASSERT_MES(parse_res, false, "Failed to parse json: \r\n" << query_info.m_body); \ + if (!parse_res) \ + { \ + MERROR("Failed to parse json: \r\n" << query_info.m_body); \ + response_info.m_response_code = 400; \ + response_info.m_response_comment = "Bad request"; \ + return true; \ + } \ uint64_t ticks1 = epee::misc_utils::get_tick_count(); \ boost::value_initialized<command_type::response> resp;\ MINFO(m_conn_context << "calling " << s_pattern); \ @@ -104,7 +110,13 @@ uint64_t ticks = misc_utils::get_tick_count(); \ boost::value_initialized<command_type::request> req; \ bool parse_res = epee::serialization::load_t_from_binary(static_cast<command_type::request&>(req), epee::strspan<uint8_t>(query_info.m_body)); \ - CHECK_AND_ASSERT_MES(parse_res, false, "Failed to parse bin body data, body size=" << query_info.m_body.size()); \ + if (!parse_res) \ + { \ + MERROR("Failed to parse bin body data, body size=" << query_info.m_body.size()); \ + response_info.m_response_code = 400; \ + response_info.m_response_comment = "Bad request"; \ + return true; \ + } \ uint64_t ticks1 = misc_utils::get_tick_count(); \ boost::value_initialized<command_type::response> resp;\ MINFO(m_conn_context << "calling " << s_pattern); \ diff --git a/contrib/epee/include/net/levin_base.h b/contrib/epee/include/net/levin_base.h index fce6d4b7e..df59a6c44 100644 --- a/contrib/epee/include/net/levin_base.h +++ b/contrib/epee/include/net/levin_base.h @@ -31,6 +31,7 @@ #include <cstdint> +#include "byte_stream.h" #include "net_utils_base.h" #include "span.h" @@ -83,11 +84,12 @@ namespace levin #define LEVIN_PROTOCOL_VER_0 0 #define LEVIN_PROTOCOL_VER_1 1 + template<class t_connection_context = net_utils::connection_context_base> struct levin_commands_handler { - virtual int invoke(int command, const epee::span<const uint8_t> in_buff, byte_slice& buff_out, t_connection_context& context)=0; + virtual int invoke(int command, const epee::span<const uint8_t> in_buff, byte_stream& buff_out, t_connection_context& context)=0; virtual int notify(int command, const epee::span<const uint8_t> in_buff, t_connection_context& context)=0; virtual void callback(t_connection_context& context){}; @@ -125,12 +127,41 @@ namespace levin } } + //! Provides space for levin (p2p) header, so that payload can be sent without copy + class message_writer + { + byte_slice finalize(uint32_t command, uint32_t flags, uint32_t return_code, bool expect_response); + public: + using header = bucket_head2; + + explicit message_writer(std::size_t reserve = 8192); + + message_writer(const message_writer&) = delete; + message_writer(message_writer&&) = default; + ~message_writer() = default; + message_writer& operator=(const message_writer&) = delete; + message_writer& operator=(message_writer&&) = default; + + //! \return Size of payload (excludes header size). + std::size_t payload_size() const noexcept + { + return buffer.size() < sizeof(header) ? 0 : buffer.size() - sizeof(header); + } + + byte_slice finalize_invoke(uint32_t command) { return finalize(command, LEVIN_PACKET_REQUEST, 0, true); } + byte_slice finalize_notify(uint32_t command) { return finalize(command, LEVIN_PACKET_REQUEST, 0, false); } + byte_slice finalize_response(uint32_t command, uint32_t return_code) + { + return finalize(command, LEVIN_PACKET_RESPONSE, return_code, false); + } + + //! Has space for levin header until a finalize method is used + byte_stream buffer; + }; + //! \return Intialized levin header. bucket_head2 make_header(uint32_t command, uint64_t msg_size, uint32_t flags, bool expect_response) noexcept; - //! \return A levin notification message. - byte_slice make_notify(int command, epee::span<const std::uint8_t> payload); - /*! Generate a dummy levin message. \param noise_bytes Total size of the returned `byte_slice`. @@ -140,12 +171,11 @@ namespace levin /*! Generate 1+ levin messages that are identical to the noise message size. - \param noise Each levin message will be identical to the size of this - message. The bytes from this message will be used for padding. + \param noise_size Each levin message will be identical to this value. \return `nullptr` if `noise.size()` is less than the levin header size. Otherwise, a levin notification message OR 2+ levin fragment messages. Each message is `noise.size()` in length. */ - byte_slice make_fragmented_notify(const byte_slice& noise, int command, epee::span<const std::uint8_t> payload); + byte_slice make_fragmented_notify(const std::size_t noise_size, int command, message_writer message); } } diff --git a/contrib/epee/include/net/levin_protocol_handler_async.h b/contrib/epee/include/net/levin_protocol_handler_async.h index ddde701ee..a6816cafc 100644 --- a/contrib/epee/include/net/levin_protocol_handler_async.h +++ b/contrib/epee/include/net/levin_protocol_handler_async.h @@ -51,6 +51,21 @@ #define MIN_BYTES_WANTED 512 #endif +template<typename context_t> +void on_levin_traffic(const context_t &context, bool initiator, bool sent, bool error, size_t bytes, const char* category) +{ + MCINFO("net.p2p.traffic", context << bytes << " bytes " << (sent ? "sent" : "received") << (error ? "/corrupt" : "") + << " for category " << category << " initiated by " << (initiator ? "us" : "peer")); +} + +template<typename context_t> +void on_levin_traffic(const context_t &context, bool initiator, bool sent, bool error, size_t bytes, int command) +{ + char buf[32]; + snprintf(buf, sizeof(buf), "command-%u", command); + on_levin_traffic(context, initiator, sent, error, bytes, buf); +} + namespace epee { namespace levin @@ -88,11 +103,10 @@ public: uint64_t m_max_packet_size; uint64_t m_invoke_timeout; - int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, boost::uuids::uuid connection_id); + int invoke(int command, message_writer in_msg, std::string& buff_out, boost::uuids::uuid connection_id); template<class callback_t> - int invoke_async(int command, const epee::span<const uint8_t> in_buff, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED); + int invoke_async(int command, message_writer in_msg, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED); - int notify(int command, const epee::span<const uint8_t> in_buff, boost::uuids::uuid connection_id); int send(epee::byte_slice message, const boost::uuids::uuid& connection_id); bool close(boost::uuids::uuid connection_id); bool update_connection_context(const t_connection_context& contxt); @@ -122,12 +136,17 @@ class async_protocol_handler { std::string m_fragment_buffer; - bool send_message(uint32_t command, epee::span<const uint8_t> in_buff, uint32_t flags, bool expect_response) + bool send_message(byte_slice message) { - const bucket_head2 head = make_header(command, in_buff.size(), flags, expect_response); - if(!m_pservice_endpoint->do_send(byte_slice{as_byte_span(head), in_buff})) + if (message.size() < sizeof(message_writer::header)) + return false; + + message_writer::header head; + std::memcpy(std::addressof(head), message.data(), sizeof(head)); + if(!m_pservice_endpoint->do_send(std::move(message))) return false; + on_levin_traffic(m_connection_context, true, true, false, head.m_cb, head.m_command); MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << head.m_cb << ", flags" << head.m_flags << ", r?=" << head.m_have_to_return_data @@ -146,7 +165,6 @@ public: stream_state_body }; - std::atomic<bool> m_deletion_initiated; std::atomic<bool> m_protocol_released; volatile uint32_t m_invoke_buf_ready; @@ -297,7 +315,6 @@ public: m_state(stream_state_head) { m_close_called = 0; - m_deletion_initiated = false; m_protocol_released = false; m_wait_count = 0; m_oponent_protocol_ver = 0; @@ -310,7 +327,6 @@ public: try { - m_deletion_initiated = true; if(m_connection_initialized) { m_config.del_connection(this); @@ -526,26 +542,17 @@ public: { if(m_current_head.m_have_to_return_data) { - byte_slice return_buff; + levin::message_writer return_message{32 * 1024}; const uint32_t return_code = m_config.m_pcommands_handler->invoke( - m_current_head.m_command, buff_to_invoke, return_buff, m_connection_context + m_current_head.m_command, buff_to_invoke, return_message.buffer, m_connection_context ); // peer_id remains unset if dropped if (m_current_head.m_command == m_connection_context.handshake_command() && m_connection_context.handshake_complete()) m_max_packet_size = m_config.m_max_packet_size; - bucket_head2 head = make_header(m_current_head.m_command, return_buff.size(), LEVIN_PACKET_RESPONSE, false); - head.m_return_code = SWAP32LE(return_code); - - if(!m_pservice_endpoint->do_send(byte_slice{{epee::as_byte_span(head), epee::to_span(return_buff)}})) + if(!send_message(return_message.finalize_response(m_current_head.m_command, return_code))) return false; - - MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << head.m_cb - << ", flags" << head.m_flags - << ", r?=" << head.m_have_to_return_data - <<", cmd = " << head.m_command - << ", ver=" << head.m_protocol_version); } else m_config.m_pcommands_handler->notify(m_current_head.m_command, buff_to_invoke, m_connection_context); @@ -622,7 +629,7 @@ public: } template<class callback_t> - bool async_invoke(int command, const epee::span<const uint8_t> in_buff, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED) + bool async_invoke(int command, message_writer in_msg, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED) { misc_utils::auto_scope_leave_caller scope_exit_handler = misc_utils::create_scope_leave_handler( boost::bind(&async_protocol_handler::finish_outer_call, this)); @@ -633,27 +640,15 @@ public: int err_code = LEVIN_OK; do { - if(m_deletion_initiated) - { - err_code = LEVIN_ERROR_CONNECTION_DESTROYED; - break; - } - CRITICAL_REGION_LOCAL(m_call_lock); - if(m_deletion_initiated) - { - err_code = LEVIN_ERROR_CONNECTION_DESTROYED; - break; - } - boost::interprocess::ipcdetail::atomic_write32(&m_invoke_buf_ready, 0); CRITICAL_REGION_BEGIN(m_invoke_response_handlers_lock); if (command == m_connection_context.handshake_command()) m_max_packet_size = m_config.m_max_packet_size; - if(!send_message(command, in_buff, LEVIN_PACKET_REQUEST, true)) + if(!send_message(in_msg.finalize_invoke(command))) { LOG_ERROR_CC(m_connection_context, "Failed to do_send"); err_code = LEVIN_ERROR_CONNECTION; @@ -679,25 +674,19 @@ public: return true; } - int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out) + int invoke(int command, message_writer in_msg, std::string& buff_out) { misc_utils::auto_scope_leave_caller scope_exit_handler = misc_utils::create_scope_leave_handler( boost::bind(&async_protocol_handler::finish_outer_call, this)); - if(m_deletion_initiated) - return LEVIN_ERROR_CONNECTION_DESTROYED; - CRITICAL_REGION_LOCAL(m_call_lock); - if(m_deletion_initiated) - return LEVIN_ERROR_CONNECTION_DESTROYED; - boost::interprocess::ipcdetail::atomic_write32(&m_invoke_buf_ready, 0); if (command == m_connection_context.handshake_command()) m_max_packet_size = m_config.m_max_packet_size; - if (!send_message(command, in_buff, LEVIN_PACKET_REQUEST, true)) + if (!send_message(in_msg.finalize_invoke(command))) { LOG_ERROR_CC(m_connection_context, "Failed to send request"); return LEVIN_ERROR_CONNECTION; @@ -706,7 +695,7 @@ public: uint64_t ticks_start = misc_utils::get_tick_count(); size_t prev_size = 0; - while(!boost::interprocess::ipcdetail::atomic_read32(&m_invoke_buf_ready) && !m_deletion_initiated && !m_protocol_released) + while(!boost::interprocess::ipcdetail::atomic_read32(&m_invoke_buf_ready) && !m_protocol_released) { if(m_cache_in_buffer.size() - prev_size >= MIN_BYTES_WANTED) { @@ -723,7 +712,7 @@ public: return LEVIN_ERROR_CONNECTION_DESTROYED; } - if(m_deletion_initiated || m_protocol_released) + if(m_protocol_released) return LEVIN_ERROR_CONNECTION_DESTROYED; CRITICAL_REGION_BEGIN(m_local_inv_buff_lock); @@ -734,31 +723,9 @@ public: return m_invoke_result_code; } - int notify(int command, const epee::span<const uint8_t> in_buff) - { - misc_utils::auto_scope_leave_caller scope_exit_handler = misc_utils::create_scope_leave_handler( - boost::bind(&async_protocol_handler::finish_outer_call, this)); - - if(m_deletion_initiated) - return LEVIN_ERROR_CONNECTION_DESTROYED; - - CRITICAL_REGION_LOCAL(m_call_lock); - - if(m_deletion_initiated) - return LEVIN_ERROR_CONNECTION_DESTROYED; - - if (!send_message(command, in_buff, LEVIN_PACKET_REQUEST, false)) - { - LOG_ERROR_CC(m_connection_context, "Failed to send notify message"); - return -1; - } - - return 1; - } - - /*! Sends `message` without adding a levin header. The message must have - been created with `make_notify`, `make_noise_notify` or - `make_fragmented_notify`. See additional instructions for + /*! Sends `message` without adding a levin header. The message must have been + created with `make_noise_notify`, `make_fragmented_notify`, or + `message_writer::finalize_notify`. See additional instructions for `make_fragmented_notify`. \return 1 on success */ @@ -768,17 +735,11 @@ public: boost::bind(&async_protocol_handler::finish_outer_call, this) ); - if(m_deletion_initiated) - return LEVIN_ERROR_CONNECTION_DESTROYED; - - const std::size_t length = message.size(); - if (!m_pservice_endpoint->do_send(std::move(message))) + if (!send_message(std::move(message))) { LOG_ERROR_CC(m_connection_context, "Failed to send message, dropping it"); return -1; } - - MDEBUG(m_connection_context << "LEVIN_PACKET_SENT. [len=" << (length - sizeof(bucket_head2)) << ", r?=0]"); return 1; } //------------------------------------------------------------------------------------------ @@ -799,36 +760,32 @@ void async_protocol_handler_config<t_connection_context>::del_connection(async_p template<class t_connection_context> void async_protocol_handler_config<t_connection_context>::delete_connections(size_t count, bool incoming) { - std::vector <boost::uuids::uuid> connections; + std::vector<typename connections_map::mapped_type> connections; + + auto scope_exit_handler = misc_utils::create_scope_leave_handler([&connections]{ + for (auto &aph: connections) + aph->finish_outer_call(); + }); + CRITICAL_REGION_BEGIN(m_connects_lock); for (auto& c: m_connects) { if (c.second->m_connection_context.m_is_income == incoming) - connections.push_back(c.first); + if (c.second->start_outer_call()) + connections.push_back(c.second); } // close random connections from the provided set // TODO or better just keep removing random elements (performance) unsigned seed = std::chrono::system_clock::now().time_since_epoch().count(); shuffle(connections.begin(), connections.end(), std::default_random_engine(seed)); - while (count > 0 && connections.size() > 0) - { - try - { - auto i = connections.end() - 1; - async_protocol_handler<t_connection_context> *conn = m_connects.at(*i); - del_connection(conn); - conn->close(); - connections.erase(i); - } - catch (const std::out_of_range &e) - { - MWARNING("Connection not found in m_connects, continuing"); - } - --count; - } + for (size_t i = 0; i < connections.size() && i < count; ++i) + m_connects.erase(connections[i]->get_connection_id()); CRITICAL_REGION_END(); + + for (size_t i = 0; i < connections.size() && i < count; ++i) + connections[i]->close(); } //------------------------------------------------------------------------------------------ template<class t_connection_context> @@ -872,41 +829,53 @@ int async_protocol_handler_config<t_connection_context>::find_and_lock_connectio } //------------------------------------------------------------------------------------------ template<class t_connection_context> -int async_protocol_handler_config<t_connection_context>::invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, boost::uuids::uuid connection_id) +int async_protocol_handler_config<t_connection_context>::invoke(int command, message_writer in_msg, std::string& buff_out, boost::uuids::uuid connection_id) { async_protocol_handler<t_connection_context>* aph; int r = find_and_lock_connection(connection_id, aph); - return LEVIN_OK == r ? aph->invoke(command, in_buff, buff_out) : r; + return LEVIN_OK == r ? aph->invoke(command, std::move(in_msg), buff_out) : r; } //------------------------------------------------------------------------------------------ template<class t_connection_context> template<class callback_t> -int async_protocol_handler_config<t_connection_context>::invoke_async(int command, const epee::span<const uint8_t> in_buff, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout) +int async_protocol_handler_config<t_connection_context>::invoke_async(int command, message_writer in_msg, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout) { async_protocol_handler<t_connection_context>* aph; int r = find_and_lock_connection(connection_id, aph); - return LEVIN_OK == r ? aph->async_invoke(command, in_buff, cb, timeout) : r; + return LEVIN_OK == r ? aph->async_invoke(command, std::move(in_msg), cb, timeout) : r; } //------------------------------------------------------------------------------------------ template<class t_connection_context> template<class callback_t> bool async_protocol_handler_config<t_connection_context>::foreach_connection(const callback_t &cb) { - CRITICAL_REGION_LOCAL(m_connects_lock); - for(auto& c: m_connects) - { - async_protocol_handler<t_connection_context>* aph = c.second; - if(!cb(aph->get_context_ref())) + std::vector<typename connections_map::mapped_type> conn; + + auto scope_exit_handler = misc_utils::create_scope_leave_handler([&conn]{ + for (auto &aph: conn) + aph->finish_outer_call(); + }); + + CRITICAL_REGION_BEGIN(m_connects_lock); + conn.reserve(m_connects.size()); + for (auto &e: m_connects) + if (e.second->start_outer_call()) + conn.push_back(e.second); + CRITICAL_REGION_END() + + for (auto &aph: conn) + if (!cb(aph->get_context_ref())) return false; - } + return true; } //------------------------------------------------------------------------------------------ template<class t_connection_context> template<class callback_t> bool async_protocol_handler_config<t_connection_context>::for_connection(const boost::uuids::uuid &connection_id, const callback_t &cb) { - CRITICAL_REGION_LOCAL(m_connects_lock); - async_protocol_handler<t_connection_context>* aph = find_connection(connection_id); - if (!aph) + async_protocol_handler<t_connection_context>* aph = nullptr; + if (find_and_lock_connection(connection_id, aph) != LEVIN_OK) return false; + auto scope_exit_handler = misc_utils::create_scope_leave_handler( + boost::bind(&async_protocol_handler<t_connection_context>::finish_outer_call, aph)); if(!cb(aph->get_context_ref())) return false; return true; @@ -951,14 +920,6 @@ void async_protocol_handler_config<t_connection_context>::set_handler(levin_comm } //------------------------------------------------------------------------------------------ template<class t_connection_context> -int async_protocol_handler_config<t_connection_context>::notify(int command, const epee::span<const uint8_t> in_buff, boost::uuids::uuid connection_id) -{ - async_protocol_handler<t_connection_context>* aph; - int r = find_and_lock_connection(connection_id, aph); - return LEVIN_OK == r ? aph->notify(command, in_buff) : r; -} -//------------------------------------------------------------------------------------------ -template<class t_connection_context> int async_protocol_handler_config<t_connection_context>::send(byte_slice message, const boost::uuids::uuid& connection_id) { async_protocol_handler<t_connection_context>* aph; @@ -969,12 +930,14 @@ int async_protocol_handler_config<t_connection_context>::send(byte_slice message template<class t_connection_context> bool async_protocol_handler_config<t_connection_context>::close(boost::uuids::uuid connection_id) { - CRITICAL_REGION_LOCAL(m_connects_lock); - async_protocol_handler<t_connection_context>* aph = find_connection(connection_id); - if (!aph) + async_protocol_handler<t_connection_context>* aph = nullptr; + if (find_and_lock_connection(connection_id, aph) != LEVIN_OK) return false; + auto scope_exit_handler = misc_utils::create_scope_leave_handler( + boost::bind(&async_protocol_handler<t_connection_context>::finish_outer_call, aph)); if (!aph->close()) return false; + CRITICAL_REGION_LOCAL(m_connects_lock); m_connects.erase(connection_id); return true; } diff --git a/contrib/epee/include/net/net_parse_helpers.h b/contrib/epee/include/net/net_parse_helpers.h index cf637ba1d..d794dd51c 100644 --- a/contrib/epee/include/net/net_parse_helpers.h +++ b/contrib/epee/include/net/net_parse_helpers.h @@ -24,12 +24,9 @@ // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // - - - #pragma once + #include "http_base.h" -#include "reg_exp_definer.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net" @@ -38,173 +35,8 @@ namespace epee { namespace net_utils { - - inline bool parse_uri_query(const std::string& query, std::list<std::pair<std::string, std::string> >& params) - { - enum state - { - st_param_name, - st_param_val - }; - state st = st_param_name; - std::string::const_iterator start_it = query.begin(); - std::pair<std::string, std::string> e; - for(std::string::const_iterator it = query.begin(); it != query.end(); it++) - { - switch(st) - { - case st_param_name: - if(*it == '=') - { - e.first.assign(start_it, it); - start_it = it;++start_it; - st = st_param_val; - } - break; - case st_param_val: - if(*it == '&') - { - e.second.assign(start_it, it); - start_it = it;++start_it; - params.push_back(e); - e.first.clear();e.second.clear(); - st = st_param_name; - } - break; - default: - LOG_ERROR("Unknown state " << (int)st); - return false; - } - } - if(st == st_param_name) - { - if(start_it != query.end()) - { - e.first.assign(start_it, query.end()); - params.push_back(e); - } - }else - { - if(start_it != query.end()) - e.second.assign(start_it, query.end()); - - if(e.first.size()) - params.push_back(e); - } - return true; - } - - inline - bool parse_uri(const std::string uri, http::uri_content& content) - { - - ///iframe_test.html?api_url=http://api.vk.com/api.php&api_id=3289090&api_settings=1&viewer_id=562964060&viewer_type=0&sid=0aad8d1c5713130f9ca0076f2b7b47e532877424961367d81e7fa92455f069be7e21bc3193cbd0be11895&secret=368ebbc0ef&access_token=668bc03f43981d883f73876ffff4aa8564254b359cc745dfa1b3cde7bdab2e94105d8f6d8250717569c0a7&user_id=0&group_id=0&is_app_user=1&auth_key=d2f7a895ca5ff3fdb2a2a8ae23fe679a&language=0&parent_language=0&ad_info=ElsdCQBaQlxiAQRdFUVUXiN2AVBzBx5pU1BXIgZUJlIEAWcgAUoLQg==&referrer=unknown&lc_name=9834b6a3&hash= - content.m_query_params.clear(); - STATIC_REGEXP_EXPR_1(rexp_match_uri, "^([^?#]*)(\\?([^#]*))?(#(.*))?", boost::regex::icase | boost::regex::normal); - - boost::smatch result; - if(!(boost::regex_search(uri, result, rexp_match_uri, boost::match_default) && result[0].matched)) - { - LOG_PRINT_L1("[PARSE URI] regex not matched for uri: " << uri); - content.m_path = uri; - return true; - } - if(result[1].matched) - { - content.m_path = result[1]; - } - if(result[3].matched) - { - content.m_query = result[3]; - } - if(result[5].matched) - { - content.m_fragment = result[5]; - } - if(content.m_query.size()) - { - parse_uri_query(content.m_query, content.m_query_params); - } - return true; - } - - inline - bool parse_url_ipv6(const std::string url_str, http::url_content& content) - { - STATIC_REGEXP_EXPR_1(rexp_match_uri, "^(([^:]*?)://)?(\\[(.*)\\](:(\\d+))?)(.*)?", boost::regex::icase | boost::regex::normal); - // 12 3 4 5 6 7 - - content.port = 0; - boost::smatch result; - if(!(boost::regex_search(url_str, result, rexp_match_uri, boost::match_default) && result[0].matched)) - { - LOG_PRINT_L1("[PARSE URI] regex not matched for uri: " << rexp_match_uri); - //content.m_path = uri; - return false; - } - if(result[2].matched) - { - content.schema = result[2]; - } - if(result[4].matched) - { - content.host = result[4]; - } - else // if host not matched, matching should be considered failed - { - return false; - } - if(result[6].matched) - { - content.port = boost::lexical_cast<uint64_t>(result[6]); - } - if(result[7].matched) - { - content.uri = result[7]; - return parse_uri(result[7], content.m_uri_content); - } - - return true; - } - - inline - bool parse_url(const std::string url_str, http::url_content& content) - { - - if (parse_url_ipv6(url_str, content)) return true; - - ///iframe_test.html?api_url=http://api.vk.com/api.php&api_id=3289090&api_settings=1&viewer_id=562964060&viewer_type=0&sid=0aad8d1c5713130f9ca0076f2b7b47e532877424961367d81e7fa92455f069be7e21bc3193cbd0be11895&secret=368ebbc0ef&access_token=668bc03f43981d883f73876ffff4aa8564254b359cc745dfa1b3cde7bdab2e94105d8f6d8250717569c0a7&user_id=0&group_id=0&is_app_user=1&auth_key=d2f7a895ca5ff3fdb2a2a8ae23fe679a&language=0&parent_language=0&ad_info=ElsdCQBaQlxiAQRdFUVUXiN2AVBzBx5pU1BXIgZUJlIEAWcgAUoLQg==&referrer=unknown&lc_name=9834b6a3&hash= - //STATIC_REGEXP_EXPR_1(rexp_match_uri, "^([^?#]*)(\\?([^#]*))?(#(.*))?", boost::regex::icase | boost::regex::normal); - STATIC_REGEXP_EXPR_1(rexp_match_uri, "^(([^:]*?)://)?(([^/:]*)(:(\\d+))?)(.*)?", boost::regex::icase | boost::regex::normal); - // 12 34 5 6 7 - content.port = 0; - boost::smatch result; - if(!(boost::regex_search(url_str, result, rexp_match_uri, boost::match_default) && result[0].matched)) - { - LOG_PRINT_L1("[PARSE URI] regex not matched for uri: " << rexp_match_uri); - //content.m_path = uri; - return true; - } - if(result[2].matched) - { - content.schema = result[2]; - } - if(result[4].matched) - { - content.host = result[4]; - } - if(result[6].matched) - { - content.port = boost::lexical_cast<uint64_t>(result[6]); - } - if(result[7].matched) - { - content.uri = result[7]; - return parse_uri(result[7], content.m_uri_content); - } - - return true; - } - + bool parse_uri(const std::string uri, http::uri_content& content); + bool parse_url_ipv6(const std::string url_str, http::url_content& content); + bool parse_url(const std::string url_str, http::url_content& content); } } diff --git a/contrib/epee/include/net/net_ssl.h b/contrib/epee/include/net/net_ssl.h index 1b1577e77..58cd7e45f 100644 --- a/contrib/epee/include/net/net_ssl.h +++ b/contrib/epee/include/net/net_ssl.h @@ -36,6 +36,7 @@ #include <boost/utility/string_ref.hpp> #include <boost/asio/ip/tcp.hpp> #include <boost/asio/ssl.hpp> +#include <boost/filesystem/path.hpp> #include <boost/system/error_code.hpp> #define SSL_FINGERPRINT_SIZE 32 @@ -144,6 +145,9 @@ namespace net_utils bool create_ec_ssl_certificate(EVP_PKEY *&pkey, X509 *&cert); bool create_rsa_ssl_certificate(EVP_PKEY *&pkey, X509 *&cert); + + //! Store private key for `ssl` at `base + ".key"` unencrypted and certificate for `ssl` at `base + ".crt"`. + boost::system::error_code store_ssl_keys(boost::asio::ssl::context& ssl, const boost::filesystem::path& base); } } diff --git a/contrib/epee/include/net/net_utils_base.h b/contrib/epee/include/net/net_utils_base.h index d86c62c17..722206ee1 100644 --- a/contrib/epee/include/net/net_utils_base.h +++ b/contrib/epee/include/net/net_utils_base.h @@ -236,6 +236,7 @@ namespace net_utils virtual address_type get_type_id() const = 0; virtual zone get_zone() const = 0; virtual bool is_blockable() const = 0; + virtual std::uint16_t port() const = 0; }; template<typename T> @@ -266,6 +267,7 @@ namespace net_utils virtual address_type get_type_id() const override { return value.get_type_id(); } virtual zone get_zone() const override { return value.get_zone(); } virtual bool is_blockable() const override { return value.is_blockable(); } + virtual std::uint16_t port() const override { return value.port(); } }; std::shared_ptr<interface> self; @@ -312,6 +314,7 @@ namespace net_utils address_type get_type_id() const { return self ? self->get_type_id() : address_type::invalid; } zone get_zone() const { return self ? self->get_zone() : zone::invalid; } bool is_blockable() const { return self ? self->is_blockable() : false; } + std::uint16_t port() const { return self ? self->port() : 0; } template<typename Type> const Type &as() const { return as_mutable<const Type>(); } BEGIN_KV_SERIALIZE_MAP() diff --git a/contrib/epee/include/reg_exp_definer.h b/contrib/epee/include/reg_exp_definer.h index eb11c9e10..386a45f9a 100644 --- a/contrib/epee/include/reg_exp_definer.h +++ b/contrib/epee/include/reg_exp_definer.h @@ -29,6 +29,7 @@ #define _REG_EXP_DEFINER_H_ #include <boost/interprocess/detail/atomic.hpp> +#include <boost/regex.hpp> #include "syncobj.h" namespace epee diff --git a/contrib/epee/include/serialization/keyvalue_serialization.h b/contrib/epee/include/serialization/keyvalue_serialization.h index fd343865c..2e4a0faad 100644 --- a/contrib/epee/include/serialization/keyvalue_serialization.h +++ b/contrib/epee/include/serialization/keyvalue_serialization.h @@ -73,7 +73,8 @@ public: \ template<bool is_store, class t_storage> \ bool serialize_map(t_storage& stg, typename t_storage::hsection hparent_section) \ { \ - decltype(*this) &this_ref = *this; + decltype(*this) &this_ref = *this; \ + (void) this_ref; // Suppress unused var warnings. Sometimes this var is used, sometimes not. #define KV_SERIALIZE_N(varialble, val_name) \ epee::serialization::selector<is_store>::serialize(this_ref.varialble, stg, hparent_section, val_name); diff --git a/contrib/epee/include/span.h b/contrib/epee/include/span.h index b355c960a..26861f994 100644 --- a/contrib/epee/include/span.h +++ b/contrib/epee/include/span.h @@ -31,7 +31,6 @@ #include <algorithm> #include <cstdint> #include <memory> -#include <string> #include <type_traits> namespace epee @@ -167,10 +166,11 @@ namespace epee } //! make a span from a std::string - template<typename T> - span<const T> strspan(const std::string &s) noexcept + template<typename T, typename U> + span<const T> strspan(const U&s) noexcept { - static_assert(std::is_same<T, char>() || std::is_same<T, unsigned char>() || std::is_same<T, int8_t>() || std::is_same<T, uint8_t>(), "Unexpected type"); + static_assert(std::is_same<typename U::value_type, char>(), "unexpected source type"); + static_assert(std::is_same<T, char>() || std::is_same<T, unsigned char>() || std::is_same<T, int8_t>() || std::is_same<T, uint8_t>(), "Unexpected destination type"); return {reinterpret_cast<const T*>(s.data()), s.size()}; } } diff --git a/contrib/epee/include/storages/levin_abstract_invoke2.h b/contrib/epee/include/storages/levin_abstract_invoke2.h index 802e16c1b..383d67cc2 100644 --- a/contrib/epee/include/storages/levin_abstract_invoke2.h +++ b/contrib/epee/include/storages/levin_abstract_invoke2.h @@ -37,21 +37,14 @@ #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net" +template<typename context_t> +void on_levin_traffic(const context_t &context, bool initiator, bool sent, bool error, size_t bytes, const char *category); + +template<typename context_t> +void on_levin_traffic(const context_t &context, bool initiator, bool sent, bool error, size_t bytes, int command); + namespace { - template<typename context_t> - void on_levin_traffic(const context_t &context, bool initiator, bool sent, bool error, size_t bytes, const char *category) - { - MCINFO("net.p2p.traffic", context << bytes << " bytes " << (sent ? "sent" : "received") << (error ? "/corrupt" : "") - << " for category " << category << " initiated by " << (initiator ? "us" : "peer")); - } - template<typename context_t> - void on_levin_traffic(const context_t &context, bool initiator, bool sent, bool error, size_t bytes, int command) - { - char buf[32]; - snprintf(buf, sizeof(buf), "command-%u", command); - return on_levin_traffic(context, initiator, sent, error, bytes, buf); - } static const constexpr epee::serialization::portable_storage::limits_t default_levin_limits = { 8192, // objects 16384, // fields @@ -117,12 +110,11 @@ namespace epee const boost::uuids::uuid &conn_id = context.m_connection_id; typename serialization::portable_storage stg; out_struct.store(stg); - byte_slice buff_to_send; + levin::message_writer to_send{16 * 1024}; std::string buff_to_recv; - stg.store_to_binary(buff_to_send, 16 * 1024); + stg.store_to_binary(to_send.buffer); - on_levin_traffic(context, true, true, false, buff_to_send.size(), command); - int res = transport.invoke(command, boost::string_ref{reinterpret_cast<const char*>(buff_to_send.data()), buff_to_send.size()}, buff_to_recv, conn_id); + int res = transport.invoke(command, std::move(to_send), buff_to_recv, conn_id); if( res <=0 ) { LOG_PRINT_L1("Failed to invoke command " << command << " return code " << res); @@ -145,10 +137,9 @@ namespace epee const boost::uuids::uuid &conn_id = context.m_connection_id; typename serialization::portable_storage stg; const_cast<t_arg&>(out_struct).store(stg);//TODO: add true const support to searilzation - byte_slice buff_to_send; - stg.store_to_binary(buff_to_send, 16 * 1024); - on_levin_traffic(context, true, true, false, buff_to_send.size(), command); - int res = transport.invoke_async(command, epee::to_span(buff_to_send), conn_id, [cb, command](int code, const epee::span<const uint8_t> buff, typename t_transport::connection_context& context)->bool + levin::message_writer to_send{16 * 1024}; + stg.store_to_binary(to_send.buffer); + int res = transport.invoke_async(command, std::move(to_send), conn_id, [cb, command](int code, const epee::span<const uint8_t> buff, typename t_transport::connection_context& context)->bool { t_result result_struct = AUTO_VAL_INIT(result_struct); if( code <=0 ) @@ -192,11 +183,10 @@ namespace epee const boost::uuids::uuid &conn_id = context.m_connection_id; serialization::portable_storage stg; out_struct.store(stg); - byte_slice buff_to_send; - stg.store_to_binary(buff_to_send); + levin::message_writer to_send; + stg.store_to_binary(to_send.buffer); - on_levin_traffic(context, true, true, false, buff_to_send.size(), command); - int res = transport.notify(command, epee::to_span(buff_to_send), conn_id); + int res = transport.send(to_send.finalize_notify(command), conn_id); if(res <=0 ) { MERROR("Failed to notify command " << command << " return code " << res); @@ -207,7 +197,7 @@ namespace epee //---------------------------------------------------------------------------------------------------- //---------------------------------------------------------------------------------------------------- template<class t_owner, class t_in_type, class t_out_type, class t_context, class callback_t> - int buff_to_t_adapter(int command, const epee::span<const uint8_t> in_buff, byte_slice& buff_out, callback_t cb, t_context& context ) + int buff_to_t_adapter(int command, const epee::span<const uint8_t> in_buff, byte_stream& buff_out, callback_t cb, t_context& context ) { serialization::portable_storage strg; if(!strg.load_from_binary(in_buff, &default_levin_limits)) @@ -230,12 +220,11 @@ namespace epee serialization::portable_storage strg_out; static_cast<t_out_type&>(out_struct).store(strg_out); - if(!strg_out.store_to_binary(buff_out, 32 * 1024)) + if(!strg_out.store_to_binary(buff_out)) { LOG_ERROR("Failed to store_to_binary in command" << command); return -1; } - on_levin_traffic(context, false, true, false, buff_out.size(), command); return res; } @@ -262,7 +251,7 @@ namespace epee } #define CHAIN_LEVIN_INVOKE_MAP2(context_type) \ - int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, context_type& context) \ + int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_stream& buff_out, context_type& context) \ { \ bool handled = false; \ return handle_invoke_map(false, command, in_buff, buff_out, context, handled); \ @@ -271,13 +260,13 @@ namespace epee #define CHAIN_LEVIN_NOTIFY_MAP2(context_type) \ int notify(int command, const epee::span<const uint8_t> in_buff, context_type& context) \ { \ - bool handled = false; epee::byte_slice fake_str; \ - return handle_invoke_map(true, command, in_buff, fake_str, context, handled); \ + bool handled = false; epee::byte_stream fake_str; \ + return handle_invoke_map(true, command, in_buff, fake_str, context, handled); \ } #define CHAIN_LEVIN_INVOKE_MAP() \ - int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, epee::net_utils::connection_context_base& context) \ + int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_stream& buff_out, epee::net_utils::connection_context_base& context) \ { \ bool handled = false; \ return handle_invoke_map(false, command, in_buff, buff_out, context, handled); \ @@ -297,7 +286,7 @@ namespace epee } #define BEGIN_INVOKE_MAP2(owner_type) \ - template <class t_context> int handle_invoke_map(bool is_notify, int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, t_context& context, bool& handled) \ + template <class t_context> int handle_invoke_map(bool is_notify, int command, const epee::span<const uint8_t> in_buff, epee::byte_stream& buff_out, t_context& context, bool& handled) \ { \ try { \ typedef owner_type internal_owner_type_name; diff --git a/contrib/epee/include/storages/parserse_base_utils.h b/contrib/epee/include/storages/parserse_base_utils.h index 5a6cc0b51..e59cbcf5f 100644 --- a/contrib/epee/include/storages/parserse_base_utils.h +++ b/contrib/epee/include/storages/parserse_base_utils.h @@ -28,13 +28,8 @@ #pragma once -#include <algorithm> -#include <boost/utility/string_ref.hpp> - -#include "misc_log_ex.h" - -#undef MONERO_DEFAULT_LOG_CATEGORY -#define MONERO_DEFAULT_LOG_CATEGORY "serialization" +#include <boost/utility/string_ref_fwd.hpp> +#include <string> namespace epee { @@ -97,46 +92,7 @@ namespace misc_utils return lut[(uint8_t)c] & 1; } - inline std::string transform_to_escape_sequence(const std::string& src) - { - static const char escaped[] = "\b\f\n\r\t\v\"\\/"; - std::string::const_iterator it = std::find_first_of(src.begin(), src.end(), escaped, escaped + sizeof(escaped)); - if (it == src.end()) - return src; - - std::string res; - res.reserve(2 * src.size()); - res.assign(src.begin(), it); - for(; it!=src.end(); ++it) - { - switch(*it) - { - case '\b': //Backspace (ascii code 08) - res+="\\b"; break; - case '\f': //Form feed (ascii code 0C) - res+="\\f"; break; - case '\n': //New line - res+="\\n"; break; - case '\r': //Carriage return - res+="\\r"; break; - case '\t': //Tab - res+="\\t"; break; - case '\v': //Vertical tab - res+="\\v"; break; - //case '\'': //Apostrophe or single quote - // res+="\\'"; break; - case '"': //Double quote - res+="\\\""; break; - case '\\': //Backslash caracter - res+="\\\\"; break; - case '/': //Backslash caracter - res+="\\/"; break; - default: - res.push_back(*it); - } - } - return res; - } + std::string transform_to_escape_sequence(const std::string& src); /* \b Backspace (ascii code 08) @@ -150,98 +106,7 @@ namespace misc_utils \\ Backslash character */ - inline void match_string2(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, std::string& val) - { - bool escape_mode = false; - std::string::const_iterator it = star_end_string; - ++it; - std::string::const_iterator fi = it; - while (fi != buf_end && ((lut[(uint8_t)*fi] & 32)) == 0) - ++fi; - val.assign(it, fi); - it = fi; - for(;it != buf_end;it++) - { - if(escape_mode/*prev_ch == '\\'*/) - { - switch(*it) - { - case 'b': //Backspace (ascii code 08) - val.push_back(0x08);break; - case 'f': //Form feed (ascii code 0C) - val.push_back(0x0C);break; - case 'n': //New line - val.push_back('\n');break; - case 'r': //Carriage return - val.push_back('\r');break; - case 't': //Tab - val.push_back('\t');break; - case 'v': //Vertical tab - val.push_back('\v');break; - case '\'': //Apostrophe or single quote - val.push_back('\'');break; - case '"': //Double quote - val.push_back('"');break; - case '\\': //Backslash character - val.push_back('\\');break; - case '/': //Slash character - val.push_back('/');break; - case 'u': //Unicode code point - if (buf_end - it < 4) - { - ASSERT_MES_AND_THROW("Invalid Unicode escape sequence"); - } - else - { - uint32_t dst = 0; - for (int i = 0; i < 4; ++i) - { - const unsigned char tmp = isx[(unsigned char)*++it]; - CHECK_AND_ASSERT_THROW_MES(tmp != 0xff, "Bad Unicode encoding"); - dst = dst << 4 | tmp; - } - // encode as UTF-8 - if (dst <= 0x7f) - { - val.push_back(dst); - } - else if (dst <= 0x7ff) - { - val.push_back(0xc0 | (dst >> 6)); - val.push_back(0x80 | (dst & 0x3f)); - } - else if (dst <= 0xffff) - { - val.push_back(0xe0 | (dst >> 12)); - val.push_back(0x80 | ((dst >> 6) & 0x3f)); - val.push_back(0x80 | (dst & 0x3f)); - } - else - { - ASSERT_MES_AND_THROW("Unicode code point is out or range"); - } - } - break; - default: - val.push_back(*it); - LOG_PRINT_L0("Unknown escape sequence :\"\\" << *it << "\""); - } - escape_mode = false; - }else if(*it == '"') - { - star_end_string = it; - return; - }else if(*it == '\\') - { - escape_mode = true; - } - else - { - val.push_back(*it); - } - } - ASSERT_MES_AND_THROW("Failed to match string in json entry: " << std::string(star_end_string, buf_end)); - } + void match_string2(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, std::string& val); inline bool match_string(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, std::string& val) { try @@ -255,42 +120,7 @@ namespace misc_utils return false; } } - inline void match_number2(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, boost::string_ref& val, bool& is_float_val, bool& is_signed_val) - { - val.clear(); - uint8_t float_flag = 0; - is_signed_val = false; - size_t chars = 0; - std::string::const_iterator it = star_end_string; - if (it != buf_end && *it == '-') - { - is_signed_val = true; - ++chars; - ++it; - } - for(;it != buf_end;it++) - { - const uint8_t flags = lut[(uint8_t)*it]; - if (flags & 16) - { - float_flag |= flags; - ++chars; - } - else - { - val = boost::string_ref(&*star_end_string, chars); - if(val.size()) - { - star_end_string = --it; - is_float_val = !!(float_flag & 2); - return; - } - else - ASSERT_MES_AND_THROW("wrong number in json entry: " << std::string(star_end_string, buf_end)); - } - } - ASSERT_MES_AND_THROW("wrong number in json entry: " << std::string(star_end_string, buf_end)); - } + void match_number2(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, boost::string_ref& val, bool& is_float_val, bool& is_signed_val); inline bool match_number(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, boost::string_ref& val) { try @@ -304,25 +134,7 @@ namespace misc_utils return false; } } - inline void match_word2(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, boost::string_ref& val) - { - val.clear(); - - for(std::string::const_iterator it = star_end_string;it != buf_end;it++) - { - if (!(lut[(uint8_t)*it] & 4)) - { - val = boost::string_ref(&*star_end_string, std::distance(star_end_string, it)); - if(val.size()) - { - star_end_string = --it; - return; - }else - ASSERT_MES_AND_THROW("failed to match word number in json entry: " << std::string(star_end_string, buf_end)); - } - } - ASSERT_MES_AND_THROW("failed to match word number in json entry: " << std::string(star_end_string, buf_end)); - } + void match_word2(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, boost::string_ref& val); inline bool match_word(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, boost::string_ref& val) { try @@ -335,44 +147,8 @@ namespace misc_utils return false; } } - inline bool match_word_with_extrasymb(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, std::string& val) - { - val.clear(); - - for(std::string::const_iterator it = star_end_string;it != buf_end;it++) - { - if(!isalnum(*it) && *it != '-' && *it != '_') - { - val.assign(star_end_string, it); - if(val.size()) - { - star_end_string = --it; - return true; - }else - return false; - } - } - return false; - } - inline bool match_word_til_equal_mark(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, std::string::const_iterator& word_end) - { - word_end = star_end_string; - - for(std::string::const_iterator it = star_end_string;it != buf_end;it++) - { - if(isspace(*it)) - { - - continue; - }else if( *it == '=' ) - { - star_end_string = it; - word_end = it; - return true; - } - } - return false; - } + bool match_word_with_extrasymb(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, std::string& val); + bool match_word_til_equal_mark(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, std::string::const_iterator& word_end); } } } diff --git a/contrib/epee/include/storages/portable_storage.h b/contrib/epee/include/storages/portable_storage.h index f77e89cb6..60f3e672f 100644 --- a/contrib/epee/include/storages/portable_storage.h +++ b/contrib/epee/include/storages/portable_storage.h @@ -24,24 +24,19 @@ // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // - - #pragma once -#include <type_traits> - -#include "misc_language.h" #include "portable_storage_base.h" -#include "portable_storage_from_bin.h" -#include "portable_storage_to_json.h" -#include "portable_storage_from_json.h" #include "portable_storage_val_converters.h" +#include "misc_log_ex.h" #include "span.h" -#include "int-util.h" + +#include <boost/mpl/contains.hpp> namespace epee { class byte_slice; + class byte_stream; namespace serialization { /************************************************************************/ @@ -91,8 +86,13 @@ namespace epee //------------------------------------------------------------------------------- bool store_to_binary(byte_slice& target, std::size_t initial_buffer_size = 8192); - bool load_from_binary(const epee::span<const uint8_t> target, const limits_t *limits = NULL); - bool load_from_binary(const std::string& target, const limits_t *limits = NULL) { return load_from_binary(epee::strspan<uint8_t>(target), limits); } + bool store_to_binary(byte_stream& ss); + bool load_from_binary(const epee::span<const uint8_t> target, const limits_t *limits = nullptr); + bool load_from_binary(const std::string& target, const limits_t *limits = nullptr) + { + return load_from_binary(epee::strspan<uint8_t>(target), limits); + } + template<class trace_policy> bool dump_as_xml(std::string& targetObj, const std::string& root_name = ""); bool dump_as_json(std::string& targetObj, size_t indent = 0, bool insert_newlines = true); @@ -117,85 +117,13 @@ namespace epee }; #pragma pack(pop) }; - inline - bool portable_storage::dump_as_json(std::string& buff, size_t indent, bool insert_newlines) - { - TRY_ENTRY(); - std::stringstream ss; - epee::serialization::dump_as_json(ss, m_root, indent, insert_newlines); - buff = ss.str(); - return true; - CATCH_ENTRY("portable_storage::dump_as_json", false) - } - inline - bool portable_storage::load_from_json(const std::string& source) - { - TRY_ENTRY(); - return json::load_from_json(source, *this); - CATCH_ENTRY("portable_storage::load_from_json", false) - } - + template<class trace_policy> bool portable_storage::dump_as_xml(std::string& targetObj, const std::string& root_name) { return false;//TODO: don't think i ever again will use xml - ambiguous and "overtagged" format - } - inline - bool portable_storage::load_from_binary(const epee::span<const uint8_t> source, const limits_t *limits) - { - m_root.m_entries.clear(); - if(source.size() < sizeof(storage_block_header)) - { - LOG_ERROR("portable_storage: wrong binary format, packet size = " << source.size() << " less than expected sizeof(storage_block_header)=" << sizeof(storage_block_header)); - return false; - } - storage_block_header* pbuff = (storage_block_header*)source.data(); - if(pbuff->m_signature_a != SWAP32LE(PORTABLE_STORAGE_SIGNATUREA) || - pbuff->m_signature_b != SWAP32LE(PORTABLE_STORAGE_SIGNATUREB) - ) - { - LOG_ERROR("portable_storage: wrong binary format - signature mismatch"); - return false; - } - if(pbuff->m_ver != PORTABLE_STORAGE_FORMAT_VER) - { - LOG_ERROR("portable_storage: wrong binary format - unknown format ver = " << pbuff->m_ver); - return false; - } - TRY_ENTRY(); - throwable_buffer_reader buf_reader(source.data()+sizeof(storage_block_header), source.size()-sizeof(storage_block_header)); - if (limits) - buf_reader.set_limits(limits->n_objects, limits->n_fields, limits->n_strings); - buf_reader.read(m_root); - return true;//TODO: - CATCH_ENTRY("portable_storage::load_from_binary", false); - } - //--------------------------------------------------------------------------------------------------------------- - inline - hsection portable_storage::open_section(const std::string& section_name, hsection hparent_section, bool create_if_notexist) - { - TRY_ENTRY(); - hparent_section = hparent_section ? hparent_section:&m_root; - storage_entry* pentry = find_storage_entry(section_name, hparent_section); - if(!pentry) - { - if(!create_if_notexist) - return nullptr; - return insert_new_section(section_name, hparent_section); - } - CHECK_AND_ASSERT(pentry , nullptr); - //check that section_entry we find is real "CSSection" - if(pentry->type() != typeid(section)) - { - if(create_if_notexist) - *pentry = storage_entry(section());//replace - else - return nullptr; - } - return &boost::get<section>(*pentry); - CATCH_ENTRY("portable_storage::open_section", nullptr); - } - //--------------------------------------------------------------------------------------------------------------- + } + template<class to_type> struct get_value_visitor: boost::static_visitor<void> { @@ -221,20 +149,6 @@ namespace epee //CATCH_ENTRY("portable_storage::template<>get_value", false); } //--------------------------------------------------------------------------------------------------------------- - inline - bool portable_storage::get_value(const std::string& value_name, storage_entry& val, hsection hparent_section) - { - //TRY_ENTRY(); - if(!hparent_section) hparent_section = &m_root; - storage_entry* pentry = find_storage_entry(value_name, hparent_section); - if(!pentry) - return false; - - val = *pentry; - return true; - //CATCH_ENTRY("portable_storage::template<>get_value", false); - } - //--------------------------------------------------------------------------------------------------------------- template<class t_value> bool portable_storage::set_value(const std::string& value_name, t_value&& v, hsection hparent_section) { @@ -256,19 +170,6 @@ namespace epee CATCH_ENTRY("portable_storage::template<>set_value", false); } //--------------------------------------------------------------------------------------------------------------- - inline - storage_entry* portable_storage::find_storage_entry(const std::string& pentry_name, hsection psection) - { - TRY_ENTRY(); - CHECK_AND_ASSERT(psection, nullptr); - auto it = psection->m_entries.find(pentry_name); - if(it == psection->m_entries.end()) - return nullptr; - - return &it->second; - CATCH_ENTRY("portable_storage::find_storage_entry", nullptr); - } - //--------------------------------------------------------------------------------------------------------------- template<class entry_type> storage_entry* portable_storage::insert_new_entry_get_storage_entry(const std::string& pentry_name, hsection psection, entry_type&& entry) { @@ -281,16 +182,6 @@ namespace epee CATCH_ENTRY("portable_storage::insert_new_entry_get_storage_entry", nullptr); } //--------------------------------------------------------------------------------------------------------------- - inline - hsection portable_storage::insert_new_section(const std::string& pentry_name, hsection psection) - { - TRY_ENTRY(); - storage_entry* pse = insert_new_entry_get_storage_entry(pentry_name, psection, section()); - if(!pse) return nullptr; - return &boost::get<section>(*pse); - CATCH_ENTRY("portable_storage::insert_new_section", nullptr); - } - //--------------------------------------------------------------------------------------------------------------- template<class to_type> struct get_first_value_visitor: boost::static_visitor<bool> { @@ -344,7 +235,6 @@ namespace epee } }; - template<class t_value> bool portable_storage::get_next_value(harray hval_array, t_value& target) { @@ -402,83 +292,5 @@ namespace epee return true; CATCH_ENTRY("portable_storage::insert_next_value", false); } - //--------------------------------------------------------------------------------------------------------------- - //sections - inline - harray portable_storage::get_first_section(const std::string& sec_name, hsection& h_child_section, hsection hparent_section) - { - TRY_ENTRY(); - if(!hparent_section) hparent_section = &m_root; - storage_entry* pentry = find_storage_entry(sec_name, hparent_section); - if(!pentry) - return nullptr; - if(pentry->type() != typeid(array_entry)) - return nullptr; - array_entry& ar_entry = boost::get<array_entry>(*pentry); - if(ar_entry.type() != typeid(array_entry_t<section>)) - return nullptr; - array_entry_t<section>& sec_array = boost::get<array_entry_t<section>>(ar_entry); - section* psec = sec_array.get_first_val(); - if(!psec) - return nullptr; - h_child_section = psec; - return &ar_entry; - CATCH_ENTRY("portable_storage::get_first_section", nullptr); - } - //--------------------------------------------------------------------------------------------------------------- - inline - bool portable_storage::get_next_section(harray hsec_array, hsection& h_child_section) - { - TRY_ENTRY(); - CHECK_AND_ASSERT(hsec_array, false); - if(hsec_array->type() != typeid(array_entry_t<section>)) - return false; - array_entry_t<section>& sec_array = boost::get<array_entry_t<section>>(*hsec_array); - h_child_section = sec_array.get_next_val(); - if(!h_child_section) - return false; - return true; - CATCH_ENTRY("portable_storage::get_next_section", false); - } - //--------------------------------------------------------------------------------------------------------------- - inline - harray portable_storage::insert_first_section(const std::string& sec_name, hsection& hinserted_childsection, hsection hparent_section) - { - TRY_ENTRY(); - if(!hparent_section) hparent_section = &m_root; - storage_entry* pentry = find_storage_entry(sec_name, hparent_section); - if(!pentry) - { - pentry = insert_new_entry_get_storage_entry(sec_name, hparent_section, array_entry(array_entry_t<section>())); - if(!pentry) - return nullptr; - } - if(pentry->type() != typeid(array_entry)) - *pentry = storage_entry(array_entry(array_entry_t<section>())); - - array_entry& ar_entry = boost::get<array_entry>(*pentry); - if(ar_entry.type() != typeid(array_entry_t<section>)) - ar_entry = array_entry(array_entry_t<section>()); - - array_entry_t<section>& sec_array = boost::get<array_entry_t<section>>(ar_entry); - hinserted_childsection = &sec_array.insert_first_val(section()); - return &ar_entry; - CATCH_ENTRY("portable_storage::insert_first_section", nullptr); - } - //--------------------------------------------------------------------------------------------------------------- - inline - bool portable_storage::insert_next_section(harray hsec_array, hsection& hinserted_childsection) - { - TRY_ENTRY(); - CHECK_AND_ASSERT(hsec_array, false); - CHECK_AND_ASSERT_MES(hsec_array->type() == typeid(array_entry_t<section>), - false, "unexpected type(not 'section') in insert_next_section, type: " << hsec_array->type().name()); - - array_entry_t<section>& sec_array = boost::get<array_entry_t<section>>(*hsec_array); - hinserted_childsection = &sec_array.insert_next_value(section()); - return true; - CATCH_ENTRY("portable_storage::insert_next_section", false); - } - //--------------------------------------------------------------------------------------------------------------- } } diff --git a/contrib/epee/include/storages/portable_storage_base.h b/contrib/epee/include/storages/portable_storage_base.h index 1676f41fb..ae0be6a34 100644 --- a/contrib/epee/include/storages/portable_storage_base.h +++ b/contrib/epee/include/storages/portable_storage_base.h @@ -29,10 +29,10 @@ #pragma once #include <boost/variant.hpp> -#include <boost/any.hpp> #include <string> #include <vector> #include <deque> +#include <map> #define PORTABLE_STORAGE_SIGNATUREA 0x01011101 #define PORTABLE_STORAGE_SIGNATUREB 0x01020101 // bender's nightmare diff --git a/contrib/epee/include/storages/portable_storage_from_json.h b/contrib/epee/include/storages/portable_storage_from_json.h index 2b2dc7ff9..3021598f5 100644 --- a/contrib/epee/include/storages/portable_storage_from_json.h +++ b/contrib/epee/include/storages/portable_storage_from_json.h @@ -26,6 +26,7 @@ #pragma once #include <boost/lexical_cast.hpp> +#include <boost/utility/string_ref.hpp> #include <boost/algorithm/string/predicate.hpp> #include "parserse_base_utils.h" #include "file_io_utils.h" diff --git a/contrib/epee/include/storages/portable_storage_template_helper.h b/contrib/epee/include/storages/portable_storage_template_helper.h index 16dd565ec..7f6596f36 100644 --- a/contrib/epee/include/storages/portable_storage_template_helper.h +++ b/contrib/epee/include/storages/portable_storage_template_helper.h @@ -29,12 +29,15 @@ #include <string> #include "byte_slice.h" -#include "parserse_base_utils.h" +#include "parserse_base_utils.h" /// TODO: (mj-xmr) This will be reduced in an another PR #include "portable_storage.h" #include "file_io_utils.h" +#include "span.h" namespace epee { + class byte_stream; + namespace serialization { //----------------------------------------------------------------------------------------------------------- @@ -126,5 +129,14 @@ namespace epee store_t_to_binary(str_in, binary_buff, initial_buffer_size); return binary_buff; } + //----------------------------------------------------------------------------------------------------------- + template<class t_struct> + bool store_t_to_binary(t_struct& str_in, byte_stream& binary_buff) + { + portable_storage ps; + str_in.store(ps); + return ps.store_to_binary(binary_buff); + } + } } diff --git a/contrib/epee/include/storages/portable_storage_to_bin.h b/contrib/epee/include/storages/portable_storage_to_bin.h index 49a7be185..b82cf532b 100644 --- a/contrib/epee/include/storages/portable_storage_to_bin.h +++ b/contrib/epee/include/storages/portable_storage_to_bin.h @@ -32,6 +32,7 @@ #include "misc_language.h" #include "portable_storage_base.h" #include "portable_storage_bin_utils.h" +#include "misc_log_ex.h" namespace epee { diff --git a/contrib/epee/include/storages/portable_storage_val_converters.h b/contrib/epee/include/storages/portable_storage_val_converters.h index e54cda828..96b0c024c 100644 --- a/contrib/epee/include/storages/portable_storage_val_converters.h +++ b/contrib/epee/include/storages/portable_storage_val_converters.h @@ -28,12 +28,17 @@ #pragma once -#include <time.h> #include <boost/regex.hpp> #include "misc_language.h" #include "portable_storage_base.h" +#include "parserse_base_utils.h" #include "warnings.h" +#include "misc_log_ex.h" + +#include <boost/lexical_cast.hpp> +#include <typeinfo> +#include <iomanip> namespace epee { diff --git a/contrib/epee/include/string_tools.h b/contrib/epee/include/string_tools.h index 2d9317d60..dbbe1906e 100644 --- a/contrib/epee/include/string_tools.h +++ b/contrib/epee/include/string_tools.h @@ -24,33 +24,16 @@ // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // - - #ifndef _STRING_TOOLS_H_ #define _STRING_TOOLS_H_ -// Previously pulled in by ASIO, further cleanup still required ... -#ifdef _WIN32 -# include <winsock2.h> -# include <windows.h> -#endif - -#include <string.h> -#include <locale> -#include <cstdlib> -#include <string> -#include <type_traits> -#include <boost/lexical_cast.hpp> -#include <boost/algorithm/string/predicate.hpp> -#include <boost/utility/string_ref.hpp> -#include "misc_log_ex.h" -#include "storages/parserse_base_utils.h" #include "hex.h" -#include "memwipe.h" #include "mlocker.h" -#include "span.h" -#include "warnings.h" +#include <boost/utility/string_ref.hpp> +#include <sstream> +#include <string> +#include <cstdint> #ifndef OUT #define OUT @@ -74,202 +57,38 @@ namespace string_tools { return from_hex::to_string(res, s); } - //---------------------------------------------------------------------------- -PUSH_WARNINGS -DISABLE_GCC_WARNING(maybe-uninitialized) - template<class XType> - inline bool get_xtype_from_string(OUT XType& val, const std::string& str_id) - { - if (std::is_integral<XType>::value && !std::numeric_limits<XType>::is_signed && !std::is_same<XType, bool>::value) - { - for (char c : str_id) - { - if (!epee::misc_utils::parse::isdigit(c)) - return false; - } - } - - try - { - val = boost::lexical_cast<XType>(str_id); - return true; - } - catch(const std::exception& /*e*/) - { - //const char* pmsg = e.what(); - return false; - } - catch(...) - { - return false; - } - - return true; - } -POP_WARNINGS - //---------------------------------------------------------------------------- - template<class XType> - inline bool xtype_to_string(const XType& val, std::string& str) - { - try - { - str = boost::lexical_cast<std::string>(val); - } - catch(...) - { - return false; - } - - return true; - } - //---------------------------------------------------------------------------- - std::string get_ip_string_from_int32(uint32_t ip); - //---------------------------------------------------------------------------- - bool get_ip_int32_from_string(uint32_t& ip, const std::string& ip_str); - //---------------------------------------------------------------------------- - inline bool parse_peer_from_string(uint32_t& ip, uint16_t& port, const std::string& addres) - { - //parse ip and address - std::string::size_type p = addres.find(':'); - std::string ip_str, port_str; - if(p == std::string::npos) - { - port = 0; - ip_str = addres; - } - else - { - ip_str = addres.substr(0, p); - port_str = addres.substr(p+1, addres.size()); - } - - if(!get_ip_int32_from_string(ip, ip_str)) - { - return false; - } - - if(p != std::string::npos && !get_xtype_from_string(port, port_str)) - { - return false; - } - return true; - } - - inline std::string num_to_string_fast(int64_t val) - { - /* - char buff[30] = {0}; - i64toa_s(val, buff, sizeof(buff)-1, 10); - return buff;*/ - return boost::lexical_cast<std::string>(val); - } - //---------------------------------------------------------------------------- - template<typename T> - inline std::string to_string_hex(const T &val) - { - static_assert(std::is_arithmetic<T>::value, "only arithmetic types"); - std::stringstream ss; - ss << std::hex << val; - std::string s; - ss >> s; - return s; - } - //---------------------------------------------------------------------------- + + std::string get_ip_string_from_int32(uint32_t ip); + bool get_ip_int32_from_string(uint32_t& ip, const std::string& ip_str); + bool parse_peer_from_string(uint32_t& ip, uint16_t& port, const std::string& addres); + std::string num_to_string_fast(int64_t val); - inline bool compare_no_case(const std::string& str1, const std::string& str2) - { - - return !boost::iequals(str1, str2); - } - //---------------------------------------------------------------------------- - inline std::string& get_current_module_name() - { - static std::string module_name; - return module_name; - } - //---------------------------------------------------------------------------- - inline std::string& get_current_module_folder() - { - static std::string module_folder; - return module_folder; - } - //---------------------------------------------------------------------------- + bool compare_no_case(const std::string& str1, const std::string& str2); + std::string& get_current_module_name(); + std::string& get_current_module_folder(); #ifdef _WIN32 - inline std::string get_current_module_path() - { - char pname [5000] = {0}; - GetModuleFileNameA( NULL, pname, sizeof(pname)); - pname[sizeof(pname)-1] = 0; //be happy ;) - return pname; - } + std::string get_current_module_path(); #endif - //---------------------------------------------------------------------------- - inline bool set_module_name_and_folder(const std::string& path_to_process_) - { - std::string path_to_process = path_to_process_; -#ifdef _WIN32 - path_to_process = get_current_module_path(); -#endif - std::string::size_type a = path_to_process.rfind( '\\' ); - if(a == std::string::npos ) - { - a = path_to_process.rfind( '/' ); - } - if ( a != std::string::npos ) - { - get_current_module_name() = path_to_process.substr(a+1, path_to_process.size()); - get_current_module_folder() = path_to_process.substr(0, a); - return true; - }else - return false; - - } - - //---------------------------------------------------------------------------- - inline bool trim_left(std::string& str) - { - for(std::string::iterator it = str.begin(); it!= str.end() && isspace(static_cast<unsigned char>(*it));) - str.erase(str.begin()); - - return true; - } - //---------------------------------------------------------------------------- - inline bool trim_right(std::string& str) - { - - for(std::string::reverse_iterator it = str.rbegin(); it!= str.rend() && isspace(static_cast<unsigned char>(*it));) - str.erase( --((it++).base())); - - return true; - } - //---------------------------------------------------------------------------- - inline std::string& trim(std::string& str) - { - - trim_left(str); - trim_right(str); - return str; - } + bool set_module_name_and_folder(const std::string& path_to_process_); + bool trim_left(std::string& str); + bool trim_right(std::string& str); //---------------------------------------------------------------------------- - inline std::string trim(const std::string& str_) + inline std::string& trim(std::string& str) { - std::string str = str_; trim_left(str); trim_right(str); return str; } //---------------------------------------------------------------------------- - inline std::string pad_string(std::string s, size_t n, char c = ' ', bool prepend = false) + inline std::string trim(const std::string& str_) { - if (s.size() < n) - { - if (prepend) - s = std::string(n - s.size(), c) + s; - else - s.append(n - s.size(), c); - } - return s; + std::string str = str_; + trim_left(str); + trim_right(str); + return str; } + std::string pad_string(std::string s, size_t n, char c = ' ', bool prepend = false); + //---------------------------------------------------------------------------- template<class t_pod_type> std::string pod_to_hex(const t_pod_type& s) @@ -296,64 +115,25 @@ POP_WARNINGS { return hex_to_pod(hex_str, unwrap(s)); } - //---------------------------------------------------------------------------- - bool validate_hex(uint64_t length, const std::string& str); - //---------------------------------------------------------------------------- - inline std::string get_extension(const std::string& str) - { - std::string res; - std::string::size_type pos = str.rfind('.'); - if(std::string::npos == pos) - return res; - - res = str.substr(pos+1, str.size()-pos); - return res; - } - //---------------------------------------------------------------------------- - inline std::string cut_off_extension(const std::string& str) + //---------------------------------------------------------------------------- + template<typename T> + inline std::string to_string_hex(const T &val) { - std::string res; - std::string::size_type pos = str.rfind('.'); - if(std::string::npos == pos) - return str; - - res = str.substr(0, pos); - return res; + static_assert(std::is_arithmetic<T>::value, "only arithmetic types"); + std::stringstream ss; + ss << std::hex << val; + std::string s; + ss >> s; + return s; } - //---------------------------------------------------------------------------- + + bool validate_hex(uint64_t length, const std::string& str); + std::string get_extension(const std::string& str); + std::string cut_off_extension(const std::string& str); + #ifdef _WIN32 - inline std::wstring utf8_to_utf16(const std::string& str) - { - if (str.empty()) - return {}; - int wstr_size = MultiByteToWideChar(CP_UTF8, 0, &str[0], str.size(), NULL, 0); - if (wstr_size == 0) - { - throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message()); - } - std::wstring wstr(wstr_size, wchar_t{}); - if (!MultiByteToWideChar(CP_UTF8, 0, &str[0], str.size(), &wstr[0], wstr_size)) - { - throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message()); - } - return wstr; - } - inline std::string utf16_to_utf8(const std::wstring& wstr) - { - if (wstr.empty()) - return {}; - int str_size = WideCharToMultiByte(CP_UTF8, 0, &wstr[0], wstr.size(), NULL, 0, NULL, NULL); - if (str_size == 0) - { - throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message()); - } - std::string str(str_size, char{}); - if (!WideCharToMultiByte(CP_UTF8, 0, &wstr[0], wstr.size(), &str[0], str_size, NULL, NULL)) - { - throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message()); - } - return str; - } + std::wstring utf8_to_utf16(const std::string& str); + std::string utf16_to_utf8(const std::wstring& wstr); #endif } } diff --git a/contrib/epee/include/string_tools_lexical.h b/contrib/epee/include/string_tools_lexical.h new file mode 100644 index 000000000..01be562bc --- /dev/null +++ b/contrib/epee/include/string_tools_lexical.h @@ -0,0 +1,91 @@ +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#ifndef _STRING_TOOLS_LEXICAL_H_ +#define _STRING_TOOLS_LEXICAL_H_ + +#include "warnings.h" +#include "storages/parserse_base_utils.h" +#include <boost/lexical_cast.hpp> // A heavy header, that was extracted from the rest + +#ifndef OUT + #define OUT +#endif + +namespace epee +{ +namespace string_tools +{ +PUSH_WARNINGS +DISABLE_GCC_WARNING(maybe-uninitialized) + template<class XType> + inline bool get_xtype_from_string(OUT XType& val, const std::string& str_id) + { + if (std::is_integral<XType>::value && !std::numeric_limits<XType>::is_signed && !std::is_same<XType, bool>::value) + { + for (char c : str_id) + { + if (!epee::misc_utils::parse::isdigit(c)) + return false; + } + } + + try + { + val = boost::lexical_cast<XType>(str_id); + return true; + } + catch(const std::exception& /*e*/) + { + //const char* pmsg = e.what(); + return false; + } + catch(...) + { + return false; + } + + return true; + } +POP_WARNINGS + + template<class XType> + inline bool xtype_to_string(const XType& val, std::string& str) + { + try + { + str = boost::lexical_cast<std::string>(val); + } + catch(...) + { + return false; + } + + return true; + } +} +} +#endif //_STRING_TOOLS_LEXICAL_H_ diff --git a/contrib/epee/include/tiny_ini.h b/contrib/epee/include/tiny_ini.h index 2bc71fc1a..6ced548eb 100644 --- a/contrib/epee/include/tiny_ini.h +++ b/contrib/epee/include/tiny_ini.h @@ -28,8 +28,6 @@ #ifndef _TINY_INI_H_ #define _TINY_INI_H_ -#include <boost/regex.hpp> -#include <boost/lexical_cast.hpp> #include "string_tools.h" namespace epee @@ -37,20 +35,8 @@ namespace epee namespace tiny_ini { - inline - bool get_param_value(const std::string& param_name, const std::string& ini_entry, std::string& res) - { - std::string expr_str = std::string() + "^("+ param_name +") *=(.*?)$"; - const boost::regex match_ini_entry( expr_str, boost::regex::icase | boost::regex::normal); - boost::smatch result; - if(!boost::regex_search(ini_entry, result, match_ini_entry, boost::match_default)) - return false; - res = result[2]; - string_tools::trim(res); - return true; - } - inline - std::string get_param_value(const std::string& param_name, const std::string& ini_entry) + bool get_param_value(const std::string& param_name, const std::string& ini_entry, std::string& res); + inline std::string get_param_value(const std::string& param_name, const std::string& ini_entry) { std::string buff; get_param_value(param_name, ini_entry, buff); diff --git a/contrib/epee/src/CMakeLists.txt b/contrib/epee/src/CMakeLists.txt index 5e101a86a..07f119ba4 100644 --- a/contrib/epee/src/CMakeLists.txt +++ b/contrib/epee/src/CMakeLists.txt @@ -26,13 +26,25 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +set(EPEE_INCLUDE_DIR_BASE "${CMAKE_CURRENT_SOURCE_DIR}/../include") -add_library(epee STATIC byte_slice.cpp byte_stream.cpp hex.cpp abstract_http_client.cpp http_auth.cpp mlog.cpp net_helper.cpp net_utils_base.cpp string_tools.cpp +# Add headers to the file list, to be able to search for them and autosave in IDEs. +monero_find_all_headers(EPEE_HEADERS_PUBLIC "${EPEE_INCLUDE_DIR_BASE}") + +monero_add_library(epee byte_slice.cpp byte_stream.cpp hex.cpp abstract_http_client.cpp http_auth.cpp mlog.cpp net_helper.cpp net_utils_base.cpp string_tools.cpp parserse_base_utils.cpp wipeable_string.cpp levin_base.cpp memwipe.c connection_basic.cpp network_throttle.cpp network_throttle-detail.cpp mlocker.cpp buffer.cpp net_ssl.cpp - int-util.cpp portable_storage.cpp) + int-util.cpp portable_storage.cpp + misc_language.cpp + misc_os_dependent.cpp + file_io_utils.cpp + net_parse_helpers.cpp + http_base.cpp + tiny_ini.cpp + ${EPEE_HEADERS_PUBLIC} + ) if (USE_READLINE AND (GNU_READLINE_FOUND OR (DEPENDS AND NOT MINGW))) - add_library(epee_readline STATIC readline_buffer.cpp) + monero_add_library(epee_readline readline_buffer.cpp) endif() if(HAVE_C11) @@ -60,14 +72,23 @@ target_link_libraries(epee ${Boost_CHRONO_LIBRARY} ${Boost_FILESYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY} - PRIVATE + ${Boost_REGEX_LIBRARY} + ${Boost_SYSTEM_LIBRARY} ${OPENSSL_LIBRARIES} + PRIVATE ${EXTRA_LIBRARIES}) if (USE_READLINE AND (GNU_READLINE_FOUND OR (DEPENDS AND NOT MINGW))) target_link_libraries(epee_readline PUBLIC easylogging + ${Boost_SYSTEM_LIBRARY} PRIVATE ${GNU_READLINE_LIBRARY}) endif() + +target_include_directories(epee + PUBLIC + "${EPEE_INCLUDE_DIR_BASE}" + "${OPENSSL_INCLUDE_DIR}") + diff --git a/contrib/epee/src/abstract_http_client.cpp b/contrib/epee/src/abstract_http_client.cpp index 540917873..3ae09c90e 100644 --- a/contrib/epee/src/abstract_http_client.cpp +++ b/contrib/epee/src/abstract_http_client.cpp @@ -1,6 +1,7 @@ #include "net/abstract_http_client.h" #include "net/http_base.h" #include "net/net_parse_helpers.h" +#include "misc_log_ex.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "net.http" diff --git a/contrib/epee/src/byte_slice.cpp b/contrib/epee/src/byte_slice.cpp index faf7689be..430853c64 100644 --- a/contrib/epee/src/byte_slice.cpp +++ b/contrib/epee/src/byte_slice.cpp @@ -36,6 +36,11 @@ #include "byte_slice.h" #include "byte_stream.h" +namespace +{ + const std::size_t page_size = 4096; +} + namespace epee { struct byte_slice_data @@ -146,7 +151,7 @@ namespace epee : byte_slice() { std::size_t space_needed = 0; - for (const auto source : sources) + for (const auto& source : sources) space_needed += source.size(); if (space_needed) @@ -155,7 +160,7 @@ namespace epee span<std::uint8_t> out{reinterpret_cast<std::uint8_t*>(storage.get() + 1), space_needed}; portion_ = {out.data(), out.size()}; - for (const auto source : sources) + for (const auto& source : sources) { std::memcpy(out.data(), source.data(), source.size()); if (out.remove_prefix(source.size()) < source.size()) @@ -173,16 +178,27 @@ namespace epee : byte_slice(adapt_buffer{}, std::move(buffer)) {} - byte_slice::byte_slice(byte_stream&& stream) noexcept + byte_slice::byte_slice(byte_stream&& stream, const bool shrink) : storage_(nullptr), portion_(stream.data(), stream.size()) { - if (stream.size()) + if (portion_.size()) { - std::uint8_t* const data = stream.take_buffer().release() - sizeof(raw_byte_slice); + byte_buffer buf; + if (shrink && page_size <= stream.available()) + { + buf = byte_buffer_resize(stream.take_buffer(), portion_.size()); + if (!buf) + throw std::bad_alloc{}; + portion_ = {buf.get(), portion_.size()}; + } + else // no need to shrink buffer + buf = stream.take_buffer(); + + std::uint8_t* const data = buf.release() - sizeof(raw_byte_slice); new (data) raw_byte_slice{}; storage_.reset(reinterpret_cast<raw_byte_slice*>(data)); } - else + else // empty stream portion_ = nullptr; } diff --git a/contrib/epee/src/file_io_utils.cpp b/contrib/epee/src/file_io_utils.cpp new file mode 100644 index 000000000..5072adcbc --- /dev/null +++ b/contrib/epee/src/file_io_utils.cpp @@ -0,0 +1,231 @@ +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "file_io_utils.h" + +#include <fstream> +#include <boost/filesystem/path.hpp> +#include <boost/filesystem/operations.hpp> +#ifdef WIN32 +#include <windows.h> +#include "string_tools.h" +#endif + +// On Windows there is a problem with non-ASCII characters in path and file names +// as far as support by the standard components used is concerned: + +// The various file stream classes, e.g. std::ifstream and std::ofstream, are +// part of the GNU C++ Library / libstdc++. On the most basic level they use the +// fopen() call as defined / made accessible to programs compiled within MSYS2 +// by the stdio.h header file maintained by the MinGW project. + +// The critical point: The implementation of fopen() is part of MSVCRT, the +// Microsoft Visual C/C++ Runtime Library, and this method does NOT offer any +// Unicode support. + +// Monero code that would want to continue to use the normal file stream classes +// but WITH Unicode support could therefore not solve this problem on its own, +// but 2 different projects from 2 different maintaining groups would need changes +// in this particular direction - something probably difficult to achieve and +// with a long time to wait until all new versions / releases arrive. + +// Implemented solution approach: Circumvent the problem by stopping to use std +// file stream classes on Windows and directly use Unicode-capable WIN32 API +// calls. Most of the code doing so is concentrated in this header file here. + +namespace epee +{ +namespace file_io_utils +{ + + bool is_file_exist(const std::string& path) + { + boost::filesystem::path p(path); + return boost::filesystem::exists(p); + } + + + bool save_string_to_file(const std::string& path_to_file, const std::string& str) + { +#ifdef WIN32 + std::wstring wide_path; + try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; } + HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL); + if (file_handle == INVALID_HANDLE_VALUE) + return false; + DWORD bytes_written; + DWORD bytes_to_write = (DWORD)str.size(); + BOOL result = WriteFile(file_handle, str.data(), bytes_to_write, &bytes_written, NULL); + CloseHandle(file_handle); + if (bytes_written != bytes_to_write) + result = FALSE; + return result; +#else + try + { + std::ofstream fstream; + fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); + fstream.open(path_to_file, std::ios_base::binary | std::ios_base::out | std::ios_base::trunc); + fstream << str; + fstream.close(); + return true; + } + + catch(...) + { + return false; + } +#endif + } + + + bool get_file_time(const std::string& path_to_file, time_t& ft) + { + boost::system::error_code ec; + ft = boost::filesystem::last_write_time(boost::filesystem::path(path_to_file), ec); + if(!ec) + return true; + else + return false; + } + + + bool set_file_time(const std::string& path_to_file, const time_t& ft) + { + boost::system::error_code ec; + boost::filesystem::last_write_time(boost::filesystem::path(path_to_file), ft, ec); + if(!ec) + return true; + else + return false; + } + + + + bool load_file_to_string(const std::string& path_to_file, std::string& target_str, size_t max_size) + { +#ifdef WIN32 + std::wstring wide_path; + try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; } + HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + if (file_handle == INVALID_HANDLE_VALUE) + return false; + DWORD file_size = GetFileSize(file_handle, NULL); + if ((file_size == INVALID_FILE_SIZE) || (uint64_t)file_size > (uint64_t)max_size) { + CloseHandle(file_handle); + return false; + } + target_str.resize(file_size); + DWORD bytes_read; + BOOL result = ReadFile(file_handle, &target_str[0], file_size, &bytes_read, NULL); + CloseHandle(file_handle); + if (bytes_read != file_size) + result = FALSE; + return result; +#else + try + { + std::ifstream fstream; + fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); + fstream.open(path_to_file, std::ios_base::binary | std::ios_base::in | std::ios::ate); + + std::ifstream::pos_type file_size = fstream.tellg(); + + if((uint64_t)file_size > (uint64_t)max_size) // ensure a large domain for comparison, and negative -> too large + return false;//don't go crazy + size_t file_size_t = static_cast<size_t>(file_size); + + target_str.resize(file_size_t); + + fstream.seekg (0, std::ios::beg); + fstream.read((char*)target_str.data(), target_str.size()); + fstream.close(); + return true; + } + + catch(...) + { + return false; + } +#endif + } + + + bool append_string_to_file(const std::string& path_to_file, const std::string& str) + { + // No special Windows implementation because so far not used in Monero code + try + { + std::ofstream fstream; + fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); + fstream.open(path_to_file.c_str(), std::ios_base::binary | std::ios_base::out | std::ios_base::app); + fstream << str; + fstream.close(); + return true; + } + + catch(...) + { + return false; + } + } + + + bool get_file_size(const std::string& path_to_file, uint64_t &size) + { +#ifdef WIN32 + std::wstring wide_path; + try { wide_path = string_tools::utf8_to_utf16(path_to_file); } catch (...) { return false; } + HANDLE file_handle = CreateFileW(wide_path.c_str(), GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + if (file_handle == INVALID_HANDLE_VALUE) + return false; + LARGE_INTEGER file_size; + BOOL result = GetFileSizeEx(file_handle, &file_size); + CloseHandle(file_handle); + if (result) { + size = file_size.QuadPart; + } + return size; +#else + try + { + std::ifstream fstream; + fstream.exceptions(std::ifstream::failbit | std::ifstream::badbit); + fstream.open(path_to_file, std::ios_base::binary | std::ios_base::in | std::ios::ate); + size = fstream.tellg(); + fstream.close(); + return true; + } + + catch(...) + { + return false; + } +#endif + } + +} +} diff --git a/contrib/epee/src/http_base.cpp b/contrib/epee/src/http_base.cpp new file mode 100644 index 000000000..f6d7568c5 --- /dev/null +++ b/contrib/epee/src/http_base.cpp @@ -0,0 +1,71 @@ +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "net/http_base.h" +#include "memwipe.h" +#include "string_tools.h" + +#include <boost/regex.hpp> +#include <string> +#include <utility> + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "net.http" + +namespace epee +{ +namespace net_utils +{ +namespace http +{ + std::string get_value_from_fields_list(const std::string& param_name, const net_utils::http::fields_list& fields) + { + fields_list::const_iterator it = fields.begin(); + for(; it != fields.end(); ++it) + if(!string_tools::compare_no_case(param_name, it->first)) + break; + + if(it==fields.end()) + return std::string(); + + return it->second; + } + + std::string get_value_from_uri_line(const std::string& param_name, const std::string& uri) + { + std::string buff = "([\\?|&])"; + buff += param_name + "=([^&]*)"; + boost::regex match_param(buff.c_str(), boost::regex::icase | boost::regex::normal); + boost::smatch result; + if(boost::regex_search(uri, result, match_param, boost::match_default) && result[0].matched) + { + return result[2]; + } + return std::string(); + } +} +} +} diff --git a/contrib/epee/src/levin_base.cpp b/contrib/epee/src/levin_base.cpp index 5ec86b3d6..7c5cd5a78 100644 --- a/contrib/epee/src/levin_base.cpp +++ b/contrib/epee/src/levin_base.cpp @@ -34,6 +34,25 @@ namespace epee { namespace levin { + message_writer::message_writer(const std::size_t reserve) + : buffer() + { + buffer.reserve(reserve); + buffer.put_n(0, sizeof(header)); + } + + byte_slice message_writer::finalize(const uint32_t command, const uint32_t flags, const uint32_t return_code, const bool expect_response) + { + if (buffer.size() < sizeof(header)) + throw std::runtime_error{"levin_writer::finalize already called"}; + + header head = make_header(command, payload_size(), flags, expect_response); + head.m_return_code = SWAP32LE(return_code); + + std::memcpy(buffer.tellp() - buffer.size(), std::addressof(head), sizeof(head)); + return byte_slice{std::move(buffer)}; + } + bucket_head2 make_header(uint32_t command, uint64_t msg_size, uint32_t flags, bool expect_response) noexcept { bucket_head2 head = {0}; @@ -47,12 +66,6 @@ namespace levin return head; } - byte_slice make_notify(int command, epee::span<const std::uint8_t> payload) - { - const bucket_head2 head = make_header(command, payload.size(), LEVIN_PACKET_REQUEST, false); - return byte_slice{epee::as_byte_span(head), payload}; - } - byte_slice make_noise_notify(const std::size_t noise_bytes) { static constexpr const std::uint32_t flags = @@ -68,46 +81,40 @@ namespace levin return byte_slice{std::move(buffer)}; } - byte_slice make_fragmented_notify(const byte_slice& noise_message, int command, epee::span<const std::uint8_t> payload) + byte_slice make_fragmented_notify(const std::size_t noise_size, const int command, message_writer message) { - const size_t noise_size = noise_message.size(); if (noise_size < sizeof(bucket_head2) * 2) return nullptr; - if (payload.size() <= noise_size - sizeof(bucket_head2)) + if (message.buffer.size() <= noise_size) { /* The entire message can be sent at once, and the levin binary parser will ignore extra bytes. So just pad with zeroes and otherwise send a "normal", not fragmented message. */ - const size_t padding = noise_size - sizeof(bucket_head2) - payload.size(); - const span<const uint8_t> padding_bytes{noise_message.end() - padding, padding}; - const bucket_head2 head = make_header(command, noise_size - sizeof(bucket_head2), LEVIN_PACKET_REQUEST, false); - return byte_slice{as_byte_span(head), payload, padding_bytes}; + message.buffer.put_n(0, noise_size - message.buffer.size()); + return message.finalize_notify(command); } // fragment message + const byte_slice payload_bytes = message.finalize_notify(command); + span<const std::uint8_t> payload = to_span(payload_bytes); + const size_t payload_space = noise_size - sizeof(bucket_head2); const size_t expected_fragments = ((payload.size() - 2) / payload_space) + 1; - std::string buffer{}; - buffer.reserve((expected_fragments + 1) * noise_size); // +1 here overselects for internal bucket_head2 value + byte_stream buffer{}; + buffer.reserve(expected_fragments * noise_size); - bucket_head2 head = make_header(0, noise_size - sizeof(bucket_head2), LEVIN_PACKET_BEGIN, false); - buffer.append(reinterpret_cast<const char*>(&head), sizeof(head)); + bucket_head2 head = make_header(0, payload_space, LEVIN_PACKET_BEGIN, false); + buffer.write(as_byte_span(head)); - head.m_command = command; - head.m_flags = LEVIN_PACKET_REQUEST; - head.m_cb = payload.size(); - buffer.append(reinterpret_cast<const char*>(&head), sizeof(head)); + // internal levin header is in payload already - size_t copy_size = payload.remove_prefix(payload_space - sizeof(bucket_head2)); - buffer.append(reinterpret_cast<const char*>(payload.data()) - copy_size, copy_size); + size_t copy_size = payload.remove_prefix(payload_space); + buffer.write(payload.data() - copy_size, copy_size); - head.m_command = 0; head.m_flags = 0; - head.m_cb = noise_size - sizeof(bucket_head2); - while (!payload.empty()) { copy_size = payload.remove_prefix(payload_space); @@ -115,12 +122,12 @@ namespace levin if (payload.empty()) head.m_flags = LEVIN_PACKET_END; - buffer.append(reinterpret_cast<const char*>(&head), sizeof(head)); - buffer.append(reinterpret_cast<const char*>(payload.data()) - copy_size, copy_size); + buffer.write(as_byte_span(head)); + buffer.write(payload.data() - copy_size, copy_size); } const size_t padding = noise_size - copy_size - sizeof(bucket_head2); - buffer.append(reinterpret_cast<const char*>(noise_message.end()) - padding, padding); + buffer.put_n(0, padding); return byte_slice{std::move(buffer)}; } diff --git a/contrib/epee/src/misc_language.cpp b/contrib/epee/src/misc_language.cpp new file mode 100644 index 000000000..6e8f2daef --- /dev/null +++ b/contrib/epee/src/misc_language.cpp @@ -0,0 +1,44 @@ +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "misc_language.h" + +#include <boost/thread.hpp> + +namespace epee +{ +namespace misc_utils +{ + bool sleep_no_w(long ms ) + { + boost::this_thread::sleep( + boost::get_system_time() + + boost::posix_time::milliseconds( std::max<long>(ms,0) ) ); + + return true; + } +} +} diff --git a/contrib/epee/src/misc_os_dependent.cpp b/contrib/epee/src/misc_os_dependent.cpp new file mode 100644 index 000000000..cd4967131 --- /dev/null +++ b/contrib/epee/src/misc_os_dependent.cpp @@ -0,0 +1,44 @@ +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "misc_os_dependent.h" +#include <boost/lexical_cast.hpp> + +namespace epee +{ +namespace misc_utils +{ + // TODO: (vtnerd) This function is weird since boost::this_thread::get_id() exists but returns a different value. + std::string get_thread_string_id() + { +#if defined(_WIN32) + return boost::lexical_cast<std::string>(GetCurrentThreadId()); +#elif defined(__GNUC__) + return boost::lexical_cast<std::string>(pthread_self()); +#endif + } +} +} diff --git a/contrib/epee/src/net_parse_helpers.cpp b/contrib/epee/src/net_parse_helpers.cpp new file mode 100644 index 000000000..2697bdac4 --- /dev/null +++ b/contrib/epee/src/net_parse_helpers.cpp @@ -0,0 +1,206 @@ +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "net/net_parse_helpers.h" +#include "net/http_base.h" +#include "misc_log_ex.h" +#include "reg_exp_definer.h" +#include <boost/lexical_cast.hpp> + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "net" + +namespace epee +{ +namespace net_utils +{ + + static bool parse_uri_query(const std::string& query, std::list<std::pair<std::string, std::string> >& params) + { + enum state + { + st_param_name, + st_param_val + }; + state st = st_param_name; + std::string::const_iterator start_it = query.begin(); + std::pair<std::string, std::string> e; + for(std::string::const_iterator it = query.begin(); it != query.end(); ++it) + { + switch(st) + { + case st_param_name: + if(*it == '=') + { + e.first.assign(start_it, it); + start_it = it;++start_it; + st = st_param_val; + } + break; + case st_param_val: + if(*it == '&') + { + e.second.assign(start_it, it); + start_it = it;++start_it; + params.push_back(e); + e.first.clear();e.second.clear(); + st = st_param_name; + } + break; + default: + LOG_ERROR("Unknown state " << (int)st); + return false; + } + } + if(st == st_param_name) + { + if(start_it != query.end()) + { + e.first.assign(start_it, query.end()); + params.push_back(e); + } + }else + { + if(start_it != query.end()) + e.second.assign(start_it, query.end()); + + if(e.first.size()) + params.push_back(e); + } + return true; + } + + bool parse_uri(const std::string uri, http::uri_content& content) + { + + ///iframe_test.html?api_url=http://api.vk.com/api.php&api_id=3289090&api_settings=1&viewer_id=562964060&viewer_type=0&sid=0aad8d1c5713130f9ca0076f2b7b47e532877424961367d81e7fa92455f069be7e21bc3193cbd0be11895&secret=368ebbc0ef&access_token=668bc03f43981d883f73876ffff4aa8564254b359cc745dfa1b3cde7bdab2e94105d8f6d8250717569c0a7&user_id=0&group_id=0&is_app_user=1&auth_key=d2f7a895ca5ff3fdb2a2a8ae23fe679a&language=0&parent_language=0&ad_info=ElsdCQBaQlxiAQRdFUVUXiN2AVBzBx5pU1BXIgZUJlIEAWcgAUoLQg==&referrer=unknown&lc_name=9834b6a3&hash= + content.m_query_params.clear(); + STATIC_REGEXP_EXPR_1(rexp_match_uri, "^([^?#]*)(\\?([^#]*))?(#(.*))?", boost::regex::icase | boost::regex::normal); + + boost::smatch result; + if(!(boost::regex_search(uri, result, rexp_match_uri, boost::match_default) && result[0].matched)) + { + LOG_PRINT_L1("[PARSE URI] regex not matched for uri: " << uri); + content.m_path = uri; + return true; + } + if(result[1].matched) + { + content.m_path = result[1]; + } + if(result[3].matched) + { + content.m_query = result[3]; + } + if(result[5].matched) + { + content.m_fragment = result[5]; + } + if(content.m_query.size()) + { + parse_uri_query(content.m_query, content.m_query_params); + } + return true; + } + + bool parse_url_ipv6(const std::string url_str, http::url_content& content) + { + STATIC_REGEXP_EXPR_1(rexp_match_uri, "^(([^:]*?)://)?(\\[(.*)\\](:(\\d+))?)(.*)?", boost::regex::icase | boost::regex::normal); + // 12 3 4 5 6 7 + + content.port = 0; + boost::smatch result; + if(!(boost::regex_search(url_str, result, rexp_match_uri, boost::match_default) && result[0].matched)) + { + LOG_PRINT_L1("[PARSE URI] regex not matched for uri: " << rexp_match_uri); + //content.m_path = uri; + return false; + } + if(result[2].matched) + { + content.schema = result[2]; + } + if(result[4].matched) + { + content.host = result[4]; + } + else // if host not matched, matching should be considered failed + { + return false; + } + if(result[6].matched) + { + content.port = boost::lexical_cast<uint64_t>(result[6]); + } + if(result[7].matched) + { + content.uri = result[7]; + return parse_uri(result[7], content.m_uri_content); + } + + return true; + } + + bool parse_url(const std::string url_str, http::url_content& content) + { + + if (parse_url_ipv6(url_str, content)) return true; + + ///iframe_test.html?api_url=http://api.vk.com/api.php&api_id=3289090&api_settings=1&viewer_id=562964060&viewer_type=0&sid=0aad8d1c5713130f9ca0076f2b7b47e532877424961367d81e7fa92455f069be7e21bc3193cbd0be11895&secret=368ebbc0ef&access_token=668bc03f43981d883f73876ffff4aa8564254b359cc745dfa1b3cde7bdab2e94105d8f6d8250717569c0a7&user_id=0&group_id=0&is_app_user=1&auth_key=d2f7a895ca5ff3fdb2a2a8ae23fe679a&language=0&parent_language=0&ad_info=ElsdCQBaQlxiAQRdFUVUXiN2AVBzBx5pU1BXIgZUJlIEAWcgAUoLQg==&referrer=unknown&lc_name=9834b6a3&hash= + //STATIC_REGEXP_EXPR_1(rexp_match_uri, "^([^?#]*)(\\?([^#]*))?(#(.*))?", boost::regex::icase | boost::regex::normal); + STATIC_REGEXP_EXPR_1(rexp_match_uri, "^(([^:]*?)://)?(([^/:]*)(:(\\d+))?)(.*)?", boost::regex::icase | boost::regex::normal); + // 12 34 5 6 7 + content.port = 0; + boost::smatch result; + if(!(boost::regex_search(url_str, result, rexp_match_uri, boost::match_default) && result[0].matched)) + { + LOG_PRINT_L1("[PARSE URI] regex not matched for uri: " << rexp_match_uri); + //content.m_path = uri; + return true; + } + if(result[2].matched) + { + content.schema = result[2]; + } + if(result[4].matched) + { + content.host = result[4]; + } + if(result[6].matched) + { + content.port = boost::lexical_cast<uint64_t>(result[6]); + } + if(result[7].matched) + { + content.uri = result[7]; + return parse_uri(result[7], content.m_uri_content); + } + + return true; + } + +} +} diff --git a/contrib/epee/src/net_ssl.cpp b/contrib/epee/src/net_ssl.cpp index 6ed27efa9..70cd0d0ad 100644 --- a/contrib/epee/src/net_ssl.cpp +++ b/contrib/epee/src/net_ssl.cpp @@ -29,6 +29,8 @@ #include <string.h> #include <thread> #include <boost/asio/ssl.hpp> +#include <boost/cerrno.hpp> +#include <boost/filesystem/operations.hpp> #include <boost/lambda/lambda.hpp> #include <openssl/ssl.h> #include <openssl/pem.h> @@ -567,6 +569,52 @@ bool ssl_support_from_string(ssl_support_t &ssl, boost::string_ref s) return true; } +boost::system::error_code store_ssl_keys(boost::asio::ssl::context& ssl, const boost::filesystem::path& base) +{ + EVP_PKEY* ssl_key = nullptr; + X509* ssl_cert = nullptr; + const auto ctx = ssl.native_handle(); + CHECK_AND_ASSERT_MES(ctx, boost::system::error_code(EINVAL, boost::system::system_category()), "Context is null"); + CHECK_AND_ASSERT_MES(base.has_filename(), boost::system::error_code(EINVAL, boost::system::system_category()), "Need filename"); + std::unique_ptr<SSL, decltype(&SSL_free)> dflt_SSL(SSL_new(ctx), SSL_free); + if (!dflt_SSL || !(ssl_key = SSL_get_privatekey(dflt_SSL.get())) || !(ssl_cert = SSL_get_certificate(dflt_SSL.get()))) + return {EINVAL, boost::system::system_category()}; + + using file_closer = int(std::FILE*); + boost::system::error_code error{}; + std::unique_ptr<std::FILE, file_closer*> file{nullptr, std::fclose}; + + // write key file unencrypted + { + const boost::filesystem::path key_file{base.string() + ".key"}; + file.reset(std::fopen(key_file.string().c_str(), "wb")); + if (!file) + return {errno, boost::system::system_category()}; + boost::filesystem::permissions(key_file, boost::filesystem::owner_read, error); + if (error) + return error; + if (!PEM_write_PrivateKey(file.get(), ssl_key, nullptr, nullptr, 0, nullptr, nullptr)) + return boost::asio::error::ssl_errors(ERR_get_error()); + if (std::fclose(file.release()) != 0) + return {errno, boost::system::system_category()}; + } + + // write certificate file in standard SSL X.509 unencrypted + const boost::filesystem::path cert_file{base.string() + ".crt"}; + file.reset(std::fopen(cert_file.string().c_str(), "wb")); + if (!file) + return {errno, boost::system::system_category()}; + const auto cert_perms = (boost::filesystem::owner_read | boost::filesystem::group_read | boost::filesystem::others_read); + boost::filesystem::permissions(cert_file, cert_perms, error); + if (error) + return error; + if (!PEM_write_X509(file.get(), ssl_cert)) + return boost::asio::error::ssl_errors(ERR_get_error()); + if (std::fclose(file.release()) != 0) + return {errno, boost::system::system_category()}; + return error; +} + } // namespace } // namespace diff --git a/contrib/epee/src/parserse_base_utils.cpp b/contrib/epee/src/parserse_base_utils.cpp new file mode 100644 index 000000000..e96c2dede --- /dev/null +++ b/contrib/epee/src/parserse_base_utils.cpp @@ -0,0 +1,282 @@ +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "storages/parserse_base_utils.h" + +#include "misc_log_ex.h" +#include <boost/utility/string_ref.hpp> +#include <algorithm> + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "serialization" + +namespace epee +{ +namespace misc_utils +{ + namespace parse + { + std::string transform_to_escape_sequence(const std::string& src) + { + static const char escaped[] = "\b\f\n\r\t\v\"\\/"; + std::string::const_iterator it = std::find_first_of(src.begin(), src.end(), escaped, escaped + sizeof(escaped)); + if (it == src.end()) + return src; + + std::string res; + res.reserve(2 * src.size()); + res.assign(src.begin(), it); + for(; it!=src.end(); ++it) + { + switch(*it) + { + case '\b': //Backspace (ascii code 08) + res+="\\b"; break; + case '\f': //Form feed (ascii code 0C) + res+="\\f"; break; + case '\n': //New line + res+="\\n"; break; + case '\r': //Carriage return + res+="\\r"; break; + case '\t': //Tab + res+="\\t"; break; + case '\v': //Vertical tab + res+="\\v"; break; + //case '\'': //Apostrophe or single quote + // res+="\\'"; break; + case '"': //Double quote + res+="\\\""; break; + case '\\': //Backslash caracter + res+="\\\\"; break; + case '/': //Backslash caracter + res+="\\/"; break; + default: + res.push_back(*it); + } + } + return res; + } + /* + + \b Backspace (ascii code 08) + \f Form feed (ascii code 0C) + \n New line + \r Carriage return + \t Tab + \v Vertical tab + \' Apostrophe or single quote + \" Double quote + \\ Backslash character + + */ + void match_string2(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, std::string& val) + { + bool escape_mode = false; + std::string::const_iterator it = star_end_string; + ++it; + std::string::const_iterator fi = it; + while (fi != buf_end && ((lut[(uint8_t)*fi] & 32)) == 0) + ++fi; + val.assign(it, fi); + it = fi; + for(;it != buf_end;++it) + { + if(escape_mode/*prev_ch == '\\'*/) + { + switch(*it) + { + case 'b': //Backspace (ascii code 08) + val.push_back(0x08);break; + case 'f': //Form feed (ascii code 0C) + val.push_back(0x0C);break; + case 'n': //New line + val.push_back('\n');break; + case 'r': //Carriage return + val.push_back('\r');break; + case 't': //Tab + val.push_back('\t');break; + case 'v': //Vertical tab + val.push_back('\v');break; + case '\'': //Apostrophe or single quote + val.push_back('\'');break; + case '"': //Double quote + val.push_back('"');break; + case '\\': //Backslash character + val.push_back('\\');break; + case '/': //Slash character + val.push_back('/');break; + case 'u': //Unicode code point + if (buf_end - it < 4) + { + ASSERT_MES_AND_THROW("Invalid Unicode escape sequence"); + } + else + { + uint32_t dst = 0; + for (int i = 0; i < 4; ++i) + { + const unsigned char tmp = isx[(unsigned char)*++it]; + CHECK_AND_ASSERT_THROW_MES(tmp != 0xff, "Bad Unicode encoding"); + dst = dst << 4 | tmp; + } + // encode as UTF-8 + if (dst <= 0x7f) + { + val.push_back(dst); + } + else if (dst <= 0x7ff) + { + val.push_back(0xc0 | (dst >> 6)); + val.push_back(0x80 | (dst & 0x3f)); + } + else if (dst <= 0xffff) + { + val.push_back(0xe0 | (dst >> 12)); + val.push_back(0x80 | ((dst >> 6) & 0x3f)); + val.push_back(0x80 | (dst & 0x3f)); + } + else + { + ASSERT_MES_AND_THROW("Unicode code point is out or range"); + } + } + break; + default: + val.push_back(*it); + LOG_PRINT_L0("Unknown escape sequence :\"\\" << *it << "\""); + } + escape_mode = false; + }else if(*it == '"') + { + star_end_string = it; + return; + }else if(*it == '\\') + { + escape_mode = true; + } + else + { + val.push_back(*it); + } + } + ASSERT_MES_AND_THROW("Failed to match string in json entry: " << std::string(star_end_string, buf_end)); + } + void match_number2(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, boost::string_ref& val, bool& is_float_val, bool& is_signed_val) + { + val.clear(); + uint8_t float_flag = 0; + is_signed_val = false; + size_t chars = 0; + std::string::const_iterator it = star_end_string; + if (it != buf_end && *it == '-') + { + is_signed_val = true; + ++chars; + ++it; + } + for(;it != buf_end;++it) + { + const uint8_t flags = lut[(uint8_t)*it]; + if (flags & 16) + { + float_flag |= flags; + ++chars; + } + else + { + val = boost::string_ref(&*star_end_string, chars); + if(val.size()) + { + star_end_string = --it; + is_float_val = !!(float_flag & 2); + return; + } + else + ASSERT_MES_AND_THROW("wrong number in json entry: " << std::string(star_end_string, buf_end)); + } + } + ASSERT_MES_AND_THROW("wrong number in json entry: " << std::string(star_end_string, buf_end)); + } + void match_word2(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, boost::string_ref& val) + { + val.clear(); + + for(std::string::const_iterator it = star_end_string;it != buf_end;++it) + { + if (!(lut[(uint8_t)*it] & 4)) + { + val = boost::string_ref(&*star_end_string, std::distance(star_end_string, it)); + if(val.size()) + { + star_end_string = --it; + return; + }else + ASSERT_MES_AND_THROW("failed to match word number in json entry: " << std::string(star_end_string, buf_end)); + } + } + ASSERT_MES_AND_THROW("failed to match word number in json entry: " << std::string(star_end_string, buf_end)); + } + bool match_word_with_extrasymb(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, std::string& val) + { + val.clear(); + + for(std::string::const_iterator it = star_end_string;it != buf_end;++it) + { + if(!isalnum(*it) && *it != '-' && *it != '_') + { + val.assign(star_end_string, it); + if(val.size()) + { + star_end_string = --it; + return true; + }else + return false; + } + } + return false; + } + bool match_word_til_equal_mark(std::string::const_iterator& star_end_string, std::string::const_iterator buf_end, std::string::const_iterator& word_end) + { + word_end = star_end_string; + + for(std::string::const_iterator it = star_end_string;it != buf_end;++it) + { + if(isspace(*it)) + { + + continue; + }else if( *it == '=' ) + { + star_end_string = it; + word_end = it; + return true; + } + } + return false; + } + } +} +} diff --git a/contrib/epee/src/portable_storage.cpp b/contrib/epee/src/portable_storage.cpp index 4534deff3..b922cc9e3 100644 --- a/contrib/epee/src/portable_storage.cpp +++ b/contrib/epee/src/portable_storage.cpp @@ -1,10 +1,43 @@ +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// #include "byte_slice.h" #include "byte_stream.h" #include "misc_log_ex.h" #include "span.h" #include "storages/portable_storage.h" +#include "storages/portable_storage_to_json.h" +#include "storages/portable_storage_from_json.h" #include "storages/portable_storage_to_bin.h" +#include "storages/portable_storage_from_bin.h" + +#include <boost/utility/string_ref.hpp> + +#include <string> +#include <sstream> namespace epee { @@ -15,15 +48,200 @@ namespace serialization TRY_ENTRY(); byte_stream ss; ss.reserve(initial_buffer_size); + store_to_binary(ss); + target = epee::byte_slice{std::move(ss)}; + return true; + CATCH_ENTRY("portable_storage::store_to_binary", false); + } + + bool portable_storage::store_to_binary(byte_stream& ss) + { + TRY_ENTRY(); storage_block_header sbh{}; sbh.m_signature_a = SWAP32LE(PORTABLE_STORAGE_SIGNATUREA); sbh.m_signature_b = SWAP32LE(PORTABLE_STORAGE_SIGNATUREB); sbh.m_ver = PORTABLE_STORAGE_FORMAT_VER; ss.write(epee::as_byte_span(sbh)); pack_entry_to_buff(ss, m_root); - target = epee::byte_slice{std::move(ss)}; return true; - CATCH_ENTRY("portable_storage::store_to_binary", false) + CATCH_ENTRY("portable_storage::store_to_binary", false); } + + bool portable_storage::dump_as_json(std::string& buff, size_t indent, bool insert_newlines) + { + TRY_ENTRY(); + std::stringstream ss; + epee::serialization::dump_as_json(ss, m_root, indent, insert_newlines); + buff = ss.str(); + return true; + CATCH_ENTRY("portable_storage::dump_as_json", false) + } + + bool portable_storage::load_from_json(const std::string& source) + { + TRY_ENTRY(); + return json::load_from_json(source, *this); + CATCH_ENTRY("portable_storage::load_from_json", false) + } + + bool portable_storage::load_from_binary(const epee::span<const uint8_t> source, const limits_t *limits) + { + m_root.m_entries.clear(); + if(source.size() < sizeof(storage_block_header)) + { + LOG_ERROR("portable_storage: wrong binary format, packet size = " << source.size() << " less than expected sizeof(storage_block_header)=" << sizeof(storage_block_header)); + return false; + } + storage_block_header* pbuff = (storage_block_header*)source.data(); + if(pbuff->m_signature_a != SWAP32LE(PORTABLE_STORAGE_SIGNATUREA) || + pbuff->m_signature_b != SWAP32LE(PORTABLE_STORAGE_SIGNATUREB) + ) + { + LOG_ERROR("portable_storage: wrong binary format - signature mismatch"); + return false; + } + if(pbuff->m_ver != PORTABLE_STORAGE_FORMAT_VER) + { + LOG_ERROR("portable_storage: wrong binary format - unknown format ver = " << pbuff->m_ver); + return false; + } + TRY_ENTRY(); + throwable_buffer_reader buf_reader(source.data()+sizeof(storage_block_header), source.size()-sizeof(storage_block_header)); + if (limits) + buf_reader.set_limits(limits->n_objects, limits->n_fields, limits->n_strings); + buf_reader.read(m_root); + return true;//TODO: + CATCH_ENTRY("portable_storage::load_from_binary", false); + } + + hsection portable_storage::open_section(const std::string& section_name, hsection hparent_section, bool create_if_notexist) + { + TRY_ENTRY(); + hparent_section = hparent_section ? hparent_section:&m_root; + storage_entry* pentry = find_storage_entry(section_name, hparent_section); + if(!pentry) + { + if(!create_if_notexist) + return nullptr; + return insert_new_section(section_name, hparent_section); + } + CHECK_AND_ASSERT(pentry , nullptr); + //check that section_entry we find is real "CSSection" + if(pentry->type() != typeid(section)) + { + if(create_if_notexist) + *pentry = storage_entry(section());//replace + else + return nullptr; + } + return &boost::get<section>(*pentry); + CATCH_ENTRY("portable_storage::open_section", nullptr); + } + + bool portable_storage::get_value(const std::string& value_name, storage_entry& val, hsection hparent_section) + { + //TRY_ENTRY(); + if(!hparent_section) hparent_section = &m_root; + storage_entry* pentry = find_storage_entry(value_name, hparent_section); + if(!pentry) + return false; + + val = *pentry; + return true; + //CATCH_ENTRY("portable_storage::template<>get_value", false); + } + + storage_entry* portable_storage::find_storage_entry(const std::string& pentry_name, hsection psection) + { + TRY_ENTRY(); + CHECK_AND_ASSERT(psection, nullptr); + auto it = psection->m_entries.find(pentry_name); + if(it == psection->m_entries.end()) + return nullptr; + + return &it->second; + CATCH_ENTRY("portable_storage::find_storage_entry", nullptr); + } + + hsection portable_storage::insert_new_section(const std::string& pentry_name, hsection psection) + { + TRY_ENTRY(); + storage_entry* pse = insert_new_entry_get_storage_entry(pentry_name, psection, section()); + if(!pse) return nullptr; + return &boost::get<section>(*pse); + CATCH_ENTRY("portable_storage::insert_new_section", nullptr); + } + + harray portable_storage::get_first_section(const std::string& sec_name, hsection& h_child_section, hsection hparent_section) + { + TRY_ENTRY(); + if(!hparent_section) hparent_section = &m_root; + storage_entry* pentry = find_storage_entry(sec_name, hparent_section); + if(!pentry) + return nullptr; + if(pentry->type() != typeid(array_entry)) + return nullptr; + array_entry& ar_entry = boost::get<array_entry>(*pentry); + if(ar_entry.type() != typeid(array_entry_t<section>)) + return nullptr; + array_entry_t<section>& sec_array = boost::get<array_entry_t<section>>(ar_entry); + section* psec = sec_array.get_first_val(); + if(!psec) + return nullptr; + h_child_section = psec; + return &ar_entry; + CATCH_ENTRY("portable_storage::get_first_section", nullptr); + } + + bool portable_storage::get_next_section(harray hsec_array, hsection& h_child_section) + { + TRY_ENTRY(); + CHECK_AND_ASSERT(hsec_array, false); + if(hsec_array->type() != typeid(array_entry_t<section>)) + return false; + array_entry_t<section>& sec_array = boost::get<array_entry_t<section>>(*hsec_array); + h_child_section = sec_array.get_next_val(); + if(!h_child_section) + return false; + return true; + CATCH_ENTRY("portable_storage::get_next_section", false); + } + + harray portable_storage::insert_first_section(const std::string& sec_name, hsection& hinserted_childsection, hsection hparent_section) + { + TRY_ENTRY(); + if(!hparent_section) hparent_section = &m_root; + storage_entry* pentry = find_storage_entry(sec_name, hparent_section); + if(!pentry) + { + pentry = insert_new_entry_get_storage_entry(sec_name, hparent_section, array_entry(array_entry_t<section>())); + if(!pentry) + return nullptr; + } + if(pentry->type() != typeid(array_entry)) + *pentry = storage_entry(array_entry(array_entry_t<section>())); + + array_entry& ar_entry = boost::get<array_entry>(*pentry); + if(ar_entry.type() != typeid(array_entry_t<section>)) + ar_entry = array_entry(array_entry_t<section>()); + + array_entry_t<section>& sec_array = boost::get<array_entry_t<section>>(ar_entry); + hinserted_childsection = &sec_array.insert_first_val(section()); + return &ar_entry; + CATCH_ENTRY("portable_storage::insert_first_section", nullptr); + } + + bool portable_storage::insert_next_section(harray hsec_array, hsection& hinserted_childsection) + { + TRY_ENTRY(); + CHECK_AND_ASSERT(hsec_array, false); + CHECK_AND_ASSERT_MES(hsec_array->type() == typeid(array_entry_t<section>), + false, "unexpected type(not 'section') in insert_next_section, type: " << hsec_array->type().name()); + + array_entry_t<section>& sec_array = boost::get<array_entry_t<section>>(*hsec_array); + hinserted_childsection = &sec_array.insert_next_value(section()); + return true; + CATCH_ENTRY("portable_storage::insert_next_section", false); + } } } diff --git a/contrib/epee/src/string_tools.cpp b/contrib/epee/src/string_tools.cpp index fd0254016..984a151b5 100644 --- a/contrib/epee/src/string_tools.cpp +++ b/contrib/epee/src/string_tools.cpp @@ -25,6 +25,29 @@ // #include "string_tools.h" +#include "string_tools_lexical.h" + + +// Previously pulled in by ASIO, further cleanup still required ... +#ifdef _WIN32 +# include <winsock2.h> +# include <windows.h> +#endif + +#include <locale> +#include <cstdlib> +#include <string> +#include <type_traits> +#include <boost/lexical_cast.hpp> +#include <boost/algorithm/string/predicate.hpp> +#include <boost/utility/string_ref.hpp> +#include "misc_log_ex.h" +#include "storages/parserse_base_utils.h" +#include "hex.h" +#include "memwipe.h" +#include "mlocker.h" +#include "span.h" +#include "warnings.h" #include <ctype.h> @@ -68,6 +91,180 @@ namespace string_tools return false; return true; } + //---------------------------------------------------------------------------- + bool parse_peer_from_string(uint32_t& ip, uint16_t& port, const std::string& addres) + { + //parse ip and address + std::string::size_type p = addres.find(':'); + std::string ip_str, port_str; + if(p == std::string::npos) + { + port = 0; + ip_str = addres; + } + else + { + ip_str = addres.substr(0, p); + port_str = addres.substr(p+1, addres.size()); + } + + if(!get_ip_int32_from_string(ip, ip_str)) + { + return false; + } + + if(p != std::string::npos && !get_xtype_from_string(port, port_str)) + { + return false; + } + return true; + } + + std::string num_to_string_fast(int64_t val) + { + /* + char buff[30] = {0}; + i64toa_s(val, buff, sizeof(buff)-1, 10); + return buff;*/ + return boost::lexical_cast<std::string>(val); + } + + + bool compare_no_case(const std::string& str1, const std::string& str2) + { + + return !boost::iequals(str1, str2); + } + //---------------------------------------------------------------------------- + std::string& get_current_module_name() + { + static std::string module_name; + return module_name; + } + //---------------------------------------------------------------------------- + std::string& get_current_module_folder() + { + static std::string module_folder; + return module_folder; + } + +#ifdef _WIN32 + std::string get_current_module_path() + { + char pname [5000] = {0}; + GetModuleFileNameA( NULL, pname, sizeof(pname)); + pname[sizeof(pname)-1] = 0; //be happy ;) + return pname; + } +#endif + + bool set_module_name_and_folder(const std::string& path_to_process_) + { + std::string path_to_process = path_to_process_; +#ifdef _WIN32 + path_to_process = get_current_module_path(); +#endif + std::string::size_type a = path_to_process.rfind( '\\' ); + if(a == std::string::npos ) + { + a = path_to_process.rfind( '/' ); + } + if ( a != std::string::npos ) + { + get_current_module_name() = path_to_process.substr(a+1, path_to_process.size()); + get_current_module_folder() = path_to_process.substr(0, a); + return true; + }else + return false; + + } + + //---------------------------------------------------------------------------- + bool trim_left(std::string& str) + { + for(std::string::iterator it = str.begin(); it!= str.end() && isspace(static_cast<unsigned char>(*it));) + str.erase(str.begin()); + + return true; + } + //---------------------------------------------------------------------------- + bool trim_right(std::string& str) + { + + for(std::string::reverse_iterator it = str.rbegin(); it!= str.rend() && isspace(static_cast<unsigned char>(*it));) + str.erase( --((it++).base())); + + return true; + } + //---------------------------------------------------------------------------- + std::string pad_string(std::string s, size_t n, char c, bool prepend) + { + if (s.size() < n) + { + if (prepend) + s = std::string(n - s.size(), c) + s; + else + s.append(n - s.size(), c); + } + return s; + } + + std::string get_extension(const std::string& str) + { + std::string res; + std::string::size_type pos = str.rfind('.'); + if(std::string::npos == pos) + return res; + + res = str.substr(pos+1, str.size()-pos); + return res; + } + //---------------------------------------------------------------------------- + std::string cut_off_extension(const std::string& str) + { + std::string res; + std::string::size_type pos = str.rfind('.'); + if(std::string::npos == pos) + return str; + + res = str.substr(0, pos); + return res; + } + //---------------------------------------------------------------------------- +#ifdef _WIN32 + std::wstring utf8_to_utf16(const std::string& str) + { + if (str.empty()) + return {}; + int wstr_size = MultiByteToWideChar(CP_UTF8, 0, &str[0], str.size(), NULL, 0); + if (wstr_size == 0) + { + throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message()); + } + std::wstring wstr(wstr_size, wchar_t{}); + if (!MultiByteToWideChar(CP_UTF8, 0, &str[0], str.size(), &wstr[0], wstr_size)) + { + throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message()); + } + return wstr; + } + std::string utf16_to_utf8(const std::wstring& wstr) + { + if (wstr.empty()) + return {}; + int str_size = WideCharToMultiByte(CP_UTF8, 0, &wstr[0], wstr.size(), NULL, 0, NULL, NULL); + if (str_size == 0) + { + throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message()); + } + std::string str(str_size, char{}); + if (!WideCharToMultiByte(CP_UTF8, 0, &wstr[0], wstr.size(), &str[0], str_size, NULL, NULL)) + { + throw std::runtime_error(std::error_code(GetLastError(), std::system_category()).message()); + } + return str; + } +#endif } } diff --git a/contrib/epee/src/tiny_ini.cpp b/contrib/epee/src/tiny_ini.cpp new file mode 100644 index 000000000..577ebf7c6 --- /dev/null +++ b/contrib/epee/src/tiny_ini.cpp @@ -0,0 +1,46 @@ +// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// * Neither the name of the Andrey N. Sabelnikov nor the +// names of its contributors may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER BE LIABLE FOR ANY +// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// + +#include "string_tools.h" +#include <boost/regex.hpp> + +namespace epee +{ +namespace tiny_ini +{ + bool get_param_value(const std::string& param_name, const std::string& ini_entry, std::string& res) + { + std::string expr_str = std::string() + "^("+ param_name +") *=(.*?)$"; + const boost::regex match_ini_entry( expr_str, boost::regex::icase | boost::regex::normal); + boost::smatch result; + if(!boost::regex_search(ini_entry, result, match_ini_entry, boost::match_default)) + return false; + res = result[2]; + string_tools::trim(res); + return true; + } +} +} diff --git a/contrib/epee/tests/src/CMakeLists.txt b/contrib/epee/tests/src/CMakeLists.txt index 4807fa7ea..0c42f87ca 100644 --- a/contrib/epee/tests/src/CMakeLists.txt +++ b/contrib/epee/tests/src/CMakeLists.txt @@ -1,5 +1,5 @@ -cmake_minimum_required(VERSION 2.8) +cmake_minimum_required(VERSION 3.5) set(Boost_USE_MULTITHREADED ON) diff --git a/contrib/gitian/README.md b/contrib/gitian/README.md index e2e1d0b94..0a40d4608 100644 --- a/contrib/gitian/README.md +++ b/contrib/gitian/README.md @@ -36,6 +36,10 @@ This guide explains how to set up the environment, and how to start the builds. You need to create a new user called `gitianuser` and be logged in as that user. The user needs `sudo` access. +```bash +sudo adduser gitianuser +sudo usermod -aG sudo gitianuser +``` LXC --- @@ -83,7 +87,7 @@ Docker Prepare for building with docker: ```bash -sudo apt-get install git make curl docker.io +sudo bash -c 'apt-get update && apt-get upgrade -y && apt-get install git curl docker.io' ``` Consider adding `gitianuser` to the `docker` group after reading about [the security implications](https://docs.docker.com/v17.09/engine/installation/linux/linux-postinstall/): @@ -96,13 +100,12 @@ sudo usermod -aG docker gitianuser Optionally add yourself to the docker group. Note that this will give docker root access to your system. ```bash -sudo usermod -aG docker gitianuser +sudo usermod -aG docker $USER ``` Manual Building ------------------- -The instructions below use the automated script [gitian-build.py](gitian-build.py) which only works in Ubuntu. ======= The script automatically installs some packages with apt. If you are not running it on a debian-like system, pass `--no-apt` along with the other arguments to it. It calls all available .yml descriptors, which in turn pass the build configurations for different platforms to gitian. @@ -122,17 +125,23 @@ cp monero/contrib/gitian/gitian-build.py . ### Setup the required environment -Setup for LXC: +Common setup part: ```bash -GH_USER=fluffypony -VERSION=v0.17.0.0 +su - gitianuser -./gitian-build.py --setup $GH_USER $VERSION +GH_USER=YOUR_GITHUB_USER_NAME +VERSION=v0.17.2.0 ``` Where `GH_USER` is your Github user name and `VERSION` is the version tag you want to build. +Setup for LXC: + +```bash +./gitian-build.py --setup $GH_USER $VERSION +``` + Setup for docker: ```bash @@ -145,8 +154,10 @@ fork the [gitian.sigs repository](https://github.com/monero-project/gitian.sigs) or pass the signed assert file back to your build machine. ```bash -git clone git@github.com:monero-project/gitian.sigs.git -git remote add $GH_USER git@github.com:$GH_USER/gitian.sigs.git +git clone https://github.com/monero-project/gitian.sigs/ +pushd gitian.sigs +git remote add $GH_USER https://github.com/$GH_USER/gitian.sigs +popd ``` Build the binaries @@ -154,13 +165,26 @@ Build the binaries **Note:** if you intend to build MacOS binaries, please follow [these instructions](https://github.com/bitcoin-core/docs/blob/master/gitian-building/gitian-building-mac-os-sdk.md) to get the required SDK. +Currently working MacOS solution: + +```bash +curl -O https://bitcoincore.org/depends-sources/sdks/MacOSX10.11.sdk.tar.gz +mv MacOSX10.11.sdk.tar.gz builder/inputs +``` + To build the most recent tag (pass in `--docker` if using docker): ```bash ./gitian-build.py --detach-sign --no-commit --build $GH_USER $VERSION ``` -To speed up the build, use `-j 5 --memory 5000` as the first arguments, where `5` is the number of CPU's you allocated to the VM plus one, and 5000 is a little bit less than then the MB's of RAM you allocated. If there is memory corruption on your machine, try to tweak these values. +To speed up the build, use `-j 5 --memory 10000` as the first arguments, where `5` is the number of CPU's you allocated to the VM plus one, and 10000 is a little bit less than then the MB's of RAM you allocated. If there is memory corruption on your machine, try to tweak these values. A good rule of thumb is, that Monero currently needs about 2 GB of RAM per core. + +A full example for `docker` would look like the following: + +```bash +./gitian-build.py -j 5 --memory 10000 --docker --detach-sign --no-commit --build $GH_USER $VERSION +``` If all went well, this produces a number of (uncommitted) `.assert` files in the gitian.sigs directory. @@ -171,6 +195,22 @@ Take a look in the assert files and note the SHA256 checksums listed there. You should verify that the checksum that is listed matches each of the binaries you actually built. This may be done on Linux using the `sha256sum` command or on MacOS using `shasum --algorithm 256` for example. +An example script to verify the checksums would be: + +```bash +pushd out/${VERSION} + +for ASSERT in ../../sigs/${VERSION}-*/*/*.assert; do + if ! sha256sum --ignore-missing -c "${ASSERT}" ; then + echo "FAILED for ${ASSERT} ! Please inspect manually." + fi +done + +popd +``` + +Don't ignore the incorrect formatting of the found assert files. These files you'll have to compare manually (currently OSX and FreeBSD). + You can also look in the [gitian.sigs](https://github.com/monero-project/gitian.sigs/) repo and / or [getmonero.org release checksums](https://web.getmonero.org/downloads/hashes.txt) to see if others got the same checksum for the same version tag. If there is ever a mismatch -- **STOP! Something is wrong**. Contact others on IRC / github to figure out what is going on. @@ -181,14 +221,7 @@ Signing assert files If you chose to do detached signing using `--detach-sign` above (recommended), you need to copy these uncommitted changes to your host machine, then sign them using your gpg key like so: ```bash -GH_USER=fluffypony -VERSION=v0.17.0.0 - -gpg --detach-sign ${VERSION}-linux/${GH_USER}/monero-linux-*-build.assert -gpg --detach-sign ${VERSION}-win/${GH_USER}/monero-win-*-build.assert -gpg --detach-sign ${VERSION}-osx/${GH_USER}/monero-osx-*-build.assert -gpg --detach-sign ${VERSION}-android/${GH_USER}/monero-android-*-build.assert -gpg --detach-sign ${VERSION}-freebsd/${GH_USER}/monero-freebsd-*-build.assert +for ASSERT in sigs/${VERSION}-*/*/*.assert; do gpg --detach-sign ${ASSERT}; done ``` This will create a `.sig` file for each `.assert` file above (2 files for each platform). @@ -201,6 +234,7 @@ Make a pull request (both the `.assert` and `.assert.sig` files) to the [monero-project/gitian.sigs](https://github.com/monero-project/gitian.sigs/) repository: ```bash +cd gitian.sigs git checkout -b $VERSION # add your assert and sig files... git commit -S -a -m "Add $GH_USER $VERSION" diff --git a/contrib/gitian/gitian-build.py b/contrib/gitian/gitian-build.py index a8140a8a6..6bf936958 100755 --- a/contrib/gitian/gitian-build.py +++ b/contrib/gitian/gitian-build.py @@ -94,10 +94,6 @@ def build(): os.chdir('builder') os.makedirs('inputs', exist_ok=True) - subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://downloads.sourceforge.net/project/osslsigncode/osslsigncode/osslsigncode-1.7.1.tar.gz']) - subprocess.check_call(['wget', '-N', '-P', 'inputs', 'https://bitcoincore.org/cfields/osslsigncode-Backports-to-1.7.1.patch']) - subprocess.check_output(["echo 'a8c4e9cafba922f89de0df1f2152e7be286aba73f78505169bc351a7938dd911 inputs/osslsigncode-Backports-to-1.7.1.patch' | sha256sum -c"], shell=True) - subprocess.check_output(["echo 'f9a8cdb38b9c309326764ebc937cba1523a3a751a7ab05df3ecc99d18ae466c9 inputs/osslsigncode-1.7.1.tar.gz' | sha256sum -c"], shell=True) subprocess.check_call(['make', '-C', 'inputs/monero/contrib/depends', 'download', 'SOURCES_PATH=' + os.getcwd() + '/cache/common']) rebuild() diff --git a/docs/ANONYMITY_NETWORKS.md b/docs/ANONYMITY_NETWORKS.md index 3337b5fc3..f8d08b05f 100644 --- a/docs/ANONYMITY_NETWORKS.md +++ b/docs/ANONYMITY_NETWORKS.md @@ -36,10 +36,6 @@ with additional exclusive IPv4 address(es). ## Usage -Anonymity networks have no seed nodes (the feature is still considered -experimental), so a user must specify an address. If configured properly, -additional peers can be found through typical p2p peerlist sharing. - ### Outbound Connections Connecting to an anonymous address requires the command line option @@ -54,8 +50,9 @@ separate process. On most systems the configuration will look like: which tells `monerod` that ".onion" p2p addresses can be forwarded to a socks proxy at IP 127.0.0.1 port 9050 with a max of 10 outgoing connections and ".b32.i2p" p2p addresses can be forwarded to a socks proxy at IP 127.0.0.1 port -9000 with the default max outgoing connections. Since there are no seed nodes -for anonymity connections, peers must be manually specified: +9000 with the default max outgoing connections. + +If desired, peers can be manually specified: ``` --add-exclusive-node rveahdfho7wo4b2m.onion:28083 diff --git a/docs/COMPILING_DEBUGGING_TESTING.md b/docs/COMPILING_DEBUGGING_TESTING.md new file mode 100644 index 000000000..f5c202303 --- /dev/null +++ b/docs/COMPILING_DEBUGGING_TESTING.md @@ -0,0 +1,86 @@ +# Compiling, debugging and testing efficiently + +This document describes ways of compiling, debugging and testing efficiently for various use cases. +The intented audience are developers, who want to leverage newly added tricks to Monero via `CMake`. The document will lower the entry point for these developers. +Before reading this document, please consult section "Build instructions" in the main README.md. +Some information from README.md will be repeated here, but the aim is to go beyond it. + +## Basic compilation + +Monero can be compiled via the main `Makefile`, using one of several targets listed there. +The targets are actually presets for `CMake` calls with various options, plus `make` commands for building or in some cases `make test` for testing. +It is possible to extract these `CMake` calls and modify them for your specific needs. For example, a minimal external cmake command to compile Monero, executed from within a newly created build directory could look like: + +`cmake -S "$DIR_SRC" -DCMAKE_BUILD_TYPE=Release && make` + +where the variable `DIR_SRC` is expected to store the path to the Monero source code. + +## Use cases + +### Test Driven Development (TDD) - shared libraries for release builds + +Building shared libraries spares a lot of disk space and linkage time. By default only the debug builds produce shared libraries. If you'd like to produce dynamic libraries for the release build for the same reasons as it's being done for the debug version, then you need to add the `BUILD_SHARED_LIBS=ON` flag to the `CMake` call, like the following: + +`cmake -S "$DIR_SRC" -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON && make` + +A perfect use case for the above call is following the Test Driven Development (TDD) principles. In a nutshell, you'd first write a couple of tests, which describe the (new) requirements of the class/method that you're about to write or modify. The tests will typically compile for quite a long time, so ideally write them once. After you're done with the tests, the only thing left to do is to keep modifying the implementation for as long as the tests are failing. If the implementation is contained properly within a .cpp file, then the only time cost to be paid will be compiling the single source file and generating the implementation's shared library. The test itself will not have to be touched and will pick up the new version of the implementation (via the shared library) upon the next execution of the test. + +### Project generation for IDEs + +CMake allows to generate project files for many IDEs. The list of supported project files can be obtained by writing in the console: + +`cmake -G` + +For instance, in order to generate Makefiles and project files for the Code::Blocks IDE, this part of the call would look like the following: + +`cmake -G "CodeBlocks - Unix Makefiles" (...)` + +The additional artifact of the above call is the `monero.cbp` Code::Blocks project file in the build directory. + +### Debugging in Code::Blocks (CB) + +First prepare the build directory for debugging using the following example command, assuming, that the path to the source dir is being held in the DIR_SRC variable, and using 2 cores: + +`cmake -S "$DIR_SRC" -G "CodeBlocks - Unix Makefiles" -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=ON && make -j 2` + +After a successful build, open the `monero.cbp` with CB. From the CB's menu bar select the target, that you want debug. Assuming these are unit tests: + +`Build -> Select target -> Select target -> unit_tests` + +In order to lower the turnaround times, we will run a specific portion of code of interest, without having to go through all the time costly initialization and execution of unrelated parts. For this we'll use GTest's capabilities of test filtering. From the build directory run the following command to learn all the registered tests: + +`tests/unit_tests/unit_tests --gtest_list_tests` + +For example, if you're only interested in logging, you'd find in the list the label `logging.` and its subtests. To execute all the logging tests, you'd write in the console: + +`tests/unit_tests/unit_tests --gtest_filter="logging.*"` + +This parameter is what we need to transfer to CB, in order to reflect the same behaviour in the CB's debugger. From the main menu select: + +`Project -> Set program's arguments...` + +Then in the `Program's arguments` textbox you'd write in this case: + +`--gtest_filter="logging.*"` + +Verify if the expected UTs are being properly executed with `F9` or select: + +`Build -> Build and run` + +If everything looks fine, then after setting some breakpoints of your choice, the target is ready for debugging in CB via: + +`Debug -> Start/Continue` + +## To be done (and merged): +### Multihost parallel compilation +https://github.com/monero-project/monero/pull/7160 + +### Faster core_tests with caching +https://github.com/monero-project/monero/pull/5821 + +### Precompiled headers +https://github.com/monero-project/monero/pull/7216 + +### Unity builds +https://github.com/monero-project/monero/pull/7217 + diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 7b184c00a..e12295412 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -15,7 +15,7 @@ of software solid and usable. Patches are preferably to be sent via a Github pull request. If that can't be done, patches in "git format-patch" format can be sent (eg, posted to fpaste.org with a long enough timeout and a link -posted to #monero-dev on irc.freenode.net). +posted to #monero-dev on irc.libera.chat). Patches should be self contained. A good rule of thumb is to have one patch per separate issue, feature, or logical change. Also, no diff --git a/docs/README.i18n.md b/docs/README.i18n.md index 5df277624..0a3dc23a9 100644 --- a/docs/README.i18n.md +++ b/docs/README.i18n.md @@ -1,7 +1,7 @@ Monero daemon internationalization ================================== -The Monero command line tools can be translated in various languages. If you wish to contribute and need help/support, contact the [Monero Localization Workgroup on Taiga](https://taiga.getmonero.org/project/erciccione-monero-localization/) or come chat on `#monero-translations` (Freenode/IRC, riot/matrix, MatterMost) +The Monero command line tools can be translated in various languages. If you wish to contribute and need help/support, contact the [Monero Localization Workgroup on Taiga](https://taiga.getmonero.org/project/erciccione-monero-localization/) or come chat on `#monero-translations` (Libera/IRC, riot/matrix, MatterMost) In order to use the same translation workflow as the [Monero Core GUI](https://github.com/monero-project/monero-gui), they use Qt Linguist translation files. However, to avoid the dependencies on Qt this normally implies, they use a custom loader to read those files at runtime. diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index a8916a7d0..7ae4ba750 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -38,8 +38,10 @@ find_package(Miniupnpc REQUIRED) message(STATUS "Using in-tree miniupnpc") +set(UPNPC_NO_INSTALL TRUE CACHE BOOL "Disable miniupnp installation" FORCE) add_subdirectory(miniupnp/miniupnpc) set_property(TARGET libminiupnpc-static PROPERTY FOLDER "external") +set_property(TARGET libminiupnpc-static PROPERTY POSITION_INDEPENDENT_CODE ON) if(MSVC) set_property(TARGET libminiupnpc-static APPEND_STRING PROPERTY COMPILE_FLAGS " -wd4244 -wd4267") elseif(NOT MSVC) diff --git a/external/db_drivers/liblmdb/mdb.c b/external/db_drivers/liblmdb/mdb.c index ba1315401..6314d5775 100644 --- a/external/db_drivers/liblmdb/mdb.c +++ b/external/db_drivers/liblmdb/mdb.c @@ -3461,9 +3461,9 @@ mdb_freelist_save(MDB_txn *txn) } else { x = mdb_mid2l_search(dl, mp->mp_pgno); mdb_tassert(txn, dl[x].mid == mp->mp_pgno); + mdb_dpage_free(env, mp); } dl[x].mptr = NULL; - mdb_dpage_free(env, mp); } { /* squash freed slots out of the dirty list */ @@ -4882,9 +4882,6 @@ mdb_env_open2(MDB_env *env, int prev) #endif env->me_maxpg = env->me_mapsize / env->me_psize; - if (env->me_txns) - env->me_txns->mti_txnid = meta.mm_txnid; - #if MDB_DEBUG { MDB_meta *meta = mdb_env_pick_meta(env); @@ -4984,6 +4981,9 @@ static int ESECT mdb_env_share_locks(MDB_env *env, int *excl) { int rc = 0; + MDB_meta *meta = mdb_env_pick_meta(env); + + env->me_txns->mti_txnid = meta->mm_txnid; #ifdef _WIN32 { @@ -7867,7 +7867,7 @@ put_sub: xdata.mv_size = 0; xdata.mv_data = ""; leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); - if (flags == MDB_CURRENT) { + if ((flags & (MDB_CURRENT|MDB_APPENDDUP)) == MDB_CURRENT) { xflags = MDB_CURRENT|MDB_NOSPILL; } else { mdb_xcursor_init1(mc, leaf); diff --git a/external/easylogging++/CMakeLists.txt b/external/easylogging++/CMakeLists.txt index 35fb86552..9aa4c08bc 100644 --- a/external/easylogging++/CMakeLists.txt +++ b/external/easylogging++/CMakeLists.txt @@ -26,17 +26,22 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -cmake_minimum_required(VERSION 2.8.7) +cmake_minimum_required(VERSION 3.5) project(easylogging CXX) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") +monero_enable_coverage() find_package(Threads) find_package(Backtrace) +monero_find_all_headers(EASYLOGGING_HEADERS "${CMAKE_CURRENT_SOURCE_DIR}") + add_library(easylogging - easylogging++.cc) + easylogging++.cc + ${EASYLOGGING_HEADERS} + ) include_directories("${CMAKE_CURRENT_SOURCE_DIR}") include_directories("${CMAKE_CURRENT_BINARY_DIR}") diff --git a/external/easylogging++/easylogging++.cc b/external/easylogging++/easylogging++.cc index bf877c018..267770074 100644 --- a/external/easylogging++/easylogging++.cc +++ b/external/easylogging++/easylogging++.cc @@ -17,6 +17,7 @@ #define EASYLOGGING_CC #include "easylogging++.h" +#include <atomic> #include <unistd.h> #if defined(AUTO_INITIALIZE_EASYLOGGINGPP) @@ -713,9 +714,8 @@ Logger::Logger(const std::string& id, const Configurations& configurations, } Logger::Logger(const Logger& logger) { - base::utils::safeDelete(m_typedConfigurations); m_id = logger.m_id; - m_typedConfigurations = logger.m_typedConfigurations; + m_typedConfigurations = logger.m_typedConfigurations ? new base::TypedConfigurations(*logger.m_typedConfigurations) : nullptr; m_parentApplicationName = logger.m_parentApplicationName; m_isConfigured = logger.m_isConfigured; m_configurations = logger.m_configurations; @@ -727,7 +727,7 @@ Logger& Logger::operator=(const Logger& logger) { if (&logger != this) { base::utils::safeDelete(m_typedConfigurations); m_id = logger.m_id; - m_typedConfigurations = logger.m_typedConfigurations; + m_typedConfigurations = logger.m_typedConfigurations ? new base::TypedConfigurations(*logger.m_typedConfigurations) : nullptr; m_parentApplicationName = logger.m_parentApplicationName; m_isConfigured = logger.m_isConfigured; m_configurations = logger.m_configurations; @@ -2035,7 +2035,7 @@ void RegisteredLoggers::unsafeFlushAll(void) { // VRegistry -VRegistry::VRegistry(base::type::VerboseLevel level, base::type::EnumType* pFlags) : m_level(level), m_pFlags(pFlags), m_lowest_priority(INT_MAX) { +VRegistry::VRegistry(base::type::VerboseLevel level, base::type::EnumType* pFlags) : m_level(level), m_pFlags(pFlags) { } /// @brief Sets verbose level. Accepted range is 0-9 @@ -2131,18 +2131,30 @@ static int priority(Level level) { return 7; } +namespace +{ + std::atomic<int> s_lowest_priority{INT_MAX}; +} + +void VRegistry::clearCategories(void) { + const base::threading::ScopedLock scopedLock(lock()); + m_categories.clear(); + m_cached_allowed_categories.clear(); + s_lowest_priority = INT_MAX; +} + void VRegistry::setCategories(const char* categories, bool clear) { base::threading::ScopedLock scopedLock(lock()); auto insert = [&](std::stringstream& ss, Level level) { m_categories.push_back(std::make_pair(ss.str(), level)); m_cached_allowed_categories.clear(); int pri = priority(level); - if (pri > m_lowest_priority) - m_lowest_priority = pri; + if (pri > s_lowest_priority) + s_lowest_priority = pri; }; if (clear) { - m_lowest_priority = 0; + s_lowest_priority = 0; m_categories.clear(); m_cached_allowed_categories.clear(); m_categoriesString.clear(); @@ -2200,9 +2212,9 @@ std::string VRegistry::getCategories() { } bool VRegistry::allowed(Level level, const std::string &category) { - const int pri = priority(level); - if (pri > m_lowest_priority) - return false; + return priority_allowed(priority(level), category); +} +bool VRegistry::priority_allowed(const int pri, const std::string &category) { base::threading::ScopedLock scopedLock(lock()); const std::map<std::string, int>::const_iterator it = m_cached_allowed_categories.find(category); if (it != m_cached_allowed_categories.end()) @@ -2477,20 +2489,20 @@ void DefaultLogDispatchCallback::handle(const LogDispatchData* data) { template<typename Transform> -static inline std::string utf8canonical(const std::string &s, Transform t = [](wint_t c)->wint_t { return c; }) +static inline void utf8canonical(std::string &s, Transform t = [](wint_t c)->wint_t { return c; }) { - std::string sc = ""; size_t avail = s.size(); const char *ptr = s.data(); wint_t cp = 0; - int bytes = 1; + int rbytes = 1, bytes = -1; char wbuf[8], *wptr; + size_t w_offset = 0; while (avail--) { if ((*ptr & 0x80) == 0) { cp = *ptr++; - bytes = 1; + rbytes = 1; } else if ((*ptr & 0xe0) == 0xc0) { @@ -2499,7 +2511,7 @@ static inline std::string utf8canonical(const std::string &s, Transform t = [](w cp = (*ptr++ & 0x1f) << 6; cp |= *ptr++ & 0x3f; --avail; - bytes = 2; + rbytes = 2; } else if ((*ptr & 0xf0) == 0xe0) { @@ -2509,7 +2521,7 @@ static inline std::string utf8canonical(const std::string &s, Transform t = [](w cp |= (*ptr++ & 0x3f) << 6; cp |= *ptr++ & 0x3f; avail -= 2; - bytes = 3; + rbytes = 3; } else if ((*ptr & 0xf8) == 0xf0) { @@ -2520,7 +2532,7 @@ static inline std::string utf8canonical(const std::string &s, Transform t = [](w cp |= (*ptr++ & 0x3f) << 6; cp |= *ptr++ & 0x3f; avail -= 3; - bytes = 4; + rbytes = 4; } else throw std::runtime_error("Invalid UTF-8"); @@ -2537,6 +2549,9 @@ static inline std::string utf8canonical(const std::string &s, Transform t = [](w else throw std::runtime_error("Invalid code point UTF-8 transformation"); + if (bytes > rbytes) + throw std::runtime_error("In place sanitization requires replacements to not take more space than the original code points"); + wptr = wbuf; switch (bytes) { @@ -2547,16 +2562,17 @@ static inline std::string utf8canonical(const std::string &s, Transform t = [](w default: throw std::runtime_error("Invalid UTF-8"); } *wptr = 0; - sc.append(wbuf, bytes); + memcpy(&s[w_offset], wbuf, bytes); + w_offset += bytes; cp = 0; bytes = 1; } - return sc; + s.resize(w_offset); } void sanitize(std::string &s) { - s = utf8canonical(s, [](wint_t c)->wint_t { + utf8canonical(s, [](wint_t c)->wint_t { if (c == 9 || c == 10 || c == 13) return c; if (c < 0x20) @@ -3335,6 +3351,14 @@ void Helpers::logCrashReason(int sig, bool stackTraceIfAvailable, Level level, c // Loggers +bool Loggers::allowed(Level level, const char* cat) +{ + const int pri = base::priority(level); + if (pri > base::s_lowest_priority) + return false; + return ELPP->vRegistry()->priority_allowed(pri, std::string{cat}); +} + Logger* Loggers::getLogger(const std::string& identity, bool registerIfNotAvailable) { return ELPP->registeredLoggers()->get(identity, registerIfNotAvailable); } diff --git a/external/easylogging++/easylogging++.h b/external/easylogging++/easylogging++.h index 0b65461bc..c4a88339f 100644 --- a/external/easylogging++/easylogging++.h +++ b/external/easylogging++/easylogging++.h @@ -412,7 +412,6 @@ ELPP_INTERNAL_DEBUGGING_OUT_INFO << ELPP_INTERNAL_DEBUGGING_MSG(internalInfoStre #include <sstream> #include <memory> #include <type_traits> -#include <atomic> #if ELPP_THREADING_ENABLED # if ELPP_USE_STD_THREADING # include <mutex> @@ -2464,12 +2463,7 @@ class VRegistry : base::NoCopy, public base::threading::ThreadSafe { return m_level; } - inline void clearCategories(void) { - base::threading::ScopedLock scopedLock(lock()); - m_categories.clear(); - m_cached_allowed_categories.clear(); - m_lowest_priority = INT_MAX; - } + void clearCategories(void); inline void clearModules(void) { base::threading::ScopedLock scopedLock(lock()); @@ -2482,6 +2476,7 @@ class VRegistry : base::NoCopy, public base::threading::ThreadSafe { void setModules(const char* modules); + bool priority_allowed(int priority, const std::string &category); bool allowed(Level level, const std::string &category); bool allowed(base::type::VerboseLevel vlevel, const char* file); @@ -2513,7 +2508,6 @@ class VRegistry : base::NoCopy, public base::threading::ThreadSafe { std::map<std::string, int> m_cached_allowed_categories; std::string m_categoriesString; std::string m_filenameCommonPrefix; - std::atomic<int> m_lowest_priority; }; } // namespace base class LogMessage { @@ -3868,6 +3862,8 @@ class Helpers : base::StaticClass { /// @brief Static helpers to deal with loggers and their configurations class Loggers : base::StaticClass { public: + /// @brief Determines whether logging will occur at this level and category + static bool allowed(Level leve, const char* cat); /// @brief Gets existing or registers new logger static Logger* getLogger(const std::string& identity, bool registerIfNotAvailable = true); /// @brief Changes default log builder for future loggers diff --git a/external/miniupnp b/external/miniupnp -Subproject 4c700e09526a7d546394e85628c57e9490feefa +Subproject 544e6fcc73c5ad9af48a8985c94f0f1d742ef2e diff --git a/external/randomx b/external/randomx -Subproject 5ce5f4906c1eb166be980f6d83cc80f4112ffc2 +Subproject fe4324e8c0c035fec3affd6e4c49241c2e5b995 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9904c5de7..aaaae3a09 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -77,30 +77,7 @@ function (monero_add_executable name) enable_stack_trace("${name}") monero_set_target_no_relink("${name}") -endfunction () - -function (monero_add_library name) - monero_add_library_with_deps(NAME "${name}" SOURCES ${ARGN}) -endfunction() - -function (monero_add_library_with_deps) - cmake_parse_arguments(MONERO_ADD_LIBRARY "" "NAME" "DEPENDS;SOURCES" ${ARGN}) - source_group("${MONERO_ADD_LIBRARY_NAME}" FILES ${MONERO_ADD_LIBRARY_SOURCES}) - - # Define a ("virtual") object library and an actual library that links those - # objects together. The virtual libraries can be arbitrarily combined to link - # any subset of objects into one library archive. This is used for releasing - # libwallet, which combines multiple components. - set(objlib obj_${MONERO_ADD_LIBRARY_NAME}) - add_library(${objlib} OBJECT ${MONERO_ADD_LIBRARY_SOURCES}) - add_library("${MONERO_ADD_LIBRARY_NAME}" $<TARGET_OBJECTS:${objlib}>) - monero_set_target_no_relink("${MONERO_ADD_LIBRARY_NAME}") - if (MONERO_ADD_LIBRARY_DEPENDS) - add_dependencies(${objlib} ${MONERO_ADD_LIBRARY_DEPENDS}) - endif() - set_property(TARGET "${MONERO_ADD_LIBRARY_NAME}" PROPERTY FOLDER "libs") - target_compile_definitions(${objlib} - PRIVATE $<TARGET_PROPERTY:${MONERO_ADD_LIBRARY_NAME},INTERFACE_COMPILE_DEFINITIONS>) + monero_set_target_strip ("${name}") endfunction () include(Version) diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp index a9a7d035f..a84a4148d 100644 --- a/src/blockchain_db/blockchain_db.cpp +++ b/src/blockchain_db/blockchain_db.cpp @@ -216,15 +216,8 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const std::pair } else { - LOG_PRINT_L1("Unsupported input type, removing key images and aborting transaction addition"); - for (const txin_v& tx_input : tx.vin) - { - if (tx_input.type() == typeid(txin_to_key)) - { - remove_spent_key(boost::get<txin_to_key>(tx_input).k_image); - } - } - return; + LOG_PRINT_L1("Unsupported input type, aborting transaction addition"); + throw std::runtime_error("Unexpected input type, aborting"); } } @@ -287,7 +280,7 @@ uint64_t BlockchainDB::add_block( const std::pair<block, blobdata>& blck num_rct_outs += blk.miner_tx.vout.size(); int tx_i = 0; crypto::hash tx_hash = crypto::null_hash; - for (const std::pair<transaction, blobdata_ref>& tx : txs) + for (const std::pair<transaction, blobdata>& tx : txs) { tx_hash = blk.tx_hashes[tx_i]; add_transaction(blk_hash, tx, &tx_hash); diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 0aa2aee6f..1d7b10648 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -897,7 +897,6 @@ uint64_t BlockchainLMDB::add_transaction_data(const crypto::hash& blk_hash, cons throw0(DB_ERROR(lmdb_error("Failed to add tx data to db transaction: ", result).c_str())); const cryptonote::blobdata_ref &blob = txp.second; - MDB_val_sized(blobval, blob); unsigned int unprunable_size = tx.unprunable_size; if (unprunable_size == 0) @@ -1355,7 +1354,8 @@ void BlockchainLMDB::open(const std::string& filename, const int db_flags) m_folder = filename; - check_mmap_support(); + try { check_mmap_support(); } + catch(...) { MERROR("Failed to check for mmap support, proceeding"); } #ifdef __OpenBSD__ if ((mdb_flags & MDB_WRITEMAP) == 0) { @@ -3190,9 +3190,8 @@ bool BlockchainLMDB::get_blocks_from(uint64_t start_height, size_t min_block_cou uint64_t size = 0; size_t num_txes = 0; MDB_val_copy<uint64_t> key(start_height); - MDB_val k, v, val_tx_id; + MDB_val v, val_tx_id; uint64_t tx_id = ~0; - MDB_cursor_op op = MDB_SET; for (uint64_t h = start_height; h < blockchain_height && blocks.size() < max_block_count && (size < max_size || blocks.size() < min_block_count); ++h) { MDB_cursor_op op = h == start_height ? MDB_SET : MDB_NEXT; @@ -3314,7 +3313,7 @@ bool BlockchainLMDB::get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::h RCURSOR(txs_prunable_hash); MDB_val_set(v, tx_hash); - MDB_val result, val_tx_prunable_hash; + MDB_val result; auto get_result = mdb_cursor_get(m_cur_tx_indices, (MDB_val *)&zerokval, &v, MDB_GET_BOTH); if (get_result == 0) { @@ -4311,7 +4310,6 @@ bool BlockchainLMDB::get_output_distribution(uint64_t amount, uint64_t from_heig return false; distribution.resize(db_height - from_height, 0); - bool fret = true; MDB_val_set(k, amount); MDB_val v; MDB_cursor_op op = MDB_SET; @@ -5106,11 +5104,10 @@ void BlockchainLMDB::migrate_0_1() void BlockchainLMDB::migrate_1_2() { LOG_PRINT_L3("BlockchainLMDB::" << __func__); - uint64_t i, z; + uint64_t i; int result; mdb_txn_safe txn(false); - MDB_val k, v; - char *ptr; + MDB_val v; MGINFO_YELLOW("Migrating blockchain from DB version 1 to 2 - this may take a while:"); MINFO("updating txs_pruned and txs_prunable tables..."); @@ -5311,7 +5308,6 @@ void BlockchainLMDB::migrate_2_3() if (result) throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_info: ", result).c_str())); if (!i) { - MDB_stat db_stat; result = mdb_stat(txn, m_block_info, &db_stats); if (result) throw0(DB_ERROR(lmdb_error("Failed to query m_block_info: ", result).c_str())); @@ -5443,7 +5439,6 @@ void BlockchainLMDB::migrate_3_4() if (result) throw0(DB_ERROR(lmdb_error("Failed to open a cursor for blocks: ", result).c_str())); if (!i) { - MDB_stat db_stat; result = mdb_stat(txn, m_block_info, &db_stats); if (result) throw0(DB_ERROR(lmdb_error("Failed to query m_block_info: ", result).c_str())); @@ -5597,7 +5592,6 @@ void BlockchainLMDB::migrate_4_5() if (result) throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_info: ", result).c_str())); if (!i) { - MDB_stat db_stat; result = mdb_stat(txn, m_block_info, &db_stats); if (result) throw0(DB_ERROR(lmdb_error("Failed to query m_block_info: ", result).c_str())); diff --git a/src/blockchain_utilities/blockchain-stats-readme.md b/src/blockchain_utilities/blockchain-stats-readme.md new file mode 100644 index 000000000..9552a3e67 --- /dev/null +++ b/src/blockchain_utilities/blockchain-stats-readme.md @@ -0,0 +1,56 @@ +# Monero Blockchain Stats + +Monero Blockchain Stats utlity exports daily statistics for the monero blockchain from creation through current state. + +## Usage: + +See also the utility's help option. `monero-blockchain-stats --help` + +From the command line run: + +`$ monero-blockchain-stats` + +This loads the existing blockchain and prints the results to the terminal. Default printed data includes Blocks per Day, Total Blocks, Transactions per Day, Total Transactions, Bytes per Day and Total Bytes. The format of the output is in tab delimited csv which is printed to the console. Redirecting or piping the output of the command allows for saving the output to a csv file or feeding your own script accordingly, i.e.: + +- `monero-blockchain-stats > stats-$(date +'%Y-%m-%d').csv` +- `monero-blockchain-stats | save-to-database.sh` + +### Options +`--data-dir arg` +to specify location of blockchain storage + +`--testnet` +Run on testnet. + +`--stagenet` +Run on stagenet. + +`--log-level arg` +0-4 or categories + +`--block-start arg (=0)` +start at block number + +`--block-stop arg (=0)` +Stop at block number + +`--with-inputs` +with input stats + +`--with-outputs` +with output stats + +`--with-ringsize` +with ringsize stats + +`--with-hours` +with txns per hour + +`--with-emission` +with coin emission + +`--with-fees` +with txn fees + +`--with-diff` +with difficulty diff --git a/src/blockchain_utilities/blockchain_ancestry.cpp b/src/blockchain_utilities/blockchain_ancestry.cpp index 89b932e4f..99a84606d 100644 --- a/src/blockchain_utilities/blockchain_ancestry.cpp +++ b/src/blockchain_utilities/blockchain_ancestry.cpp @@ -32,6 +32,7 @@ #include <boost/algorithm/string.hpp> #include <boost/archive/portable_binary_iarchive.hpp> #include <boost/archive/portable_binary_oarchive.hpp> +#include <boost/filesystem/path.hpp> #include "common/unordered_containers_boost_serialization.h" #include "common/command_line.h" #include "common/varint.h" @@ -149,7 +150,7 @@ struct ancestry_state_t { std::unordered_map<crypto::hash, cryptonote::transaction> old_tx_cache; a & old_tx_cache; - for (const auto i: old_tx_cache) + for (const auto& i: old_tx_cache) tx_cache.insert(std::make_pair(i.first, ::tx_data_t(i.second))); } else @@ -161,7 +162,7 @@ struct ancestry_state_t std::unordered_map<uint64_t, cryptonote::block> old_block_cache; a & old_block_cache; block_cache.resize(old_block_cache.size()); - for (const auto i: old_block_cache) + for (const auto& i: old_block_cache) block_cache[i.first] = i.second; } else @@ -575,7 +576,6 @@ int main(int argc, char* argv[]) { add_ancestry(state.ancestry, txid, ancestor{amount, offset}); // find the tx which created this output - bool found = false; crypto::hash output_txid; if (!get_output_txid(state, db, amount, offset, output_txid)) { @@ -693,7 +693,6 @@ int main(int argc, char* argv[]) add_ancestor(ancestry, amount, offset); // find the tx which created this output - bool found = false; crypto::hash output_txid; if (!get_output_txid(state, db, amount, offset, output_txid)) { diff --git a/src/blockchain_utilities/blockchain_blackball.cpp b/src/blockchain_utilities/blockchain_blackball.cpp index a8197483f..d53251fd3 100644 --- a/src/blockchain_utilities/blockchain_blackball.cpp +++ b/src/blockchain_utilities/blockchain_blackball.cpp @@ -28,6 +28,7 @@ #include <boost/range/adaptor/transformed.hpp> #include <boost/algorithm/string.hpp> +#include <boost/filesystem.hpp> #include "common/unordered_containers_boost_serialization.h" #include "common/command_line.h" #include "common/varint.h" @@ -47,9 +48,6 @@ namespace po = boost::program_options; using namespace epee; using namespace cryptonote; -static const char zerokey[8] = {0}; -static const MDB_val zerokval = { sizeof(zerokey), (void *)zerokey }; - static uint64_t records_per_sync = 200; static uint64_t db_flags = 0; static MDB_dbi dbi_relative_rings; @@ -390,9 +388,7 @@ static bool for_all_transactions(const std::string &filename, uint64_t &start_id cryptonote::transaction_prefix tx; blobdata bd; bd.assign(reinterpret_cast<char*>(v.mv_data), v.mv_size); - std::stringstream ss; - ss << bd; - binary_archive<false> ba(ss); + binary_archive<false> ba{epee::strspan<std::uint8_t>(bd)}; bool r = do_serialize(ba, tx); CHECK_AND_ASSERT_MES(r, false, "Failed to parse transaction from blob"); @@ -703,7 +699,6 @@ static void get_per_amount_outputs(MDB_txn *txn, uint64_t amount, uint64_t &tota int dbr = mdb_cursor_open(txn, dbi_per_amount, &cur); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for per amount outputs: " + std::string(mdb_strerror(dbr))); MDB_val k, v; - mdb_size_t count = 0; k.mv_size = sizeof(uint64_t); k.mv_data = (void*)&amount; dbr = mdb_cursor_get(cur, &k, &v, MDB_SET); @@ -726,7 +721,6 @@ static void inc_per_amount_outputs(MDB_txn *txn, uint64_t amount, uint64_t total int dbr = mdb_cursor_open(txn, dbi_per_amount, &cur); CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for per amount outputs: " + std::string(mdb_strerror(dbr))); MDB_val k, v; - mdb_size_t count = 0; k.mv_size = sizeof(uint64_t); k.mv_data = (void*)&amount; dbr = mdb_cursor_get(cur, &k, &v, MDB_SET); @@ -1077,7 +1071,6 @@ static std::vector<std::pair<uint64_t, uint64_t>> load_outputs(const std::string s[len - 1] = 0; if (!s[0]) continue; - std::pair<uint64_t, uint64_t> output; uint64_t offset, num_offsets; if (sscanf(s, "@%" PRIu64, &amount) == 1) { @@ -1269,8 +1262,6 @@ int main(int argc, char* argv[]) LOG_PRINT_L0("Scanning for spent outputs..."); - size_t done = 0; - const uint64_t start_blackballed_outputs = get_num_spent_outputs(); tools::ringdb ringdb(output_file_path.string(), epee::string_tools::pod_to_hex(get_genesis_block_hash(inputs[0]))); diff --git a/src/blockchain_utilities/blockchain_depth.cpp b/src/blockchain_utilities/blockchain_depth.cpp index 6199586bf..8c3c3a009 100644 --- a/src/blockchain_utilities/blockchain_depth.cpp +++ b/src/blockchain_utilities/blockchain_depth.cpp @@ -27,6 +27,7 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <boost/range/adaptor/transformed.hpp> +#include <boost/filesystem/path.hpp> #include <boost/algorithm/string.hpp> #include "common/command_line.h" #include "common/varint.h" diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp index 60c069c3b..8d81ef54d 100644 --- a/src/blockchain_utilities/blockchain_import.cpp +++ b/src/blockchain_utilities/blockchain_import.cpp @@ -146,7 +146,7 @@ int check_flush(cryptonote::core &core, std::vector<block_complete_entry> &block if (!parse_and_validate_block_from_blob(b.block, block)) { MERROR("Failed to parse block: " - << epee::string_tools::pod_to_hex(get_blob_hash(b.block))); + << epee::string_tools::buff_to_hex_nodelimer(b.block)); core.cleanup_handle_incoming_blocks(); return 1; } @@ -177,8 +177,11 @@ int check_flush(cryptonote::core &core, std::vector<block_complete_entry> &block core.handle_incoming_tx(tx_blob, tvc, relay_method::block, true); if(tvc.m_verifivation_failed) { - MERROR("transaction verification failed, tx_id = " - << epee::string_tools::pod_to_hex(get_blob_hash(tx_blob.blob))); + cryptonote::transaction transaction; + if (cryptonote::parse_and_validate_tx_from_blob(tx_blob.blob, transaction)) + MERROR("Transaction verification failed, tx_id = " << cryptonote::get_transaction_hash(transaction)); + else + MERROR("Transaction verification failed, transaction is unparsable"); core.cleanup_handle_incoming_blocks(); return 1; } @@ -192,8 +195,11 @@ int check_flush(cryptonote::core &core, std::vector<block_complete_entry> &block if(bvc.m_verifivation_failed) { - MERROR("Block verification failed, id = " - << epee::string_tools::pod_to_hex(get_blob_hash(block_entry.block))); + cryptonote::block block; + if (cryptonote::parse_and_validate_block_from_blob(block_entry.block, block)) + MERROR("Block verification failed, id = " << cryptonote::get_block_hash(block)); + else + MERROR("Block verification failed, block is unparsable"); core.cleanup_handle_incoming_blocks(); return 1; } @@ -227,7 +233,7 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path return false; } - uint64_t block_first, block_last; + uint64_t block_first; uint64_t start_height = 1, seek_height; if (opt_resume) start_height = core.get_blockchain_storage().get_current_blockchain_height(); diff --git a/src/blockchain_utilities/blockchain_prune.cpp b/src/blockchain_utilities/blockchain_prune.cpp index 9a9d58c46..b1c599f3a 100644 --- a/src/blockchain_utilities/blockchain_prune.cpp +++ b/src/blockchain_utilities/blockchain_prune.cpp @@ -29,6 +29,8 @@ #include <array> #include <lmdb.h> #include <boost/algorithm/string.hpp> +#include <boost/system/error_code.hpp> +#include <boost/filesystem.hpp> #include "common/command_line.h" #include "common/pruning.h" #include "cryptonote_core/cryptonote_core.h" diff --git a/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp b/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp index f8763710e..78a662134 100644 --- a/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp +++ b/src/blockchain_utilities/blockchain_prune_known_spent_data.cpp @@ -27,6 +27,7 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <boost/algorithm/string.hpp> +#include <boost/filesystem.hpp> #include "common/command_line.h" #include "serialization/crypto.h" #include "cryptonote_core/tx_pool.h" @@ -66,7 +67,6 @@ static std::map<uint64_t, uint64_t> load_outputs(const std::string &filename) s[len - 1] = 0; if (!s[0]) continue; - std::pair<uint64_t, uint64_t> output; uint64_t offset, num_offsets; if (sscanf(s, "@%" PRIu64, &amount) == 1) { diff --git a/src/blockchain_utilities/blockchain_stats.cpp b/src/blockchain_utilities/blockchain_stats.cpp index 1f728b4e5..5f5ca6abf 100644 --- a/src/blockchain_utilities/blockchain_stats.cpp +++ b/src/blockchain_utilities/blockchain_stats.cpp @@ -27,6 +27,7 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <boost/algorithm/string.hpp> +#include <boost/filesystem.hpp> #include "common/command_line.h" #include "common/varint.h" #include "cryptonote_basic/cryptonote_boost_serialization.h" diff --git a/src/blockchain_utilities/blockchain_usage.cpp b/src/blockchain_utilities/blockchain_usage.cpp index 6e87a0974..8356ef420 100644 --- a/src/blockchain_utilities/blockchain_usage.cpp +++ b/src/blockchain_utilities/blockchain_usage.cpp @@ -28,6 +28,7 @@ #include <boost/range/adaptor/transformed.hpp> #include <boost/algorithm/string.hpp> +#include <boost/filesystem/path.hpp> #include "common/command_line.h" #include "common/varint.h" #include "cryptonote_core/tx_pool.h" @@ -180,7 +181,6 @@ int main(int argc, char* argv[]) LOG_PRINT_L0("Building usage patterns..."); - size_t done = 0; std::unordered_map<output_data, std::list<reference>> outputs; std::unordered_map<uint64_t,uint64_t> indices; @@ -195,7 +195,7 @@ int main(int argc, char* argv[]) { if (opt_rct_only && out.amount) continue; - uint64_t index = indices[out.amount]++; + indices[out.amount]++; output_data od(out.amount, indices[out.amount], coinbase, height); auto itb = outputs.emplace(od, std::list<reference>()); itb.first->first.info(coinbase, height); diff --git a/src/blocks/CMakeLists.txt b/src/blocks/CMakeLists.txt index 445596a66..d2003316d 100644 --- a/src/blocks/CMakeLists.txt +++ b/src/blocks/CMakeLists.txt @@ -28,20 +28,32 @@ set(GENERATED_SOURCES "") +set(GENERATOR "${CMAKE_CURRENT_BINARY_DIR}/blocks_generator.cmake") +file(GENERATE OUTPUT ${GENERATOR} CONTENT [=[ +file(READ "${INPUT_DAT_FILE}" DATA HEX) +string(REGEX REPLACE "[0-9a-fA-F][0-9a-fA-F]" "0x\\0," DATA "${DATA}") +file(WRITE "${OUTPUT_C_SOURCE}" " +#include <stddef.h> +const unsigned char ${BLOB_NAME}[]={ + ${DATA} +}; +const size_t ${BLOB_NAME}_len = sizeof(${BLOB_NAME}); +" +) +]=]) foreach(BLOB_NAME checkpoints testnet_blocks stagenet_blocks) set(OUTPUT_C_SOURCE "generated_${BLOB_NAME}.c") list(APPEND GENERATED_SOURCES ${OUTPUT_C_SOURCE}) set(INPUT_DAT_FILE "${BLOB_NAME}.dat") add_custom_command( - OUTPUT ${OUTPUT_C_SOURCE} - MAIN_DEPENDENCY ${INPUT_DAT_FILE} - COMMAND - cd ${CMAKE_CURRENT_BINARY_DIR} && - echo "'#include\t<stddef.h>'" > ${OUTPUT_C_SOURCE} && - echo "'const\tunsigned\tchar\t${BLOB_NAME}[]={'" >> ${OUTPUT_C_SOURCE} && - od -v -An -tx1 ${CMAKE_CURRENT_SOURCE_DIR}/${INPUT_DAT_FILE} | sed -e "'s/[0-9a-fA-F]\\{1,\\}/0x&,/g'" -e "'$$s/.$$//'" >> ${OUTPUT_C_SOURCE} && - echo "'};'" >> ${OUTPUT_C_SOURCE} && - echo "'const\tsize_t\t${BLOB_NAME}_len\t=\tsizeof(${BLOB_NAME});'" >> ${OUTPUT_C_SOURCE} + OUTPUT ${OUTPUT_C_SOURCE} + MAIN_DEPENDENCY ${INPUT_DAT_FILE} + DEPENDS ${GENERATOR} + COMMAND ${CMAKE_COMMAND} + "-DINPUT_DAT_FILE=${CMAKE_CURRENT_SOURCE_DIR}/${INPUT_DAT_FILE}" + "-DBLOB_NAME=${BLOB_NAME}" + "-DOUTPUT_C_SOURCE=${CMAKE_CURRENT_BINARY_DIR}/${OUTPUT_C_SOURCE}" + -P "${GENERATOR}" ) endforeach() diff --git a/src/checkpoints/checkpoints.cpp b/src/checkpoints/checkpoints.cpp index c88a630cc..6b48d8723 100644 --- a/src/checkpoints/checkpoints.cpp +++ b/src/checkpoints/checkpoints.cpp @@ -34,6 +34,8 @@ #include "string_tools.h" #include "storages/portable_storage_template_helper.h" // epee json include #include "serialization/keyvalue_serialization.h" +#include <boost/system/error_code.hpp> +#include <boost/filesystem.hpp> #include <functional> #include <vector> diff --git a/src/common/dns_utils.cpp b/src/common/dns_utils.cpp index 4f4efcd81..f0b617798 100644 --- a/src/common/dns_utils.cpp +++ b/src/common/dns_utils.cpp @@ -37,6 +37,7 @@ #include <boost/thread/mutex.hpp> #include <boost/algorithm/string/join.hpp> #include <boost/optional.hpp> +#include <boost/utility/string_ref.hpp> using namespace epee; #undef MONERO_DEFAULT_LOG_CATEGORY @@ -124,6 +125,7 @@ static const char *get_record_name(int record_type) case DNS_TYPE_A: return "A"; case DNS_TYPE_TXT: return "TXT"; case DNS_TYPE_AAAA: return "AAAA"; + case DNS_TYPE_TLSA: return "TLSA"; default: return "unknown"; } } @@ -186,6 +188,13 @@ boost::optional<std::string> txt_to_string(const char* src, size_t len) return std::string(src+1, len-1); } +boost::optional<std::string> tlsa_to_string(const char* src, size_t len) +{ + if (len < 4) + return boost::none; + return std::string(src, len); +} + // custom smart pointer. // TODO: see if std::auto_ptr and the like support custom destructors template<typename type, void (*freefunc)(type*)> @@ -326,11 +335,15 @@ std::vector<std::string> DNSResolver::get_record(const std::string& url, int rec // destructor takes care of cleanup ub_result_ptr result; + MDEBUG("Performing DNSSEC " << get_record_name(record_type) << " record query for " << url); + // call DNS resolver, blocking. if return value not zero, something went wrong if (!ub_resolve(m_data->m_ub_context, string_copy(url.c_str()), record_type, DNS_CLASS_IN, &result)) { dnssec_available = (result->secure || result->bogus); dnssec_valid = result->secure && !result->bogus; + if (dnssec_available && !dnssec_valid) + MWARNING("Invalid DNSSEC " << get_record_name(record_type) << " record signature for " << url << ": " << result->why_bogus); if (result->havedata) { for (size_t i=0; result->data[i] != NULL; i++) @@ -338,8 +351,9 @@ std::vector<std::string> DNSResolver::get_record(const std::string& url, int rec boost::optional<std::string> res = (*reader)(result->data[i], result->len[i]); if (res) { - MINFO("Found \"" << *res << "\" in " << get_record_name(record_type) << " record for " << url); - addresses.push_back(*res); + // do not dump dns record directly from dns into log + MINFO("Found " << get_record_name(record_type) << " record for " << url); + addresses.push_back(std::move(*res)); } } } @@ -363,6 +377,17 @@ std::vector<std::string> DNSResolver::get_txt_record(const std::string& url, boo return get_record(url, DNS_TYPE_TXT, txt_to_string, dnssec_available, dnssec_valid); } +std::vector<std::string> DNSResolver::get_tlsa_tcp_record(const boost::string_ref url, const boost::string_ref port, bool& dnssec_available, bool& dnssec_valid) +{ + std::string service_addr; + service_addr.reserve(url.size() + port.size() + 7); + service_addr.push_back('_'); + service_addr.append(port.data(), port.size()); + service_addr.append("._tcp."); + service_addr.append(url.data(), url.size()); + return get_record(service_addr, DNS_TYPE_TLSA, tlsa_to_string, dnssec_available, dnssec_valid); +} + std::string DNSResolver::get_dns_format_from_oa_address(const std::string& oa_addr) { std::string addr(oa_addr); @@ -484,36 +509,12 @@ std::string get_account_address_as_str_from_url(const std::string& url, bool& dn return dns_confirm(url, addresses, dnssec_valid); } -namespace -{ - bool dns_records_match(const std::vector<std::string>& a, const std::vector<std::string>& b) - { - if (a.size() != b.size()) return false; - - for (const auto& record_in_a : a) - { - bool ok = false; - for (const auto& record_in_b : b) - { - if (record_in_a == record_in_b) - { - ok = true; - break; - } - } - if (!ok) return false; - } - - return true; - } -} - bool load_txt_records_from_dns(std::vector<std::string> &good_records, const std::vector<std::string> &dns_urls) { // Prevent infinite recursion when distributing if (dns_urls.empty()) return false; - std::vector<std::vector<std::string> > records; + std::vector<std::set<std::string> > records; records.resize(dns_urls.size()); size_t first_index = crypto::rand_idx(dns_urls.size()); @@ -525,7 +526,9 @@ bool load_txt_records_from_dns(std::vector<std::string> &good_records, const std for (size_t n = 0; n < dns_urls.size(); ++n) { tpool.submit(&waiter,[n, dns_urls, &records, &avail, &valid](){ - records[n] = tools::DNSResolver::instance().get_txt_record(dns_urls[n], avail[n], valid[n]); + const auto res = tools::DNSResolver::instance().get_txt_record(dns_urls[n], avail[n], valid[n]); + for (const auto &s: res) + records[n].insert(s); }); } waiter.wait(); @@ -568,29 +571,31 @@ bool load_txt_records_from_dns(std::vector<std::string> &good_records, const std return false; } - int good_records_index = -1; - for (size_t i = 0; i < records.size() - 1; ++i) + typedef std::map<std::set<std::string>, uint32_t> map_t; + map_t record_count; + for (const auto &e: records) { - if (records[i].size() == 0) continue; + if (!e.empty()) + ++record_count[e]; + } - for (size_t j = i + 1; j < records.size(); ++j) - { - if (dns_records_match(records[i], records[j])) - { - good_records_index = i; - break; - } - } - if (good_records_index >= 0) break; + map_t::const_iterator good_record = record_count.end(); + for (map_t::const_iterator i = record_count.begin(); i != record_count.end(); ++i) + { + if (good_record == record_count.end() || i->second > good_record->second) + good_record = i; } - if (good_records_index < 0) + MDEBUG("Found " << (good_record == record_count.end() ? 0 : good_record->second) << "/" << dns_urls.size() << " matching records from " << num_valid_records << " valid records"); + if (good_record == record_count.end() || good_record->second < dns_urls.size() / 2 + 1) { - LOG_PRINT_L0("WARNING: no two DNS TXT records matched"); + LOG_PRINT_L0("WARNING: no majority of DNS TXT records matched (only " << good_record->second << "/" << dns_urls.size() << ")"); return false; } - good_records = records[good_records_index]; + good_records = {}; + for (const auto &s: good_record->first) + good_records.push_back(s); return true; } diff --git a/src/common/dns_utils.h b/src/common/dns_utils.h index 30c4cced2..99e91bc54 100644 --- a/src/common/dns_utils.h +++ b/src/common/dns_utils.h @@ -31,15 +31,17 @@ #include <string> #include <functional> #include <boost/optional/optional_fwd.hpp> +#include <boost/utility/string_ref_fwd.hpp> namespace tools { // RFC defines for record types and classes for DNS, gleaned from ldns source -const static int DNS_CLASS_IN = 1; -const static int DNS_TYPE_A = 1; -const static int DNS_TYPE_TXT = 16; -const static int DNS_TYPE_AAAA = 8; +constexpr const int DNS_CLASS_IN = 1; +constexpr const int DNS_TYPE_A = 1; +constexpr const int DNS_TYPE_TXT = 16; +constexpr const int DNS_TYPE_AAAA = 8; +constexpr const int DNS_TYPE_TLSA = 52; struct DNSResolverData; @@ -106,6 +108,17 @@ public: std::vector<std::string> get_txt_record(const std::string& url, bool& dnssec_available, bool& dnssec_valid); /** + * @brief gets all TLSA TCP records from a DNS query for the supplied URL; + * if no TLSA record present returns an empty vector. + * + * @param url A string containing a URL to query for + * @param port The service port number (as string) to query + * + * @return A vector of strings containing all TLSA records; or an empty vector + */ + std::vector<std::string> get_tlsa_tcp_record(boost::string_ref url, boost::string_ref port, bool& dnssec_available, bool& dnssec_valid); + + /** * @brief Gets a DNS address from OpenAlias format * * If the address looks good, but contains one @ symbol, replace that with a . diff --git a/src/common/i18n.cpp b/src/common/i18n.cpp index ebc367b3a..051220ee1 100644 --- a/src/common/i18n.cpp +++ b/src/common/i18n.cpp @@ -35,6 +35,10 @@ #include "common/i18n.h" #include "translation_files.h" +#include <boost/system/error_code.hpp> +#include <boost/filesystem.hpp> +#include <algorithm> + #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "i18n" diff --git a/src/common/updates.cpp b/src/common/updates.cpp index 61a76f5da..af38d7a54 100644 --- a/src/common/updates.cpp +++ b/src/common/updates.cpp @@ -48,7 +48,10 @@ namespace tools static const std::vector<std::string> dns_urls = { "updates.moneropulse.org", "updates.moneropulse.net", - "updates.moneropulse.co", + "updates.moneropulse.fr", + "updates.moneropulse.de", + "updates.moneropulse.no", + "updates.moneropulse.ch", "updates.moneropulse.se" }; diff --git a/src/common/util.cpp b/src/common/util.cpp index 445a11a75..af9843b95 100644 --- a/src/common/util.cpp +++ b/src/common/util.cpp @@ -1355,8 +1355,12 @@ std::string get_nix_version_display_string() 100743, 92152, 57565, 22533, 37564, 21823, 19980, 18277, 18402, 14344, 12142, 15842, 13677, 17631, 18294, 22270, 41422, 39296, 36688, 33512, 33831, 27582, 22276, 27516, 27317, 25505, 24426, 20566, 23045, 26766, - 28185, 26169, 27011, - 28642 // Blocks 1,990,000 to 1,999,999 in December 2019 + 28185, 26169, 27011, 28642, 34994, 34442, 30682, 34357, 31640, 41167, + 41301, 48616, 51075, 55061, 49909, 44606, 47091, 53828, 42520, 39023, + 55245, 56145, 51119, 60398, 71821, 48142, 60310, 56041, 54176, 66220, + 56336, 55248, 56656, 63305, 54029, 77136, 71902, 71618, 83587, 81068, + 69062, 54848, 53681, 53555, + 50616 // Blocks 2,400,000 to 2,409,999 in July 2021 }; const uint64_t block_range_size = 10000; diff --git a/src/crypto/CMakeLists.txt b/src/crypto/CMakeLists.txt index 3b33fe90a..3f0f7d34b 100644 --- a/src/crypto/CMakeLists.txt +++ b/src/crypto/CMakeLists.txt @@ -116,6 +116,7 @@ endif() # cheat because cmake and ccache hate each other set_property(SOURCE CryptonightR_template.S PROPERTY LANGUAGE C) +set_property(SOURCE CryptonightR_template.S PROPERTY XCODE_EXPLICIT_FILE_TYPE sourcecode.asm) # Must be done last, because it references libraries in this directory add_subdirectory(wallet) diff --git a/src/crypto/hash-ops.h b/src/crypto/hash-ops.h index 7dfc5151d..1cd502994 100644 --- a/src/crypto/hash-ops.h +++ b/src/crypto/hash-ops.h @@ -87,6 +87,10 @@ void hash_extra_jh(const void *data, size_t length, char *hash); void hash_extra_skein(const void *data, size_t length, char *hash); void tree_hash(const char (*hashes)[HASH_SIZE], size_t count, char *root_hash); +bool tree_path(size_t count, size_t idx, uint32_t *path); +bool tree_branch(const char (*hashes)[HASH_SIZE], size_t count, const char *hash, char (*branch)[HASH_SIZE], size_t *depth, uint32_t *path); +bool tree_branch_hash(const char hash[HASH_SIZE], const char (*branch)[HASH_SIZE], size_t depth, uint32_t path, char root[HASH_SIZE]); +bool is_branch_in_tree(const char hash[HASH_SIZE], const char root[HASH_SIZE], const char (*branch)[HASH_SIZE], size_t depth, uint32_t path); #define RX_BLOCK_VERSION 12 void rx_slow_hash_allocate_state(void); diff --git a/src/crypto/rx-slow-hash.c b/src/crypto/rx-slow-hash.c index fa35a32e2..801987e37 100644 --- a/src/crypto/rx-slow-hash.c +++ b/src/crypto/rx-slow-hash.c @@ -264,12 +264,14 @@ void rx_slow_hash(const uint64_t mainheight, const uint64_t seedheight, const ch cache = rx_sp->rs_cache; if (cache == NULL) { - if (cache == NULL) { + if (!(disabled_flags() & RANDOMX_FLAG_LARGE_PAGES)) { cache = randomx_alloc_cache(flags | RANDOMX_FLAG_LARGE_PAGES); if (cache == NULL) { mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX cache"); - cache = randomx_alloc_cache(flags); } + } + if (cache == NULL) { + cache = randomx_alloc_cache(flags); if (cache == NULL) local_abort("Couldn't allocate RandomX cache"); } @@ -291,11 +293,14 @@ void rx_slow_hash(const uint64_t mainheight, const uint64_t seedheight, const ch CTHR_MUTEX_LOCK(rx_dataset_mutex); if (!rx_dataset_nomem) { if (rx_dataset == NULL) { - rx_dataset = randomx_alloc_dataset(RANDOMX_FLAG_LARGE_PAGES); - if (rx_dataset == NULL) { - mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX dataset"); - rx_dataset = randomx_alloc_dataset(RANDOMX_FLAG_DEFAULT); + if (!(disabled_flags() & RANDOMX_FLAG_LARGE_PAGES)) { + rx_dataset = randomx_alloc_dataset(RANDOMX_FLAG_LARGE_PAGES); + if (rx_dataset == NULL) { + mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX dataset"); + } } + if (rx_dataset == NULL) + rx_dataset = randomx_alloc_dataset(RANDOMX_FLAG_DEFAULT); if (rx_dataset != NULL) rx_initdata(rx_sp->rs_cache, miners, seedheight); } @@ -311,11 +316,14 @@ void rx_slow_hash(const uint64_t mainheight, const uint64_t seedheight, const ch } CTHR_MUTEX_UNLOCK(rx_dataset_mutex); } - rx_vm = randomx_create_vm(flags | RANDOMX_FLAG_LARGE_PAGES, rx_sp->rs_cache, rx_dataset); - if(rx_vm == NULL) { //large pages failed - mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX VM"); - rx_vm = randomx_create_vm(flags, rx_sp->rs_cache, rx_dataset); + if (!(disabled_flags() & RANDOMX_FLAG_LARGE_PAGES)) { + rx_vm = randomx_create_vm(flags | RANDOMX_FLAG_LARGE_PAGES, rx_sp->rs_cache, rx_dataset); + if(rx_vm == NULL) { //large pages failed + mdebug(RX_LOGCAT, "Couldn't use largePages for RandomX VM"); + } } + if (rx_vm == NULL) + rx_vm = randomx_create_vm(flags, rx_sp->rs_cache, rx_dataset); if(rx_vm == NULL) {//fallback if everything fails flags = RANDOMX_FLAG_DEFAULT | (miners ? RANDOMX_FLAG_FULL_MEM : 0); rx_vm = randomx_create_vm(flags, rx_sp->rs_cache, rx_dataset); diff --git a/src/crypto/tree-hash.c b/src/crypto/tree-hash.c index 643e95121..8f3ea3339 100644 --- a/src/crypto/tree-hash.c +++ b/src/crypto/tree-hash.c @@ -104,3 +104,154 @@ void tree_hash(const char (*hashes)[HASH_SIZE], size_t count, char *root_hash) { free(ints); } } + +bool tree_path(size_t count, size_t idx, uint32_t *path) +{ + if (count == 0) + return false; + + if (count == 1) { + *path = 0; + } else if (count == 2) { + *path = idx == 0 ? 0 : 1; + } else { + size_t i, j; + + *path = 0; + size_t cnt = tree_hash_cnt( count ); + + for (i = 2 * cnt - count, j = 2 * cnt - count; j < cnt; i += 2, ++j) { + if (idx == i || idx == i+1) + { + *path = (*path << 1) | (idx == i ? 0 : 1); + idx = j; + } + } + assert(i == count); + + while (cnt > 2) { + cnt >>= 1; + for (i = 0, j = 0; j < cnt; i += 2, ++j) { + if (idx == i || idx == i + 1) + { + *path = (*path << 1) | (idx == i ? 0 : 1); + idx = j; + } + } + } + + if (idx == 0 || idx == 1) + { + *path = (*path << 1) | (idx == 0 ? 0 : 1); + idx = 0; + } + } + return true; +} + +bool tree_branch(const char (*hashes)[HASH_SIZE], size_t count, const char *hash, char (*branch)[HASH_SIZE], size_t *depth, uint32_t *path) +{ + size_t idx; + + if (count == 0) + return false; + + for (idx = 0; idx < count; ++idx) + if (!memcmp(hash, hashes[idx], HASH_SIZE)) + break; + if (idx == count) + return false; + + assert(count > 0); + if (count == 1) { + *depth = 0; + *path = 0; + } else if (count == 2) { + *depth = 1; + *path = idx == 0 ? 0 : 1; + memcpy(branch[0], hashes[idx ^ 1], HASH_SIZE); + } else { + size_t i, j; + + *depth = 0; + *path = 0; + size_t cnt = tree_hash_cnt( count ); + + char *ints = calloc(cnt, HASH_SIZE); // zero out as extra protection for using uninitialized mem + assert(ints); + + memcpy(ints, hashes, (2 * cnt - count) * HASH_SIZE); + + for (i = 2 * cnt - count, j = 2 * cnt - count; j < cnt; i += 2, ++j) { + if (idx == i || idx == i+1) + { + memcpy(branch[*depth], hashes[idx == i ? i + 1 : i], HASH_SIZE); + ++*depth; + *path = (*path << 1) | (idx == i ? 0 : 1); + idx = j; + } + cn_fast_hash(hashes[i], 64, ints + j * HASH_SIZE); + } + assert(i == count); + + while (cnt > 2) { + cnt >>= 1; + for (i = 0, j = 0; j < cnt; i += 2, ++j) { + if (idx == i || idx == i + 1) + { + memcpy(branch[*depth], ints + (idx == i ? i + 1 : i) * HASH_SIZE, HASH_SIZE); + ++*depth; + *path = (*path << 1) | (idx == i ? 0 : 1); + idx = j; + } + cn_fast_hash(ints + i * HASH_SIZE, 64, ints + j * HASH_SIZE); + } + } + + if (idx == 0 || idx == 1) + { + memcpy(branch[*depth], ints + (idx == 0 ? 1 : 0) * HASH_SIZE, HASH_SIZE); + ++*depth; + *path = (*path << 1) | (idx == 0 ? 0 : 1); + idx = 0; + } + + free(ints); + } + return true; +} + +bool tree_branch_hash(const char hash[HASH_SIZE], const char (*branch)[HASH_SIZE], size_t depth, uint32_t path, char root[HASH_SIZE]) +{ + size_t d; + char partial[HASH_SIZE]; + + memcpy(partial, hash, HASH_SIZE); + + for (d = 0; d < depth; ++d) + { + char buffer[2 * HASH_SIZE]; + if ((path >> (depth - d - 1)) & 1) + { + memcpy(buffer, branch[d], HASH_SIZE); + memcpy(buffer + HASH_SIZE, partial, HASH_SIZE); + } + else + { + memcpy(buffer, partial, HASH_SIZE); + memcpy(buffer + HASH_SIZE, branch[d], HASH_SIZE); + } + cn_fast_hash(buffer, 2 * HASH_SIZE, partial); + } + + memcpy(root, partial, HASH_SIZE); + return true; +} + +bool is_branch_in_tree(const char hash[HASH_SIZE], const char root[HASH_SIZE], const char (*branch)[HASH_SIZE], size_t depth, uint32_t path) +{ + char res[HASH_SIZE]; + if (!tree_branch_hash(hash, branch, depth, path, res)) + return false; + return memcmp(res, root, HASH_SIZE) == 0; +} diff --git a/src/cryptonote_basic/CMakeLists.txt b/src/cryptonote_basic/CMakeLists.txt index 5286256c7..e386ec4ea 100644 --- a/src/cryptonote_basic/CMakeLists.txt +++ b/src/cryptonote_basic/CMakeLists.txt @@ -36,6 +36,14 @@ if(APPLE) endif() endif() +monero_add_library(cryptonote_format_utils_basic + cryptonote_format_utils_basic.cpp +) +target_link_libraries(cryptonote_format_utils_basic + PUBLIC + cncrypto +) + set(cryptonote_basic_sources account.cpp connection_context.cpp @@ -43,6 +51,7 @@ set(cryptonote_basic_sources cryptonote_format_utils.cpp difficulty.cpp hardfork.cpp + merge_mining.cpp miner.cpp) set(cryptonote_basic_headers) @@ -57,6 +66,7 @@ set(cryptonote_basic_private_headers cryptonote_format_utils.h difficulty.h hardfork.h + merge_mining.h miner.h tx_extra.h verification_context.h) @@ -72,6 +82,7 @@ target_link_libraries(cryptonote_basic common cncrypto checkpoints + cryptonote_format_utils_basic device ${Boost_DATE_TIME_LIBRARY} ${Boost_PROGRAM_OPTIONS_LIBRARY} diff --git a/src/cryptonote_basic/connection_context.h b/src/cryptonote_basic/connection_context.h index a7d688300..ee26a0c07 100644 --- a/src/cryptonote_basic/connection_context.h +++ b/src/cryptonote_basic/connection_context.h @@ -77,6 +77,8 @@ namespace cryptonote int m_expect_response; uint64_t m_expect_height; size_t m_num_requested; + epee::copyable_atomic m_new_stripe_notification{0}; + epee::copyable_atomic m_idle_peer_notification{0}; }; inline std::string get_protocol_state_string(cryptonote_connection_context::state s) diff --git a/src/cryptonote_basic/cryptonote_basic.h b/src/cryptonote_basic/cryptonote_basic.h index c70ae1df1..6394a7071 100644 --- a/src/cryptonote_basic/cryptonote_basic.h +++ b/src/cryptonote_basic/cryptonote_basic.h @@ -152,10 +152,6 @@ namespace cryptonote }; - template<typename T> static inline unsigned int getpos(T &ar) { return 0; } - template<> inline unsigned int getpos(binary_archive<true> &ar) { return ar.stream().tellp(); } - template<> inline unsigned int getpos(binary_archive<false> &ar) { return ar.stream().tellg(); } - class transaction_prefix { @@ -236,17 +232,17 @@ namespace cryptonote set_blob_size_valid(false); } - const unsigned int start_pos = getpos(ar); + const auto start_pos = ar.getpos(); FIELDS(*static_cast<transaction_prefix *>(this)) if (std::is_same<Archive<W>, binary_archive<W>>()) - prefix_size = getpos(ar) - start_pos; + prefix_size = ar.getpos() - start_pos; if (version == 1) { if (std::is_same<Archive<W>, binary_archive<W>>()) - unprunable_size = getpos(ar) - start_pos; + unprunable_size = ar.getpos() - start_pos; ar.tag("signatures"); ar.begin_array(); @@ -284,11 +280,11 @@ namespace cryptonote { ar.begin_object(); bool r = rct_signatures.serialize_rctsig_base(ar, vin.size(), vout.size()); - if (!r || !ar.stream().good()) return false; + if (!r || !ar.good()) return false; ar.end_object(); if (std::is_same<Archive<W>, binary_archive<W>>()) - unprunable_size = getpos(ar) - start_pos; + unprunable_size = ar.getpos() - start_pos; if (!pruned && rct_signatures.type != rct::RCTTypeNull) { @@ -296,7 +292,7 @@ namespace cryptonote ar.begin_object(); r = rct_signatures.p.serialize_rctsig_prunable(ar, rct_signatures.type, vin.size(), vout.size(), vin.size() > 0 && vin[0].type() == typeid(txin_to_key) ? boost::get<txin_to_key>(vin[0]).key_offsets.size() - 1 : 0); - if (!r || !ar.stream().good()) return false; + if (!r || !ar.good()) return false; ar.end_object(); } } @@ -320,13 +316,13 @@ namespace cryptonote { ar.begin_object(); bool r = rct_signatures.serialize_rctsig_base(ar, vin.size(), vout.size()); - if (!r || !ar.stream().good()) return false; + if (!r || !ar.good()) return false; ar.end_object(); } } if (!typename Archive<W>::is_saving()) pruned = true; - return ar.stream().good(); + return ar.good(); } private: diff --git a/src/cryptonote_basic/cryptonote_format_utils.cpp b/src/cryptonote_basic/cryptonote_format_utils.cpp index fcc96883b..17adcdc35 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.cpp +++ b/src/cryptonote_basic/cryptonote_format_utils.cpp @@ -32,6 +32,7 @@ #include <boost/algorithm/string.hpp> #include "wipeable_string.h" #include "string_tools.h" +#include "string_tools_lexical.h" #include "serialization/string.h" #include "cryptonote_format_utils.h" #include "cryptonote_config.h" @@ -104,7 +105,6 @@ namespace cryptonote uint64_t get_transaction_weight_clawback(const transaction &tx, size_t n_padded_outputs) { - const rct::rctSig &rv = tx.rct_signatures; const uint64_t bp_base = 368; const size_t n_outputs = tx.vout.size(); if (n_padded_outputs <= 2) @@ -139,22 +139,6 @@ namespace cryptonote return h; } - //--------------------------------------------------------------- - void get_transaction_prefix_hash(const transaction_prefix& tx, crypto::hash& h) - { - std::ostringstream s; - binary_archive<true> a(s); - ::serialization::serialize(a, const_cast<transaction_prefix&>(tx)); - crypto::cn_fast_hash(s.str().data(), s.str().size(), h); - } - //--------------------------------------------------------------- - crypto::hash get_transaction_prefix_hash(const transaction_prefix& tx) - { - crypto::hash h = null_hash; - get_transaction_prefix_hash(tx, h); - return h; - } - //--------------------------------------------------------------- bool expand_transaction_1(transaction &tx, bool base_only) { if (tx.version >= 2 && !is_coinbase(tx)) @@ -211,9 +195,7 @@ namespace cryptonote //--------------------------------------------------------------- bool parse_and_validate_tx_from_blob(const blobdata_ref& tx_blob, transaction& tx) { - std::stringstream ss; - ss << tx_blob; - binary_archive<false> ba(ss); + binary_archive<false> ba{epee::strspan<std::uint8_t>(tx_blob)}; bool r = ::serialization::serialize(ba, tx); CHECK_AND_ASSERT_MES(r, false, "Failed to parse transaction from blob"); CHECK_AND_ASSERT_MES(expand_transaction_1(tx, false), false, "Failed to expand transaction data"); @@ -224,9 +206,7 @@ namespace cryptonote //--------------------------------------------------------------- bool parse_and_validate_tx_base_from_blob(const blobdata_ref& tx_blob, transaction& tx) { - std::stringstream ss; - ss << tx_blob; - binary_archive<false> ba(ss); + binary_archive<false> ba{epee::strspan<std::uint8_t>(tx_blob)}; bool r = tx.serialize_base(ba); CHECK_AND_ASSERT_MES(r, false, "Failed to parse transaction from blob"); CHECK_AND_ASSERT_MES(expand_transaction_1(tx, true), false, "Failed to expand transaction data"); @@ -236,9 +216,7 @@ namespace cryptonote //--------------------------------------------------------------- bool parse_and_validate_tx_prefix_from_blob(const blobdata_ref& tx_blob, transaction_prefix& tx) { - std::stringstream ss; - ss << tx_blob; - binary_archive<false> ba(ss); + binary_archive<false> ba{epee::strspan<std::uint8_t>(tx_blob)}; bool r = ::serialization::serialize_noeof(ba, tx); CHECK_AND_ASSERT_MES(r, false, "Failed to parse transaction prefix from blob"); return true; @@ -246,9 +224,7 @@ namespace cryptonote //--------------------------------------------------------------- bool parse_and_validate_tx_from_blob(const blobdata_ref& tx_blob, transaction& tx, crypto::hash& tx_hash) { - std::stringstream ss; - ss << tx_blob; - binary_archive<false> ba(ss); + binary_archive<false> ba{epee::strspan<std::uint8_t>(tx_blob)}; bool r = ::serialization::serialize(ba, tx); CHECK_AND_ASSERT_MES(r, false, "Failed to parse transaction from blob"); CHECK_AND_ASSERT_MES(expand_transaction_1(tx, false), false, "Failed to expand transaction data"); @@ -532,22 +508,15 @@ namespace cryptonote if(tx_extra.empty()) return true; - std::string extra_str(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size()); - std::istringstream iss(extra_str); - binary_archive<false> ar(iss); + binary_archive<false> ar{epee::to_span(tx_extra)}; - bool eof = false; - while (!eof) + do { tx_extra_field field; bool r = ::do_serialize(ar, field); CHECK_AND_NO_ASSERT_MES_L1(r, false, "failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size()))); tx_extra_fields.push_back(field); - - std::ios_base::iostate state = iss.rdstate(); - eof = (EOF == iss.peek()); - iss.clear(state); - } + } while (!ar.eof()); CHECK_AND_NO_ASSERT_MES_L1(::serialization::check_stream_state(ar), false, "failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size()))); return true; @@ -578,13 +547,10 @@ namespace cryptonote return true; } - std::string extra_str(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size()); - std::istringstream iss(extra_str); - binary_archive<false> ar(iss); + binary_archive<false> ar{epee::to_span(tx_extra)}; - bool eof = false; size_t processed = 0; - while (!eof) + do { tx_extra_field field; bool r = ::do_serialize(ar, field); @@ -596,12 +562,8 @@ namespace cryptonote break; } tx_extra_fields.push_back(field); - processed = iss.tellg(); - - std::ios_base::iostate state = iss.rdstate(); - eof = (EOF == iss.peek()); - iss.clear(state); - } + processed = ar.getpos(); + } while (!ar.eof()); if (!::serialization::check_stream_state(ar)) { MWARNING("failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size()))); @@ -728,29 +690,42 @@ namespace cryptonote return true; } //--------------------------------------------------------------- + bool add_mm_merkle_root_to_tx_extra(std::vector<uint8_t>& tx_extra, const crypto::hash& mm_merkle_root, size_t mm_merkle_tree_depth) + { + CHECK_AND_ASSERT_MES(mm_merkle_tree_depth < 32, false, "merge mining merkle tree depth should be less than 32"); + size_t start_pos = tx_extra.size(); + tx_extra.resize(tx_extra.size() + 3 + 32); + //write tag + tx_extra[start_pos] = TX_EXTRA_MERGE_MINING_TAG; + //write data size + ++start_pos; + tx_extra[start_pos] = 33; + //write depth varint (always one byte here) + ++start_pos; + tx_extra[start_pos] = mm_merkle_tree_depth; + //write data + ++start_pos; + memcpy(&tx_extra[start_pos], &mm_merkle_root, 32); + return true; + } + //--------------------------------------------------------------- bool remove_field_from_tx_extra(std::vector<uint8_t>& tx_extra, const std::type_info &type) { if (tx_extra.empty()) return true; std::string extra_str(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size()); - std::istringstream iss(extra_str); - binary_archive<false> ar(iss); + binary_archive<false> ar{epee::strspan<std::uint8_t>(extra_str)}; std::ostringstream oss; binary_archive<true> newar(oss); - bool eof = false; - while (!eof) + do { tx_extra_field field; bool r = ::do_serialize(ar, field); CHECK_AND_NO_ASSERT_MES_L1(r, false, "failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size()))); if (field.type() != type) ::do_serialize(newar, field); - - std::ios_base::iostate state = iss.rdstate(); - eof = (EOF == iss.peek()); - iss.clear(state); - } + } while (!ar.eof()); CHECK_AND_NO_ASSERT_MES_L1(::serialization::check_stream_state(ar), false, "failed to deserialize extra field. extra = " << string_tools::buff_to_hex_nodelimer(std::string(reinterpret_cast<const char*>(tx_extra.data()), tx_extra.size()))); tx_extra.clear(); std::string s = oss.str(); @@ -1338,9 +1313,7 @@ namespace cryptonote //--------------------------------------------------------------- bool parse_and_validate_block_from_blob(const blobdata_ref& b_blob, block& b, crypto::hash *block_hash) { - std::stringstream ss; - ss << b_blob; - binary_archive<false> ba(ss); + binary_archive<false> ba{epee::strspan<std::uint8_t>(b_blob)}; bool r = ::serialization::serialize(ba, b); CHECK_AND_ASSERT_MES(r, false, "Failed to parse block from blob"); b.invalidate_hashes(); diff --git a/src/cryptonote_basic/cryptonote_format_utils.h b/src/cryptonote_basic/cryptonote_format_utils.h index 636a88b9a..3fe4c44e7 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.h +++ b/src/cryptonote_basic/cryptonote_format_utils.h @@ -83,6 +83,7 @@ namespace cryptonote std::vector<crypto::public_key> get_additional_tx_pub_keys_from_extra(const transaction_prefix& tx); bool add_additional_tx_pub_keys_to_extra(std::vector<uint8_t>& tx_extra, const std::vector<crypto::public_key>& additional_pub_keys); bool add_extra_nonce_to_tx_extra(std::vector<uint8_t>& tx_extra, const blobdata& extra_nonce); + bool add_mm_merkle_root_to_tx_extra(std::vector<uint8_t>& tx_extra, const crypto::hash& mm_merkle_root, size_t mm_merkle_tree_depth); bool remove_field_from_tx_extra(std::vector<uint8_t>& tx_extra, const std::type_info &type); void set_payment_id_to_tx_extra_nonce(blobdata& extra_nonce, const crypto::hash& payment_id); void set_encrypted_payment_id_to_tx_extra_nonce(blobdata& extra_nonce, const crypto::hash8& payment_id); @@ -147,9 +148,7 @@ namespace cryptonote template<class t_object> bool t_serializable_object_from_blob(t_object& to, const blobdata& b_blob) { - std::stringstream ss; - ss << b_blob; - binary_archive<false> ba(ss); + binary_archive<false> ba{epee::strspan<std::uint8_t>(b_blob)}; bool r = ::serialization::serialize(ba, to); return r; } diff --git a/src/cryptonote_basic/cryptonote_format_utils_basic.cpp b/src/cryptonote_basic/cryptonote_format_utils_basic.cpp new file mode 100644 index 000000000..29130ce46 --- /dev/null +++ b/src/cryptonote_basic/cryptonote_format_utils_basic.cpp @@ -0,0 +1,49 @@ +// Copyright (c) 2014-2021, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +#include "cryptonote_format_utils.h" + +namespace cryptonote +{ + void get_transaction_prefix_hash(const transaction_prefix& tx, crypto::hash& h) + { + std::ostringstream s; + binary_archive<true> a(s); + ::serialization::serialize(a, const_cast<transaction_prefix&>(tx)); + crypto::cn_fast_hash(s.str().data(), s.str().size(), h); + } + + crypto::hash get_transaction_prefix_hash(const transaction_prefix& tx) + { + crypto::hash h = crypto::null_hash; + get_transaction_prefix_hash(tx, h); + return h; + } +} diff --git a/src/cryptonote_basic/merge_mining.cpp b/src/cryptonote_basic/merge_mining.cpp new file mode 100644 index 000000000..fcc74859f --- /dev/null +++ b/src/cryptonote_basic/merge_mining.cpp @@ -0,0 +1,95 @@ +// Copyright (c) 2020, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <string.h> +#include "misc_log_ex.h" +#include "int-util.h" +#include "crypto/crypto.h" +#include "common/util.h" +#include "merge_mining.h" + +using namespace epee; + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "cn.mm" + +using namespace crypto; + +namespace cryptonote +{ + +//--------------------------------------------------------------- +uint32_t get_aux_slot(const crypto::hash &id, uint32_t nonce, uint32_t n_aux_chains) +{ + CHECK_AND_ASSERT_THROW_MES(n_aux_chains > 0, "n_aux_chains is 0"); + + uint8_t buf[HASH_SIZE + sizeof(uint32_t) + 1]; + memcpy(buf, &id, HASH_SIZE); + uint32_t v = SWAP32LE(nonce); + memcpy(buf + HASH_SIZE, &v, sizeof(uint32_t)); + buf[HASH_SIZE + sizeof(uint32_t)] = config::HASH_KEY_MM_SLOT; + + crypto::hash res; + tools::sha256sum(buf, sizeof(buf), res); + v = *((const uint32_t*)&res); + return SWAP32LE(v) % n_aux_chains; +} +//--------------------------------------------------------------- +uint32_t get_path_from_aux_slot(uint32_t slot, uint32_t n_aux_chains) +{ + CHECK_AND_ASSERT_THROW_MES(n_aux_chains > 0, "n_aux_chains is 0"); + CHECK_AND_ASSERT_THROW_MES(slot < n_aux_chains, "slot >= n_aux_chains"); + + uint32_t path = 0; + CHECK_AND_ASSERT_THROW_MES(tree_path(n_aux_chains, slot, &path), "Failed to get path from aux slot"); + return path; +} +//--------------------------------------------------------------- +uint32_t encode_mm_depth(uint32_t n_aux_chains, uint32_t nonce) +{ + CHECK_AND_ASSERT_THROW_MES(n_aux_chains > 0, "n_aux_chains is 0"); + + // how many bits to we need to representing n_aux_chains - 1 + uint32_t n_bits = 1; + while ((1u << n_bits) < n_aux_chains && n_bits < 16) + ++n_bits; + CHECK_AND_ASSERT_THROW_MES(n_bits <= 16, "Way too many bits required"); + + const uint32_t depth = (n_bits - 1) | ((n_aux_chains - 1) << 3) | (nonce << (3 + n_bits)); + return depth; +} +//--------------------------------------------------------------- +bool decode_mm_depth(uint32_t depth, uint32_t &n_aux_chains, uint32_t &nonce) +{ + const uint32_t n_bits = 1 + (depth & 7); + n_aux_chains = 1 + (depth >> 3 & ((1 << n_bits) - 1)); + nonce = depth >> (3 + n_bits); + return true; +} +//--------------------------------------------------------------- +} diff --git a/src/cryptonote_basic/merge_mining.h b/src/cryptonote_basic/merge_mining.h new file mode 100644 index 000000000..378438f7c --- /dev/null +++ b/src/cryptonote_basic/merge_mining.h @@ -0,0 +1,40 @@ +// Copyright (c) 2020, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +#include <stdint.h> +#include "crypto/crypto.h" + +namespace cryptonote +{ + uint32_t get_aux_slot(const crypto::hash &id, uint32_t nonce, uint32_t n_aux_chains); + uint32_t get_path_from_aux_slot(uint32_t slot, uint32_t n_aux_chains); + uint32_t encode_mm_depth(uint32_t n_aux_chains, uint32_t nonce); + bool decode_mm_depth(uint32_t depth, uint32_t &n_aux_chains, uint32_t &nonce); +} diff --git a/src/cryptonote_basic/miner.cpp b/src/cryptonote_basic/miner.cpp index 29f6dce5a..ae514aac6 100644 --- a/src/cryptonote_basic/miner.cpp +++ b/src/cryptonote_basic/miner.cpp @@ -44,6 +44,7 @@ #include "string_tools.h" #include "storages/portable_storage_template_helper.h" #include "boost/logic/tribool.hpp" +#include <boost/filesystem.hpp> #ifdef __APPLE__ #include <sys/times.h> diff --git a/src/cryptonote_basic/tx_extra.h b/src/cryptonote_basic/tx_extra.h index 50f2e1438..76efc22d3 100644 --- a/src/cryptonote_basic/tx_extra.h +++ b/src/cryptonote_basic/tx_extra.h @@ -57,11 +57,7 @@ namespace cryptonote // size - 1 - because of variant tag for (size = 1; size <= TX_EXTRA_PADDING_MAX_COUNT; ++size) { - std::ios_base::iostate state = ar.stream().rdstate(); - bool eof = EOF == ar.stream().peek(); - ar.stream().clear(state); - - if (eof) + if (ar.eof()) break; uint8_t zero; @@ -139,8 +135,7 @@ namespace cryptonote if(!::do_serialize(ar, field)) return false; - std::istringstream iss(field); - binary_archive<false> iar(iss); + binary_archive<false> iar{epee::strspan<std::uint8_t>(field)}; serialize_helper helper(*this); return ::serialization::serialize(iar, helper); } diff --git a/src/cryptonote_config.h b/src/cryptonote_config.h index 2cb28b2b1..915835d1b 100644 --- a/src/cryptonote_config.h +++ b/src/cryptonote_config.h @@ -194,7 +194,6 @@ #define CRYPTONOTE_PRUNING_STRIPE_SIZE 4096 // the smaller, the smoother the increase #define CRYPTONOTE_PRUNING_LOG_STRIPES 3 // the higher, the more space saved #define CRYPTONOTE_PRUNING_TIP_BLOCKS 5500 // the smaller, the more space saved -//#define CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED #define RPC_CREDITS_PER_HASH_SCALE ((float)(1<<24)) @@ -235,6 +234,7 @@ namespace config const unsigned char HASH_KEY_CLSAG_AGG_0[] = "CLSAG_agg_0"; const unsigned char HASH_KEY_CLSAG_AGG_1[] = "CLSAG_agg_1"; const char HASH_KEY_MESSAGE_SIGNING[] = "MoneroMessageSignature"; + const unsigned char HASH_KEY_MM_SLOT = 'm'; namespace testnet { diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 8ec624254..f0e6794b9 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -57,6 +57,7 @@ #include "common/notify.h" #include "common/varint.h" #include "common/pruning.h" +#include "time_helper.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "blockchain" @@ -565,6 +566,8 @@ void Blockchain::pop_blocks(uint64_t nblocks) return; } + CHECK_AND_ASSERT_THROW_MES(update_next_cumulative_weight_limit(), "Error updating next cumulative weight limit"); + if (stop_batch) m_db->batch_stop(); } @@ -642,7 +645,6 @@ block Blockchain::pop_block_from_blockchain() m_scan_table.clear(); m_blocks_txs_check.clear(); - CHECK_AND_ASSERT_THROW_MES(update_next_cumulative_weight_limit(), "Error updating next cumulative weight limit"); uint64_t top_block_height; crypto::hash top_block_hash = get_tail_id(top_block_height); m_tx_pool.on_blockchain_dec(top_block_height, top_block_hash); @@ -965,7 +967,7 @@ start: MGINFO("START DUMP"); MGINFO(ss.str()); MGINFO("END DUMP"); - MGINFO("Please send moneromooo on Freenode the contents of this log, from a couple dozen lines before START DUMP to END DUMP"); + MGINFO("Please send moneromooo on Libera.Chat the contents of this log, from a couple dozen lines before START DUMP to END DUMP"); } return diff; } @@ -973,7 +975,7 @@ start: std::pair<bool, uint64_t> Blockchain::check_difficulty_checkpoints() const { uint64_t res = 0; - for (const std::pair<uint64_t, difficulty_type>& i : m_checkpoints.get_difficulty_points()) + for (const std::pair<const uint64_t, difficulty_type>& i : m_checkpoints.get_difficulty_points()) { if (i.first >= m_db->height()) break; @@ -1109,6 +1111,7 @@ bool Blockchain::rollback_blockchain_switching(std::list<block>& original_chain, { pop_block_from_blockchain(); } + CHECK_AND_ASSERT_THROW_MES(update_next_cumulative_weight_limit(), "Error updating next cumulative weight limit"); // make sure the hard fork object updates its current version m_hardfork->reorganize_from_chain_height(rollback_height); @@ -1159,6 +1162,7 @@ bool Blockchain::switch_to_alternative_blockchain(std::list<block_extended_info> block b = pop_block_from_blockchain(); disconnected_chain.push_front(b); } + CHECK_AND_ASSERT_THROW_MES(update_next_cumulative_weight_limit(), "Error updating next cumulative weight limit"); auto split_height = m_db->height(); @@ -2608,7 +2612,7 @@ bool Blockchain::get_split_transactions_blobs(const t_ids_container& txs_ids, t_ } //------------------------------------------------------------------ template<class t_ids_container, class t_tx_container, class t_missed_container> -bool Blockchain::get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const +bool Blockchain::get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs, bool pruned) const { LOG_PRINT_L3("Blockchain::" << __func__); CRITICAL_REGION_LOCAL(m_blockchain_lock); @@ -2619,10 +2623,12 @@ bool Blockchain::get_transactions(const t_ids_container& txs_ids, t_tx_container try { cryptonote::blobdata tx; - if (m_db->get_tx_blob(tx_hash, tx)) + bool res = pruned ? m_db->get_pruned_tx_blob(tx_hash, tx) : m_db->get_tx_blob(tx_hash, tx); + if (res) { txs.push_back(transaction()); - if (!parse_and_validate_tx_from_blob(tx, txs.back())) + res = pruned ? parse_and_validate_tx_base_from_blob(tx, txs.back()) : parse_and_validate_tx_from_blob(tx, txs.back()); + if (!res) { LOG_ERROR("Invalid transaction"); return false; @@ -3438,7 +3444,6 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc, bool failed = false; for (size_t i = 0; i < tx.vin.size(); i++) { - const txin_to_key& in_to_key = boost::get<txin_to_key>(tx.vin[i]); if(!failed && !results[i]) failed = true; } @@ -5150,7 +5155,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete if (m_cancel) return false; - for (const auto &tx_blob : entry.txs) + for (size_t i = 0; i < entry.txs.size(); ++i) { if (tx_index >= txes.size()) SCAN_TABLE_QUIT("tx_index is out of sync"); @@ -5527,6 +5532,6 @@ void Blockchain::cache_block_template(const block &b, const cryptonote::account_ } namespace cryptonote { -template bool Blockchain::get_transactions(const std::vector<crypto::hash>&, std::vector<transaction>&, std::vector<crypto::hash>&) const; +template bool Blockchain::get_transactions(const std::vector<crypto::hash>&, std::vector<transaction>&, std::vector<crypto::hash>&, bool) const; template bool Blockchain::get_split_transactions_blobs(const std::vector<crypto::hash>&, std::vector<std::tuple<crypto::hash, cryptonote::blobdata, crypto::hash, cryptonote::blobdata>>&, std::vector<crypto::hash>&) const; } diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index 5291f1338..a0e7967de 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -721,7 +721,7 @@ namespace cryptonote template<class t_ids_container, class t_tx_container, class t_missed_container> bool get_split_transactions_blobs(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const; template<class t_ids_container, class t_tx_container, class t_missed_container> - bool get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const; + bool get_transactions(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs, bool pruned = false) const; //debug functions diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 12125fb9d..1da14221a 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -57,6 +57,8 @@ using namespace epee; #include "hardforks/hardforks.h" #include "version.h" +#include <boost/filesystem.hpp> + #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "cn" @@ -427,9 +429,9 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- - bool core::get_transactions(const std::vector<crypto::hash>& txs_ids, std::vector<cryptonote::blobdata>& txs, std::vector<crypto::hash>& missed_txs) const + bool core::get_transactions(const std::vector<crypto::hash>& txs_ids, std::vector<cryptonote::blobdata>& txs, std::vector<crypto::hash>& missed_txs, bool pruned) const { - return m_blockchain_storage.get_transactions_blobs(txs_ids, txs, missed_txs); + return m_blockchain_storage.get_transactions_blobs(txs_ids, txs, missed_txs, pruned); } //----------------------------------------------------------------------------------------------- bool core::get_split_transactions_blobs(const std::vector<crypto::hash>& txs_ids, std::vector<std::tuple<crypto::hash, cryptonote::blobdata, crypto::hash, cryptonote::blobdata>>& txs, std::vector<crypto::hash>& missed_txs) const @@ -443,9 +445,9 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- - bool core::get_transactions(const std::vector<crypto::hash>& txs_ids, std::vector<transaction>& txs, std::vector<crypto::hash>& missed_txs) const + bool core::get_transactions(const std::vector<crypto::hash>& txs_ids, std::vector<transaction>& txs, std::vector<crypto::hash>& missed_txs, bool pruned) const { - return m_blockchain_storage.get_transactions(txs_ids, txs, missed_txs); + return m_blockchain_storage.get_transactions(txs_ids, txs, missed_txs, pruned); } //----------------------------------------------------------------------------------------------- bool core::get_alternative_blocks(std::vector<block>& blocks) const @@ -630,7 +632,7 @@ namespace cryptonote void operator()(std::uint64_t, epee::span<const block> blocks) const { - for (const block bl : blocks) + for (const block& bl : blocks) cmdline.notify("%s", epee::string_tools::pod_to_hex(get_block_hash(bl)).c_str(), NULL); } }; @@ -1241,7 +1243,7 @@ namespace cryptonote std::vector<transaction> txs; std::vector<crypto::hash> missed_txs; uint64_t coinbase_amount = get_outs_money_amount(b.miner_tx); - this->get_transactions(b.tx_hashes, txs, missed_txs); + this->get_transactions(b.tx_hashes, txs, missed_txs, true); uint64_t tx_fee_amount = 0; for(const auto& tx: txs) { diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 8891540a9..8478049f9 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -385,7 +385,7 @@ namespace cryptonote * * @note see Blockchain::get_transactions */ - bool get_transactions(const std::vector<crypto::hash>& txs_ids, std::vector<cryptonote::blobdata>& txs, std::vector<crypto::hash>& missed_txs) const; + bool get_transactions(const std::vector<crypto::hash>& txs_ids, std::vector<cryptonote::blobdata>& txs, std::vector<crypto::hash>& missed_txs, bool pruned = false) const; /** * @copydoc Blockchain::get_transactions @@ -399,7 +399,7 @@ namespace cryptonote * * @note see Blockchain::get_transactions */ - bool get_transactions(const std::vector<crypto::hash>& txs_ids, std::vector<transaction>& txs, std::vector<crypto::hash>& missed_txs) const; + bool get_transactions(const std::vector<crypto::hash>& txs_ids, std::vector<transaction>& txs, std::vector<crypto::hash>& missed_txs, bool pruned = false) const; /** * @copydoc Blockchain::get_block_by_hash diff --git a/src/cryptonote_core/cryptonote_tx_utils.cpp b/src/cryptonote_core/cryptonote_tx_utils.cpp index 7400c4328..f41c63a4b 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.cpp +++ b/src/cryptonote_core/cryptonote_tx_utils.cpp @@ -622,8 +622,10 @@ namespace cryptonote if (need_additional_txkeys) { additional_tx_keys.clear(); - for (const auto &d: destinations) + for (size_t i = 0; i < destinations.size(); ++i) + { additional_tx_keys.push_back(keypair::generate(sender_account_keys.get_device()).sec); + } } bool r = construct_tx_with_tx_key(sender_account_keys, subaddresses, sources, destinations, change_addr, extra, tx, unlock_time, tx_key, additional_tx_keys, rct, rct_config, msout); diff --git a/src/cryptonote_core/cryptonote_tx_utils.h b/src/cryptonote_core/cryptonote_tx_utils.h index dbdf409b5..73cdd31cd 100644 --- a/src/cryptonote_core/cryptonote_tx_utils.h +++ b/src/cryptonote_core/cryptonote_tx_utils.h @@ -44,10 +44,10 @@ namespace cryptonote typedef std::pair<uint64_t, rct::ctkey> output_entry; std::vector<output_entry> outputs; //index + key + optional ringct commitment - size_t real_output; //index in outputs vector of real output_entry + uint64_t real_output; //index in outputs vector of real output_entry crypto::public_key real_out_tx_key; //incoming real tx public key std::vector<crypto::public_key> real_out_additional_tx_keys; //incoming real tx additional public keys - size_t real_output_in_tx_index; //index in transaction outputs vector + uint64_t real_output_in_tx_index; //index in transaction outputs vector uint64_t amount; //money bool rct; //true if the output is rct rct::key mask; //ringct amount mask diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index d059ab78f..a7e96e23a 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -383,7 +383,6 @@ namespace cryptonote bool tx_memory_pool::add_tx(transaction &tx, tx_verification_context& tvc, relay_method tx_relay, bool relayed, uint8_t version) { crypto::hash h = null_hash; - size_t blob_size = 0; cryptonote::blobdata bl; t_serializable_object_to_blob(tx, bl); if (bl.size() == 0 || !get_transaction_hash(tx, h)) @@ -1041,7 +1040,6 @@ namespace cryptonote return true; }, true, category); - txpool_tx_meta_t meta; for (const key_images_container::value_type& kee : m_spent_key_images) { const crypto::key_image& k_image = kee.first; const std::unordered_set<crypto::hash>& kei_image_set = kee.second; diff --git a/src/cryptonote_protocol/CMakeLists.txt b/src/cryptonote_protocol/CMakeLists.txt index d28b44bb7..85c25546f 100644 --- a/src/cryptonote_protocol/CMakeLists.txt +++ b/src/cryptonote_protocol/CMakeLists.txt @@ -26,7 +26,7 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -cmake_minimum_required (VERSION 2.6) +cmake_minimum_required (VERSION 3.5) project (monero CXX) file(GLOB CRYPTONOTE_PROTOCOL *) diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.h b/src/cryptonote_protocol/cryptonote_protocol_handler.h index 28530f3e7..80dd2bc39 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.h @@ -46,6 +46,8 @@ #include "block_queue.h" #include "common/perf_timer.h" #include "cryptonote_basic/connection_context.h" +#include "net/levin_base.h" +#include "p2p/net_node_common.h" #include <boost/circular_buffer.hpp> PUSH_WARNINGS @@ -195,10 +197,11 @@ namespace cryptonote bool post_notify(typename t_parameter::request& arg, cryptonote_connection_context& context) { LOG_PRINT_L2("[" << epee::net_utils::print_connection_context_short(context) << "] post " << typeid(t_parameter).name() << " -->"); - epee::byte_slice blob; - epee::serialization::store_t_to_binary(arg, blob, 256 * 1024); // optimize for block responses + + epee::levin::message_writer out{256 * 1024}; // optimize for block responses + epee::serialization::store_t_to_binary(arg, out.buffer); //handler_response_blocks_now(blob.size()); // XXX - return m_p2p->invoke_notify_to_peer(t_parameter::ID, epee::to_span(blob), context); + return m_p2p->invoke_notify_to_peer(t_parameter::ID, std::move(out), context); } }; diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler.inl b/src/cryptonote_protocol/cryptonote_protocol_handler.inl index b57fc0f0f..106253082 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler.inl +++ b/src/cryptonote_protocol/cryptonote_protocol_handler.inl @@ -137,6 +137,41 @@ namespace cryptonote CHECK_AND_ASSERT_MES_CC( context.m_callback_request_count > 0, false, "false callback fired, but context.m_callback_request_count=" << context.m_callback_request_count); --context.m_callback_request_count; + uint32_t notified = true; + if (context.m_idle_peer_notification.compare_exchange_strong(notified, not notified)) + { + if (context.m_state == cryptonote_connection_context::state_synchronizing && context.m_last_request_time != boost::date_time::not_a_date_time) + { + const boost::posix_time::ptime now = boost::posix_time::microsec_clock::universal_time(); + const boost::posix_time::time_duration dt = now - context.m_last_request_time; + const auto ms = dt.total_microseconds(); + if (ms > IDLE_PEER_KICK_TIME || (context.m_expect_response && ms > NON_RESPONSIVE_PEER_KICK_TIME)) + { + if (context.m_score-- >= 0) + { + MINFO(context << " kicking idle peer, last update " << (dt.total_microseconds() / 1.e6) << " seconds ago, expecting " << (int)context.m_expect_response); + context.m_last_request_time = boost::date_time::not_a_date_time; + context.m_expect_response = 0; + context.m_expect_height = 0; + context.m_state = cryptonote_connection_context::state_standby; // we'll go back to adding, then (if we can't), download + } + else + { + MINFO(context << "dropping idle peer with negative score"); + drop_connection_with_score(context, context.m_expect_response == 0 ? 1 : 5, false); + return false; + } + } + } + } + + notified = true; + if (context.m_new_stripe_notification.compare_exchange_strong(notified, not notified)) + { + if (context.m_state == cryptonote_connection_context::state_normal) + context.m_state = cryptonote_connection_context::state_synchronizing; + } + if(context.m_state == cryptonote_connection_context::state_synchronizing && context.m_last_request_time == boost::posix_time::not_a_date_time) { NOTIFY_REQUEST_CHAIN::request r = {}; @@ -338,10 +373,6 @@ namespace cryptonote } context.m_remote_blockchain_height = hshd.current_height; context.m_pruning_seed = hshd.pruning_seed; -#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED - context.m_pruning_seed = tools::make_pruning_seed(1 + (context.m_remote_address.as<epee::net_utils::ipv4_network_address>().ip()) % (1 << CRYPTONOTE_PRUNING_LOG_STRIPES), CRYPTONOTE_PRUNING_LOG_STRIPES); - LOG_INFO_CC(context, "New connection posing as pruning seed " << epee::string_tools::to_string_hex(context.m_pruning_seed) << ", seed address " << &context.m_pruning_seed); -#endif uint64_t target = m_core.get_target_blockchain_height(); if (target == 0) @@ -1691,7 +1722,7 @@ skip: const uint32_t peer_stripe = tools::get_pruning_stripe(context.m_pruning_seed); if (stripe && peer_stripe && peer_stripe != stripe) return true; - context.m_state = cryptonote_connection_context::state_synchronizing; + context.m_new_stripe_notification = true; LOG_PRINT_CCONTEXT_L2("requesting callback"); ++context.m_callback_request_count; m_p2p->request_callback(context); @@ -1714,7 +1745,6 @@ skip: bool t_cryptonote_protocol_handler<t_core>::kick_idle_peers() { MTRACE("Checking for idle peers..."); - std::vector<std::pair<boost::uuids::uuid, unsigned>> idle_peers; m_p2p->for_each_connection([&](cryptonote_connection_context& context, nodetool::peerid_type peer_id, uint32_t support_flags)->bool { if (context.m_state == cryptonote_connection_context::state_synchronizing && context.m_last_request_time != boost::date_time::not_a_date_time) @@ -1724,36 +1754,16 @@ skip: const auto ms = dt.total_microseconds(); if (ms > IDLE_PEER_KICK_TIME || (context.m_expect_response && ms > NON_RESPONSIVE_PEER_KICK_TIME)) { - if (context.m_score-- >= 0) - { - MINFO(context << " kicking idle peer, last update " << (dt.total_microseconds() / 1.e6) << " seconds ago, expecting " << (int)context.m_expect_response); - LOG_PRINT_CCONTEXT_L2("requesting callback"); - context.m_last_request_time = boost::date_time::not_a_date_time; - context.m_expect_response = 0; - context.m_expect_height = 0; - context.m_state = cryptonote_connection_context::state_standby; // we'll go back to adding, then (if we can't), download - ++context.m_callback_request_count; - m_p2p->request_callback(context); - } - else - { - idle_peers.push_back(std::make_pair(context.m_connection_id, context.m_expect_response == 0 ? 1 : 5)); - } + context.m_idle_peer_notification = true; + LOG_PRINT_CCONTEXT_L2("requesting callback"); + ++context.m_callback_request_count; + m_p2p->request_callback(context); + MLOG_PEER_STATE("requesting callback"); } } return true; }); - for (const auto &e: idle_peers) - { - const auto &uuid = e.first; - m_p2p->for_connection(uuid, [&](cryptonote_connection_context& ctx, nodetool::peerid_type peer_id, uint32_t f)->bool{ - MINFO(ctx << "dropping idle peer with negative score"); - drop_connection_with_score(ctx, e.second, false); - return true; - }); - } - return true; } //------------------------------------------------------------------------------------------------------------------------ @@ -1849,10 +1859,8 @@ skip: bool t_cryptonote_protocol_handler<t_core>::should_download_next_span(cryptonote_connection_context& context, bool standby) { std::vector<crypto::hash> hashes; - boost::uuids::uuid span_connection_id; boost::posix_time::ptime request_time; boost::uuids::uuid connection_id; - std::pair<uint64_t, uint64_t> span; bool filled; const uint64_t blockchain_height = m_core.get_current_blockchain_height(); @@ -1878,7 +1886,6 @@ skip: // in standby, be ready to double download early since we're idling anyway // let the fastest peer trigger first - long threshold; const double dl_speed = context.m_max_speed_down; if (standby && dt >= REQUEST_NEXT_SCHEDULED_SPAN_THRESHOLD_STANDBY && dl_speed > 0) { @@ -2074,6 +2081,8 @@ skip: } MDEBUG(context << "Nothing to get from this peer, and it's not ahead of us, all done"); context.m_state = cryptonote_connection_context::state_normal; + if (m_core.get_current_blockchain_height() >= m_core.get_target_blockchain_height()) + on_connection_synchronized(); return true; } uint64_t next_needed_height = m_block_queue.get_next_needed_height(bc_height); @@ -2221,6 +2230,8 @@ skip: } MDEBUG(context << "Nothing to get from this peer, and it's not ahead of us, all done"); context.m_state = cryptonote_connection_context::state_normal; + if (m_core.get_current_blockchain_height() >= m_core.get_target_blockchain_height()) + on_connection_synchronized(); return true; } @@ -2303,7 +2314,7 @@ skip: const uint32_t peer_stripe = tools::get_pruning_stripe(context.m_pruning_seed); const uint32_t first_stripe = tools::get_pruning_stripe(span.first, context.m_remote_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES); const uint32_t last_stripe = tools::get_pruning_stripe(span.first + span.second - 1, context.m_remote_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES); - if ((((first_stripe && peer_stripe != first_stripe) || (last_stripe && peer_stripe != last_stripe)) && !m_sync_pruned_blocks) || (m_sync_pruned_blocks && req.prune)) + if (((first_stripe && peer_stripe != first_stripe) || (last_stripe && peer_stripe != last_stripe)) && !m_sync_pruned_blocks) { MDEBUG(context << "We need full data, but the peer does not have it, dropping peer"); return false; @@ -2412,10 +2423,7 @@ skip: if (context.m_remote_blockchain_height >= m_core.get_target_blockchain_height()) { if (m_core.get_current_blockchain_height() >= m_core.get_target_blockchain_height()) - { - MGINFO_GREEN("SYNCHRONIZED OK"); on_connection_synchronized(); - } } else { @@ -2706,15 +2714,15 @@ skip: // send fluffy ones first, we want to encourage people to run that if (!fluffyConnections.empty()) { - epee::byte_slice fluffyBlob; - epee::serialization::store_t_to_binary(fluffy_arg, fluffyBlob, 32 * 1024); - m_p2p->relay_notify_to_list(NOTIFY_NEW_FLUFFY_BLOCK::ID, epee::to_span(fluffyBlob), std::move(fluffyConnections)); + epee::levin::message_writer fluffyBlob{32 * 1024}; + epee::serialization::store_t_to_binary(fluffy_arg, fluffyBlob.buffer); + m_p2p->relay_notify_to_list(NOTIFY_NEW_FLUFFY_BLOCK::ID, std::move(fluffyBlob), std::move(fluffyConnections)); } if (!fullConnections.empty()) { - epee::byte_slice fullBlob; - epee::serialization::store_t_to_binary(arg, fullBlob, 128 * 1024); - m_p2p->relay_notify_to_list(NOTIFY_NEW_BLOCK::ID, epee::to_span(fullBlob), std::move(fullConnections)); + epee::levin::message_writer fullBlob{128 * 1024}; + epee::serialization::store_t_to_binary(arg, fullBlob.buffer); + m_p2p->relay_notify_to_list(NOTIFY_NEW_BLOCK::ID, std::move(fullBlob), std::move(fullConnections)); } return true; @@ -2847,15 +2855,12 @@ skip: epee::string_tools::to_string_hex(context.m_pruning_seed) << "), score " << score << ", flush_all_spans " << flush_all_spans); - m_block_queue.flush_spans(context.m_connection_id, flush_all_spans); + if (score > 0) + m_p2p->add_host_fail(context.m_remote_address, score); - // copy since dropping the connection will invalidate the context, and thus the address - const auto remote_address = context.m_remote_address; + m_block_queue.flush_spans(context.m_connection_id, flush_all_spans); m_p2p->drop_connection(context); - - if (score > 0) - m_p2p->add_host_fail(remote_address, score); } //------------------------------------------------------------------------------------------------------------------------ template<class t_core> diff --git a/src/cryptonote_protocol/cryptonote_protocol_handler_common.h b/src/cryptonote_protocol/cryptonote_protocol_handler_common.h index 79c2edf1d..57b1d049c 100644 --- a/src/cryptonote_protocol/cryptonote_protocol_handler_common.h +++ b/src/cryptonote_protocol/cryptonote_protocol_handler_common.h @@ -30,8 +30,8 @@ #pragma once -#include "p2p/net_node_common.h" #include "cryptonote_protocol/cryptonote_protocol_defs.h" +#include "cryptonote_protocol/enums.h" #include "cryptonote_basic/connection_context.h" namespace cryptonote { diff --git a/src/cryptonote_protocol/levin_notify.cpp b/src/cryptonote_protocol/levin_notify.cpp index 1e9f3e399..0b065c3c3 100644 --- a/src/cryptonote_protocol/levin_notify.cpp +++ b/src/cryptonote_protocol/levin_notify.cpp @@ -159,7 +159,7 @@ namespace levin return get_out_connections(p2p, get_blockchain_height(p2p, core)); } - epee::byte_slice make_tx_payload(std::vector<blobdata>&& txs, const bool pad, const bool fluff) + epee::levin::message_writer make_tx_message(std::vector<blobdata>&& txs, const bool pad, const bool fluff) { NOTIFY_NEW_TRANSACTIONS::request request{}; request.txs = std::move(txs); @@ -193,21 +193,17 @@ namespace levin // if the size of _ moved enough, we might lose byte in size encoding, we don't care } - epee::byte_slice fullBlob; - if (!epee::serialization::store_t_to_binary(request, fullBlob)) + epee::levin::message_writer out; + if (!epee::serialization::store_t_to_binary(request, out.buffer)) throw std::runtime_error{"Failed to serialize to epee binary format"}; - return fullBlob; + return out; } bool make_payload_send_txs(connections& p2p, std::vector<blobdata>&& txs, const boost::uuids::uuid& destination, const bool pad, const bool fluff) { - const epee::byte_slice blob = make_tx_payload(std::move(txs), pad, fluff); - p2p.for_connection(destination, [&blob](detail::p2p_context& context) { - on_levin_traffic(context, true, true, false, blob.size(), NOTIFY_NEW_TRANSACTIONS::ID); - return true; - }); - return p2p.notify(NOTIFY_NEW_TRANSACTIONS::ID, epee::to_span(blob), destination); + epee::byte_slice blob = make_tx_message(std::move(txs), pad, fluff).finalize_notify(NOTIFY_NEW_TRANSACTIONS::ID); + return p2p.send(std::move(blob), destination); } /* The current design uses `asio::strand`s. The documentation isn't as clear @@ -653,10 +649,6 @@ namespace levin else message = zone_->noise.clone(); - zone_->p2p->for_connection(channel.connection, [&](detail::p2p_context& context) { - on_levin_traffic(context, true, true, false, message.size(), "noise"); - return true; - }); if (zone_->p2p->send(std::move(message), channel.connection)) { if (!channel.queue.empty() && channel.active.empty()) @@ -816,9 +808,8 @@ namespace levin // Padding is not useful when using noise mode. Send as stem so receiver // forwards in Dandelion++ mode. - const epee::byte_slice payload = make_tx_payload(std::move(txs), false, false); epee::byte_slice message = epee::levin::make_fragmented_notify( - zone_->noise, NOTIFY_NEW_TRANSACTIONS::ID, epee::to_span(payload) + zone_->noise.size(), NOTIFY_NEW_TRANSACTIONS::ID, make_tx_message(std::move(txs), false, false) ); if (CRYPTONOTE_MAX_FRAGMENTS * zone_->noise.size() < message.size()) { diff --git a/src/daemon/command_parser_executor.cpp b/src/daemon/command_parser_executor.cpp index 5a7560874..14233bf29 100644 --- a/src/daemon/command_parser_executor.cpp +++ b/src/daemon/command_parser_executor.cpp @@ -30,6 +30,8 @@ #include "common/command_line.h" #include "net/parse.h" #include "daemon/command_parser_executor.h" +#include <boost/filesystem.hpp> +#include <boost/algorithm/string/predicate.hpp> #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "daemon" @@ -986,17 +988,67 @@ bool t_command_parser_executor::check_blockchain_pruning(const std::vector<std:: bool t_command_parser_executor::set_bootstrap_daemon(const std::vector<std::string>& args) { - const size_t args_count = args.size(); - if (args_count < 1 || args_count > 3) + struct parsed_t + { + std::string address; + std::string user; + std::string password; + std::string proxy; + }; + + boost::optional<parsed_t> parsed = [&args]() -> boost::optional<parsed_t> { + const size_t args_count = args.size(); + if (args_count == 0) + { + return {}; + } + if (args[0] == "auto") + { + if (args_count == 1) + { + return {{args[0], "", "", ""}}; + } + if (args_count == 2) + { + return {{args[0], "", "", args[1]}}; + } + } + else if (args[0] == "none") + { + if (args_count == 1) + { + return {{"", "", "", ""}}; + } + } + else + { + if (args_count == 1) + { + return {{args[0], "", "", ""}}; + } + if (args_count == 2) + { + return {{args[0], "", "", args[1]}}; + } + if (args_count == 3) + { + return {{args[0], args[1], args[2], ""}}; + } + if (args_count == 4) + { + return {{args[0], args[1], args[2], args[3]}}; + } + } + return {}; + }(); + + if (!parsed) { std::cout << "Invalid syntax: Wrong number of parameters. For more details, use the help command." << std::endl; return true; } - return m_executor.set_bootstrap_daemon( - args[0] != "none" ? args[0] : std::string(), - args_count > 1 ? args[1] : std::string(), - args_count > 2 ? args[2] : std::string()); + return m_executor.set_bootstrap_daemon(parsed->address, parsed->user, parsed->password, parsed->proxy); } bool t_command_parser_executor::flush_cache(const std::vector<std::string>& args) diff --git a/src/daemon/command_server.cpp b/src/daemon/command_server.cpp index 4768bb842..63f44c4cd 100644 --- a/src/daemon/command_server.cpp +++ b/src/daemon/command_server.cpp @@ -326,7 +326,7 @@ t_command_server::t_command_server( m_command_lookup.set_handler( "set_bootstrap_daemon" , std::bind(&t_command_parser_executor::set_bootstrap_daemon, &m_parser, p::_1) - , "set_bootstrap_daemon (auto | none | host[:port] [username] [password])" + , "set_bootstrap_daemon (auto | none | host[:port] [username] [password]) [proxy_ip:proxy_port]" , "URL of a 'bootstrap' remote daemon that the connected wallets can use while this daemon is still not fully synced.\n" "Use 'auto' to enable automatic public nodes discovering and bootstrap daemon switching" ); diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp index 5a7d4dd4e..16e6a304c 100644 --- a/src/daemon/rpc_command_executor.cpp +++ b/src/daemon/rpc_command_executor.cpp @@ -1059,7 +1059,6 @@ bool t_rpc_command_executor::print_transaction(crypto::hash transaction_hash, // Print json if requested if (include_json) { - crypto::hash tx_hash, tx_prefix_hash; cryptonote::transaction tx; cryptonote::blobdata blob; std::string source = as_hex.empty() ? pruned_as_hex + prunable_as_hex : as_hex; @@ -2406,7 +2405,8 @@ bool t_rpc_command_executor::check_blockchain_pruning() bool t_rpc_command_executor::set_bootstrap_daemon( const std::string &address, const std::string &username, - const std::string &password) + const std::string &password, + const std::string &proxy) { cryptonote::COMMAND_RPC_SET_BOOTSTRAP_DAEMON::request req; cryptonote::COMMAND_RPC_SET_BOOTSTRAP_DAEMON::response res; @@ -2415,6 +2415,7 @@ bool t_rpc_command_executor::set_bootstrap_daemon( req.address = address; req.username = username; req.password = password; + req.proxy = proxy; if (m_is_rpc) { diff --git a/src/daemon/rpc_command_executor.h b/src/daemon/rpc_command_executor.h index 6fb5d6903..118f04731 100644 --- a/src/daemon/rpc_command_executor.h +++ b/src/daemon/rpc_command_executor.h @@ -168,7 +168,8 @@ public: bool set_bootstrap_daemon( const std::string &address, const std::string &username, - const std::string &password); + const std::string &password, + const std::string &proxy); bool rpc_payments(); diff --git a/src/debug_utilities/cn_deserialize.cpp b/src/debug_utilities/cn_deserialize.cpp index dd4701e4f..c039b93c5 100644 --- a/src/debug_utilities/cn_deserialize.cpp +++ b/src/debug_utilities/cn_deserialize.cpp @@ -133,6 +133,18 @@ int main(int argc, char* argv[]) { std::cout << "Parsed block:" << std::endl; std::cout << cryptonote::obj_to_json_str(block) << std::endl; + bool parsed = cryptonote::parse_tx_extra(block.miner_tx.extra, fields); + if (!parsed) + std::cout << "Failed to parse tx_extra" << std::endl; + + if (!fields.empty()) + { + print_extra_fields(fields); + } + else + { + std::cout << "No fields were found in tx_extra" << std::endl; + } } else if (cryptonote::parse_and_validate_tx_from_blob(blob, tx) || cryptonote::parse_and_validate_tx_base_from_blob(blob, tx)) { diff --git a/src/debug_utilities/dns_checks.cpp b/src/debug_utilities/dns_checks.cpp index 76b66c6cb..138cd4fc1 100644 --- a/src/debug_utilities/dns_checks.cpp +++ b/src/debug_utilities/dns_checks.cpp @@ -131,7 +131,7 @@ int main(int argc, char* argv[]) lookup(LOOKUP_A, {"seeds.moneroseeds.se", "seeds.moneroseeds.ae.org", "seeds.moneroseeds.ch", "seeds.moneroseeds.li"}); - lookup(LOOKUP_TXT, {"updates.moneropulse.org", "updates.moneropulse.net", "updates.moneropulse.co", "updates.moneropulse.se"}); + lookup(LOOKUP_TXT, {"updates.moneropulse.org", "updates.moneropulse.net", "updates.moneropulse.co", "updates.moneropulse.se", "updates.moneropulse.fr", "updates.moneropulse.de", "updates.moneropulse.no", "updates.moneropulse.ch"}); lookup(LOOKUP_TXT, {"checkpoints.moneropulse.org", "checkpoints.moneropulse.net", "checkpoints.moneropulse.co", "checkpoints.moneropulse.se"}); diff --git a/src/device/CMakeLists.txt b/src/device/CMakeLists.txt index ff2afba4b..3597ab336 100644 --- a/src/device/CMakeLists.txt +++ b/src/device/CMakeLists.txt @@ -71,6 +71,7 @@ target_link_libraries(device PUBLIC ${HIDAPI_LIBRARIES} cncrypto + cryptonote_format_utils_basic ringct_basic wallet-crypto ${OPENSSL_CRYPTO_LIBRARIES} diff --git a/src/device/device_io_hid.cpp b/src/device/device_io_hid.cpp index 7c61c3b1a..7aa5b39bf 100644 --- a/src/device/device_io_hid.cpp +++ b/src/device/device_io_hid.cpp @@ -181,7 +181,6 @@ namespace hw { unsigned char padding_buffer[MAX_BLOCK+1]; unsigned int result; int hid_ret; - unsigned int sw_offset; unsigned int remaining; unsigned int offset = 0; diff --git a/src/device/device_ledger.cpp b/src/device/device_ledger.cpp index 384b776ee..5caad3a1a 100644 --- a/src/device/device_ledger.cpp +++ b/src/device/device_ledger.cpp @@ -518,9 +518,7 @@ namespace hw { } bool device_ledger::init(void) { - #ifdef DEBUG_HWDEVICE this->controle_device = &hw::get_device("default"); - #endif this->release(); hw_device.init(); MDEBUG( "Device "<<this->id <<" HIDUSB inited"); @@ -686,7 +684,6 @@ namespace hw { /* ======================================================================= */ bool device_ledger::derive_subaddress_public_key(const crypto::public_key &pub, const crypto::key_derivation &derivation, const std::size_t output_index, crypto::public_key &derived_pub){ - AUTO_LOCK_CMD(); #ifdef DEBUG_HWDEVICE const crypto::public_key pub_x = pub; crypto::key_derivation derivation_x; @@ -710,7 +707,7 @@ namespace hw { MDEBUG( "derive_subaddress_public_key : PARSE mode with known viewkey"); crypto::derive_subaddress_public_key(pub, derivation, output_index,derived_pub); } else { - + AUTO_LOCK_CMD(); int offset = set_command_header_noopt(INS_DERIVE_SUBADDRESS_PUBLIC_KEY); //pub memmove(this->buffer_send+offset, pub.data, 32); @@ -739,6 +736,12 @@ namespace hw { } crypto::public_key device_ledger::get_subaddress_spend_public_key(const cryptonote::account_keys& keys, const cryptonote::subaddress_index &index) { + if (has_view_key) { + cryptonote::account_keys keys_{keys}; + keys_.m_view_secret_key = this->viewkey; + return this->controle_device->get_subaddress_spend_public_key(keys_, index); + } + AUTO_LOCK_CMD(); crypto::public_key D; @@ -790,6 +793,12 @@ namespace hw { } cryptonote::account_public_address device_ledger::get_subaddress(const cryptonote::account_keys& keys, const cryptonote::subaddress_index &index) { + if (has_view_key) { + cryptonote::account_keys keys_{keys}; + keys_.m_view_secret_key = this->viewkey; + return this->controle_device->get_subaddress(keys_, index); + } + AUTO_LOCK_CMD(); cryptonote::account_public_address address; @@ -1038,7 +1047,6 @@ namespace hw { } bool device_ledger::generate_key_derivation(const crypto::public_key &pub, const crypto::secret_key &sec, crypto::key_derivation &derivation) { - AUTO_LOCK_CMD(); bool r = false; #ifdef DEBUG_HWDEVICE @@ -1059,6 +1067,7 @@ namespace hw { assert(is_fake_view_key(sec)); r = crypto::generate_key_derivation(pub, this->viewkey, derivation); } else { + AUTO_LOCK_CMD(); int offset = set_command_header_noopt(INS_GEN_KEY_DERIVATION); //pub memmove(this->buffer_send+offset, pub.data, 32); diff --git a/src/device/device_ledger.hpp b/src/device/device_ledger.hpp index 00ff05ec5..3b6cc505f 100644 --- a/src/device/device_ledger.hpp +++ b/src/device/device_ledger.hpp @@ -181,11 +181,8 @@ namespace hw { // To speed up blockchain parsing the view key maybe handle here. crypto::secret_key viewkey; bool has_view_key; - - //extra debug - #ifdef DEBUG_HWDEVICE + device *controle_device; - #endif public: device_ledger(); diff --git a/src/device_trezor/device_trezor.cpp b/src/device_trezor/device_trezor.cpp index 7c3e8785d..c2070b0d1 100644 --- a/src/device_trezor/device_trezor.cpp +++ b/src/device_trezor/device_trezor.cpp @@ -28,6 +28,8 @@ // #include "device_trezor.hpp" +#include <boost/filesystem.hpp> +#include <boost/algorithm/string/predicate.hpp> namespace hw { namespace trezor { diff --git a/src/device_trezor/device_trezor_base.cpp b/src/device_trezor/device_trezor_base.cpp index f59be1573..b0b4342f5 100644 --- a/src/device_trezor/device_trezor_base.cpp +++ b/src/device_trezor/device_trezor_base.cpp @@ -31,6 +31,7 @@ #include "memwipe.h" #include <boost/algorithm/string/classification.hpp> #include <boost/algorithm/string/split.hpp> +#include <boost/algorithm/string/predicate.hpp> #include <boost/regex.hpp> namespace hw { @@ -365,15 +366,14 @@ namespace trezor { void device_trezor_base::device_state_initialize_unsafe() { require_connected(); - std::string tmp_session_id; auto initMsg = std::make_shared<messages::management::Initialize>(); const auto data_cleaner = epee::misc_utils::create_scope_leave_handler([&]() { - memwipe(&tmp_session_id[0], tmp_session_id.size()); + if (initMsg->has_session_id()) + memwipe(&(*initMsg->mutable_session_id())[0], initMsg->mutable_session_id()->size()); }); if(!m_device_session_id.empty()) { - tmp_session_id.assign(m_device_session_id.data(), m_device_session_id.size()); - initMsg->set_allocated_session_id(&tmp_session_id); + initMsg->set_allocated_session_id(new std::string(m_device_session_id.data(), m_device_session_id.size())); } m_features = this->client_exchange<messages::management::Features>(initMsg); @@ -382,8 +382,6 @@ namespace trezor { } else { m_device_session_id.clear(); } - - initMsg->release_session_id(); } void device_trezor_base::device_state_reset() @@ -453,18 +451,14 @@ namespace trezor { pin = m_pin; } - std::string pin_field; messages::common::PinMatrixAck m; if (pin) { - pin_field.assign(pin->data(), pin->size()); - m.set_allocated_pin(&pin_field); + m.set_allocated_pin(new std::string(pin->data(), pin->size())); } const auto data_cleaner = epee::misc_utils::create_scope_leave_handler([&]() { - m.release_pin(); - if (!pin_field.empty()){ - memwipe(&pin_field[0], pin_field.size()); - } + if (m.has_pin()) + memwipe(&(*m.mutable_pin())[0], m.mutable_pin()->size()); }); resp = call_raw(&m); @@ -499,7 +493,6 @@ namespace trezor { boost::optional<epee::wipeable_string> passphrase; TREZOR_CALLBACK_GET(passphrase, on_passphrase_request, on_device); - std::string passphrase_field; messages::common::PassphraseAck m; m.set_on_device(on_device); if (!on_device) { @@ -512,16 +505,13 @@ namespace trezor { } if (passphrase) { - passphrase_field.assign(passphrase->data(), passphrase->size()); - m.set_allocated_passphrase(&passphrase_field); + m.set_allocated_passphrase(new std::string(passphrase->data(), passphrase->size())); } } const auto data_cleaner = epee::misc_utils::create_scope_leave_handler([&]() { - m.release_passphrase(); - if (!passphrase_field.empty()){ - memwipe(&passphrase_field[0], passphrase_field.size()); - } + if (m.has_passphrase()) + memwipe(&(*m.mutable_passphrase())[0], m.mutable_passphrase()->size()); }); resp = call_raw(&m); diff --git a/src/device_trezor/device_trezor_base.hpp b/src/device_trezor/device_trezor_base.hpp index 4db8f0c8e..0162b23df 100644 --- a/src/device_trezor/device_trezor_base.hpp +++ b/src/device_trezor/device_trezor_base.hpp @@ -165,7 +165,7 @@ namespace trezor { // Scoped session closer BOOST_SCOPE_EXIT_ALL(&, this) { - if (open_session){ + if (open_session && this->get_transport()){ this->get_transport()->close(); } }; diff --git a/src/device_trezor/trezor/protocol.cpp b/src/device_trezor/trezor/protocol.cpp index 288f3ddca..92150b579 100644 --- a/src/device_trezor/trezor/protocol.cpp +++ b/src/device_trezor/trezor/protocol.cpp @@ -502,21 +502,9 @@ namespace tx { } void Signer::compute_integrated_indices(TsxData * tsx_data){ - if (m_aux_data == nullptr || m_aux_data->tx_recipients.empty()){ - return; - } - auto & chg = tsx_data->change_dts(); std::string change_hash = hash_addr(&chg.addr(), chg.amount(), chg.is_subaddress()); - std::vector<uint32_t> integrated_indices; - std::set<std::string> integrated_hashes; - for (auto & cur : m_aux_data->tx_recipients){ - if (!cur.has_payment_id){ - continue; - } - integrated_hashes.emplace(hash_addr(&cur.address.m_spend_public_key, &cur.address.m_view_public_key)); - } ssize_t idx = -1; for (auto & cur : tsx_data->outputs()){ @@ -527,8 +515,7 @@ namespace tx { continue; } - c_hash = hash_addr(&cur.addr()); - if (integrated_hashes.find(c_hash) != integrated_hashes.end()){ + if (cur.is_integrated()){ integrated_indices.push_back((uint32_t)idx); } } diff --git a/src/device_trezor/trezor/protocol.hpp b/src/device_trezor/trezor/protocol.hpp index fa824ec3b..0fdd36a51 100644 --- a/src/device_trezor/trezor/protocol.hpp +++ b/src/device_trezor/trezor/protocol.hpp @@ -66,9 +66,7 @@ namespace protocol{ template<typename T> bool cn_deserialize(const void * buff, size_t len, T & dst){ - std::stringstream ss; - ss.write(static_cast<const char *>(buff), len); //ss << tx_blob; - binary_archive<false> ba(ss); + binary_archive<false> ba{{reinterpret_cast<const std::uint8_t*>(buff), len}}; bool r = ::serialization::serialize(ba, dst); return r; } diff --git a/src/device_trezor/trezor/transport.cpp b/src/device_trezor/trezor/transport.cpp index a56090de0..881848a80 100644 --- a/src/device_trezor/trezor/transport.cpp +++ b/src/device_trezor/trezor/transport.cpp @@ -38,6 +38,7 @@ #include <boost/asio/ip/udp.hpp> #include <boost/date_time/posix_time/posix_time_types.hpp> #include <boost/format.hpp> +#include <boost/algorithm/string/predicate.hpp> #include "common/apply_permutation.h" #include "transport.hpp" #include "messages/messages-common.pb.h" @@ -156,7 +157,11 @@ namespace trezor{ #define PROTO_HEADER_SIZE 6 static size_t message_size(const google::protobuf::Message &req){ - return static_cast<size_t>(req.ByteSize()); +#if GOOGLE_PROTOBUF_VERSION < 3006001 + return size_t(req.ByteSize()); +#else + return req.ByteSizeLong(); +#endif } static size_t serialize_message_buffer_size(size_t msg_size) { @@ -572,8 +577,13 @@ namespace trezor{ std::string req = "PINGPING"; char res[8]; - m_socket->send_to(boost::asio::buffer(req.c_str(), req.size()), m_endpoint); - receive(res, 8, nullptr, false, timeout); + const auto written = m_socket->send_to(boost::asio::buffer(req.c_str(), req.size()), m_endpoint); + if (written != req.size()) + return false; + memset(res, 0, sizeof(res)); + const auto received = receive(res, 8, nullptr, false, timeout); + if (received != 8) + return false; return memcmp(res, "PONGPONG", 8) == 0; diff --git a/src/gen_multisig/gen_multisig.cpp b/src/gen_multisig/gen_multisig.cpp index 15f7e5c0a..4aa21b149 100644 --- a/src/gen_multisig/gen_multisig.cpp +++ b/src/gen_multisig/gen_multisig.cpp @@ -74,8 +74,6 @@ namespace const command_line::arg_descriptor<bool, false> arg_testnet = {"testnet", genms::tr("Create testnet multisig wallets"), false}; const command_line::arg_descriptor<bool, false> arg_stagenet = {"stagenet", genms::tr("Create stagenet multisig wallets"), false}; const command_line::arg_descriptor<bool, false> arg_create_address_file = {"create-address-file", genms::tr("Create an address file for new wallets"), false}; - - const command_line::arg_descriptor< std::vector<std::string> > arg_command = {"command", ""}; } static bool generate_multisig(uint32_t threshold, uint32_t total, const std::string &basename, network_type nettype, bool create_address_file) diff --git a/src/mnemonics/electrum-words.cpp b/src/mnemonics/electrum-words.cpp index bda18d950..8c79a53ca 100644 --- a/src/mnemonics/electrum-words.cpp +++ b/src/mnemonics/electrum-words.cpp @@ -194,7 +194,6 @@ namespace { epee::wipeable_string trimmed_words = "", word; - const auto &word_map = language->get_word_map(); const auto &trimmed_word_map = language->get_trimmed_word_map(); const uint32_t unique_prefix_length = language->get_unique_prefix_length(); for (std::vector<epee::wipeable_string>::const_iterator it = word_list.begin(); it != word_list.end(); it++) @@ -492,6 +491,14 @@ namespace crypto return "<language not found>"; } + bool is_valid_language(const std::string &language) + { + const std::vector<const Language::Base*> language_instances = get_language_list(); + for (std::vector<const Language::Base*>::const_iterator it = language_instances.begin(); it != language_instances.end(); it++) + if ((*it)->get_english_language_name() == language || (*it)->get_language_name() == language) + return true; + return false; + } } } diff --git a/src/mnemonics/electrum-words.h b/src/mnemonics/electrum-words.h index 8d4c5be66..eb0c99e0b 100644 --- a/src/mnemonics/electrum-words.h +++ b/src/mnemonics/electrum-words.h @@ -124,6 +124,8 @@ namespace crypto * \return the name of the language in English */ std::string get_english_name_for(const std::string &name); + + bool is_valid_language(const std::string &language); } } diff --git a/src/mnemonics/language_base.h b/src/mnemonics/language_base.h index bf8793aa2..1aa869e45 100644 --- a/src/mnemonics/language_base.h +++ b/src/mnemonics/language_base.h @@ -129,7 +129,7 @@ namespace Language if ((*it).size() < unique_prefix_length)
{
if (flags & ALLOW_SHORT_WORDS)
- MWARNING(language_name << " word '" << *it << "' is shorter than its prefix length, " << unique_prefix_length);
+ MINFO(language_name << " word '" << *it << "' is shorter than its prefix length, " << unique_prefix_length);
else
throw std::runtime_error("Too short word in " + language_name + " word list: " + *it);
}
diff --git a/src/net/CMakeLists.txt b/src/net/CMakeLists.txt index afcd42ef7..e93e27bcd 100644 --- a/src/net/CMakeLists.txt +++ b/src/net/CMakeLists.txt @@ -26,10 +26,10 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -set(net_sources dandelionpp.cpp error.cpp http.cpp i2p_address.cpp parse.cpp socks.cpp - socks_connect.cpp tor_address.cpp zmq.cpp) -set(net_headers dandelionpp.h error.h http.cpp i2p_address.h parse.h socks.h socks_connect.h - tor_address.h zmq.h) +set(net_sources dandelionpp.cpp error.cpp http.cpp i2p_address.cpp parse.cpp resolve.cpp + socks.cpp socks_connect.cpp tor_address.cpp zmq.cpp) +set(net_headers dandelionpp.h error.h http.cpp i2p_address.h parse.h socks.h resolve.h + socks_connect.h tor_address.h zmq.h) monero_add_library(net ${net_sources} ${net_headers}) target_link_libraries(net common epee ${ZMQ_LIB} ${Boost_ASIO_LIBRARY}) diff --git a/src/net/error.cpp b/src/net/error.cpp index 037f44d52..d2e713bc5 100644 --- a/src/net/error.cpp +++ b/src/net/error.cpp @@ -47,12 +47,18 @@ namespace { switch (net::error(value)) { + case net::error::bogus_dnssec: + return "Invalid response signature from DNSSEC enabled domain"; + case net::error::dns_query_failure: + return "Failed to retrieve desired DNS record"; case net::error::expected_tld: return "Expected top-level domain"; case net::error::invalid_host: return "Host value is not valid"; case net::error::invalid_i2p_address: return "Invalid I2P address"; + case net::error::invalid_mask: + return "CIDR netmask outside of 0-32 range"; case net::error::invalid_port: return "Invalid port value (expected 0-65535)"; case net::error::invalid_tor_address: @@ -71,6 +77,7 @@ namespace switch (net::error(value)) { case net::error::invalid_port: + case net::error::invalid_mask: return std::errc::result_out_of_range; case net::error::expected_tld: case net::error::invalid_tor_address: diff --git a/src/net/error.h b/src/net/error.h index 7c852dd20..746eb0ecb 100644 --- a/src/net/error.h +++ b/src/net/error.h @@ -37,13 +37,16 @@ namespace net enum class error : int { // 0 reserved for success (as per expect<T>) - expected_tld = 1, //!< Expected a tld + bogus_dnssec = 1, //!< Invalid response signature from DNSSEC enabled domain + dns_query_failure, //!< Failed to retrieve desired DNS record + expected_tld, //!< Expected a tld invalid_host, //!< Hostname is not valid invalid_i2p_address, + invalid_mask, //!< Outside of 0-32 range invalid_port, //!< Outside of 0-65535 range invalid_tor_address,//!< Invalid base32 or length unsupported_address,//!< Type not supported by `get_network_address` - invalid_mask, //!< Outside of 0-32 range + }; //! \return `std::error_category` for `net` namespace. diff --git a/src/net/i2p_address.cpp b/src/net/i2p_address.cpp index 6c03b3808..ada4eb0d3 100644 --- a/src/net/i2p_address.cpp +++ b/src/net/i2p_address.cpp @@ -38,7 +38,7 @@ #include "net/error.h" #include "serialization/keyvalue_serialization.h" #include "storages/portable_storage.h" -#include "string_tools.h" +#include "string_tools_lexical.h" namespace net { diff --git a/src/net/parse.cpp b/src/net/parse.cpp index 8a98e941a..298576ba4 100644 --- a/src/net/parse.cpp +++ b/src/net/parse.cpp @@ -31,6 +31,7 @@ #include "net/tor_address.h" #include "net/i2p_address.h" #include "string_tools.h" +#include "string_tools_lexical.h" namespace net { diff --git a/src/net/resolve.cpp b/src/net/resolve.cpp new file mode 100644 index 000000000..1b43cf6c4 --- /dev/null +++ b/src/net/resolve.cpp @@ -0,0 +1,71 @@ +// Copyright (c) 2020, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "net/resolve.h" + +#include <boost/utility/string_ref.hpp> +#include "common/dns_utils.h" +#include "common/expect.h" +#include "net/error.h" + +namespace net +{ +namespace dnssec +{ + expect<service_response> resolve_hostname(const std::string& addr, const std::string& tlsa_port) + { + // use basic (blocking) unbound for now, possibly refactor later + tools::DNSResolver& resolver = tools::DNSResolver::instance(); + + bool dnssec_available = false; + bool dnssec_valid = false; + std::vector<std::string> ip_records = resolver.get_ipv4(addr, dnssec_available, dnssec_valid); + + if (dnssec_available && !dnssec_valid) + return {net::error::bogus_dnssec}; + + if (ip_records.empty()) + { + ip_records = resolver.get_ipv6(addr, dnssec_available, dnssec_valid); + if (dnssec_available && !dnssec_valid) + return {net::error::bogus_dnssec}; + if (ip_records.empty()) + return {net::error::dns_query_failure}; + } + + std::vector<std::string> tlsa{}; + if (dnssec_available && !tlsa_port.empty()) + { + tlsa = resolver.get_tlsa_tcp_record(addr, tlsa_port, dnssec_available, dnssec_valid); + if (!dnssec_valid) + return {net::error::bogus_dnssec}; + } + return {{std::move(ip_records), std::move(tlsa)}}; + } +} // dnssec +} // net diff --git a/src/net/resolve.h b/src/net/resolve.h new file mode 100644 index 000000000..46bd8e617 --- /dev/null +++ b/src/net/resolve.h @@ -0,0 +1,47 @@ +// Copyright (c) 2020, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <string> +#include <vector> + +template<typename> class expect; + +namespace net +{ +namespace dnssec +{ + struct service_response + { + std::vector<std::string> ip; //!< IPv4/6 records in dotted or semicolon notation + std::vector<std::string> tlsa; //!< DANE/TLSA records + }; + + //! \return IP + (optionally) DANE/TLSA records, failing if DNSSEC signature is "bogus" + expect<service_response> resolve_hostname(const std::string& addr, const std::string& tlsa_port = {}); +} // dnssec +} // net diff --git a/src/net/socks.cpp b/src/net/socks.cpp index 3463f452c..c2330bd41 100644 --- a/src/net/socks.cpp +++ b/src/net/socks.cpp @@ -48,7 +48,6 @@ namespace socks { namespace { - constexpr const unsigned v4_reply_size = 8; constexpr const std::uint8_t v4_connect_command = 1; constexpr const std::uint8_t v4tor_resolve_command = 0xf0; constexpr const std::uint8_t v4_request_granted = 90; diff --git a/src/net/socks_connect.cpp b/src/net/socks_connect.cpp index 9db9d4483..c797a24d4 100644 --- a/src/net/socks_connect.cpp +++ b/src/net/socks_connect.cpp @@ -38,6 +38,7 @@ #include "net/net_utils_base.h" #include "net/socks.h" #include "string_tools.h" +#include "string_tools_lexical.h" namespace net { diff --git a/src/net/tor_address.cpp b/src/net/tor_address.cpp index 4414861e7..a04dcb042 100644 --- a/src/net/tor_address.cpp +++ b/src/net/tor_address.cpp @@ -38,7 +38,7 @@ #include "net/error.h" #include "serialization/keyvalue_serialization.h" #include "storages/portable_storage.h" -#include "string_tools.h" +#include "string_tools_lexical.h" namespace net { diff --git a/src/p2p/CMakeLists.txt b/src/p2p/CMakeLists.txt index cfd8273b4..0cd38f253 100644 --- a/src/p2p/CMakeLists.txt +++ b/src/p2p/CMakeLists.txt @@ -26,7 +26,7 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -cmake_minimum_required (VERSION 2.6) +cmake_minimum_required (VERSION 3.5) project (monero CXX) file(GLOB P2P *) diff --git a/src/p2p/net_node.cpp b/src/p2p/net_node.cpp index 8dd551d1e..84cc1581e 100644 --- a/src/p2p/net_node.cpp +++ b/src/p2p/net_node.cpp @@ -166,6 +166,7 @@ namespace nodetool const command_line::arg_descriptor<bool> arg_pad_transactions = { "pad-transactions", "Pad relayed transactions to help defend against traffic volume analysis", false }; + const command_line::arg_descriptor<uint32_t> arg_max_connections_per_ip = {"max-connections-per-ip", "Maximum number of connections allowed from the same IP address", 1}; boost::optional<std::vector<proxy>> get_proxies(boost::program_options::variables_map const& vm) { diff --git a/src/p2p/net_node.h b/src/p2p/net_node.h index 59a6e5091..f2888674b 100644 --- a/src/p2p/net_node.h +++ b/src/p2p/net_node.h @@ -258,7 +258,8 @@ namespace nodetool m_igd(no_igd), m_offline(false), is_closing(false), - m_network_id() + m_network_id(), + max_connections(1) {} virtual ~node_server(); @@ -343,10 +344,9 @@ namespace nodetool virtual void on_connection_close(p2p_connection_context& context); virtual void callback(p2p_connection_context& context); //----------------- i_p2p_endpoint ------------------------------------------------------------- - virtual bool relay_notify_to_list(int command, const epee::span<const uint8_t> data_buff, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections); + virtual bool relay_notify_to_list(int command, epee::levin::message_writer message, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections) final; virtual epee::net_utils::zone send_txs(std::vector<cryptonote::blobdata> txs, const epee::net_utils::zone origin, const boost::uuids::uuid& source, cryptonote::relay_method tx_relay); - virtual bool invoke_command_to_peer(int command, const epee::span<const uint8_t> req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context); - virtual bool invoke_notify_to_peer(int command, const epee::span<const uint8_t> req_buff, const epee::net_utils::connection_context_base& context); + virtual bool invoke_notify_to_peer(int command, epee::levin::message_writer message, const epee::net_utils::connection_context_base& context) final; virtual bool drop_connection(const epee::net_utils::connection_context_base& context); virtual void request_callback(const epee::net_utils::connection_context_base& context); virtual void for_each_connection(std::function<bool(typename t_payload_net_handler::connection_context&, peerid_type, uint32_t)> f); @@ -517,6 +517,8 @@ namespace nodetool epee::net_utils::ssl_support_t m_ssl_support; bool m_enable_dns_blocklist; + + uint32_t max_connections; }; const int64_t default_limit_up = P2P_DEFAULT_LIMIT_RATE_UP; // kB/s @@ -551,6 +553,7 @@ namespace nodetool extern const command_line::arg_descriptor<int64_t> arg_limit_rate_down; extern const command_line::arg_descriptor<int64_t> arg_limit_rate; extern const command_line::arg_descriptor<bool> arg_pad_transactions; + extern const command_line::arg_descriptor<uint32_t> arg_max_connections_per_ip; } POP_WARNINGS diff --git a/src/p2p/net_node.inl b/src/p2p/net_node.inl index 05af22a7f..cfeac3d37 100644 --- a/src/p2p/net_node.inl +++ b/src/p2p/net_node.inl @@ -131,6 +131,7 @@ namespace nodetool command_line::add_arg(desc, arg_limit_rate_down); command_line::add_arg(desc, arg_limit_rate); command_line::add_arg(desc, arg_pad_transactions); + command_line::add_arg(desc, arg_max_connections_per_ip); } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> @@ -269,9 +270,17 @@ namespace nodetool peerlist_entry pe{}; pe.adr = addr; - zone.second.m_peerlist.remove_from_peer_white(pe); - zone.second.m_peerlist.remove_from_peer_gray(pe); - zone.second.m_peerlist.remove_from_peer_anchor(addr); + if (addr.port() == 0) + { + zone.second.m_peerlist.evict_host_from_peerlist(true, pe); + zone.second.m_peerlist.evict_host_from_peerlist(false, pe); + } + else + { + zone.second.m_peerlist.remove_from_peer_white(pe); + zone.second.m_peerlist.remove_from_peer_gray(pe); + zone.second.m_peerlist.remove_from_peer_anchor(addr); + } for (const auto &c: conns) zone.second.m_net_server.get_config_object().close(c); @@ -331,6 +340,13 @@ namespace nodetool for (const auto &c: conns) zone.second.m_net_server.get_config_object().close(c); + for (int i = 0; i < 2; ++i) + zone.second.m_peerlist.filter(i == 0, [&subnet](const peerlist_entry &pe){ + if (pe.adr.get_type_id() != epee::net_utils::ipv4_network_address::get_type_id()) + return false; + return subnet.matches(pe.adr.as<const epee::net_utils::ipv4_network_address>()); + }); + conns.clear(); } @@ -615,6 +631,8 @@ namespace nodetool return false; } + max_connections = command_line::get_arg(vm, arg_max_connections_per_ip); + return true; } //----------------------------------------------------------------------------------- @@ -679,11 +697,15 @@ namespace nodetool full_addrs.insert("212.83.175.67:28080"); full_addrs.insert("212.83.172.165:28080"); full_addrs.insert("192.110.160.146:28080"); + full_addrs.insert("88.99.173.38:28080"); + full_addrs.insert("51.79.173.165:28080"); } else if (m_nettype == cryptonote::STAGENET) { full_addrs.insert("162.210.173.150:38080"); full_addrs.insert("192.110.160.146:38080"); + full_addrs.insert("88.99.173.38:38080"); + full_addrs.insert("51.79.173.165:38080"); } else if (m_nettype == cryptonote::FAKECHAIN) { @@ -698,6 +720,8 @@ namespace nodetool full_addrs.insert("209.250.243.248:18080"); full_addrs.insert("104.238.221.81:18080"); full_addrs.insert("66.85.74.134:18080"); + full_addrs.insert("88.99.173.38:18080"); + full_addrs.insert("51.79.173.165:18080"); } return full_addrs; } @@ -825,6 +849,7 @@ namespace nodetool "xwvz3ekocr3dkyxfkmgm2hvbpzx2ysqmaxgter7znnqrhoicygkfswid.onion:18083", "4pixvbejrvihnkxmduo2agsnmc3rrulrqc7s3cbwwrep6h6hrzsibeqd.onion:18083", "zbjkbsxc5munw3qusl7j2hpcmikhqocdf4pqhnhtpzw5nt5jrmofptid.onion:18083", + "qz43zul2x56jexzoqgkx2trzwcfnr6l3hbtfcfx54g4r3eahy3bssjyd.onion:18083", }; } return {}; @@ -894,32 +919,6 @@ namespace nodetool for(const auto& p: m_command_line_peers) m_network_zones.at(p.adr.get_zone()).m_peerlist.append_with_peer_white(p); -// all peers are now setup -#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED - for (auto& zone : m_network_zones) - { - std::list<peerlist_entry> plw; - while (zone.second.m_peerlist.get_white_peers_count()) - { - plw.push_back(peerlist_entry()); - zone.second.m_peerlist.get_white_peer_by_index(plw.back(), 0); - zone.second.m_peerlist.remove_from_peer_white(plw.back()); - } - for (auto &e:plw) - zone.second.m_peerlist.append_with_peer_white(e); - - std::list<peerlist_entry> plg; - while (zone.second.m_peerlist.get_gray_peers_count()) - { - plg.push_back(peerlist_entry()); - zone.second.m_peerlist.get_gray_peer_by_index(plg.back(), 0); - zone.second.m_peerlist.remove_from_peer_gray(plg.back()); - } - for (auto &e:plg) - zone.second.m_peerlist.append_with_peer_gray(e); - } -#endif - //only in case if we really sure that we have external visible ip m_have_address = true; @@ -1162,6 +1161,7 @@ namespace nodetool pi = context.peer_id = rsp.node_data.peer_id; context.m_rpc_port = rsp.node_data.rpc_port; context.m_rpc_credits_per_hash = rsp.node_data.rpc_credits_per_hash; + context.support_flags = rsp.node_data.support_flags; const auto azone = context.m_remote_address.get_zone(); network_zone& zone = m_network_zones.at(azone); zone.m_peerlist.set_peer_just_seen(rsp.node_data.peer_id, context.m_remote_address, context.m_pruning_seed, context.m_rpc_port, context.m_rpc_credits_per_hash); @@ -1195,10 +1195,11 @@ namespace nodetool } else if (!just_take_peerlist) { - try_get_support_flags(context_, [](p2p_connection_context& flags_context, const uint32_t& support_flags) - { - flags_context.support_flags = support_flags; - }); + if (context_.support_flags == 0) + try_get_support_flags(context_, [](p2p_connection_context& flags_context, const uint32_t& support_flags) + { + flags_context.support_flags = support_flags; + }); } return hsh_result; @@ -1224,9 +1225,8 @@ namespace nodetool if(!handle_remote_peerlist(rsp.local_peerlist_new, context)) { LOG_WARNING_CC(context, "COMMAND_TIMED_SYNC: failed to handle_remote_peerlist(...), closing connection."); - const auto remote_address = context.m_remote_address; m_network_zones.at(context.m_remote_address.get_zone()).m_net_server.get_config_object().close(context.m_connection_id ); - add_host_fail(remote_address); + add_host_fail(context.m_remote_address); } if(!context.m_is_income) m_network_zones.at(context.m_remote_address.get_zone()).m_peerlist.set_peer_just_seen(context.peer_id, context.m_remote_address, context.m_pruning_seed, context.m_rpc_port, context.m_rpc_credits_per_hash); @@ -1251,8 +1251,8 @@ namespace nodetool if(!max_index) return 0; - size_t x = crypto::rand<size_t>()%(max_index+1); - size_t res = (x*x*x)/(max_index*max_index); //parabola \/ + size_t x = crypto::rand<size_t>()%(16*max_index+1); + size_t res = (x*x*x)/(max_index*max_index*16*16*16); //parabola \/ MDEBUG("Random connection index=" << res << "(x="<< x << ", max_index=" << max_index << ")"); return res; } @@ -1390,7 +1390,7 @@ namespace nodetool if(just_take_peerlist) { zone.m_net_server.get_config_object().close(con->m_connection_id); - MDEBUG(na.str() << "CONNECTION HANDSHAKED OK AND CLOSED."); + LOG_DEBUG_CC(*con, "CONNECTION HANDSHAKED OK AND CLOSED."); return true; } @@ -1452,7 +1452,7 @@ namespace nodetool zone.m_net_server.get_config_object().close(con->m_connection_id); - MDEBUG(na.str() << "CONNECTION HANDSHAKED OK AND CLOSED."); + LOG_DEBUG_CC(*con, "CONNECTION HANDSHAKED OK AND CLOSED."); return true; } @@ -2022,15 +2022,24 @@ namespace nodetool boost::split(ips, record, boost::is_any_of(";")); for (const auto &ip: ips) { + if (ip.empty()) + continue; + auto subnet = net::get_ipv4_subnet_address(ip); + if (subnet) + { + block_subnet(*subnet, DNS_BLOCKLIST_LIFETIME); + ++good; + continue; + } const expect<epee::net_utils::network_address> parsed_addr = net::get_network_address(ip, 0); - if (!parsed_addr) + if (parsed_addr) { - MWARNING("Invalid IP address from DNS blocklist: " << ip << " - " << parsed_addr.error()); - ++bad; + block_host(*parsed_addr, DNS_BLOCKLIST_LIFETIME, true); + ++good; continue; } - block_host(*parsed_addr, DNS_BLOCKLIST_LIFETIME, true); - ++good; + MWARNING("Invalid IP address or subnet from DNS blocklist: " << ip << " - " << parsed_addr.error()); + ++bad; } } if (good > 0) @@ -2125,10 +2134,6 @@ namespace nodetool continue; } local_peerlist[i].last_seen = 0; - -#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED - be.pruning_seed = tools::make_pruning_seed(1 + (be.adr.as<epee::net_utils::ipv4_network_address>().ip()) % (1ul << CRYPTONOTE_PRUNING_LOG_STRIPES), CRYPTONOTE_PRUNING_LOG_STRIPES); -#endif } return true; } @@ -2174,6 +2179,7 @@ namespace nodetool node_data.rpc_port = zone.m_can_pingback ? m_rpc_port : 0; node_data.rpc_credits_per_hash = zone.m_can_pingback ? m_rpc_credits_per_hash : 0; node_data.network_id = m_network_id; + node_data.support_flags = zone.m_config.m_support_flags; return true; } //----------------------------------------------------------------------------------- @@ -2191,8 +2197,9 @@ namespace nodetool } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::relay_notify_to_list(int command, const epee::span<const uint8_t> data_buff, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections) + bool node_server<t_payload_net_handler>::relay_notify_to_list(int command, epee::levin::message_writer data_buff, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections) { + epee::byte_slice message = data_buff.finalize_notify(command); std::sort(connections.begin(), connections.end()); auto zone = m_network_zones.begin(); for(const auto& c_id: connections) @@ -2210,7 +2217,7 @@ namespace nodetool ++zone; } if (zone->first == c_id.first) - zone->second.m_net_server.get_config_object().notify(command, data_buff, c_id.second); + zone->second.m_net_server.get_config_object().send(message.clone(), c_id.second); } return true; } @@ -2277,24 +2284,13 @@ namespace nodetool } //----------------------------------------------------------------------------------- template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::invoke_notify_to_peer(int command, const epee::span<const uint8_t> req_buff, const epee::net_utils::connection_context_base& context) + bool node_server<t_payload_net_handler>::invoke_notify_to_peer(const int command, epee::levin::message_writer message, const epee::net_utils::connection_context_base& context) { if(is_filtered_command(context.m_remote_address, command)) return false; network_zone& zone = m_network_zones.at(context.m_remote_address.get_zone()); - int res = zone.m_net_server.get_config_object().notify(command, req_buff, context.m_connection_id); - return res > 0; - } - //----------------------------------------------------------------------------------- - template<class t_payload_net_handler> - bool node_server<t_payload_net_handler>::invoke_command_to_peer(int command, const epee::span<const uint8_t> req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context) - { - if(is_filtered_command(context.m_remote_address, command)) - return false; - - network_zone& zone = m_network_zones.at(context.m_remote_address.get_zone()); - int res = zone.m_net_server.get_config_object().invoke(command, req_buff, resp_buff, context.m_connection_id); + int res = zone.m_net_server.get_config_object().send(message.finalize_notify(command), context.m_connection_id); return res > 0; } //----------------------------------------------------------------------------------- @@ -2480,14 +2476,12 @@ namespace nodetool template<class t_payload_net_handler> int node_server<t_payload_net_handler>::handle_handshake(int command, typename COMMAND_HANDSHAKE::request& arg, typename COMMAND_HANDSHAKE::response& rsp, p2p_connection_context& context) { - // copy since dropping the connection will invalidate the context, and thus the address - const auto remote_address = context.m_remote_address; - if(arg.node_data.network_id != m_network_id) { + LOG_INFO_CC(context, "WRONG NETWORK AGENT CONNECTED! id=" << arg.node_data.network_id); drop_connection(context); - add_host_fail(remote_address); + add_host_fail(context.m_remote_address); return 1; } @@ -2495,7 +2489,7 @@ namespace nodetool { LOG_WARNING_CC(context, "COMMAND_HANDSHAKE came not from incoming connection"); drop_connection(context); - add_host_fail(remote_address); + add_host_fail(context.m_remote_address); return 1; } @@ -2544,6 +2538,7 @@ namespace nodetool context.m_in_timedsync = false; context.m_rpc_port = arg.node_data.rpc_port; context.m_rpc_credits_per_hash = arg.node_data.rpc_credits_per_hash; + context.support_flags = arg.node_data.support_flags; if(arg.node_data.my_port && zone.m_can_pingback) { @@ -2577,10 +2572,11 @@ namespace nodetool }); } - try_get_support_flags(context, [](p2p_connection_context& flags_context, const uint32_t& support_flags) - { - flags_context.support_flags = support_flags; - }); + if (context.support_flags == 0) + try_get_support_flags(context, [](p2p_connection_context& flags_context, const uint32_t& support_flags) + { + flags_context.support_flags = support_flags; + }); //fill response zone.m_peerlist.get_peerlist_head(rsp.local_peerlist_new, true); @@ -2846,8 +2842,7 @@ namespace nodetool if (address.get_zone() != epee::net_utils::zone::public_) return false; // Unable to determine how many connections from host - const size_t max_connections = 1; - size_t count = 0; + uint32_t count = 0; m_network_zones.at(epee::net_utils::zone::public_).m_net_server.get_config_object().foreach_connection([&](const p2p_connection_context& cntxt) { diff --git a/src/p2p/net_node_common.h b/src/p2p/net_node_common.h index 0da758ad4..92b7596ae 100644 --- a/src/p2p/net_node_common.h +++ b/src/p2p/net_node_common.h @@ -40,6 +40,8 @@ #include "net/net_utils_base.h" #include "p2p_protocol_defs.h" +namespace epee { namespace levin { class message_writer; } } + namespace nodetool { @@ -49,10 +51,9 @@ namespace nodetool template<class t_connection_context> struct i_p2p_endpoint { - virtual bool relay_notify_to_list(int command, const epee::span<const uint8_t> data_buff, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections)=0; + virtual bool relay_notify_to_list(int command, epee::levin::message_writer message, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections)=0; virtual epee::net_utils::zone send_txs(std::vector<cryptonote::blobdata> txs, const epee::net_utils::zone origin, const boost::uuids::uuid& source, cryptonote::relay_method tx_relay)=0; - virtual bool invoke_command_to_peer(int command, const epee::span<const uint8_t> req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context)=0; - virtual bool invoke_notify_to_peer(int command, const epee::span<const uint8_t> req_buff, const epee::net_utils::connection_context_base& context)=0; + virtual bool invoke_notify_to_peer(int command, epee::levin::message_writer message, const epee::net_utils::connection_context_base& context)=0; virtual bool drop_connection(const epee::net_utils::connection_context_base& context)=0; virtual void request_callback(const epee::net_utils::connection_context_base& context)=0; virtual uint64_t get_public_connections_count()=0; @@ -71,7 +72,7 @@ namespace nodetool template<class t_connection_context> struct p2p_endpoint_stub: public i_p2p_endpoint<t_connection_context> { - virtual bool relay_notify_to_list(int command, const epee::span<const uint8_t> data_buff, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections) + virtual bool relay_notify_to_list(int command, epee::levin::message_writer message, std::vector<std::pair<epee::net_utils::zone, boost::uuids::uuid>> connections) { return false; } @@ -79,11 +80,7 @@ namespace nodetool { return epee::net_utils::zone::invalid; } - virtual bool invoke_command_to_peer(int command, const epee::span<const uint8_t> req_buff, std::string& resp_buff, const epee::net_utils::connection_context_base& context) - { - return false; - } - virtual bool invoke_notify_to_peer(int command, const epee::span<const uint8_t> req_buff, const epee::net_utils::connection_context_base& context) + virtual bool invoke_notify_to_peer(int command, epee::levin::message_writer message, const epee::net_utils::connection_context_base& context) { return true; } diff --git a/src/p2p/net_peerlist.cpp b/src/p2p/net_peerlist.cpp index 42ab9727d..50dc6da77 100644 --- a/src/p2p/net_peerlist.cpp +++ b/src/p2p/net_peerlist.cpp @@ -289,17 +289,9 @@ namespace nodetool copy_peers(peers.anchor, m_peers_anchor.get<by_addr>()); } - void peerlist_manager::evict_host_from_white_peerlist(const peerlist_entry& pr) + void peerlist_manager::evict_host_from_peerlist(bool use_white, const peerlist_entry& pr) { - peers_indexed::index<by_time>::type& sorted_index=m_peers_white.get<by_time>(); - auto i = sorted_index.begin(); - while (i != sorted_index.end()) - { - if (i->adr.is_same_host(pr.adr)) - i = sorted_index.erase(i); - else - ++i; - } + filter(use_white, [&pr](const peerlist_entry& pe){ return pe.adr.is_same_host(pr.adr); }); } } diff --git a/src/p2p/net_peerlist.h b/src/p2p/net_peerlist.h index c794b0f3b..0662789b9 100644 --- a/src/p2p/net_peerlist.h +++ b/src/p2p/net_peerlist.h @@ -109,8 +109,8 @@ namespace nodetool bool get_white_peer_by_index(peerlist_entry& p, size_t i); bool get_gray_peer_by_index(peerlist_entry& p, size_t i); template<typename F> bool foreach(bool white, const F &f); - void evict_host_from_white_peerlist(const peerlist_entry& pr); - bool append_with_peer_white(const peerlist_entry& pr); + void evict_host_from_peerlist(bool white, const peerlist_entry& pr); + bool append_with_peer_white(const peerlist_entry& pr, bool trust_last_seen = false); bool append_with_peer_gray(const peerlist_entry& pr); bool append_with_peer_anchor(const anchor_peerlist_entry& ple); bool set_peer_just_seen(peerid_type peer, const epee::net_utils::network_address& addr, uint32_t pruning_seed, uint16_t rpc_port, uint32_t rpc_credits_per_hash); @@ -120,6 +120,7 @@ namespace nodetool bool get_and_empty_anchor_peerlist(std::vector<anchor_peerlist_entry>& apl); bool remove_from_peer_anchor(const epee::net_utils::network_address& addr); bool remove_from_peer_white(const peerlist_entry& pe); + template<typename F> size_t filter(bool white, const F &f); // f returns true: drop, false: keep private: struct by_time{}; @@ -329,12 +330,12 @@ namespace nodetool ple.pruning_seed = pruning_seed; ple.rpc_port = rpc_port; ple.rpc_credits_per_hash = rpc_credits_per_hash; - return append_with_peer_white(ple); + return append_with_peer_white(ple, true); CATCH_ENTRY_L0("peerlist_manager::set_peer_just_seen()", false); } //-------------------------------------------------------------------------------------------------- inline - bool peerlist_manager::append_with_peer_white(const peerlist_entry& ple) + bool peerlist_manager::append_with_peer_white(const peerlist_entry& ple, bool trust_last_seen) { TRY_ENTRY(); if(!is_host_allowed(ple.adr)) @@ -346,7 +347,7 @@ namespace nodetool if(by_addr_it_wt == m_peers_white.get<by_addr>().end()) { //put new record into white list - evict_host_from_white_peerlist(ple); + evict_host_from_peerlist(true, ple); m_peers_white.insert(ple); trim_white_peerlist(); }else @@ -357,7 +358,8 @@ namespace nodetool new_ple.pruning_seed = by_addr_it_wt->pruning_seed; if (by_addr_it_wt->rpc_port && ple.rpc_port == 0) // guard against older nodes not passing RPC port around new_ple.rpc_port = by_addr_it_wt->rpc_port; - new_ple.last_seen = by_addr_it_wt->last_seen; // do not overwrite the last seen timestamp, incoming peer list are untrusted + if (!trust_last_seen) + new_ple.last_seen = by_addr_it_wt->last_seen; // do not overwrite the last seen timestamp, incoming peer lists are untrusted m_peers_white.replace(by_addr_it_wt, new_ple); } //remove from gray list, if need @@ -519,5 +521,26 @@ namespace nodetool CATCH_ENTRY_L0("peerlist_manager::remove_from_peer_anchor()", false); } //-------------------------------------------------------------------------------------------------- + template<typename F> size_t peerlist_manager::filter(bool white, const F &f) + { + size_t filtered = 0; + TRY_ENTRY(); + CRITICAL_REGION_LOCAL(m_peerlist_lock); + peers_indexed::index<by_addr>::type& sorted_index = white ? m_peers_gray.get<by_addr>() : m_peers_white.get<by_addr>(); + auto i = sorted_index.begin(); + while (i != sorted_index.end()) + { + if (f(*i)) + { + i = sorted_index.erase(i); + ++filtered; + } + else + ++i; + } + CATCH_ENTRY_L0("peerlist_manager::filter()", filtered); + return filtered; + } + //-------------------------------------------------------------------------------------------------- } diff --git a/src/p2p/net_peerlist_boost_serialization.h b/src/p2p/net_peerlist_boost_serialization.h index 37a85d526..1cf4cb026 100644 --- a/src/p2p/net_peerlist_boost_serialization.h +++ b/src/p2p/net_peerlist_boost_serialization.h @@ -38,10 +38,6 @@ #include "net/i2p_address.h" #include "p2p/p2p_protocol_defs.h" -#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED -#include "common/pruning.h" -#endif - BOOST_CLASS_VERSION(nodetool::peerlist_entry, 3) namespace boost @@ -228,12 +224,6 @@ namespace boost return; } a & pl.pruning_seed; -#ifdef CRYPTONOTE_PRUNING_DEBUG_SPOOF_SEED - if (!typename Archive::is_saving()) - { - pl.pruning_seed = tools::make_pruning_seed(1+pl.adr.as<epee::net_utils::ipv4_network_address>().ip() % (1<<CRYPTONOTE_PRUNING_LOG_STRIPES), CRYPTONOTE_PRUNING_LOG_STRIPES); - } -#endif if (ver < 2) { if (!typename Archive::is_saving()) diff --git a/src/p2p/p2p_protocol_defs.h b/src/p2p/p2p_protocol_defs.h index b439dc47e..12763d4ee 100644 --- a/src/p2p/p2p_protocol_defs.h +++ b/src/p2p/p2p_protocol_defs.h @@ -188,6 +188,7 @@ namespace nodetool uint16_t rpc_port; uint32_t rpc_credits_per_hash; peerid_type peer_id; + uint32_t support_flags; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE_VAL_POD_AS_BLOB(network_id) @@ -195,6 +196,7 @@ namespace nodetool KV_SERIALIZE(my_port) KV_SERIALIZE_OPT(rpc_port, (uint16_t)(0)) KV_SERIALIZE_OPT(rpc_credits_per_hash, (uint32_t)0) + KV_SERIALIZE_OPT(support_flags, (uint32_t)0) END_KV_SERIALIZE_MAP() }; diff --git a/src/ringct/multiexp.cc b/src/ringct/multiexp.cc index f69b4a12c..620f7d0dd 100644 --- a/src/ringct/multiexp.cc +++ b/src/ringct/multiexp.cc @@ -372,7 +372,6 @@ std::shared_ptr<straus_cached_data> straus_init_cache(const std::vector<Multiexp if (N == 0) N = data.size(); CHECK_AND_ASSERT_THROW_MES(N <= data.size(), "Bad cache base data"); - ge_cached cached; ge_p1p1 p1; ge_p3 p3; std::shared_ptr<straus_cached_data> cache(new straus_cached_data()); @@ -454,7 +453,6 @@ rct::key straus(const std::vector<MultiexpData> &data, const std::shared_ptr<str std::shared_ptr<straus_cached_data> local_cache = cache == NULL ? straus_init_cache(data) : cache; ge_cached cached; ge_p1p1 p1; - ge_p3 p3; #ifdef TRACK_STRAUS_ZERO_IDENTITY MULTIEXP_PERF(PERF_TIMER_START_UNIT(skip, 1000000)); @@ -587,7 +585,6 @@ std::shared_ptr<pippenger_cached_data> pippenger_init_cache(const std::vector<Mu if (N == 0) N = data.size() - start_offset; CHECK_AND_ASSERT_THROW_MES(N <= data.size() - start_offset, "Bad cache base data"); - ge_cached cached; std::shared_ptr<pippenger_cached_data> cache(new pippenger_cached_data()); cache->size = N; diff --git a/src/ringct/rctSigs.cpp b/src/ringct/rctSigs.cpp index 93eb52d4e..f5950c53c 100644 --- a/src/ringct/rctSigs.cpp +++ b/src/ringct/rctSigs.cpp @@ -725,7 +725,6 @@ namespace rct { CHECK_AND_ASSERT_THROW_MES((kLRki && mscout) || (!kLRki && !mscout), "Only one of kLRki/mscout is present"); keyV tmp(rows + 1); keyV sk(rows + 1); - size_t i; keyM M(cols, tmp); keyV P, C, C_nonzero; @@ -899,7 +898,6 @@ namespace rct { key R; geDsmp P_precomp; geDsmp C_precomp; - geDsmp H_precomp; size_t i = 0; ge_p3 hash8_p3; geDsmp hash_precomp; diff --git a/src/ringct/rctTypes.h b/src/ringct/rctTypes.h index 00b72123a..278ff4164 100644 --- a/src/ringct/rctTypes.h +++ b/src/ringct/rctTypes.h @@ -284,7 +284,7 @@ namespace rct { { FIELD(type) if (type == RCTTypeNull) - return ar.stream().good(); + return ar.good(); if (type != RCTTypeFull && type != RCTTypeSimple && type != RCTTypeBulletproof && type != RCTTypeBulletproof2 && type != RCTTypeCLSAG) return false; VARINT_FIELD(txnFee) @@ -344,7 +344,7 @@ namespace rct { ar.delimit_array(); } ar.end_array(); - return ar.stream().good(); + return ar.good(); } BEGIN_SERIALIZE_OBJECT() @@ -375,7 +375,7 @@ namespace rct { if (mixin >= 0xffffffff) return false; if (type == RCTTypeNull) - return ar.stream().good(); + return ar.good(); if (type != RCTTypeFull && type != RCTTypeSimple && type != RCTTypeBulletproof && type != RCTTypeBulletproof2 && type != RCTTypeCLSAG) return false; if (type == RCTTypeBulletproof || type == RCTTypeBulletproof2 || type == RCTTypeCLSAG) @@ -522,7 +522,7 @@ namespace rct { } ar.end_array(); } - return ar.stream().good(); + return ar.good(); } BEGIN_SERIALIZE_OBJECT() diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt index aa4102481..15e433e10 100644 --- a/src/rpc/CMakeLists.txt +++ b/src/rpc/CMakeLists.txt @@ -39,7 +39,7 @@ set(rpc_sources core_rpc_server.cpp rpc_payment.cpp rpc_version_str.cpp - instanciations) + instanciations.cpp) set(daemon_messages_sources message.cpp diff --git a/src/rpc/bootstrap_daemon.cpp b/src/rpc/bootstrap_daemon.cpp index 2fdd28406..ffea906d5 100644 --- a/src/rpc/bootstrap_daemon.cpp +++ b/src/rpc/bootstrap_daemon.cpp @@ -7,6 +7,7 @@ #include "crypto/crypto.h" #include "cryptonote_core/cryptonote_core.h" #include "misc_log_ex.h" +#include "net/parse.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "daemon.rpc.bootstrap_daemon" @@ -16,19 +17,23 @@ namespace cryptonote bootstrap_daemon::bootstrap_daemon( std::function<std::map<std::string, bool>()> get_public_nodes, - bool rpc_payment_enabled) + bool rpc_payment_enabled, + const std::string &proxy) : m_selector(new bootstrap_node::selector_auto(std::move(get_public_nodes))) , m_rpc_payment_enabled(rpc_payment_enabled) { + set_proxy(proxy); } bootstrap_daemon::bootstrap_daemon( const std::string &address, boost::optional<epee::net_utils::http::login> credentials, - bool rpc_payment_enabled) + bool rpc_payment_enabled, + const std::string &proxy) : m_selector(nullptr) , m_rpc_payment_enabled(rpc_payment_enabled) { + set_proxy(proxy); if (!set_server(address, std::move(credentials))) { throw std::runtime_error("invalid bootstrap daemon address or credentials"); @@ -78,6 +83,18 @@ namespace cryptonote return success; } + void bootstrap_daemon::set_proxy(const std::string &address) + { + if (!address.empty() && !net::get_tcp_endpoint(address)) + { + throw std::runtime_error("invalid proxy address format"); + } + if (!m_http_client.set_proxy(address)) + { + throw std::runtime_error("failed to set proxy address"); + } + } + bool bootstrap_daemon::set_server(const std::string &address, const boost::optional<epee::net_utils::http::login> &credentials /* = boost::none */) { if (!m_http_client.set_server(address, credentials)) diff --git a/src/rpc/bootstrap_daemon.h b/src/rpc/bootstrap_daemon.h index d54042b11..1e4477123 100644 --- a/src/rpc/bootstrap_daemon.h +++ b/src/rpc/bootstrap_daemon.h @@ -8,7 +8,7 @@ #include <boost/thread/mutex.hpp> #include <boost/utility/string_ref.hpp> -#include "net/http_client.h" +#include "net/http.h" #include "storages/http_abstract_invoke.h" #include "bootstrap_node_selector.h" @@ -21,11 +21,13 @@ namespace cryptonote public: bootstrap_daemon( std::function<std::map<std::string, bool>()> get_public_nodes, - bool rpc_payment_enabled); + bool rpc_payment_enabled, + const std::string &proxy); bootstrap_daemon( const std::string &address, boost::optional<epee::net_utils::http::login> credentials, - bool rpc_payment_enabled); + bool rpc_payment_enabled, + const std::string &proxy); std::string address() const noexcept; boost::optional<std::pair<uint64_t, uint64_t>> get_height(); @@ -72,12 +74,14 @@ namespace cryptonote return handle_result(result, result_struct.status); } + void set_proxy(const std::string &address); + private: bool set_server(const std::string &address, const boost::optional<epee::net_utils::http::login> &credentials = boost::none); bool switch_server_if_needed(); private: - epee::net_utils::http::http_simple_client m_http_client; + net::http::client m_http_client; const bool m_rpc_payment_enabled; const std::unique_ptr<bootstrap_node::selector> m_selector; boost::mutex m_selector_mutex; diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index db228dd94..8d8a68efb 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -30,6 +30,7 @@ #include <boost/preprocessor/stringize.hpp> #include <boost/uuid/nil_generator.hpp> +#include <boost/filesystem.hpp> #include "include_base_utils.h" #include "string_tools.h" using namespace epee; @@ -44,6 +45,7 @@ using namespace epee; #include "cryptonote_basic/cryptonote_format_utils.h" #include "cryptonote_basic/account.h" #include "cryptonote_basic/cryptonote_basic_impl.h" +#include "cryptonote_basic/merge_mining.h" #include "cryptonote_core/tx_sanity_check.h" #include "misc_language.h" #include "net/parse.h" @@ -154,6 +156,7 @@ namespace cryptonote command_line::add_arg(desc, arg_restricted_rpc); command_line::add_arg(desc, arg_bootstrap_daemon_address); command_line::add_arg(desc, arg_bootstrap_daemon_login); + command_line::add_arg(desc, arg_bootstrap_daemon_proxy); cryptonote::rpc_args::init_options(desc, true); command_line::add_arg(desc, arg_rpc_payment_address); command_line::add_arg(desc, arg_rpc_payment_difficulty); @@ -172,7 +175,10 @@ namespace cryptonote , m_rpc_payment_allow_free_loopback(false) {} //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::set_bootstrap_daemon(const std::string &address, const std::string &username_password) + bool core_rpc_server::set_bootstrap_daemon( + const std::string &address, + const std::string &username_password, + const std::string &proxy) { boost::optional<epee::net_utils::http::login> credentials; const auto loc = username_password.find(':'); @@ -180,7 +186,7 @@ namespace cryptonote { credentials = epee::net_utils::http::login(username_password.substr(0, loc), username_password.substr(loc + 1)); } - return set_bootstrap_daemon(address, credentials); + return set_bootstrap_daemon(address, credentials, proxy); } //------------------------------------------------------------------------------------------------------------------------------ std::map<std::string, bool> core_rpc_server::get_public_nodes(uint32_t credits_per_hash_threshold/* = 0*/) @@ -217,7 +223,10 @@ namespace cryptonote return result; } //------------------------------------------------------------------------------------------------------------------------------ - bool core_rpc_server::set_bootstrap_daemon(const std::string &address, const boost::optional<epee::net_utils::http::login> &credentials) + bool core_rpc_server::set_bootstrap_daemon( + const std::string &address, + const boost::optional<epee::net_utils::http::login> &credentials, + const std::string &proxy) { boost::unique_lock<boost::shared_mutex> lock(m_bootstrap_daemon_mutex); @@ -233,11 +242,11 @@ namespace cryptonote auto get_nodes = [this]() { return get_public_nodes(credits_per_hash_threshold); }; - m_bootstrap_daemon.reset(new bootstrap_daemon(std::move(get_nodes), rpc_payment_enabled)); + m_bootstrap_daemon.reset(new bootstrap_daemon(std::move(get_nodes), rpc_payment_enabled, proxy)); } else { - m_bootstrap_daemon.reset(new bootstrap_daemon(address, credentials, rpc_payment_enabled)); + m_bootstrap_daemon.reset(new bootstrap_daemon(address, credentials, rpc_payment_enabled, proxy)); } m_should_use_bootstrap_daemon = m_bootstrap_daemon.get() != nullptr; @@ -278,6 +287,7 @@ namespace cryptonote } } disable_rpc_ban = rpc_config->disable_rpc_ban; + const std::string data_dir{command_line::get_arg(vm, cryptonote::arg_data_dir)}; std::string address = command_line::get_arg(vm, arg_rpc_payment_address); if (!address.empty() && allow_rpc_payment) { @@ -306,7 +316,7 @@ namespace cryptonote } m_rpc_payment_allow_free_loopback = command_line::get_arg(vm, arg_rpc_payment_allow_free_loopback); m_rpc_payment.reset(new rpc_payment(info.address, diff, credits)); - m_rpc_payment->load(command_line::get_arg(vm, cryptonote::arg_data_dir)); + m_rpc_payment->load(data_dir); m_p2p.set_rpc_credits_per_hash(RPC_CREDITS_PER_HASH_SCALE * (credits / (float)diff)); } @@ -318,8 +328,10 @@ namespace cryptonote MWARNING("The RPC server is accessible from the outside, but no RPC payment was setup. RPC access will be free for all."); } - if (!set_bootstrap_daemon(command_line::get_arg(vm, arg_bootstrap_daemon_address), - command_line::get_arg(vm, arg_bootstrap_daemon_login))) + if (!set_bootstrap_daemon( + command_line::get_arg(vm, arg_bootstrap_daemon_address), + command_line::get_arg(vm, arg_bootstrap_daemon_login), + command_line::get_arg(vm, arg_bootstrap_daemon_proxy))) { MFATAL("Failed to parse bootstrap daemon address"); return false; @@ -333,12 +345,32 @@ namespace cryptonote if (m_rpc_payment) m_net_server.add_idle_handler([this](){ return m_rpc_payment->on_idle(); }, 60 * 1000); + bool store_ssl_key = !restricted && rpc_config->ssl_options && rpc_config->ssl_options.auth.certificate_path.empty(); + const auto ssl_base_path = (boost::filesystem::path{data_dir} / "rpc_ssl").string(); + if (store_ssl_key && boost::filesystem::exists(ssl_base_path + ".crt")) + { + // load key from previous run, password prompted by OpenSSL + store_ssl_key = false; + rpc_config->ssl_options.auth = + epee::net_utils::ssl_authentication_t{ssl_base_path + ".key", ssl_base_path + ".crt"}; + } + auto rng = [](size_t len, uint8_t *ptr){ return crypto::rand(len, ptr); }; - return epee::http_server_impl_base<core_rpc_server, connection_context>::init( + const bool inited = epee::http_server_impl_base<core_rpc_server, connection_context>::init( rng, std::move(port), std::move(bind_ip_str), std::move(bind_ipv6_str), std::move(rpc_config->use_ipv6), std::move(rpc_config->require_ipv4), std::move(rpc_config->access_control_origins), std::move(http_login), std::move(rpc_config->ssl_options) ); + + if (store_ssl_key && inited) + { + // new keys were generated, store for next run + const auto error = epee::net_utils::store_ssl_keys(m_net_server.get_ssl_context(), ssl_base_path); + if (error) + MFATAL("Failed to store HTTP SSL cert/key for " << (restricted ? "restricted " : "") << "RPC server: " << error.message()); + return !bool(error); + } + return inited; } //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::check_payment(const std::string &client_message, uint64_t payment, const std::string &rpc, bool same_ts, std::string &message, uint64_t &credits, std::string &top_hash) @@ -367,7 +399,6 @@ namespace cryptonote message = "Client signature does not verify for " + rpc; return false; } - crypto::public_key local_client; if (!m_rpc_payment->pay(client, ts, payment, rpc, same_ts, credits)) { message = CORE_RPC_STATUS_PAYMENT_REQUIRED; @@ -574,7 +605,7 @@ namespace cryptonote if (max_blocks == 0) { res.status = CORE_RPC_STATUS_PAYMENT_REQUIRED; - return false; + return true; } } @@ -583,7 +614,7 @@ namespace cryptonote { res.status = "Failed"; add_host_fail(ctx); - return false; + return true; } CHECK_PAYMENT_SAME_TS(req, res, bs.size() * COST_PER_BLOCK); @@ -619,12 +650,12 @@ namespace cryptonote if (!r) { res.status = "Failed"; - return false; + return true; } if (indices.size() != n_txes_to_lookup || res.output_indices.back().indices.size() != (req.no_miner_tx ? 1 : 0)) { res.status = "Failed"; - return false; + return true; } for (size_t i = 0; i < indices.size(); ++i) res.output_indices.back().indices.push_back({std::move(indices[i])}); @@ -647,7 +678,7 @@ namespace cryptonote if(!m_core.get_alternative_blocks(blks)) { res.status = "Failed"; - return false; + return true; } res.blks_hashes.reserve(blks.size()); @@ -718,7 +749,7 @@ namespace cryptonote { res.status = "Failed"; add_host_fail(ctx); - return false; + return true; } CHECK_PAYMENT_SAME_TS(req, res, res.m_block_ids.size() * COST_PER_BLOCK_HASH); @@ -1013,6 +1044,7 @@ namespace cryptonote if (e.in_pool) { e.block_height = e.block_timestamp = std::numeric_limits<uint64_t>::max(); + e.confirmations = 0; auto it = per_tx_pool_tx_info.find(tx_hash); if (it != per_tx_pool_tx_info.end()) { @@ -1031,6 +1063,7 @@ namespace cryptonote else { e.block_height = m_core.get_blockchain_storage().get_db().get_tx_block_height(tx_hash); + e.confirmations = m_core.get_current_blockchain_height() - e.block_height; e.block_timestamp = m_core.get_blockchain_storage().get_db().get_block_timestamp(e.block_height); e.received_timestamp = 0; e.double_spend_seen = false; @@ -1049,7 +1082,7 @@ namespace cryptonote if (!r) { res.status = "Failed"; - return false; + return true; } } } @@ -1425,7 +1458,8 @@ namespace cryptonote res.status = peer_list_res.status; if (!success) { - return false; + res.status = "Failed to get peer list"; + return true; } if (res.status != CORE_RPC_STATUS_OK) { @@ -1596,15 +1630,15 @@ namespace cryptonote { credentials = epee::net_utils::http::login(req.username, req.password); } - - if (set_bootstrap_daemon(req.address, credentials)) + + if (set_bootstrap_daemon(req.address, credentials, req.proxy)) { res.status = CORE_RPC_STATUS_OK; } else { res.status = "Failed to set bootstrap daemon"; - } + } return true; } @@ -1627,7 +1661,7 @@ namespace cryptonote if (m_should_use_bootstrap_daemon) { res.status = "This command is unsupported for bootstrap daemon"; - return false; + return true; } } res.count = m_core.get_current_blockchain_height(); @@ -1643,7 +1677,7 @@ namespace cryptonote if (m_should_use_bootstrap_daemon) { res = "This command is unsupported for bootstrap daemon"; - return false; + return true; } } if(req.size() != 1) @@ -1806,7 +1840,6 @@ namespace cryptonote return false; } } - uint64_t seed_height; crypto::hash seed_hash, next_seed_hash; if (!get_block_template(info.address, req.prev_block.empty() ? NULL : &prev_block, blob_reserve, reserved_offset, wdiff, res.height, res.expected_reward, b, res.seed_height, seed_hash, next_seed_hash, error_resp)) return false; @@ -1828,6 +1861,125 @@ namespace cryptonote return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool core_rpc_server::on_add_aux_pow(const COMMAND_RPC_ADD_AUX_POW::request& req, COMMAND_RPC_ADD_AUX_POW::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) + { + RPC_TRACKER(add_aux_pow); + bool r; + if (use_bootstrap_daemon_if_necessary<COMMAND_RPC_ADD_AUX_POW>(invoke_http_mode::JON_RPC, "add_aux_pow", req, res, r)) + return r; + + if (req.aux_pow.empty()) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_PARAM; + error_resp.message = "Empty aux pow hash vector"; + return false; + } + + crypto::hash merkle_root; + size_t merkle_tree_depth = 0; + std::vector<std::pair<crypto::hash, crypto::hash>> aux_pow; + std::vector<crypto::hash> aux_pow_raw; + aux_pow.reserve(req.aux_pow.size()); + aux_pow_raw.reserve(req.aux_pow.size()); + for (const auto &s: req.aux_pow) + { + aux_pow.push_back({}); + if (!epee::string_tools::hex_to_pod(s.id, aux_pow.back().first)) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_PARAM; + error_resp.message = "Invalid aux pow id"; + return false; + } + if (!epee::string_tools::hex_to_pod(s.hash, aux_pow.back().second)) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_PARAM; + error_resp.message = "Invalid aux pow hash"; + return false; + } + aux_pow_raw.push_back(aux_pow.back().second); + } + + size_t path_domain = 1; + while ((1u << path_domain) < aux_pow.size()) + ++path_domain; + uint32_t nonce; + const uint32_t max_nonce = 65535; + bool collision = true; + for (nonce = 0; nonce <= max_nonce; ++nonce) + { + std::vector<bool> slots(aux_pow.size(), false); + collision = false; + for (size_t idx = 0; idx < aux_pow.size(); ++idx) + { + const uint32_t slot = cryptonote::get_aux_slot(aux_pow[idx].first, nonce, aux_pow.size()); + if (slot >= aux_pow.size()) + { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = "Computed slot is out of range"; + return false; + } + if (slots[slot]) + { + collision = true; + break; + } + slots[slot] = true; + } + if (!collision) + break; + } + if (collision) + { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = "Failed to find a suitable nonce"; + return false; + } + + crypto::tree_hash((const char(*)[crypto::HASH_SIZE])aux_pow_raw.data(), aux_pow_raw.size(), merkle_root.data); + res.merkle_root = epee::string_tools::pod_to_hex(merkle_root); + res.merkle_tree_depth = cryptonote::encode_mm_depth(aux_pow.size(), nonce); + + blobdata blocktemplate_blob; + if (!epee::string_tools::parse_hexstr_to_binbuff(req.blocktemplate_blob, blocktemplate_blob)) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_PARAM; + error_resp.message = "Invalid blocktemplate_blob"; + return false; + } + + block b; + if (!parse_and_validate_block_from_blob(blocktemplate_blob, b)) + { + error_resp.code = CORE_RPC_ERROR_CODE_WRONG_BLOCKBLOB; + error_resp.message = "Wrong blocktemplate_blob"; + return false; + } + + if (!remove_field_from_tx_extra(b.miner_tx.extra, typeid(cryptonote::tx_extra_merge_mining_tag))) + { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = "Error removing existing merkle root"; + return false; + } + if (!add_mm_merkle_root_to_tx_extra(b.miner_tx.extra, merkle_root, merkle_tree_depth)) + { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = "Error adding merkle root"; + return false; + } + b.invalidate_hashes(); + b.miner_tx.invalidate_hashes(); + + const blobdata block_blob = t_serializable_object_to_blob(b); + const blobdata hashing_blob = get_block_hashing_blob(b); + + res.blocktemplate_blob = string_tools::buff_to_hex_nodelimer(block_blob); + res.blockhashing_blob = string_tools::buff_to_hex_nodelimer(hashing_blob); + res.aux_pow = req.aux_pow; + res.status = CORE_RPC_STATUS_OK; + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool core_rpc_server::on_submitblock(const COMMAND_RPC_SUBMITBLOCK::request& req, COMMAND_RPC_SUBMITBLOCK::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx) { RPC_TRACKER(submitblock); @@ -1835,7 +1987,8 @@ namespace cryptonote boost::shared_lock<boost::shared_mutex> lock(m_bootstrap_daemon_mutex); if (m_should_use_bootstrap_daemon) { - res.status = "This command is unsupported for bootstrap daemon"; + error_resp.code = CORE_RPC_ERROR_CODE_UNSUPPORTED_BOOTSTRAP; + error_resp.message = "This command is unsupported for bootstrap daemon"; return false; } } @@ -2527,7 +2680,7 @@ namespace cryptonote if (!m_core.get_blockchain_storage().flush_txes_from_pool(txids)) { res.status = "Failed to remove one or more tx(es)"; - return false; + return true; } if (failed) @@ -2536,7 +2689,7 @@ namespace cryptonote res.status = "Failed to parse txid"; else res.status = "Failed to parse some of the txids"; - return false; + return true; } res.status = CORE_RPC_STATUS_OK; @@ -2695,7 +2848,7 @@ namespace cryptonote if (req.limit_down != -1) { res.status = CORE_RPC_ERROR_CODE_WRONG_PARAM; - return false; + return true; } epee::net_utils::connection_basic::set_rate_down_limit(nodetool::default_limit_down); } @@ -2709,7 +2862,7 @@ namespace cryptonote if (req.limit_up != -1) { res.status = CORE_RPC_ERROR_CODE_WRONG_PARAM; - return false; + return true; } epee::net_utils::connection_basic::set_rate_up_limit(nodetool::default_limit_up); } @@ -2813,17 +2966,17 @@ namespace cryptonote if (!tools::download(path.string(), res.auto_uri)) { MERROR("Failed to download " << res.auto_uri); - return false; + return true; } if (!tools::sha256sum(path.string(), file_hash)) { MERROR("Failed to hash " << path); - return false; + return true; } if (hash != epee::string_tools::pod_to_hex(file_hash)) { MERROR("Download from " << res.auto_uri << " does not match the expected hash"); - return false; + return true; } MINFO("New version downloaded to " << path); } @@ -2898,6 +3051,8 @@ namespace cryptonote if (failed) { + error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR; + error_resp.message = res.status; return false; } @@ -3009,7 +3164,7 @@ namespace cryptonote if (!req.binary) { res.status = "Binary only call"; - return false; + return true; } try { @@ -3021,7 +3176,7 @@ namespace cryptonote if (!data) { res.status = "Failed to get output distribution"; - return false; + return true; } res.distributions.push_back({std::move(*data), amount, "", req.binary, req.compress}); @@ -3030,7 +3185,7 @@ namespace cryptonote catch (const std::exception &e) { res.status = "Failed to get output distribution"; - return false; + return true; } res.status = CORE_RPC_STATUS_OK; @@ -3264,7 +3419,8 @@ namespace cryptonote if (!m_rpc_payment) { - res.status = "Payments not enabled"; + error_resp.code = CORE_RPC_ERROR_CODE_PAYMENTS_NOT_ENABLED; + error_resp.message = "Payments not enabled"; return false; } @@ -3292,7 +3448,8 @@ namespace cryptonote if (!m_rpc_payment) { - res.status = "Payments not enabled"; + error_resp.code = CORE_RPC_ERROR_CODE_PAYMENTS_NOT_ENABLED; + error_resp.message = "Payments not enabled"; return false; } @@ -3349,6 +3506,12 @@ namespace cryptonote , "" }; + const command_line::arg_descriptor<std::string> core_rpc_server::arg_bootstrap_daemon_proxy = { + "bootstrap-daemon-proxy" + , "<ip>:<port> socks proxy to use for bootstrap daemon connections" + , "" + }; + const command_line::arg_descriptor<std::string> core_rpc_server::arg_rpc_payment_address = { "rpc-payment-address" , "Restrict RPC to clients sending micropayment to this address" diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h index dcf6b4e4b..b21e43ab0 100644 --- a/src/rpc/core_rpc_server.h +++ b/src/rpc/core_rpc_server.h @@ -72,6 +72,7 @@ namespace cryptonote static const command_line::arg_descriptor<bool> arg_rpc_ssl_allow_any_cert; static const command_line::arg_descriptor<std::string> arg_bootstrap_daemon_address; static const command_line::arg_descriptor<std::string> arg_bootstrap_daemon_login; + static const command_line::arg_descriptor<std::string> arg_bootstrap_daemon_proxy; static const command_line::arg_descriptor<std::string> arg_rpc_payment_address; static const command_line::arg_descriptor<uint64_t> arg_rpc_payment_difficulty; static const command_line::arg_descriptor<uint64_t> arg_rpc_payment_credits; @@ -146,6 +147,7 @@ namespace cryptonote MAP_JON_RPC_WE("on_getblockhash", on_getblockhash, COMMAND_RPC_GETBLOCKHASH) MAP_JON_RPC_WE("get_block_template", on_getblocktemplate, COMMAND_RPC_GETBLOCKTEMPLATE) MAP_JON_RPC_WE("getblocktemplate", on_getblocktemplate, COMMAND_RPC_GETBLOCKTEMPLATE) + MAP_JON_RPC_WE("add_aux_pow", on_add_aux_pow, COMMAND_RPC_ADD_AUX_POW) MAP_JON_RPC_WE("submit_block", on_submitblock, COMMAND_RPC_SUBMITBLOCK) MAP_JON_RPC_WE("submitblock", on_submitblock, COMMAND_RPC_SUBMITBLOCK) MAP_JON_RPC_WE_IF("generateblocks", on_generateblocks, COMMAND_RPC_GENERATEBLOCKS, !m_restricted) @@ -226,6 +228,7 @@ namespace cryptonote bool on_getblockcount(const COMMAND_RPC_GETBLOCKCOUNT::request& req, COMMAND_RPC_GETBLOCKCOUNT::response& res, const connection_context *ctx = NULL); bool on_getblockhash(const COMMAND_RPC_GETBLOCKHASH::request& req, COMMAND_RPC_GETBLOCKHASH::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_getblocktemplate(const COMMAND_RPC_GETBLOCKTEMPLATE::request& req, COMMAND_RPC_GETBLOCKTEMPLATE::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); + bool on_add_aux_pow(const COMMAND_RPC_ADD_AUX_POW::request& req, COMMAND_RPC_ADD_AUX_POW::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_submitblock(const COMMAND_RPC_SUBMITBLOCK::request& req, COMMAND_RPC_SUBMITBLOCK::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_generateblocks(const COMMAND_RPC_GENERATEBLOCKS::request& req, COMMAND_RPC_GENERATEBLOCKS::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); bool on_get_last_block_header(const COMMAND_RPC_GET_LAST_BLOCK_HEADER::request& req, COMMAND_RPC_GET_LAST_BLOCK_HEADER::response& res, epee::json_rpc::error& error_resp, const connection_context *ctx = NULL); @@ -268,8 +271,14 @@ private: uint64_t get_block_reward(const block& blk); bool fill_block_header_response(const block& blk, bool orphan_status, uint64_t height, const crypto::hash& hash, block_header_response& response, bool fill_pow_hash); std::map<std::string, bool> get_public_nodes(uint32_t credits_per_hash_threshold = 0); - bool set_bootstrap_daemon(const std::string &address, const std::string &username_password); - bool set_bootstrap_daemon(const std::string &address, const boost::optional<epee::net_utils::http::login> &credentials); + bool set_bootstrap_daemon( + const std::string &address, + const std::string &username_password, + const std::string &proxy); + bool set_bootstrap_daemon( + const std::string &address, + const boost::optional<epee::net_utils::http::login> &credentials, + const std::string &proxy); enum invoke_http_mode { JON, BIN, JON_RPC }; template <typename COMMAND_TYPE> bool use_bootstrap_daemon_if_necessary(const invoke_http_mode &mode, const std::string &command_name, const typename COMMAND_TYPE::request& req, typename COMMAND_TYPE::response& res, bool &r); diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index bbcb27f1c..ff8c98b98 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -88,7 +88,7 @@ namespace cryptonote // advance which version they will stop working with // Don't go over 32767 for any of these #define CORE_RPC_VERSION_MAJOR 3 -#define CORE_RPC_VERSION_MINOR 5 +#define CORE_RPC_VERSION_MINOR 7 #define MAKE_CORE_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define CORE_RPC_VERSION MAKE_CORE_RPC_VERSION(CORE_RPC_VERSION_MAJOR, CORE_RPC_VERSION_MINOR) @@ -350,6 +350,7 @@ namespace cryptonote bool in_pool; bool double_spend_seen; uint64_t block_height; + uint64_t confirmations; uint64_t block_timestamp; uint64_t received_timestamp; std::vector<uint64_t> output_indices; @@ -367,6 +368,7 @@ namespace cryptonote if (!this_ref.in_pool) { KV_SERIALIZE(block_height) + KV_SERIALIZE(confirmations) KV_SERIALIZE(block_timestamp) KV_SERIALIZE(output_indices) } @@ -938,6 +940,52 @@ namespace cryptonote typedef epee::misc_utils::struct_init<response_t> response; }; + struct COMMAND_RPC_ADD_AUX_POW + { + struct aux_pow_t + { + std::string id; + std::string hash; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(id) + KV_SERIALIZE(hash) + END_KV_SERIALIZE_MAP() + }; + + struct request_t: public rpc_request_base + { + blobdata blocktemplate_blob; + std::vector<aux_pow_t> aux_pow; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_PARENT(rpc_request_base) + KV_SERIALIZE(blocktemplate_blob) + KV_SERIALIZE(aux_pow) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<request_t> request; + + struct response_t: public rpc_response_base + { + blobdata blocktemplate_blob; + blobdata blockhashing_blob; + std::string merkle_root; + uint32_t merkle_tree_depth; + std::vector<aux_pow_t> aux_pow; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE_PARENT(rpc_response_base) + KV_SERIALIZE(blocktemplate_blob) + KV_SERIALIZE(blockhashing_blob) + KV_SERIALIZE(merkle_root) + KV_SERIALIZE(merkle_tree_depth) + KV_SERIALIZE(aux_pow) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<response_t> response; + }; + struct COMMAND_RPC_SUBMITBLOCK { typedef std::vector<std::string> request; @@ -1613,11 +1661,13 @@ namespace cryptonote std::string address; std::string username; std::string password; + std::string proxy; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(address) KV_SERIALIZE(username) KV_SERIALIZE(password) + KV_SERIALIZE(proxy) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<request_t> request; diff --git a/src/rpc/core_rpc_server_error_codes.h b/src/rpc/core_rpc_server_error_codes.h index 1458049ab..232df0373 100644 --- a/src/rpc/core_rpc_server_error_codes.h +++ b/src/rpc/core_rpc_server_error_codes.h @@ -49,6 +49,8 @@ #define CORE_RPC_ERROR_CODE_DUPLICATE_PAYMENT -17 #define CORE_RPC_ERROR_CODE_STALE_PAYMENT -18 #define CORE_RPC_ERROR_CODE_RESTRICTED -19 +#define CORE_RPC_ERROR_CODE_UNSUPPORTED_BOOTSTRAP -20 +#define CORE_RPC_ERROR_CODE_PAYMENTS_NOT_ENABLED -21 static inline const char *get_rpc_server_error_message(int64_t code) { @@ -72,6 +74,8 @@ static inline const char *get_rpc_server_error_message(int64_t code) case CORE_RPC_ERROR_CODE_DUPLICATE_PAYMENT: return "Duplicate payment"; case CORE_RPC_ERROR_CODE_STALE_PAYMENT: return "Stale payment"; case CORE_RPC_ERROR_CODE_RESTRICTED: return "Parameters beyond restricted allowance"; + case CORE_RPC_ERROR_CODE_UNSUPPORTED_BOOTSTRAP: return "Command is unsupported in bootstrap mode"; + case CORE_RPC_ERROR_CODE_PAYMENTS_NOT_ENABLED: return "Payments not enabled"; default: MERROR("Unknown error: " << code); return "Unknown error"; } } diff --git a/src/rpc/rpc_payment.cpp b/src/rpc/rpc_payment.cpp index 176f11fa3..f0c513911 100644 --- a/src/rpc/rpc_payment.cpp +++ b/src/rpc/rpc_payment.cpp @@ -27,6 +27,7 @@ // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include <boost/archive/portable_binary_iarchive.hpp> +#include <boost/filesystem.hpp> #include "cryptonote_config.h" #include "include_base_utils.h" #include "string_tools.h" @@ -292,12 +293,13 @@ namespace cryptonote MINFO("loading rpc payments data from " << state_file_path); std::ifstream data; data.open(state_file_path, std::ios_base::binary | std::ios_base::in); + std::string bytes(std::istream_iterator<char>{data}, std::istream_iterator<char>{}); if (!data.fail()) { bool loaded = false; try { - binary_archive<false> ar(data); + binary_archive<false> ar{epee::strspan<std::uint8_t>(bytes)}; if (::serialization::serialize(ar, *this)) if (::serialization::check_stream_state(ar)) loaded = true; @@ -305,6 +307,8 @@ namespace cryptonote catch (...) {} if (!loaded) { + bytes.clear(); + bytes.shrink_to_fit(); try { boost::archive::portable_binary_iarchive a(data); diff --git a/src/serialization/binary_archive.h b/src/serialization/binary_archive.h index 80104c8f7..acda70039 100644 --- a/src/serialization/binary_archive.h +++ b/src/serialization/binary_archive.h @@ -36,9 +36,11 @@ #include <cassert> #include <iostream> #include <iterator> +#include <boost/endian/conversion.hpp> #include <boost/type_traits/make_unsigned.hpp> #include "common/varint.h" +#include "span.h" #include "warnings.h" /* I have no clue what these lines means */ @@ -55,16 +57,15 @@ DISABLE_VS_WARNINGS(4244) * purpse is to define the functions used for the binary_archive. Its * a header, basically. I think it was declared simply to save typing... */ -template <class Stream, bool IsSaving> +template <bool IsSaving> struct binary_archive_base { - typedef Stream stream_type; - typedef binary_archive_base<Stream, IsSaving> base_type; + typedef binary_archive_base<IsSaving> base_type; typedef boost::mpl::bool_<IsSaving> is_saving; typedef uint8_t variant_tag_type; - explicit binary_archive_base(stream_type &s) : stream_(s) { } + explicit binary_archive_base() { } /* definition of standard API functions */ void tag(const char *) { } @@ -72,12 +73,6 @@ struct binary_archive_base void end_object() { } void begin_variant() { } void end_variant() { } - /* I just want to leave a comment saying how this line really shows - flaws in the ownership model of many OOP languages, that is all. */ - stream_type &stream() { return stream_; } - -protected: - stream_type &stream_; }; /* \struct binary_archive @@ -95,15 +90,18 @@ struct binary_archive; template <> -struct binary_archive<false> : public binary_archive_base<std::istream, false> +struct binary_archive<false> : public binary_archive_base<false> { + explicit binary_archive(epee::span<const std::uint8_t> s) + : base_type(), bytes_(s), begin_(s.begin()), good_(true), varint_bug_backward_compatibility_(false) + {} - explicit binary_archive(stream_type &s) : base_type(s) { - stream_type::pos_type pos = stream_.tellg(); - stream_.seekg(0, std::ios_base::end); - eof_pos_ = stream_.tellg(); - stream_.seekg(pos); - } + bool good() const noexcept { return good_; } + void set_fail() noexcept { good_ = false; } + + //! If implementing as `std::istream`, reset stream error state after `peek()` call. + bool eof() const noexcept { return bytes_.empty(); } + std::size_t getpos() const noexcept { return bytes_.begin() - begin_; } template <class T> void serialize_int(T &v) @@ -116,24 +114,24 @@ struct binary_archive<false> : public binary_archive_base<std::istream, false> * \brief serializes an unsigned integer */ template <class T> - void serialize_uint(T &v, size_t width = sizeof(T)) + void serialize_uint(T &v) { - T ret = 0; - unsigned shift = 0; - for (size_t i = 0; i < width; i++) { - //std::cerr << "tell: " << stream_.tellg() << " value: " << ret << std::endl; - char c; - stream_.get(c); - T b = (unsigned char)c; - ret += (b << shift); // can this be changed to OR, i think it can. - shift += 8; + const std::size_t actual = bytes_.remove_prefix(sizeof(T)); + good_ &= (actual == sizeof(T)); + if (actual == sizeof(T)) + { + std::memcpy(std::addressof(v), bytes_.data() - sizeof(T), sizeof(T)); + boost::endian::little_to_native_inplace(v); // epee isn't templated } - v = ret; + else + v = 0; // ensures initialization } void serialize_blob(void *buf, size_t len, const char *delimiter="") { - stream_.read((char *)buf, len); + const std::size_t actual = bytes_.remove_prefix(len); + good_ &= (len == actual); + std::memcpy(buf, bytes_.data() - actual, actual); } template <class T> @@ -145,9 +143,11 @@ struct binary_archive<false> : public binary_archive_base<std::istream, false> template <class T> void serialize_uvarint(T &v) { - typedef std::istreambuf_iterator<char> it; - if (tools::read_varint(it(stream_), it(), v) < 0) - stream_.setstate(std::ios_base::failbit); + auto current = bytes_.cbegin(); + auto end = bytes_.cend(); + good_ &= (0 <= tools::read_varint(current, end, v)); + current = std::min(current, bytes_.cend()); + bytes_ = {current, std::size_t(bytes_.cend() - current)}; } void begin_array(size_t &s) @@ -166,21 +166,26 @@ struct binary_archive<false> : public binary_archive_base<std::istream, false> serialize_int(t); } - size_t remaining_bytes() { - if (!stream_.good()) - return 0; - //std::cerr << "tell: " << stream_.tellg() << std::endl; - assert(stream_.tellg() <= eof_pos_); - return eof_pos_ - stream_.tellg(); - } + size_t remaining_bytes() const noexcept { return good() ? bytes_.size() : 0; } + void enable_varint_bug_backward_compatibility() { varint_bug_backward_compatibility_ = true; } + bool varint_bug_backward_compatibility_enabled() const { return varint_bug_backward_compatibility_; } protected: - std::streamoff eof_pos_; + epee::span<const std::uint8_t> bytes_; + std::uint8_t const* const begin_; + bool good_; + bool varint_bug_backward_compatibility_; }; template <> -struct binary_archive<true> : public binary_archive_base<std::ostream, true> +struct binary_archive<true> : public binary_archive_base<true> { - explicit binary_archive(stream_type &s) : base_type(s) { } + typedef std::ostream stream_type; + explicit binary_archive(stream_type &s) : base_type(), stream_(s) { } + + bool good() const { return stream_.good(); } + void set_fail() { stream_.setstate(std::ios::failbit); } + + std::streampos getpos() const { return stream_.tellp(); } template <class T> void serialize_int(T v) @@ -227,6 +232,10 @@ struct binary_archive<true> : public binary_archive_base<std::ostream, true> void write_variant_tag(variant_tag_type t) { serialize_int(t); } + + bool varint_bug_backward_compatibility_enabled() const { return false; } +protected: + stream_type& stream_; }; POP_WARNINGS diff --git a/src/serialization/binary_utils.h b/src/serialization/binary_utils.h index 8635585af..8444f220e 100644 --- a/src/serialization/binary_utils.h +++ b/src/serialization/binary_utils.h @@ -39,8 +39,7 @@ namespace serialization { template <class T> bool parse_binary(const std::string &blob, T &v) { - std::istringstream istr(blob); - binary_archive<false> iar(istr); + binary_archive<false> iar{epee::strspan<std::uint8_t>(blob)}; return ::serialization::serialize(iar, v); } diff --git a/src/serialization/container.h b/src/serialization/container.h index d5e75bb4f..77681ec93 100644 --- a/src/serialization/container.h +++ b/src/serialization/container.h @@ -32,22 +32,27 @@ namespace serialization { namespace detail { - template <typename Archive, class T> - bool serialize_container_element(Archive& ar, T& e) + template<typename T> + inline constexpr bool use_container_varint() noexcept { - return ::do_serialize(ar, e); + return std::is_integral<T>::value && std::is_unsigned<T>::value && sizeof(T) > 1; } - template <typename Archive> - bool serialize_container_element(Archive& ar, uint32_t& e) + template <typename Archive, class T> + typename std::enable_if<!use_container_varint<T>(), bool>::type + serialize_container_element(Archive& ar, T& e) { - ar.serialize_varint(e); - return true; + return ::do_serialize(ar, e); } - template <typename Archive> - bool serialize_container_element(Archive& ar, uint64_t& e) + template<typename Archive, typename T> + typename std::enable_if<use_container_varint<T>(), bool>::type + serialize_container_element(Archive& ar, T& e) { + static constexpr const bool previously_varint = std::is_same<uint64_t, T>() || std::is_same<uint32_t, T>(); + + if (!previously_varint && ar.varint_bug_backward_compatibility_enabled() && !typename Archive::is_saving()) + return ::do_serialize(ar, e); ar.serialize_varint(e); return true; } @@ -62,13 +67,13 @@ bool do_serialize_container(Archive<false> &ar, C &v) { size_t cnt; ar.begin_array(cnt); - if (!ar.stream().good()) + if (!ar.good()) return false; v.clear(); // very basic sanity check if (ar.remaining_bytes() < cnt) { - ar.stream().setstate(std::ios::failbit); + ar.set_fail(); return false; } @@ -81,7 +86,7 @@ bool do_serialize_container(Archive<false> &ar, C &v) if (!::serialization::detail::serialize_container_element(ar, e)) return false; ::serialization::detail::do_add(v, std::move(e)); - if (!ar.stream().good()) + if (!ar.good()) return false; } ar.end_array(); @@ -95,13 +100,13 @@ bool do_serialize_container(Archive<true> &ar, C &v) ar.begin_array(cnt); for (auto i = v.begin(); i != v.end(); ++i) { - if (!ar.stream().good()) + if (!ar.good()) return false; if (i != v.begin()) ar.delimit_array(); if(!::serialization::detail::serialize_container_element(ar, (typename C::value_type&)*i)) return false; - if (!ar.stream().good()) + if (!ar.good()) return false; } ar.end_array(); diff --git a/src/serialization/crypto.h b/src/serialization/crypto.h index 42bd06e92..d635a9ee9 100644 --- a/src/serialization/crypto.h +++ b/src/serialization/crypto.h @@ -47,7 +47,7 @@ bool do_serialize(Archive<false> &ar, std::vector<crypto::signature> &v) // very basic sanity check if (ar.remaining_bytes() < cnt*sizeof(crypto::signature)) { - ar.stream().setstate(std::ios::failbit); + ar.set_fail(); return false; } @@ -55,7 +55,7 @@ bool do_serialize(Archive<false> &ar, std::vector<crypto::signature> &v) for (size_t i = 0; i < cnt; i++) { v.resize(i+1); ar.serialize_blob(&(v[i]), sizeof(crypto::signature), ""); - if (!ar.stream().good()) + if (!ar.good()) return false; } return true; @@ -70,7 +70,7 @@ bool do_serialize(Archive<true> &ar, std::vector<crypto::signature> &v) size_t cnt = v.size(); for (size_t i = 0; i < cnt; i++) { ar.serialize_blob(&(v[i]), sizeof(crypto::signature), ""); - if (!ar.stream().good()) + if (!ar.good()) return false; } ar.end_string(); diff --git a/src/serialization/debug_archive.h b/src/serialization/debug_archive.h index b04d6ae19..e8ccb9a58 100644 --- a/src/serialization/debug_archive.h +++ b/src/serialization/debug_archive.h @@ -38,6 +38,7 @@ struct debug_archive : public json_archive<W> { typedef typename json_archive<W>::stream_type stream_type; debug_archive(stream_type &s) : json_archive<W>(s) { } + stream_type& stream() { return this->stream_; } }; template <class T> diff --git a/src/serialization/difficulty_type.h b/src/serialization/difficulty_type.h index 56c0312e7..75d1fd13f 100644 --- a/src/serialization/difficulty_type.h +++ b/src/serialization/difficulty_type.h @@ -38,10 +38,10 @@ inline bool do_serialize(Archive<false>& ar, cryptonote::difficulty_type &diff) { uint64_t hi, lo; ar.serialize_varint(hi); - if (!ar.stream().good()) + if (!ar.good()) return false; ar.serialize_varint(lo); - if (!ar.stream().good()) + if (!ar.good()) return false; diff = hi; diff <<= 64; @@ -52,13 +52,13 @@ inline bool do_serialize(Archive<false>& ar, cryptonote::difficulty_type &diff) template <template <bool> class Archive> inline bool do_serialize(Archive<true>& ar, cryptonote::difficulty_type &diff) { - if (!ar.stream().good()) + if (!ar.good()) return false; const uint64_t hi = ((diff >> 64) & 0xffffffffffffffff).convert_to<uint64_t>(); const uint64_t lo = (diff & 0xffffffffffffffff).convert_to<uint64_t>(); ar.serialize_varint(hi); ar.serialize_varint(lo); - if (!ar.stream().good()) + if (!ar.good()) return false; return true; } diff --git a/src/serialization/json_archive.h b/src/serialization/json_archive.h index 50dd5bbd0..bab6dfa94 100644 --- a/src/serialization/json_archive.h +++ b/src/serialization/json_archive.h @@ -58,6 +58,10 @@ struct json_archive_base json_archive_base(stream_type &s, bool indent = false) : stream_(s), indent_(indent), object_begin(false), depth_(0) { } + bool good() const { return stream_.good(); } + void set_fail() { stream_.setstate(std::ios::failbit); } + void clear_fail() { stream_.clear(); } + void tag(const char *tag) { if (!object_begin) stream_ << ", "; @@ -82,7 +86,8 @@ struct json_archive_base void begin_variant() { begin_object(); } void end_variant() { end_object(); } - Stream &stream() { return stream_; } + + bool varint_bug_backward_compatibility_enabled() const { return false; } protected: void make_indent() @@ -115,6 +120,8 @@ struct json_archive<true> : public json_archive_base<std::ostream, true> { json_archive(stream_type &s, bool indent = false) : base_type(s, indent), inner_array_size_(0) { } + std::streampos getpos() const { return stream_.tellp(); } + template<typename T> static auto promote_to_printable_integer_type(T v) -> decltype(+v) { diff --git a/src/serialization/json_object.cpp b/src/serialization/json_object.cpp index 67f042c2e..28e207ff2 100644 --- a/src/serialization/json_object.cpp +++ b/src/serialization/json_object.cpp @@ -1091,9 +1091,12 @@ void toJsonValue(rapidjson::Writer<epee::byte_stream>& dest, const rct::rctSig& }; INSERT_INTO_JSON_OBJECT(dest, type, sig.type); - INSERT_INTO_JSON_OBJECT(dest, encrypted, sig.ecdhInfo); - INSERT_INTO_JSON_OBJECT(dest, commitments, transform(sig.outPk, just_mask)); - INSERT_INTO_JSON_OBJECT(dest, fee, sig.txnFee); + if (sig.type != rct::RCTTypeNull) + { + INSERT_INTO_JSON_OBJECT(dest, encrypted, sig.ecdhInfo); + INSERT_INTO_JSON_OBJECT(dest, commitments, transform(sig.outPk, just_mask)); + INSERT_INTO_JSON_OBJECT(dest, fee, sig.txnFee); + } // prunable if (!sig.p.bulletproofs.empty() || !sig.p.rangeSigs.empty() || !sig.p.MGs.empty() || !sig.get_pseudo_outs().empty()) @@ -1122,9 +1125,12 @@ void fromJsonValue(const rapidjson::Value& val, rct::rctSig& sig) } GET_FROM_JSON_OBJECT(val, sig.type, type); - GET_FROM_JSON_OBJECT(val, sig.ecdhInfo, encrypted); - GET_FROM_JSON_OBJECT(val, sig.outPk, commitments); - GET_FROM_JSON_OBJECT(val, sig.txnFee, fee); + if (sig.type != rct::RCTTypeNull) + { + GET_FROM_JSON_OBJECT(val, sig.ecdhInfo, encrypted); + GET_FROM_JSON_OBJECT(val, sig.outPk, commitments); + GET_FROM_JSON_OBJECT(val, sig.txnFee, fee); + } // prunable const auto prunable = val.FindMember("prunable"); diff --git a/src/serialization/json_object.h b/src/serialization/json_object.h index de14c8911..35ea990b3 100644 --- a/src/serialization/json_object.h +++ b/src/serialization/json_object.h @@ -365,7 +365,7 @@ inline typename std::enable_if<sfinae::is_vector_like<Vec>::value, void>::type t static_assert(!std::is_same<value_type, unsigned char>::value, "encoding an array of unsigned char is faster as hex"); dest.StartArray(); - for (const auto& t : vec) + for (auto t : vec) toJsonValue(dest, t); dest.EndArray(); } diff --git a/src/serialization/pair.h b/src/serialization/pair.h index 18280d837..2d9a89242 100644 --- a/src/serialization/pair.h +++ b/src/serialization/pair.h @@ -30,21 +30,34 @@ #pragma once #include <memory> +#include <boost/type_traits/make_unsigned.hpp> #include "serialization.h" namespace serialization { namespace detail { + template<typename T> + inline constexpr bool use_pair_varint() noexcept + { + return std::is_integral<T>::value && std::is_unsigned<T>::value && sizeof(T) > 1; + } + template <typename Archive, class T> - bool serialize_pair_element(Archive& ar, T& e) + typename std::enable_if<!use_pair_varint<T>(), bool>::type + serialize_pair_element(Archive& ar, T& e) { return ::do_serialize(ar, e); } - template <typename Archive> - bool serialize_pair_element(Archive& ar, uint64_t& e) + template<typename Archive, typename T> + typename std::enable_if<use_pair_varint<T>(), bool>::type + serialize_pair_element(Archive& ar, T& e) { + static constexpr const bool previously_varint = std::is_same<uint64_t, T>(); + + if (!previously_varint && ar.varint_bug_backward_compatibility_enabled() && !typename Archive::is_saving()) + return ::do_serialize(ar, e); ar.serialize_varint(e); return true; } @@ -56,19 +69,19 @@ inline bool do_serialize(Archive<false>& ar, std::pair<F,S>& p) { size_t cnt; ar.begin_array(cnt); - if (!ar.stream().good()) + if (!ar.good()) return false; if (cnt != 2) return false; if (!::serialization::detail::serialize_pair_element(ar, p.first)) return false; - if (!ar.stream().good()) + if (!ar.good()) return false; ar.delimit_array(); if (!::serialization::detail::serialize_pair_element(ar, p.second)) return false; - if (!ar.stream().good()) + if (!ar.good()) return false; ar.end_array(); @@ -79,16 +92,16 @@ template <template <bool> class Archive, class F, class S> inline bool do_serialize(Archive<true>& ar, std::pair<F,S>& p) { ar.begin_array(2); - if (!ar.stream().good()) + if (!ar.good()) return false; if(!::serialization::detail::serialize_pair_element(ar, p.first)) return false; - if (!ar.stream().good()) + if (!ar.good()) return false; ar.delimit_array(); if(!::serialization::detail::serialize_pair_element(ar, p.second)) return false; - if (!ar.stream().good()) + if (!ar.good()) return false; ar.end_array(); return true; diff --git a/src/serialization/serialization.h b/src/serialization/serialization.h index 3ebeed171..4acadeab3 100644 --- a/src/serialization/serialization.h +++ b/src/serialization/serialization.h @@ -213,7 +213,7 @@ inline bool do_serialize(Archive &ar, bool &v) * \brief self-explanatory */ #define END_SERIALIZE() \ - return ar.stream().good(); \ + return ar.good(); \ } /*! \macro VALUE(f) @@ -223,7 +223,7 @@ inline bool do_serialize(Archive &ar, bool &v) do { \ ar.tag(#f); \ bool r = ::do_serialize(ar, f); \ - if (!r || !ar.stream().good()) return false; \ + if (!r || !ar.good()) return false; \ } while(0); /*! \macro FIELD_N(t,f) @@ -234,7 +234,7 @@ inline bool do_serialize(Archive &ar, bool &v) do { \ ar.tag(t); \ bool r = ::do_serialize(ar, f); \ - if (!r || !ar.stream().good()) return false; \ + if (!r || !ar.good()) return false; \ } while(0); /*! \macro FIELD(f) @@ -245,7 +245,7 @@ inline bool do_serialize(Archive &ar, bool &v) do { \ ar.tag(#f); \ bool r = ::do_serialize(ar, f); \ - if (!r || !ar.stream().good()) return false; \ + if (!r || !ar.good()) return false; \ } while(0); /*! \macro FIELDS(f) @@ -255,7 +255,7 @@ inline bool do_serialize(Archive &ar, bool &v) #define FIELDS(f) \ do { \ bool r = ::do_serialize(ar, f); \ - if (!r || !ar.stream().good()) return false; \ + if (!r || !ar.good()) return false; \ } while(0); /*! \macro VARINT_FIELD(f) @@ -265,7 +265,7 @@ inline bool do_serialize(Archive &ar, bool &v) do { \ ar.tag(#f); \ ar.serialize_varint(f); \ - if (!ar.stream().good()) return false; \ + if (!ar.good()) return false; \ } while(0); /*! \macro VARINT_FIELD_N(t, f) @@ -276,7 +276,7 @@ inline bool do_serialize(Archive &ar, bool &v) do { \ ar.tag(t); \ ar.serialize_varint(f); \ - if (!ar.stream().good()) return false; \ + if (!ar.good()) return false; \ } while(0); /*! \macro MAGIC_FIELD(m) @@ -286,7 +286,7 @@ inline bool do_serialize(Archive &ar, bool &v) do { \ ar.tag("magic"); \ ar.serialize_blob((void*)magic.data(), magic.size()); \ - if (!ar.stream().good()) return false; \ + if (!ar.good()) return false; \ if (magic != m) return false; \ } while(0); @@ -297,7 +297,7 @@ inline bool do_serialize(Archive &ar, bool &v) do { \ ar.tag("version"); \ ar.serialize_varint(version); \ - if (!ar.stream().good()) return false; \ + if (!ar.good()) return false; \ } while(0); @@ -339,10 +339,10 @@ namespace serialization { * * \brief self explanatory */ - template<class Stream> - bool do_check_stream_state(Stream& s, boost::mpl::bool_<true>, bool noeof) + template<class Archive> + bool do_check_stream_state(Archive& ar, boost::mpl::bool_<true>, bool noeof) { - return s.good(); + return ar.good(); } /*! \fn do_check_stream_state * @@ -350,15 +350,13 @@ namespace serialization { * * \detailed Also checks to make sure that the stream is not at EOF */ - template<class Stream> - bool do_check_stream_state(Stream& s, boost::mpl::bool_<false>, bool noeof) + template<class Archive> + bool do_check_stream_state(Archive& ar, boost::mpl::bool_<false>, bool noeof) { bool result = false; - if (s.good()) + if (ar.good()) { - std::ios_base::iostate state = s.rdstate(); - result = noeof || EOF == s.peek(); - s.clear(state); + result = noeof || ar.eof(); } return result; } @@ -371,7 +369,7 @@ namespace serialization { template<class Archive> bool check_stream_state(Archive& ar, bool noeof = false) { - return detail::do_check_stream_state(ar.stream(), typename Archive::is_saving(), noeof); + return detail::do_check_stream_state(ar, typename Archive::is_saving(), noeof); } /*! \fn serialize diff --git a/src/serialization/string.h b/src/serialization/string.h index f5f69833a..f1f8f4ab0 100644 --- a/src/serialization/string.h +++ b/src/serialization/string.h @@ -39,7 +39,7 @@ inline bool do_serialize(Archive<false>& ar, std::string& str) ar.serialize_varint(size); if (ar.remaining_bytes() < size) { - ar.stream().setstate(std::ios::failbit); + ar.set_fail(); return false; } diff --git a/src/serialization/variant.h b/src/serialization/variant.h index b08cf6bf0..6debb63d1 100644 --- a/src/serialization/variant.h +++ b/src/serialization/variant.h @@ -74,7 +74,7 @@ struct variant_reader current_type x; if(!::do_serialize(ar, x)) { - ar.stream().setstate(std::ios::failbit); + ar.set_fail(); return false; } v = x; @@ -95,7 +95,7 @@ struct variant_reader<Archive, Variant, TBegin, TBegin> static inline bool read(Archive &ar, Variant &v, variant_tag_type t) { - ar.stream().setstate(std::ios::failbit); + ar.set_fail(); return false; } }; @@ -116,7 +116,7 @@ struct serializer<Archive<false>, boost::variant<BOOST_VARIANT_ENUM_PARAMS(T)>> typename boost::mpl::begin<types>::type, typename boost::mpl::end<types>::type>::read(ar, v, t)) { - ar.stream().setstate(std::ios::failbit); + ar.set_fail(); return false; } ar.end_variant(); @@ -143,7 +143,7 @@ struct serializer<Archive<true>, boost::variant<BOOST_VARIANT_ENUM_PARAMS(T)>> ar.write_variant_tag(variant_serialization_traits<Archive<true>, T>::get_tag()); if(!::do_serialize(ar, rv)) { - ar.stream().setstate(std::ios::failbit); + ar.set_fail(); return false; } ar.end_variant(); diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index dfd6adf3a..dc031b36c 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -50,6 +50,7 @@ #include <boost/format.hpp> #include <boost/regex.hpp> #include <boost/range/adaptor/transformed.hpp> +#include <boost/filesystem.hpp> #include "include_base_utils.h" #include "console_handler.h" #include "common/i18n.h" @@ -195,7 +196,7 @@ namespace const char* USAGE_SWEEP_BELOW("sweep_below <amount_threshold> [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <address> [<payment_id (obsolete)>]"); const char* USAGE_SWEEP_SINGLE("sweep_single [<priority>] [<ring_size>] [outputs=<N>] <key_image> <address> [<payment_id (obsolete)>]"); const char* USAGE_DONATE("donate [index=<N1>[,<N2>,...]] [<priority>] [<ring_size>] <amount> [<payment_id (obsolete)>]"); - const char* USAGE_SIGN_TRANSFER("sign_transfer [export_raw]"); + const char* USAGE_SIGN_TRANSFER("sign_transfer [export_raw] [<filename>]"); const char* USAGE_SET_LOG("set_log <level>|{+,-,}<categories>"); const char* USAGE_ACCOUNT("account\n" " account new <label text with white spaces allowed>\n" @@ -276,12 +277,13 @@ namespace const char* USAGE_PUBLIC_NODES("public_nodes"); const char* USAGE_WELCOME("welcome"); const char* USAGE_RPC_PAYMENT_INFO("rpc_payment_info"); - const char* USAGE_START_MINING_FOR_RPC("start_mining_for_rpc"); + const char* USAGE_START_MINING_FOR_RPC("start_mining_for_rpc [<number_of_threads>]"); const char* USAGE_STOP_MINING_FOR_RPC("stop_mining_for_rpc"); const char* USAGE_SHOW_QR_CODE("show_qr_code [<subaddress_index>]"); const char* USAGE_VERSION("version"); const char* USAGE_HELP("help [<command> | all]"); const char* USAGE_APROPOS("apropos <keyword> [<keyword> ...]"); + const char* USAGE_SCAN_TX("scan_tx <txid> [<txid> ...]"); std::string input_line(const std::string& prompt, bool yesno = false) { @@ -606,9 +608,14 @@ void simple_wallet::handle_transfer_exception(const std::exception_ptr &e, bool fail_msg_writer() << e.what(); warn_of_possible_attack = false; } + catch (const tools::error::zero_amount&) + { + fail_msg_writer() << sw::tr("destination amount is zero"); + warn_of_possible_attack = false; + } catch (const tools::error::zero_destination&) { - fail_msg_writer() << sw::tr("one of destinations is zero"); + fail_msg_writer() << sw::tr("transaction has no destination"); warn_of_possible_attack = false; } catch (const tools::error::tx_too_big& e) @@ -1360,7 +1367,7 @@ bool simple_wallet::import_multisig_main(const std::vector<std::string> &args, b size_t n_outputs = m_wallet->import_multisig(info); // Clear line "Height xxx of xxx" std::cout << "\r \r"; - success_msg_writer() << tr("Multisig info imported"); + success_msg_writer() << tr("Multisig info imported. Number of outputs updated: ") << n_outputs; } catch (const std::exception &e) { @@ -2360,6 +2367,24 @@ bool simple_wallet::start_mining_for_rpc(const std::vector<std::string> &args) if (!try_connect_to_daemon()) return true; + bool ok = true; + if(args.size() >= 1) + { + uint16_t num = 0; + ok = string_tools::get_xtype_from_string(num, args[0]); + m_rpc_payment_threads = num; + } + else + { + m_rpc_payment_threads = 0; + } + + if (!ok) + { + PRINT_USAGE(USAGE_START_MINING_FOR_RPC); + return true; + } + LOCK_IDLE_SCOPE(); bool payment_required; @@ -3214,6 +3239,45 @@ bool simple_wallet::apropos(const std::vector<std::string> &args) return true; } +bool simple_wallet::scan_tx(const std::vector<std::string> &args) +{ + if (args.empty()) + { + PRINT_USAGE(USAGE_SCAN_TX); + return true; + } + + // Parse and dedup args + std::unordered_set<crypto::hash> txids; + for (const auto &s : args) { + crypto::hash txid; + if (!epee::string_tools::hex_to_pod(s, txid)) { + fail_msg_writer() << tr("Invalid txid specified: ") << s; + return true; + } + txids.insert(txid); + } + std::vector<crypto::hash> txids_v(txids.begin(), txids.end()); + + if (!m_wallet->is_trusted_daemon()) { + message_writer(console_color_red, true) << tr("WARNING: this operation may reveal the txids to the remote node and affect your privacy"); + if (!command_line::is_yes(input_line("Do you want to continue?", true))) { + message_writer() << tr("You have canceled the operation"); + return true; + } + } + + LOCK_IDLE_SCOPE(); + m_in_manual_refresh.store(true); + try { + m_wallet->scan_tx(txids_v); + } catch (const std::exception &e) { + fail_msg_writer() << e.what(); + } + m_in_manual_refresh.store(false); + return true; +} + simple_wallet::simple_wallet() : m_allow_mismatched_daemon_version(false) , m_refresh_progress_reporter(*this) @@ -3301,7 +3365,8 @@ simple_wallet::simple_wallet() m_cmd_binder.set_handler("sign_transfer", boost::bind(&simple_wallet::on_command, this, &simple_wallet::sign_transfer, _1), tr(USAGE_SIGN_TRANSFER), - tr("Sign a transaction from a file. If the parameter \"export_raw\" is specified, transaction raw hex data suitable for the daemon RPC /sendrawtransaction is exported.")); + tr("Sign a transaction from a file. If the parameter \"export_raw\" is specified, transaction raw hex data suitable for the daemon RPC /sendrawtransaction is exported.\n" + "Use the parameter <filename> to specify the file to read from. If not specified, the default \"unsigned_monero_tx\" will be used.")); m_cmd_binder.set_handler("submit_transfer", boost::bind(&simple_wallet::on_command, this, &simple_wallet::submit_transfer, _1), tr("Submit a signed transaction from a file.")); @@ -3763,6 +3828,10 @@ simple_wallet::simple_wallet() boost::bind(&simple_wallet::on_command, this, &simple_wallet::apropos, _1), tr(USAGE_APROPOS), tr("Search all command descriptions for keyword(s)")); + m_cmd_binder.set_handler("scan_tx", + boost::bind(&simple_wallet::on_command, this, &simple_wallet::scan_tx, _1), + tr(USAGE_SCAN_TX), + tr("Scan the transactions given by <txid>(s), processing them and looking for outputs")); m_cmd_binder.set_unknown_command_handler(boost::bind(&simple_wallet::on_command, this, &simple_wallet::on_unknown_command, _1)); m_cmd_binder.set_empty_command_handler(boost::bind(&simple_wallet::on_empty_command, this)); m_cmd_binder.set_cancel_handler(boost::bind(&simple_wallet::on_cancelled_command, this)); @@ -4519,7 +4588,7 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm) password = *r; welcome = true; // if no block_height is specified, assume its a new account and start it "now" - if(m_wallet->get_refresh_from_block_height() == 0) { + if (command_line::is_arg_defaulted(vm, arg_restore_height)) { { tools::scoped_message_writer wrt = tools::msg_writer(); wrt << tr("No restore height is specified.") << " "; @@ -5826,7 +5895,7 @@ bool simple_wallet::refresh_main(uint64_t start_height, enum ResetType reset, bo if (reset != ResetNone) { if (reset == ResetSoftKeepKI) - height_pre = m_wallet->hash_m_transfers(-1, transfer_hash_pre); + height_pre = m_wallet->hash_m_transfers(boost::none, transfer_hash_pre); m_wallet->rescan_blockchain(reset == ResetHard, false, reset == ResetSoftKeepKI); } @@ -5937,7 +6006,7 @@ bool simple_wallet::show_balance_unlocked(bool detailed) if (m_wallet->has_multisig_partial_key_images()) extra = tr(" (Some owned outputs have partial key images - import_multisig_info needed)"); else if (m_wallet->has_unknown_key_images()) - extra += tr(" (Some owned outputs have missing key images - import_key_images needed)"); + extra += tr(" (Some owned outputs have missing key images - export_outputs, import_outputs, export_key_images, and import_key_images needed)"); success_msg_writer() << tr("Currently selected account: [") << m_current_subaddress_account << tr("] ") << m_wallet->get_subaddress_label({m_current_subaddress_account, 0}); const std::string tag = m_wallet->get_account_tags().second[m_current_subaddress_account]; success_msg_writer() << tr("Tag: ") << (tag.empty() ? std::string{tr("(No tag assigned)")} : tag); @@ -6468,8 +6537,6 @@ void simple_wallet::check_for_inactivity_lock(bool user) //---------------------------------------------------------------------------------------------------- bool simple_wallet::on_command(bool (simple_wallet::*cmd)(const std::vector<std::string>&), const std::vector<std::string> &args) { - const time_t now = time(NULL); - time_t dt = now - m_last_activity_time; m_last_activity_time = time(NULL); m_in_command = true; @@ -7435,7 +7502,6 @@ bool simple_wallet::sweep_single(const std::vector<std::string> &args_) if (local_args.size() == 3) { crypto::hash payment_id; - crypto::hash8 payment_id8; std::string extra_nonce; if (tools::wallet2::parse_long_payment_id(local_args.back(), payment_id)) { @@ -7891,19 +7957,33 @@ bool simple_wallet::sign_transfer(const std::vector<std::string> &args_) fail_msg_writer() << tr("This is a watch only wallet"); return true; } - if (args_.size() > 1 || (args_.size() == 1 && args_[0] != "export_raw")) + + bool export_raw = false; + std::string unsigned_filename = "unsigned_monero_tx"; + if (args_.size() > 2 || (args_.size() == 2 && args_[0] != "export_raw")) { PRINT_USAGE(USAGE_SIGN_TRANSFER); return true; } + else if (args_.size() == 2) + { + export_raw = true; + unsigned_filename = args_[1]; + } + else if (args_.size() == 1) + { + if (args_[0] == "export_raw") + export_raw = true; + else + unsigned_filename = args_[0]; + } SCOPED_WALLET_UNLOCK(); - const bool export_raw = args_.size() == 1; std::vector<tools::wallet2::pending_tx> ptx; try { - bool r = m_wallet->sign_tx("unsigned_monero_tx", "signed_monero_tx", ptx, [&](const tools::wallet2::unsigned_tx_set &tx){ return accept_loaded_tx(tx); }, export_raw); + bool r = m_wallet->sign_tx(unsigned_filename, "signed_monero_tx", ptx, [&](const tools::wallet2::unsigned_tx_set &tx){ return accept_loaded_tx(tx); }, export_raw); if (!r) { fail_msg_writer() << tr("Failed to sign transaction"); @@ -8601,7 +8681,6 @@ bool simple_wallet::get_transfers(std::vector<std::string>& local_args, std::vec if (!unlocked) { locked_msg = "locked"; - const uint64_t unlock_time = pd.m_unlock_time; if (pd.m_unlock_time < CRYPTONOTE_MAX_BLOCK_NUMBER) { uint64_t bh = std::max(pd.m_unlock_time, pd.m_block_height + CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE); @@ -8843,12 +8922,12 @@ bool simple_wallet::export_transfers(const std::vector<std::string>& args_) // header file << - boost::format("%8.8s,%9.9s,%8.8s,%25.25s,%20.20s,%20.20s,%64.64s,%16.16s,%14.14s,%100.100s,%20.20s,%s,%s") % + boost::format("%8.8s,%9.9s,%8.8s,%25.25s,%20.20s,%20.20s,%64.64s,%16.16s,%14.14s,%106.106s,%20.20s,%s,%s") % tr("block") % tr("direction") % tr("unlocked") % tr("timestamp") % tr("amount") % tr("running balance") % tr("hash") % tr("payment ID") % tr("fee") % tr("destination") % tr("amount") % tr("index") % tr("note") << std::endl; uint64_t running_balance = 0; - auto formatter = boost::format("%8.8llu,%9.9s,%8.8s,%25.25s,%20.20s,%20.20s,%64.64s,%16.16s,%14.14s,%100.100s,%20.20s,\"%s\",%s"); + auto formatter = boost::format("%8.8llu,%9.9s,%8.8s,%25.25s,%20.20s,%20.20s,%64.64s,%16.16s,%14.14s,%106.106s,%20.20s,\"%s\",%s"); for (const auto& transfer : all_transfers) { @@ -9272,7 +9351,7 @@ bool simple_wallet::check_rpc_payment() fail_msg_writer() << tr("Error mining to daemon: ") << error; m_cmd_binder.print_prompt(); }; - bool ret = m_wallet->search_for_rpc_payment(target, startfunc, contfunc, foundfunc, errorfunc); + bool ret = m_wallet->search_for_rpc_payment(target, m_rpc_payment_threads, startfunc, contfunc, foundfunc, errorfunc); if (!ret) { fail_msg_writer() << tr("Failed to start mining for RPC payment"); @@ -9466,7 +9545,7 @@ void simple_wallet::print_accounts() { const std::pair<std::map<std::string, std::string>, std::vector<std::string>>& account_tags = m_wallet->get_account_tags(); size_t num_untagged_accounts = m_wallet->get_num_subaddress_accounts(); - for (const std::pair<std::string, std::string>& p : account_tags.first) + for (const std::pair<const std::string, std::string>& p : account_tags.first) { const std::string& tag = p.first; print_accounts(tag); @@ -11180,7 +11259,6 @@ void simple_wallet::mms_next(const std::vector<std::string> &args) void simple_wallet::mms_sync(const std::vector<std::string> &args) { - mms::message_store& ms = m_wallet->get_message_store(); if (args.size() != 0) { fail_msg_writer() << tr("Usage: mms sync"); @@ -11278,7 +11356,6 @@ void simple_wallet::mms_export(const std::vector<std::string> &args) return; } LOCK_IDLE_SCOPE(); - mms::message_store& ms = m_wallet->get_message_store(); mms::message m; bool valid_id = get_message_from_arg(args[0], m); if (valid_id) @@ -11347,7 +11424,6 @@ void simple_wallet::mms_show(const std::vector<std::string> &args) return; } LOCK_IDLE_SCOPE(); - mms::message_store& ms = m_wallet->get_message_store(); mms::message m; bool valid_id = get_message_from_arg(args[0], m); if (valid_id) @@ -11653,4 +11729,3 @@ bool simple_wallet::mms(const std::vector<std::string> &args) return true; } // End MMS ------------------------------------------------------------------------------------------------ - diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h index 61104c87f..8780bee1d 100644 --- a/src/simplewallet/simplewallet.h +++ b/src/simplewallet/simplewallet.h @@ -158,6 +158,7 @@ namespace cryptonote bool set_credits_target(const std::vector<std::string> &args = std::vector<std::string>()); bool help(const std::vector<std::string> &args = std::vector<std::string>()); bool apropos(const std::vector<std::string> &args); + bool scan_tx(const std::vector<std::string> &args); bool start_mining(const std::vector<std::string> &args); bool stop_mining(const std::vector<std::string> &args); bool set_daemon(const std::vector<std::string> &args); @@ -463,6 +464,7 @@ namespace cryptonote std::atomic<bool> m_need_payment; boost::posix_time::ptime m_last_rpc_payment_mining_time; bool m_rpc_payment_mining_requested; + uint32_t m_rpc_payment_threads = 0; bool m_daemon_rpc_payment_message_displayed; float m_rpc_payment_hash_rate; std::atomic<bool> m_suspend_rpc_payment_mining; diff --git a/src/wallet/CMakeLists.txt b/src/wallet/CMakeLists.txt index bf238ae37..2dd64a38f 100644 --- a/src/wallet/CMakeLists.txt +++ b/src/wallet/CMakeLists.txt @@ -40,18 +40,7 @@ set(wallet_sources wallet_rpc_payments.cpp ) -set(wallet_private_headers - wallet2.h - wallet_args.h - wallet_errors.h - wallet_rpc_server.h - wallet_rpc_server_commands_defs.h - wallet_rpc_server_error_codes.h - ringdb.h - node_rpc_proxy.h - message_store.h - message_transporter.h - wallet_rpc_helpers.h) +monero_find_all_headers(wallet_private_headers "${CMAKE_CURRENT_SOURCE_DIR}") monero_private_headers(wallet ${wallet_private_headers}) @@ -115,45 +104,4 @@ if(NOT IOS) install(TARGETS wallet_rpc_server DESTINATION bin) endif() -# build and install libwallet_merged only if we building for GUI -if (BUILD_GUI_DEPS) - set(libs_to_merge - wallet_api - wallet - rpc_base - multisig - blockchain_db - cryptonote_core - cryptonote_basic - mnemonics - common - cncrypto - device - hardforks - ringct - ringct_basic - checkpoints - version - net - device_trezor) - - foreach(lib ${libs_to_merge}) - list(APPEND objlibs $<TARGET_OBJECTS:obj_${lib}>) # matches naming convention in src/CMakeLists.txt - endforeach() - add_library(wallet_merged STATIC ${objlibs}) - if(IOS) - set(lib_folder lib-${ARCH}) - else() - set(lib_folder lib) - endif() - install(TARGETS wallet_merged - ARCHIVE DESTINATION ${lib_folder}) - - install(FILES ${TREZOR_DEP_LIBS} - DESTINATION ${lib_folder}) - file(WRITE "trezor_link_flags.txt" ${TREZOR_DEP_LINKER}) - install(FILES "trezor_link_flags.txt" - DESTINATION ${lib_folder}) -endif() - add_subdirectory(api) diff --git a/src/wallet/api/CMakeLists.txt b/src/wallet/api/CMakeLists.txt index 30eb4ce03..655cdfefd 100644 --- a/src/wallet/api/CMakeLists.txt +++ b/src/wallet/api/CMakeLists.txt @@ -71,10 +71,13 @@ target_link_libraries(wallet_api mnemonics ${LMDB_LIBRARY} ${Boost_CHRONO_LIBRARY} + ${Boost_LOCALE_LIBRARY} + ${ICU_LIBRARIES} ${Boost_SERIALIZATION_LIBRARY} ${Boost_FILESYSTEM_LIBRARY} ${Boost_SYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} ${Boost_REGEX_LIBRARY} PRIVATE ${EXTRA_LIBRARIES}) diff --git a/src/wallet/api/address_book.cpp b/src/wallet/api/address_book.cpp index 96090d9f5..0b0e17464 100644 --- a/src/wallet/api/address_book.cpp +++ b/src/wallet/api/address_book.cpp @@ -81,7 +81,7 @@ bool AddressBookImpl::setDescription(std::size_t index, const std::string &descr tools::wallet2::address_book_row entry = ab[index]; entry.m_description = description; - bool r = m_wallet->m_wallet->set_address_book_row(index, entry.m_address, NULL, entry.m_description, entry.m_is_subaddress); + bool r = m_wallet->m_wallet->set_address_book_row(index, entry.m_address, entry.m_has_payment_id ? &entry.m_payment_id : nullptr, entry.m_description, entry.m_is_subaddress); if (r) refresh(); else @@ -166,5 +166,3 @@ AddressBookImpl::~AddressBookImpl() } } // namespace - -namespace Bitmonero = Monero; diff --git a/src/wallet/api/address_book.h b/src/wallet/api/address_book.h index e22f474fb..40da46853 100644 --- a/src/wallet/api/address_book.h +++ b/src/wallet/api/address_book.h @@ -66,6 +66,3 @@ private: }; } - -namespace Bitmonero = Monero; - diff --git a/src/wallet/api/pending_transaction.cpp b/src/wallet/api/pending_transaction.cpp index 24f6d37db..f7e74591f 100644 --- a/src/wallet/api/pending_transaction.cpp +++ b/src/wallet/api/pending_transaction.cpp @@ -40,6 +40,7 @@ #include <vector> #include <sstream> #include <boost/format.hpp> +#include <boost/filesystem.hpp> using namespace std; @@ -263,6 +264,3 @@ std::vector<std::string> PendingTransactionImpl::signersKeys() const { } } - -namespace Bitmonero = Monero; - diff --git a/src/wallet/api/pending_transaction.h b/src/wallet/api/pending_transaction.h index bd0ca80a6..274c60851 100644 --- a/src/wallet/api/pending_transaction.h +++ b/src/wallet/api/pending_transaction.h @@ -73,5 +73,3 @@ private: } - -namespace Bitmonero = Monero; diff --git a/src/wallet/api/transaction_history.cpp b/src/wallet/api/transaction_history.cpp index b6d52c58d..4649089ea 100644 --- a/src/wallet/api/transaction_history.cpp +++ b/src/wallet/api/transaction_history.cpp @@ -139,6 +139,7 @@ void TransactionHistoryImpl::refresh() ti->m_paymentid = payment_id; ti->m_coinbase = pd.m_coinbase; ti->m_amount = pd.m_amount; + ti->m_fee = pd.m_fee; ti->m_direction = TransactionInfo::Direction_In; ti->m_hash = string_tools::pod_to_hex(pd.m_tx_hash); ti->m_blockheight = pd.m_block_height; @@ -265,5 +266,3 @@ void TransactionHistoryImpl::refresh() } } // namespace - -namespace Bitmonero = Monero; diff --git a/src/wallet/api/transaction_history.h b/src/wallet/api/transaction_history.h index 60f12d771..fe77253e6 100644 --- a/src/wallet/api/transaction_history.h +++ b/src/wallet/api/transaction_history.h @@ -56,6 +56,3 @@ private: }; } - -namespace Bitmonero = Monero; - diff --git a/src/wallet/api/transaction_info.cpp b/src/wallet/api/transaction_info.cpp index 33e7856db..edbdc469a 100644 --- a/src/wallet/api/transaction_info.cpp +++ b/src/wallet/api/transaction_info.cpp @@ -150,5 +150,3 @@ uint64_t TransactionInfoImpl::unlockTime() const } } // namespace - -namespace Bitmonero = Monero; diff --git a/src/wallet/api/transaction_info.h b/src/wallet/api/transaction_info.h index 8bc36a8e9..5eeeb04c2 100644 --- a/src/wallet/api/transaction_info.h +++ b/src/wallet/api/transaction_info.h @@ -87,5 +87,3 @@ private: }; } // namespace - -namespace Bitmonero = Monero; diff --git a/src/wallet/api/unsigned_transaction.cpp b/src/wallet/api/unsigned_transaction.cpp index fb96674a7..4ccfafebd 100644 --- a/src/wallet/api/unsigned_transaction.cpp +++ b/src/wallet/api/unsigned_transaction.cpp @@ -316,6 +316,3 @@ uint64_t UnsignedTransactionImpl::minMixinCount() const } } // namespace - -namespace Bitmonero = Monero; - diff --git a/src/wallet/api/unsigned_transaction.h b/src/wallet/api/unsigned_transaction.h index 81fc098ff..07649e39e 100644 --- a/src/wallet/api/unsigned_transaction.h +++ b/src/wallet/api/unsigned_transaction.h @@ -71,5 +71,3 @@ private: } - -namespace Bitmonero = Monero; diff --git a/src/wallet/api/utils.cpp b/src/wallet/api/utils.cpp index 34debee9d..19151b5f6 100644 --- a/src/wallet/api/utils.cpp +++ b/src/wallet/api/utils.cpp @@ -60,5 +60,3 @@ void onStartup() } // namespace - -namespace Bitmonero = Monero; diff --git a/src/wallet/api/wallet.cpp b/src/wallet/api/wallet.cpp index 9acc8484c..0afbda705 100644 --- a/src/wallet/api/wallet.cpp +++ b/src/wallet/api/wallet.cpp @@ -45,10 +45,8 @@ #include <sstream> #include <unordered_map> -#ifdef WIN32 #include <boost/locale.hpp> #include <boost/filesystem.hpp> -#endif using namespace std; using namespace cryptonote; @@ -60,7 +58,6 @@ namespace Monero { namespace { // copy-pasted from simplewallet - static const size_t DEFAULT_MIXIN = 6; static const int DEFAULT_REFRESH_INTERVAL_MILLIS = 1000 * 10; // limit maximum refresh interval as one minute static const int MAX_REFRESH_INTERVAL_MILLIS = 1000 * 60 * 1; @@ -791,11 +788,11 @@ bool WalletImpl::close(bool store) return result; } -std::string WalletImpl::seed() const +std::string WalletImpl::seed(const std::string& seed_offset) const { epee::wipeable_string seed; if (m_wallet) - m_wallet->get_seed(seed); + m_wallet->get_seed(seed, seed_offset); return std::string(seed.data(), seed.size()); // TODO } @@ -839,6 +836,11 @@ bool WalletImpl::setPassword(const std::string &password) return status() == Status_Ok; } +const std::string& WalletImpl::getPassword() const +{ + return m_password; +} + bool WalletImpl::setDevicePin(const std::string &pin) { clearStatus(); @@ -1168,7 +1170,7 @@ bool WalletImpl::submitTransaction(const string &fileName) { return true; } -bool WalletImpl::exportKeyImages(const string &filename) +bool WalletImpl::exportKeyImages(const string &filename, bool all) { if (m_wallet->watch_only()) { @@ -1178,7 +1180,7 @@ bool WalletImpl::exportKeyImages(const string &filename) try { - if (!m_wallet->export_key_images(filename)) + if (!m_wallet->export_key_images(filename, all)) { setStatusError(tr("failed to save file ") + filename); return false; @@ -1216,6 +1218,68 @@ bool WalletImpl::importKeyImages(const string &filename) return true; } +bool WalletImpl::exportOutputs(const string &filename, bool all) +{ + if (m_wallet->key_on_device()) + { + setStatusError(string(tr("Not supported on HW wallets.")) + filename); + return false; + } + + try + { + std::string data = m_wallet->export_outputs_to_str(all); + bool r = m_wallet->save_to_file(filename, data); + if (!r) + { + LOG_ERROR("Failed to save file " << filename); + setStatusError(string(tr("Failed to save file: ")) + filename); + return false; + } + } + catch (const std::exception &e) + { + LOG_ERROR("Error exporting outputs: " << e.what()); + setStatusError(string(tr("Error exporting outputs: ")) + e.what()); + return false; + } + + LOG_PRINT_L2("Outputs exported to " << filename); + return true; +} + +bool WalletImpl::importOutputs(const string &filename) +{ + if (m_wallet->key_on_device()) + { + setStatusError(string(tr("Not supported on HW wallets.")) + filename); + return false; + } + + std::string data; + bool r = m_wallet->load_from_file(filename, data); + if (!r) + { + LOG_ERROR("Failed to read file: " << filename); + setStatusError(string(tr("Failed to read file: ")) + filename); + return false; + } + + try + { + size_t n_outputs = m_wallet->import_outputs_from_str(data); + LOG_PRINT_L2(std::to_string(n_outputs) << " outputs imported"); + } + catch (const std::exception &e) + { + LOG_ERROR("Failed to import outputs: " << e.what()); + setStatusError(string(tr("Failed to import outputs: ")) + e.what()); + return false; + } + + return true; +} + void WalletImpl::addSubaddressAccount(const std::string& label) { m_wallet->add_subaddress_account(label); @@ -1431,13 +1495,6 @@ PendingTransaction *WalletImpl::createTransactionMultDest(const std::vector<stri cryptonote::address_parse_info info; - // indicates if dst_addr is integrated address (address + payment_id) - // TODO: (https://bitcointalk.org/index.php?topic=753252.msg9985441#msg9985441) - size_t fake_outs_count = mixin_count > 0 ? mixin_count : m_wallet->default_mixin(); - if (fake_outs_count == 0) - fake_outs_count = DEFAULT_MIXIN; - fake_outs_count = m_wallet->adjust_mixin(fake_outs_count); - uint32_t adjusted_priority = m_wallet->adjust_priority(static_cast<uint32_t>(priority)); PendingTransactionImpl * transaction = new PendingTransactionImpl(*this); @@ -1503,6 +1560,9 @@ PendingTransaction *WalletImpl::createTransactionMultDest(const std::vector<stri break; } try { + size_t fake_outs_count = mixin_count > 0 ? mixin_count : m_wallet->default_mixin(); + fake_outs_count = m_wallet->adjust_mixin(mixin_count); + if (amount) { transaction->m_pending_tx = m_wallet->create_transactions_2(dsts, fake_outs_count, 0 /* unlock_time */, adjusted_priority, @@ -1567,8 +1627,10 @@ PendingTransaction *WalletImpl::createTransactionMultDest(const std::vector<stri setStatusError(writer.str()); } catch (const tools::error::tx_sum_overflow& e) { setStatusError(e.what()); + } catch (const tools::error::zero_amount&) { + setStatusError(tr("destination amount is zero")); } catch (const tools::error::zero_destination&) { - setStatusError(tr("one of destinations is zero")); + setStatusError(tr("transaction has no destination")); } catch (const tools::error::tx_too_big& e) { setStatusError(tr("failed to find a suitable way to split transactions")); } catch (const tools::error::transfer_error& e) { @@ -2001,9 +2063,24 @@ bool WalletImpl::checkReserveProof(const std::string &address, const std::string } } -std::string WalletImpl::signMessage(const std::string &message) +std::string WalletImpl::signMessage(const std::string &message, const std::string &address) { - return m_wallet->sign(message, tools::wallet2::sign_with_spend_key); + if (address.empty()) { + return m_wallet->sign(message, tools::wallet2::sign_with_spend_key); + } + + cryptonote::address_parse_info info; + if (!cryptonote::get_account_address_from_str(info, m_wallet->nettype(), address)) { + setStatusError(tr("Failed to parse address")); + return ""; + } + auto index = m_wallet->get_subaddress_index(info.address); + if (!index) { + setStatusError(tr("Address doesn't belong to the wallet")); + return ""; + } + + return m_wallet->sign(message, tools::wallet2::sign_with_spend_key, *index); } bool WalletImpl::verifySignedMessage(const std::string &message, const std::string &address, const std::string &signature) const @@ -2104,6 +2181,11 @@ bool WalletImpl::watchOnly() const return m_wallet->watch_only(); } +bool WalletImpl::isDeterministic() const +{ + return m_wallet->is_deterministic(); +} + void WalletImpl::clearStatus() const { boost::lock_guard<boost::mutex> l(m_statusMutex); @@ -2283,6 +2365,11 @@ bool WalletImpl::parse_uri(const std::string &uri, std::string &address, std::st return m_wallet->parse_uri(uri, address, payment_id, amount, tx_description, recipient_name, unknown_parameters, error); } +std::string WalletImpl::make_uri(const std::string &address, const std::string &payment_id, uint64_t amount, const std::string &tx_description, const std::string &recipient_name, std::string &error) const +{ + return m_wallet->make_uri(address, payment_id, amount, tx_description, recipient_name, error); +} + std::string WalletImpl::getDefaultDataDir() const { return tools::get_default_data_dir(); @@ -2305,6 +2392,15 @@ bool WalletImpl::rescanSpent() return true; } +void WalletImpl::setOffline(bool offline) +{ + m_wallet->set_offline(offline); +} + +bool WalletImpl::isOffline() const +{ + return m_wallet->is_offline(); +} void WalletImpl::hardForkInfo(uint8_t &version, uint64_t &earliest_height) const { @@ -2505,6 +2601,32 @@ void WalletImpl::deviceShowAddress(uint32_t accountIndex, uint32_t addressIndex, m_wallet->device_show_address(accountIndex, addressIndex, payment_id_param); } -} // namespace -namespace Bitmonero = Monero; +bool WalletImpl::reconnectDevice() +{ + clearStatus(); + + bool r; + try { + r = m_wallet->reconnect_device(); + } + catch (const std::exception &e) { + LOG_ERROR(__FUNCTION__ << " error: " << e.what()); + setStatusError(e.what()); + return false; + } + + return r; +} + +uint64_t WalletImpl::getBytesReceived() +{ + return m_wallet->get_bytes_received(); +} + +uint64_t WalletImpl::getBytesSent() +{ + return m_wallet->get_bytes_sent(); +} + +} // namespace diff --git a/src/wallet/api/wallet.h b/src/wallet/api/wallet.h index 3bf3e759b..67fc2c08a 100644 --- a/src/wallet/api/wallet.h +++ b/src/wallet/api/wallet.h @@ -81,7 +81,7 @@ public: const std::string &device_name); Device getDeviceType() const override; bool close(bool store = true); - std::string seed() const override; + std::string seed(const std::string& seed_offset = "") const override; std::string getSeedLanguage() const override; void setSeedLanguage(const std::string &arg) override; // void setListener(Listener *) {} @@ -89,6 +89,7 @@ public: std::string errorString() const override; void statusWithErrorString(int& status, std::string& errorString) const override; bool setPassword(const std::string &password) override; + const std::string& getPassword() const override; bool setDevicePin(const std::string &password) override; bool setDevicePassphrase(const std::string &password) override; std::string address(uint32_t accountIndex = 0, uint32_t addressIndex = 0) const override; @@ -129,6 +130,7 @@ public: void setRecoveringFromDevice(bool recoveringFromDevice) override; void setSubaddressLookahead(uint32_t major, uint32_t minor) override; bool watchOnly() const override; + bool isDeterministic() const override; bool rescanSpent() override; NetworkType nettype() const override {return static_cast<NetworkType>(m_wallet->nettype());} void hardForkInfo(uint8_t &version, uint64_t &earliest_height) const override; @@ -164,8 +166,10 @@ public: virtual PendingTransaction * createSweepUnmixableTransaction() override; bool submitTransaction(const std::string &fileName) override; virtual UnsignedTransaction * loadUnsignedTx(const std::string &unsigned_filename) override; - bool exportKeyImages(const std::string &filename) override; + bool exportKeyImages(const std::string &filename, bool all = false) override; bool importKeyImages(const std::string &filename) override; + bool exportOutputs(const std::string &filename, bool all = false) override; + bool importOutputs(const std::string &filename) override; virtual void disposeTransaction(PendingTransaction * t) override; virtual uint64_t estimateTransactionFee(const std::vector<std::pair<std::string, uint64_t>> &destinations, @@ -181,6 +185,9 @@ public: virtual bool setCacheAttribute(const std::string &key, const std::string &val) override; virtual std::string getCacheAttribute(const std::string &key) const override; + virtual void setOffline(bool offline) override; + virtual bool isOffline() const override; + virtual bool setUserNote(const std::string &txid, const std::string ¬e) override; virtual std::string getUserNote(const std::string &txid) const override; virtual std::string getTxKey(const std::string &txid) const override; @@ -191,13 +198,14 @@ public: virtual bool checkSpendProof(const std::string &txid, const std::string &message, const std::string &signature, bool &good) const override; virtual std::string getReserveProof(bool all, uint32_t account_index, uint64_t amount, const std::string &message) const override; virtual bool checkReserveProof(const std::string &address, const std::string &message, const std::string &signature, bool &good, uint64_t &total, uint64_t &spent) const override; - virtual std::string signMessage(const std::string &message) override; + virtual std::string signMessage(const std::string &message, const std::string &address) override; virtual bool verifySignedMessage(const std::string &message, const std::string &address, const std::string &signature) const override; virtual std::string signMultisigParticipant(const std::string &message) const override; virtual bool verifyMessageWithPublicKey(const std::string &message, const std::string &publicKey, const std::string &signature) const override; virtual void startRefresh() override; virtual void pauseRefresh() override; virtual bool parse_uri(const std::string &uri, std::string &address, std::string &payment_id, uint64_t &amount, std::string &tx_description, std::string &recipient_name, std::vector<std::string> &unknown_parameters, std::string &error) override; + virtual std::string make_uri(const std::string &address, const std::string &payment_id, uint64_t amount, const std::string &tx_description, const std::string &recipient_name, std::string &error) const override; virtual std::string getDefaultDataDir() const override; virtual bool lightWalletLogin(bool &isNewWallet) const override; virtual bool lightWalletImportWalletRequest(std::string &payment_id, uint64_t &fee, bool &new_request, bool &request_fulfilled, std::string &payment_address, std::string &status) override; @@ -215,6 +223,9 @@ public: virtual bool isKeysFileLocked() override; virtual uint64_t coldKeyImageSync(uint64_t &spent, uint64_t &unspent) override; virtual void deviceShowAddress(uint32_t accountIndex, uint32_t addressIndex, const std::string &paymentId) override; + virtual bool reconnectDevice() override; + virtual uint64_t getBytesReceived() override; + virtual uint64_t getBytesSent() override; private: void clearStatus() const; @@ -276,7 +287,4 @@ private: } // namespace -namespace Bitmonero = Monero; - #endif - diff --git a/src/wallet/api/wallet2_api.h b/src/wallet/api/wallet2_api.h index 44928a422..f9c421a93 100644 --- a/src/wallet/api/wallet2_api.h +++ b/src/wallet/api/wallet2_api.h @@ -446,7 +446,7 @@ struct Wallet }; virtual ~Wallet() = 0; - virtual std::string seed() const = 0; + virtual std::string seed(const std::string& seed_offset = "") const = 0; virtual std::string getSeedLanguage() const = 0; virtual void setSeedLanguage(const std::string &arg) = 0; //! returns wallet status (Status_Ok | Status_Error) @@ -456,6 +456,7 @@ struct Wallet //! returns both error and error string atomically. suggested to use in instead of status() and errorString() virtual void statusWithErrorString(int& status, std::string& errorString) const = 0; virtual bool setPassword(const std::string &password) = 0; + virtual const std::string& getPassword() const = 0; virtual bool setDevicePin(const std::string &pin) { (void)pin; return false; }; virtual bool setDevicePassphrase(const std::string &passphrase) { (void)passphrase; return false; }; virtual std::string address(uint32_t accountIndex = 0, uint32_t addressIndex = 0) const = 0; @@ -627,6 +628,12 @@ struct Wallet virtual bool watchOnly() const = 0; /** + * @brief isDeterministic - checks if wallet keys are deterministic + * @return - true if deterministic + */ + virtual bool isDeterministic() const = 0; + + /** * @brief blockChainHeight - returns current blockchain height * @return */ @@ -901,9 +908,10 @@ struct Wallet /*! * \brief exportKeyImages - exports key images to file * \param filename + * \param all - export all key images or only those that have not yet been exported * \return - true on success */ - virtual bool exportKeyImages(const std::string &filename) = 0; + virtual bool exportKeyImages(const std::string &filename, bool all = false) = 0; /*! * \brief importKeyImages - imports key images from file @@ -912,6 +920,19 @@ struct Wallet */ virtual bool importKeyImages(const std::string &filename) = 0; + /*! + * \brief importOutputs - exports outputs to file + * \param filename + * \return - true on success + */ + virtual bool exportOutputs(const std::string &filename, bool all = false) = 0; + + /*! + * \brief importOutputs - imports outputs from file + * \param filename + * \return - true on success + */ + virtual bool importOutputs(const std::string &filename) = 0; virtual TransactionHistory * history() = 0; virtual AddressBook * addressBook() = 0; @@ -973,7 +994,7 @@ struct Wallet * \param message - the message to sign (arbitrary byte data) * \return the signature */ - virtual std::string signMessage(const std::string &message) = 0; + virtual std::string signMessage(const std::string &message, const std::string &address = "") = 0; /*! * \brief verifySignedMessage - verify a signature matches a given message * \param message - the message (arbitrary byte data) @@ -999,6 +1020,7 @@ struct Wallet virtual bool verifyMessageWithPublicKey(const std::string &message, const std::string &publicKey, const std::string &signature) const = 0; virtual bool parse_uri(const std::string &uri, std::string &address, std::string &payment_id, uint64_t &amount, std::string &tx_description, std::string &recipient_name, std::vector<std::string> &unknown_parameters, std::string &error) = 0; + virtual std::string make_uri(const std::string &address, const std::string &payment_id, uint64_t amount, const std::string &tx_description, const std::string &recipient_name, std::string &error) const = 0; virtual std::string getDefaultDataDir() const = 0; @@ -1007,6 +1029,13 @@ struct Wallet * \return true on success */ virtual bool rescanSpent() = 0; + + /* + * \brief setOffline - toggle set offline on/off + * \param offline - true/false + */ + virtual void setOffline(bool offline) = 0; + virtual bool isOffline() const = 0; //! blackballs a set of outputs virtual bool blackballOutputs(const std::vector<std::string> &outputs, bool add) = 0; @@ -1058,6 +1087,15 @@ struct Wallet //! shows address on device display virtual void deviceShowAddress(uint32_t accountIndex, uint32_t addressIndex, const std::string &paymentId) = 0; + + //! attempt to reconnect to hardware device + virtual bool reconnectDevice() = 0; + + //! get bytes received + virtual uint64_t getBytesReceived() = 0; + + //! get bytes sent + virtual uint64_t getBytesSent() = 0; }; /** @@ -1336,6 +1374,3 @@ struct WalletManagerFactory } - -namespace Bitmonero = Monero; - diff --git a/src/wallet/api/wallet_manager.cpp b/src/wallet/api/wallet_manager.cpp index 900fe91e5..417a27db5 100644 --- a/src/wallet/api/wallet_manager.cpp +++ b/src/wallet/api/wallet_manager.cpp @@ -406,5 +406,3 @@ void WalletManagerFactory::setLogCategories(const std::string &categories) } - -namespace Bitmonero = Monero; diff --git a/src/wallet/api/wallet_manager.h b/src/wallet/api/wallet_manager.h index 2f603b0a9..cf3056a17 100644 --- a/src/wallet/api/wallet_manager.h +++ b/src/wallet/api/wallet_manager.h @@ -102,5 +102,3 @@ private: }; } // namespace - -namespace Bitmonero = Monero; diff --git a/src/wallet/message_store.cpp b/src/wallet/message_store.cpp index 303b576c7..34b4f440b 100644 --- a/src/wallet/message_store.cpp +++ b/src/wallet/message_store.cpp @@ -30,6 +30,8 @@ #include <boost/archive/portable_binary_iarchive.hpp> #include <boost/format.hpp> #include <boost/algorithm/string.hpp> +#include <boost/system/error_code.hpp> +#include <boost/filesystem.hpp> #include <fstream> #include <sstream> #include "file_io_utils.h" @@ -191,9 +193,7 @@ void message_store::unpack_signer_config(const multisig_wallet_state &state, con { try { - std::stringstream iss; - iss << signer_config; - binary_archive<false> ar(iss); + binary_archive<false> ar{epee::strspan<std::uint8_t>(signer_config)}; THROW_WALLET_EXCEPTION_IF(!::serialization::serialize(ar, signers), tools::error::wallet_internal_error, "Failed to serialize signer config"); } catch (...) @@ -381,9 +381,7 @@ void message_store::process_auto_config_data_message(uint32_t id) auto_config_data data; try { - std::stringstream iss; - iss << m.content; - binary_archive<false> ar(iss); + binary_archive<false> ar{epee::strspan<std::uint8_t>(m.content)}; THROW_WALLET_EXCEPTION_IF(!::serialization::serialize(ar, data), tools::error::wallet_internal_error, "Failed to serialize auto config data"); } catch (...) @@ -788,9 +786,7 @@ void message_store::read_from_file(const multisig_wallet_state &state, const std file_data read_file_data; try { - std::stringstream iss; - iss << buf; - binary_archive<false> ar(iss); + binary_archive<false> ar{epee::strspan<std::uint8_t>(buf)}; if (::serialization::serialize(ar, read_file_data)) if (::serialization::check_stream_state(ar)) loaded = true; @@ -827,9 +823,7 @@ void message_store::read_from_file(const multisig_wallet_state &state, const std loaded = false; try { - std::stringstream iss; - iss << decrypted_data; - binary_archive<false> ar(iss); + binary_archive<false> ar{epee::strspan<std::uint8_t>(decrypted_data)}; if (::serialization::serialize(ar, *this)) if (::serialization::check_stream_state(ar)) loaded = true; @@ -1340,7 +1334,10 @@ bool message_store::check_for_messages(const multisig_wallet_state &state, std:: } } std::vector<transport_message> transport_messages; - bool r = m_transporter.receive_messages(destinations, transport_messages); + if (!m_transporter.receive_messages(destinations, transport_messages)) + { + return false; + } if (!m_run.load(std::memory_order_relaxed)) { // Stop was called, don't waste time processing the messages diff --git a/src/wallet/node_rpc_proxy.cpp b/src/wallet/node_rpc_proxy.cpp index 48a602bf3..a576c267c 100644 --- a/src/wallet/node_rpc_proxy.cpp +++ b/src/wallet/node_rpc_proxy.cpp @@ -32,6 +32,8 @@ #include "rpc/rpc_payment_costs.h" #include "storages/http_abstract_invoke.h" +#include <boost/thread.hpp> + #define RETURN_ON_RPC_RESPONSE_ERROR(r, error, res, method) \ do { \ CHECK_AND_ASSERT_MES(error.code == 0, error.message, error.message); \ diff --git a/src/wallet/node_rpc_proxy.h b/src/wallet/node_rpc_proxy.h index 51b7f01dd..f5e3fca5f 100644 --- a/src/wallet/node_rpc_proxy.h +++ b/src/wallet/node_rpc_proxy.h @@ -30,6 +30,7 @@ #include <string> #include <boost/thread/mutex.hpp> +#include <boost/thread/recursive_mutex.hpp> #include "include_base_utils.h" #include "net/abstract_http_client.h" #include "rpc/core_rpc_server_commands_defs.h" diff --git a/src/wallet/ringdb.cpp b/src/wallet/ringdb.cpp index dfeb987ca..025a2037f 100644 --- a/src/wallet/ringdb.cpp +++ b/src/wallet/ringdb.cpp @@ -42,9 +42,6 @@ #define V1TAG ((uint64_t)798237759845202) -static const char zerokey[8] = {0}; -static const MDB_val zerokeyval = { sizeof(zerokey), (void *)zerokey }; - static int compare_hash32(const MDB_val *a, const MDB_val *b) { uint32_t *va = (uint32_t*) a->mv_data; diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 6dd7bc0aa..cc0dff999 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -30,12 +30,14 @@ #include <numeric> #include <tuple> +#include <queue> #include <boost/format.hpp> #include <boost/optional/optional.hpp> #include <boost/algorithm/string/classification.hpp> #include <boost/algorithm/string/trim.hpp> #include <boost/algorithm/string/split.hpp> #include <boost/algorithm/string/join.hpp> +#include <boost/algorithm/string/predicate.hpp> #include <boost/asio/ip/address.hpp> #include <boost/range/adaptor/transformed.hpp> #include <boost/preprocessor/stringize.hpp> @@ -142,6 +144,9 @@ using namespace cryptonote; #define IGNORE_LONG_PAYMENT_ID_FROM_BLOCK_VERSION 12 +#define DEFAULT_UNLOCK_TIME (CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE * DIFFICULTY_TARGET_V2) +#define RECENT_SPEND_WINDOW (50 * DIFFICULTY_TARGET_V2) + static const std::string MULTISIG_SIGNATURE_MAGIC = "SigMultisigPkV1"; static const std::string MULTISIG_EXTRA_INFO_MAGIC = "MultisigxV1"; @@ -344,8 +349,6 @@ std::string get_weight_string(const cryptonote::transaction &tx, size_t blob_siz std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variables_map& vm, bool unattended, const options& opts, const std::function<boost::optional<tools::password_container>(const char *, bool)> &password_prompter) { - namespace ip = boost::asio::ip; - const bool testnet = command_line::get_arg(vm, opts.testnet); const bool stagenet = command_line::get_arg(vm, opts.stagenet); const network_type nettype = testnet ? TESTNET : stagenet ? STAGENET : MAINNET; @@ -367,7 +370,7 @@ std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variabl // user specified CA file or fingeprints implies enabled SSL by default epee::net_utils::ssl_options_t ssl_options = epee::net_utils::ssl_support_t::e_ssl_support_enabled; - if (command_line::get_arg(vm, opts.daemon_ssl_allow_any_cert)) + if (daemon_ssl_allow_any_cert) ssl_options.verification = epee::net_utils::ssl_verification_t::none; else if (!daemon_ssl_ca_file.empty() || !daemon_ssl_allowed_fingerprints.empty()) { @@ -1004,9 +1007,6 @@ bool get_pruned_tx(const cryptonote::COMMAND_RPC_GET_TRANSACTIONS::entry &entry, namespace tools { -// for now, limit to 30 attempts. TODO: discuss a good number to limit to. -const size_t MAX_SPLIT_ATTEMPTS = 30; - constexpr const std::chrono::seconds wallet2::rpc_timeout; const char* wallet2::tr(const char* str) { return i18n_translate(str, "tools::wallet2"); } @@ -1022,7 +1022,13 @@ gamma_picker::gamma_picker(const std::vector<uint64_t> &rct_offsets, double shap end = rct_offsets.data() + rct_offsets.size() - CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE; num_rct_outputs = *(end - 1); THROW_WALLET_EXCEPTION_IF(num_rct_outputs == 0, error::wallet_internal_error, "No rct outputs"); + THROW_WALLET_EXCEPTION_IF(outputs_to_consider == 0, error::wallet_internal_error, "No rct outputs to consider"); average_output_time = DIFFICULTY_TARGET_V2 * blocks_to_consider / outputs_to_consider; // this assumes constant target over the whole rct range + if (average_output_time == 0) { + // TODO: apply this to all cases; do so alongside a hard fork, where all clients will update at the same time, preventing anonymity puddle formation + average_output_time = DIFFICULTY_TARGET_V2 * blocks_to_consider / static_cast<double>(outputs_to_consider); + } + THROW_WALLET_EXCEPTION_IF(average_output_time == 0, error::wallet_internal_error, "Average seconds per output cannot be 0."); }; gamma_picker::gamma_picker(const std::vector<uint64_t> &rct_offsets): gamma_picker(rct_offsets, GAMMA_SHAPE, GAMMA_SCALE) {} @@ -1031,6 +1037,34 @@ uint64_t gamma_picker::pick() { double x = gamma(engine); x = exp(x); + + if (x > DEFAULT_UNLOCK_TIME) + { + // We are trying to select an output from the chain that appeared 'x' seconds before the + // current chain tip, where 'x' is selected from the gamma distribution recommended in Miller et al. + // (https://arxiv.org/pdf/1704.04299/). + // Our method is to get the average time delta between outputs in the recent past, estimate the number of + // outputs 'n' that would have appeared between 'chain_tip - x' and 'chain_tip', select the real output at + // 'current_num_outputs - n', then randomly select an output from the block where that output appears. + // Source code to paper: https://github.com/maltemoeser/moneropaper + // + // Due to the 'default spendable age' mechanic in Monero, 'current_num_outputs' only contains + // currently *unlocked* outputs, which means the earliest output that can be selected is not at the chain tip! + // Therefore, we must offset 'x' so it matches up with the timing of the outputs being considered. We do + // this by saying if 'x` equals the expected age of the first unlocked output (compared to the current + // chain tip - i.e. DEFAULT_UNLOCK_TIME), then select the first unlocked output. + x -= DEFAULT_UNLOCK_TIME; + } + else + { + // If the spent time suggested by the gamma is less than the unlock time, that means the gamma is suggesting an output + // that is no longer feasible to be spent (possible since the gamma was constructed when consensus rules did not enforce the + // lock time). The assumption made in this code is that an output expected spent quicker than the unlock time would likely + // be spent within RECENT_SPEND_WINDOW after allowed. So it returns an output that falls between 0 and the RECENT_SPEND_WINDOW. + // The RECENT_SPEND_WINDOW was determined with empirical analysis of observed data. + x = crypto::rand_idx(static_cast<uint64_t>(RECENT_SPEND_WINDOW)); + } + uint64_t output_index = x / average_output_time; if (output_index >= num_rct_outputs) return std::numeric_limits<uint64_t>::max(); // bad pick @@ -1216,6 +1250,7 @@ wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended, std wallet2::~wallet2() { + deinit(); } bool wallet2::has_testnet_option(const boost::program_options::variables_map& vm) @@ -1323,6 +1358,9 @@ bool wallet2::set_daemon(std::string daemon_address, boost::optional<epee::net_u m_trusted_daemon = trusted_daemon; if (changed) { + if (!m_persistent_rpc_client_id) { + set_rpc_client_secret_key(rct::rct2sk(rct::skGen())); + } m_rpc_payment_state.expected_spent = 0; m_rpc_payment_state.discrepancy = 0; m_node_rpc_proxy.invalidate(); @@ -1602,6 +1640,47 @@ std::string wallet2::get_subaddress_label(const cryptonote::subaddress_index& in return m_subaddress_labels[index.major][index.minor]; } //---------------------------------------------------------------------------------------------------- +void wallet2::scan_tx(const std::vector<crypto::hash> &txids) +{ + // Get the transactions from daemon in batches and add them to a priority queue ordered in chronological order + auto cmp_tx_entry = [](const cryptonote::COMMAND_RPC_GET_TRANSACTIONS::entry& l, const cryptonote::COMMAND_RPC_GET_TRANSACTIONS::entry& r) + { return l.block_height > r.block_height; }; + + std::priority_queue<cryptonote::COMMAND_RPC_GET_TRANSACTIONS::entry, std::vector<COMMAND_RPC_GET_TRANSACTIONS::entry>, decltype(cmp_tx_entry)> txq(cmp_tx_entry); + const size_t SLICE_SIZE = 100; // RESTRICTED_TRANSACTIONS_COUNT as defined in rpc/core_rpc_server.cpp, hardcoded in daemon code + for(size_t slice = 0; slice < txids.size(); slice += SLICE_SIZE) { + cryptonote::COMMAND_RPC_GET_TRANSACTIONS::request req = AUTO_VAL_INIT(req); + cryptonote::COMMAND_RPC_GET_TRANSACTIONS::response res = AUTO_VAL_INIT(res); + req.decode_as_json = false; + req.prune = true; + + size_t ntxes = slice + SLICE_SIZE > txids.size() ? txids.size() - slice : SLICE_SIZE; + for (size_t i = slice; i < slice + ntxes; ++i) + req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txids[i])); + + { + const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; + req.client = get_client_signature(); + bool r = epee::net_utils::invoke_http_json("/gettransactions", req, res, *m_http_client, rpc_timeout); + THROW_WALLET_EXCEPTION_IF(!r, error::wallet_internal_error, "Failed to get transaction from daemon"); + THROW_WALLET_EXCEPTION_IF(res.txs.size() != req.txs_hashes.size(), error::wallet_internal_error, "Failed to get transaction from daemon"); + } + + for (auto& tx_info : res.txs) + txq.push(tx_info); + } + + // Process the transactions in chronologically ascending order + while(!txq.empty()) { + auto& tx_info = txq.top(); + cryptonote::transaction tx; + crypto::hash tx_hash; + THROW_WALLET_EXCEPTION_IF(!get_pruned_tx(tx_info, tx, tx_hash), error::wallet_internal_error, "Failed to get transaction from daemon (2)"); + process_new_transaction(tx_hash, tx, tx_info.output_indices, tx_info.block_height, 0, tx_info.block_timestamp, false, tx_info.in_pool, tx_info.double_spend_seen, {}, {}); + txq.pop(); + } +} +//---------------------------------------------------------------------------------------------------- void wallet2::set_subaddress_label(const cryptonote::subaddress_index& index, const std::string &label) { THROW_WALLET_EXCEPTION_IF(index.major >= m_subaddress_labels.size(), error::account_index_outofbound); @@ -1859,7 +1938,7 @@ void wallet2::cache_tx_data(const cryptonote::transaction& tx, const crypto::has const bool is_miner = tx.vin.size() == 1 && tx.vin[0].type() == typeid(cryptonote::txin_gen); if (!is_miner || m_refresh_type != RefreshType::RefreshNoCoinbase) { - const size_t rec_size = is_miner && m_refresh_type == RefreshType::RefreshOptimizeCoinbase ? 1 : tx.vout.size(); + const size_t rec_size = (is_miner && m_refresh_type == RefreshType::RefreshOptimizeCoinbase && tx.version < 2) ? 1 : tx.vout.size(); if (!tx.vout.empty()) { // if tx.vout is not empty, we loop through all tx pubkeys @@ -2008,7 +2087,7 @@ void wallet2::process_new_transaction(const crypto::hash &txid, const cryptonote { // assume coinbase isn't for us } - else if (miner_tx && m_refresh_type == RefreshOptimizeCoinbase) + else if (miner_tx && m_refresh_type == RefreshOptimizeCoinbase && tx.version < 2) { check_acc_out_precomp_once(tx.vout[0], derivation, additional_derivations, 0, is_out_data_ptr, tx_scan_info[0], output_found[0]); THROW_WALLET_EXCEPTION_IF(tx_scan_info[0].error, error::acc_outs_lookup_error, tx, tx_pub_key, m_account.get_keys()); @@ -2735,9 +2814,8 @@ void wallet2::process_parsed_blocks(uint64_t start_height, const std::vector<cry { if (tx_cache_data[i].empty()) continue; - tpool.submit(&waiter, [&hwdev, &gender, &tx_cache_data, i]() { + tpool.submit(&waiter, [&gender, &tx_cache_data, i]() { auto &slot = tx_cache_data[i]; - boost::unique_lock<hw::device> hwdev_lock(hwdev); for (auto &iod: slot.primary) gender(iod); for (auto &iod: slot.additional) @@ -2780,8 +2858,9 @@ void wallet2::process_parsed_blocks(uint64_t start_height, const std::vector<cry if (m_refresh_type != RefreshType::RefreshNoCoinbase) { THROW_WALLET_EXCEPTION_IF(txidx >= tx_cache_data.size(), error::wallet_internal_error, "txidx out of range"); - const size_t n_vouts = m_refresh_type == RefreshType::RefreshOptimizeCoinbase ? 1 : parsed_blocks[i].block.miner_tx.vout.size(); - tpool.submit(&waiter, [&, i, n_vouts, txidx](){ geniod(parsed_blocks[i].block.miner_tx, n_vouts, txidx); }, true); + const cryptonote::transaction& tx = parsed_blocks[i].block.miner_tx; + const size_t n_vouts = (m_refresh_type == RefreshType::RefreshOptimizeCoinbase && tx.version < 2) ? 1 : tx.vout.size(); + tpool.submit(&waiter, [&, i, n_vouts, txidx](){ geniod(tx, n_vouts, txidx); }, true); } ++txidx; for (size_t j = 0; j < parsed_blocks[i].txes.size(); ++j) @@ -3453,14 +3532,6 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo blocks_fetched += added_blocks; } THROW_WALLET_EXCEPTION_IF(!waiter.wait(), error::wallet_internal_error, "Exception in thread pool"); - if(!first && blocks_start_height == next_blocks_start_height) - { - m_node_rpc_proxy.set_height(m_blockchain.size()); - refreshed = true; - break; - } - - first = false; // handle error from async fetching thread if (error) @@ -3471,6 +3542,15 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo throw std::runtime_error("proxy exception in refresh thread"); } + if(!first && blocks_start_height == next_blocks_start_height) + { + m_node_rpc_proxy.set_height(m_blockchain.size()); + refreshed = true; + break; + } + + first = false; + if (!next_blocks.empty()) { const uint64_t expected_start_height = std::max(static_cast<uint64_t>(m_blockchain.size()), uint64_t(1)) - 1; @@ -3707,9 +3787,11 @@ void wallet2::detach_blockchain(uint64_t height, std::map<std::pair<uint64_t, ui //---------------------------------------------------------------------------------------------------- bool wallet2::deinit() { - m_is_initialized=false; - unlock_keys_file(); - m_account.deinit(); + if(m_is_initialized) { + m_is_initialized = false; + unlock_keys_file(); + m_account.deinit(); + } return true; } //---------------------------------------------------------------------------------------------------- @@ -4384,7 +4466,7 @@ bool wallet2::load_keys_buf(const std::string& keys_buf, const epee::wipeable_st account_public_address device_account_public_address; THROW_WALLET_EXCEPTION_IF(!hwdev.get_public_address(device_account_public_address), error::wallet_internal_error, "Cannot get a device address"); - THROW_WALLET_EXCEPTION_IF(device_account_public_address != m_account.get_keys().m_account_address, error::wallet_internal_error, "Device wallet does not match wallet address. " + THROW_WALLET_EXCEPTION_IF(device_account_public_address != m_account.get_keys().m_account_address, error::wallet_internal_error, "Device wallet does not match wallet address. If the device uses the passphrase feature, please check whether the passphrase was entered correctly (it may have been misspelled - different passphrases generate different wallets, passphrase is case-sensitive). " "Device address: " + cryptonote::get_account_address_as_str(m_nettype, false, device_account_public_address) + ", wallet address: " + m_account.get_public_address_str(m_nettype)); LOG_PRINT_L0("Device inited..."); @@ -5663,12 +5745,18 @@ void wallet2::load(const std::string& wallet_, const epee::wipeable_string& pass try { - std::stringstream iss; - iss << cache_data; - binary_archive<false> ar(iss); + binary_archive<false> ar{epee::strspan<std::uint8_t>(cache_data)}; if (::serialization::serialize(ar, *this)) if (::serialization::check_stream_state(ar)) loaded = true; + if (!loaded) + { + binary_archive<false> ar{epee::strspan<std::uint8_t>(cache_data)}; + ar.enable_varint_bug_backward_compatibility(); + if (::serialization::serialize(ar, *this)) + if (::serialization::check_stream_state(ar)) + loaded = true; + } } catch(...) { } @@ -5988,7 +6076,7 @@ uint64_t wallet2::balance(uint32_t index_major, bool strict) const { uint64_t amount = 0; if(m_light_wallet) - return m_light_wallet_unlocked_balance; + return m_light_wallet_balance; for (const auto& i : balance_per_subaddress(index_major, strict)) amount += i.second; return amount; @@ -6002,7 +6090,7 @@ uint64_t wallet2::unlocked_balance(uint32_t index_major, bool strict, uint64_t * if (time_to_unlock) *time_to_unlock = 0; if(m_light_wallet) - return m_light_wallet_balance; + return m_light_wallet_unlocked_balance; for (const auto& i : unlocked_balance_per_subaddress(index_major, strict)) { amount += i.second.first; @@ -6042,6 +6130,14 @@ std::map<uint32_t, uint64_t> wallet2::balance_per_subaddress(uint32_t index_majo found->second += utx.second.m_change; } } + + for (const auto& utx: m_unconfirmed_payments) + { + if (utx.second.m_pd.m_subaddr_index.major == index_major) + { + amount_per_subaddr[utx.second.m_pd.m_subaddr_index.minor] += utx.second.m_pd.m_amount; + } + } } return amount_per_subaddr; } @@ -6730,8 +6826,7 @@ bool wallet2::parse_unsigned_tx_from_str(const std::string &unsigned_tx_st, unsi catch(const std::exception &e) { LOG_PRINT_L0("Failed to decrypt unsigned tx: " << e.what()); return false; } try { - std::istringstream iss(s); - binary_archive<false> ar(iss); + binary_archive<false> ar{epee::strspan<std::uint8_t>(s)}; if (!::serialization::serialize(ar, exported_txs)) { LOG_PRINT_L0("Failed to parse data from unsigned tx"); @@ -7045,8 +7140,7 @@ bool wallet2::parse_tx_from_str(const std::string &signed_tx_st, std::vector<too catch (const std::exception &e) { LOG_PRINT_L0("Failed to decrypt signed transaction: " << e.what()); return false; } try { - std::istringstream iss(s); - binary_archive<false> ar(iss); + binary_archive<false> ar{epee::strspan<std::uint8_t>(s)}; if (!::serialization::serialize(ar, signed_txs)) { LOG_PRINT_L0("Failed to deserialize signed transaction"); @@ -7181,8 +7275,7 @@ bool wallet2::parse_multisig_tx_from_str(std::string multisig_tx_st, multisig_tx bool loaded = false; try { - std::istringstream iss(multisig_tx_st); - binary_archive<false> ar(iss); + binary_archive<false> ar{epee::strspan<std::uint8_t>(multisig_tx_st)}; if (::serialization::serialize(ar, exported_txs)) if (::serialization::check_stream_state(ar)) loaded = true; @@ -7984,7 +8077,6 @@ void wallet2::light_wallet_get_outs(std::vector<std::vector<tools::wallet2::get_ // Check if we got enough outputs for each amount for(auto& out: ores.amount_outs) { - const uint64_t out_amount = boost::lexical_cast<uint64_t>(out.amount); THROW_WALLET_EXCEPTION_IF(out.outputs.size() < light_wallet_requested_outputs_count , error::wallet_internal_error, "Not enough outputs for amount: " + boost::lexical_cast<std::string>(out.amount)); MDEBUG(out.outputs.size() << " outputs for amount "+ boost::lexical_cast<std::string>(out.amount) + " received from light wallet node"); } @@ -8539,18 +8631,30 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> } // get the keys for those - req.get_txid = false; - + // the response can get large and end up rejected by the anti DoS limits, so chunk it if needed + size_t offset = 0; + while (offset < req.outputs.size()) { + static const size_t chunk_size = 1000; + COMMAND_RPC_GET_OUTPUTS_BIN::request chunk_req = AUTO_VAL_INIT(chunk_req); + COMMAND_RPC_GET_OUTPUTS_BIN::response chunk_daemon_resp = AUTO_VAL_INIT(chunk_daemon_resp); + chunk_req.get_txid = false; + for (size_t i = 0; i < std::min<size_t>(req.outputs.size() - offset, chunk_size); ++i) + chunk_req.outputs.push_back(req.outputs[offset + i]); + const boost::lock_guard<boost::recursive_mutex> lock{m_daemon_rpc_mutex}; uint64_t pre_call_credits = m_rpc_payment_state.credits; - req.client = get_client_signature(); - bool r = epee::net_utils::invoke_http_bin("/get_outs.bin", req, daemon_resp, *m_http_client, rpc_timeout); - THROW_ON_RPC_RESPONSE_ERROR(r, {}, daemon_resp, "get_outs.bin", error::get_outs_error, get_rpc_status(daemon_resp.status)); - THROW_WALLET_EXCEPTION_IF(daemon_resp.outs.size() != req.outputs.size(), error::wallet_internal_error, + chunk_req.client = get_client_signature(); + bool r = epee::net_utils::invoke_http_bin("/get_outs.bin", chunk_req, chunk_daemon_resp, *m_http_client, rpc_timeout); + THROW_ON_RPC_RESPONSE_ERROR(r, {}, chunk_daemon_resp, "get_outs.bin", error::get_outs_error, get_rpc_status(chunk_daemon_resp.status)); + THROW_WALLET_EXCEPTION_IF(chunk_daemon_resp.outs.size() != chunk_req.outputs.size(), error::wallet_internal_error, "daemon returned wrong response for get_outs.bin, wrong amounts count = " + - std::to_string(daemon_resp.outs.size()) + ", expected " + std::to_string(req.outputs.size())); - check_rpc_cost("/get_outs.bin", daemon_resp.credits, pre_call_credits, daemon_resp.outs.size() * COST_PER_OUT); + std::to_string(chunk_daemon_resp.outs.size()) + ", expected " + std::to_string(chunk_req.outputs.size())); + check_rpc_cost("/get_outs.bin", chunk_daemon_resp.credits, pre_call_credits, chunk_daemon_resp.outs.size() * COST_PER_OUT); + + offset += chunk_size; + for (size_t i = 0; i < chunk_daemon_resp.outs.size(); ++i) + daemon_resp.outs.push_back(std::move(chunk_daemon_resp.outs[i])); } std::unordered_map<uint64_t, uint64_t> scanty_outs; @@ -8593,7 +8697,8 @@ void wallet2::get_outs(std::vector<std::vector<tools::wallet2::get_outs_entry>> if (req.outputs[i].index == td.m_global_output_index) if (daemon_resp.outs[i].key == boost::get<txout_to_key>(td.m_tx.vout[td.m_internal_output_index].target).key) if (daemon_resp.outs[i].mask == mask) - real_out_found = true; + if (daemon_resp.outs[i].unlocked) + real_out_found = true; } THROW_WALLET_EXCEPTION_IF(!real_out_found, error::wallet_internal_error, "Daemon response did not include the requested real output"); @@ -8706,7 +8811,7 @@ void wallet2::transfer_selected(const std::vector<cryptonote::tx_destination_ent // throw if total amount overflows uint64_t for(auto& dt: dsts) { - THROW_WALLET_EXCEPTION_IF(0 == dt.amount, error::zero_destination); + THROW_WALLET_EXCEPTION_IF(0 == dt.amount, error::zero_amount); needed_money += dt.amount; LOG_PRINT_L2("transfer: adding " << print_money(dt.amount) << ", for a total of " << print_money (needed_money)); THROW_WALLET_EXCEPTION_IF(needed_money < dt.amount, error::tx_sum_overflow, dsts, fee, m_nettype); @@ -8865,7 +8970,7 @@ void wallet2::transfer_selected_rct(std::vector<cryptonote::tx_destination_entry // throw if total amount overflows uint64_t for(auto& dt: dsts) { - THROW_WALLET_EXCEPTION_IF(0 == dt.amount, error::zero_destination); + THROW_WALLET_EXCEPTION_IF(0 == dt.amount, error::zero_amount); needed_money += dt.amount; LOG_PRINT_L2("transfer: adding " << print_money(dt.amount) << ", for a total of " << print_money (needed_money)); THROW_WALLET_EXCEPTION_IF(needed_money < dt.amount, error::tx_sum_overflow, dsts, fee, m_nettype); @@ -9778,13 +9883,18 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp TX() : weight(0), needed_fee(0) {} - void add(const cryptonote::tx_destination_entry &de, uint64_t amount, unsigned int original_output_index, bool merge_destinations) { + /* Add an output to the transaction. + * Returns True if the output was added, False if there are no more available output slots. + */ + bool add(const cryptonote::tx_destination_entry &de, uint64_t amount, unsigned int original_output_index, bool merge_destinations, size_t max_dsts) { if (merge_destinations) { std::vector<cryptonote::tx_destination_entry>::iterator i; i = std::find_if(dsts.begin(), dsts.end(), [&](const cryptonote::tx_destination_entry &d) { return !memcmp (&d.addr, &de.addr, sizeof(de.addr)); }); if (i == dsts.end()) { + if (dsts.size() >= max_dsts) + return false; dsts.push_back(de); i = dsts.end() - 1; i->amount = 0; @@ -9797,12 +9907,15 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp std::string("original_output_index too large: ") + std::to_string(original_output_index) + " > " + std::to_string(dsts.size())); if (original_output_index == dsts.size()) { + if (dsts.size() >= max_dsts) + return false; dsts.push_back(de); dsts.back().amount = 0; } THROW_WALLET_EXCEPTION_IF(memcmp(&dsts[original_output_index].addr, &de.addr, sizeof(de.addr)), error::wallet_internal_error, "Mismatched destination address"); dsts[original_output_index].amount += amount; } + return true; } }; std::vector<TX> txes; @@ -9830,14 +9943,14 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp needed_money = 0; for(auto& dt: dsts) { - THROW_WALLET_EXCEPTION_IF(0 == dt.amount, error::zero_destination); + THROW_WALLET_EXCEPTION_IF(0 == dt.amount, error::zero_amount); needed_money += dt.amount; LOG_PRINT_L2("transfer: adding " << print_money(dt.amount) << ", for a total of " << print_money (needed_money)); THROW_WALLET_EXCEPTION_IF(needed_money < dt.amount, error::tx_sum_overflow, dsts, 0, m_nettype); } // throw if attempting a transaction with no money - THROW_WALLET_EXCEPTION_IF(needed_money == 0, error::zero_destination); + THROW_WALLET_EXCEPTION_IF(needed_money == 0, error::zero_amount); std::map<uint32_t, std::pair<uint64_t, std::pair<uint64_t, uint64_t>>> unlocked_balance_per_subaddr = unlocked_balance_per_subaddress(subaddr_account, false); std::map<uint32_t, uint64_t> balance_per_subaddr = balance_per_subaddress(subaddr_account, false); @@ -10072,6 +10185,7 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp // clear any fake outs we'd already gathered, since we'll need a new set outs.clear(); + bool out_slots_exhausted = false; if (adding_fee) { LOG_PRINT_L2("We need more fee, adding it to fee"); @@ -10084,20 +10198,32 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp // we can fully pay that destination LOG_PRINT_L2("We can fully pay " << get_account_address_as_str(m_nettype, dsts[0].is_subaddress, dsts[0].addr) << " for " << print_money(dsts[0].amount)); - tx.add(dsts[0], dsts[0].amount, original_output_index, m_merge_destinations); + if (!tx.add(dsts[0], dsts[0].amount, original_output_index, m_merge_destinations, BULLETPROOF_MAX_OUTPUTS-1)) + { + LOG_PRINT_L2("Didn't pay: ran out of output slots"); + out_slots_exhausted = true; + break; + } available_amount -= dsts[0].amount; dsts[0].amount = 0; pop_index(dsts, 0); ++original_output_index; } - if (available_amount > 0 && !dsts.empty() && estimate_tx_weight(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size()+1, extra.size(), bulletproof, clsag) < TX_WEIGHT_TARGET(upper_transaction_weight_limit)) { + if (!out_slots_exhausted && available_amount > 0 && !dsts.empty() && estimate_tx_weight(use_rct, tx.selected_transfers.size(), fake_outs_count, tx.dsts.size()+1, extra.size(), bulletproof, clsag) < TX_WEIGHT_TARGET(upper_transaction_weight_limit)) { // we can partially fill that destination LOG_PRINT_L2("We can partially pay " << get_account_address_as_str(m_nettype, dsts[0].is_subaddress, dsts[0].addr) << " for " << print_money(available_amount) << "/" << print_money(dsts[0].amount)); - tx.add(dsts[0], available_amount, original_output_index, m_merge_destinations); - dsts[0].amount -= available_amount; - available_amount = 0; + if (tx.add(dsts[0], available_amount, original_output_index, m_merge_destinations, BULLETPROOF_MAX_OUTPUTS-1)) + { + dsts[0].amount -= available_amount; + available_amount = 0; + } + else + { + LOG_PRINT_L2("Didn't pay: ran out of output slots"); + out_slots_exhausted = true; + } } } @@ -10105,8 +10231,15 @@ std::vector<wallet2::pending_tx> wallet2::create_transactions_2(std::vector<cryp LOG_PRINT_L2("Considering whether to create a tx now, " << tx.selected_transfers.size() << " inputs, tx limit " << upper_transaction_weight_limit); bool try_tx = false; + + // If the new transaction is full, create it and start a new one + if (out_slots_exhausted) + { + LOG_PRINT_L2("Transaction is full, will create it and start a new tx"); + try_tx = true; + } // if we have preferred picks, but haven't yet used all of them, continue - if (preferred_inputs.empty()) + else if (preferred_inputs.empty()) { if (adding_fee) { @@ -10317,8 +10450,6 @@ bool wallet2::sanity_check(const std::vector<wallet2::pending_tx> &ptx_vector, s { MDEBUG("sanity_check: " << ptx_vector.size() << " txes, " << dsts.size() << " destinations"); - hw::device &hwdev = m_account.get_device(); - THROW_WALLET_EXCEPTION_IF(ptx_vector.empty(), error::wallet_internal_error, "No transactions"); // check every party in there does receive at least the required amount @@ -10355,7 +10486,6 @@ bool wallet2::sanity_check(const std::vector<wallet2::pending_tx> &ptx_vector, s for (const auto &r: required) { const account_public_address &address = r.first; - const crypto::public_key &view_pkey = address.m_view_public_key; uint64_t total_received = 0; for (const auto &ptx: ptx_vector) @@ -11450,6 +11580,9 @@ void wallet2::check_tx_key_helper(const cryptonote::transaction &tx, const crypt void wallet2::check_tx_key_helper(const crypto::hash &txid, const crypto::key_derivation &derivation, const std::vector<crypto::key_derivation> &additional_derivations, const cryptonote::account_public_address &address, uint64_t &received, bool &in_pool, uint64_t &confirmations) { + uint32_t rpc_version; + THROW_WALLET_EXCEPTION_IF(!check_connection(&rpc_version), error::wallet_internal_error, "Failed to connect to daemon: " + get_daemon_address()); + COMMAND_RPC_GET_TRANSACTIONS::request req; COMMAND_RPC_GET_TRANSACTIONS::response res; req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txid)); @@ -11495,10 +11628,17 @@ void wallet2::check_tx_key_helper(const crypto::hash &txid, const crypto::key_de confirmations = 0; if (!in_pool) { - std::string err; - uint64_t bc_height = get_daemon_blockchain_height(err); - if (err.empty()) - confirmations = bc_height - res.txs.front().block_height; + if (rpc_version < MAKE_CORE_RPC_VERSION(3, 7)) + { + std::string err; + uint64_t bc_height = get_daemon_blockchain_height(err); + if (err.empty() && bc_height > res.txs.front().block_height) + confirmations = bc_height - res.txs.front().block_height; + } + else + { + confirmations = res.txs.front().confirmations; + } } } @@ -11961,8 +12101,7 @@ bool wallet2::check_reserve_proof(const cryptonote::account_public_address &addr serializable_unordered_map<crypto::public_key, crypto::signature> subaddr_spendkeys; try { - std::istringstream iss(sig_decoded); - binary_archive<false> ar(iss); + binary_archive<false> ar{epee::strspan<std::uint8_t>(sig_decoded)}; if (::serialization::serialize_noeof(ar, proofs)) if (::serialization::serialize_noeof(ar, subaddr_spendkeys)) if (::serialization::check_stream_state(ar)) @@ -12162,7 +12301,7 @@ uint64_t wallet2::get_approximate_blockchain_height() const // Calculated blockchain height uint64_t approx_blockchain_height = fork_block + (time(NULL) - fork_time)/seconds_per_block; // testnet got some huge rollbacks, so the estimation is way off - static const uint64_t approximate_testnet_rolled_back_blocks = 303967; + static const uint64_t approximate_testnet_rolled_back_blocks = 342100; if (m_nettype == TESTNET && approx_blockchain_height > approximate_testnet_rolled_back_blocks) approx_blockchain_height -= approximate_testnet_rolled_back_blocks; LOG_PRINT_L2("Calculated blockchain height: " << approx_blockchain_height); @@ -12473,7 +12612,7 @@ crypto::public_key wallet2::get_tx_pub_key_from_received_outs(const tools::walle bool wallet2::export_key_images(const std::string &filename, bool all) const { PERF_TIMER(export_key_images); - std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> ski = export_key_images(all); + std::pair<uint64_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> ski = export_key_images(all); std::string magic(KEY_IMAGE_EXPORT_FILE_MAGIC, strlen(KEY_IMAGE_EXPORT_FILE_MAGIC)); const cryptonote::account_public_address &keys = get_account().get_keys().m_account_address; const uint32_t offset = ski.first; @@ -12500,7 +12639,7 @@ bool wallet2::export_key_images(const std::string &filename, bool all) const } //---------------------------------------------------------------------------------------------------- -std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> wallet2::export_key_images(bool all) const +std::pair<uint64_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> wallet2::export_key_images(bool all) const { PERF_TIMER(export_key_images_raw); std::vector<std::pair<crypto::key_image, crypto::signature>> ski; @@ -12997,7 +13136,7 @@ void wallet2::import_blockchain(const std::tuple<size_t, crypto::hash, std::vect m_last_block_reward = cryptonote::get_outs_money_amount(genesis.miner_tx); } //---------------------------------------------------------------------------------------------------- -std::pair<size_t, std::vector<tools::wallet2::transfer_details>> wallet2::export_outputs(bool all) const +std::pair<uint64_t, std::vector<tools::wallet2::transfer_details>> wallet2::export_outputs(bool all) const { PERF_TIMER(export_outputs); std::vector<tools::wallet2::transfer_details> outs; @@ -13037,7 +13176,7 @@ std::string wallet2::export_outputs_to_str(bool all) const return magic + ciphertext; } //---------------------------------------------------------------------------------------------------- -size_t wallet2::import_outputs(const std::pair<size_t, std::vector<tools::wallet2::transfer_details>> &outputs) +size_t wallet2::import_outputs(const std::pair<uint64_t, std::vector<tools::wallet2::transfer_details>> &outputs) { PERF_TIMER(import_outputs); @@ -13143,12 +13282,10 @@ size_t wallet2::import_outputs_from_str(const std::string &outputs_st) try { std::string body(data, headerlen); - std::pair<size_t, std::vector<tools::wallet2::transfer_details>> outputs; + std::pair<uint64_t, std::vector<tools::wallet2::transfer_details>> outputs; try { - std::stringstream iss; - iss << body; - binary_archive<false> ar(iss); + binary_archive<false> ar{epee::strspan<std::uint8_t>(body)}; if (::serialization::serialize(ar, outputs)) if (::serialization::check_stream_state(ar)) loaded = true; @@ -13243,7 +13380,6 @@ rct::multisig_kLRki wallet2::get_multisig_composite_kLRki(size_t n, const std::u { CHECK_AND_ASSERT_THROW_MES(n < m_transfers.size(), "Bad transfer index"); - const transfer_details &td = m_transfers[n]; rct::multisig_kLRki kLRki = get_multisig_kLRki(n, rct::skGen()); // pick a L/R pair from every other participant but one @@ -13401,8 +13537,7 @@ size_t wallet2::import_multisig(std::vector<cryptonote::blobdata> blobs) bool loaded = false; try { - std::istringstream iss(body); - binary_archive<false> ar(iss); + binary_archive<false> ar{epee::strspan<std::uint8_t>(body)}; if (::serialization::serialize(ar, i)) if (::serialization::check_stream_state(ar)) loaded = true; @@ -14124,15 +14259,15 @@ void wallet2::hash_m_transfer(const transfer_details & transfer, crypto::hash &h KECCAK_CTX state; keccak_init(&state); keccak_update(&state, (const uint8_t *) transfer.m_txid.data, sizeof(transfer.m_txid.data)); - keccak_update(&state, (const uint8_t *) transfer.m_internal_output_index, sizeof(transfer.m_internal_output_index)); - keccak_update(&state, (const uint8_t *) transfer.m_global_output_index, sizeof(transfer.m_global_output_index)); - keccak_update(&state, (const uint8_t *) transfer.m_amount, sizeof(transfer.m_amount)); + keccak_update(&state, (const uint8_t *) &transfer.m_internal_output_index, sizeof(transfer.m_internal_output_index)); + keccak_update(&state, (const uint8_t *) &transfer.m_global_output_index, sizeof(transfer.m_global_output_index)); + keccak_update(&state, (const uint8_t *) &transfer.m_amount, sizeof(transfer.m_amount)); keccak_finish(&state, (uint8_t *) hash.data); } //---------------------------------------------------------------------------------------------------- -uint64_t wallet2::hash_m_transfers(int64_t transfer_height, crypto::hash &hash) const +uint64_t wallet2::hash_m_transfers(boost::optional<uint64_t> transfer_height, crypto::hash &hash) const { - CHECK_AND_ASSERT_THROW_MES(transfer_height > (int64_t)m_transfers.size(), "Hash height is greater than number of transfers"); + CHECK_AND_ASSERT_THROW_MES(!transfer_height || *transfer_height <= m_transfers.size(), "Hash height is greater than number of transfers"); KECCAK_CTX state; crypto::hash tmp_hash{}; @@ -14140,12 +14275,12 @@ uint64_t wallet2::hash_m_transfers(int64_t transfer_height, crypto::hash &hash) keccak_init(&state); for(const transfer_details & transfer : m_transfers){ - if (transfer_height >= 0 && current_height >= (uint64_t)transfer_height){ + if (transfer_height && current_height >= *transfer_height){ break; } hash_m_transfer(transfer, tmp_hash); - keccak_update(&state, (const uint8_t *) transfer.m_block_height, sizeof(transfer.m_block_height)); + keccak_update(&state, (const uint8_t *) &transfer.m_block_height, sizeof(transfer.m_block_height)); keccak_update(&state, (const uint8_t *) tmp_hash.data, sizeof(tmp_hash.data)); current_height += 1; } @@ -14157,23 +14292,28 @@ uint64_t wallet2::hash_m_transfers(int64_t transfer_height, crypto::hash &hash) void wallet2::finish_rescan_bc_keep_key_images(uint64_t transfer_height, const crypto::hash &hash) { // Compute hash of m_transfers, if differs there had to be BC reorg. - crypto::hash new_transfers_hash{}; - hash_m_transfers((int64_t) transfer_height, new_transfers_hash); + if (transfer_height <= m_transfers.size()) { + crypto::hash new_transfers_hash{}; + hash_m_transfers(transfer_height, new_transfers_hash); - if (new_transfers_hash != hash) - { - // Soft-Reset to avoid inconsistency in case of BC reorg. - clear_soft(false); // keep_key_images works only with soft reset. - THROW_WALLET_EXCEPTION_IF(true, error::wallet_internal_error, "Transfers changed during rescan, soft or hard rescan is needed"); - } + if (new_transfers_hash == hash) { + // Restore key images in m_transfers from m_key_images + for(auto it = m_key_images.begin(); it != m_key_images.end(); it++) + { + THROW_WALLET_EXCEPTION_IF(it->second >= m_transfers.size(), + error::wallet_internal_error, + "Key images cache contains illegal transfer offset"); + m_transfers[it->second].m_key_image = it->first; + m_transfers[it->second].m_key_image_known = true; + } - // Restore key images in m_transfers from m_key_images - for(auto it = m_key_images.begin(); it != m_key_images.end(); it++) - { - THROW_WALLET_EXCEPTION_IF(it->second >= m_transfers.size(), error::wallet_internal_error, "Key images cache contains illegal transfer offset"); - m_transfers[it->second].m_key_image = it->first; - m_transfers[it->second].m_key_image_known = true; + return; + } } + + // Soft-Reset to avoid inconsistency in case of BC reorg. + clear_soft(false); // keep_key_images works only with soft reset. + THROW_WALLET_EXCEPTION_IF(true, error::wallet_internal_error, "Transfers changed during rescan, soft or hard rescan is needed"); } //---------------------------------------------------------------------------------------------------- uint64_t wallet2::get_bytes_sent() const diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 2e455c40c..facf9878d 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -327,7 +327,7 @@ private: uint64_t m_block_height; cryptonote::transaction_prefix m_tx; crypto::hash m_txid; - size_t m_internal_output_index; + uint64_t m_internal_output_index; uint64_t m_global_output_index; bool m_spent; bool m_frozen; @@ -338,7 +338,7 @@ private: bool m_rct; bool m_key_image_known; bool m_key_image_request; // view wallets: we want to request it; cold wallets: it was requested - size_t m_pk_index; + uint64_t m_pk_index; cryptonote::subaddress_index m_subaddr_index; bool m_key_image_partial; std::vector<rct::key> m_multisig_k; @@ -1264,6 +1264,8 @@ private: std::string get_spend_proof(const crypto::hash &txid, const std::string &message); bool check_spend_proof(const crypto::hash &txid, const std::string &message, const std::string &sig_str); + void scan_tx(const std::vector<crypto::hash> &txids); + /*! * \brief Generates a proof that proves the reserve of unspent funds * \param account_minreserve When specified, collect outputs only belonging to the given account and prove the smallest reserve above the given amount @@ -1370,9 +1372,9 @@ private: bool verify_with_public_key(const std::string &data, const crypto::public_key &public_key, const std::string &signature) const; // Import/Export wallet data - std::pair<size_t, std::vector<tools::wallet2::transfer_details>> export_outputs(bool all = false) const; + std::pair<uint64_t, std::vector<tools::wallet2::transfer_details>> export_outputs(bool all = false) const; std::string export_outputs_to_str(bool all = false) const; - size_t import_outputs(const std::pair<size_t, std::vector<tools::wallet2::transfer_details>> &outputs); + size_t import_outputs(const std::pair<uint64_t, std::vector<tools::wallet2::transfer_details>> &outputs); size_t import_outputs_from_str(const std::string &outputs_st); payment_container export_payments() const; void import_payments(const payment_container &payments); @@ -1380,7 +1382,7 @@ private: std::tuple<size_t, crypto::hash, std::vector<crypto::hash>> export_blockchain() const; void import_blockchain(const std::tuple<size_t, crypto::hash, std::vector<crypto::hash>> &bc); bool export_key_images(const std::string &filename, bool all = false) const; - std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> export_key_images(bool all = false) const; + std::pair<uint64_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> export_key_images(bool all = false) const; uint64_t import_key_images(const std::vector<std::pair<crypto::key_image, crypto::signature>> &signed_key_images, size_t offset, uint64_t &spent, uint64_t &unspent, bool check_spent = true); uint64_t import_key_images(const std::string &filename, uint64_t &spent, uint64_t &unspent); bool import_key_images(std::vector<crypto::key_image> key_images, size_t offset=0, boost::optional<std::unordered_set<size_t>> selected_transfers=boost::none); @@ -1426,7 +1428,7 @@ private: bool get_rpc_payment_info(bool mining, bool &payment_required, uint64_t &credits, uint64_t &diff, uint64_t &credits_per_hash_found, cryptonote::blobdata &hashing_blob, uint64_t &height, uint64_t &seed_height, crypto::hash &seed_hash, crypto::hash &next_seed_hash, uint32_t &cookie); bool daemon_requires_payment(); bool make_rpc_payment(uint32_t nonce, uint32_t cookie, uint64_t &credits, uint64_t &balance); - bool search_for_rpc_payment(uint64_t credits_target, const std::function<bool(uint64_t, uint64_t)> &startfunc, const std::function<bool(unsigned)> &contfunc, const std::function<bool(uint64_t)> &foundfunc = NULL, const std::function<void(const std::string&)> &errorfunc = NULL); + bool search_for_rpc_payment(uint64_t credits_target, uint32_t n_threads, const std::function<bool(uint64_t, uint64_t)> &startfunc, const std::function<bool(unsigned)> &contfunc, const std::function<bool(uint64_t)> &foundfunc = NULL, const std::function<void(const std::string&)> &errorfunc = NULL); template<typename T> void handle_payment_changes(const T &res, std::true_type) { if (res.status == CORE_RPC_STATUS_OK || res.status == CORE_RPC_STATUS_PAYMENT_REQUIRED) m_rpc_payment_state.credits = res.credits; @@ -1545,7 +1547,7 @@ private: bool is_tx_spendtime_unlocked(uint64_t unlock_time, uint64_t block_height); void hash_m_transfer(const transfer_details & transfer, crypto::hash &hash) const; - uint64_t hash_m_transfers(int64_t transfer_height, crypto::hash &hash) const; + uint64_t hash_m_transfers(boost::optional<uint64_t> transfer_height, crypto::hash &hash) const; void finish_rescan_bc_keep_key_images(uint64_t transfer_height, const crypto::hash &hash); void enable_dns(bool enable) { m_use_dns = enable; } void set_offline(bool offline = true); diff --git a/src/wallet/wallet_errors.h b/src/wallet/wallet_errors.h index 4a89ed81a..011780f43 100644 --- a/src/wallet/wallet_errors.h +++ b/src/wallet/wallet_errors.h @@ -83,6 +83,7 @@ namespace tools // tx_rejected // tx_sum_overflow // tx_too_big + // zero_amount // zero_destination // wallet_rpc_error * // daemon_busy @@ -750,10 +751,18 @@ namespace tools uint64_t m_tx_weight_limit; }; //---------------------------------------------------------------------------------------------------- + struct zero_amount: public transfer_error + { + explicit zero_amount(std::string&& loc) + : transfer_error(std::move(loc), "destination amount is zero") + { + } + }; + //---------------------------------------------------------------------------------------------------- struct zero_destination : public transfer_error { explicit zero_destination(std::string&& loc) - : transfer_error(std::move(loc), "destination amount is zero") + : transfer_error(std::move(loc), "transaction has no destination") { } }; diff --git a/src/wallet/wallet_rpc_payments.cpp b/src/wallet/wallet_rpc_payments.cpp index 6527b1384..bf278f695 100644 --- a/src/wallet/wallet_rpc_payments.cpp +++ b/src/wallet/wallet_rpc_payments.cpp @@ -42,6 +42,7 @@ #include "cryptonote_basic/blobdatatype.h" #include "common/i18n.h" #include "common/util.h" +#include "common/threadpool.h" #undef MONERO_DEFAULT_LOG_CATEGORY #define MONERO_DEFAULT_LOG_CATEGORY "wallet.wallet2.rpc_payments" @@ -101,7 +102,7 @@ bool wallet2::make_rpc_payment(uint32_t nonce, uint32_t cookie, uint64_t &credit return true; } //---------------------------------------------------------------------------------------------------- -bool wallet2::search_for_rpc_payment(uint64_t credits_target, const std::function<bool(uint64_t, uint64_t)> &startfunc, const std::function<bool(unsigned)> &contfunc, const std::function<bool(uint64_t)> &foundfunc, const std::function<void(const std::string&)> &errorfunc) +bool wallet2::search_for_rpc_payment(uint64_t credits_target, uint32_t n_threads, const std::function<bool(uint64_t, uint64_t)> &startfunc, const std::function<bool(unsigned)> &contfunc, const std::function<bool(uint64_t)> &foundfunc, const std::function<void(const std::string&)> &errorfunc) { bool need_payment = false; bool payment_required; @@ -139,49 +140,65 @@ bool wallet2::search_for_rpc_payment(uint64_t credits_target, const std::functio continue; } - crypto::hash hash; - const uint32_t local_nonce = nonce++; // wrapping's OK - *(uint32_t*)(hashing_blob.data() + 39) = SWAP32LE(local_nonce); - const uint8_t major_version = hashing_blob[0]; - if (major_version >= RX_BLOCK_VERSION) - { - const int miners = 1; - crypto::rx_slow_hash(height, seed_height, seed_hash.data, hashing_blob.data(), hashing_blob.size(), hash.data, miners, 0); - } - else + if(n_threads == 0) + n_threads = boost::thread::hardware_concurrency(); + + std::vector<crypto::hash> hash(n_threads); + tools::threadpool& tpool = tools::threadpool::getInstance(); + tools::threadpool::waiter waiter(tpool); + + const uint32_t local_nonce = nonce += n_threads; // wrapping's OK + for (size_t i = 0; i < n_threads; i++) { - int cn_variant = hashing_blob[0] >= 7 ? hashing_blob[0] - 6 : 0; - crypto::cn_slow_hash(hashing_blob.data(), hashing_blob.size(), hash, cn_variant, height); + tpool.submit(&waiter, [&, i] { + *(uint32_t*)(hashing_blob.data() + 39) = SWAP32LE(local_nonce-i); + const uint8_t major_version = hashing_blob[0]; + if (major_version >= RX_BLOCK_VERSION) + { + const int miners = 1; + crypto::rx_slow_hash(height, seed_height, seed_hash.data, hashing_blob.data(), hashing_blob.size(), hash[i].data, miners, 0); + } + else + { + int cn_variant = hashing_blob[0] >= 7 ? hashing_blob[0] - 6 : 0; + crypto::cn_slow_hash(hashing_blob.data(), hashing_blob.size(), hash[i], cn_variant, height); + } + }); } - ++n_hashes; - if (cryptonote::check_hash(hash, diff)) + waiter.wait(); + n_hashes += n_threads; + + for(size_t i=0; i < n_threads; i++) { - uint64_t credits, balance; - try + if (cryptonote::check_hash(hash[i], diff)) { - make_rpc_payment(local_nonce, cookie, credits, balance); - if (credits != credits_per_hash_found) + uint64_t credits, balance; + try { - MERROR("Found nonce, but daemon did not credit us with the expected amount"); + make_rpc_payment(local_nonce-i, cookie, credits, balance); + if (credits != credits_per_hash_found) + { + MERROR("Found nonce, but daemon did not credit us with the expected amount"); + if (errorfunc) + errorfunc("Found nonce, but daemon did not credit us with the expected amount"); + return false; + } + MDEBUG("Found nonce " << local_nonce-i << " at diff " << diff << ", gets us " << credits_per_hash_found << ", now " << balance << " credits"); + if (!foundfunc(credits)) + break; + } + catch (const tools::error::wallet_coded_rpc_error &e) + { + MWARNING("Found a local_nonce at diff " << diff << ", but failed to send it to the daemon"); if (errorfunc) - errorfunc("Found nonce, but daemon did not credit us with the expected amount"); - return false; + errorfunc("Found nonce, but daemon errored out with error " + std::to_string(e.code()) + ": " + e.status() + ", continuing"); + } + catch (const std::exception &e) + { + MWARNING("Found a local_nonce at diff " << diff << ", but failed to send it to the daemon"); + if (errorfunc) + errorfunc("Found nonce, but daemon errored out with: '" + std::string(e.what()) + "', continuing"); } - MDEBUG("Found nonce " << local_nonce << " at diff " << diff << ", gets us " << credits_per_hash_found << ", now " << balance << " credits"); - if (!foundfunc(credits)) - break; - } - catch (const tools::error::wallet_coded_rpc_error &e) - { - MWARNING("Found a local_nonce at diff " << diff << ", but failed to send it to the daemon"); - if (errorfunc) - errorfunc("Found nonce, but daemon errored out with error " + std::to_string(e.code()) + ": " + e.status() + ", continuing"); - } - catch (const std::exception &e) - { - MWARNING("Found a local_nonce at diff " << diff << ", but failed to send it to the daemon"); - if (errorfunc) - errorfunc("Found nonce, but daemon errored out with: '" + std::string(e.what()) + "', continuing"); } } } diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index 03db8b70f..e1a06886b 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -687,7 +687,7 @@ namespace tools { if (!m_wallet) return not_open(er); const std::pair<std::map<std::string, std::string>, std::vector<std::string>> account_tags = m_wallet->get_account_tags(); - for (const std::pair<std::string, std::string>& p : account_tags.first) + for (const std::pair<const std::string, std::string>& p : account_tags.first) { res.account_tags.resize(res.account_tags.size() + 1); auto& info = res.account_tags.back(); @@ -762,6 +762,90 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_freeze(const wallet_rpc::COMMAND_RPC_FREEZE::request& req, wallet_rpc::COMMAND_RPC_FREEZE::response& res, epee::json_rpc::error& er, const connection_context *ctx) + { + if (!m_wallet) return not_open(er); + try + { + if (req.key_image.empty()) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = std::string("Must specify key image to freeze"); + return false; + } + crypto::key_image ki; + if (!epee::string_tools::hex_to_pod(req.key_image, ki)) + { + er.code = WALLET_RPC_ERROR_CODE_WRONG_KEY_IMAGE; + er.message = "failed to parse key image"; + return false; + } + m_wallet->freeze(ki); + } + catch (const std::exception& e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_thaw(const wallet_rpc::COMMAND_RPC_THAW::request& req, wallet_rpc::COMMAND_RPC_THAW::response& res, epee::json_rpc::error& er, const connection_context *ctx) + { + if (!m_wallet) return not_open(er); + try + { + if (req.key_image.empty()) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = std::string("Must specify key image to thaw"); + return false; + } + crypto::key_image ki; + if (!epee::string_tools::hex_to_pod(req.key_image, ki)) + { + er.code = WALLET_RPC_ERROR_CODE_WRONG_KEY_IMAGE; + er.message = "failed to parse key image"; + return false; + } + m_wallet->thaw(ki); + } + catch (const std::exception& e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_frozen(const wallet_rpc::COMMAND_RPC_FROZEN::request& req, wallet_rpc::COMMAND_RPC_FROZEN::response& res, epee::json_rpc::error& er, const connection_context *ctx) + { + if (!m_wallet) return not_open(er); + try + { + if (req.key_image.empty()) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = std::string("Must specify key image to check if frozen"); + return false; + } + crypto::key_image ki; + if (!epee::string_tools::hex_to_pod(req.key_image, ki)) + { + er.code = WALLET_RPC_ERROR_CODE_WRONG_KEY_IMAGE; + er.message = "failed to parse key image"; + return false; + } + res.frozen = m_wallet->frozen(ki); + } + catch (const std::exception& e) + { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::validate_transfer(const std::list<wallet_rpc::transfer_destination>& destinations, const std::string& payment_id, std::vector<cryptonote::tx_destination_entry>& dsts, std::vector<uint8_t>& extra, bool at_least_one_destination, epee::json_rpc::error& er) { crypto::hash8 integrated_payment_id = crypto::null_hash8; @@ -822,7 +906,7 @@ namespace tools if (at_least_one_destination && dsts.empty()) { er.code = WALLET_RPC_ERROR_CODE_ZERO_DESTINATION; - er.message = "No destinations for this transfer"; + er.message = "Transaction has no destination"; return false; } @@ -877,10 +961,10 @@ namespace tools return amount; } //------------------------------------------------------------------------------------------------------------------------------ - template<typename Ts, typename Tu> + template<typename Ts, typename Tu, typename Tk> bool wallet_rpc_server::fill_response(std::vector<tools::wallet2::pending_tx> &ptx_vector, bool get_tx_key, Ts& tx_key, Tu &amount, Tu &fee, Tu &weight, std::string &multisig_txset, std::string &unsigned_txset, bool do_not_relay, - Ts &tx_hash, bool get_tx_hex, Ts &tx_blob, bool get_tx_metadata, Ts &tx_metadata, epee::json_rpc::error &er) + Ts &tx_hash, bool get_tx_hex, Ts &tx_blob, bool get_tx_metadata, Ts &tx_metadata, Tk &spent_key_images, epee::json_rpc::error &er) { for (const auto & ptx : ptx_vector) { @@ -895,6 +979,17 @@ namespace tools fill(amount, total_amount(ptx)); fill(fee, ptx.fee); fill(weight, cryptonote::get_transaction_weight(ptx.tx)); + + // add spent key images + tools::wallet_rpc::key_image_list key_image_list; + bool all_are_txin_to_key = std::all_of(ptx.tx.vin.begin(), ptx.tx.vin.end(), [&](const cryptonote::txin_v& s_e) -> bool + { + CHECKED_GET_SPECIFIC_VARIANT(s_e, const cryptonote::txin_to_key, in, false); + key_image_list.key_images.push_back(epee::string_tools::pod_to_hex(in.k_image)); + return true; + }); + THROW_WALLET_EXCEPTION_IF(!all_are_txin_to_key, error::unexpected_txin_type, ptx.tx); + fill(spent_key_images, key_image_list); } if (m_wallet->multisig()) @@ -981,7 +1076,7 @@ namespace tools } return fill_response(ptx_vector, req.get_tx_key, res.tx_key, res.amount, res.fee, res.weight, res.multisig_txset, res.unsigned_txset, req.do_not_relay, - res.tx_hash, req.get_tx_hex, res.tx_blob, req.get_tx_metadata, res.tx_metadata, er); + res.tx_hash, req.get_tx_hex, res.tx_blob, req.get_tx_metadata, res.tx_metadata, res.spent_key_images, er); } catch (const std::exception& e) { @@ -1027,7 +1122,7 @@ namespace tools } return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.weight_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay, - res.tx_hash_list, req.get_tx_hex, res.tx_blob_list, req.get_tx_metadata, res.tx_metadata_list, er); + res.tx_hash_list, req.get_tx_hex, res.tx_blob_list, req.get_tx_metadata, res.tx_metadata_list, res.spent_key_images_list, er); } catch (const std::exception& e) { @@ -1197,13 +1292,20 @@ namespace tools try { // gather info to ask the user - std::unordered_map<cryptonote::account_public_address, std::pair<std::string, uint64_t>> dests; + std::unordered_map<cryptonote::account_public_address, std::pair<std::string, uint64_t>> tx_dests; + std::unordered_map<cryptonote::account_public_address, std::pair<std::string, uint64_t>> all_dests; int first_known_non_zero_change_index = -1; + res.summary.amount_in = 0; + res.summary.amount_out = 0; + res.summary.change_amount = 0; + res.summary.fee = 0; for (size_t n = 0; n < tx_constructions.size(); ++n) { const tools::wallet2::tx_construction_data &cd = tx_constructions[n]; res.desc.push_back({0, 0, std::numeric_limits<uint32_t>::max(), 0, {}, "", 0, "", 0, 0, ""}); wallet_rpc::COMMAND_RPC_DESCRIBE_TRANSFER::transfer_description &desc = res.desc.back(); + // Clear the recipients collection ready for this loop iteration + tx_dests.clear(); std::vector<cryptonote::tx_extra_field> tx_extra_fields; bool has_encrypted_payment_id = false; @@ -1242,17 +1344,17 @@ namespace tools std::string address = cryptonote::get_account_address_as_str(m_wallet->nettype(), entry.is_subaddress, entry.addr); if (has_encrypted_payment_id && !entry.is_subaddress && address != entry.original) address = cryptonote::get_account_integrated_address_as_str(m_wallet->nettype(), entry.addr, payment_id8); - auto i = dests.find(entry.addr); - if (i == dests.end()) - dests.insert(std::make_pair(entry.addr, std::make_pair(address, entry.amount))); + auto i = tx_dests.find(entry.addr); + if (i == tx_dests.end()) + tx_dests.insert(std::make_pair(entry.addr, std::make_pair(address, entry.amount))); else i->second.second += entry.amount; desc.amount_out += entry.amount; } if (cd.change_dts.amount > 0) { - auto it = dests.find(cd.change_dts.addr); - if (it == dests.end()) + auto it = tx_dests.find(cd.change_dts.addr); + if (it == tx_dests.end()) { er.code = WALLET_RPC_ERROR_CODE_BAD_UNSIGNED_TX_DATA; er.message = "Claimed change does not go to a paid address"; @@ -1279,30 +1381,45 @@ namespace tools desc.change_amount += cd.change_dts.amount; it->second.second -= cd.change_dts.amount; if (it->second.second == 0) - dests.erase(cd.change_dts.addr); + tx_dests.erase(cd.change_dts.addr); } - size_t n_dummy_outputs = 0; - for (auto i = dests.begin(); i != dests.end(); ) + for (auto i = tx_dests.begin(); i != tx_dests.end(); ++i) { if (i->second.second > 0) { desc.recipients.push_back({i->second.first, i->second.second}); + auto it_in_all = all_dests.find(i->first); + if (it_in_all == all_dests.end()) + all_dests.insert(std::make_pair(i->first, i->second)); + else + it_in_all->second.second += i->second.second; } else ++desc.dummy_outputs; - ++i; } if (desc.change_amount > 0) { const tools::wallet2::tx_construction_data &cd0 = tx_constructions[0]; desc.change_address = get_account_address_as_str(m_wallet->nettype(), cd0.subaddr_account > 0, cd0.change_dts.addr); + res.summary.change_address = desc.change_address; } desc.fee = desc.amount_in - desc.amount_out; desc.unlock_time = cd.unlock_time; desc.extra = epee::to_hex::string({cd.extra.data(), cd.extra.size()}); + + // Update summary items + res.summary.amount_in += desc.amount_in; + res.summary.amount_out += desc.amount_out; + res.summary.change_amount += desc.change_amount; + res.summary.fee += desc.fee; + } + // Populate the summary recipients list + for (auto i = all_dests.begin(); i != all_dests.end(); ++i) + { + res.summary.recipients.push_back({i->second.first, i->second.second}); } } catch (const std::exception &e) @@ -1390,7 +1507,7 @@ namespace tools std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_unmixable_sweep_transactions(); return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.weight_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay, - res.tx_hash_list, req.get_tx_hex, res.tx_blob_list, req.get_tx_metadata, res.tx_metadata_list, er); + res.tx_hash_list, req.get_tx_hex, res.tx_blob_list, req.get_tx_metadata, res.tx_metadata_list, res.spent_key_images_list, er); } catch (const std::exception& e) { @@ -1448,7 +1565,7 @@ namespace tools std::vector<wallet2::pending_tx> ptx_vector = m_wallet->create_transactions_all(req.below_amount, dsts[0].addr, dsts[0].is_subaddress, req.outputs, mixin, req.unlock_time, priority, extra, req.account_index, subaddr_indices); return fill_response(ptx_vector, req.get_tx_keys, res.tx_key_list, res.amount_list, res.fee_list, res.weight_list, res.multisig_txset, res.unsigned_txset, req.do_not_relay, - res.tx_hash_list, req.get_tx_hex, res.tx_blob_list, req.get_tx_metadata, res.tx_metadata_list, er); + res.tx_hash_list, req.get_tx_hex, res.tx_blob_list, req.get_tx_metadata, res.tx_metadata_list, res.spent_key_images_list, er); } catch (const std::exception& e) { @@ -1523,7 +1640,7 @@ namespace tools } return fill_response(ptx_vector, req.get_tx_key, res.tx_key, res.amount, res.fee, res.weight, res.multisig_txset, res.unsigned_txset, req.do_not_relay, - res.tx_hash, req.get_tx_hex, res.tx_blob, req.get_tx_metadata, res.tx_metadata, er); + res.tx_hash, req.get_tx_hex, res.tx_blob, req.get_tx_metadata, res.tx_metadata, res.spent_key_images, er); } catch (const std::exception& e) { @@ -1556,8 +1673,7 @@ namespace tools try { - std::istringstream iss(blob); - binary_archive<false> ar(iss); + binary_archive<false> ar{epee::strspan<std::uint8_t>(blob)}; if (::serialization::serialize(ar, ptx)) loaded = true; } @@ -1888,6 +2004,7 @@ namespace tools rpc_transfers.tx_hash = epee::string_tools::pod_to_hex(td.m_txid); rpc_transfers.subaddr_index = {td.m_subaddr_index.major, td.m_subaddr_index.minor}; rpc_transfers.key_image = td.m_key_image_known ? epee::string_tools::pod_to_hex(td.m_key_image) : ""; + rpc_transfers.pubkey = epee::string_tools::pod_to_hex(td.get_public_key()); rpc_transfers.block_height = td.m_block_height; rpc_transfers.frozen = td.m_frozen; rpc_transfers.unlocked = m_wallet->is_transfer_unlocked(td); @@ -2702,7 +2819,7 @@ namespace tools if (!m_wallet) return not_open(er); try { - std::pair<size_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> ski = m_wallet->export_key_images(req.all); + std::pair<uint64_t, std::vector<std::pair<crypto::key_image, crypto::signature>>> ski = m_wallet->export_key_images(req.all); res.offset = ski.first; res.signed_key_images.resize(ski.second.size()); for (size_t n = 0; n < ski.second.size(); ++n) @@ -3014,6 +3131,41 @@ namespace tools return true; } //------------------------------------------------------------------------------------------------------------------------------ + bool wallet_rpc_server::on_scan_tx(const wallet_rpc::COMMAND_RPC_SCAN_TX::request& req, wallet_rpc::COMMAND_RPC_SCAN_TX::response& res, epee::json_rpc::error& er, const connection_context *ctx) + { + if (!m_wallet) return not_open(er); + if (m_restricted) + { + er.code = WALLET_RPC_ERROR_CODE_DENIED; + er.message = "Command unavailable in restricted mode."; + return false; + } + + std::vector<crypto::hash> txids; + std::list<std::string>::const_iterator i = req.txids.begin(); + while (i != req.txids.end()) + { + cryptonote::blobdata txid_blob; + if(!epee::string_tools::parse_hexstr_to_binbuff(*i++, txid_blob) || txid_blob.size() != sizeof(crypto::hash)) + { + er.code = WALLET_RPC_ERROR_CODE_WRONG_TXID; + er.message = "TX ID has invalid format"; + return false; + } + + crypto::hash txid = *reinterpret_cast<const crypto::hash*>(txid_blob.data()); + txids.push_back(txid); + } + + try { + m_wallet->scan_tx(txids); + } catch (const std::exception &e) { + handle_rpc_exception(std::current_exception(), er, WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR); + return false; + } + return true; + } + //------------------------------------------------------------------------------------------------------------------------------ bool wallet_rpc_server::on_rescan_spent(const wallet_rpc::COMMAND_RPC_RESCAN_SPENT::request& req, wallet_rpc::COMMAND_RPC_RESCAN_SPENT::response& res, epee::json_rpc::error& er, const connection_context *ctx) { if (!m_wallet) return not_open(er); @@ -3119,17 +3271,7 @@ namespace tools } std::string wallet_file = req.filename.empty() ? "" : (m_wallet_dir + "/" + req.filename); { - std::vector<std::string> languages; - crypto::ElectrumWords::get_language_list(languages, false); - std::vector<std::string>::iterator it; - - it = std::find(languages.begin(), languages.end(), req.language); - if (it == languages.end()) - { - crypto::ElectrumWords::get_language_list(languages, true); - it = std::find(languages.begin(), languages.end(), req.language); - } - if (it == languages.end()) + if (!crypto::ElectrumWords::is_valid_language(req.language)) { er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; er.message = "Unknown language: " + req.language; @@ -3334,6 +3476,11 @@ namespace tools er.code = WALLET_RPC_ERROR_CODE_DAEMON_IS_BUSY; er.message = e.what(); } + catch (const tools::error::zero_amount& e) + { + er.code = WALLET_RPC_ERROR_CODE_ZERO_AMOUNT; + er.message = e.what(); + } catch (const tools::error::zero_destination& e) { er.code = WALLET_RPC_ERROR_CODE_ZERO_DESTINATION; @@ -3547,6 +3694,17 @@ namespace tools return false; } + if (!req.language.empty()) + { + if (!crypto::ElectrumWords::is_valid_language(req.language)) + { + er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; + er.message = "The specified seed language is invalid."; + return false; + } + wal->set_seed_language(req.language); + } + // set blockheight if given try { @@ -3690,12 +3848,7 @@ namespace tools er.message = "Wallet was using the old seed language. You need to specify a new seed language."; return false; } - std::vector<std::string> language_list; - std::vector<std::string> language_list_en; - crypto::ElectrumWords::get_language_list(language_list); - crypto::ElectrumWords::get_language_list(language_list_en, true); - if (std::find(language_list.begin(), language_list.end(), req.language) == language_list.end() && - std::find(language_list_en.begin(), language_list_en.end(), req.language) == language_list_en.end()) + if (!crypto::ElectrumWords::is_valid_language(req.language)) { er.code = WALLET_RPC_ERROR_CODE_UNKNOWN_ERROR; er.message = "Wallet was using the old seed language, and the specified new seed language is invalid."; @@ -4438,7 +4591,14 @@ public: wal->stop(); }); - wal->refresh(wal->is_trusted_daemon()); + try + { + wal->refresh(wal->is_trusted_daemon()); + } + catch (const std::exception& e) + { + LOG_ERROR(tools::wallet_rpc_server::tr("Initial refresh failed: ") << e.what()); + } // if we ^C during potentially length load/refresh, there's no server loop yet if (quit) { diff --git a/src/wallet/wallet_rpc_server.h b/src/wallet/wallet_rpc_server.h index 6e39eca1e..7169c9136 100644 --- a/src/wallet/wallet_rpc_server.h +++ b/src/wallet/wallet_rpc_server.h @@ -84,6 +84,9 @@ namespace tools MAP_JON_RPC_WE("set_account_tag_description", on_set_account_tag_description, wallet_rpc::COMMAND_RPC_SET_ACCOUNT_TAG_DESCRIPTION) MAP_JON_RPC_WE("get_height", on_getheight, wallet_rpc::COMMAND_RPC_GET_HEIGHT) MAP_JON_RPC_WE("getheight", on_getheight, wallet_rpc::COMMAND_RPC_GET_HEIGHT) + MAP_JON_RPC_WE("freeze", on_freeze, wallet_rpc::COMMAND_RPC_FREEZE) + MAP_JON_RPC_WE("thaw", on_thaw, wallet_rpc::COMMAND_RPC_THAW) + MAP_JON_RPC_WE("frozen", on_frozen, wallet_rpc::COMMAND_RPC_FROZEN) MAP_JON_RPC_WE("transfer", on_transfer, wallet_rpc::COMMAND_RPC_TRANSFER) MAP_JON_RPC_WE("transfer_split", on_transfer_split, wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT) MAP_JON_RPC_WE("sign_transfer", on_sign_transfer, wallet_rpc::COMMAND_RPC_SIGN_TRANSFER) @@ -131,6 +134,7 @@ namespace tools MAP_JON_RPC_WE("delete_address_book",on_delete_address_book,wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY) MAP_JON_RPC_WE("refresh", on_refresh, wallet_rpc::COMMAND_RPC_REFRESH) MAP_JON_RPC_WE("auto_refresh", on_auto_refresh, wallet_rpc::COMMAND_RPC_AUTO_REFRESH) + MAP_JON_RPC_WE("scan_tx", on_scan_tx, wallet_rpc::COMMAND_RPC_SCAN_TX) MAP_JON_RPC_WE("rescan_spent", on_rescan_spent, wallet_rpc::COMMAND_RPC_RESCAN_SPENT) MAP_JON_RPC_WE("start_mining", on_start_mining, wallet_rpc::COMMAND_RPC_START_MINING) MAP_JON_RPC_WE("stop_mining", on_stop_mining, wallet_rpc::COMMAND_RPC_STOP_MINING) @@ -173,6 +177,9 @@ namespace tools bool on_untag_accounts(const wallet_rpc::COMMAND_RPC_UNTAG_ACCOUNTS::request& req, wallet_rpc::COMMAND_RPC_UNTAG_ACCOUNTS::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_set_account_tag_description(const wallet_rpc::COMMAND_RPC_SET_ACCOUNT_TAG_DESCRIPTION::request& req, wallet_rpc::COMMAND_RPC_SET_ACCOUNT_TAG_DESCRIPTION::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_getheight(const wallet_rpc::COMMAND_RPC_GET_HEIGHT::request& req, wallet_rpc::COMMAND_RPC_GET_HEIGHT::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_freeze(const wallet_rpc::COMMAND_RPC_FREEZE::request& req, wallet_rpc::COMMAND_RPC_FREEZE::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_thaw(const wallet_rpc::COMMAND_RPC_THAW::request& req, wallet_rpc::COMMAND_RPC_THAW::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_frozen(const wallet_rpc::COMMAND_RPC_FROZEN::request& req, wallet_rpc::COMMAND_RPC_FROZEN::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_transfer(const wallet_rpc::COMMAND_RPC_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_TRANSFER::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_transfer_split(const wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT::request& req, wallet_rpc::COMMAND_RPC_TRANSFER_SPLIT::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_sign_transfer(const wallet_rpc::COMMAND_RPC_SIGN_TRANSFER::request& req, wallet_rpc::COMMAND_RPC_SIGN_TRANSFER::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); @@ -218,6 +225,7 @@ namespace tools bool on_delete_address_book(const wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY::request& req, wallet_rpc::COMMAND_RPC_DELETE_ADDRESS_BOOK_ENTRY::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_refresh(const wallet_rpc::COMMAND_RPC_REFRESH::request& req, wallet_rpc::COMMAND_RPC_REFRESH::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_auto_refresh(const wallet_rpc::COMMAND_RPC_AUTO_REFRESH::request& req, wallet_rpc::COMMAND_RPC_AUTO_REFRESH::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); + bool on_scan_tx(const wallet_rpc::COMMAND_RPC_SCAN_TX::request& req, wallet_rpc::COMMAND_RPC_SCAN_TX::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_rescan_spent(const wallet_rpc::COMMAND_RPC_RESCAN_SPENT::request& req, wallet_rpc::COMMAND_RPC_RESCAN_SPENT::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_start_mining(const wallet_rpc::COMMAND_RPC_START_MINING::request& req, wallet_rpc::COMMAND_RPC_START_MINING::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); bool on_stop_mining(const wallet_rpc::COMMAND_RPC_STOP_MINING::request& req, wallet_rpc::COMMAND_RPC_STOP_MINING::response& res, epee::json_rpc::error& er, const connection_context *ctx = NULL); @@ -255,10 +263,10 @@ namespace tools bool not_open(epee::json_rpc::error& er); void handle_rpc_exception(const std::exception_ptr& e, epee::json_rpc::error& er, int default_error_code); - template<typename Ts, typename Tu> + template<typename Ts, typename Tu, typename Tk> bool fill_response(std::vector<tools::wallet2::pending_tx> &ptx_vector, bool get_tx_key, Ts& tx_key, Tu &amount, Tu &fee, Tu &weight, std::string &multisig_txset, std::string &unsigned_txset, bool do_not_relay, - Ts &tx_hash, bool get_tx_hex, Ts &tx_blob, bool get_tx_metadata, Ts &tx_metadata, epee::json_rpc::error &er); + Ts &tx_hash, bool get_tx_hex, Ts &tx_blob, bool get_tx_metadata, Ts &tx_metadata, Tk &spent_key_images, epee::json_rpc::error &er); bool validate_transfer(const std::list<wallet_rpc::transfer_destination>& destinations, const std::string& payment_id, std::vector<cryptonote::tx_destination_entry>& dsts, std::vector<uint8_t>& extra, bool at_least_one_destination, epee::json_rpc::error& er); diff --git a/src/wallet/wallet_rpc_server_commands_defs.h b/src/wallet/wallet_rpc_server_commands_defs.h index 81f83fb18..248d31aa4 100644 --- a/src/wallet/wallet_rpc_server_commands_defs.h +++ b/src/wallet/wallet_rpc_server_commands_defs.h @@ -47,7 +47,7 @@ // advance which version they will stop working with // Don't go over 32767 for any of these #define WALLET_RPC_VERSION_MAJOR 1 -#define WALLET_RPC_VERSION_MINOR 20 +#define WALLET_RPC_VERSION_MINOR 23 #define MAKE_WALLET_RPC_VERSION(major,minor) (((major)<<16)|(minor)) #define WALLET_RPC_VERSION MAKE_WALLET_RPC_VERSION(WALLET_RPC_VERSION_MAJOR, WALLET_RPC_VERSION_MINOR) namespace tools @@ -456,6 +456,78 @@ namespace wallet_rpc END_KV_SERIALIZE_MAP() }; + struct COMMAND_RPC_FREEZE + { + struct request_t + { + std::string key_image; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(key_image) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<request_t> request; + + struct response_t + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<response_t> response; + }; + + struct COMMAND_RPC_THAW + { + struct request_t + { + std::string key_image; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(key_image) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<request_t> request; + + struct response_t + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<response_t> response; + }; + + struct COMMAND_RPC_FROZEN + { + struct request_t + { + std::string key_image; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(key_image) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<request_t> request; + + struct response_t + { + bool frozen; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(frozen) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<response_t> response; + }; + + struct key_image_list + { + std::list<std::string> key_images; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(key_images) + END_KV_SERIALIZE_MAP() + }; + struct COMMAND_RPC_TRANSFER { struct request_t @@ -499,6 +571,7 @@ namespace wallet_rpc std::string tx_metadata; std::string multisig_txset; std::string unsigned_txset; + key_image_list spent_key_images; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tx_hash) @@ -510,6 +583,7 @@ namespace wallet_rpc KV_SERIALIZE(tx_metadata) KV_SERIALIZE(multisig_txset) KV_SERIALIZE(unsigned_txset) + KV_SERIALIZE(spent_key_images) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<response_t> response; @@ -567,6 +641,7 @@ namespace wallet_rpc std::list<std::string> tx_metadata_list; std::string multisig_txset; std::string unsigned_txset; + std::list<key_image_list> spent_key_images_list; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tx_hash_list) @@ -578,6 +653,7 @@ namespace wallet_rpc KV_SERIALIZE(tx_metadata_list) KV_SERIALIZE(multisig_txset) KV_SERIALIZE(unsigned_txset) + KV_SERIALIZE(spent_key_images_list) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<response_t> response; @@ -625,6 +701,25 @@ namespace wallet_rpc END_KV_SERIALIZE_MAP() }; + struct txset_summary + { + uint64_t amount_in; + uint64_t amount_out; + std::list<recipient> recipients; + uint64_t change_amount; + std::string change_address; + uint64_t fee; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(amount_in) + KV_SERIALIZE(amount_out) + KV_SERIALIZE(recipients) + KV_SERIALIZE(change_amount) + KV_SERIALIZE(change_address) + KV_SERIALIZE(fee) + END_KV_SERIALIZE_MAP() + }; + struct request_t { std::string unsigned_txset; @@ -640,8 +735,10 @@ namespace wallet_rpc struct response_t { std::list<transfer_description> desc; + struct txset_summary summary; BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(summary) KV_SERIALIZE(desc) END_KV_SERIALIZE_MAP() }; @@ -742,6 +839,7 @@ namespace wallet_rpc std::list<std::string> tx_metadata_list; std::string multisig_txset; std::string unsigned_txset; + std::list<key_image_list> spent_key_images_list; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tx_hash_list) @@ -753,6 +851,7 @@ namespace wallet_rpc KV_SERIALIZE(tx_metadata_list) KV_SERIALIZE(multisig_txset) KV_SERIALIZE(unsigned_txset) + KV_SERIALIZE(spent_key_images_list) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<response_t> response; @@ -816,6 +915,7 @@ namespace wallet_rpc std::list<std::string> tx_metadata_list; std::string multisig_txset; std::string unsigned_txset; + std::list<key_image_list> spent_key_images_list; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tx_hash_list) @@ -827,6 +927,7 @@ namespace wallet_rpc KV_SERIALIZE(tx_metadata_list) KV_SERIALIZE(multisig_txset) KV_SERIALIZE(unsigned_txset) + KV_SERIALIZE(spent_key_images_list) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<response_t> response; @@ -875,6 +976,7 @@ namespace wallet_rpc std::string tx_metadata; std::string multisig_txset; std::string unsigned_txset; + key_image_list spent_key_images; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tx_hash) @@ -886,6 +988,7 @@ namespace wallet_rpc KV_SERIALIZE(tx_metadata) KV_SERIALIZE(multisig_txset) KV_SERIALIZE(unsigned_txset) + KV_SERIALIZE(spent_key_images) END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init<response_t> response; @@ -1010,6 +1113,7 @@ namespace wallet_rpc std::string tx_hash; cryptonote::subaddress_index subaddr_index; std::string key_image; + std::string pubkey; // owned output public key found uint64_t block_height; bool frozen; bool unlocked; @@ -1021,6 +1125,7 @@ namespace wallet_rpc KV_SERIALIZE(tx_hash) KV_SERIALIZE(subaddr_index) KV_SERIALIZE(key_image) + KV_SERIALIZE(pubkey); KV_SERIALIZE(block_height) KV_SERIALIZE(frozen) KV_SERIALIZE(unlocked) @@ -2028,6 +2133,26 @@ namespace wallet_rpc typedef epee::misc_utils::struct_init<response_t> response; }; + struct COMMAND_RPC_SCAN_TX + { + struct request_t + { + std::list<std::string> txids; + + BEGIN_KV_SERIALIZE_MAP() + KV_SERIALIZE(txids) + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<request_t> request; + + struct response_t + { + BEGIN_KV_SERIALIZE_MAP() + END_KV_SERIALIZE_MAP() + }; + typedef epee::misc_utils::struct_init<response_t> response; + }; + struct COMMAND_RPC_START_MINING { struct request_t @@ -2192,6 +2317,7 @@ namespace wallet_rpc std::string viewkey; std::string password; bool autosave_current; + std::string language; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE_OPT(restore_height, (uint64_t)0) @@ -2201,6 +2327,7 @@ namespace wallet_rpc KV_SERIALIZE(viewkey) KV_SERIALIZE(password) KV_SERIALIZE_OPT(autosave_current, true) + KV_SERIALIZE(language) END_KV_SERIALIZE_MAP() }; diff --git a/src/wallet/wallet_rpc_server_error_codes.h b/src/wallet/wallet_rpc_server_error_codes.h index f7c5bb0e1..b991029a9 100644 --- a/src/wallet/wallet_rpc_server_error_codes.h +++ b/src/wallet/wallet_rpc_server_error_codes.h @@ -76,4 +76,5 @@ #define WALLET_RPC_ERROR_CODE_NON_DETERMINISTIC -43 #define WALLET_RPC_ERROR_CODE_INVALID_LOG_LEVEL -44 #define WALLET_RPC_ERROR_CODE_ATTRIBUTE_NOT_FOUND -45 +#define WALLET_RPC_ERROR_CODE_ZERO_AMOUNT -46 #define WALLET_RPC_ERROR_CODE_INVALID_SIGNATURE_TYPE -47 diff --git a/tests/README.md b/tests/README.md index f6dd25ead..908482c99 100644 --- a/tests/README.md +++ b/tests/README.md @@ -50,7 +50,12 @@ To run the same tests on a release build, replace `debug` with `release`. # Functional tests [TODO] -Functional tests are located under the `tests/functional` directory. +Functional tests are located under the `tests/functional_tests` directory. + +Building all the tests requires installing the following dependencies: +```bash +pip install requests psutil monotonic +``` First, run a regtest daemon in the offline mode and with a fixed difficulty: ```bash @@ -65,6 +70,25 @@ velvet lymph giddy number token physics poetry unquoted nibs useful sabotage lim Open the wallet file with `monero-wallet-rpc` with RPC port 18083. Finally, start tests by invoking ./blockchain.py or ./speed.py +## Parameters + +Configuration of individual tests. + +### Mining test + +The following environment variables may be set to control the mining test: + +- `MINING_NO_MEASUREMENT` - set to anything to use large enough and fixed mining timeouts (use case: very slow PCs and no intention to change the mining code) +- `MINING_SILENT` - set to anything to disable mining logging + +For example, to customize the run of the functional tests, you may run the following commands from the build directory: + +```bash +export MINING_NO_MEASUREMENT=1 +ctest -V -R functional_tests_rpc +unset MINING_NO_MEASUREMENT +``` + # Fuzz tests Fuzz tests are written using American Fuzzy Lop (AFL), and located under the `tests/fuzz` directory. diff --git a/tests/core_tests/chaingen.h b/tests/core_tests/chaingen.h index a5fd35028..3f9f01a11 100644 --- a/tests/core_tests/chaingen.h +++ b/tests/core_tests/chaingen.h @@ -648,11 +648,9 @@ public: bvc.m_verifivation_failed = true; cryptonote::block blk; - std::stringstream ss; - ss << sr_block.data; - binary_archive<false> ba(ss); + binary_archive<false> ba{epee::strspan<std::uint8_t>(sr_block.data)}; ::serialization::serialize(ba, blk); - if (!ss.good()) + if (!ba.good()) { blk = cryptonote::block(); } @@ -671,11 +669,9 @@ public: bool tx_added = pool_size + 1 == m_c.get_pool_transactions_count(); cryptonote::transaction tx; - std::stringstream ss; - ss << sr_tx.data; - binary_archive<false> ba(ss); + binary_archive<false> ba{epee::strspan<std::uint8_t>(sr_tx.data)}; ::serialization::serialize(ba, tx); - if (!ss.good()) + if (!ba.good()) { tx = cryptonote::transaction(); } @@ -1084,7 +1080,7 @@ inline bool do_replay_file(const std::string& filename) } #define QUOTEME(x) #x -#define DEFINE_TESTS_ERROR_CONTEXT(text) const char* perr_context = text; +#define DEFINE_TESTS_ERROR_CONTEXT(text) const char* perr_context = text; (void) perr_context; #define CHECK_TEST_CONDITION(cond) CHECK_AND_ASSERT_MES(cond, false, "[" << perr_context << "] failed: \"" << QUOTEME(cond) << "\"") #define CHECK_EQ(v1, v2) CHECK_AND_ASSERT_MES(v1 == v2, false, "[" << perr_context << "] failed: \"" << QUOTEME(v1) << " == " << QUOTEME(v2) << "\", " << v1 << " != " << v2) #define CHECK_NOT_EQ(v1, v2) CHECK_AND_ASSERT_MES(!(v1 == v2), false, "[" << perr_context << "] failed: \"" << QUOTEME(v1) << " != " << QUOTEME(v2) << "\", " << v1 << " == " << v2) diff --git a/tests/core_tests/transaction_tests.cpp b/tests/core_tests/transaction_tests.cpp index d0b55dcf4..d19bdd1f9 100644 --- a/tests/core_tests/transaction_tests.cpp +++ b/tests/core_tests/transaction_tests.cpp @@ -78,8 +78,6 @@ bool test_transaction_generation_and_ring_signature() tx_source_entry& src = sources.back(); src.amount = 70368744177663; { - tx_output_entry oe; - src.push_output(0, boost::get<txout_to_key>(tx_mine_1.vout[0].target).key, src.amount); src.push_output(1, boost::get<txout_to_key>(tx_mine_2.vout[0].target).key, src.amount); diff --git a/tests/core_tests/tx_pool.cpp b/tests/core_tests/tx_pool.cpp index 1e496f3da..166eb6075 100644 --- a/tests/core_tests/tx_pool.cpp +++ b/tests/core_tests/tx_pool.cpp @@ -165,9 +165,6 @@ bool txpool_double_spend_base::timestamp_change_pause(cryptonote::core& /*c*/, s bool txpool_double_spend_base::check_changed(cryptonote::core& c, const size_t ev_index, relay_test condition) { - const std::size_t public_hash_count = m_broadcasted_hashes.size(); - const std::size_t all_hash_count = m_all_hashes.size(); - const std::size_t new_broadcasted_hash_count = m_broadcasted_hashes.size() + unsigned(condition == relay_test::broadcasted); const std::size_t new_all_hash_count = m_all_hashes.size() + unsigned(condition == relay_test::hidden) + unsigned(condition == relay_test::no_relay); @@ -393,7 +390,7 @@ bool txpool_double_spend_base::check_changed(cryptonote::core& c, const size_t e } } - for (const std::pair<crypto::hash, uint64_t>& hash : m_all_hashes) + for (const std::pair<const crypto::hash, uint64_t>& hash : m_all_hashes) { cryptonote::blobdata tx_blob{}; if (!c.get_pool_transaction(hash.first, tx_blob, cryptonote::relay_category::all)) @@ -411,7 +408,7 @@ bool txpool_double_spend_base::check_changed(cryptonote::core& c, const size_t e for (const crypto::hash& hash : m_no_relay_hashes) difference.erase(hash); - for (const std::pair<crypto::hash, uint64_t>& hash : difference) + for (const std::pair<const crypto::hash, uint64_t>& hash : difference) { if (c.pool_has_tx(hash.first)) { diff --git a/tests/functional_tests/CMakeLists.txt b/tests/functional_tests/CMakeLists.txt index 462fd4b77..337097a34 100644 --- a/tests/functional_tests/CMakeLists.txt +++ b/tests/functional_tests/CMakeLists.txt @@ -64,16 +64,18 @@ target_link_libraries(make_test_signature ${CMAKE_THREAD_LIBS_INIT} ${EXTRA_LIBRARIES}) -execute_process(COMMAND ${PYTHON_EXECUTABLE} "-c" "import requests; print('OK')" OUTPUT_VARIABLE REQUESTS_OUTPUT OUTPUT_STRIP_TRAILING_WHITESPACE) +monero_add_minimal_executable(cpu_power_test cpu_power_test.cpp) +find_program(PYTHON3_FOUND python3 REQUIRED) + +execute_process(COMMAND ${PYTHON3_FOUND} "-c" "import requests; import psutil; import monotonic; print('OK')" OUTPUT_VARIABLE REQUESTS_OUTPUT OUTPUT_STRIP_TRAILING_WHITESPACE) if (REQUESTS_OUTPUT STREQUAL "OK") add_test( NAME functional_tests_rpc - COMMAND ${PYTHON_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/functional_tests_rpc.py" "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_BINARY_DIR}" all) + COMMAND ${PYTHON3_FOUND} "${CMAKE_CURRENT_SOURCE_DIR}/functional_tests_rpc.py" "${PYTHON3_FOUND}" "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_BINARY_DIR}" all) + add_test( + NAME check_missing_rpc_methods + COMMAND ${PYTHON3_FOUND} "${CMAKE_CURRENT_SOURCE_DIR}/check_missing_rpc_methods.py" "${CMAKE_SOURCE_DIR}") else() - message(WARNING "functional_tests_rpc skipped, needs the 'requests' python module") - set(CTEST_CUSTOM_TESTS_IGNORE ${CTEST_CUSTOM_TESTS_IGNORE} functional_tests_rpc) + message(WARNING "functional_tests_rpc and check_missing_rpc_methods skipped, needs the 'requests', 'psutil' and 'monotonic' python modules") + set(CTEST_CUSTOM_TESTS_IGNORE ${CTEST_CUSTOM_TESTS_IGNORE} functional_tests_rpc check_missing_rpc_methods) endif() - -add_test( - NAME check_missing_rpc_methods - COMMAND ${PYTHON_EXECUTABLE} "${CMAKE_CURRENT_SOURCE_DIR}/check_missing_rpc_methods.py" "${CMAKE_SOURCE_DIR}") diff --git a/tests/functional_tests/cpu_power_test.cpp b/tests/functional_tests/cpu_power_test.cpp new file mode 100644 index 000000000..54c4bfc9f --- /dev/null +++ b/tests/functional_tests/cpu_power_test.cpp @@ -0,0 +1,112 @@ +// Copyright (c) 2014-2021, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +const char * descr = +"This program prints the time (ms) needed to calculate a mathematical challenge. It's purpose \n\ +is to be able to use the result to compare the CPU power available on the machine it's being executed \n\ +and under the given circumstances, which can differ for the same machine. For example: \n\ +The printed value will be different when using 2 of 2 cores, when there an another process \n\ +running on one of the cores core, and when no other intense process running. \n\ + \n\ +The program expects a one argument: a numerical value of the threads to start the calculations on. \n\ + \n\ +Prints: \n\ +Time to calculate a mathematical challenge in MILLISECONDS. \n\ + \n\ +Returns: \n\ +0 on success, \n\ +1 on a missing argument, \n\ +2 on an incorrect format of the argument.\n"; + +#include <iostream> +#include <future> +#include <vector> +#include <sstream> +#include <chrono> + +using namespace std; + +/** +Uses Leibniz Formula for Pi. +https://en.wikipedia.org/wiki/Leibniz_formula_for_%CF%80 +*/ +static double calcPi(const size_t max_iter) +{ + const double n = 4; + double pi = 0; + double d = 1; + for (size_t i = 1; i < max_iter; ++i) + { + const double a = 2.0 * (i % 2) - 1.0; + pi += a * n / d; + d += 2; + } + return pi; +} + +int main(int argc, const char ** argv) +{ + const size_t max_iter = 1e9; + if (argc < 2) + { + cout << "Please pass the number of threads to run.\n"; + + cout << '\n' << descr << '\n'; + return 1; + } + + // Convert argument to an integer. + int numThreads = 1; + const char * numThreadsStr = argv[1]; + std::istringstream iss(numThreadsStr); + if (! (iss >> numThreads) ) + { + cout << "Incorrect format of number of threads = '" << numThreadsStr << "'\n"; + return 2; + } + // Run the calculation in parallel. + std::vector<std::future<double>> futures; + for(int i = 0; i < numThreads; ++i) + { + futures.push_back(std::async(calcPi, max_iter)); + } + + // Start measuring the time. + const std::chrono::steady_clock::time_point tbegin = std::chrono::steady_clock::now(); + for(auto & e : futures) + { + e.get(); + } + // Stop measuring the time. + const std::chrono::steady_clock::time_point tend = std::chrono::steady_clock::now(); + + // Print the measured duration. + cout << std::chrono::duration_cast<std::chrono::milliseconds>(tend - tbegin).count() << std::endl; + return 0; +} + diff --git a/tests/functional_tests/functional_tests_rpc.py b/tests/functional_tests/functional_tests_rpc.py index 79e04b8a6..450552cf8 100755 --- a/tests/functional_tests/functional_tests_rpc.py +++ b/tests/functional_tests/functional_tests_rpc.py @@ -44,6 +44,7 @@ N_MONERODS = 4 N_WALLETS = 5 WALLET_DIRECTORY = builddir + "/functional-tests-directory" +FUNCTIONAL_TESTS_DIRECTORY = builddir + "/tests/functional_tests" DIFFICULTY = 10 monerod_base = [builddir + "/bin/monerod", "--regtest", "--fixed-difficulty", str(DIFFICULTY), "--no-igd", "--p2p-bind-port", "monerod_p2p_port", "--rpc-bind-port", "monerod_rpc_port", "--zmq-rpc-bind-port", "monerod_zmq_port", "--non-interactive", "--disable-dns-checkpoints", "--check-updates", "disabled", "--rpc-ssl", "disabled", "--data-dir", "monerod_data_dir", "--log-level", "1"] @@ -71,14 +72,14 @@ for i in range(N_MONERODS): command_lines.append([str(18180+i) if x == "monerod_rpc_port" else str(18280+i) if x == "monerod_p2p_port" else str(18380+i) if x == "monerod_zmq_port" else builddir + "/functional-tests-directory/monerod" + str(i) if x == "monerod_data_dir" else x for x in monerod_base]) if i < len(monerod_extra): command_lines[-1] += monerod_extra[i] - outputs.append(open(builddir + '/tests/functional_tests/monerod' + str(i) + '.log', 'a+')) + outputs.append(open(FUNCTIONAL_TESTS_DIRECTORY + '/monerod' + str(i) + '.log', 'a+')) ports.append(18180+i) for i in range(N_WALLETS): command_lines.append([str(18090+i) if x == "wallet_port" else x for x in wallet_base]) if i < len(wallet_extra): command_lines[-1] += wallet_extra[i] - outputs.append(open(builddir + '/tests/functional_tests/wallet' + str(i) + '.log', 'a+')) + outputs.append(open(FUNCTIONAL_TESTS_DIRECTORY + '/wallet' + str(i) + '.log', 'a+')) ports.append(18090+i) print('Starting servers...') @@ -89,9 +90,11 @@ try: PYTHONPATH += srcdir + '/../../utils/python-rpc' os.environ['PYTHONPATH'] = PYTHONPATH os.environ['WALLET_DIRECTORY'] = WALLET_DIRECTORY + os.environ['FUNCTIONAL_TESTS_DIRECTORY'] = FUNCTIONAL_TESTS_DIRECTORY + os.environ['SOURCE_DIRECTORY'] = srcdir os.environ['PYTHONIOENCODING'] = 'utf-8' os.environ['DIFFICULTY'] = str(DIFFICULTY) - os.environ['MAKE_TEST_SIGNATURE'] = builddir + '/tests/functional_tests/make_test_signature' + os.environ['MAKE_TEST_SIGNATURE'] = FUNCTIONAL_TESTS_DIRECTORY + '/make_test_signature' os.environ['SEEDHASH_EPOCH_BLOCKS'] = "8" os.environ['SEEDHASH_EPOCH_LAG'] = "4" diff --git a/tests/functional_tests/make_test_signature.cc b/tests/functional_tests/make_test_signature.cc index 07cc3eea6..cb1723847 100644 --- a/tests/functional_tests/make_test_signature.cc +++ b/tests/functional_tests/make_test_signature.cc @@ -28,6 +28,7 @@ #include <stdio.h> #include "misc_language.h" +#include "misc_log_ex.h" #include "string_tools.h" #include "rpc/rpc_payment_signature.h" diff --git a/tests/functional_tests/mining.py b/tests/functional_tests/mining.py index 1eb6b0ba7..7ecbdeed5 100755 --- a/tests/functional_tests/mining.py +++ b/tests/functional_tests/mining.py @@ -31,6 +31,10 @@ from __future__ import print_function import time import os +import math +import monotonic +import util_resources +import multiprocessing """Test daemon mining RPC calls @@ -38,6 +42,10 @@ Test the following RPCs: - start_mining - stop_mining - mining_status + +Control the behavior with these environment variables: + MINING_NO_MEASUREMENT - set to anything to use large enough and fixed mining timeouts + MINING_SILENT - set to anything to disable mining logging """ from framework.daemon import Daemon @@ -71,12 +79,23 @@ class MiningTest(): def mine(self, via_daemon): print("Test mining via " + ("daemon" if via_daemon else "wallet")) + cores_init = multiprocessing.cpu_count() # RX init uses all cores + cores_mine = 1 # Mining uses a parametric number of cores + is_mining_measurent = 'MINING_NO_MEASUREMENT' not in os.environ + + if is_mining_measurent: # A dynamic calculation of the CPU power requested + time_pi_single_cpu = self.measure_cpu_power_get_time(cores_mine) + time_pi_all_cores = self.measure_cpu_power_get_time(cores_init) + # This is the last measurement, since it takes very little time and can be placed timewise-closer to the mining itself. + available_ram = self.get_available_ram() # So far no ideas how to use this var, other than printing it + + start = monotonic.monotonic() daemon = Daemon() wallet = Wallet() # check info/height/balance before generating blocks res_info = daemon.get_info() - prev_height = res_info.height + initial_height = res_info.height res_getbalance = wallet.get_balance() prev_balance = res_getbalance.balance @@ -85,29 +104,75 @@ class MiningTest(): if via_daemon: res = daemon.start_mining('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', threads_count = 1) else: - res = wallet.start_mining(threads_count = 1) + res = wallet.start_mining(threads_count = cores_mine) res_status = daemon.mining_status() assert res_status.active == True - assert res_status.threads_count == 1 + assert res_status.threads_count == cores_mine assert res_status.address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm' assert res_status.is_background_mining_enabled == False assert res_status.block_reward >= 600000000000 # wait till we mined a few of them - target_height = prev_height + 5 - height = prev_height - timeout = 60 # randomx is slow to init + target_height = initial_height + 5 + height = initial_height + + if not is_mining_measurent: + timeout_init = 600 + timeout_mine = 300 + else: + """ + Randomx init has high variance on CI machines due to noisy neighbors, + taking up resources in parallel (including by our own jobs). + + Mining is organized in the following scheme: + 1) first loop's pass: RandomX init and mining + 2) every next pass: only mining + Pass 1) takes much more time than pass 2) + Pass 1) uses all cores, pass 2) just one (currently) + For the above reasons both passes need separate timeouts and adjustments. + After the first pass, the timeout is being reset to a lower value. + """ + + def calc_timeout(seconds_constant, time_pi, cores): + """ + The time it took to calculate pi under certain conditions + is proportional to the time it will take to calculate the real job. + + The number of cores used decreases the time almost linearly. + """ + timeout = float(seconds_constant) * time_pi / float(cores) + return timeout + + timeout_base_init = 60 # RX init needs more time + timeout_base_mine = 20 + timeout_init = calc_timeout(timeout_base_init, time_pi_all_cores, cores_init) + timeout_mine = calc_timeout(timeout_base_mine, time_pi_single_cpu, cores_mine) + + msg_timeout_src = "adjusted for the currently available CPU power" if is_mining_measurent else "selected to have the default value" + msg = "Timeout for {} {}, is {:.1f} s" + self.print_mining_info(msg.format("init, ", msg_timeout_src, timeout_init)) + self.print_mining_info(msg.format("mining,", msg_timeout_src, timeout_mine)) + timeout = timeout_init + rx_inited = False # Gets initialized in the first pass of the below loop while height < target_height: seen_height = height - for _ in range(timeout): + for _ in range(int(math.ceil(timeout))): time.sleep(1) + seconds_passed = monotonic.monotonic() - start height = daemon.get_info().height if height > seen_height: break else: - assert False, 'Failed to mine successor to block %d (initial block = %d)' % (seen_height, prev_height) - timeout = 10 + assert False, 'Failed to mine successor to block %d (initial block = %d) after %d s. RX initialized = %r' % (seen_height, initial_height, round(seconds_passed), rx_inited) + if not rx_inited: + rx_inited = True + timeout = timeout_mine # Resetting the timeout after first mined block and RX init + self.print_time_taken(start, "RX init + mining 1st block") + else: + self.print_time_taken(start, "mining iteration") + + self.print_time_taken(start, "mining total") if via_daemon: res = daemon.stop_mining() @@ -123,7 +188,7 @@ class MiningTest(): wallet.refresh() res_getbalance = wallet.get_balance() balance = res_getbalance.balance - assert balance >= prev_balance + (new_height - prev_height) * 600000000000 + assert balance >= prev_balance + (new_height - initial_height) * 600000000000 if via_daemon: res = daemon.start_mining('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', threads_count = 1, do_background_mining = True) @@ -144,6 +209,23 @@ class MiningTest(): res_status = daemon.mining_status() assert res_status.active == False + def measure_cpu_power_get_time(self, cores): + self.print_mining_info("Measuring the currently available CPU power...") + build_dir_funcional_tests = os.environ['FUNCTIONAL_TESTS_DIRECTORY'] + time_pi = util_resources.get_time_pi_seconds(cores, build_dir_funcional_tests) + self.print_mining_info("Time taken to calculate Pi on {} core(s) was {:.2f} s.".format(cores, time_pi)) + return time_pi + + def get_available_ram(self): + available_ram = util_resources.available_ram_gb() + threshold_ram = 3 + self.print_mining_info("Available RAM = " + str(round(available_ram, 1)) + " GB") + if available_ram < threshold_ram: + print("Warning! Available RAM =", round(available_ram, 1), + "GB is less than the reasonable threshold =", threshold_ram, + ". The RX init might exceed the calculated timeout.") + return available_ram + def submitblock(self): print("Test submitblock") @@ -171,6 +253,20 @@ class MiningTest(): assert res.height == height + i + 1 assert res.hash == block_hash + def is_mining_silent(self): + return 'MINING_SILENT' in os.environ + + def print_mining_info(self, msg): + if self.is_mining_silent(): + return + print(msg) + + def print_time_taken(self, start, msg_context): + if self.is_mining_silent(): + return + seconds_passed = monotonic.monotonic() - start + print("Time taken for", msg_context, "=", round(seconds_passed, 1), "s.") + def test_randomx(self): print("Test RandomX") diff --git a/tests/functional_tests/transactions_flow_test.cpp b/tests/functional_tests/transactions_flow_test.cpp index cec6825f5..3f5f90d3b 100644 --- a/tests/functional_tests/transactions_flow_test.cpp +++ b/tests/functional_tests/transactions_flow_test.cpp @@ -41,7 +41,6 @@ using namespace cryptonote; namespace { uint64_t const TEST_FEE = 5000000000; // 5 * 10^9 - uint64_t const TEST_DUST_THRESHOLD = 5000000000; // 5 * 10^9 } std::string generate_random_wallet_name() diff --git a/tests/functional_tests/util_resources.py b/tests/functional_tests/util_resources.py new file mode 100755 index 000000000..0ea96c129 --- /dev/null +++ b/tests/functional_tests/util_resources.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2021 The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +""" + Help determine how much CPU power is available at the given time + by running numerical calculations +""" + +from __future__ import print_function +import subprocess +import psutil + +def available_ram_gb(): + ram_bytes = psutil.virtual_memory().available + kilo = 1024.0 + ram_gb = ram_bytes / kilo**3 + return ram_gb + +def get_time_pi_seconds(cores, app_dir='.'): + app_path = '{}/cpu_power_test'.format(app_dir) + time_calc = subprocess.check_output([app_path, str(cores)]) + decoded = time_calc.decode('utf-8') + miliseconds = int(decoded) + + return miliseconds / 1000.0 diff --git a/tests/fuzz/bulletproof.cpp b/tests/fuzz/bulletproof.cpp index e0f183bc5..06416fe34 100644 --- a/tests/fuzz/bulletproof.cpp +++ b/tests/fuzz/bulletproof.cpp @@ -37,9 +37,7 @@ BEGIN_INIT_SIMPLE_FUZZER() END_INIT_SIMPLE_FUZZER() BEGIN_SIMPLE_FUZZER() - std::stringstream ss; - ss << std::string((const char*)buf, len); - binary_archive<false> ba(ss); + binary_archive<false> ba{{buf, len}}; rct::Bulletproof proof = AUTO_VAL_INIT(proof); ::serialization::serialize(ba, proof); END_SIMPLE_FUZZER() diff --git a/tests/fuzz/cold-outputs.cpp b/tests/fuzz/cold-outputs.cpp index 2698a36ba..69eb7b61f 100644 --- a/tests/fuzz/cold-outputs.cpp +++ b/tests/fuzz/cold-outputs.cpp @@ -50,11 +50,8 @@ BEGIN_INIT_SIMPLE_FUZZER() END_INIT_SIMPLE_FUZZER() BEGIN_SIMPLE_FUZZER() - std::string s((const char*)buf, len); - std::pair<size_t, std::vector<tools::wallet2::transfer_details>> outputs; - std::stringstream iss; - iss << s; - binary_archive<false> ar(iss); + std::pair<uint64_t, std::vector<tools::wallet2::transfer_details>> outputs; + binary_archive<false> ar{{buf, len}}; ::serialization::serialize(ar, outputs); size_t n_outputs = wallet->import_outputs(outputs); std::cout << boost::lexical_cast<std::string>(n_outputs) << " outputs imported" << std::endl; diff --git a/tests/fuzz/cold-transaction.cpp b/tests/fuzz/cold-transaction.cpp index 135343704..bf5fba1f7 100644 --- a/tests/fuzz/cold-transaction.cpp +++ b/tests/fuzz/cold-transaction.cpp @@ -50,11 +50,8 @@ BEGIN_INIT_SIMPLE_FUZZER() END_INIT_SIMPLE_FUZZER() BEGIN_SIMPLE_FUZZER() - std::string s((const char*)buf, len); tools::wallet2::unsigned_tx_set exported_txs; - std::stringstream iss; - iss << s; - binary_archive<false> ar(iss); + binary_archive<false> ar{{buf, len}}; ::serialization::serialize(ar, exported_txs); std::vector<tools::wallet2::pending_tx> ptx; bool success = wallet->sign_tx(exported_txs, "/tmp/cold-transaction-test-signed", ptx); diff --git a/tests/fuzz/levin.cpp b/tests/fuzz/levin.cpp index 78b7b6863..209a75221 100644 --- a/tests/fuzz/levin.cpp +++ b/tests/fuzz/levin.cpp @@ -68,13 +68,13 @@ namespace { } - virtual int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, test_levin_connection_context& context) + virtual int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_stream& buff_out, test_levin_connection_context& context) { m_invoke_counter.inc(); boost::unique_lock<boost::mutex> lock(m_mutex); m_last_command = command; m_last_in_buf = std::string((const char*)in_buff.data(), in_buff.size()); - buff_out = m_invoke_out_buf.clone(); + buff_out.write(epee::to_span(m_invoke_out_buf)); return m_return_code; } diff --git a/tests/gtest/CMakeLists.txt b/tests/gtest/CMakeLists.txt index 621d0f042..50941b8d8 100644 --- a/tests/gtest/CMakeLists.txt +++ b/tests/gtest/CMakeLists.txt @@ -45,7 +45,7 @@ endif() # ${gtest_BINARY_DIR}. # Language "C" is required for find_package(Threads). project(gtest CXX C) -cmake_minimum_required(VERSION 2.6.2) +cmake_minimum_required(VERSION 3.5) if (COMMAND set_up_hermetic_build) set_up_hermetic_build() diff --git a/tests/gtest/cmake/internal_utils.cmake b/tests/gtest/cmake/internal_utils.cmake index ae45d7d7a..b33197989 100644 --- a/tests/gtest/cmake/internal_utils.cmake +++ b/tests/gtest/cmake/internal_utils.cmake @@ -96,7 +96,7 @@ macro(config_compiler_and_linker) set(cxx_no_rtti_flags "-fno-rtti -DGTEST_HAS_RTTI=0") set(cxx_strict_flags "-Wextra -Wno-unused-parameter -Wno-missing-field-initializers") - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") set(cxx_base_flags "-Wall -Wshadow -fPIC") set(cxx_exception_flags "-fexceptions") set(cxx_no_exception_flags "-fno-exceptions") diff --git a/tests/net_load_tests/net_load_tests.h b/tests/net_load_tests/net_load_tests.h index baab07d31..59eef7bd1 100644 --- a/tests/net_load_tests/net_load_tests.h +++ b/tests/net_load_tests/net_load_tests.h @@ -67,7 +67,7 @@ namespace net_load_tests { } - virtual int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, test_connection_context& context) + virtual int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_stream& buff_out, test_connection_context& context) { //m_invoke_counter.inc(); //std::unique_lock<std::mutex> lock(m_mutex); diff --git a/tests/net_load_tests/srv.cpp b/tests/net_load_tests/srv.cpp index 84d97ebd4..fabe14f49 100644 --- a/tests/net_load_tests/srv.cpp +++ b/tests/net_load_tests/srv.cpp @@ -169,7 +169,7 @@ namespace LOG_PRINT_L0("Closing connections. Number of opened connections: " << m_tcp_server.get_config_object().get_connections_count()); size_t count = 0; - bool r = m_tcp_server.get_config_object().foreach_connection([&](test_connection_context& ctx) { + m_tcp_server.get_config_object().foreach_connection([&](test_connection_context& ctx) { if (ctx.m_connection_id != cmd_conn_id) { ++count; diff --git a/tests/performance_tests/performance_tests.h b/tests/performance_tests/performance_tests.h index ae7aabe08..7bedfdd4e 100644 --- a/tests/performance_tests/performance_tests.h +++ b/tests/performance_tests/performance_tests.h @@ -215,7 +215,6 @@ void run_test(const std::string &filter, Params ¶ms, const char* test_name) if (params.stats) { uint64_t mins = min / scale; - uint64_t maxs = max / scale; uint64_t meds = med / scale; uint64_t p95s = quantiles[9] / scale; uint64_t stddevs = stddev / scale; diff --git a/tests/unit_tests/bulletproofs.cpp b/tests/unit_tests/bulletproofs.cpp index 7c6e459f5..ee617938c 100644 --- a/tests/unit_tests/bulletproofs.cpp +++ b/tests/unit_tests/bulletproofs.cpp @@ -296,7 +296,6 @@ TEST(bulletproof, weight_pruned) ASSERT_TRUE(tx.version == 2); ASSERT_FALSE(tx.pruned); ASSERT_TRUE(rct::is_rct_bulletproof(tx.rct_signatures.type)); - const uint64_t tx_size = bd.size(); const uint64_t tx_weight = cryptonote::get_transaction_weight(tx); ASSERT_TRUE(parse_and_validate_tx_base_from_blob(bd, pruned_tx)); ASSERT_TRUE(pruned_tx.version == 2); diff --git a/tests/unit_tests/crypto.cpp b/tests/unit_tests/crypto.cpp index 7ac87ca7c..b550b35b0 100644 --- a/tests/unit_tests/crypto.cpp +++ b/tests/unit_tests/crypto.cpp @@ -33,6 +33,7 @@ #include <string> #include "cryptonote_basic/cryptonote_basic_impl.h" +#include "cryptonote_basic/merge_mining.h" namespace { @@ -99,3 +100,215 @@ TEST(Crypto, verify_32) } } } + +TEST(Crypto, tree_branch) +{ + crypto::hash inputs[6]; + crypto::hash branch[8]; + crypto::hash root, root2; + size_t depth; + uint32_t path, path2; + + auto hasher = [](const crypto::hash &h0, const crypto::hash &h1) -> crypto::hash + { + char buffer[64]; + memcpy(buffer, &h0, 32); + memcpy(buffer + 32, &h1, 32); + crypto::hash res; + cn_fast_hash(buffer, 64, res); + return res; + }; + + for (int n = 0; n < 6; ++n) + { + memset(&inputs[n], 0, 32); + inputs[n].data[0] = n + 1; + } + + // empty + ASSERT_FALSE(crypto::tree_branch((const char(*)[32])inputs, 0, crypto::null_hash.data, (char(*)[32])branch, &depth, &path)); + + // one, matching + ASSERT_TRUE(crypto::tree_branch((const char(*)[32])inputs, 1, inputs[0].data, (char(*)[32])branch, &depth, &path)); + ASSERT_EQ(depth, 0); + ASSERT_EQ(path, 0); + ASSERT_TRUE(crypto::tree_path(1, 0, &path2)); + ASSERT_EQ(path, path2); + crypto::tree_hash((const char(*)[32])inputs, 1, root.data); + ASSERT_EQ(root, inputs[0]); + ASSERT_TRUE(crypto::is_branch_in_tree(inputs[0].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[1].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(crypto::null_hash.data, root.data, (const char(*)[32])branch, depth, path)); + + // one, not found + ASSERT_FALSE(crypto::tree_branch((const char(*)[32])inputs, 1, inputs[1].data, (char(*)[32])branch, &depth, &path)); + + // two, index 0 + ASSERT_TRUE(crypto::tree_branch((const char(*)[32])inputs, 2, inputs[0].data, (char(*)[32])branch, &depth, &path)); + ASSERT_EQ(depth, 1); + ASSERT_EQ(path, 0); + ASSERT_TRUE(crypto::tree_path(2, 0, &path2)); + ASSERT_EQ(path, path2); + ASSERT_EQ(branch[0], inputs[1]); + crypto::tree_hash((const char(*)[32])inputs, 2, root.data); + ASSERT_EQ(root, hasher(inputs[0], inputs[1])); + ASSERT_TRUE(crypto::is_branch_in_tree(inputs[0].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[1].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[2].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(crypto::null_hash.data, root.data, (const char(*)[32])branch, depth, path)); + + // two, index 1 + ASSERT_TRUE(crypto::tree_branch((const char(*)[32])inputs, 2, inputs[1].data, (char(*)[32])branch, &depth, &path)); + ASSERT_EQ(depth, 1); + ASSERT_EQ(path, 1); + ASSERT_TRUE(crypto::tree_path(2, 1, &path2)); + ASSERT_EQ(path, path2); + ASSERT_EQ(branch[0], inputs[0]); + crypto::tree_hash((const char(*)[32])inputs, 2, root.data); + ASSERT_EQ(root, hasher(inputs[0], inputs[1])); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[0].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_TRUE(crypto::is_branch_in_tree(inputs[1].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[2].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(crypto::null_hash.data, root.data, (const char(*)[32])branch, depth, path)); + + // two, not found + ASSERT_FALSE(crypto::tree_branch((const char(*)[32])inputs, 2, inputs[2].data, (char(*)[32])branch, &depth, &path)); + + // a b c 0 + // x y + // z + + // three, index 0 + ASSERT_TRUE(crypto::tree_branch((const char(*)[32])inputs, 3, inputs[0].data, (char(*)[32])branch, &depth, &path)); + ASSERT_GE(depth, 1); + ASSERT_LE(depth, 2); + ASSERT_TRUE(crypto::tree_path(3, 0, &path2)); + ASSERT_EQ(path, path2); + crypto::tree_hash((const char(*)[32])inputs, 3, root.data); + ASSERT_EQ(root, hasher(inputs[0], hasher(inputs[1], inputs[2]))); + ASSERT_TRUE(crypto::is_branch_in_tree(inputs[0].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[1].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[2].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[3].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(crypto::null_hash.data, root.data, (const char(*)[32])branch, depth, path)); + + // three, index 1 + ASSERT_TRUE(crypto::tree_branch((const char(*)[32])inputs, 3, inputs[1].data, (char(*)[32])branch, &depth, &path)); + ASSERT_GE(depth, 1); + ASSERT_LE(depth, 2); + ASSERT_TRUE(crypto::tree_path(3, 1, &path2)); + ASSERT_EQ(path, path2); + crypto::tree_hash((const char(*)[32])inputs, 3, root.data); + ASSERT_EQ(root, hasher(inputs[0], hasher(inputs[1], inputs[2]))); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[0].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_TRUE(crypto::is_branch_in_tree(inputs[1].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[2].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[3].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(crypto::null_hash.data, root.data, (const char(*)[32])branch, depth, path)); + + // three, index 2 + ASSERT_TRUE(crypto::tree_branch((const char(*)[32])inputs, 3, inputs[2].data, (char(*)[32])branch, &depth, &path)); + ASSERT_GE(depth, 1); + ASSERT_LE(depth, 2); + ASSERT_TRUE(crypto::tree_path(3, 2, &path2)); + ASSERT_EQ(path, path2); + crypto::tree_hash((const char(*)[32])inputs, 3, root.data); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[0].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[1].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_TRUE(crypto::is_branch_in_tree(inputs[2].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[3].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(crypto::null_hash.data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_TRUE(crypto::tree_branch_hash(inputs[2].data, (const char(*)[32])branch, depth, path, root2.data)); + ASSERT_EQ(root, root2); + + // three, not found + ASSERT_FALSE(crypto::tree_branch((const char(*)[32])inputs, 3, inputs[3].data, (char(*)[32])branch, &depth, &path)); + + // a b c d e 0 0 0 + // x y + // z + // w + + // five, index 0 + ASSERT_TRUE(crypto::tree_branch((const char(*)[32])inputs, 5, inputs[0].data, (char(*)[32])branch, &depth, &path)); + ASSERT_GE(depth, 2); + ASSERT_LE(depth, 3); + ASSERT_TRUE(crypto::tree_path(5, 0, &path2)); + ASSERT_EQ(path, path2); + crypto::tree_hash((const char(*)[32])inputs, 5, root.data); + ASSERT_TRUE(crypto::is_branch_in_tree(inputs[0].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[1].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[2].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[3].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[4].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[5].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(crypto::null_hash.data, root.data, (const char(*)[32])branch, depth, path)); + + // five, index 1 + ASSERT_TRUE(crypto::tree_branch((const char(*)[32])inputs, 5, inputs[1].data, (char(*)[32])branch, &depth, &path)); + ASSERT_GE(depth, 2); + ASSERT_LE(depth, 3); + ASSERT_TRUE(crypto::tree_path(5, 1, &path2)); + ASSERT_EQ(path, path2); + crypto::tree_hash((const char(*)[32])inputs, 5, root.data); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[0].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_TRUE(crypto::is_branch_in_tree(inputs[1].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[2].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[3].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[4].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[5].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(crypto::null_hash.data, root.data, (const char(*)[32])branch, depth, path)); + + // five, index 2 + ASSERT_TRUE(crypto::tree_branch((const char(*)[32])inputs, 5, inputs[2].data, (char(*)[32])branch, &depth, &path)); + ASSERT_GE(depth, 2); + ASSERT_LE(depth, 3); + ASSERT_TRUE(crypto::tree_path(5, 2, &path2)); + ASSERT_EQ(path, path2); + crypto::tree_hash((const char(*)[32])inputs, 5, root.data); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[0].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[1].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_TRUE(crypto::is_branch_in_tree(inputs[2].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[3].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[4].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[5].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(crypto::null_hash.data, root.data, (const char(*)[32])branch, depth, path)); + + // five, index 4 + ASSERT_TRUE(crypto::tree_branch((const char(*)[32])inputs, 5, inputs[4].data, (char(*)[32])branch, &depth, &path)); + ASSERT_GE(depth, 2); + ASSERT_LE(depth, 3); + ASSERT_TRUE(crypto::tree_path(5, 4, &path2)); + ASSERT_EQ(path, path2); + crypto::tree_hash((const char(*)[32])inputs, 5, root.data); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[0].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[1].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[2].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[3].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_TRUE(crypto::is_branch_in_tree(inputs[4].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[5].data, root.data, (const char(*)[32])branch, depth, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(crypto::null_hash.data, root.data, (const char(*)[32])branch, depth, path)); + + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[4].data, root.data, (const char(*)[32])branch, depth - 1, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[4].data, root.data, (const char(*)[32])branch, depth + 1, path)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[4].data, root.data, (const char(*)[32])branch, depth, path ^ 1)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[4].data, root.data, (const char(*)[32])branch, depth, path ^ 2)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[4].data, root.data, (const char(*)[32])branch, depth, path ^ 3)); + ASSERT_FALSE(crypto::is_branch_in_tree(inputs[4].data, root.data, (const char(*)[32])(branch + 1), depth, path)); + + // five, not found + ASSERT_FALSE(crypto::tree_branch((const char(*)[32])inputs, 5, crypto::null_hash.data, (char(*)[32])branch, &depth, &path)); + + // depth encoding roundtrip + for (uint32_t n_chains = 1; n_chains <= 65; ++n_chains) + { + for (uint32_t nonce = 0; nonce < 1024; ++nonce) + { + const uint32_t depth = cryptonote::encode_mm_depth(n_chains, nonce); + uint32_t n_chains_2, nonce_2; + ASSERT_TRUE(cryptonote::decode_mm_depth(depth, n_chains_2, nonce_2)); + ASSERT_EQ(n_chains, n_chains_2); + ASSERT_EQ(nonce, nonce_2); + } + } +} diff --git a/tests/unit_tests/epee_boosted_tcp_server.cpp b/tests/unit_tests/epee_boosted_tcp_server.cpp index 71098f612..992122dd2 100644 --- a/tests/unit_tests/epee_boosted_tcp_server.cpp +++ b/tests/unit_tests/epee_boosted_tcp_server.cpp @@ -37,6 +37,7 @@ #include "include_base_utils.h" #include "string_tools.h" #include "net/abstract_tcp_server2.h" +#include "net/levin_protocol_handler_async.h" namespace { @@ -110,8 +111,7 @@ TEST(boosted_tcp_server, worker_threads_are_exception_resistant) { boost::unique_lock<boost::mutex> lock(mtx); - ASSERT_NE(boost::cv_status::timeout, cond.wait_for(lock, boost::chrono::seconds(5))); - ASSERT_EQ(4, counter); + ASSERT_TRUE(cond.wait_for(lock, boost::chrono::seconds(5), [&counter]{ return counter == 4; })); } // Check if threads are alive @@ -124,11 +124,341 @@ TEST(boosted_tcp_server, worker_threads_are_exception_resistant) { boost::unique_lock<boost::mutex> lock(mtx); - ASSERT_NE(boost::cv_status::timeout, cond.wait_for(lock, boost::chrono::seconds(5))); - ASSERT_EQ(4, counter); + ASSERT_TRUE(cond.wait_for(lock, boost::chrono::seconds(5), [&counter]{ return counter == 4; })); } srv.send_stop_signal(); ASSERT_TRUE(srv.timed_wait_server_stop(5 * 1000)); ASSERT_TRUE(srv.deinit_server()); } + + +TEST(test_epee_connection, test_lifetime) +{ + struct context_t: epee::net_utils::connection_context_base { + static constexpr size_t get_max_bytes(int) noexcept { return -1; } + static constexpr int handshake_command() noexcept { return 1001; } + static constexpr bool handshake_complete() noexcept { return true; } + }; + + using functional_obj_t = std::function<void ()>; + struct command_handler_t: epee::levin::levin_commands_handler<context_t> { + size_t delay; + functional_obj_t on_connection_close_f; + command_handler_t(size_t delay = 0, + functional_obj_t on_connection_close_f = nullptr + ): + delay(delay), + on_connection_close_f(on_connection_close_f) + {} + virtual int invoke(int, const epee::span<const uint8_t>, epee::byte_stream&, context_t&) override { epee::misc_utils::sleep_no_w(delay); return {}; } + virtual int notify(int, const epee::span<const uint8_t>, context_t&) override { return {}; } + virtual void callback(context_t&) override {} + virtual void on_connection_new(context_t&) override {} + virtual void on_connection_close(context_t&) override { + if (on_connection_close_f) + on_connection_close_f(); + } + virtual ~command_handler_t() override {} + static void destroy(epee::levin::levin_commands_handler<context_t>* ptr) { delete ptr; } + }; + + using handler_t = epee::levin::async_protocol_handler<context_t>; + using connection_t = epee::net_utils::connection<handler_t>; + using connection_ptr = boost::shared_ptr<connection_t>; + using shared_state_t = typename connection_t::shared_state; + using shared_state_ptr = std::shared_ptr<shared_state_t>; + using shared_states_t = std::vector<shared_state_ptr>; + using tag_t = boost::uuids::uuid; + using tags_t = std::vector<tag_t>; + using io_context_t = boost::asio::io_service; + using endpoint_t = boost::asio::ip::tcp::endpoint; + using work_t = boost::asio::io_service::work; + using work_ptr = std::shared_ptr<work_t>; + using workers_t = std::vector<std::thread>; + using server_t = epee::net_utils::boosted_tcp_server<handler_t>; + using lock_t = std::mutex; + using lock_guard_t = std::lock_guard<lock_t>; + using connection_weak_ptr = boost::weak_ptr<connection_t>; + struct shared_conn_t { + lock_t lock; + connection_weak_ptr conn; + }; + using shared_conn_ptr = std::shared_ptr<shared_conn_t>; + + io_context_t io_context; + work_ptr work(std::make_shared<work_t>(io_context)); + + workers_t workers; + while (workers.size() < 4) { + workers.emplace_back([&io_context]{ + io_context.run(); + }); + } + + endpoint_t endpoint(boost::asio::ip::address::from_string("127.0.0.1"), 5262); + server_t server(epee::net_utils::e_connection_type_P2P); + server.init_server(endpoint.port(), + endpoint.address().to_string(), + 0, + "", + false, + true, + epee::net_utils::ssl_support_t::e_ssl_support_disabled + ); + server.run_server(2, false); + server.get_config_shared()->set_handler(new command_handler_t, &command_handler_t::destroy); + + io_context.post([&io_context, &work, &endpoint, &server]{ + auto scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&work]{ + work.reset(); + }); + + shared_state_ptr shared_state(std::make_shared<shared_state_t>()); + shared_state->set_handler(new command_handler_t, &command_handler_t::destroy); + + auto create_connection = [&io_context, &endpoint, &shared_state] { + connection_ptr conn(new connection_t(io_context, shared_state, {}, {})); + conn->socket().connect(endpoint); + conn->start({}, {}); + context_t context; + conn->get_context(context); + auto tag = context.m_connection_id; + return tag; + }; + + ASSERT_TRUE(shared_state->get_connections_count() == 0); + auto tag = create_connection(); + ASSERT_TRUE(shared_state->get_connections_count() == 1); + bool success = shared_state->for_connection(tag, [shared_state](context_t& context){ + shared_state->close(context.m_connection_id); + context.m_remote_address.get_zone(); + return true; + }); + ASSERT_TRUE(success); + + ASSERT_TRUE(shared_state->get_connections_count() == 0); + constexpr auto N = 8; + tags_t tags(N); + for(auto &t: tags) + t = create_connection(); + ASSERT_TRUE(shared_state->get_connections_count() == N); + size_t index = 0; + success = shared_state->foreach_connection([&index, shared_state, &tags, &create_connection](context_t& context){ + if (!index) + for (const auto &t: tags) + shared_state->close(t); + + shared_state->close(context.m_connection_id); + context.m_remote_address.get_zone(); + ++index; + + for(auto i = 0; i < N; ++i) + create_connection(); + return true; + }); + ASSERT_TRUE(success); + ASSERT_TRUE(index == N); + ASSERT_TRUE(shared_state->get_connections_count() == N * N); + + index = 0; + success = shared_state->foreach_connection([&index, shared_state](context_t& context){ + shared_state->close(context.m_connection_id); + context.m_remote_address.get_zone(); + ++index; + return true; + }); + ASSERT_TRUE(success); + ASSERT_TRUE(index == N * N); + ASSERT_TRUE(shared_state->get_connections_count() == 0); + + while (shared_state->sock_count); + ASSERT_TRUE(shared_state->get_connections_count() == 0); + constexpr auto DELAY = 30; + constexpr auto TIMEOUT = 1; + server.get_config_shared()->set_handler(new command_handler_t(DELAY), &command_handler_t::destroy); + for (auto i = 0; i < N; ++i) { + tag = create_connection(); + ASSERT_TRUE(shared_state->get_connections_count() == 1); + success = shared_state->invoke_async(1, epee::levin::message_writer{}, tag, [](int, const epee::span<const uint8_t>, context_t&){}, TIMEOUT); + ASSERT_TRUE(success); + while (shared_state->sock_count == 1) { + success = shared_state->foreach_connection([&shared_state, &tag](context_t&){ + return shared_state->request_callback(tag); + }); + ASSERT_TRUE(success); + } + shared_state->close(tag); + ASSERT_TRUE(shared_state->get_connections_count() == 0); + } + + while (shared_state->sock_count); + constexpr auto ZERO_DELAY = 0; + size_t counter = 0; + shared_state->set_handler(new command_handler_t(ZERO_DELAY, + [&counter]{ + ASSERT_TRUE(counter++ == 0); + } + ), + &command_handler_t::destroy + ); + connection_ptr conn(new connection_t(io_context, shared_state, {}, {})); + conn->socket().connect(endpoint); + conn->start({}, {}); + ASSERT_TRUE(shared_state->get_connections_count() == 1); + shared_state->del_out_connections(1); + ASSERT_TRUE(shared_state->get_connections_count() == 0); + conn.reset(); + + while (shared_state->sock_count); + shared_conn_ptr shared_conn(std::make_shared<shared_conn_t>()); + shared_state->set_handler(new command_handler_t(ZERO_DELAY, + [shared_state, shared_conn]{ + { + connection_ptr conn; + { + lock_guard_t guard(shared_conn->lock); + conn = shared_conn->conn.lock(); + shared_conn->conn.reset(); + } + if (conn) + conn->cancel(); + } + const auto success = shared_state->foreach_connection([](context_t&){ + return true; + }); + ASSERT_TRUE(success); + } + ), + &command_handler_t::destroy + ); + for (auto i = 0; i < N; ++i) { + { + connection_ptr conn(new connection_t(io_context, shared_state, {}, {})); + conn->socket().connect(endpoint); + conn->start({}, {}); + lock_guard_t guard(shared_conn->lock); + shared_conn->conn = conn; + } + ASSERT_TRUE(shared_state->get_connections_count() == 1); + shared_state->del_out_connections(1); + ASSERT_TRUE(shared_state->get_connections_count() == 0); + } + + shared_states_t shared_states; + while (shared_states.size() < 2) { + shared_states.emplace_back(std::make_shared<shared_state_t>()); + shared_states.back()->set_handler(new command_handler_t(ZERO_DELAY, + [&shared_states]{ + for (auto &s: shared_states) { + auto success = s->foreach_connection([](context_t&){ + return true; + }); + ASSERT_TRUE(success); + } + } + ), + &command_handler_t::destroy + ); + } + workers_t workers; + + for (auto &s: shared_states) { + workers.emplace_back([&io_context, &s, &endpoint]{ + for (auto i = 0; i < N * N; ++i) { + connection_ptr conn(new connection_t(io_context, s, {}, {})); + conn->socket().connect(endpoint); + conn->start({}, {}); + io_context.post([conn]{ + conn->cancel(); + }); + conn.reset(); + s->del_out_connections(1); + while (s->sock_count); + } + }); + } + for (;workers.size(); workers.pop_back()) + workers.back().join(); + + for (auto &s: shared_states) { + workers.emplace_back([&io_context, &s, &endpoint]{ + for (auto i = 0; i < N * N; ++i) { + connection_ptr conn(new connection_t(io_context, s, {}, {})); + conn->socket().connect(endpoint); + conn->start({}, {}); + conn->cancel(); + while (conn.use_count() > 1); + s->foreach_connection([&io_context, &s, &endpoint, &conn](context_t& context){ + conn.reset(new connection_t(io_context, s, {}, {})); + conn->socket().connect(endpoint); + conn->start({}, {}); + conn->cancel(); + while (conn.use_count() > 1); + conn.reset(); + return true; + }); + while (s->sock_count); + } + }); + } + for (;workers.size(); workers.pop_back()) + workers.back().join(); + + for (auto &s: shared_states) { + workers.emplace_back([&io_context, &s, &endpoint]{ + for (auto i = 0; i < N; ++i) { + connection_ptr conn(new connection_t(io_context, s, {}, {})); + conn->socket().connect(endpoint); + conn->start({}, {}); + context_t context; + conn->get_context(context); + auto tag = context.m_connection_id; + conn->cancel(); + while (conn.use_count() > 1); + s->for_connection(tag, [&io_context, &s, &endpoint, &conn](context_t& context){ + conn.reset(new connection_t(io_context, s, {}, {})); + conn->socket().connect(endpoint); + conn->start({}, {}); + conn->cancel(); + while (conn.use_count() > 1); + conn.reset(); + return true; + }); + while (s->sock_count); + } + }); + } + for (;workers.size(); workers.pop_back()) + workers.back().join(); + + for (auto &s: shared_states) { + workers.emplace_back([&io_context, &s, &endpoint]{ + for (auto i = 0; i < N; ++i) { + connection_ptr conn(new connection_t(io_context, s, {}, {})); + conn->socket().connect(endpoint); + conn->start({}, {}); + context_t context; + conn->get_context(context); + auto tag = context.m_connection_id; + io_context.post([conn]{ + conn->cancel(); + }); + conn.reset(); + s->close(tag); + while (s->sock_count); + } + }); + } + for (;workers.size(); workers.pop_back()) + workers.back().join(); + + }); + + for (auto& w: workers) { + w.join(); + } + server.send_stop_signal(); + server.timed_wait_server_stop(5 * 1000); + server.deinit_server(); +} diff --git a/tests/unit_tests/epee_levin_protocol_handler_async.cpp b/tests/unit_tests/epee_levin_protocol_handler_async.cpp index a499fa608..d9de99b8b 100644 --- a/tests/unit_tests/epee_levin_protocol_handler_async.cpp +++ b/tests/unit_tests/epee_levin_protocol_handler_async.cpp @@ -59,13 +59,13 @@ namespace { } - virtual int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, test_levin_connection_context& context) + virtual int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_stream& buff_out, test_levin_connection_context& context) { m_invoke_counter.inc(); boost::unique_lock<boost::mutex> lock(m_mutex); m_last_command = command; m_last_in_buf = std::string((const char*)in_buff.data(), in_buff.size()); - buff_out = m_invoke_out_buf.clone(); + buff_out.write(epee::to_span(m_invoke_out_buf)); return m_return_code; } @@ -434,8 +434,11 @@ TEST_F(positive_test_connection_to_levin_protocol_handler_calls, handler_process const int expected_command = 4673261; const std::string in_data(256, 'e'); + epee::levin::message_writer message{}; + message.buffer.write(epee::to_span(in_data)); + const epee::byte_slice noise = epee::levin::make_noise_notify(1024); - const epee::byte_slice notify = epee::levin::make_notify(expected_command, epee::strspan<std::uint8_t>(in_data)); + const epee::byte_slice notify = message.finalize_notify(expected_command); test_connection_ptr conn = create_connection(); @@ -468,11 +471,16 @@ TEST_F(positive_test_connection_to_levin_protocol_handler_calls, handler_process const int expected_command = 4673261; const int expected_fragmented_command = 46732; const std::string in_data(256, 'e'); - std::string in_fragmented_data(1024 * 4, 'c'); + + epee::levin::message_writer message{}; + message.buffer.write(epee::to_span(in_data)); + + epee::levin::message_writer in_fragmented_data; + in_fragmented_data.buffer.put_n('c', 1024 * 4); const epee::byte_slice noise = epee::levin::make_noise_notify(1024); - const epee::byte_slice notify = epee::levin::make_notify(expected_command, epee::strspan<std::uint8_t>(in_data)); - epee::byte_slice fragmented = epee::levin::make_fragmented_notify(noise, expected_fragmented_command, epee::strspan<std::uint8_t>(in_fragmented_data)); + const epee::byte_slice notify = message.finalize_notify(expected_command); + epee::byte_slice fragmented = epee::levin::make_fragmented_notify(noise.size(), expected_fragmented_command, std::move(in_fragmented_data)); EXPECT_EQ(5u, fragmented.size() / 1024); EXPECT_EQ(0u, fragmented.size() % 1024); @@ -497,11 +505,13 @@ TEST_F(positive_test_connection_to_levin_protocol_handler_calls, handler_process ASSERT_TRUE(conn->m_protocol_handler.handle_recv(next.data(), next.size())); } - in_fragmented_data.resize(((1024 - sizeof(epee::levin::bucket_head2)) * 5) - sizeof(epee::levin::bucket_head2)); // add padding zeroes + std::string compare_buffer(1024 * 4, 'c'); + compare_buffer.resize(((1024 - sizeof(epee::levin::bucket_head2)) * 5) - sizeof(epee::levin::bucket_head2)); // add padding zeroes + ASSERT_EQ(4u, m_commands_handler.notify_counter()); ASSERT_EQ(0u, m_commands_handler.invoke_counter()); ASSERT_EQ(expected_fragmented_command, m_commands_handler.last_command()); - ASSERT_EQ(in_fragmented_data, m_commands_handler.last_in_buf()); + ASSERT_EQ(compare_buffer, m_commands_handler.last_in_buf()); ASSERT_EQ(0u, conn->send_counter()); ASSERT_TRUE(conn->last_send_data().empty()); diff --git a/tests/unit_tests/epee_serialization.cpp b/tests/unit_tests/epee_serialization.cpp index cade16f0d..95a2b6ecd 100644 --- a/tests/unit_tests/epee_serialization.cpp +++ b/tests/unit_tests/epee_serialization.cpp @@ -30,6 +30,7 @@ #include <gtest/gtest.h> #include "storages/portable_storage.h" +#include "span.h" TEST(epee_binary, two_keys) { diff --git a/tests/unit_tests/epee_utils.cpp b/tests/unit_tests/epee_utils.cpp index 256f8c3c2..b83a809b7 100644 --- a/tests/unit_tests/epee_utils.cpp +++ b/tests/unit_tests/epee_utils.cpp @@ -1115,11 +1115,13 @@ TEST(ByteStream, ToByteSlice) epee::byte_stream stream; + stream.reserve(128*1024); stream.write(source); EXPECT_EQ(sizeof(source), stream.size()); + EXPECT_EQ(128*1024, stream.capacity()); EXPECT_TRUE(equal(source, byte_span{stream.data(), stream.size()})); - const epee::byte_slice slice{std::move(stream)}; + const epee::byte_slice slice{std::move(stream), true}; EXPECT_EQ(0u, stream.size()); EXPECT_EQ(0u, stream.available()); EXPECT_EQ(0u, stream.capacity()); @@ -1510,6 +1512,7 @@ TEST(NetUtils, NetworkAddress) constexpr static epee::net_utils::address_type get_type_id() noexcept { return epee::net_utils::address_type(-1); } constexpr static epee::net_utils::zone get_zone() noexcept { return epee::net_utils::zone::invalid; } constexpr static bool is_blockable() noexcept { return false; } + constexpr static uint16_t port() { return 0; } }; const epee::net_utils::network_address empty; diff --git a/tests/unit_tests/get_xtype_from_string.cpp b/tests/unit_tests/get_xtype_from_string.cpp index 78bbe6657..17492ed19 100644 --- a/tests/unit_tests/get_xtype_from_string.cpp +++ b/tests/unit_tests/get_xtype_from_string.cpp @@ -31,6 +31,7 @@ #include "gtest/gtest.h" #include <string_tools.h> +#include <string_tools_lexical.h> using namespace epee::string_tools; diff --git a/tests/unit_tests/hardfork.cpp b/tests/unit_tests/hardfork.cpp index 6d7e9a8aa..912651de4 100644 --- a/tests/unit_tests/hardfork.cpp +++ b/tests/unit_tests/hardfork.cpp @@ -432,7 +432,6 @@ TEST(voting, info) // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 static const uint8_t block_versions[] = { 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4 }; - static const uint8_t expected_versions[] = { 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4 }; static const uint8_t expected_thresholds[] = { 0, 1, 1, 2, 2, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 4, 2, 2, 2, 2 }; for (uint64_t h = 0; h < sizeof(block_versions) / sizeof(block_versions[0]); ++h) { diff --git a/tests/unit_tests/json_serialization.cpp b/tests/unit_tests/json_serialization.cpp index f76199e57..9fa589139 100644 --- a/tests/unit_tests/json_serialization.cpp +++ b/tests/unit_tests/json_serialization.cpp @@ -51,7 +51,7 @@ namespace test if (!cryptonote::find_tx_extra_field_by_type(extra_fields, key_field)) throw std::runtime_error{"invalid transaction"}; - for (auto const& input : boost::adaptors::index(source.vout)) + for (auto const input : boost::adaptors::index(source.vout)) { source_amount += input.value().amount; auto const& key = boost::get<cryptonote::txout_to_key>(input.value().target); diff --git a/tests/unit_tests/levin.cpp b/tests/unit_tests/levin.cpp index 53a7f7b67..30d6f8133 100644 --- a/tests/unit_tests/levin.cpp +++ b/tests/unit_tests/levin.cpp @@ -245,9 +245,9 @@ namespace return out; } - virtual int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_slice& buff_out, cryptonote::levin::detail::p2p_context& context) override final + virtual int invoke(int command, const epee::span<const uint8_t> in_buff, epee::byte_stream& buff_out, cryptonote::levin::detail::p2p_context& context) override final { - buff_out = nullptr; + buff_out.clear(); invoked_.push_back( {context.m_connection_id, command, std::string{reinterpret_cast<const char*>(in_buff.data()), in_buff.size()}} ); @@ -384,21 +384,50 @@ TEST(make_header, expect_return) EXPECT_EQ(0u, header1.m_flags); } -TEST(make_notify, empty_payload) +TEST(message_writer, invoke_with_empty_payload) { - const epee::byte_slice message = epee::levin::make_notify(443, nullptr); + const epee::byte_slice message = epee::levin::message_writer{}.finalize_invoke(443); + const epee::levin::bucket_head2 header = + epee::levin::make_header(443, 0, LEVIN_PACKET_REQUEST, true); + ASSERT_EQ(sizeof(header), message.size()); + EXPECT_TRUE(std::memcmp(std::addressof(header), message.data(), sizeof(header)) == 0); +} + +TEST(message_writer, invoke_with_payload) +{ + std::string bytes(100, 'a'); + std::generate(bytes.begin(), bytes.end(), crypto::random_device{}); + + epee::levin::message_writer writer{}; + writer.buffer.write(epee::to_span(bytes)); + + const epee::byte_slice message = writer.finalize_invoke(443); + const epee::levin::bucket_head2 header = + epee::levin::make_header(443, bytes.size(), LEVIN_PACKET_REQUEST, true); + + ASSERT_EQ(sizeof(header) + bytes.size(), message.size()); + EXPECT_TRUE(std::memcmp(std::addressof(header), message.data(), sizeof(header)) == 0); + EXPECT_TRUE(std::memcmp(bytes.data(), message.data() + sizeof(header), bytes.size()) == 0); +} + +TEST(message_writer, notify_with_empty_payload) +{ + const epee::byte_slice message = epee::levin::message_writer{}.finalize_notify(443); const epee::levin::bucket_head2 header = epee::levin::make_header(443, 0, LEVIN_PACKET_REQUEST, false); ASSERT_EQ(sizeof(header), message.size()); EXPECT_TRUE(std::memcmp(std::addressof(header), message.data(), sizeof(header)) == 0); } -TEST(make_notify, with_payload) +TEST(message_writer, notify_with_payload) { std::string bytes(100, 'a'); std::generate(bytes.begin(), bytes.end(), crypto::random_device{}); - const epee::byte_slice message = epee::levin::make_notify(443, epee::strspan<std::uint8_t>(bytes)); + epee::levin::message_writer writer{}; + writer.buffer.write(epee::to_span(bytes)); + + const epee::byte_slice message = writer.finalize_notify(443); const epee::levin::bucket_head2 header = epee::levin::make_header(443, bytes.size(), LEVIN_PACKET_REQUEST, false); @@ -407,6 +436,44 @@ TEST(make_notify, with_payload) EXPECT_TRUE(std::memcmp(bytes.data(), message.data() + sizeof(header), bytes.size()) == 0); } +TEST(message_writer, response_with_empty_payload) +{ + const epee::byte_slice message = epee::levin::message_writer{}.finalize_response(443, 1); + epee::levin::bucket_head2 header = + epee::levin::make_header(443, 0, LEVIN_PACKET_RESPONSE, false); + header.m_return_code = SWAP32LE(1); + ASSERT_EQ(sizeof(header), message.size()); + EXPECT_TRUE(std::memcmp(std::addressof(header), message.data(), sizeof(header)) == 0); +} + +TEST(message_writer, response_with_payload) +{ + std::string bytes(100, 'a'); + std::generate(bytes.begin(), bytes.end(), crypto::random_device{}); + + epee::levin::message_writer writer{}; + writer.buffer.write(epee::to_span(bytes)); + + const epee::byte_slice message = writer.finalize_response(443, 6450); + epee::levin::bucket_head2 header = + epee::levin::make_header(443, bytes.size(), LEVIN_PACKET_RESPONSE, false); + header.m_return_code = SWAP32LE(6450); + + ASSERT_EQ(sizeof(header) + bytes.size(), message.size()); + EXPECT_TRUE(std::memcmp(std::addressof(header), message.data(), sizeof(header)) == 0); + EXPECT_TRUE(std::memcmp(bytes.data(), message.data() + sizeof(header), bytes.size()) == 0); +} + +TEST(message_writer, error) +{ + epee::levin::message_writer writer{}; + writer.buffer.clear(); + + EXPECT_THROW(writer.finalize_invoke(0), std::runtime_error); + EXPECT_THROW(writer.finalize_notify(0), std::runtime_error); + EXPECT_THROW(writer.finalize_response(0, 0), std::runtime_error); +} + TEST(make_noise, invalid) { EXPECT_TRUE(epee::levin::make_noise_notify(sizeof(epee::levin::bucket_head2) - 1).empty()); @@ -428,13 +495,13 @@ TEST(make_noise, valid) TEST(make_fragment, invalid) { - EXPECT_TRUE(epee::levin::make_fragmented_notify(nullptr, 0, nullptr).empty()); + EXPECT_TRUE(epee::levin::make_fragmented_notify(0, 0, epee::levin::message_writer{}).empty()); } TEST(make_fragment, single) { const epee::byte_slice noise = epee::levin::make_noise_notify(1024); - const epee::byte_slice fragment = epee::levin::make_fragmented_notify(noise, 11, nullptr); + const epee::byte_slice fragment = epee::levin::make_fragmented_notify(noise.size(), 11, epee::levin::message_writer{}); const epee::levin::bucket_head2 header = epee::levin::make_header(11, 1024 - sizeof(epee::levin::bucket_head2), LEVIN_PACKET_REQUEST, false); @@ -449,8 +516,13 @@ TEST(make_fragment, multiple) std::string bytes(1024 * 3 - 150, 'a'); std::generate(bytes.begin(), bytes.end(), crypto::random_device{}); + epee::levin::message_writer message; + message.buffer.write(epee::to_span(bytes)); + const epee::byte_slice noise = epee::levin::make_noise_notify(1024); - epee::byte_slice fragment = epee::levin::make_fragmented_notify(noise, 114, epee::strspan<std::uint8_t>(bytes)); + epee::byte_slice fragment = epee::levin::make_fragmented_notify(noise.size(), 114, std::move(message)); + + EXPECT_EQ(1024 * 3, fragment.size()); epee::levin::bucket_head2 header = epee::levin::make_header(0, 1024 - sizeof(epee::levin::bucket_head2), LEVIN_PACKET_BEGIN, false); @@ -497,6 +569,7 @@ TEST(make_fragment, multiple) fragment.take_slice(bytes.size()); + EXPECT_EQ(18, fragment.size()); EXPECT_EQ(18, std::count(fragment.cbegin(), fragment.cend(), 0)); } @@ -664,8 +737,9 @@ TEST_F(levin_notify, stem_no_outs_without_padding) ASSERT_LT(0u, io_service_.poll()); EXPECT_EQ(txs, events_.take_relayed(cryptonote::relay_method::fluff)); if (events_.has_stem_txes()) + { EXPECT_EQ(txs, events_.take_relayed(cryptonote::relay_method::stem)); - + } notifier.run_fluff(); ASSERT_LT(0u, io_service_.poll()); @@ -1034,7 +1108,9 @@ TEST_F(levin_notify, stem_no_outs_with_padding) ASSERT_LT(0u, io_service_.poll()); EXPECT_EQ(txs, events_.take_relayed(cryptonote::relay_method::fluff)); if (events_.has_stem_txes()) + { EXPECT_EQ(txs, events_.take_relayed(cryptonote::relay_method::stem)); + } notifier.run_fluff(); ASSERT_LT(0u, io_service_.poll()); @@ -2161,20 +2237,31 @@ TEST_F(levin_notify, command_max_bytes) add_connection(true); - std::string bytes(4096, 'h'); + std::string payload(4096, 'h'); + epee::byte_slice bytes; + { + epee::levin::message_writer dest{}; + dest.buffer.write(epee::to_span(payload)); + bytes = dest.finalize_notify(ping_command); + } - EXPECT_EQ(1, get_connections().notify(ping_command, epee::strspan<std::uint8_t>(bytes), contexts_.front().get_id())); + EXPECT_EQ(1, get_connections().send(bytes.clone(), contexts_.front().get_id())); EXPECT_EQ(1u, contexts_.front().process_send_queue(true)); EXPECT_EQ(1u, receiver_.notified_size()); const received_message msg = receiver_.get_raw_notification(); EXPECT_EQ(ping_command, msg.command); EXPECT_EQ(contexts_.front().get_id(), msg.connection); - EXPECT_EQ(bytes, msg.payload); + EXPECT_EQ(payload, msg.payload); - bytes.push_back('e'); + { + payload.push_back('h'); + epee::levin::message_writer dest{}; + dest.buffer.write(epee::to_span(payload)); + bytes = dest.finalize_notify(ping_command); + } - EXPECT_EQ(1, get_connections().notify(ping_command, epee::strspan<std::uint8_t>(bytes), contexts_.front().get_id())); + EXPECT_EQ(1, get_connections().send(std::move(bytes), contexts_.front().get_id())); EXPECT_EQ(1u, contexts_.front().process_send_queue(false)); EXPECT_EQ(0u, receiver_.notified_size()); } diff --git a/tests/unit_tests/logging.cpp b/tests/unit_tests/logging.cpp index b3afba243..f11b17412 100644 --- a/tests/unit_tests/logging.cpp +++ b/tests/unit_tests/logging.cpp @@ -195,3 +195,16 @@ TEST(logging, multiline) cleanup(); } +// These operations might segfault +TEST(logging, copy_ctor_segfault) +{ + const el::Logger log1("id1", nullptr); + const el::Logger log2(log1); +} + +TEST(logging, operator_equals_segfault) +{ + const el::Logger log1("id1", nullptr); + el::Logger log2("id2", nullptr); + log2 = log1; +} diff --git a/tests/unit_tests/multiexp.cpp b/tests/unit_tests/multiexp.cpp index 722c568da..212aa0e40 100644 --- a/tests/unit_tests/multiexp.cpp +++ b/tests/unit_tests/multiexp.cpp @@ -260,7 +260,7 @@ TEST(multiexp, scalarmult_triple) rct::key res; ge_p3 Gp3; - ge_frombytes_vartime(&Gp3, rct::G.bytes); + ASSERT_EQ(ge_frombytes_vartime(&Gp3, rct::G.bytes), 0); static const rct::key scalars[] = { rct::Z, diff --git a/tests/unit_tests/net.cpp b/tests/unit_tests/net.cpp index dffda5e4e..2836fd948 100644 --- a/tests/unit_tests/net.cpp +++ b/tests/unit_tests/net.cpp @@ -1351,7 +1351,7 @@ TEST(dandelionpp_map, dropped_connection) } EXPECT_EQ(3u, used.size()); - for (const std::pair<boost::uuids::uuid, std::size_t>& entry : used) + for (const std::pair<const boost::uuids::uuid, std::size_t>& entry : used) EXPECT_EQ(3u, entry.second); for (const boost::uuids::uuid& connection : in_connections) @@ -1409,7 +1409,7 @@ TEST(dandelionpp_map, dropped_connection) } EXPECT_EQ(3u, used.size()); - for (const std::pair<boost::uuids::uuid, std::size_t>& entry : used) + for (const std::pair<const boost::uuids::uuid, std::size_t>& entry : used) EXPECT_EQ(3u, entry.second); } { @@ -1472,7 +1472,7 @@ TEST(dandelionpp_map, dropped_connection_remapped) } EXPECT_EQ(3u, used.size()); - for (const std::pair<boost::uuids::uuid, std::size_t>& entry : used) + for (const std::pair<const boost::uuids::uuid, std::size_t>& entry : used) EXPECT_EQ(3u, entry.second); for (const boost::uuids::uuid& connection : in_connections) @@ -1511,7 +1511,7 @@ TEST(dandelionpp_map, dropped_connection_remapped) } EXPECT_EQ(2u, used.size()); - for (const std::pair<boost::uuids::uuid, std::size_t>& entry : used) + for (const std::pair<const boost::uuids::uuid, std::size_t>& entry : used) EXPECT_EQ(5u, entry.second); } // select 3 of 3 connections but do not remap existing links @@ -1532,7 +1532,7 @@ TEST(dandelionpp_map, dropped_connection_remapped) } EXPECT_EQ(2u, used.size()); - for (const std::pair<boost::uuids::uuid, std::size_t>& entry : used) + for (const std::pair<const boost::uuids::uuid, std::size_t>& entry : used) EXPECT_EQ(5u, entry.second); } // map 8 new incoming connections across 3 outgoing links @@ -1555,7 +1555,7 @@ TEST(dandelionpp_map, dropped_connection_remapped) } EXPECT_EQ(3u, used.size()); - for (const std::pair<boost::uuids::uuid, std::size_t>& entry : used) + for (const std::pair<const boost::uuids::uuid, std::size_t>& entry : used) EXPECT_EQ(6u, entry.second); } } @@ -1609,7 +1609,7 @@ TEST(dandelionpp_map, dropped_all_connections) } EXPECT_EQ(3u, used.size()); - for (const std::pair<boost::uuids::uuid, std::size_t>& entry : used) + for (const std::pair<const boost::uuids::uuid, std::size_t>& entry : used) EXPECT_EQ(3u, entry.second); for (const boost::uuids::uuid& connection : in_connections) @@ -1641,7 +1641,7 @@ TEST(dandelionpp_map, dropped_all_connections) } EXPECT_EQ(3u, used.size()); - for (const std::pair<boost::uuids::uuid, std::size_t>& entry : used) + for (const std::pair<const boost::uuids::uuid, std::size_t>& entry : used) EXPECT_EQ(3u, entry.second); } } diff --git a/tests/unit_tests/node_server.cpp b/tests/unit_tests/node_server.cpp index 4d6f09e69..cab600b3d 100644 --- a/tests/unit_tests/node_server.cpp +++ b/tests/unit_tests/node_server.cpp @@ -250,7 +250,6 @@ TEST(ban, subnet) TEST(ban, ignores_port) { - time_t seconds; test_core pr_core; cryptonote::t_cryptonote_protocol_handler<test_core> cprotocol(pr_core, NULL); Server server(cprotocol); @@ -294,6 +293,18 @@ TEST(node_server, bind_same_p2p_port) boost::program_options::variables_map vm; boost::program_options::store(boost::program_options::parse_command_line(1, argv, desc_options), vm); + /* + Reason for choosing '127.0.0.2' as the IP: + + A TCP local socket address that has been bound is unavailable for some time after closing, unless the SO_REUSEADDR flag has been set. + That's why connections with automatically assigned source port 48080/58080 from previous test blocks the next to bind acceptor + so solution is to either set reuse_addr option for each socket in all tests + or use ip different from localhost for acceptors in order to not interfere with automatically assigned source endpoints + + Relevant part about REUSEADDR from man: + https://www.man7.org/linux/man-pages/man7/ip.7.html + */ + vm.find(nodetool::arg_p2p_bind_ip.name)->second = boost::program_options::variable_value(std::string("127.0.0.2"), false); vm.find(nodetool::arg_p2p_bind_port.name)->second = boost::program_options::variable_value(std::string(port), false); boost::program_options::notify(vm); @@ -311,5 +322,589 @@ TEST(node_server, bind_same_p2p_port) EXPECT_TRUE(init(new_node(), port_another)); } +TEST(cryptonote_protocol_handler, race_condition) +{ + struct contexts { + using basic = epee::net_utils::connection_context_base; + using cryptonote = cryptonote::cryptonote_connection_context; + using p2p = nodetool::p2p_connection_context_t<cryptonote>; + }; + using context_t = contexts::p2p; + using handler_t = epee::levin::async_protocol_handler<context_t>; + using connection_t = epee::net_utils::connection<handler_t>; + using connection_ptr = boost::shared_ptr<connection_t>; + using connections_t = std::vector<connection_ptr>; + using shared_state_t = typename connection_t::shared_state; + using shared_state_ptr = std::shared_ptr<shared_state_t>; + using io_context_t = boost::asio::io_service; + using event_t = epee::simple_event; + using ec_t = boost::system::error_code; + auto create_conn_pair = [](connection_ptr in, connection_ptr out) { + using endpoint_t = boost::asio::ip::tcp::endpoint; + using acceptor_t = boost::asio::ip::tcp::acceptor; + io_context_t io_context; + endpoint_t endpoint(boost::asio::ip::address::from_string("127.0.0.1"), 5262); + acceptor_t acceptor(io_context); + ec_t ec; + acceptor.open(endpoint.protocol(), ec); + EXPECT_EQ(ec.value(), 0); + acceptor.set_option(boost::asio::ip::tcp::acceptor::reuse_address(true)); + acceptor.bind(endpoint, ec); + EXPECT_EQ(ec.value(), 0); + acceptor.listen(boost::asio::socket_base::max_connections, ec); + EXPECT_EQ(ec.value(), 0); + out->socket().open(endpoint.protocol(), ec); + EXPECT_EQ(ec.value(), 0); + acceptor.async_accept(in->socket(), [](const ec_t &ec){}); + out->socket().async_connect(endpoint, [](const ec_t &ec){}); + io_context.run(); + acceptor.close(ec); + EXPECT_EQ(ec.value(), 0); + EXPECT_TRUE(in->start(true, true)); + EXPECT_TRUE(out->start(false, true)); + return std::make_pair<>(std::move(in), std::move(out)); + }; + auto get_conn_tag = [](connection_t &conn){ + context_t context; + conn.get_context(context); + return context.m_connection_id; + }; + using work_t = boost::asio::io_service::work; + using work_ptr = std::shared_ptr<work_t>; + using workers_t = std::vector<std::thread>; + using commands_handler_t = epee::levin::levin_commands_handler<context_t>; + using p2p_endpoint_t = nodetool::i_p2p_endpoint<contexts::cryptonote>; + using core_t = cryptonote::core; + using core_ptr = std::unique_ptr<core_t>; + using core_protocol_t = cryptonote::t_cryptonote_protocol_handler<core_t>; + using core_protocol_ptr = std::shared_ptr<core_protocol_t>; + using block_t = cryptonote::block; + using diff_t = cryptonote::difficulty_type; + using reward_t = uint64_t; + using height_t = uint64_t; + struct span { + using blocks = epee::span<const block_t>; + }; + auto get_block_template = []( + core_t &core, + block_t &block, + diff_t &diff, + reward_t &reward + ){ + auto &storage = core.get_blockchain_storage(); + const auto height = storage.get_current_blockchain_height(); + const auto hardfork = storage.get_current_hard_fork_version(); + block.major_version = hardfork; + block.minor_version = storage.get_ideal_hard_fork_version(); + block.prev_id = storage.get_tail_id(); + auto &db = storage.get_db(); + block.timestamp = db.get_top_block_timestamp(); + block.nonce = 0xACAB; + block.miner_tx.vin.clear(); + block.miner_tx.vout.clear(); + block.miner_tx.extra.clear(); + block.miner_tx.version = hardfork >= 4 ? 2 : 1; + block.miner_tx.unlock_time = height + CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW; + block.miner_tx.vin.push_back(cryptonote::txin_gen{height}); + cryptonote::add_tx_pub_key_to_extra(block.miner_tx, {}); + cryptonote::get_block_reward( + db.get_block_weight(height - 1), + {}, + db.get_block_already_generated_coins(height - 1), + reward, + hardfork + ); + block.miner_tx.vout.push_back(cryptonote::tx_out{reward, cryptonote::txout_to_key{}}); + diff = storage.get_difficulty_for_next_block(); + }; + struct stat { + struct chain { + diff_t diff; + reward_t reward; + }; + }; + auto add_block = []( + core_t &core, + const block_t &block, + const stat::chain &stat + ){ + core.get_blockchain_storage().get_db().batch_start({}, {}); + core.get_blockchain_storage().get_db().add_block( + {block, cryptonote::block_to_blob(block)}, + cryptonote::get_transaction_weight(block.miner_tx), + core.get_blockchain_storage().get_next_long_term_block_weight( + cryptonote::get_transaction_weight(block.miner_tx) + ), + stat.diff, + stat.reward, + {} + ); + core.get_blockchain_storage().get_db().batch_stop(); + }; + struct messages { + struct core { + using sync = cryptonote::CORE_SYNC_DATA; + }; + using handshake = nodetool::COMMAND_HANDSHAKE_T<core::sync>; + }; + struct net_node_t: commands_handler_t, p2p_endpoint_t { + using span_t = epee::span<const uint8_t>; + using zone_t = epee::net_utils::zone; + using uuid_t = boost::uuids::uuid; + using relay_t = cryptonote::relay_method; + using blobs_t = std::vector<cryptonote::blobdata>; + using id_t = nodetool::peerid_type; + using callback_t = std::function<bool(contexts::cryptonote &, id_t, uint32_t)>; + using address_t = epee::net_utils::network_address; + using connections_t = std::vector<std::pair<zone_t, uuid_t>>; + struct bans { + using subnets = std::map<epee::net_utils::ipv4_network_subnet, time_t>; + using hosts = std::map<std::string, time_t>; + }; + shared_state_ptr shared_state; + core_protocol_ptr core_protocol; + virtual int invoke(int command, const span_t in, epee::byte_stream &out, context_t &context) override { + if (core_protocol) { + if (command == messages::handshake::ID) { + return epee::net_utils::buff_to_t_adapter<void, typename messages::handshake::request, typename messages::handshake::response>( + command, + in, + out, + [this](int command, typename messages::handshake::request &in, typename messages::handshake::response &out, context_t &context){ + core_protocol->process_payload_sync_data(in.payload_data, context, true); + core_protocol->get_payload_sync_data(out.payload_data); + return 1; + }, + context + ); + } + bool handled; + return core_protocol->handle_invoke_map(false, command, in, out, context, handled); + } + else + return {}; + } + virtual int notify(int command, const span_t in, context_t &context) override { + if (core_protocol) { + bool handled; + epee::byte_stream out; + return core_protocol->handle_invoke_map(true, command, in, out, context, handled); + } + else + return {}; + } + virtual void callback(context_t &context) override { + if (core_protocol) + core_protocol->on_callback(context); + } + virtual void on_connection_new(context_t&) override {} + virtual void on_connection_close(context_t &context) override { + if (core_protocol) + core_protocol->on_connection_close(context); + } + virtual ~net_node_t() override {} + virtual bool add_host_fail(const address_t&, unsigned int = {}) override { + return {}; + } + virtual bool block_host(address_t address, time_t = {}, bool = {}) override { + return {}; + } + virtual bool drop_connection(const contexts::basic& context) override { + if (shared_state) + return shared_state->close(context.m_connection_id); + else + return {}; + } + virtual bool for_connection(const uuid_t& uuid, callback_t f) override { + if (shared_state) + return shared_state->for_connection(uuid,[&f](context_t &context){ + return f(context, context.peer_id, context.support_flags); + }); + else + return {}; + } + virtual bool invoke_notify_to_peer(int command, epee::levin::message_writer in, const contexts::basic& context) override { + if (shared_state) + return shared_state->send(in.finalize_notify(command), context.m_connection_id); + else + return {}; + } + virtual bool relay_notify_to_list(int command, epee::levin::message_writer in, connections_t connections) override { + if (shared_state) { + for (auto &e: connections) + shared_state->send(in.finalize_notify(command), e.second); + } + return {}; + } + virtual bool unblock_host(const address_t&) override { + return {}; + } + virtual zone_t send_txs(blobs_t, const zone_t, const uuid_t&, relay_t) override { + return {}; + } + virtual bans::subnets get_blocked_subnets() override { + return {}; + } + virtual bans::hosts get_blocked_hosts() override { + return {}; + } + virtual uint64_t get_public_connections_count() override { + if (shared_state) + return shared_state->get_connections_count(); + else + return {}; + } + virtual void add_used_stripe_peer(const contexts::cryptonote&) override {} + virtual void clear_used_stripe_peers() override {} + virtual void remove_used_stripe_peer(const contexts::cryptonote&) override {} + virtual void for_each_connection(callback_t f) override { + if (shared_state) + shared_state->foreach_connection([&f](context_t &context){ + return f(context, context.peer_id, context.support_flags); + }); + } + virtual void request_callback(const contexts::basic &context) override { + if (shared_state) + shared_state->request_callback(context.m_connection_id); + } + }; + auto conduct_handshake = [get_conn_tag](net_node_t &net_node, connection_ptr conn){ + event_t handshaked; + net_node.shared_state->for_connection( + get_conn_tag(*conn), + [&handshaked, &net_node](context_t &context){ + typename messages::handshake::request msg; + net_node.core_protocol->get_payload_sync_data(msg.payload_data); + epee::net_utils::async_invoke_remote_command2<typename messages::handshake::response>( + context, + messages::handshake::ID, + msg, + *net_node.shared_state, + [&handshaked, &net_node](int code, const typename messages::handshake::response &msg, context_t &context){ + EXPECT_TRUE(code >= 0); + net_node.core_protocol->process_payload_sync_data(msg.payload_data, context, true); + handshaked.raise(); + }, + P2P_DEFAULT_HANDSHAKE_INVOKE_TIMEOUT + ); + return true; + } + ); + handshaked.wait(); + }; + using path_t = boost::filesystem::path; + auto create_dir = []{ + ec_t ec; + path_t path = boost::filesystem::temp_directory_path() / boost::filesystem::unique_path("daemon-%%%%%%%%%%%%%%%%", ec); + if (ec) + return path_t{}; + auto success = boost::filesystem::create_directory(path, ec); + if (not ec && success) + return path; + return path_t{}; + }; + auto remove_tree = [](const path_t &path){ + ec_t ec; + boost::filesystem::remove_all(path, ec); + }; + using options_t = boost::program_options::variables_map; + struct daemon_t { + options_t options; + core_ptr core; + core_protocol_ptr core_protocol; + net_node_t net_node; + shared_state_ptr shared_state; + connections_t conn; + }; + struct daemons_t { + daemon_t main; + daemon_t alt; + }; + using options_description_t = boost::program_options::options_description; + + const auto dir = create_dir(); + ASSERT_TRUE(not dir.empty()); + + daemons_t daemon{ + { + [&dir]{ + options_t options; + boost::program_options::store( + boost::program_options::command_line_parser({ + "--data-dir", + (dir / "main").string(), + "--disable-dns-checkpoints", + "--check-updates=disabled", + "--fixed-difficulty=1", + "--block-sync-size=1", + "--db-sync-mode=fastest:async:50000", + }).options([]{ + options_description_t options_description{}; + cryptonote::core::init_options(options_description); + return options_description; + }()).run(), + options + ); + return options; + }(), + {}, + {}, + {}, + {}, + {}, + }, + { + [&dir]{ + options_t options; + boost::program_options::store( + boost::program_options::command_line_parser({ + "--data-dir", + (dir / "alt").string(), + "--disable-dns-checkpoints", + "--check-updates=disabled", + "--fixed-difficulty=1", + "--block-sync-size=1", + "--db-sync-mode=fastest:async:50000", + }).options([]{ + options_description_t options_description{}; + cryptonote::core::init_options(options_description); + return options_description; + }()).run(), + options + ); + return options; + }(), + {}, + {}, + {}, + {}, + {}, + }, + }; + + io_context_t io_context; + work_ptr work = std::make_shared<work_t>(io_context); + workers_t workers; + while (workers.size() < 4) { + workers.emplace_back([&io_context]{ + io_context.run(); + }); + } + + connection_t::set_rate_up_limit(std::numeric_limits<int64_t>::max()); + connection_t::set_rate_down_limit(std::numeric_limits<int64_t>::max()); + + { + daemon.main.core = core_ptr(new core_t(nullptr)); + daemon.main.core->init(daemon.main.options, nullptr, nullptr); + daemon.main.net_node.core_protocol = daemon.main.core_protocol = core_protocol_ptr(new core_protocol_t( + *daemon.main.core, &daemon.main.net_node, {} + )); + daemon.main.core->set_cryptonote_protocol(daemon.main.core_protocol.get()); + daemon.main.core_protocol->init(daemon.main.options); + daemon.main.net_node.shared_state = daemon.main.shared_state = std::make_shared<shared_state_t>(); + daemon.main.shared_state->set_handler(&daemon.main.net_node); + daemon.alt.shared_state = std::make_shared<shared_state_t>(); + daemon.alt.shared_state->set_handler(&daemon.alt.net_node); + + struct { + event_t prepare; + event_t check; + event_t finish; + } events; + auto connections = create_conn_pair( + connection_ptr(new connection_t(io_context, daemon.main.shared_state, {}, {})), + connection_ptr(new connection_t(io_context, daemon.alt.shared_state, {}, {})) + ); + { + auto conn = connections.first; + auto shared_state = daemon.main.shared_state; + const auto tag = get_conn_tag(*conn); + conn->strand_.post([tag, conn, shared_state, &events]{ + shared_state->for_connection(tag, [](context_t &context){ + context.m_expect_height = -1; + context.m_expect_response = -1; + context.m_last_request_time = boost::date_time::min_date_time; + context.m_score = 0; + context.m_state = contexts::cryptonote::state_synchronizing; + return true; + }); + events.prepare.raise(); + events.check.wait(); + shared_state->for_connection(tag, [](context_t &context){ + EXPECT_TRUE(context.m_expect_height == -1); + EXPECT_TRUE(context.m_expect_response == -1); + EXPECT_TRUE(context.m_last_request_time == boost::date_time::min_date_time); + EXPECT_TRUE(context.m_score == 0); + EXPECT_TRUE(context.m_state == contexts::cryptonote::state_synchronizing); + return true; + }); + events.finish.raise(); + }); + } + events.prepare.wait(); + daemon.main.core_protocol->on_idle(); + events.check.raise(); + events.finish.wait(); + + connections.first->strand_.post([connections]{ + connections.first->cancel(); + }); + connections.second->strand_.post([connections]{ + connections.second->cancel(); + }); + connections.first.reset(); + connections.second.reset(); + while (daemon.main.shared_state->sock_count); + while (daemon.alt.shared_state->sock_count); + daemon.main.core_protocol->deinit(); + daemon.main.core->stop(); + daemon.main.core->deinit(); + daemon.main.net_node.shared_state.reset(); + daemon.main.shared_state.reset(); + daemon.main.core_protocol.reset(); + daemon.main.core.reset(); + daemon.alt.shared_state.reset(); + } + + { + daemon.main.core = core_ptr(new core_t(nullptr)); + daemon.main.core->init(daemon.main.options, nullptr, nullptr); + daemon.main.net_node.core_protocol = daemon.main.core_protocol = core_protocol_ptr(new core_protocol_t( + *daemon.main.core, &daemon.main.net_node, {} + )); + daemon.main.core->set_cryptonote_protocol(daemon.main.core_protocol.get()); + daemon.main.core->set_checkpoints({}); + daemon.main.core_protocol->init(daemon.main.options); + daemon.main.net_node.shared_state = daemon.main.shared_state = std::make_shared<shared_state_t>(); + daemon.main.shared_state->set_handler(&daemon.main.net_node); + daemon.alt.core = core_ptr(new core_t(nullptr)); + daemon.alt.core->init(daemon.alt.options, nullptr, nullptr); + daemon.alt.net_node.core_protocol = daemon.alt.core_protocol = core_protocol_ptr(new core_protocol_t( + *daemon.alt.core, &daemon.alt.net_node, {} + )); + daemon.alt.core->set_cryptonote_protocol(daemon.alt.core_protocol.get()); + daemon.alt.core->set_checkpoints({}); + daemon.alt.core_protocol->init(daemon.alt.options); + daemon.alt.net_node.shared_state = daemon.alt.shared_state = std::make_shared<shared_state_t>(); + daemon.alt.shared_state->set_handler(&daemon.alt.net_node); + + struct { + io_context_t io_context; + work_ptr work; + workers_t workers; + } check; + check.work = std::make_shared<work_t>(check.io_context); + while (check.workers.size() < 2) { + check.workers.emplace_back([&check]{ + check.io_context.run(); + }); + } + while (daemon.main.conn.size() < 1) { + daemon.main.conn.emplace_back(new connection_t(check.io_context, daemon.main.shared_state, {}, {})); + daemon.alt.conn.emplace_back(new connection_t(io_context, daemon.alt.shared_state, {}, {})); + create_conn_pair(daemon.main.conn.back(), daemon.alt.conn.back()); + conduct_handshake(daemon.alt.net_node, daemon.alt.conn.back()); + } + struct { + event_t prepare; + event_t sync; + event_t finish; + } events; + { + auto conn = daemon.main.conn.back(); + auto shared_state = daemon.main.shared_state; + const auto tag = get_conn_tag(*conn); + conn->strand_.post([tag, conn, shared_state, &events]{ + shared_state->for_connection(tag, [](context_t &context){ + EXPECT_TRUE(context.m_state == contexts::cryptonote::state_normal); + return true; + }); + events.prepare.raise(); + events.sync.wait(); + shared_state->for_connection(tag, [](context_t &context){ + EXPECT_TRUE(context.m_state == contexts::cryptonote::state_normal); + return true; + }); + events.finish.raise(); + }); + } + events.prepare.wait(); + daemon.main.core->get_blockchain_storage().add_block_notify( + [&events](height_t height, span::blocks blocks){ + if (height >= CRYPTONOTE_PRUNING_STRIPE_SIZE) + events.sync.raise(); + } + ); + { + stat::chain stat{ + daemon.alt.core->get_blockchain_storage().get_db().get_block_cumulative_difficulty( + daemon.alt.core->get_current_blockchain_height() - 1 + ), + daemon.alt.core->get_blockchain_storage().get_db().get_block_already_generated_coins( + daemon.alt.core->get_current_blockchain_height() - 1 + ), + }; + while (daemon.alt.core->get_current_blockchain_height() < CRYPTONOTE_PRUNING_STRIPE_SIZE + CRYPTONOTE_PRUNING_TIP_BLOCKS) { + block_t block; + diff_t diff; + reward_t reward; + get_block_template(*daemon.alt.core, block, diff, reward); + stat.diff += diff; + stat.reward = stat.reward < (MONEY_SUPPLY - stat.reward) ? stat.reward + reward : MONEY_SUPPLY; + add_block(*daemon.alt.core, block, stat); + if (daemon.main.core->get_current_blockchain_height() + 1 < CRYPTONOTE_PRUNING_STRIPE_SIZE) + add_block(*daemon.main.core, block, stat); + } + } + while (daemon.main.conn.size() < 2) { + daemon.main.conn.emplace_back(new connection_t(check.io_context, daemon.main.shared_state, {}, {})); + daemon.alt.conn.emplace_back(new connection_t(io_context, daemon.alt.shared_state, {}, {})); + create_conn_pair(daemon.main.conn.back(), daemon.alt.conn.back()); + conduct_handshake(daemon.alt.net_node, daemon.alt.conn.back()); + } + events.finish.wait(); + + for (;daemon.main.conn.size(); daemon.main.conn.pop_back()) { + auto conn = daemon.main.conn.back(); + conn->strand_.post([conn]{ + conn->cancel(); + }); + } + for (;daemon.alt.conn.size(); daemon.alt.conn.pop_back()) { + auto conn = daemon.alt.conn.back(); + conn->strand_.post([conn]{ + conn->cancel(); + }); + } + while (daemon.main.shared_state->sock_count); + while (daemon.alt.shared_state->sock_count); + daemon.main.core_protocol->deinit(); + daemon.main.core->stop(); + daemon.main.core->deinit(); + daemon.main.net_node.shared_state.reset(); + daemon.main.shared_state.reset(); + daemon.main.core_protocol.reset(); + daemon.main.core.reset(); + daemon.alt.core_protocol->deinit(); + daemon.alt.core->stop(); + daemon.alt.core->deinit(); + daemon.alt.net_node.shared_state.reset(); + daemon.alt.shared_state.reset(); + daemon.alt.core_protocol.reset(); + daemon.alt.core.reset(); + check.work.reset(); + for (auto& w: check.workers) { + w.join(); + } + } + + work.reset(); + for (auto& w: workers) { + w.join(); + } + remove_tree(dir); +} + namespace nodetool { template class node_server<cryptonote::t_cryptonote_protocol_handler<test_core>>; } namespace cryptonote { template class t_cryptonote_protocol_handler<test_core>; } diff --git a/tests/unit_tests/output_selection.cpp b/tests/unit_tests/output_selection.cpp index cff509451..20dc58e03 100644 --- a/tests/unit_tests/output_selection.cpp +++ b/tests/unit_tests/output_selection.cpp @@ -206,7 +206,7 @@ TEST(select_outputs, same_distribution) for (size_t i = 0; i < chain_picks.size(); ++i) chain_norm[i * 100 / chain_picks.size()] += chain_picks[i]; - double max_dev = 0.0, avg_dev = 0.0; + double avg_dev = 0.0; for (size_t i = 0; i < 100; ++i) { const double diff = (double)output_norm[i] - (double)chain_norm[i]; diff --git a/tests/unit_tests/serialization.cpp b/tests/unit_tests/serialization.cpp index 7b8a291d0..535752665 100644 --- a/tests/unit_tests/serialization.cpp +++ b/tests/unit_tests/serialization.cpp @@ -132,11 +132,10 @@ TEST(Serialization, BinaryArchiveInts) { ASSERT_EQ(8, oss.str().size()); ASSERT_EQ(string("\0\0\0\0\xff\0\0\0", 8), oss.str()); - istringstream iss(oss.str()); - binary_archive<false> iar(iss); + binary_archive<false> iar{epee::strspan<std::uint8_t>(oss.str())}; iar.serialize_int(x1); - ASSERT_EQ(8, iss.tellg()); - ASSERT_TRUE(iss.good()); + ASSERT_EQ(8, iar.getpos()); + ASSERT_TRUE(iar.good()); ASSERT_EQ(x, x1); } @@ -151,10 +150,9 @@ TEST(Serialization, BinaryArchiveVarInts) { ASSERT_EQ(6, oss.str().size()); ASSERT_EQ(string("\x80\x80\x80\x80\xF0\x1F", 6), oss.str()); - istringstream iss(oss.str()); - binary_archive<false> iar(iss); + binary_archive<false> iar{epee::strspan<std::uint8_t>(oss.str())}; iar.serialize_varint(x1); - ASSERT_TRUE(iss.good()); + ASSERT_TRUE(iar.good()); ASSERT_EQ(x, x1); } diff --git a/tests/unit_tests/test_tx_utils.cpp b/tests/unit_tests/test_tx_utils.cpp index 9425e2432..0d0dc8819 100644 --- a/tests/unit_tests/test_tx_utils.cpp +++ b/tests/unit_tests/test_tx_utils.cpp @@ -287,3 +287,73 @@ TEST(sort_tx_extra, invalid_suffix_partial) std::vector<uint8_t> expected(&expected_arr[0], &expected_arr[0] + sizeof(expected_arr)); ASSERT_EQ(sorted, expected); } + +TEST(remove_field_from_tx_extra, remove_first) +{ + const uint8_t extra_arr[] = {1, 30, 208, 98, 162, 133, 64, 85, 83, 112, 91, 188, 89, 211, 24, 131, 39, 154, 22, 228, + 80, 63, 198, 141, 173, 111, 244, 183, 4, 149, 186, 140, 230, 2, 1, 42}; + std::vector<uint8_t> extra(&extra_arr[0], &extra_arr[0] + sizeof(extra_arr)); + + std::vector<cryptonote::tx_extra_field> tx_extra_fields; + ASSERT_TRUE(cryptonote::parse_tx_extra(extra, tx_extra_fields)); + ASSERT_EQ(2, tx_extra_fields.size()); + ASSERT_EQ(typeid(cryptonote::tx_extra_pub_key), tx_extra_fields[0].type()); + ASSERT_EQ(typeid(cryptonote::tx_extra_nonce), tx_extra_fields[1].type()); + + tx_extra_fields.clear(); + ASSERT_TRUE(cryptonote::remove_field_from_tx_extra(extra, typeid(cryptonote::tx_extra_pub_key))); + ASSERT_TRUE(cryptonote::parse_tx_extra(extra, tx_extra_fields)); + ASSERT_EQ(1, tx_extra_fields.size()); + ASSERT_EQ(typeid(cryptonote::tx_extra_nonce), tx_extra_fields[0].type()); +} + +TEST(remove_field_from_tx_extra, remove_last) +{ + const uint8_t extra_arr[] = {1, 30, 208, 98, 162, 133, 64, 85, 83, 112, 91, 188, 89, 211, 24, 131, 39, 154, 22, 228, + 80, 63, 198, 141, 173, 111, 244, 183, 4, 149, 186, 140, 230, 2, 1, 42}; + std::vector<uint8_t> extra(&extra_arr[0], &extra_arr[0] + sizeof(extra_arr)); + + std::vector<cryptonote::tx_extra_field> tx_extra_fields; + ASSERT_TRUE(cryptonote::parse_tx_extra(extra, tx_extra_fields)); + ASSERT_EQ(2, tx_extra_fields.size()); + ASSERT_EQ(typeid(cryptonote::tx_extra_pub_key), tx_extra_fields[0].type()); + ASSERT_EQ(typeid(cryptonote::tx_extra_nonce), tx_extra_fields[1].type()); + + tx_extra_fields.clear(); + ASSERT_TRUE(cryptonote::remove_field_from_tx_extra(extra, typeid(cryptonote::tx_extra_nonce))); + ASSERT_TRUE(cryptonote::parse_tx_extra(extra, tx_extra_fields)); + ASSERT_EQ(1, tx_extra_fields.size()); + ASSERT_EQ(typeid(cryptonote::tx_extra_pub_key), tx_extra_fields[0].type()); +} + +TEST(remove_field_from_tx_extra, remove_middle) +{ + const uint8_t extra_arr[] = {1, 30, 208, 98, 162, 133, 64, 85, 83, 112, 91, 188, 89, 211, 24, 131, 39, 154, 22, 228, + 80, 63, 198, 141, 173, 111, 244, 183, 4, 149, 186, 140, 230, 2, 1, 42, 1, 30, 208, 98, 162, 133, 64, 85, 83, 112, + 91, 188, 89, 211, 24, 131, 39, 154, 22, 228, 80, 63, 198, 141, 173, 111, 244, 183, 4, 149, 186, 140, 230}; + std::vector<uint8_t> extra(&extra_arr[0], &extra_arr[0] + sizeof(extra_arr)); + + std::vector<cryptonote::tx_extra_field> tx_extra_fields; + ASSERT_TRUE(cryptonote::parse_tx_extra(extra, tx_extra_fields)); + ASSERT_EQ(3, tx_extra_fields.size()); + ASSERT_EQ(typeid(cryptonote::tx_extra_pub_key), tx_extra_fields[0].type()); + ASSERT_EQ(typeid(cryptonote::tx_extra_nonce), tx_extra_fields[1].type()); + ASSERT_EQ(typeid(cryptonote::tx_extra_pub_key), tx_extra_fields[2].type()); + + tx_extra_fields.clear(); + ASSERT_TRUE(cryptonote::remove_field_from_tx_extra(extra, typeid(cryptonote::tx_extra_nonce))); + ASSERT_TRUE(cryptonote::parse_tx_extra(extra, tx_extra_fields)); + ASSERT_EQ(2, tx_extra_fields.size()); + ASSERT_EQ(typeid(cryptonote::tx_extra_pub_key), tx_extra_fields[0].type()); + ASSERT_EQ(typeid(cryptonote::tx_extra_pub_key), tx_extra_fields[0].type()); +} + +TEST(remove_field_from_tx_extra, invalid_varint) +{ + const uint8_t extra_arr[] = {1, 30, 208, 98, 162, 133, 64, 85, 83, 112, 91, 188, 89, 211, 24, 131, 39, 154, 22, 228, + 80, 63, 198, 141, 173, 111, 244, 183, 4, 149, 186, 140, 230, 2, 0x80, 0}; + std::vector<uint8_t> extra(&extra_arr[0], &extra_arr[0] + sizeof(extra_arr)); + + ASSERT_FALSE(cryptonote::remove_field_from_tx_extra(extra, typeid(cryptonote::tx_extra_nonce))); + ASSERT_EQ(sizeof(extra_arr), extra.size()); +} diff --git a/tests/unit_tests/tx_proof.cpp b/tests/unit_tests/tx_proof.cpp index c5d06bc68..0adb8713e 100644 --- a/tests/unit_tests/tx_proof.cpp +++ b/tests/unit_tests/tx_proof.cpp @@ -58,14 +58,14 @@ TEST(tx_proof, prove_verify_v2) // R_B = rB crypto::public_key R_B; ge_p3 B_p3; - ge_frombytes_vartime(&B_p3,&B); + ASSERT_EQ(ge_frombytes_vartime(&B_p3,&B), 0); ge_p2 R_B_p2; ge_scalarmult(&R_B_p2, &unwrap(r), &B_p3); ge_tobytes(&R_B, &R_B_p2); // R_G = rG crypto::public_key R_G; - ge_frombytes_vartime(&B_p3,&B); + ASSERT_EQ(ge_frombytes_vartime(&B_p3,&B), 0); ge_p3 R_G_p3; ge_scalarmult_base(&R_G_p3, &unwrap(r)); ge_p3_tobytes(&R_G, &R_G_p3); @@ -73,7 +73,7 @@ TEST(tx_proof, prove_verify_v2) // D = rA crypto::public_key D; ge_p3 A_p3; - ge_frombytes_vartime(&A_p3,&A); + ASSERT_EQ(ge_frombytes_vartime(&A_p3,&A), 0); ge_p2 D_p2; ge_scalarmult(&D_p2, &unwrap(r), &A_p3); ge_tobytes(&D, &D_p2); diff --git a/tests/unit_tests/wipeable_string.cpp b/tests/unit_tests/wipeable_string.cpp index 9911bd6f4..0e0bf8906 100644 --- a/tests/unit_tests/wipeable_string.cpp +++ b/tests/unit_tests/wipeable_string.cpp @@ -189,20 +189,20 @@ TEST(wipeable_string, parse_hexstr) { boost::optional<epee::wipeable_string> s; - ASSERT_EQ(boost::none, epee::wipeable_string("x").parse_hexstr()); - ASSERT_EQ(boost::none, epee::wipeable_string("x0000000000000000").parse_hexstr()); - ASSERT_EQ(boost::none, epee::wipeable_string("0000000000000000x").parse_hexstr()); - ASSERT_EQ(boost::none, epee::wipeable_string("0").parse_hexstr()); - ASSERT_EQ(boost::none, epee::wipeable_string("000").parse_hexstr()); + ASSERT_TRUE(boost::none == epee::wipeable_string("x").parse_hexstr()); + ASSERT_TRUE(boost::none == epee::wipeable_string("x0000000000000000").parse_hexstr()); + ASSERT_TRUE(boost::none == epee::wipeable_string("0000000000000000x").parse_hexstr()); + ASSERT_TRUE(boost::none == epee::wipeable_string("0").parse_hexstr()); + ASSERT_TRUE(boost::none == epee::wipeable_string("000").parse_hexstr()); ASSERT_TRUE((s = epee::wipeable_string("").parse_hexstr()) != boost::none); - ASSERT_EQ(*s, ""); + ASSERT_TRUE(*s == ""); ASSERT_TRUE((s = epee::wipeable_string("00").parse_hexstr()) != boost::none); - ASSERT_EQ(*s, epee::wipeable_string("", 1)); + ASSERT_TRUE(*s == epee::wipeable_string("", 1)); ASSERT_TRUE((s = epee::wipeable_string("41").parse_hexstr()) != boost::none); - ASSERT_EQ(*s, epee::wipeable_string("A")); + ASSERT_TRUE(*s == epee::wipeable_string("A")); ASSERT_TRUE((s = epee::wipeable_string("414243").parse_hexstr()) != boost::none); - ASSERT_EQ(*s, epee::wipeable_string("ABC")); + ASSERT_TRUE(*s == epee::wipeable_string("ABC")); } TEST(wipeable_string, to_hex) diff --git a/tests/unit_tests/zmq_rpc.cpp b/tests/unit_tests/zmq_rpc.cpp index 59759bed8..66a7f3ac1 100644 --- a/tests/unit_tests/zmq_rpc.cpp +++ b/tests/unit_tests/zmq_rpc.cpp @@ -202,7 +202,6 @@ namespace MASSERT(!expected.empty()); std::size_t actual_height = 0; - crypto::hash actual_id{}; crypto::hash actual_prev_id{}; std::vector<crypto::hash> actual_ids{}; GET_FROM_JSON_OBJECT(pub.second, actual_height, first_height); diff --git a/translations/CMakeLists.txt b/translations/CMakeLists.txt index 0081746f2..927d1733a 100644 --- a/translations/CMakeLists.txt +++ b/translations/CMakeLists.txt @@ -26,7 +26,7 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -cmake_minimum_required(VERSION 2.8.7) +cmake_minimum_required(VERSION 3.5) project(translations) @@ -73,7 +73,7 @@ string(REPLACE ".ts" ".qm" qm_files "${ts_files}") add_custom_command(TARGET generate_translations_header POST_BUILD - COMMAND ./generate_translations_header ${qm_files} + COMMAND $<TARGET_FILE:generate_translations_header> ${qm_files} WORKING_DIRECTORY "${CMAKE_CURRENT_BIN_DIR}" COMMENT "Generating embedded translations header") diff --git a/utils/build_scripts/android32.Dockerfile b/utils/build_scripts/android32.Dockerfile deleted file mode 100644 index a2d0edbb3..000000000 --- a/utils/build_scripts/android32.Dockerfile +++ /dev/null @@ -1,134 +0,0 @@ -FROM debian:stable - -RUN apt-get update && apt-get install -y unzip automake build-essential curl file pkg-config git python libtool libtinfo5 - -WORKDIR /opt/android -## INSTALL ANDROID SDK -ENV ANDROID_SDK_REVISION 4333796 -ENV ANDROID_SDK_HASH 92ffee5a1d98d856634e8b71132e8a95d96c83a63fde1099be3d86df3106def9 -RUN curl -s -O https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_REVISION}.zip \ - && echo "${ANDROID_SDK_HASH} sdk-tools-linux-${ANDROID_SDK_REVISION}.zip" | sha256sum -c \ - && unzip sdk-tools-linux-${ANDROID_SDK_REVISION}.zip \ - && rm -f sdk-tools-linux-${ANDROID_SDK_REVISION}.zip - -## INSTALL ANDROID NDK -ENV ANDROID_NDK_REVISION 17b -ENV ANDROID_NDK_HASH 5dfbbdc2d3ba859fed90d0e978af87c71a91a5be1f6e1c40ba697503d48ccecd -RUN curl -s -O https://dl.google.com/android/repository/android-ndk-r${ANDROID_NDK_REVISION}-linux-x86_64.zip \ - && echo "${ANDROID_NDK_HASH} android-ndk-r${ANDROID_NDK_REVISION}-linux-x86_64.zip" | sha256sum -c \ - && unzip android-ndk-r${ANDROID_NDK_REVISION}-linux-x86_64.zip \ - && rm -f android-ndk-r${ANDROID_NDK_REVISION}-linux-x86_64.zip - -ENV WORKDIR /opt/android -ENV ANDROID_SDK_ROOT ${WORKDIR}/tools -ENV ANDROID_NDK_ROOT ${WORKDIR}/android-ndk-r${ANDROID_NDK_REVISION} -ENV PREFIX /opt/android/prefix - -ENV TOOLCHAIN_DIR ${WORKDIR}/toolchain-arm -RUN ${ANDROID_NDK_ROOT}/build/tools/make_standalone_toolchain.py \ - --arch arm \ - --api 21 \ - --install-dir ${TOOLCHAIN_DIR} \ - --stl=libc++ - -#INSTALL cmake -ENV CMAKE_VERSION 3.12.1 -RUN cd /usr \ - && curl -s -O https://cmake.org/files/v3.12/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz \ - && tar -xzf /usr/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz \ - && rm -f /usr/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz -ENV PATH /usr/cmake-${CMAKE_VERSION}-Linux-x86_64/bin:$PATH - -## Boost -ARG BOOST_VERSION=1_68_0 -ARG BOOST_VERSION_DOT=1.68.0 -ARG BOOST_HASH=7f6130bc3cf65f56a618888ce9d5ea704fa10b462be126ad053e80e553d6d8b7 -RUN set -ex \ - && curl -s -L -o boost_${BOOST_VERSION}.tar.bz2 https://dl.bintray.com/boostorg/release/${BOOST_VERSION_DOT}/source/boost_${BOOST_VERSION}.tar.bz2 \ - && echo "${BOOST_HASH} boost_${BOOST_VERSION}.tar.bz2" | sha256sum -c \ - && tar -xvf boost_${BOOST_VERSION}.tar.bz2 \ - && rm -f boost_${BOOST_VERSION}.tar.bz2 \ - && cd boost_${BOOST_VERSION} \ - && ./bootstrap.sh --prefix=${PREFIX} - -ENV HOST_PATH $PATH -ENV PATH $TOOLCHAIN_DIR/arm-linux-androideabi/bin:$TOOLCHAIN_DIR/bin:$PATH - -ARG NPROC=1 - -# Build iconv for lib boost locale -ENV ICONV_VERSION 1.15 -ENV ICONV_HASH ccf536620a45458d26ba83887a983b96827001e92a13847b45e4925cc8913178 -RUN curl -s -O http://ftp.gnu.org/pub/gnu/libiconv/libiconv-${ICONV_VERSION}.tar.gz \ - && echo "${ICONV_HASH} libiconv-${ICONV_VERSION}.tar.gz" | sha256sum -c \ - && tar -xzf libiconv-${ICONV_VERSION}.tar.gz \ - && rm -f libiconv-${ICONV_VERSION}.tar.gz \ - && cd libiconv-${ICONV_VERSION} \ - && CC=arm-linux-androideabi-clang CXX=arm-linux-androideabi-clang++ ./configure --build=x86_64-linux-gnu --host=arm-eabi --prefix=${PREFIX} --disable-rpath \ - && make -j${NPROC} && make install - -## Build BOOST -RUN cd boost_${BOOST_VERSION} \ - && ./b2 --build-type=minimal link=static runtime-link=static --with-chrono --with-date_time --with-filesystem --with-program_options --with-regex --with-serialization --with-system --with-thread --with-locale --build-dir=android --stagedir=android toolset=clang threading=multi threadapi=pthread target-os=android -sICONV_PATH=${PREFIX} install -j${NPROC} - -#Note : we build openssl because the default lacks DSA1 - -# download, configure and make Zlib -ENV ZLIB_VERSION 1.2.11 -ENV ZLIB_HASH c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1 -RUN curl -s -O https://zlib.net/zlib-${ZLIB_VERSION}.tar.gz \ - && echo "${ZLIB_HASH} zlib-${ZLIB_VERSION}.tar.gz" | sha256sum -c \ - && tar -xzf zlib-${ZLIB_VERSION}.tar.gz \ - && rm zlib-${ZLIB_VERSION}.tar.gz \ - && mv zlib-${ZLIB_VERSION} zlib \ - && cd zlib && CC=clang CXX=clang++ ./configure --static \ - && make -j${NPROC} - -# open ssl -ARG OPENSSL_VERSION=1.0.2p -ARG OPENSSL_HASH=50a98e07b1a89eb8f6a99477f262df71c6fa7bef77df4dc83025a2845c827d00 -RUN curl -s -O https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz \ - && echo "${OPENSSL_HASH} openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c \ - && tar -xzf openssl-${OPENSSL_VERSION}.tar.gz \ - && rm openssl-${OPENSSL_VERSION}.tar.gz \ - && cd openssl-${OPENSSL_VERSION} \ - && sed -i -e "s/mandroid/target\ armv7\-none\-linux\-androideabi/" Configure \ - && CC=clang CXX=clang++ \ - ./Configure android-armv7 \ - no-asm \ - no-shared --static \ - --with-zlib-include=${WORKDIR}/zlib/include --with-zlib-lib=${WORKDIR}/zlib/lib \ - --prefix=${PREFIX} --openssldir=${PREFIX} \ - && make -j${NPROC} \ - && make install - -# ZMQ -ARG ZMQ_VERSION=v4.2.5 -ARG ZMQ_HASH=d062edd8c142384792955796329baf1e5a3377cd -RUN git clone https://github.com/zeromq/libzmq.git -b ${ZMQ_VERSION} \ - && cd libzmq \ - && test `git rev-parse HEAD` = ${ZMQ_HASH} || exit 1 \ - && ./autogen.sh \ - && CC=clang CXX=clang++ ./configure --prefix=${PREFIX} --host=arm-linux-androideabi --enable-static --disable-shared \ - && make -j${NPROC} \ - && make install - -# Sodium -ARG SODIUM_VERSION=1.0.16 -ARG SODIUM_HASH=675149b9b8b66ff44152553fb3ebf9858128363d -RUN set -ex \ - && git clone https://github.com/jedisct1/libsodium.git -b ${SODIUM_VERSION} \ - && cd libsodium \ - && test `git rev-parse HEAD` = ${SODIUM_HASH} || exit 1 \ - && ./autogen.sh \ - && CC=clang CXX=clang++ ./configure --prefix=${PREFIX} --host=arm-linux-androideabi --enable-static --disable-shared \ - && make -j${NPROC} \ - && make install - -ADD . /src -RUN cd /src \ - && CMAKE_INCLUDE_PATH="${PREFIX}/include" \ - CMAKE_LIBRARY_PATH="${PREFIX}/lib" \ - ANDROID_STANDALONE_TOOLCHAIN_PATH=${TOOLCHAIN_DIR} \ - USE_SINGLE_BUILDDIR=1 \ - PATH=${HOST_PATH} make release-static-android-armv7 -j${NPROC} diff --git a/utils/build_scripts/android64.Dockerfile b/utils/build_scripts/android64.Dockerfile deleted file mode 100644 index eca2d4da1..000000000 --- a/utils/build_scripts/android64.Dockerfile +++ /dev/null @@ -1,134 +0,0 @@ -FROM debian:stable - -RUN apt-get update && apt-get install -y unzip automake build-essential curl file pkg-config git python libtool libtinfo5 - -WORKDIR /opt/android -## INSTALL ANDROID SDK -ENV ANDROID_SDK_REVISION 4333796 -ENV ANDROID_SDK_HASH 92ffee5a1d98d856634e8b71132e8a95d96c83a63fde1099be3d86df3106def9 -RUN curl -s -O https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_REVISION}.zip \ - && echo "${ANDROID_SDK_HASH} sdk-tools-linux-${ANDROID_SDK_REVISION}.zip" | sha256sum -c \ - && unzip sdk-tools-linux-${ANDROID_SDK_REVISION}.zip \ - && rm -f sdk-tools-linux-${ANDROID_SDK_REVISION}.zip - -## INSTALL ANDROID NDK -ENV ANDROID_NDK_REVISION 17b -ENV ANDROID_NDK_HASH 5dfbbdc2d3ba859fed90d0e978af87c71a91a5be1f6e1c40ba697503d48ccecd -RUN curl -s -O https://dl.google.com/android/repository/android-ndk-r${ANDROID_NDK_REVISION}-linux-x86_64.zip \ - && echo "${ANDROID_NDK_HASH} android-ndk-r${ANDROID_NDK_REVISION}-linux-x86_64.zip" | sha256sum -c \ - && unzip android-ndk-r${ANDROID_NDK_REVISION}-linux-x86_64.zip \ - && rm -f android-ndk-r${ANDROID_NDK_REVISION}-linux-x86_64.zip - -ENV WORKDIR /opt/android -ENV ANDROID_SDK_ROOT ${WORKDIR}/tools -ENV ANDROID_NDK_ROOT ${WORKDIR}/android-ndk-r${ANDROID_NDK_REVISION} -ENV PREFIX /opt/android/prefix - -ENV TOOLCHAIN_DIR ${WORKDIR}/toolchain-arm -RUN ${ANDROID_NDK_ROOT}/build/tools/make_standalone_toolchain.py \ - --arch arm64 \ - --api 21 \ - --install-dir ${TOOLCHAIN_DIR} \ - --stl=libc++ - -#INSTALL cmake -ENV CMAKE_VERSION 3.12.1 -RUN cd /usr \ - && curl -s -O https://cmake.org/files/v3.12/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz \ - && tar -xzf /usr/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz \ - && rm -f /usr/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz -ENV PATH /usr/cmake-${CMAKE_VERSION}-Linux-x86_64/bin:$PATH - -## Boost -ARG BOOST_VERSION=1_68_0 -ARG BOOST_VERSION_DOT=1.68.0 -ARG BOOST_HASH=7f6130bc3cf65f56a618888ce9d5ea704fa10b462be126ad053e80e553d6d8b7 -RUN set -ex \ - && curl -s -L -o boost_${BOOST_VERSION}.tar.bz2 https://dl.bintray.com/boostorg/release/${BOOST_VERSION_DOT}/source/boost_${BOOST_VERSION}.tar.bz2 \ - && echo "${BOOST_HASH} boost_${BOOST_VERSION}.tar.bz2" | sha256sum -c \ - && tar -xvf boost_${BOOST_VERSION}.tar.bz2 \ - && rm -f boost_${BOOST_VERSION}.tar.bz2 \ - && cd boost_${BOOST_VERSION} \ - && ./bootstrap.sh --prefix=${PREFIX} - -ENV HOST_PATH $PATH -ENV PATH $TOOLCHAIN_DIR/aarch64-linux-android/bin:$TOOLCHAIN_DIR/bin:$PATH - -ARG NPROC=1 - -# Build iconv for lib boost locale -ENV ICONV_VERSION 1.15 -ENV ICONV_HASH ccf536620a45458d26ba83887a983b96827001e92a13847b45e4925cc8913178 -RUN curl -s -O http://ftp.gnu.org/pub/gnu/libiconv/libiconv-${ICONV_VERSION}.tar.gz \ - && echo "${ICONV_HASH} libiconv-${ICONV_VERSION}.tar.gz" | sha256sum -c \ - && tar -xzf libiconv-${ICONV_VERSION}.tar.gz \ - && rm -f libiconv-${ICONV_VERSION}.tar.gz \ - && cd libiconv-${ICONV_VERSION} \ - && CC=aarch64-linux-android-clang CXX=aarch64-linux-android-clang++ ./configure --build=x86_64-linux-gnu --host=arm-eabi --prefix=${PREFIX} --disable-rpath \ - && make -j${NPROC} && make install - -## Build BOOST -RUN cd boost_${BOOST_VERSION} \ - && ./b2 --build-type=minimal link=static runtime-link=static --with-chrono --with-date_time --with-filesystem --with-program_options --with-regex --with-serialization --with-system --with-thread --with-locale --build-dir=android --stagedir=android toolset=clang threading=multi threadapi=pthread target-os=android -sICONV_PATH=${PREFIX} install -j${NPROC} - -#Note : we build openssl because the default lacks DSA1 - -# download, configure and make Zlib -ENV ZLIB_VERSION 1.2.11 -ENV ZLIB_HASH c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1 -RUN curl -s -O https://zlib.net/zlib-${ZLIB_VERSION}.tar.gz \ - && echo "${ZLIB_HASH} zlib-${ZLIB_VERSION}.tar.gz" | sha256sum -c \ - && tar -xzf zlib-${ZLIB_VERSION}.tar.gz \ - && rm zlib-${ZLIB_VERSION}.tar.gz \ - && mv zlib-${ZLIB_VERSION} zlib \ - && cd zlib && CC=clang CXX=clang++ ./configure --static \ - && make -j${NPROC} - -# open ssl -ARG OPENSSL_VERSION=1.0.2p -ARG OPENSSL_HASH=50a98e07b1a89eb8f6a99477f262df71c6fa7bef77df4dc83025a2845c827d00 -RUN curl -s -O https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz \ - && echo "${OPENSSL_HASH} openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c \ - && tar -xzf openssl-${OPENSSL_VERSION}.tar.gz \ - && rm openssl-${OPENSSL_VERSION}.tar.gz \ - && cd openssl-${OPENSSL_VERSION} \ - && sed -i -e "s/mandroid/target\ aarch64\-linux\-android/" Configure \ - && CC=clang CXX=clang++ \ - ./Configure android \ - no-asm \ - no-shared --static \ - --with-zlib-include=${WORKDIR}/zlib/include --with-zlib-lib=${WORKDIR}/zlib/lib \ - --prefix=${PREFIX} --openssldir=${PREFIX} \ - && make -j${NPROC} \ - && make install - -# ZMQ -ARG ZMQ_VERSION=master -ARG ZMQ_HASH=501d0815bf2b0abb93be8214fc66519918ef6c40 -RUN git clone https://github.com/zeromq/libzmq.git -b ${ZMQ_VERSION} \ - && cd libzmq \ - && git checkout ${ZMQ_HASH} \ - && ./autogen.sh \ - && CC=clang CXX=clang++ ./configure --prefix=${PREFIX} --host=aarch64-linux-android --enable-static --disable-shared \ - && make -j${NPROC} \ - && make install - -# Sodium -ARG SODIUM_VERSION=1.0.16 -ARG SODIUM_HASH=675149b9b8b66ff44152553fb3ebf9858128363d -RUN set -ex \ - && git clone https://github.com/jedisct1/libsodium.git -b ${SODIUM_VERSION} \ - && cd libsodium \ - && test `git rev-parse HEAD` = ${SODIUM_HASH} || exit 1 \ - && ./autogen.sh \ - && CC=clang CXX=clang++ ./configure --prefix=${PREFIX} --host=aarch64-linux-android --enable-static --disable-shared \ - && make -j${NPROC} \ - && make install - -ADD . /src -RUN cd /src \ - && CMAKE_INCLUDE_PATH="${PREFIX}/include" \ - CMAKE_LIBRARY_PATH="${PREFIX}/lib" \ - ANDROID_STANDALONE_TOOLCHAIN_PATH=${TOOLCHAIN_DIR} \ - USE_SINGLE_BUILDDIR=1 \ - PATH=${HOST_PATH} make release-static-android-armv8 -j${NPROC} diff --git a/utils/gpg_keys/anon.asc b/utils/gpg_keys/anon.asc new file mode 100644 index 000000000..83e1a0cd3 --- /dev/null +++ b/utils/gpg_keys/anon.asc @@ -0,0 +1,13 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mDMEAAAAGBYJKwYBBAHaRw8BAQdAtk3N9Qcw9x3s5frY00j/jO10wfOCZ59mTLtN +tsYJ1620GGFub24gPGFub24gW2F0XSBub3doZXJlPoiQBBMWCAA4FiEEn2DdNuXc +KHIjsNT304V8F6p/losFAmAhnpUCGyMFCwkIBwIGFQoJCAsCBBYCAwECHgECF4AA +CgkQ04V8F6p/loumGwEAxjmzDa6ZK7zsps0sYbOvlh0tmxByUqZy1GvIaiOpdboA +/2guWA0tzK+fyQS2L2oXzFwh+oTk5bZ2KqnnfCr9a6kDuDgEYCG1oRIKKwYBBAGX +VQEFAQEHQCnTdYrNM/yw++BSJPgPAoImGhrKaqBoe5VBi9TUk+8aAwEIB4h4BBgW +CAAgFiEEn2DdNuXcKHIjsNT304V8F6p/losFAmAhtaECGwwACgkQ04V8F6p/lose +3AD9E3e4olBj7e6t5NtlMkkBzYKs16RNZRMJ5GMOw8lH8NUA/imPWARO2Y4ryswV +LAtQE1x0SCd4wEP8QEta4p2OYscI +=9Mvr +-----END PGP PUBLIC KEY BLOCK----- diff --git a/utils/gpg_keys/mj-xmr.asc b/utils/gpg_keys/mj-xmr.asc new file mode 100644 index 000000000..e02d3e511 --- /dev/null +++ b/utils/gpg_keys/mj-xmr.asc @@ -0,0 +1,52 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBGBxrvMBEADD7dDjSfrtiuQJEuMMMGYt3wGQ1iJdiLWe4THJDbtPDCLdBxh4 +fpYGMVkUYT8LJFkf41JD/hAIKMNQHdw7qmZcfN9F/9U+jInHRYBM1dbA/mHs8Jhi +YEGrD+v7c5fEuFXANW7z2uPjszw67tv3K5z8pFkt6K0pCgnxSKBu10WYkV0fvnxz +e3naXAnWQQ/qeGP2xT2F3S+bn5GvCTPw1pI1HAaFcYVD+XZpNdKQHnHMWlDrQ0qe +zymqrknjMmtD70Z7gEOogNxjd2QXUsQDq7XwxfHnhVZHHm98O95gh4Uw2Pve/TMI +f0ItA+MYp+hDqN7wmq3TjNSVI9L02m/RgpO7cxpEgQ9QFYJo3dHC8cP7x/uqH3vI +usYjOJtzq3UZWn4hnHRviEMYSDPtfFmO5CRElueVsRTiG0h7cyQYZ8HrEC9qDyR8 +vn0shqrUbcwsplyhwtpoyuJKcK/mFnmETUQ7ZF5NqgIlOaU7Du8WBt84YrDKRe5q +TKMTtDEf4PrejqhYvb7GNOK3D/LJ/dZTJIdETZ2rRCCftgKjTLJr9fRSrQnxw5NQ +8fnnK/O8JNpRvnDvR2HlmI8w/sgEDT99U4XgSi9BIUrM8HYkNykENXsk2BvEIGEN +txXwZKDJmxK787+y4WX2JfbhFGPNCXxuwrXUqNhKy4Qit140Z7WH6UIvSwARAQAB +tEdtai14bXIgKEJlY2F1c2UgV29ybGQgZG9taW5hdGlvbiBzaG91bGQgYmUgZnVu +LikgPG1qeG1yQHByb3Rvbm1haWwuY29tPokCVAQTAQoAPhYhBLMwkesQonK53VcC ++sEBv5QJNFHgBQJgca7zAhsDBQkJZgGABQsJCAcCBhUKCQgLAgQWAgMBAh4BAheA +AAoJEMEBv5QJNFHgPykP/0YOJYU0ap8BpAtG9Dfv46UPy77XMw2ag2AJ0MYPrqy6 +omjha15qLHeXxCr3CJNYa4EEvX0hahSGn4k3VmcDHVYYpD4f/RZfu8fBpk0xQtsv +wF7zuAn0KSeg7maopdWfcPn02uTOm2crzTMMm5scXZ/YmMviUFPsgpyIkM5oCre6 +/RhIK953OUpgc/JKIlhpABA7KBTWUbc18BGHHdcPntVKrIxrIvzchVwI2P8KDQki +NS33Fc9EoOlDuzbTkQldZJrsEsMZWKfzoNNlZsLtroaS6SlqQNllYjSEJx7ccjEU ++zvMGLXhxqL7XTFO1HdvacWwPsSPR8MXrwUFcKrDDzC57rHNPrHncPY1JwEkN0Aj +V3VzTM+5QPvo6QQ+aNISxy9Hs470SpLDmmBl//Dn4qbFWSqAQ3qSD7S34MgW0FeC +4L1/uTsDokkx+ZIf7WQUl2p4vi8/puFw2y3VLs4WGaUYko1gjhoFIDs3gP7YIeE9 +Lp+6/+RxMkuSGTrHM9rcpeClp3jEN/I7n/epKpJOSgroN+/RdbOOnVECC7+kWkM0 +U9fPi+O3fRIlaR+Q8NWkokgkbtw6gYV3bKnW78AnF5INs3rrtOH1WtGSp7ghKP4p +7GE5cCIbUsB3dT6Ifo5tAi3faYFyhUZ2GsYNM0fsge10OwtPs+4YbSuurWUyIwuW +uQINBGBxrvMBEACxiDPPivT0Tv4DWtkGZ7tHRF0IT9nSZuL5F+qMX4D3hef3RfVY +4hxAQe2elvMU4PV/3neaRyiWSd4/Q8DnWrGQGvn3gxx8AHZTQwv/01Ae0InyNI9t +pzpBCMvOY73dba1JRklEMMyJPAUrdvmyRTEuZEIAxoVlaKKSAr4oeaENGLwWxB4o +NsAPxKNWw8drGQfJixMA+IeqeMzRjo6o5gJzrlaTb3+S78a1rD0pMmY7s6j3LqmF +kd0L/JlGrs+ILCxXjddg6FRL/qUtvOZX+jdogb1hKvPFCMMc1BfUdq9WbrCpoj54 +OhBnO+yCHi2faUi0YP8Fumu6RTCDN+MK0nEji+peaJApHf15ZVd9DAX9NzRl/iLb +sMjf4KBE2BI6FaUpdzkTbH6F713O2ZGWGEROBTjKucO3OkjGfHKf9rxdGRt37I/z +b59xRzSn91Vz1nEXCmDSO/iPAGZ3bQKh+zSGwjwJsZrBRCvS34AhGaEs8un6nd9M +FoMClpyjmm3W6WUZnELmcdlKhY2xbi5Yub21+mrjjjS6BuFpSYIC1xKJrCdpXtAm +i8x5mcKbY24uRuqsY9pQrDHVXK/Pwj6+k9rGwbe5P6VkbZOpJ2pWSL68EoxIo1zp +bv6moIJtxipydUOD2k76W0oGplexoxtW5f1WKcb3ykJRRrdPQxtPaPK14QARAQAB +iQI8BBgBCgAmFiEEszCR6xCicrndVwL6wQG/lAk0UeAFAmBxrvMCGwwFCQlmAYAA +CgkQwQG/lAk0UeClZhAAwQa+XO2GhCfZgrYOOSR5h2iib5k8OWiTsV68/DHOMsFJ +1WVDsqjlNEOhPPg0CbH4FMHJOndcsjJb+Rs6Y35YsgTmQl1SHlJXZinV1AJ+IaW/ +Esx+qOpoGUzCzCrjGpwSYHxGSQ52WpLvTaHbosH3x2XXTZ2znOdwTlFB/36Yco3c +ROWVLT/FhjugxTP0KddZR9Lu7ThK7txQhTQp5oBwWgpv0YAPaRPA+BpvrtzI7xSx +2AOMtr7jiGAu0yFL5WestRy37UmunxS8Rd8PGILcfFBIvfqNTMwU55SW1nS9XbRh +bUb/iil+Ly6lVsGAPW7iUgEUO7HtsPxeF+Bk/Hh+ZCHglQrGpzGcPmgbG3IiMF9H +nISbh6sFrkDXjyLNP1zl6wg0cTyk12OcYo0nX4jp1CY3Oztd3rVdcMT8Qot4JSHj +rDx+b3FTErVCnitEeALuBt/JSNH0VMcgOmFen9goJRVXcW/Dnsb+7QWXooj3aC++ +O6eghDN1YyJClQqLxvVThr7pdlMiWgWzHLmRkI2o84r14qOC15yTh7qr3Y5LOMPN +px9K2f9f2l6KJ0w7vgKwa6pYiVf1AAw2Ym98LAJn+6v9+nRx9mSq9Nw+M5eo9DXW +gclwPV1SSWUls8jjOmluI6/j8TE9JMBy5UKX1g6TOKJWW6wXcOUfzNVtG1oRNPA= +=egkb +-----END PGP PUBLIC KEY BLOCK----- diff --git a/utils/health/README.md b/utils/health/README.md index 8fadd3908..05eb057a1 100644 --- a/utils/health/README.md +++ b/utils/health/README.md @@ -1,26 +1,32 @@ -#Intro +# Intro + This directory contains tools, which can be used for checking the health of the project, like build/run time analyzers, lints, etc. -#Usage +# Usage + Unless it's stated differently, these scripts should be called from a given source directory, where you want the checks to be performed, for instance: `og@ghetto:~/dev/monero$ utils/health/clang-build-time-analyzer-run.sh` -##ClangBuildAnalyzer +## ClangBuildAnalyzer + `utils/health/clang-build-time-analyzer-run.sh` The CBA helps in finding culprints of slow compilation. On the first run, the script will complain about the missing ClangBuildAnalyzer binary and will point you to another script, which is able to clone and build the required binary. -##clang-tidy +## clang-tidy + `utils/health/clang-tidy-run.sh` Performs Lint checks on the source code and stores the result in the build directory. More information on the [home page](https://clang.llvm.org/extra/clang-tidy/). -##include-what-you-use +## include-what-you-use + `utils/health/clang-include-what-you-use-run.sh` Analyses the header file hierarchy and delivers hints on how to reduce their complexity. More information on the [home page](https://include-what-you-use.org/). -##Valgrind checks +## Valgrind checks + `utils/health/valgrind-tests.sh` This script is able to run valgrind's callgrind, cachegrind and memcheck for a given set of executables. It expects ONE PARAMETER, which points to a file with paths to executables and their arguments, written line by line. For example: @@ -33,6 +39,7 @@ build/tests/unit_tests/unit_tests The `*.out` results can be interpreted with the `kcachegrind` tool. The memcheck output is just a readable text file with a summary at the end. -#Footer +# Footer + Responsible: mj-xmr diff --git a/utils/python-rpc/console.py b/utils/python-rpc/console.py index 57a04528b..4b3c18f5d 100755 --- a/utils/python-rpc/console.py +++ b/utils/python-rpc/console.py @@ -4,7 +4,16 @@ from __future__ import print_function import sys import subprocess import socket -import urlparse +try: + import urllib.parse + url_parser = urllib.parse.urlparse +except: + try: + import urlparse + url_parser = urlparse.urlparse + except: + print('urllib or urlparse is needed') + sys.exit(1) import framework.rpc import framework.daemon import framework.wallet @@ -21,7 +30,7 @@ for n in range(1, len(sys.argv)): try: port = int(sys.argv[n]) except: - t = urlparse.urlparse(sys.argv[n], allow_fragments = False) + t = url_parser(sys.argv[n], allow_fragments = False) scheme = t.scheme or scheme host = t.hostname or host port = t.port or port @@ -29,7 +38,7 @@ for n in range(1, len(sys.argv)): raise Exception(USAGE) if port <= 0 or port > 65535: raise Exception(USAGE) - except Exception, e: + except Exception as e: print('Error: ' + str(e)) raise Exception(USAGE) @@ -49,7 +58,7 @@ for n in range(1, len(sys.argv)): } try: res = rpc.send_json_rpc_request(get_version) - except Exception, e: + except Exception as e: raise Exception('Failed to call version RPC: ' + str(e)) if 'version' not in res: diff --git a/utils/python-rpc/framework/daemon.py b/utils/python-rpc/framework/daemon.py index 435bc93f8..207565e6f 100644 --- a/utils/python-rpc/framework/daemon.py +++ b/utils/python-rpc/framework/daemon.py @@ -53,6 +53,19 @@ class Daemon(object): return self.rpc.send_json_rpc_request(getblocktemplate) get_block_template = getblocktemplate + def add_aux_pow(self, blocktemplate_blob, aux_pow, client = ""): + add_aux_pow = { + 'method': 'add_aux_pow', + 'params': { + 'blocktemplate_blob': blocktemplate_blob, + 'aux_pow' : aux_pow, + 'client' : client, + }, + 'jsonrpc': '2.0', + 'id': '0' + } + return self.rpc.send_json_rpc_request(add_aux_pow) + def send_raw_transaction(self, tx_as_hex, do_not_relay = False, do_sanity_checks = True, client = ""): send_raw_transaction = { 'client': client, diff --git a/utils/python-rpc/framework/wallet.py b/utils/python-rpc/framework/wallet.py index d97c24143..02084620c 100644 --- a/utils/python-rpc/framework/wallet.py +++ b/utils/python-rpc/framework/wallet.py @@ -1088,3 +1088,47 @@ class Wallet(object): 'id': '0' } return self.rpc.send_json_rpc_request(get_version) + + def scan_tx(self, txids): + scan_tx = { + 'method': 'scan_tx', + 'jsonrpc': '2.0', + 'params' : { + 'txids': txids, + }, + 'id': '0' + } + return self.rpc.send_json_rpc_request(scan_tx) + + def freeze(self, key_image): + freeze = { + 'method': 'freeze', + 'jsonrpc': '2.0', + 'params' : { + 'key_image': key_image, + }, + 'id': '0' + } + return self.rpc.send_json_rpc_request(freeze) + + def thaw(self, key_image): + thaw = { + 'method': 'thaw', + 'jsonrpc': '2.0', + 'params' : { + 'key_image': key_image, + }, + 'id': '0' + } + return self.rpc.send_json_rpc_request(thaw) + + def frozen(self, key_image): + frozen = { + 'method': 'frozen', + 'jsonrpc': '2.0', + 'params' : { + 'key_image': key_image, + }, + 'id': '0' + } + return self.rpc.send_json_rpc_request(frozen) |